diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da1fd112e583b..75745b103c461 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,14 +32,20 @@ repos: hooks: - id: identity name: Print checked files + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Print input to the static check hooks for troubleshooting - id: check-hooks-apply name: Check if all hooks apply to the repository + env: + PYTHONPYCACHEPREFIX: .build/pycache - repo: https://github.com/thlorenz/doctoc.git rev: 59dcae910a6ad4bbe8711275b854e3cd5c30f08b # frozen: v2.5.0 hooks: - id: doctoc name: Add TOC for Markdown and RST files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: (?x) ^README\.md$| @@ -63,6 +69,8 @@ repos: hooks: - id: insert-license name: Add license for all SQL files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: \.sql$ exclude: | (?x) @@ -79,6 +87,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all RST files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --comment-style - "||" @@ -93,6 +103,8 @@ repos: ^scripts/ci/license-templates/ - id: insert-license name: Add license for CSS/JS/JSX/PUML/TS/TSX + env: + PYTHONPYCACHEPREFIX: .build/pycache files: \.(css|jsx?|puml|tsx?)$ exclude: ^\.github/.*$|ui/openapi-gen/|www/openapi-gen/|.*/dist/.* args: @@ -103,6 +115,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Shell files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$ files: \.bash$|\.sh$ args: @@ -113,6 +127,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all toml files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$ files: \.toml$ args: @@ -123,6 +139,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Python files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$|^.*/_vendor/.*$|^airflow-ctl/.*/.*generated\.py$ files: \.py$|\.pyi$ args: @@ -133,6 +151,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all XML files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$ files: \.xml$ args: @@ -143,6 +163,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all YAML files except Helm templates + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^\.github/.*$| @@ -162,6 +184,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Markdown files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: \.md$ args: - --comment-style @@ -180,6 +204,8 @@ repos: ^scripts/ci/license-templates/ - id: insert-license name: Add short license for agentic Markdown files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --comment-style - "||" @@ -201,6 +227,8 @@ repos: ^scripts/ci/license-templates/ - id: insert-license name: Add license for all other files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --comment-style - "|#|" @@ -225,11 +253,15 @@ repos: hooks: - id: check-min-python-version name: Check minimum Python version + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_min_python_version.py language: python require_serial: true - id: update-notice-year name: Update NOTICE files to current year (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_notice_year.py stages: ['manual'] language: python @@ -237,11 +269,15 @@ repos: require_serial: true - id: check-notice-files name: Check NOTICE files for current year and ASF references + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_notice_files.py language: python files: ^.*NOTICE$ - id: check-version-consistency name: Check version consistency + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_version_consistency.py language: python files: > @@ -254,6 +290,8 @@ repos: require_serial: true - id: check-applies-to-roots-in-sync name: Check plugin applies_to destination tables stay in sync + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "The UI mirrors plugins_manager's destination -> record map; drift silently widens plugin scope" entry: ./scripts/ci/prek/check_applies_to_roots_in_sync.py language: python @@ -265,6 +303,8 @@ repos: require_serial: true - id: check-ci-workflows-in-sync name: Check ci-arm.yml and ci-amd.yml stay in sync + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_ci_workflows_in_sync.py language: python files: ^\.github/workflows/ci-(arm|amd)\.yml$ @@ -272,6 +312,8 @@ repos: require_serial: true - id: check-java-sdk-version-in-sync name: Check Java SDK JDK version is consistent across build files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_java_sdk_version_in_sync.py language: python files: > @@ -286,6 +328,8 @@ repos: require_serial: true - id: sync-java-sdk-supervisor-schema name: Sync Java SDK supervisor schema with configured version + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Download the Supervisor Schema when Java SDK's bundled schema.json drifts" entry: ./scripts/ci/prek/sync_java_sdk_supervisor_schema.py language: python @@ -296,6 +340,8 @@ repos: ^java-sdk/sdk/schema/schema\.json$ - id: regenerate-java-sdk-verification-metadata name: Regenerate the Java SDK dependency verification metadata + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Drop trusted checksums that no Java SDK build task resolves any more" entry: ./scripts/ci/prek/regenerate_java_sdk_verification_metadata.py language: python @@ -315,6 +361,8 @@ repos: ^java-sdk/(?!example/|scala_spark_example/)[^/]+/(?:build\.gradle\.kts|gradle\.properties)$ - id: update-java-sdk-readme-matrix name: Update the Java SDK compatibility matrix in java-sdk/README.md and Dokka module doc + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_java_sdk_readme_matrix.py language: python files: > @@ -330,6 +378,8 @@ repos: require_serial: true - id: update-ts-sdk-readme-matrix name: Update the TypeScript SDK compatibility matrix in ts-sdk/README.md + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_ts_sdk_readme_matrix.py language: python files: > @@ -344,6 +394,8 @@ repos: require_serial: true - id: sync-ts-sdk-schemas name: Sync TypeScript SDK vendored schemas with the distributions that own them + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Copy airflow-core's Dag schema and the Task SDK's supervisor schema when TS SDK's vendored copies drift" entry: ./scripts/ci/prek/sync_ts_sdk_schemas.py language: python @@ -360,6 +412,8 @@ repos: ^scripts/ci/prek/sync_ts_sdk_schemas\.py$ - id: sync-java-sdk-dag-schema name: Sync Java SDK Dag serialization schema with airflow-core + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Copy airflow-core's serialization schema when Java SDK's vendored dag-schema.json drifts" entry: ./java-sdk/gradlew -p ./java-sdk :sdk:syncDagSchema -PfailOnDagSchemaDrift language: system @@ -377,6 +431,8 @@ repos: ^java-sdk/sdk/schema/dag-schema\.json$ - id: sync-go-sdk-schemas name: Sync Go SDK vendored schemas with the distributions that own them + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Copy airflow-core's Dag schema and task-sdk's supervisor schema when go-sdk's vendored copies drift" entry: ./scripts/ci/prek/sync_go_sdk_schemas.py language: python @@ -396,6 +452,8 @@ repos: ^scripts/ci/prek/sync_go_sdk_schemas\.py$ - id: check-go-sdk-generated-drift name: Check Go SDK generated files against the schemas they generate from + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Regenerate the Dag specs and the coordinator-protocol models, and fail when a committed file differs" entry: ./scripts/ci/prek/check_go_sdk_generated_drift.py # golang so prek provisions the toolchain the generators need, the way the @@ -424,6 +482,8 @@ repos: # check when the shared harness or Airflow's serializer changes, which that project cannot see. - id: check-ts-sdk-serialization-conformance-shared name: Check the TS SDK serializes Dags the way Airflow does, after a shared change + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the TS SDK and with Airflow, and compare the two" entry: ./ts-sdk/scripts/ci/prek/check_serialization_conformance.py language: node @@ -437,6 +497,8 @@ repos: ^scripts/ci/lang_sdk_serialization/.*$ - id: check-go-sdk-serialization-conformance name: Check the Go SDK serializes Dags the way Airflow does + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the Go SDK and with Airflow, and compare the two" entry: ./scripts/ci/prek/check_go_sdk_serialization_conformance.py # golang so prek provisions the toolchain that builds the Go serializer. @@ -453,6 +515,8 @@ repos: ^scripts/ci/prek/check_go_sdk_serialization_conformance\.py$ - id: check-java-sdk-serialization-conformance name: Check the Java SDK serializes Dags the way Airflow does + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the Java SDK and with Airflow, and compare the two" entry: ./java-sdk/scripts/ci/prek/check_serialization_conformance.py language: system @@ -472,6 +536,8 @@ repos: ^scripts/ci/lang_sdk_serialization/.*$ - id: check-go-version-in-sync name: Check Go toolchain version is consistent across build files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_go_version_in_sync.py language: python files: > @@ -487,6 +553,8 @@ repos: require_serial: true - id: update-go-sdk-readme-matrix name: Update the Go SDK compatibility matrix in go-sdk/README.md + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_go_sdk_readme_matrix.py language: python files: > @@ -501,6 +569,8 @@ repos: require_serial: true - id: check-go-example-mod-tidy name: Check lang-SDK Go example module is tidy against the Go SDK + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_go_example_mod_tidy.py # golang so prek provisions the toolchain the check needs: static checks run on a # runner whose preinstalled toolchains are wiped to free disk space, and the SDK's @@ -518,6 +588,8 @@ repos: require_serial: true - id: check-partition-mapper-defaults-in-sync name: Check partition-mapper core/SDK sync (FanOutMapper table + SegmentWindow/FixedKeyMapper) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py language: python files: > @@ -532,6 +604,8 @@ repos: require_serial: true - id: check-window-in-sync name: Check Window definitions stay in sync (core/SDK) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_window_in_sync.py language: python files: > @@ -542,6 +616,8 @@ repos: require_serial: true - id: sync-uv-min-version-markers name: Sync `# sync-uv-min-version` markers with [tool.uv] required-version + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sync_uv_min_version_markers.py language: python # Hook reads pyproject.toml itself; the file list selects places that may carry @@ -555,6 +631,8 @@ repos: require_serial: true - id: check-distribution-gitignore name: Check distribution .gitignore files have *.iml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_distribution_gitignore.py language: python files: > @@ -567,9 +645,11 @@ repos: require_serial: true - id: upgrade-important-versions name: Upgrade important versions (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache + UV_FROZEN: "0" entry: ./scripts/ci/prek/upgrade_important_versions.py stages: ['manual'] - env: {UV_FROZEN: "0"} language: python files: > (?x) @@ -580,9 +660,11 @@ repos: require_serial: true - id: upgrade-dependency-floors name: Upgrade curated dependency floors (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache + UV_FROZEN: "0" entry: ./scripts/ci/prek/upgrade_dependency_floors.py stages: ['manual'] - env: {UV_FROZEN: "0"} language: python files: > (?x) @@ -596,6 +678,8 @@ repos: hooks: - id: blacken-docs name: Run black on docs + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --line-length=110 - --target-version=py311 @@ -612,15 +696,25 @@ repos: hooks: - id: check-merge-conflict name: Check that merge conflicts are not being committed + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: debug-statements name: Detect accidentally committed debug statements + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: check-builtin-literals name: Require literal syntax when initializing builtins + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: detect-private-key name: Detect if private key is added to the repository + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^providers/ssh/docs/connections/ssh\.rst$ - id: end-of-file-fixer name: Make sure that there is an empty line at the end + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^airflow-core/docs/img/.*\.dot| @@ -630,15 +724,23 @@ repos: .*/openapi-gen/.* - id: mixed-line-ending name: Detect if mixed line ending is used (\r vs. \r\n) + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: check-executables-have-shebangs name: Check that executables have shebang + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: check-xml name: Check XML files with xmllint + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^scripts/ci/docker-compose/gremlin/. - id: trailing-whitespace name: Remove trailing whitespace at end of line + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^airflow-core/docs/img/.*\.dot$| @@ -650,13 +752,19 @@ repos: hooks: - id: rst-backticks name: Check if RST files use double backticks for code + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: python-no-log-warn name: Check if there are no deprecate log warn + env: + PYTHONPYCACHEPREFIX: .build/pycache - repo: https://github.com/adrienverge/yamllint rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # frozen: v1.38.0 hooks: - id: yamllint name: Check YAML files with yamllint + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: yamllint -c yamllint-config.yml --strict types: [yaml] exclude: > @@ -677,6 +785,8 @@ repos: hooks: - id: flynt name: Run flynt string format converter for Python + env: + PYTHONPYCACHEPREFIX: .build/pycache args: # If flynt detects too long text it ignores it. So we set a very large limit to make it easy # to split the text by hand. Too long lines are detected by flake8 (below), @@ -688,6 +798,8 @@ repos: hooks: - id: codespell name: Run codespell + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Run codespell to check for common misspellings in files entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s) to the codespell dictionary at docs/spelling_wordlist.txt. @@ -716,6 +828,8 @@ repos: hooks: - id: zizmor name: Run zizmor to check for github workflow syntax errors + env: + PYTHONPYCACHEPREFIX: .build/pycache types: [yaml] files: ^\.github/workflows/.*$|^\.github/actions/.*$ require_serial: true @@ -731,6 +845,8 @@ repos: # bundles its own libc and is portable across runners. - id: lychee-docker name: Check internal Markdown links with lychee (offline mode) + env: + PYTHONPYCACHEPREFIX: .build/pycache description: >- Validate intra-repo links in Markdown files (relative paths, anchors, and fragments) without making any HTTP requests. Auto-generated client @@ -761,6 +877,8 @@ repos: hooks: - id: check-shared-distributions-structure name: Check shared distributions structure + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_shared_distributions_structure.py language: python pass_filenames: false @@ -773,12 +891,16 @@ repos: ^task-sdk/.*\.py$ - id: check-shared-distributions-usage name: Check shared distributions usage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_shared_distributions_usage.py language: python pass_filenames: false files: ^shared/.*$|^.*/pyproject.toml$|^.*/_shared/.*$ - id: check-airflow-imports-in-shared name: Check for core/sdk imports in shared libraries + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_airflow_imports_in_shared.py language: python pass_filenames: true @@ -791,6 +913,8 @@ repos: ^shared/secrets_backend/src/airflow_shared/secrets_backend/base\.py$ - id: check-test-only-imports-in-src name: Check for test-only imports in production source + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_test_only_imports_in_src.py language: python pass_filenames: true @@ -803,18 +927,24 @@ repos: ^shared/.*/src/.*\.py$ - id: check-secrets-search-path-sync name: Check sync between sdk and core + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_secrets_search_path_sync.py language: python pass_filenames: false files: ^airflow-core/src/airflow/secrets/base_secrets\.py$|^task-sdk/src/airflow/sdk/execution_time/secrets/__init__\.py$ - id: check-registry-types-json-sync name: Check registry types.json in sync with types.py + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_registry_types_json_sync.py language: python pass_filenames: false files: ^dev/registry/registry_tools/types\.py$|^registry/src/_data/types\.json$ - id: ruff name: Run 'ruff' for extremely fast Python linting + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Run 'ruff' for extremely fast Python linting" entry: ruff check --force-exclude language: python @@ -825,6 +955,8 @@ repos: exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$ - id: ruff-format name: Run 'ruff format' + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Run 'ruff format' for extremely fast Python formatting" entry: ./scripts/ci/prek/ruff_format.py language: python @@ -834,6 +966,8 @@ repos: exclude: ^airflow-core/tests/unit/dags/test_imports\.py$ - id: replace-bad-characters name: Replace bad characters + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/replace_bad_characters.py language: python types: [file, text] @@ -851,6 +985,8 @@ repos: /go\.(mod|sum)$ - id: lint-dockerfile name: Lint Dockerfile + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/lint_dockerfile.py files: Dockerfile.*$ @@ -858,24 +994,32 @@ repos: require_serial: true - id: check-airflow-bug-report-template name: Sort airflow-bug-report provider list + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^\.github/ISSUE_TEMPLATE/1-airflow_bug_report\.yml$ require_serial: true entry: ./scripts/ci/prek/check_airflow_bug_report_template.py - id: update-local-yml-file name: Update mounts in the local yml file + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/local_yml_mounts.py language: python files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false - id: check-extras-order name: Check order of extras in Dockerfile + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_order_dockerfile_extras.py language: python files: ^Dockerfile$ pass_filenames: false - id: generate-airflow-diagrams name: Generate airflow diagrams + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_airflow_diagrams.py language: python files: > @@ -887,6 +1031,8 @@ repos: pass_filenames: true - id: prevent-deprecated-sqlalchemy-usage name: Prevent deprecated sqlalchemy usage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py language: python files: > @@ -902,12 +1048,16 @@ repos: pass_filenames: true - id: update-supported-versions name: Updates supported versions in documentation + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/supported_versions.py language: python files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/prek/supported_versions\.py$|^README\.md$ pass_filenames: false - id: generate-api-permissions-doc name: Generate REST API permission reference documentation + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/extract_permissions.py language: python files: > @@ -919,6 +1069,8 @@ repos: pass_filenames: false - id: generate-fab-permissions-doc name: Generate FAB auth manager REST API permission table + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/fab_permissions_doc.py language: python files: > @@ -934,12 +1086,16 @@ repos: pass_filenames: false - id: update-tested-versions name: Update tested Python/DB/Kubernetes versions in docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_tested_versions.py language: python files: ^airflow-core/docs/installation/prerequisites\.rst$|^README\.md$|^dev/breeze/src/airflow_breeze/global_constants\.py$|^scripts/ci/prek/update_tested_versions\.py$|^scripts/ci/prek/common_prek_utils\.py$ pass_filenames: false - id: check-revision-heads-map name: Check that the REVISION_HEADS_MAP is up-to-date + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_revision_heads_map.py pass_filenames: false @@ -951,17 +1107,23 @@ repos: ^airflow-core/src/airflow/utils/db\.py$ - id: update-version name: Update versions in docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_versions.py language: python files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$ pass_filenames: false - id: check-pydevd-left-in-code + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check for pydevd debug statements accidentally left entry: "pydevd.*settrace\\(" pass_filenames: true files: \.py$ - id: check-safe-filter-usage-in-html + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Don't use safe in templates description: the Safe filter is error-prone, use Markup() in code instead @@ -969,6 +1131,8 @@ repos: files: \.html$ pass_filenames: true - id: check-urlparse-usage-in-code + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Don't use urlparse in code description: urlparse is not recommended, use urlsplit() in code instead @@ -977,6 +1141,8 @@ repos: files: \.py$ - id: check-auth-manager-mock-targets name: Patch SimpleAuthManager, not BaseAuthManager, for methods it overrides + env: + PYTHONPYCACHEPREFIX: .build/pycache description: The API test client uses SimpleAuthManager, so patching an overridden BaseAuthManager method has no effect entry: ./scripts/ci/prek/check_auth_manager_mock_targets.py language: python @@ -984,6 +1150,8 @@ repos: files: ^airflow-core/tests/unit/api_fastapi/.*\.py$ exclude: ^airflow-core/tests/unit/api_fastapi/auth/managers/simple/ - id: check-for-inclusive-language + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check for language that we do not accept as community description: Please use more appropriate words for community documentation. @@ -1095,6 +1263,8 @@ repos: ^scripts/ci/prek/vendor_k8s_json_schema\.py$ - id: check-template-context-variable-in-sync name: Sync template context variable refs + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py files: @@ -1103,6 +1273,8 @@ repos: ^task-sdk/src/airflow/sdk/definitions/context\.py$| ^airflow-core/docs/templates-ref\.rst$ - id: check-base-operator-usage + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check BaseOperator core imports description: Make sure BaseOperator is imported from airflow.models.baseoperator in core @@ -1116,6 +1288,8 @@ repos: ^airflow-core/src/airflow/operators/.*$| ^providers/.*$ - id: check-base-operator-usage + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check BaseOperatorLink core imports description: Make sure BaseOperatorLink is not imported from airflow.models in core @@ -1130,6 +1304,8 @@ repos: ^providers/.*/src/airflow/providers/.*$| ^providers/.*/src/airflow/providers/standard/sensors/.*$ - id: check-core-deprecation-classes + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Verify usage of Airflow deprecation classes in core entry: category=DeprecationWarning|category=PendingDeprecationWarning @@ -1145,6 +1321,8 @@ repos: ^devel-common/ pass_filenames: true - id: check-provide-create-sessions-imports + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check session util imports description: NEW_SESSION, provide_session, and create_session should be imported from airflow.utils.session to avoid import cycles. @@ -1152,12 +1330,16 @@ repos: files: \.py$ pass_filenames: true - id: check-incorrect-use-of-LoggingMixin + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Make sure LoggingMixin is not used alone entry: "LoggingMixin\\(\\)" files: \.py$ pass_filenames: true - id: check-start-date-not-used-in-defaults + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: start_date not in default_args entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):" @@ -1165,24 +1347,32 @@ repos: pass_filenames: true - id: check-example-dag-tags name: Check example DAGs carry the "example" tag + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_example_dag_tags.py language: python files: ^.*/example_dags/.*\.py$ pass_filenames: true - id: check-example-dag-shell-commands name: Check example Dags do not build shell commands from runtime values + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_example_dag_shell_commands.py language: python files: ^.*/example_dags/.*\.py$ pass_filenames: true - id: check-apache-license-rat name: Check if licenses are OK for Apache + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_license.py language: python files: ^LICENSE$ pass_filenames: false - id: check-metrics-synced-with-registry name: Check that metrics in the codebase are in sync with the metrics registry YAML file. + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_metrics_synced_with_the_registry.py language: python files: \.py$|^shared/observability/src/airflow_shared/observability/metrics/metrics_template\.yaml$ @@ -1192,12 +1382,16 @@ repos: additional_dependencies: ["PyYAML>=6.0", "rich>=13.6.0"] - id: check-boring-cyborg-configuration name: Checks for Boring Cyborg configuration consistency + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/boring_cyborg.py pass_filenames: false require_serial: true - id: update-in-the-wild-to-be-sorted name: Sort INTHEWILD.md alphabetically + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sort_in_the_wild.py language: python files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$ @@ -1205,6 +1399,8 @@ repos: require_serial: true - id: update-installed-providers-to-be-sorted name: Sort and uniquify installed_providers.txt + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sort_installed_providers.py language: python files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$ @@ -1212,6 +1408,8 @@ repos: require_serial: true - id: update-spelling-wordlist-to-be-sorted name: Sort spelling_wordlist.txt + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sort_spelling_wordlist.py language: python files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ @@ -1219,12 +1417,16 @@ repos: pass_filenames: false - id: shellcheck name: Check Shell scripts syntax correctness + env: + PYTHONPYCACHEPREFIX: .build/pycache language: docker_image entry: koalaman/shellcheck:v0.8.0 -x -a files: \.(bash|sh)$|^hooks/build$|^hooks/push$ exclude: ^dev/breeze/autocomplete/.*$ - id: check-integrations-list-consistent name: Sync integrations list with docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_integrations_list.py language: python files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests\.rst$ @@ -1232,6 +1434,8 @@ repos: pass_filenames: false - id: sync-translation-namespaces name: Sync translation namespace file list + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sync_translation_namespaces.py language: python files: > @@ -1241,6 +1445,8 @@ repos: pass_filenames: false - id: generate-agent-skills name: Sync AGENTS.md commands from contributing docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_agent_skills.py language: python files: > @@ -1252,6 +1458,8 @@ repos: additional_dependencies: ['pyyaml', 'rich>=13.6.0'] - id: check-documented-commands name: Check documented breeze and prek commands exist + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_documented_commands.py language: python files: > @@ -1269,6 +1477,8 @@ repos: # its installation there, so update the action alongside any id change. - id: run-skill-eval name: Run skill-eval against AGENTS.md and cases + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./dev/skill-evals/eval.py language: node language_version: '22.22.0' @@ -1283,6 +1493,8 @@ repos: verbose: true - id: run-skill-eval-codex name: Run skill-eval against AGENTS.md and cases (Codex runtime) + env: + PYTHONPYCACHEPREFIX: .build/pycache # Kept separate from run-skill-eval so only contributors who actually run # the Codex arm build an env carrying the Codex CLI binary. promptfoo # bundles its own @openai/codex-sdk, but that build ships a binary whose @@ -1304,6 +1516,8 @@ repos: verbose: true - id: view-skill-eval name: View skill-eval results in browser (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: env PROMPTFOO_CONFIG_DIR=.build/promptfoo promptfoo view language: node language_version: '22.22.0' @@ -1317,6 +1531,8 @@ repos: require_serial: true - id: check-eval-hash name: Verify skill-eval ran on current AGENTS.md and cases + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_eval_hash.py language: python files: > @@ -1327,6 +1543,8 @@ repos: pass_filenames: false - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/update_airflow_pyproject_toml.py files: > @@ -1339,6 +1557,8 @@ repos: require_serial: true - id: check-excluded-provider-markers name: Check excluded-provider python_version and platform markers in pyproject.toml + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_excluded_provider_markers.py files: > @@ -1350,6 +1570,8 @@ repos: additional_dependencies: ['packaging>=25', 'pyyaml', 'rich>=13.6.0'] - id: check-provider-config-fallback-defaults name: Check provider_config_fallback_defaults.cfg is in sync with provider.yaml files + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_provider_config_fallback_defaults.py files: > @@ -1362,6 +1584,8 @@ repos: additional_dependencies: ['pyyaml', 'rich>=13.6.0'] - id: check-dependency-lower-bounds name: Check that dependencies in pyproject.toml have lower bounds + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_dependency_lower_bounds.py files: (^|/)pyproject\.toml$ @@ -1369,12 +1593,16 @@ repos: additional_dependencies: ['packaging>=25', 'rich>=13.6.0'] - id: update-reproducible-source-date-epoch name: Update Source Date Epoch for reproducible builds + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/update_source_date_epoch.py files: ^RELEASE_NOTES\.rst$|^chart/RELEASE_NOTES\.rst$ require_serial: true - id: check-breeze-top-dependencies-limited name: Check top-level breeze deps + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Breeze should have small number of top-level dependencies language: python entry: ./scripts/tools/check_if_limited_dependencies.py @@ -1383,18 +1611,24 @@ repos: require_serial: true - id: check-system-tests-present name: Check if system tests have required segments of code + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_system_tests.py language: python files: ^.*/tests/system/.*/example_[^/]*\.py$ pass_filenames: true - id: generate-pypi-readme name: Generate PyPI README + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_pypi_readme.py language: python files: ^README\.md$ pass_filenames: false - id: sync-provider-readme name: Sync provider README.rst Requirements table with pyproject.toml + env: + PYTHONPYCACHEPREFIX: .build/pycache description: | Provider README.rst files are regenerated from pyproject.toml at distribution-build time. Between releases, dependency bumps in pyproject.toml do not propagate to the @@ -1406,6 +1640,8 @@ repos: pass_filenames: true - id: lint-markdown name: Run markdownlint + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Checks the style of Markdown files. entry: markdownlint language: node @@ -1417,6 +1653,8 @@ repos: additional_dependencies: ['markdownlint-cli@0.38.0'] - id: lint-json-schema name: Lint JSON Schema files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file @@ -1427,6 +1665,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint NodePort Service + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url @@ -1437,6 +1677,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint Docker compose files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url @@ -1452,6 +1694,8 @@ repos: require_serial: true - id: check-persist-credentials-disabled-in-github-workflows name: Check persistent creds in workflow files + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check that workflow files have persist-credentials disabled entry: ./scripts/ci/prek/checkout_no_credentials.py language: python @@ -1459,12 +1703,16 @@ repos: files: ^\.github/workflows/.*\.yml$ - id: check-docstring-param-types name: Check that docstrings do not specify param types + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/docstring_param_type.py language: python pass_filenames: true files: \.py$ - id: check-zip-file-is-not-committed name: Check no zip files are committed + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Zip files are not allowed in the repository language: fail entry: | @@ -1473,6 +1721,8 @@ repos: files: \.zip$ - id: update-inlined-dockerfile-scripts name: Inline Dockerfile and Dockerfile.ci scripts + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/inline_scripts_in_docker.py language: python pass_filenames: false @@ -1480,6 +1730,8 @@ repos: require_serial: true - id: update-docker-gpg-keys name: Update Docker build GPG keys (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_docker_gpg_keys.py stages: ['manual'] language: python @@ -1488,12 +1740,16 @@ repos: require_serial: true - id: check-changelog-has-no-duplicates name: Check changelogs for duplicate entries + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: changelog\.(rst|txt)$ entry: ./scripts/ci/prek/changelog_duplicates.py pass_filenames: true - id: check-no-conventional-commit-message name: Check commit message is not Conventional Commits style + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Reject feat:/fix:/chore: style commit subjects - Airflow uses plain prose" language: python entry: ./scripts/ci/prek/check_no_conventional_commit_message.py @@ -1502,12 +1758,16 @@ repos: pass_filenames: true - id: check-changelog-format name: Check changelog format + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: changelog\.(rst|txt)$ entry: ./scripts/ci/prek/check_changelog_format.py pass_filenames: true - id: check-newsfragments-are-valid name: Check newsfragments are valid + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: newsfragments/.*\.rst$ entry: ./scripts/ci/prek/newsfragments.py @@ -1517,6 +1777,8 @@ repos: always_run: true - id: update-breeze-cmd-output name: Update breeze docs + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Update output of breeze commands in Breeze documentation entry: ./scripts/ci/prek/breeze_cmd_line.py language: python @@ -1529,6 +1791,8 @@ repos: pass_filenames: false - id: check-example-dags-urls name: Check that example dags url include provider versions + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_example_dags_paths.py language: python pass_filenames: true @@ -1540,12 +1804,16 @@ repos: always_run: true - id: check-lazy-logging name: Check that all logging methods are lazy + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_lazy_logging.py language: python pass_filenames: true files: \.py$ - id: check-xcom-model-columns name: Check XComModel columns are read through xcom_entity, not directly in SQLAlchemy queries + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_xcom_model_columns.py language: python pass_filenames: true @@ -1553,24 +1821,32 @@ repos: exclude: ^airflow-core/src/airflow/(models/xcom\.py|api_fastapi/common/parameters/xcom\.py)$ - id: check-no-new-airflow-exceptions name: Check that no new raise AirflowException usages are added + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_new_airflow_exception_usage.py language: python pass_filenames: true files: ^(airflow-core|airflow-ctl|task-sdk|providers|shared)/.*\.py$ - id: check-no-new-provide-session-positional name: Check that no new @provide_session functions declare `session` positionally + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_provide_session_kwargs.py language: python pass_filenames: true files: ^(airflow-core|providers)/.*\.py$|^scripts/ci/prek/known_provide_session_positional\.txt$|^scripts/ci/prek/check_provide_session_kwargs\.py$ - id: check-no-new-airflow-core-utils-modules name: Check that no new modules are added under airflow-core/src/airflow/utils + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_no_new_airflow_core_utils_modules.py language: python pass_filenames: false files: ^airflow-core/src/airflow/utils/[^/]+$|^scripts/ci/prek/known_airflow_core_utils_modules\.txt$|^scripts/ci/prek/check_no_new_airflow_core_utils_modules\.py$ - id: check-mapped-column-nullable-annotations name: Check that nullable mapped_column attributes are annotated Mapped[X | None] + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_mapped_column_nullable_annotations.py language: python pass_filenames: true @@ -1582,6 +1858,8 @@ repos: ^scripts/ci/prek/check_mapped_column_nullable_annotations\.py$ - id: bandit name: bandit + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Bandit is a tool for finding common security issues in Python code" entry: bandit language: python @@ -1599,6 +1877,8 @@ repos: - "high" # TODO: remove this line when we fix all the issues - id: check-k8s-schemas-published name: Check K8s schemas are published on airflow.apache.org + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_k8s_schemas_published.py language: python pass_filenames: false @@ -1612,8 +1892,10 @@ repos: # because under it `--check` only validates the lock, it does not detect staleness. - id: update-uv-lock name: Update uv.lock + env: + PYTHONPYCACHEPREFIX: .build/pycache + UV_FROZEN: "0" entry: uv lock --check - env: {UV_FROZEN: "0"} language: system files: > (?x) @@ -1623,6 +1905,8 @@ repos: require_serial: true - id: mypy-devel-common name: Run mypy for devel-common + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py devel-common pass_filenames: false @@ -1630,13 +1914,25 @@ repos: require_serial: true - id: check-shared-mypy-hooks name: Every shared/ has a mypy-shared- prek hook + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_shared_mypy_hooks.py pass_filenames: false files: ^shared/.*|^scripts/ci/prek/check_shared_mypy_hooks\.py$ require_serial: true + - id: check-prek-hooks-pycache-prefix + name: Every prek hook writes Python bytecode outside the source tree + env: + PYTHONPYCACHEPREFIX: .build/pycache + language: python + entry: ./scripts/ci/prek/check_prek_hooks_pycache_prefix.py + pass_filenames: false + files: (^|/)\.pre-commit-config\.yaml$|^scripts/ci/prek/check_prek_hooks_pycache_prefix\.py$ - id: check-migration-patterns name: Check migration files for anti-patterns and style (MIG001-MIG004) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_migration_patterns.py language: python pass_filenames: true @@ -1648,12 +1944,16 @@ repos: ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_template_fields.py files: ^(providers/[^/]+|airflow-core)/src/airflow/(.*/)?(sensors|operators)/.*\.py$ require_serial: true - id: check-execution-api-versions name: Check execution API datamodel changes have corresponding version updates + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_execution_api_versions.py language: python pass_filenames: true @@ -1661,6 +1961,8 @@ repos: require_serial: true - id: generate-supervisor-schemas-snapshot name: Regenerate supervisor schema snapshot + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_supervisor_schemas_snapshot.py language: python pass_filenames: false @@ -1668,6 +1970,8 @@ repos: require_serial: true - id: check-supervisor-schemas-versions name: Check supervisor schema changes have corresponding version updates + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_supervisor_schemas_versions.py language: python pass_filenames: true @@ -1675,6 +1979,8 @@ repos: require_serial: true - id: generate-yaml-importer-schema-snapshot name: Regenerate YAML DAG importer schema snapshot + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: uv run -p 3.12 --frozen --no-progress --project task-sdk -s scripts/ci/prek/generate_yaml_importer_schema.py language: python pass_filenames: false @@ -1682,6 +1988,8 @@ repos: require_serial: true - id: check-yaml-importer-schema-versions name: Check YAML DAG importer schema changes have corresponding version updates + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_yaml_importer_schema_versions.py language: python pass_filenames: true @@ -1689,6 +1997,8 @@ repos: require_serial: true - id: generate-tasksdk-datamodels name: Generate Datamodels for TaskSDK client + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: uv run -p 3.12 --no-progress --active --group codegen --package apache-airflow-task-sdk --directory task-sdk -s dev/generate_task_sdk_models.py pass_filenames: false @@ -1696,6 +2006,8 @@ repos: require_serial: true - id: generate-airflowctl-datamodels name: Generate Datamodels for AirflowCTL + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: > bash -c ' @@ -1709,12 +2021,16 @@ repos: require_serial: true - id: check-contextmanager-class-decorators name: Check for problematic context manager class decorators + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py language: python files: .*test.*\.py$ pass_filenames: true - id: ktlint name: Run ktlint format + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Use ktlint (via Gradle) to format Kotlin and Java files" entry: ./java-sdk/gradlew -p ./java-sdk ktlintFormat language: system @@ -1722,6 +2038,8 @@ repos: files: ^java-sdk/.*$ - id: spotless name: Run spotless apply + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Use spotless (via Gradle) to format Java files" entry: ./java-sdk/gradlew -p ./java-sdk spotlessApply language: system diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml index a16401bf481d6..293a892159b06 100644 --- a/airflow-core/.pre-commit-config.yaml +++ b/airflow-core/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: check-first-limit name: Check single-row lookups have an explicit limit + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_first_limit.py language: python files: ^src/airflow/.*\.py$ pass_filenames: true - id: check-airflow-k8s-not-used name: Check airflow.kubernetes imports are not used + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/.*\.py$ require_serial: true @@ -41,6 +45,8 @@ repos: --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`." - id: check-common-compat-used-for-openlineage name: Check common.compat is used for OL deprecated classes + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/.*\.py$ require_serial: true @@ -50,6 +56,8 @@ repos: --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead." - id: check-cncf-k8s-only-for-executors name: Check cncf.kubernetes imports used for executors only + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/.*\.py$ require_serial: true @@ -66,12 +74,16 @@ repos: "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`." - id: check-extra-packages-references name: Checks setup extra packages + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Checks if extras defined in hatch_build.py are listed in extra-packages-ref.rst file language: python files: ^docs/extra-packages-ref\.rst$|^hatch_build\.py$ pass_filenames: false entry: ../scripts/ci/prek/check_extra_packages_ref.py - id: check-no-providers-in-core-examples + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: No providers imports in core example DAGs description: The core example DAGs have no dependencies other than standard provider or core Airflow @@ -80,6 +92,8 @@ repos: files: ^src/airflow/example_dags/.*\.py$ - id: check-only-new-session-with-provide-session name: Check NEW_SESSION is only used with @provide_session + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/new_session_in_provide_session.py pass_filenames: true @@ -87,18 +101,24 @@ repos: exclude: ^src/airflow/serialization/pydantic/.* - id: check-base-operator-partial-arguments name: Check BaseOperator and partial() arguments + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_base_operator_partial_arguments.py pass_filenames: false files: ^src/airflow/models/(?:base|mapped)operator\.py$ - id: check-decorated-operator-implements-custom-name name: Check @task decorator implements custom_operator_name + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/decorator_operator_implements_custom_name.py pass_filenames: true files: ^src/airflow/.*\.py$ - id: compile-ui-assets name: Compile ui assets (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['manual'] types_or: [javascript, ts, tsx] @@ -111,6 +131,8 @@ repos: additional_dependencies: ['pnpm@10.25.0'] - id: compile-ui-assets-dev name: Compile ui assets in dev mode (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['manual'] types_or: [javascript, ts, tsx] @@ -123,6 +145,8 @@ repos: additional_dependencies: ['pnpm@10.25.0'] - id: check-tests-in-the-right-folders name: Check if tests are in the right folders + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_tests_in_right_folders.py language: python files: ^tests/.*\.py$ @@ -130,6 +154,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint config_templates/config.yml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_json_schema.py args: - --spec-file @@ -140,12 +166,16 @@ repos: require_serial: true - id: check-code-deprecations name: Check deprecations categories in decorators + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_deprecations.py language: python pass_filenames: true files: ^src/airflow/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true @@ -157,24 +187,32 @@ repos: ^tests/unit/utils/test_serve_logs\.py$ - id: check-openapi-exception-doc-in-sync name: Check API routes declare the HTTP statuses they raise + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_openapi_exception_doc_in_sync.py language: python pass_filenames: true files: ^src/airflow/api_fastapi/.*/routes/.*\.py$ - id: create-missing-init-py-files-tests name: Create missing init.py files in tests + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_init_in_tests.py language: python pass_filenames: false files: ^tests/.*\.py$ - id: check-tests-unittest-testcase name: Unit tests do not inherit from unittest.TestCase + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check that unit tests do not inherit from unittest.TestCase entry: ../scripts/ci/prek/unittest_testcase.py language: python pass_filenames: true files: ^tests/.*\.py$ - id: check-no-fab-migrations + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check no migration is done on FAB related table description: > @@ -202,6 +240,8 @@ repos: ^src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key\.py$ - id: ts-compile-lint-ui name: Compile / format / lint UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files language: node files: | @@ -218,6 +258,8 @@ repos: require_serial: true - id: ts-compile-lint-simple-auth-manager-ui name: Compile / format / lint simple auth manager UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files language: node files: | @@ -235,6 +277,8 @@ repos: require_serial: true - id: mypy-airflow-core name: Run mypy for airflow-core + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-core pass_filenames: false @@ -244,6 +288,8 @@ repos: # The below prek hooks are those requiring CI image to be built - id: generate-openapi-spec name: Generate the FastAPI API spec + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/generate_openapi_spec.py pass_filenames: false @@ -254,6 +300,8 @@ repos: exclude: ^src/airflow/api_fastapi/execution_api/.* - id: check-i18n-json name: Check i18n files validity + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check i18n files are valid json, have no TODOs, and auto-format them language: python files: ^src/airflow/ui/public/i18n/locales/.*\.json$ @@ -261,6 +309,8 @@ repos: pass_filenames: false - id: update-migration-references name: Update migration ref doc + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/migration_reference.py args: ["--app", "airflow"] @@ -271,6 +321,8 @@ repos: ^docs/migrations-ref\.rst$ - id: migration-round-trip name: Migration round-trip SQLite FK enforcement + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/migration_round_trip.py pass_filenames: false @@ -279,6 +331,8 @@ repos: files: ^src/airflow/migrations/versions/.*\.py$ - id: check-default-configuration name: Check the default configuration + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_default_configuration.py language: python require_serial: true @@ -286,6 +340,8 @@ repos: files: ^src/airflow/config_templates/config\.yml$ - id: check-security-doc-constants name: Check security docs match config.yml constants + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_security_doc_constants.py language: python pass_filenames: false @@ -295,6 +351,8 @@ repos: ^docs/security/jwt_token_authentication\.rst$| ^docs/security/security_model\.rst$ - id: check-airflow-version-checks-in-core + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: No AIRFLOW_V_* imports in airflow-core entry: "import AIRFLOW_V_" @@ -312,6 +370,8 @@ repos: ^tests/unit/models/test_variable\.py$ - id: check-sdk-imports-in-core name: Check for SDK imports in core files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_sdk_imports_in_core.py language: python types: [python] @@ -319,6 +379,8 @@ repos: ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-schema-defaults name: Check schema defaults match server-side defaults + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_schema_defaults.py language: python files: diff --git a/airflow-ctl-tests/.pre-commit-config.yaml b/airflow-ctl-tests/.pre-commit-config.yaml index 1016fa939ebca..791ddc4d8eb6a 100644 --- a/airflow-ctl-tests/.pre-commit-config.yaml +++ b/airflow-ctl-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: check-airflowctl-command-coverage name: Check airflowctl CLI command test coverage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_airflowctl_command_coverage.py language: python pass_filenames: false @@ -32,6 +34,8 @@ repos: ^tests/airflowctl_tests/.*\.py$ - id: mypy-airflow-ctl-tests name: Run mypy for airflow-ctl-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl-tests pass_filenames: false diff --git a/airflow-ctl/.pre-commit-config.yaml b/airflow-ctl/.pre-commit-config.yaml index 4561533c574d0..24353137cbb6f 100644 --- a/airflow-ctl/.pre-commit-config.yaml +++ b/airflow-ctl/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: mypy-airflow-ctl name: Run mypy for airflow-ctl + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl pass_filenames: false @@ -33,6 +35,8 @@ repos: require_serial: true - id: generate-airflowctl-help-images name: Generate SVG from Airflow CTL Commands + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/capture_airflowctl_help.py language: python pass_filenames: false @@ -43,6 +47,8 @@ repos: ^src/airflowctl/ctl/commands/.*\.py$ - id: check-airflowctl-command-coverage name: Check airflowctl CLI command test coverage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_airflowctl_command_coverage.py language: python pass_filenames: false @@ -52,6 +58,8 @@ repos: ^docs/images/command_hashes.txt$ - id: check-airflowctl-help-texts name: Check airflowctl CLI commands have help texts + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_airflowctl_help_texts.py language: python pass_filenames: false diff --git a/airflow-e2e-tests/.pre-commit-config.yaml b/airflow-e2e-tests/.pre-commit-config.yaml index e3d6413706dfe..0819cbdda0dda 100644 --- a/airflow-e2e-tests/.pre-commit-config.yaml +++ b/airflow-e2e-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-airflow-e2e-tests name: Run mypy for airflow-e2e-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-e2e-tests pass_filenames: false diff --git a/chart/.pre-commit-config.yaml b/chart/.pre-commit-config.yaml index 6267bd1bfc378..fd60314f6ceaa 100644 --- a/chart/.pre-commit-config.yaml +++ b/chart/.pre-commit-config.yaml @@ -27,6 +27,8 @@ repos: hooks: - id: insert-license name: Add license for all Helm template files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: ^templates/.* args: - --comment-style @@ -38,6 +40,8 @@ repos: hooks: - id: update-chart-dependencies name: Update chart dependencies to latest (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_chart_dependencies.py stages: ['manual'] language: python @@ -46,6 +50,8 @@ repos: require_serial: true - id: mypy-helm-tests name: Run mypy for helm-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py chart/tests pass_filenames: false @@ -56,6 +62,8 @@ repos: hooks: - id: pretty-format-json name: Format JSON files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --autofix - --no-sort-keys @@ -70,6 +78,8 @@ repos: hooks: - id: lint-helm-chart name: Lint Helm Chart + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_helm.py language: python pass_filenames: false @@ -77,12 +87,16 @@ repos: require_serial: true - id: validate-chart-annotations name: Validate chart annotations + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/validate_chart_annotations.py language: python pass_filenames: false files: ^Chart\.yaml$ - id: kubeconform name: Kubeconform check on our helm chart + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_kubeconform.py language: python pass_filenames: false @@ -90,6 +104,8 @@ repos: require_serial: true - id: build-kustomize-overlays name: Build and smoke-test Kustomize overlays + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/build_kustomize_overlays.py language: python pass_filenames: false @@ -97,6 +113,8 @@ repos: require_serial: true - id: sort-helm-values-schema-file-type-field name: Sort Helm Values Schema File (type field) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/sort_helm_values_schema_file_type_field.py language: python pass_filenames: false @@ -104,6 +122,8 @@ repos: require_serial: true - id: update-helm-values-schema-schema-file-type-field-match name: Update Helm Values Schema Schema File (type field match) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_helm_schema_schema_type_fields_match.py language: python pass_filenames: false @@ -111,6 +131,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint chart/values.schema.json + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_json_schema.py args: - --spec-file @@ -122,11 +144,15 @@ repos: require_serial: true - id: update-vendored-in-k8s-json-schema name: Vendor k8s definitions into values.schema.json + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/vendor_k8s_json_schema.py language: python files: ^values\.schema\.json$ - id: lint-json-schema name: Lint chart/values.yaml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_json_schema.py args: - --enforce-defaults @@ -139,6 +165,8 @@ repos: require_serial: true - id: lint-chart-schema name: Lint chart/values.schema.json file + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/chart_schema.py language: python pass_filenames: false diff --git a/dev/.pre-commit-config.yaml b/dev/.pre-commit-config.yaml index f401bd9754594..98e0d6f1d3d35 100644 --- a/dev/.pre-commit-config.yaml +++ b/dev/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-dev name: Run mypy for dev + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py dev pass_filenames: false diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 814e27b4f311f..3ae4775745911 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -21,6 +21,7 @@ from __future__ import annotations import contextlib +import errno import hashlib import os import platform @@ -622,6 +623,11 @@ def cleanup_python_generated_files(): pass except PermissionError: permission_errors.append(Path(dirpath)) + except OSError as e: + # A concurrent breeze process (e.g. parallel prek hooks) can write a fresh .pyc + # between rmtree emptying the directory and removing it. + if e.errno != errno.ENOTEMPTY: + raise dirnames.clear() if permission_errors: if platform.uname().system.lower() == "linux": diff --git a/dev/breeze/tests/test_path_utils.py b/dev/breeze/tests/test_path_utils.py new file mode 100644 index 0000000000000..2c161a01d4e4a --- /dev/null +++ b/dev/breeze/tests/test_path_utils.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import errno +from unittest import mock + +import pytest + +from airflow_breeze.utils import path_utils +from airflow_breeze.utils.path_utils import cleanup_python_generated_files + + +@pytest.fixture +def root_with_pycache(tmp_path, monkeypatch): + pycache = tmp_path / "pkg" / "__pycache__" + pycache.mkdir(parents=True) + (pycache / "mod.cpython-310.pyc").write_bytes(b"") + monkeypatch.setattr(path_utils, "AIRFLOW_ROOT_PATH", tmp_path) + return pycache + + +@mock.patch.object(path_utils.shutil, "rmtree", autospec=True) +def test_cleanup_ignores_pycache_repopulated_concurrently(mock_rmtree, root_with_pycache): + mock_rmtree.side_effect = OSError(errno.ENOTEMPTY, "Directory not empty") + + cleanup_python_generated_files() + + mock_rmtree.assert_called_once_with(str(root_with_pycache)) + + +@mock.patch.object(path_utils.shutil, "rmtree", autospec=True) +def test_cleanup_reraises_other_os_errors(mock_rmtree, root_with_pycache): + mock_rmtree.side_effect = OSError(errno.EIO, "I/O error") + + with pytest.raises(OSError, match="I/O error"): + cleanup_python_generated_files() diff --git a/dev/mypy/.pre-commit-config.yaml b/dev/mypy/.pre-commit-config.yaml index a75260a92762d..2e7f12f2a9fa7 100644 --- a/dev/mypy/.pre-commit-config.yaml +++ b/dev/mypy/.pre-commit-config.yaml @@ -25,6 +25,8 @@ repos: - repo: local hooks: - id: mypy-airflow-mypy + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['pre-push'] name: Run mypy for airflow-mypy plugins language: python @@ -35,6 +37,8 @@ repos: ^tests/.*\.py$ require_serial: true - id: mypy-airflow-mypy-manual + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['manual'] name: Run mypy for airflow-mypy (manual) language: python diff --git a/docker-tests/.pre-commit-config.yaml b/docker-tests/.pre-commit-config.yaml index 27e73ecc067d4..c3cc7803445a1 100644 --- a/docker-tests/.pre-commit-config.yaml +++ b/docker-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-docker-tests name: Run mypy for docker-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py docker-tests pass_filenames: false diff --git a/go-sdk/.pre-commit-config.yaml b/go-sdk/.pre-commit-config.yaml index 523b64d8dd077..ee0f25b2ebaea 100644 --- a/go-sdk/.pre-commit-config.yaml +++ b/go-sdk/.pre-commit-config.yaml @@ -26,14 +26,20 @@ repos: hooks: - id: identity name: Print checked files + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Print input to the static check hooks for troubleshooting - id: check-hooks-apply name: Check if all hooks apply to the repository + env: + PYTHONPYCACHEPREFIX: .build/pycache - repo: https://github.com/Lucas-C/pre-commit-hooks rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 hooks: - id: insert-license name: Add license for all Go files + env: + PYTHONPYCACHEPREFIX: .build/pycache types: [go] exclude: mocks/.*\.go$|\.pb\.go$ args: @@ -48,6 +54,8 @@ repos: hooks: - id: go-mod-tidy name: Run go mod tidy + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: go mod tidy types: [go] exclude: mocks/.*\.go$ @@ -55,6 +63,8 @@ repos: language: golang - id: imports name: Consistent import ordering for Go files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: goimports-reviser -rm-unused -set-exit-status additional_dependencies: [github.com/incu6us/goimports-reviser/v3@v3.10.0] types: [go] @@ -62,12 +72,16 @@ repos: require_serial: true - id: gofmt name: Format go code + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: golines --base-formatter=gofumpt --write-output --max-len=100 --chain-split-dots additional_dependencies: [github.com/segmentio/golines@latest, mvdan.cc/gofumpt@v0.8.0] types: [go] language: golang - id: staticcheck name: Check naming in Go code + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: staticcheck -checks ST1003 ./... additional_dependencies: [honnef.co/go/tools/cmd/staticcheck@v0.7.0] types: [go] diff --git a/kubernetes-tests/.pre-commit-config.yaml b/kubernetes-tests/.pre-commit-config.yaml index 4bfb8077b9c3b..0c5ae79e0d544 100644 --- a/kubernetes-tests/.pre-commit-config.yaml +++ b/kubernetes-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-kubernetes-tests name: Run mypy for kubernetes-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py kubernetes-tests pass_filenames: false diff --git a/providers/.pre-commit-config.yaml b/providers/.pre-commit-config.yaml index 9e48891a1e6aa..b2010ad83a7d2 100644 --- a/providers/.pre-commit-config.yaml +++ b/providers/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: check-deferrable-default name: Check and fix default value of default_deferrable + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_deferrable_default.py pass_filenames: false files: ^(.*/)?airflow/.*/(sensors|operators)/.*\.py$ - id: check-trigger-serialize-init name: Check trigger __init__ and serialize() are in sync + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Every trigger __init__ parameter must appear in its serialize() return dict language: python entry: ../scripts/ci/prek/check_trigger_serialize_init.py @@ -39,6 +43,8 @@ repos: files: ^.*/src/airflow/providers/.*/triggers/[^/]+\.py$ - id: check-deferred-hook-configuration name: Check deferred AWS tasks pass their hook configuration + env: + PYTHONPYCACHEPREFIX: .build/pycache description: An operator that defers must hand region_name, verify and botocore_config to its trigger language: python entry: ../scripts/ci/prek/check_deferred_hook_configuration.py @@ -46,6 +52,8 @@ repos: files: ^amazon/src/airflow/providers/amazon/aws/.*\.py$ - id: check-migration-patterns name: Check migration files for anti-patterns and style (MIG001-MIG004) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_migration_patterns.py language: python pass_filenames: true @@ -54,12 +62,16 @@ repos: require_serial: true - id: update-providers-dependencies name: Update dependencies for providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_providers_dependencies.py language: python always_run: true pass_filenames: false - id: validate-operators-init name: Validate templated fields in operator __init__ + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Require plain assignment of templated fields and forbid reading their un-rendered values language: python entry: ../scripts/ci/prek/validate_operators_init.py @@ -67,6 +79,8 @@ repos: files: ^.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ - id: check-conn-id-templated name: Check connection ids accepted by operators are in template_fields + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Every *conn_id argument an operator, sensor or notifier accepts must be templated language: python entry: ../scripts/ci/prek/check_conn_id_templated.py @@ -76,6 +90,8 @@ repos: require_serial: true - id: update-providers-build-files name: Update providers build files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_providers_build_files.py language: python pass_filenames: true @@ -91,24 +107,32 @@ repos: require_serial: true - id: check-airflow-v-imports-in-tests name: Check AIRFLOW_V imports in tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_airflow_v_imports_in_tests.py pass_filenames: true files: ^.*/tests/.+\.py$ - id: check-sql-dependency-common-data-structure name: Check dependency of SQL providers + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check dependency of SQL Providers with common data structure entry: ../scripts/ci/prek/check_common_sql_dependency.py language: python files: ^.*/src/airflow/providers/.*/hooks/.*\.py$ - id: generate-volumes-for-sources name: Generate volumes for docker compose + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/generate_volumes_for_sources.py language: python files: ^.*/provider\.yaml$ pass_filenames: false require_serial: true - id: check-pytest-mark-db-test-in-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check pytest.mark.db_test use in providers entry: pytest\.mark\.db_test @@ -169,6 +193,8 @@ repos: ^zendesk/.*\.py$ - id: check-links-to-example-dags-do-not-use-hardcoded-versions name: Verify no hard-coded version in example dags + env: + PYTHONPYCACHEPREFIX: .build/pycache description: The links to example dags should use |version| as version specification language: pygrep entry: > @@ -178,6 +204,8 @@ repos: pass_filenames: true files: ^.*/docs/.*\.rst - id: check-no-airflow-deprecation-in-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Do not use DeprecationWarning in providers description: Use AirflowProviderDeprecationWarning in providers @@ -185,6 +213,8 @@ repos: pass_filenames: true files: ^.*/src/airflow/providers/.*\.py$ - id: check-base-operator-usage + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check BaseOperator other imports description: Make sure BaseOperator is imported from airflow.models outside of core @@ -193,6 +223,8 @@ repos: files: ^.*/src/airflow/providers/.*\.py$ exclude: ^standard/.*/.*\.py$ - id: check-get-lineage-collector-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python name: Check providers import hook lineage code from compat description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of @@ -205,6 +237,8 @@ repos: exclude: ^common/compat/src/airflow/providers/common/compat/.*\.py$ - id: check-aiobotocore-optional name: Check if aiobotocore is an optional dependency only + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_aiobotocore_optional.py language: python files: ^.*/provider\.yaml$ @@ -212,6 +246,8 @@ repos: require_serial: true - id: check-providers-subpackages-init-file-exist name: Provider subpackage init files are there + env: + PYTHONPYCACHEPREFIX: .build/pycache pass_filenames: false always_run: true entry: ../scripts/ci/prek/check_providers_subpackages_all_have_init.py @@ -219,12 +255,16 @@ repos: require_serial: true - id: check-system-tests-tocs name: Check that system tests is properly added + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_system_tests_hidden_in_index.py language: python pass_filenames: true files: ^.*/docs/index\.rst$ - id: check-provider-docs-valid name: Validate provider doc files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_provider_docs.py language: python files: ^.*/provider\.yaml$|^.*/docs/.* @@ -232,6 +272,8 @@ repos: pass_filenames: false - id: check-connection-doc-labels name: Validate connection doc labels match provider.yaml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_connection_doc_labels.py language: python files: ^.*/provider\.yaml$|^.*/docs/.*/connections.*\.rst$|^.*/docs/connections\.rst$ @@ -239,12 +281,16 @@ repos: pass_filenames: false - id: check-common-ai-operators-index name: Check common.ai operators are listed in the docs index + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_common_ai_operators_index.py language: python files: ^common/ai/(src/airflow/providers/common/ai/operators/.*\.py|docs/operators/index\.rst)$ require_serial: true pass_filenames: false - id: mypy-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['pre-push'] name: Run mypy for providers language: python @@ -252,6 +298,8 @@ repos: files: ^.*\.py$ require_serial: true - id: mypy-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['manual'] name: Run mypy for providers (manual) language: python @@ -261,6 +309,8 @@ repos: require_serial: true - id: check-provider-yaml-valid name: Validate provider.yaml files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_provider_yaml_files.py language: python files: > @@ -271,18 +321,24 @@ repos: require_serial: true - id: check-cli-definition-imports name: Check CLI definition files only import allowed modules + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_cli_definition_imports.py language: python files: ^.*/cli/definition\.py$ require_serial: false - id: check-imports-in-providers name: Check imports in providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_imports_in_providers.py language: python files: ^.*/src/airflow/providers/.*version_compat.*\.py$ require_serial: true - id: check-conf-import-in-providers name: Check conf is imported from compat SDK in providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_conf_import_in_providers.py language: python types: [python] @@ -291,6 +347,8 @@ repos: require_serial: false - id: provider-version-compat name: Check for correct version_compat imports in providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_provider_version_compat.py language: python types: [python] @@ -301,6 +359,8 @@ repos: hooks: - id: ast-grep name: Check providers import httpx2 rather than httpx + env: + PYTHONPYCACHEPREFIX: .build/pycache types: [python] # Pinned to the providers config rather than relying on ast-grep walking up # from the working directory, so a future root sgconfig.yml cannot shadow it. diff --git a/providers/amazon/.pre-commit-config.yaml b/providers/amazon/.pre-commit-config.yaml index 3e634c3ef2df7..eda8ca78672ac 100644 --- a/providers/amazon/.pre-commit-config.yaml +++ b/providers/amazon/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true diff --git a/providers/common/ai/.pre-commit-config.yaml b/providers/common/ai/.pre-commit-config.yaml index 396081858a82e..10929d328da3b 100644 --- a/providers/common/ai/.pre-commit-config.yaml +++ b/providers/common/ai/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: ts-compile-lint-hitl-review-ui name: Compile / format / lint HITL Review UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files in HITL Review language: node files: | @@ -41,6 +43,8 @@ repos: require_serial: true - id: compile-common-ai-provider-assets name: Compile Common AI provider assets + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['pre-commit', 'manual'] files: ^src/airflow/providers/common/ai/plugins/www/ @@ -49,6 +53,8 @@ repos: additional_dependencies: ['pnpm@10.25.0'] - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true diff --git a/providers/common/compat/.pre-commit-config.yaml b/providers/common/compat/.pre-commit-config.yaml index 71cfbd6dad26e..537eeaf88b253 100644 --- a/providers/common/compat/.pre-commit-config.yaml +++ b/providers/common/compat/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: check-common-compat-sdk-imports-in-sync name: Check common.compat sdk TYPE_CHECKING matches runtime maps + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/providers/common/compat/sdk\.py$ pass_filenames: false diff --git a/providers/edge3/.pre-commit-config.yaml b/providers/edge3/.pre-commit-config.yaml index 000f8b6dab6f5..d1b7ab19cc7e2 100644 --- a/providers/edge3/.pre-commit-config.yaml +++ b/providers/edge3/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: generate-openapi-spec-edge name: Generate the FastAPI API spec for Edge + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/generate_openapi_spec_providers.py edge pass_filenames: false files: ^src/airflow/providers/edge3/worker_api/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true @@ -41,6 +45,8 @@ repos: ^tests/unit/edge3/(worker_api|plugins)/.*\.py$ - id: ts-compile-lint-edge-ui name: Compile / format / lint edge UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files in Edge Provider language: node files: | @@ -57,6 +63,8 @@ repos: require_serial: true - id: update-migration-references-edge3 name: Update migration ref doc for Edge3 + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/migration_reference.py args: ["--app", "edge3"] @@ -67,6 +75,8 @@ repos: ^(providers/edge3/)?docs/migrations-ref\.rst$ - id: check-revision-heads-map-edge3 name: Check that the REVISION_HEADS_MAP is up-to-date + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/check_revision_heads_map.py pass_filenames: false @@ -76,6 +86,8 @@ repos: ^(providers/edge3/)?src/airflow/providers/edge3/models/db\.py$ - id: compile-edge-assets name: Compile Edge provider assets + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['pre-commit', 'manual'] files: ^src/airflow/providers/edge3/plugins/www diff --git a/providers/fab/.pre-commit-config.yaml b/providers/fab/.pre-commit-config.yaml index 2b5032ea3650f..560e3c9fe9cd2 100644 --- a/providers/fab/.pre-commit-config.yaml +++ b/providers/fab/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: compile-fab-assets name: Compile FAB provider assets + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['pre-commit', 'manual'] files: ^src/airflow/providers/fab/www/ @@ -38,12 +40,16 @@ repos: additional_dependencies: ['pnpm@10.25.0'] - id: generate-openapi-spec-fab name: Generate the FastAPI API spec for FAB + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/generate_openapi_spec_providers.py fab pass_filenames: false files: ^src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true @@ -53,6 +59,8 @@ repos: ^tests/unit/fab/auth_manager/api_fastapi/.*\.py$ - id: update-migration-references-fab name: Update migration ref doc for FAB + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/migration_reference.py args: ["--app", "fab"] @@ -63,6 +71,8 @@ repos: ^(providers/fab/)?docs/migrations-ref\.rst$ - id: check-revision-heads-map-fab name: Check that the REVISION_HEADS_MAP is up-to-date + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/check_revision_heads_map.py pass_filenames: false @@ -72,6 +82,8 @@ repos: ^(providers/fab/)?src/airflow/providers/fab/auth_manager/models/db\.py$ - id: prevent-deprecated-sqlalchemy-usage-fab name: Prevent deprecated sqlalchemy usage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py language: python files: ^.*\.py$ diff --git a/providers/keycloak/.pre-commit-config.yaml b/providers/keycloak/.pre-commit-config.yaml index 2215a6cd5a0f5..d7c5319f98ce2 100644 --- a/providers/keycloak/.pre-commit-config.yaml +++ b/providers/keycloak/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: generate-openapi-spec-keycloak name: Generate the FastAPI API spec for Keycloak + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/generate_openapi_spec_providers.py keycloak pass_filenames: false files: ^src/airflow/providers/keycloak/auth_manager/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true diff --git a/scripts/.pre-commit-config.yaml b/scripts/.pre-commit-config.yaml index 2b55cb7cfc9a4..dc44bf615ef86 100644 --- a/scripts/.pre-commit-config.yaml +++ b/scripts/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-scripts name: Run mypy for scripts + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py scripts pass_filenames: false diff --git a/scripts/ci/prek/AGENTS.md b/scripts/ci/prek/AGENTS.md index bcaf0b8a4d51c..d8b605f668483 100644 --- a/scripts/ci/prek/AGENTS.md +++ b/scripts/ci/prek/AGENTS.md @@ -32,3 +32,7 @@ When adding a new breeze-dependent hook: - Register the script in the appropriate `.pre-commit-config.yaml` (`/.pre-commit-config.yaml` for repo-wide hooks, `/airflow-core/.pre-commit-config.yaml` for core-specific hooks, or a provider-level config). +- Give every hook `env: {PYTHONPYCACHEPREFIX: .build/pycache}`. Hooks run in parallel and + breeze deletes `__pycache__` directories on start-up, so a hook writing `.pyc` next to the + sources can make a concurrent breeze hook fail with `Directory not empty`. + `check-prek-hooks-pycache-prefix` enforces this. diff --git a/scripts/ci/prek/check_prek_hooks_pycache_prefix.py b/scripts/ci/prek/check_prek_hooks_pycache_prefix.py new file mode 100755 index 0000000000000..c22e3cc2fc7e6 --- /dev/null +++ b/scripts/ci/prek/check_prek_hooks_pycache_prefix.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0.3", +# ] +# /// +"""Fail when a prek hook does not redirect Python bytecode out of the source tree. + +prek runs hooks in parallel, and breeze removes every ``__pycache__`` under the +repository on start-up. A hook that writes ``.pyc`` files next to the sources can +refill a directory while a concurrent breeze is removing it, failing that hook +with ``Directory not empty``. Every hook therefore sets ``PYTHONPYCACHEPREFIX`` +to a hidden ``.build/`` directory, which breeze skips and git ignores. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import yaml + +PYCACHE_PREFIX = ".build/pycache" + + +def find_hooks_without_pycache_prefix(config_path: Path) -> list[str]: + config = yaml.safe_load(config_path.read_text()) or {} + return [ + hook["id"] + for repo in config.get("repos", []) + for hook in repo.get("hooks", []) + if (hook.get("env") or {}).get("PYTHONPYCACHEPREFIX") != PYCACHE_PREFIX + ] + + +def main() -> int: + config_paths = subprocess.check_output( + ["git", "ls-files", "*.pre-commit-config.yaml", ".pre-commit-config.yaml"], text=True + ).split() + errors = [ + f"{config_path}: {hook_id}" + for config_path in config_paths + for hook_id in find_hooks_without_pycache_prefix(Path(config_path)) + ] + if errors: + print("ERROR: These prek hooks do not set PYTHONPYCACHEPREFIX:\n") + print("\n".join(f" {error}" for error in errors)) + print( + "\nAdd the following to each of them, right after `name:`:\n\n" + " env:\n" + f" PYTHONPYCACHEPREFIX: {PYCACHE_PREFIX}\n" + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/prek/check_shared_mypy_hooks.py b/scripts/ci/prek/check_shared_mypy_hooks.py index fe30f36a3aa22..6aea7fd4a7ec0 100755 --- a/scripts/ci/prek/check_shared_mypy_hooks.py +++ b/scripts/ci/prek/check_shared_mypy_hooks.py @@ -47,6 +47,8 @@ hooks: - id: mypy-shared-{dist} name: Run mypy for shared-{dist} + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: >- ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py diff --git a/scripts/tests/ci/prek/test_check_prek_hooks_pycache_prefix.py b/scripts/tests/ci/prek/test_check_prek_hooks_pycache_prefix.py new file mode 100644 index 0000000000000..c3ed9ad9d0ff4 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_prek_hooks_pycache_prefix.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from check_prek_hooks_pycache_prefix import find_hooks_without_pycache_prefix + +CONFIG = """\ +repos: + - repo: meta + hooks: + - id: with-prefix + env: + PYTHONPYCACHEPREFIX: .build/pycache + - repo: local + hooks: + - id: no-env + - id: other-env + env: + FOO: bar + - id: wrong-prefix + env: + PYTHONPYCACHEPREFIX: /tmp/pycache +""" + + +def test_find_hooks_without_pycache_prefix(tmp_path): + config = tmp_path / ".pre-commit-config.yaml" + config.write_text(CONFIG) + + assert find_hooks_without_pycache_prefix(config) == ["no-env", "other-env", "wrong-prefix"] + + +def test_find_hooks_without_pycache_prefix_empty_config(tmp_path): + config = tmp_path / ".pre-commit-config.yaml" + config.write_text("") + + assert find_hooks_without_pycache_prefix(config) == [] diff --git a/shared/configuration/.pre-commit-config.yaml b/shared/configuration/.pre-commit-config.yaml index 9706d8602d6f3..52dbc4dbcdd78 100644 --- a/shared/configuration/.pre-commit-config.yaml +++ b/shared/configuration/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-configuration name: Run mypy for shared-configuration + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/configuration pass_filenames: false diff --git a/shared/dagnode/.pre-commit-config.yaml b/shared/dagnode/.pre-commit-config.yaml index 34f7fe42212b1..523a4709c6f73 100644 --- a/shared/dagnode/.pre-commit-config.yaml +++ b/shared/dagnode/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-dagnode name: Run mypy for shared-dagnode + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/dagnode pass_filenames: false diff --git a/shared/listeners/.pre-commit-config.yaml b/shared/listeners/.pre-commit-config.yaml index 62f226fab7407..abf48f879ab1e 100644 --- a/shared/listeners/.pre-commit-config.yaml +++ b/shared/listeners/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-listeners name: Run mypy for shared-listeners + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/listeners pass_filenames: false diff --git a/shared/logging/.pre-commit-config.yaml b/shared/logging/.pre-commit-config.yaml index 7440bc4add9a1..07bb047675985 100644 --- a/shared/logging/.pre-commit-config.yaml +++ b/shared/logging/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-logging name: Run mypy for shared-logging + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/logging pass_filenames: false diff --git a/shared/module_loading/.pre-commit-config.yaml b/shared/module_loading/.pre-commit-config.yaml index fd00e315b0e53..a4d31f74c4dd8 100644 --- a/shared/module_loading/.pre-commit-config.yaml +++ b/shared/module_loading/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-module_loading name: Run mypy for shared-module_loading + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/module_loading pass_filenames: false diff --git a/shared/observability/.pre-commit-config.yaml b/shared/observability/.pre-commit-config.yaml index eab5206f13303..0184d22126b19 100644 --- a/shared/observability/.pre-commit-config.yaml +++ b/shared/observability/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-observability name: Run mypy for shared-observability + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/observability pass_filenames: false diff --git a/shared/plugins_manager/.pre-commit-config.yaml b/shared/plugins_manager/.pre-commit-config.yaml index 9180e2b10358e..b3fa3ccc116a1 100644 --- a/shared/plugins_manager/.pre-commit-config.yaml +++ b/shared/plugins_manager/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-plugins_manager name: Run mypy for shared-plugins_manager + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/plugins_manager pass_filenames: false diff --git a/shared/providers_discovery/.pre-commit-config.yaml b/shared/providers_discovery/.pre-commit-config.yaml index 2c26b25a954ff..dd3093def9d73 100644 --- a/shared/providers_discovery/.pre-commit-config.yaml +++ b/shared/providers_discovery/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-providers_discovery name: Run mypy for shared-providers_discovery + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: >- ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py diff --git a/shared/secrets_backend/.pre-commit-config.yaml b/shared/secrets_backend/.pre-commit-config.yaml index 93175830f9f9b..7db5719454566 100644 --- a/shared/secrets_backend/.pre-commit-config.yaml +++ b/shared/secrets_backend/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-secrets_backend name: Run mypy for shared-secrets_backend + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/secrets_backend pass_filenames: false diff --git a/shared/secrets_masker/.pre-commit-config.yaml b/shared/secrets_masker/.pre-commit-config.yaml index 8025b4429c216..296a9d4c90faa 100644 --- a/shared/secrets_masker/.pre-commit-config.yaml +++ b/shared/secrets_masker/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-secrets_masker name: Run mypy for shared-secrets_masker + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/secrets_masker pass_filenames: false diff --git a/shared/serialization/.pre-commit-config.yaml b/shared/serialization/.pre-commit-config.yaml index e0619c11dfd61..a5ec870dc90ea 100644 --- a/shared/serialization/.pre-commit-config.yaml +++ b/shared/serialization/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-serialization name: Run mypy for shared-serialization + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/serialization pass_filenames: false diff --git a/shared/state/.pre-commit-config.yaml b/shared/state/.pre-commit-config.yaml index 9c22fb9a0ec90..7317ae12afd71 100644 --- a/shared/state/.pre-commit-config.yaml +++ b/shared/state/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-state name: Run mypy for shared-state + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/state pass_filenames: false diff --git a/shared/template_rendering/.pre-commit-config.yaml b/shared/template_rendering/.pre-commit-config.yaml index cbe27c38eba39..cfb2023474b8f 100644 --- a/shared/template_rendering/.pre-commit-config.yaml +++ b/shared/template_rendering/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-template_rendering name: Run mypy for shared-template_rendering + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: >- ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py diff --git a/shared/timezones/.pre-commit-config.yaml b/shared/timezones/.pre-commit-config.yaml index 25c6cf6d8a599..01da7365f6fd4 100644 --- a/shared/timezones/.pre-commit-config.yaml +++ b/shared/timezones/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-timezones name: Run mypy for shared-timezones + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/timezones pass_filenames: false diff --git a/task-sdk-integration-tests/.pre-commit-config.yaml b/task-sdk-integration-tests/.pre-commit-config.yaml index f09e0e1f6fe6f..75cee84f0d339 100644 --- a/task-sdk-integration-tests/.pre-commit-config.yaml +++ b/task-sdk-integration-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-task-sdk-integration-tests name: Run mypy for task-sdk-integration-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk-integration-tests pass_filenames: false diff --git a/task-sdk/.pre-commit-config.yaml b/task-sdk/.pre-commit-config.yaml index a44abd29c1566..219e9783209de 100644 --- a/task-sdk/.pre-commit-config.yaml +++ b/task-sdk/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: check-core-imports name: Check for core imports in task-sdk files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_core_imports_in_sdk.py language: python types: [python] @@ -54,12 +56,16 @@ repos: ^src/airflow/sdk/types.py$ - id: check-init-decorator-arguments name: Sync model __init__ and decorator arguments + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_init_decorator_arguments.py pass_filenames: false files: ^src/airflow/sdk/definitions/dag\.py$|^src/airflow/sdk/definitions/decorators/task_group\.py$ - id: mypy-task-sdk name: Run mypy for task-sdk + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk pass_filenames: false diff --git a/ts-sdk/.pre-commit-config.yaml b/ts-sdk/.pre-commit-config.yaml index ea1ad619478ba..0a927e4dd0140 100644 --- a/ts-sdk/.pre-commit-config.yaml +++ b/ts-sdk/.pre-commit-config.yaml @@ -25,6 +25,8 @@ repos: hooks: - id: ts-sdk-lint name: Lint TypeScript SDK + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/ts_sdk_lint.py language: node files: | @@ -40,6 +42,8 @@ repos: require_serial: true - id: check-ts-sdk-supervisor-schema name: Check TypeScript SDK supervisor schema is up to date + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_supervisor_schema.py language: node files: > @@ -52,6 +56,8 @@ repos: require_serial: true - id: check-ts-sdk-dag-schema name: Check TypeScript SDK Dag schema fields are up to date + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_dag_schema.py language: node files: > @@ -64,6 +70,8 @@ repos: require_serial: true - id: check-ts-sdk-serialization-conformance name: Check the TS SDK serializes Dags the way Airflow does + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the TS SDK and with Airflow, and compare the two" entry: ./scripts/ci/prek/check_serialization_conformance.py language: node @@ -78,6 +86,8 @@ repos: require_serial: true - id: check-ts-sdk-docs-package-version-in-sync name: Check ts-sdk/docs package.json versions match ts-sdk/package.json + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_ts_sdk_docs_package_version_in_sync.py language: python files: > @@ -88,6 +98,8 @@ repos: require_serial: true - id: compile-ts-sdk name: Compile TypeScript SDK + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/compile_ts_sdk.py language: node stages: [manual] @@ -103,6 +115,8 @@ repos: require_serial: true - id: verify-ts-sdk-package name: Verify TypeScript SDK package artifact + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/verify_ts_sdk_package.py language: node stages: [manual]