From 72f71a891e52e7261eb15cdc8f1f0a832f2d348b Mon Sep 17 00:00:00 2001 From: Amogh Date: Mon, 18 Mar 2024 14:28:43 +0530 Subject: [PATCH 1/5] Introducing --latest for generate-issue-content-core command --- .github/workflows/check-providers.yml | 3 +- ...management_generate-issue-content-core.svg | 44 +++++++++--------- ...management_generate-issue-content-core.txt | 2 +- .../commands/release_management_commands.py | 45 ++++++++++++++++--- .../release_management_commands_config.py | 1 + 5 files changed, 66 insertions(+), 29 deletions(-) diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index 04f140b536e5c..6590d42d429d0 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -100,8 +100,7 @@ jobs: --only-available-in-dist --disable-progress - name: "Test airflow core issue generation automatically" run: > - breeze release-management generate-issue-content-core - --previous-release 1.9.0 --current-release 1.10.0 + breeze release-management generate-issue-content-core --latest - name: "Generate source constraints from CI image" shell: bash run: > diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-core.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-core.svg index 2b77d88347350..bdf8bb692bd7c 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-core.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-core.svg @@ -1,4 +1,4 @@ - + - + @@ -101,17 +99,20 @@ + + + - Command: release-management generate-issue-content-core + Command: release-management generate-issue-content-core - + - + Usage:breeze release-management generate-issue-content-core[OPTIONS] @@ -119,19 +120,20 @@ Generates content for issue to test the core release. ╭─ Generate issue flags ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env        │ -│variable set. Can be generated with:                                                       │ -│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status        │ -│(TEXT)                                                                                     │ -│*--previous-releasecommit reference (for example hash or tag) of the previous release.(TEXT)[required]│ -│*--current-releasecommit reference (for example hash or tag) of the current release.(TEXT)[required]│ -│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ -│--limit-pr-countLimit PR count processes (useful for testing small subset of PRs).(INTEGER)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env variable  │ +│set. Can be generated with:                                                                   │ +│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status           │ +│(TEXT)                                                                                        │ +│--previous-releasecommit reference (for example hash or tag) of the previous release.(TEXT)│ +│--current-releasecommit reference (for example hash or tag) of the current release.(TEXT)│ +│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ +│--limit-pr-countLimit PR count processes (useful for testing small subset of PRs).(INTEGER)│ +│--latestRun the command against latest released version of airflow│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-core.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-core.txt index 825c34ee36d67..bf380665aa306 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-core.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-core.txt @@ -1 +1 @@ -8e911f09e161d4fff654119656017be6 +15723131610a3be9a3093f346e72ed06 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 5d725ffa32888..828a1934fafa2 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2227,7 +2227,13 @@ def generate_issue_content_helm_chart( limit_pr_count: int | None, ): generate_issue_content( - github_token, previous_release, current_release, excluded_pr_list, limit_pr_count, is_helm_chart=True + github_token, + previous_release, + current_release, + excluded_pr_list, + limit_pr_count, + is_helm_chart=True, + latest=False, ) @@ -2248,13 +2254,11 @@ def generate_issue_content_helm_chart( @click.option( "--previous-release", type=str, - required=True, help="commit reference (for example hash or tag) of the previous release.", ) @click.option( "--current-release", type=str, - required=True, help="commit reference (for example hash or tag) of the current release.", ) @click.option("--excluded-pr-list", type=str, help="Coma-separated list of PRs to exclude from the issue.") @@ -2264,6 +2268,11 @@ def generate_issue_content_helm_chart( default=None, help="Limit PR count processes (useful for testing small subset of PRs).", ) +@click.option( + "--latest", + is_flag=True, + help="Run the command against latest released version of airflow", +) @option_verbose def generate_issue_content_core( github_token: str, @@ -2271,9 +2280,16 @@ def generate_issue_content_core( current_release: str, excluded_pr_list: str, limit_pr_count: int | None, + latest: bool, ): generate_issue_content( - github_token, previous_release, current_release, excluded_pr_list, limit_pr_count, is_helm_chart=False + github_token, + previous_release, + current_release, + excluded_pr_list, + limit_pr_count, + is_helm_chart=False, + latest=latest, ) @@ -3149,12 +3165,31 @@ def generate_issue_content( excluded_pr_list: str, limit_pr_count: int | None, is_helm_chart: bool, + latest: bool, ): from github import Github, Issue, PullRequest, UnknownObjectException PullRequestOrIssue = Union[PullRequest.PullRequest, Issue.Issue] verbose = get_verbose() - changes = get_changes(verbose, previous_release, current_release, is_helm_chart) + + previous = previous_release + current = current_release + + if latest: + import requests + + response = requests.get("https://pypi.org/pypi/apache-airflow/json") + response.raise_for_status() + latest_released_version = response.json()["info"]["version"] + previous = str(latest_released_version) + current = os.getenv("VERSION", previous + "rc1") + if current == previous + "rc1": + get_console().print( + f"\n[warning]Environment variable VERSION not set, setting current release " + f"version as {previous + 'rc1'}\n" + ) + + changes = get_changes(verbose, previous, current, is_helm_chart) change_prs = [change.pr for change in changes] if excluded_pr_list: excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index 67bdc340a64f7..048b93d5e1115 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -274,6 +274,7 @@ "--current-release", "--excluded-pr-list", "--limit-pr-count", + "--latest", ], } ], From 437751fdbec69f499b0539ea6fddf39070b72b84 Mon Sep 17 00:00:00 2001 From: Amogh Date: Tue, 26 Mar 2024 13:22:18 +0530 Subject: [PATCH 2/5] updating the fallback to main --- .github/workflows/check-providers.yml | 2 +- .../commands/release_management_commands.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index 6590d42d429d0..ac85ef9658ecd 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -100,7 +100,7 @@ jobs: --only-available-in-dist --disable-progress - name: "Test airflow core issue generation automatically" run: > - breeze release-management generate-issue-content-core --latest + breeze release-management generate-issue-content-core --latest --verbose - name: "Generate source constraints from CI image" shell: bash run: > diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 828a1934fafa2..833a92990e41e 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -3182,11 +3182,11 @@ def generate_issue_content( response.raise_for_status() latest_released_version = response.json()["info"]["version"] previous = str(latest_released_version) - current = os.getenv("VERSION", previous + "rc1") - if current == previous + "rc1": + current = os.getenv("VERSION", "main") + if current == "main": get_console().print( - f"\n[warning]Environment variable VERSION not set, setting current release " - f"version as {previous + 'rc1'}\n" + "\n[warning]Environment variable VERSION not set, setting current release " + "version as 'main'\n" ) changes = get_changes(verbose, previous, current, is_helm_chart) From 30d3e8cf41c4028b5b53a5bb4bde2d84d4f818a2 Mon Sep 17 00:00:00 2001 From: Amogh Date: Tue, 26 Mar 2024 14:04:49 +0530 Subject: [PATCH 3/5] Adding max-count --- .../commands/release_management_commands.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 833a92990e41e..7b339f953f375 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2066,6 +2066,7 @@ def get_git_log_command( from_commit: str | None = None, to_commit: str | None = None, is_helm_chart: bool = True, + max_count: int = 0, ) -> list[str]: git_cmd = [ "git", @@ -2073,6 +2074,8 @@ def get_git_log_command( "--pretty=format:%H %h %cd %s", "--date=short", ] + if max_count: + git_cmd.extend(["--max-count", str(max_count)]) if from_commit and to_commit: git_cmd.append(f"{from_commit}...{to_commit}") elif from_commit: @@ -2115,12 +2118,20 @@ def get_change_from_line(line: str): def get_changes( - verbose: bool, previous_release: str, current_release: str, is_helm_chart: bool = False + verbose: bool, + previous_release: str, + current_release: str, + is_helm_chart: bool = False, + max_count: int = 0, ) -> list[Change]: print(MY_DIR_PATH, SOURCE_DIR_PATH) change_strings = subprocess.check_output( get_git_log_command( - verbose, from_commit=previous_release, to_commit=current_release, is_helm_chart=is_helm_chart + verbose, + from_commit=previous_release, + to_commit=current_release, + is_helm_chart=is_helm_chart, + max_count=max_count, ), cwd=SOURCE_DIR_PATH, text=True, @@ -3174,6 +3185,7 @@ def generate_issue_content( previous = previous_release current = current_release + max_count = 0 if latest: import requests @@ -3188,8 +3200,9 @@ def generate_issue_content( "\n[warning]Environment variable VERSION not set, setting current release " "version as 'main'\n" ) + max_count = 30 - changes = get_changes(verbose, previous, current, is_helm_chart) + changes = get_changes(verbose, previous, current, is_helm_chart, max_count) change_prs = [change.pr for change in changes] if excluded_pr_list: excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] From 73e00b2a9735f5a9a4cc9ceea559d7f63ec0bb49 Mon Sep 17 00:00:00 2001 From: Amogh Date: Tue, 26 Mar 2024 15:15:56 +0530 Subject: [PATCH 4/5] using limit-pr-count instead --- .github/workflows/check-providers.yml | 2 +- .../commands/release_management_commands.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index ac85ef9658ecd..31a9b318b3b12 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -100,7 +100,7 @@ jobs: --only-available-in-dist --disable-progress - name: "Test airflow core issue generation automatically" run: > - breeze release-management generate-issue-content-core --latest --verbose + breeze release-management generate-issue-content-core --limit-pr-count 25 --latest --verbose - name: "Generate source constraints from CI image" shell: bash run: > diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 7b339f953f375..1a1740f8a7030 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2066,7 +2066,6 @@ def get_git_log_command( from_commit: str | None = None, to_commit: str | None = None, is_helm_chart: bool = True, - max_count: int = 0, ) -> list[str]: git_cmd = [ "git", @@ -2074,8 +2073,6 @@ def get_git_log_command( "--pretty=format:%H %h %cd %s", "--date=short", ] - if max_count: - git_cmd.extend(["--max-count", str(max_count)]) if from_commit and to_commit: git_cmd.append(f"{from_commit}...{to_commit}") elif from_commit: @@ -2131,7 +2128,6 @@ def get_changes( from_commit=previous_release, to_commit=current_release, is_helm_chart=is_helm_chart, - max_count=max_count, ), cwd=SOURCE_DIR_PATH, text=True, @@ -3185,7 +3181,6 @@ def generate_issue_content( previous = previous_release current = current_release - max_count = 0 if latest: import requests @@ -3200,9 +3195,8 @@ def generate_issue_content( "\n[warning]Environment variable VERSION not set, setting current release " "version as 'main'\n" ) - max_count = 30 - changes = get_changes(verbose, previous, current, is_helm_chart, max_count) + changes = get_changes(verbose, previous, current, is_helm_chart) change_prs = [change.pr for change in changes] if excluded_pr_list: excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] From f21727d5080f1a23724ba4aedfe8f0d09d5c78e8 Mon Sep 17 00:00:00 2001 From: Amogh Date: Tue, 26 Mar 2024 17:52:04 +0530 Subject: [PATCH 5/5] making it as HEAD --- .../airflow_breeze/commands/release_management_commands.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 1a1740f8a7030..e45dc25f7db87 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2119,7 +2119,6 @@ def get_changes( previous_release: str, current_release: str, is_helm_chart: bool = False, - max_count: int = 0, ) -> list[Change]: print(MY_DIR_PATH, SOURCE_DIR_PATH) change_strings = subprocess.check_output( @@ -3189,11 +3188,11 @@ def generate_issue_content( response.raise_for_status() latest_released_version = response.json()["info"]["version"] previous = str(latest_released_version) - current = os.getenv("VERSION", "main") - if current == "main": + current = os.getenv("VERSION", "HEAD") + if current == "HEAD": get_console().print( "\n[warning]Environment variable VERSION not set, setting current release " - "version as 'main'\n" + "version as 'HEAD'\n" ) changes = get_changes(verbose, previous, current, is_helm_chart)