diff --git a/.github/workflows/cpu-torch-latest.yml b/.github/workflows/cpu-torch-latest.yml index 7a4952f18987..3aaf30bc42ad 100644 --- a/.github/workflows/cpu-torch-latest.yml +++ b/.github/workflows/cpu-torch-latest.yml @@ -32,11 +32,6 @@ on: default: 'main' type: string pull_request: - paths-ignore: - - 'docs/**' - - 'blogs/**' - - 'deepspeed/inference/v2/**' - - 'tests/unit/inference/v2/**' merge_group: branches: [ master ] schedule: @@ -47,8 +42,39 @@ concurrency: cancel-in-progress: true jobs: + check-paths: + name: cpu-torch-latest / check paths + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.non_pr.outputs.should_run || steps.filter.outputs.run_tests }} + steps: + - id: non_pr + if: github.event_name != 'pull_request' + run: echo "should_run=true" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + if: github.event_name == 'pull_request' + + - uses: dorny/paths-filter@v3 + id: filter + if: github.event_name == 'pull_request' + with: + predicate-quantifier: every + filters: | + run_tests: + - '**' + - '!docs/**' + - '!blogs/**' + - '!deepspeed/inference/v2/**' + - '!tests/unit/inference/v2/**' + unit-tests: name: cpu-torch-latest / unit tests + needs: check-paths + if: always() && (needs.check-paths.result != 'success' || needs.check-paths.outputs.should_run == 'true') runs-on: ubuntu-24.04 env: @@ -58,6 +84,10 @@ jobs: DEFAULT_TRANSFORMERS_REF: 'main' steps: + - name: Fail if path filter failed + if: needs.check-paths.result != 'success' + run: exit 1 + - uses: actions/checkout@v4 - id: setup-venv diff --git a/.github/workflows/modal-torch-latest.yml b/.github/workflows/modal-torch-latest.yml index dec100331472..ffa5ef9c8a0f 100644 --- a/.github/workflows/modal-torch-latest.yml +++ b/.github/workflows/modal-torch-latest.yml @@ -47,11 +47,6 @@ on: - master pull_request_target: - paths-ignore: - - 'docs/**' - - 'blogs/**' - - 'deepspeed/inference/v2/**' - - 'tests/unit/inference/v2/**' types: [review_requested, ready_for_review, synchronize] branches: - master @@ -68,7 +63,7 @@ jobs: contents: read pull-requests: read outputs: - deepspeed: ${{ steps.filter.outputs.deepspeed }} + deepspeed: ${{ steps.filter.outputs.deepspeed == 'true' || steps.filter.outputs.modal_workflow == 'true' || steps.filter.outputs.ci == 'true' || steps.filter.outputs.unit_tests == 'true' || steps.filter.outputs.csrc == 'true' }} steps: - name: Checkout repository @@ -81,29 +76,32 @@ jobs: id: filter with: token: ${{ secrets.GITHUB_TOKEN }} + predicate-quantifier: every filters: | deepspeed: - 'deepspeed/**' + - '!deepspeed/inference/v2/**' + modal_workflow: - '.github/workflows/modal*.yml' + ci: - 'ci/**' + unit_tests: - 'tests/unit/**' + - '!tests/unit/inference/v2/**' + csrc: - 'csrc/**' deploy: name: modal-torch-latest / DeepSpeedAI CI runs-on: ubuntu-latest needs: collect-tests - env: - # these are created at https://modal.com/settings/deepspeedai/tokens - # they are then added to the repo's secrets at https://github.com/deepspeedai/deepspeed/settings/secrets/actions - MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} - MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - # this one comes from https://huggingface.co/settings/profile of the bot user - # and it too is then updated at https://github.com/deepspeedai/deepspeed/settings/secrets/actions - HF_TOKEN: ${{ secrets.HF_TOKEN }} - if: github.event_name == 'workflow_dispatch' || needs.collect-tests.outputs.deepspeed == 'true' + if: always() && (github.event_name == 'workflow_dispatch' || needs.collect-tests.result != 'success' || needs.collect-tests.outputs.deepspeed == 'true') steps: + - name: Fail if path filter failed + if: github.event_name != 'workflow_dispatch' && needs.collect-tests.result != 'success' + run: exit 1 + - name: Checkout Repository uses: actions/checkout@v4 with: @@ -125,5 +123,12 @@ jobs: MODAL_TORCH_PRESET: ${{ github.event.inputs.torch_preset || '2.10.0-cuda12.8' }} MODAL_TRANSFORMERS_SOURCE: ${{ github.event.inputs.transformers_source || 'git' }} MODAL_TRANSFORMERS_REF: ${{ github.event.inputs.transformers_ref || 'main' }} + # these are created at https://modal.com/settings/deepspeedai/tokens + # they are then added to the repo's secrets at https://github.com/deepspeedai/deepspeed/settings/secrets/actions + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + # this one comes from https://huggingface.co/settings/profile of the bot user + # and it too is then updated at https://github.com/deepspeedai/deepspeed/settings/secrets/actions + HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | modal run -m ci.torch_latest diff --git a/.github/workflows/nv-pre-compile-ops.yml b/.github/workflows/nv-pre-compile-ops.yml index 3e82eddaba26..1f369755775b 100644 --- a/.github/workflows/nv-pre-compile-ops.yml +++ b/.github/workflows/nv-pre-compile-ops.yml @@ -5,11 +5,6 @@ on: pull_request: branches: '**' - paths-ignore: - - 'docs/**' - - 'blogs/**' - - 'deepspeed/inference/v2/**' - - 'tests/unit/inference/v2/**' merge_group: branches: [ master ] schedule: @@ -20,13 +15,48 @@ concurrency: cancel-in-progress: true jobs: + check-paths: + name: nv-pre-compile-ops / check paths + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.non_pr.outputs.should_run || steps.filter.outputs.run_tests }} + steps: + - id: non_pr + if: github.event_name != 'pull_request' + run: echo "should_run=true" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + if: github.event_name == 'pull_request' + + - uses: dorny/paths-filter@v3 + id: filter + if: github.event_name == 'pull_request' + with: + predicate-quantifier: every + filters: | + run_tests: + - '**' + - '!docs/**' + - '!blogs/**' + - '!deepspeed/inference/v2/**' + - '!tests/unit/inference/v2/**' + unit-tests: name: nv-pre-compile-ops / precompile ops + needs: check-paths + if: always() && (needs.check-paths.result != 'success' || needs.check-paths.outputs.should_run == 'true') runs-on: ubuntu-24.04 container: image: nvidia/cuda:12.6.3-devel-ubuntu22.04 steps: + - name: Fail if path filter failed + if: needs.check-paths.result != 'success' + run: exit 1 + - name: Install system dependencies run: | apt-get update && apt-get install -y git python3 python3-pip libaio-dev ninja-build diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f9a2d9f3fb2d..f9fe1fbf7854 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -5,9 +5,6 @@ on: pull_request: branches: '**' - paths-ignore: - - 'docs/**' - - 'blogs/**' merge_group: branches: [ master ] schedule: @@ -18,8 +15,37 @@ concurrency: cancel-in-progress: true jobs: + check-paths: + name: python / check paths + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.non_pr.outputs.should_run || steps.filter.outputs.run_tests }} + steps: + - id: non_pr + if: github.event_name != 'pull_request' + run: echo "should_run=true" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + if: github.event_name == 'pull_request' + + - uses: dorny/paths-filter@v3 + id: filter + if: github.event_name == 'pull_request' + with: + predicate-quantifier: every + filters: | + run_tests: + - '**' + - '!docs/**' + - '!blogs/**' + unit-tests: name: python / install smoke (Python ${{ matrix.pyVersion }}) + needs: check-paths + if: always() strategy: matrix: pyVersion: ["3.10", "3.11", "3.12"] @@ -30,21 +56,35 @@ jobs: image: python:${{ matrix.pyVersion }}-slim steps: + - name: Fail if path filter failed + if: needs.check-paths.result != 'success' + run: exit 1 + + - name: Skip ignored-path install smoke + if: needs.check-paths.outputs.should_run != 'true' + run: echo "Only ignored paths changed; install smoke intentionally skipped." + - uses: actions/checkout@v4 + if: needs.check-paths.outputs.should_run == 'true' - name: Install build dependencies + if: needs.check-paths.outputs.should_run == 'true' run: | apt-get update && apt-get install -y build-essential ninja-build - name: environment + if: needs.check-paths.outputs.should_run == 'true' run: | which python python --version - name: Install PyTorch (CPU) + if: needs.check-paths.outputs.should_run == 'true' run: | pip install torch --index-url https://download.pytorch.org/whl/cpu - name: Install deepspeed + if: needs.check-paths.outputs.should_run == 'true' run: | pip install . - name: DS Report + if: needs.check-paths.outputs.should_run == 'true' run: | ds_report