Check #20609 #196
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check PR | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| type: number | |
| description: PR to check | |
| action: | |
| type: choice | |
| description: Action to perform | |
| required: true | |
| default: check | |
| options: [check, bench] | |
| left_rev: | |
| type: string | |
| description: Left (old) revision | |
| left_origin: | |
| type: string | |
| description: Left (old) origin | |
| default: pypi | |
| right_rev: | |
| type: string | |
| description: Right (new) revision | |
| right_origin: | |
| type: string | |
| description: Right (new) origin | |
| default: pypi | |
| issues: | |
| types: [opened, reopened] | |
| issue_comment: | |
| types: [created] | |
| permissions: | |
| contents: read | |
| env: | |
| BENCH_RUNS_SELFCHECK: 30 | |
| BENCH_RUNS_SIMPLE: 50 | |
| MARKER: <!-- marker --> | |
| jobs: | |
| get-target: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request }} | |
| permissions: | |
| issues: write | |
| outputs: | |
| pr_number: ${{ steps.pr-number.outputs.pr }} | |
| action: ${{ (github.event_name == 'issues' || github.event_name == 'issue_comment') && steps.pr-number.outputs.action || inputs.action }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Get issue | |
| id: get-issue | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: get-issue | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Get PR number to check | |
| id: pr-number | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| run: | | |
| if [ -z "${INPUT_PR}" ] || [ -z "$ACTION" ]; then | |
| case "${ISSUE_TITLE,,}" in | |
| 'check #'*) | |
| pr=${ISSUE_TITLE##*#} | |
| action="check" | |
| ;; | |
| 'bench #'*) | |
| pr=${ISSUE_TITLE##*#} | |
| action="bench" | |
| ;; | |
| *) | |
| pr="invalid" | |
| action="invalid" | |
| esac | |
| else | |
| pr=${INPUT_PR} | |
| action=${ACTION} | |
| fi | |
| re='^[0-9]+$' | |
| if ! [[ "$pr" =~ $re ]]; then | |
| msg="No PR number found" | |
| elif ! gh --repo python/mypy pr view "$pr"; then | |
| msg=$(printf "PR #%s not found in python/mypy" "$pr") | |
| else | |
| printf "pr=%s\n" "${pr}" >> "$GITHUB_OUTPUT" | |
| printf "action=%s\n" "${action}" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| printf "%s\n" "$msg" >&2 | |
| printf "msg=%s\n" "$msg" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| env: | |
| GH_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| INPUT_PR: ${{ inputs.pr }} | |
| ACTION: ${{ inputs.action }} | |
| ISSUE_TITLE: ${{ steps.get-issue.outputs.issue-title }} | |
| - name: Report progress | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| run: | | |
| gh issue view "$ISSUE" --json body | jq -r '.body' >body.txt | |
| cat >>body.txt <<-EOF | |
| ${{ env.MARKER }} | |
| Running $ACTION on #$INPUT_PR... | |
| EOF | |
| gh issue edit "$ISSUE" --body-file body.txt | |
| rm body.txt | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_PR: ${{ steps.pr-number.outputs.pr }} | |
| ACTION: ${{ steps.pr-number.outputs.action }} | |
| ISSUE: ${{ github.event.issue.number }} | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| ${{ steps.pr-number.outputs.msg }} | |
| Comment with any text to retry (feel free to edit the title beforehand). | |
| Depending on the desider action, use the following issue titles: | |
| * Check a PR against all open tickets: `Check #xxxxx` | |
| * Run a few benchmarks for a PR: `Bench #xxxxx` | |
| fetch: | |
| runs-on: ubuntu-latest | |
| needs: [get-target] | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }} | |
| steps: | |
| - uses: actions/cache/restore@v5 | |
| id: restore-issues | |
| name: Restore cache | |
| with: | |
| path: downloaded/ | |
| key: issues-dump-v2-${{ github.run_id }} | |
| restore-keys: issues-dump-v2- | |
| - uses: actions/checkout@v6 | |
| if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }} | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@v7 | |
| if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }} | |
| with: | |
| python-version: '3.13' | |
| - name: Fetch issues | |
| # cache-hit is false for partial matches | |
| if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }} | |
| run: | | |
| uv run fetch | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: issues-dump | |
| path: downloaded/ | |
| retention-days: 1 | |
| apply: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }} | |
| permissions: | |
| issues: write | |
| needs: [get-target, fetch] | |
| strategy: | |
| matrix: | |
| side: [left, right] | |
| shard: [0, 1] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: '3.13' | |
| - uses: actions/download-artifact@v5 | |
| with: | |
| name: issues-dump | |
| path: downloaded/ | |
| - name: Apply mypy (PR) | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' || inputs.pr != null }} | |
| run: | | |
| uv run run \ | |
| --only-${{ matrix.side }} \ | |
| --pr "${PR}" \ | |
| --shard "${{ matrix.shard }}" \ | |
| --total-shards 2 | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| PR: ${{ needs.get-target.outputs.pr_number }} | |
| TQDM_MININTERVAL: '5' | |
| - name: Apply mypy (two versions) | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.pr == null }} | |
| run: | | |
| uv run run \ | |
| --only-${{ matrix.side }} \ | |
| --left-rev "$LEFT_REV" \ | |
| --left-origin "$LEFT_ORIGIN" \ | |
| --right-rev "$RIGHT_REV" \ | |
| --right-origin "$RIGHT_ORIGIN" \ | |
| --shard "${{ matrix.shard }}" \ | |
| --total-shards 2 | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| LEFT_REV: ${{ inputs.left_rev }} | |
| LEFT_ORIGIN: ${{ inputs.left_origin }} | |
| RIGHT_REV: ${{ inputs.right_rev }} | |
| RIGHT_ORIGIN: ${{ inputs.right_origin }} | |
| TQDM_MININTERVAL: '5' | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.shard }}-${{ matrix.side }} | |
| path: outputs/ | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Sorry, something went wrong when running mypy. | |
| diff: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }} | |
| needs: [apply, get-target] | |
| permissions: | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: '3.13' | |
| - uses: actions/download-artifact@v5 | |
| with: | |
| name: issues-dump | |
| path: downloaded/ | |
| - uses: actions/download-artifact@v5 | |
| with: | |
| pattern: results-* | |
| path: outputs/ | |
| merge-multiple: true | |
| - name: Compare outputs | |
| id: compare | |
| run: | | |
| uv run diff --no-snippets --diff-originals | tee out.txt | |
| { | |
| echo 'diff_text<<EOF' | |
| sed -e 's/\x1b\[1;3.m//g' -e 's/\x1b\[0m//g' out.txt | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - name: Post results | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Check results are ready! | |
| PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }} | |
| ```diff | |
| ${{ steps.compare.outputs.diff_text }} | |
| ``` | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Sorry, something went wrong during diff generation. | |
| bench: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'bench' }} | |
| needs: [get-target] | |
| permissions: | |
| issues: write | |
| strategy: | |
| matrix: | |
| iter: [1, 2, 3] | |
| sample: [selfcheck, simple] | |
| outputs: | |
| result_run_selfcheck_1: ${{ steps.run-bench.outputs.result_run_selfcheck_1 }} | |
| result_run_selfcheck_2: ${{ steps.run-bench.outputs.result_run_selfcheck_2 }} | |
| result_run_selfcheck_3: ${{ steps.run-bench.outputs.result_run_selfcheck_3 }} | |
| result_run_simple_1: ${{ steps.run-bench.outputs.result_run_simple_1 }} | |
| result_run_simple_2: ${{ steps.run-bench.outputs.result_run_simple_2 }} | |
| result_run_simple_3: ${{ steps.run-bench.outputs.result_run_simple_3 }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| repository: python/mypy | |
| fetch-depth: 0 # We need to have history access for merging | |
| token: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: '3.12' | |
| - name: Checkout PR | |
| id: get-hash | |
| run: | | |
| git config --global advice.detachedHead false | |
| git config user.name "Never pushes" | |
| git config user.email "[email protected]" | |
| gh pr checkout "$PR_NUMBER" | |
| info=$(gh --repo python/mypy pr view 19623 --json baseRefName,baseRefOid) | |
| base_branch=$(jq -r '.baseRefName' <<<"$info") | |
| git merge "origin/$base_branch" --no-edit | |
| hash=$(git rev-parse HEAD) | |
| printf 'head_hash=%s\n' "$hash" >>"$GITHUB_OUTPUT" | |
| printf 'base_branch=%s\n' "$base_branch" >>"$GITHUB_OUTPUT" | |
| env: | |
| PR_NUMBER: ${{ needs.get-target.outputs.pr_number }} | |
| GH_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - name: Run benchmark | |
| id: run-bench | |
| run: | | |
| set -o pipefail | |
| uv venv | |
| . .venv/bin/activate | |
| uv pip install '.[mypyc]' | |
| uv pip install -r build-requirements.txt | |
| uv pip install -r test-requirements.txt | |
| extra_args=() | |
| if [[ "$CHECK" = "simple" ]]; then | |
| extra_args+=( '-c' 'import typing' ) | |
| fi | |
| python ./misc/perf_compare.py \ | |
| "$PR_BASE_BRANCH" "$PR_HEAD_HASH" \ | |
| --num-runs "$NUM_RUNS" \ | |
| "${extra_args[@]}" \ | |
| | tee results.txt | |
| { | |
| printf 'result_run_%s_%s<<EOF\n' "$CHECK" "$SEQUENCE_NO" | |
| sed -ne '/=== Results/,$ p' results.txt | |
| echo EOF | |
| } >>"$GITHUB_OUTPUT" | |
| env: | |
| PR_HEAD_HASH: ${{ steps.get-hash.outputs.head_hash }} | |
| PR_BASE_BRANCH: ${{ steps.get-hash.outputs.base_branch }} | |
| NUM_RUNS: ${{ matrix.sample == 'simple' && env.BENCH_RUNS_SIMPLE || env.BENCH_RUNS_SELFCHECK }} | |
| CHECK: ${{ matrix.sample }} | |
| SEQUENCE_NO: ${{ matrix.iter }} | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Sorry, something went wrong during benchmark execution. | |
| bench-report: | |
| runs-on: ubuntu-latest | |
| if: | | |
| !github.event.issue.pull_request | |
| && needs.get-target.outputs.action == 'bench' | |
| && (github.event_name == 'issues' || github.event_name == 'issue_comment') | |
| needs: [get-target, bench] | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: Post results | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Benchmark results are ready! | |
| PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }} | |
| Results over three rounds for selfcheck (N=${{ env.BENCH_RUNS_SELFCHECK }}): | |
| ``` | |
| ${{ needs.bench.outputs.result_run_selfcheck_1 }} | |
| ``` | |
| ``` | |
| ${{ needs.bench.outputs.result_run_selfcheck_2 }} | |
| ``` | |
| ``` | |
| ${{ needs.bench.outputs.result_run_selfcheck_3 }} | |
| ``` | |
| Results over three rounds for `import typing` (N=${{ env.BENCH_RUNS_SIMPLE }}): | |
| ``` | |
| ${{ needs.bench.outputs.result_run_simple_1 }} | |
| ``` | |
| ``` | |
| ${{ needs.bench.outputs.result_run_simple_2 }} | |
| ``` | |
| ``` | |
| ${{ needs.bench.outputs.result_run_simple_3 }} | |
| ``` | |
| clear-body: | |
| runs-on: ubuntu-latest | |
| name: Remove progress report from issue body | |
| if: | | |
| always() && !cancelled() | |
| && !github.event.issue.pull_request | |
| && (github.event_name == 'issues' || github.event_name == 'issue_comment') | |
| needs: [bench-report, diff] | |
| permissions: | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Clear body | |
| run: | | |
| gh issue view "$ISSUE" --json body | jq -r '.body' >body.txt | |
| sed -e '/${{ env.MARKER }}/Q' body.txt >body.new.txt | |
| gh issue edit "$ISSUE" --body-file body.new.txt | |
| rm body.txt body.new.txt | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE: ${{ github.event.issue.number }} |