Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ repos:
files: ^\.github/
types: [file] # Include actions.lock.
stages: [pre-commit]
- id: zizmor
name: zizmor (templates)
args: [--offline]
files: ^bench_runner/templates/.*\.src\.yml$
stages: [pre-commit]

- repo: local
hooks:
Expand Down
3 changes: 2 additions & 1 deletion DEVELOPER.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ prek install
prek run --all-files
```

The hooks include zizmor checks for `.github`.
The hooks include zizmor checks for `.github` and the workflow templates in
`bench_runner/templates`.

After changing action references or adding workflows under `.github/workflows`,
run `gh actions-lock` and commit the updated lockfile with the workflow changes.
Expand Down
7 changes: 4 additions & 3 deletions bench_runner/scripts/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,9 @@ def add_flag_env(jobs: dict[str, Any]):
job.setdefault("env", {})
job["env"]["flags"] = flag_value
for step in job["steps"]:
if "run" in step:
step["run"] = step["run"].replace("${{ env.flags }}", flag_value)
for key, value in step.get("env", {}).items():
if isinstance(value, str):
step["env"][key] = value.replace("${{ env.flags }}", flag_value)


def generate__benchmark(src: Any) -> Any:
Expand Down Expand Up @@ -271,6 +272,7 @@ def generate__weekly(dst: Any) -> Any:
)
job = {
"uses": "./.github/workflows/_benchmark.yml",
"permissions": {"contents": "write"},
"needs": "determine_head",
"with": {
"fork": "python",
Expand All @@ -280,7 +282,6 @@ def generate__weekly(dst: Any) -> Any:
"pgo": True,
**flags.flags_to_gha_variables_yml(weekly_cfg.flags),
},
"secrets": "inherit",
}
job_name = f"weekly-{name}-{runner.nickname}"
dst["jobs"][job_name] = job
Expand Down
38 changes: 35 additions & 3 deletions bench_runner/templates/_benchmark.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,13 @@ name: _benchmark
description: "Collect Linux perf profiling data (Linux only)"
type: boolean

permissions: {}

jobs:
benchmark-windows:
runs-on: [self-hosted, windows, bare-metal]
permissions:
contents: write

steps:
# Tell git to checkout repos with symlinks (required by pyston
Expand All @@ -68,12 +72,20 @@ jobs:
git config --global core.symlinks true
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
- name: git gc
run: |
git gc
- name: Building Python and running pyperformance
env:
FORK: ${{ inputs.fork }}
REF: ${{ inputs.ref }}
MACHINE: ${{ inputs.machine }}
BENCHMARKS: ${{ inputs.benchmarks || 'all' }}
FLAGS: ${{ env.flags }}
run: |
py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
py workflow_bootstrap.py $env:FORK $env:REF $env:MACHINE $env:BENCHMARKS "$env:FLAGS" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
# Pull again, since another job may have committed results in the meantime
- name: Pull benchmarking
run: |
Expand All @@ -97,10 +109,14 @@ jobs:
benchmark-linux:
runs-on: [self-hosted, linux, bare-metal]
timeout-minutes: 1440
permissions:
contents: write

steps:
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
- name: git gc
run: |
git gc
Expand All @@ -111,8 +127,14 @@ jobs:
with:
python-version: "3.15"
- name: Building Python and running pyperformance
env:
FORK: ${{ inputs.fork }}
REF: ${{ inputs.ref }}
MACHINE: ${{ inputs.machine }}
BENCHMARKS: ${{ inputs.benchmarks || 'all' }}
FLAGS: ${{ env.flags }}
run: |
python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
python workflow_bootstrap.py "$FORK" "$REF" "$MACHINE" "$BENCHMARKS" "$FLAGS" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
# Pull again, since another job may have committed results in the meantime
- name: Pull benchmarking
if: ${{ !inputs.perf }}
Expand Down Expand Up @@ -145,16 +167,26 @@ jobs:

benchmark-darwin:
runs-on: [self-hosted, macos, bare-metal]
permissions:
contents: write

steps:
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
- name: git gc
run: |
git gc
- name: Building Python and running pyperformance
env:
FORK: ${{ inputs.fork }}
REF: ${{ inputs.ref }}
MACHINE: ${{ inputs.machine }}
BENCHMARKS: ${{ inputs.benchmarks || 'all' }}
FLAGS: ${{ env.flags }}
run: |
python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
python3 workflow_bootstrap.py "$FORK" "$REF" "$MACHINE" "$BENCHMARKS" "$FLAGS" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
Comment on lines +182 to +189

@maurycy maurycy Nov 13, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a wow finding to me

# Pull again, since another job may have committed results in the meantime
- name: Pull benchmarking
run: |
Expand Down
7 changes: 7 additions & 0 deletions bench_runner/templates/_find_failures.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,19 @@ name: _find_failures
- cron: "0 22 * * 0"
workflow_dispatch: {}

permissions: {}

jobs:
find_failures:
runs-on: ubuntu-latest
permissions:
actions: read
contents: write
steps:
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
- name: Setup system Python
uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
Expand Down
5 changes: 5 additions & 0 deletions bench_runner/templates/_generate.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,18 @@ name: _generate
type: boolean
default: false

permissions: {}

jobs:
generate-results:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
ref: main
- name: Checkout CPython
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand Down
16 changes: 14 additions & 2 deletions bench_runner/templates/_notify.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,15 @@ name: _notify
version:
type: string

permissions: {}

jobs:
notify:
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
issues: write
steps:
- uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 #v3.0.3
- name: Checkout benchmarking
Expand All @@ -34,6 +40,12 @@ jobs:
pip install -r requirements.txt
- name: Send notifications
if: env.WORKFLOW_CONCLUSION == 'success'
run: python -m bench_runner notify --fork ${{ inputs.fork }} --ref ${{ inputs.ref }} --head ${{ inputs.head }} --date ${{ inputs.date }} --version ${{ inputs.version }} --flags ${{ env.flags }}
run: python -m bench_runner notify --fork "$FORK" --ref "$REF" --head "$HEAD" --date "$DATE" --version "$VERSION" --flags "$FLAGS"
env:
GH_TOKEN: ${{ github.token }}
GH_TOKEN: ${{ github.token }}
FORK: ${{ inputs.fork }}
REF: ${{ inputs.ref }}
HEAD: ${{ inputs.head }}
DATE: ${{ inputs.date }}
VERSION: ${{ inputs.version }}
FLAGS: ${{ env.flags }}
10 changes: 9 additions & 1 deletion bench_runner/templates/_publish.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ name: _publish
dry_run:
type: boolean
default: false
secrets:
BENCHMARK_MIRROR:
required: true

workflow_dispatch:
inputs:
Expand All @@ -15,19 +18,24 @@ name: _publish
type: boolean
default: false

permissions: {}

jobs:
mirror:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
ref: main
# This has to be enough depth to handle since the last time we published.
# As a rule of thumb, (number of machines * 2, plus the generated stuff) * 2
fetch-depth: 20
token: ${{ secrets.BENCHMARK_MIRROR }}
- name: Mirror
run: |
git remote add public https://github.com/${{ github.repository }}-public
git remote add public "https://github.com/$GITHUB_REPOSITORY-public"
git push -f public main
13 changes: 12 additions & 1 deletion bench_runner/templates/_pystats.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,18 @@ name: _pystats
description: "Rerun and replace results if commit already exists"
type: boolean

permissions: {}

jobs:
collect-stats:
runs-on: [self-hosted, linux, cloud]
permissions:
contents: write
steps:
- name: Checkout benchmarking
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: true
- name: git gc
run: |
git gc
Expand All @@ -51,8 +57,13 @@ jobs:
with:
python-version: "3.15"
- name: Build CPython and run pyperformance benchmarks
env:
FORK: ${{ inputs.fork }}
REF: ${{ inputs.ref }}
BENCHMARKS: ${{ inputs.benchmarks || 'all' }}
FLAGS: ${{ env.flags }}
run: |
python workflow_bootstrap.py --pystats ${{ inputs.fork }} ${{ inputs.ref }} all ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }}
python workflow_bootstrap.py --pystats "$FORK" "$REF" all "$BENCHMARKS" "$FLAGS" --run_id ${{ github.run_id }}
- name: Pull benchmarking
run: |
# Another benchmarking task may have created results for the same
Expand Down
20 changes: 15 additions & 5 deletions bench_runner/templates/_weekly.src.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@ name: _weekly
- cron: "0 0 * * 0"
workflow_dispatch: {}

permissions: {}

jobs:
# Determine the base commit of the selected commit. The output is passed to
# the `base` job below. If the data already exists for this commit, it will be
# skipped.
determine_head:
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
commit: ${{ steps.head.outputs.commit }}
steps:
Expand All @@ -26,38 +30,44 @@ jobs:
id: head
run: |
cd cpython
git log -n 1 --format="commit=%H" >> $GITHUB_OUTPUT
git log -n 1 --format="commit=%H" >> "$GITHUB_OUTPUT"

pystats:
uses: ./.github/workflows/_pystats.yml
permissions:
contents: write
needs: determine_head
with:
fork: python
ref: ${{ needs.determine_head.outputs.commit }}
benchmarks: all
tier2: false
secrets: inherit

pystats-tier2:
uses: ./.github/workflows/_pystats.yml
permissions:
contents: write
needs: determine_head
with:
fork: python
ref: ${{ needs.determine_head.outputs.commit }}
benchmarks: all
tier2: true
secrets: inherit

generate:
uses: ./.github/workflows/_generate.yml
permissions:
contents: write
if: ${{ always() }}
with:
force: false
needs: [pystats, pystats-tier2]
secrets: inherit

publish:
uses: ./.github/workflows/_publish.yml
permissions:
contents: read
if: ${{ always() }}
needs: [generate]
secrets: inherit

@maurycy maurycy Nov 13, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might have implications but I'm not exactly sure.

secrets:
BENCHMARK_MIRROR: ${{ secrets.BENCHMARK_MIRROR }}
Loading
Loading