diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index c1e86edfab0..27259c7e8c9 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -396,6 +396,10 @@ jobs: # results travel with the screenshots. - name: ParparVM vs JDK 25 (performance gate) if: success() || failure() + # Names this pull request's overlay (perf-baseline/pr/.json) in the + # comment. A workflow_dispatch run carries no pull request, so the gate asks gh. + env: + GH_TOKEN: ${{ github.token }} run: | vm/selfhost/ci-perf-gate.sh run linux-${{ matrix.arch }} artifacts/linux-port/raw/perf \ --hello-workload "$GITHUB_WORKSPACE/artifacts/linux-port/raw/translation-record.jsonl" \ diff --git a/.github/workflows/parparvm-selfhost.yml b/.github/workflows/parparvm-selfhost.yml index 6818d5577d0..322b8b0ee70 100644 --- a/.github/workflows/parparvm-selfhost.yml +++ b/.github/workflows/parparvm-selfhost.yml @@ -133,7 +133,7 @@ jobs: # - Keeping it in both places would pay for the expensive -O3 self-host build # twice on any PR carrying the `selfhost` label. # - # The baselines and tolerance live in vm/selfhost/perf-baseline.json, the + # The baselines and tolerance live in vm/selfhost/perf-baseline/, the # measurement in vm/selfhost/perf-gate.py. # Both trees, so a divergence can be inspected rather than guessed at from diff --git a/.github/workflows/parparvm-tests-windows.yml b/.github/workflows/parparvm-tests-windows.yml index 0bd6e65bc04..669fd8a6cdc 100644 --- a/.github/workflows/parparvm-tests-windows.yml +++ b/.github/workflows/parparvm-tests-windows.yml @@ -558,6 +558,10 @@ jobs: # builds only ask javac for -source/-target 1.8 against an explicit bootclasspath. - name: ParparVM vs JDK 25 (performance gate, x64) if: success() || failure() + # Names this pull request's overlay (perf-baseline/pr/.json) in the + # comment. A workflow_dispatch run carries no pull request, so the gate asks gh. + env: + GH_TOKEN: ${{ github.token }} shell: bash run: | export JDK_8_HOME="${JDK_8_HOME:-$JAVA_HOME}" @@ -763,6 +767,10 @@ jobs: # builds only ask javac for -source/-target 1.8 against an explicit bootclasspath. - name: ParparVM vs JDK 25 (performance gate, arm64) if: success() || failure() + # Names this pull request's overlay (perf-baseline/pr/.json) in the + # comment. A workflow_dispatch run carries no pull request, so the gate asks gh. + env: + GH_TOKEN: ${{ github.token }} shell: bash run: | export JDK_8_HOME="${JDK_8_HOME:-$JAVA_HOME}" diff --git a/.github/workflows/perf-baseline.yml b/.github/workflows/perf-baseline.yml new file mode 100644 index 00000000000..f8a7a92c440 --- /dev/null +++ b/.github/workflows/perf-baseline.yml @@ -0,0 +1,117 @@ +name: Performance baselines + +# The ParparVM performance gate's baselines live in vm/selfhost/perf-baseline/: +# base/ holds the consolidated rows, and each pull request that changes a baseline +# writes only its own pr/.json, so baseline changes stay in the pull +# request's diff without two branches ever editing the same file +# (vm/selfhost/perf_baseline.py explains the layout). +# +# check every pull request: the overlays resolve against base/ without +# contradicting each other, and the pull request touched only its own +# overlay (base/ is the fold's alone). No paths filter: several of this +# repository's pull requests exceed GitHub's paths-filter diff limit, and a +# filtered check would silently not run on exactly those. +# fold nightly on master: every overlay on master belongs to a merged pull +# request, so it is folded into base/ and deleted, in one commit that +# provably changes no verdict. Then the website is rebuilt, because the +# Port Status page's ParparVM vs JDK 25 table is rendered from these rows. +# A push made with GITHUB_TOKEN triggers no workflow, hence the dispatch. + +on: + pull_request: + branches: + - master + push: + branches: + - master + schedule: + - cron: '20 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check: + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + # The pull request's base commit has to be present for the diff. + fetch-depth: 0 + + - name: The gate's own unit tests + working-directory: vm/selfhost + run: python3 -m unittest test_perf_gate + + - name: Check the baselines + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if [ -n "${BASE_SHA}" ]; then + python3 vm/selfhost/perf_baseline.py check --base "${BASE_SHA}" --pr "${PR_NUMBER}" + else + python3 vm/selfhost/perf_baseline.py check + fi + + fold: + if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' + runs-on: ubuntu-24.04 + permissions: + contents: write + actions: write + concurrency: + group: perf-baseline-fold + cancel-in-progress: false + steps: + - uses: actions/checkout@v6 + with: + ref: master + + - name: Fold merged overlays into base/ + id: fold + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # A merge landing while this runs makes the push non-fast-forward. Start over + # from the new master rather than rebasing: the fold is cheap, and redoing it + # folds the overlay that just arrived as well. + for attempt in 1 2 3; do + git fetch --quiet origin master + git reset --quiet --hard origin/master + python3 vm/selfhost/perf_baseline.py fold + git add -A vm/selfhost/perf-baseline + if git diff --staged --quiet; then + echo "No overlays to fold." + echo "folded=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git diff --staged --stat + git commit --quiet -m "ci: fold merged performance baselines into perf-baseline/base" + if git push origin HEAD:master; then + echo "folded=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "master moved during the fold (attempt ${attempt}); retrying." + done + echo "Could not push the fold after 3 attempts." >&2 + exit 1 + + # One-shot on purpose. If this dispatch fails after the fold was pushed, a re-run + # finds nothing to fold and skips it -- and that is acceptable: website-docs.yml + # also rebuilds on its own daily schedule (and port-status-nightly dispatches it), + # so the JDK 25 table, which shows gate baselines rather than live results, is at + # most a day behind. Persisting a "rebuild owed" flag to close that window would + # add state for no visible benefit. + - name: Rebuild the Port Status page + if: steps.fold.outputs.folded == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow run website-docs.yml --ref master -f deploy_production=true diff --git a/.github/workflows/scripts-macos.yml b/.github/workflows/scripts-macos.yml index efa6a5794c9..b0d3cb8f7f1 100644 --- a/.github/workflows/scripts-macos.yml +++ b/.github/workflows/scripts-macos.yml @@ -247,6 +247,10 @@ jobs: # got it, and JDK 25 is fetched privately so no later step sees a different JDK. - name: ParparVM vs JDK 25 (performance gate) if: success() || failure() + # Names this pull request's overlay (perf-baseline/pr/.json) in the + # comment. A workflow_dispatch run carries no pull request, so the gate asks gh. + env: + GH_TOKEN: ${{ github.token }} run: | set -u ENV_FILE="${TMPDIR%/}/codenameone-tools/tools/env.sh" diff --git a/.github/workflows/website-docs.yml b/.github/workflows/website-docs.yml index 8b1353f71cb..079e677b2c5 100644 --- a/.github/workflows/website-docs.yml +++ b/.github/workflows/website-docs.yml @@ -45,6 +45,11 @@ on: - 'vm/backend/src/**' - 'vm/backend/impl/**' - 'vm/backend/shared-sources.sh' + # build.sh renders the Port Status JDK 25 table with this script's `summary`, so a + # change to its output shape must go through the Hugo build and validate_port_status. + # The baseline DATA (vm/selfhost/perf-baseline/**) is left out on purpose: a row + # changes numbers, not shape, and the nightly fold dispatches a rebuild for it. + - 'vm/selfhost/perf_baseline.py' - '.github/workflows/website-docs.yml' push: branches: [main, master] @@ -78,6 +83,8 @@ on: - 'vm/backend/src/**' - 'vm/backend/impl/**' - 'vm/backend/shared-sources.sh' + # As above: the JDK 25 table's renderer. + - 'vm/selfhost/perf_baseline.py' - '.github/workflows/website-docs.yml' workflow_dispatch: inputs: diff --git a/.gitignore b/.gitignore index 9f336a95563..2e669e77b79 100644 --- a/.gitignore +++ b/.gitignore @@ -196,3 +196,6 @@ cn1-build-hints.json # Concatenated CSS the native-theme build feeds the compiler. Regenerated on # every run; the parts under native-themes//*.css are the source. native-themes/*/target/ + +# Rendered from vm/selfhost/perf-baseline by scripts/website/build.sh; never committed. +docs/website/data/port_status_jdk25.json diff --git a/docs/website/assets/css/extended/cn1-port-status.css b/docs/website/assets/css/extended/cn1-port-status.css index 20e7179f398..8d48938ad1e 100644 --- a/docs/website/assets/css/extended/cn1-port-status.css +++ b/docs/website/assets/css/extended/cn1-port-status.css @@ -490,6 +490,21 @@ min-width: 1320px; } +.cn1-port-status__matrix--jdk25 table { + min-width: 900px; +} + +.cn1-port-status__ratio { + display: block; + white-space: nowrap; +} + +.cn1-port-status__ratio small { + color: var(--secondary); + display: block; + font-size: .68rem; +} + .cn1-port-status__matrix--performance tbody th small { color: var(--secondary); display: block; diff --git a/docs/website/layouts/_default/port-status.html b/docs/website/layouts/_default/port-status.html index e98ae000d65..c01640aeb8d 100644 --- a/docs/website/layouts/_default/port-status.html +++ b/docs/website/layouts/_default/port-status.html @@ -319,6 +319,63 @@

{{ $support.benchmark.title }}

+ {{- /* Rendered by scripts/website/build.sh from vm/selfhost/perf-baseline: the ratios + every platform build's performance gate holds ParparVM to, so this table and the + gate cannot disagree. Absent from a bare `hugo` run, hence the guard. */ -}} + {{- with site.Data.port_status_jdk25 }} + {{- $jdk := . -}} +
+
+

Gated performance

+

ParparVM against JDK 25

+

+ Each value is ParparVM's time or peak memory divided by HotSpot JDK 25 running the same code on the same CI runner, + as the median of interleaved, paired runs; below 1.00x ParparVM is faster or smaller. These are the baselines every + platform build is gated on: a pull request that moves one past its tolerance, in either direction, fails until the new value is + recorded in the repository. A platform measured on several runner CPU models shows the median across them and, below it, the range. +

+
+
+ + + + + {{- range $jdk.platforms }}{{- end }} + + + + {{- range $jdk.benchmarks }} + {{- $benchmark := . -}} + + + {{- range $jdk.platforms }} + {{- $platform := . -}} + {{- $cell := index .benchmarks $benchmark.id -}} + {{- $covered := 0 -}}{{- with $cell -}}{{- $covered = len .cpus -}}{{- end -}} + + {{- end }} + + {{- end }} + +
Workload{{ .name }}
{{ .name }}

{{ .description }}

Time and RAM, ParparVM / JDK 25		 + {{- with $cell -}} + {{- range $metric := slice "time" "memory" -}} + {{- $m := index $cell $metric -}} + {{- $lo := printf "%.2f" (float $m.min) -}} + {{- $hi := printf "%.2f" (float $m.max) -}} + {{ if eq $metric "time" }}Time{{ else }}RAM{{ end }} {{ printf "%.2fx" (float $m.median) }} + {{- if ne $lo $hi }}{{ $lo }}-{{ $hi }}x{{ end -}} + + {{- end -}} + {{- else -}}Not measured{{- end -}} +
+
+

+ Performance gate baselines +

+
+ {{- end }} +