From aa0a9191ca744f4e402ca079355dee9666fa2a98 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 17:01:05 +0100 Subject: [PATCH 1/3] Use a separate retry timeout for tensor tests in CI The tensor tests run in a few minutes but shared the main suite's retry timeout, so a sporadic hang there burned the whole timeout before a retry. Add a dedicated rerun-tensor-tests-timeout in conda-package.yml (15 min) and check-onemath.yaml (10 min). In check-onemath.yaml also raise the main suite retry timeout to 30 min, since healthy runs take up to ~19 min, and the oneMath develop job timeout from 60 to 120 min to match the tag job, so a retry can still fit. --- .github/workflows/check-onemath.yaml | 9 +++++---- .github/workflows/conda-package.yml | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 50aef0954b7..1e02ec05b93 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -23,7 +23,8 @@ env: test-env-name: 'test_onemath' rerun-tests-on-failure: 'true' rerun-tests-max-attempts: 2 - rerun-tests-timeout: 20 + rerun-tests-timeout: 30 + rerun-tensor-tests-timeout: 10 jobs: build_env_file: @@ -196,7 +197,7 @@ jobs: id: run_tensor_tests uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: - timeout_minutes: ${{ env.rerun-tests-timeout }} + timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | @@ -221,7 +222,7 @@ jobs: os: [ubuntu-latest] # windows-2022 - no DFT support for Windows in oneMKL runs-on: ${{ matrix.os }} - timeout-minutes: 60 + timeout-minutes: 120 defaults: run: @@ -330,7 +331,7 @@ jobs: id: run_tensor_tests_branch uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: - timeout_minutes: ${{ env.rerun-tests-timeout }} + timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index c68402601ae..bc77f3ce741 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -24,6 +24,7 @@ env: rerun-tests-on-failure: 'true' rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 + rerun-tensor-tests-timeout: 15 jobs: build: @@ -308,7 +309,7 @@ jobs: id: run_tests_tensor_linux uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: - timeout_minutes: ${{ env.rerun-tests-timeout }} + timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | @@ -617,7 +618,7 @@ jobs: id: run_tests_tensor_win uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: - timeout_minutes: ${{ env.rerun-tests-timeout }} + timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any shell: pwsh From 640928b9d9df183f4462a4ce56e29ffe976e5f69 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 12:41:35 +0100 Subject: [PATCH 2/3] Use a longer tensor tests timeout on Windows and in conda-forge CI Healthy Windows tensor runs take up to ~19 min, so a 15 min timeout could kill a slow but passing run. Add rerun-tensor-tests-timeout-win (20 min) and use it for the Windows tensor step in conda-package.yml. Also use the dedicated tensor timeouts (15 min on Linux, 20 min on Windows) in the reusable conda-build-test.yml, since conda-package-cf.yml hits the same tensor hang. --- .github/workflows/conda-build-test.yml | 5 ++++- .github/workflows/conda-package.yml | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index 570312ba8d9..16d9aac7045 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -35,6 +35,9 @@ env: test-env-name: 'test' rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 + rerun-tensor-tests-timeout: 15 + # Healthy Windows tensor runs take up to ~19 min + rerun-tensor-tests-timeout-win: 20 jobs: build: @@ -285,7 +288,7 @@ jobs: id: run_tests_tensor uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: - timeout_minutes: ${{ env.rerun-tests-timeout }} + timeout_minutes: ${{ runner.os == 'Windows' && env.rerun-tensor-tests-timeout-win || env.rerun-tensor-tests-timeout }} max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any shell: bash diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index bc77f3ce741..b25bd9caf94 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -25,6 +25,8 @@ env: rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 + # Healthy Windows tensor runs take up to ~19 min + rerun-tensor-tests-timeout-win: 20 jobs: build: @@ -618,7 +620,7 @@ jobs: id: run_tests_tensor_win uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: - timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} + timeout_minutes: ${{ env.rerun-tensor-tests-timeout-win }} max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any shell: pwsh From 6901db927f5c1358aceff303c834a1e706969954 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 12:47:01 +0100 Subject: [PATCH 3/3] Allow a third tensor tests attempt in CI With the shorter tensor timeouts, three attempts cost less than the old two, and a hang in both attempts does happen. Add rerun-tensor-tests-max-attempts (3) and use it in the tensor retry steps. Raise the conda-package.yml Linux test job timeout from 100 to 120 min, the same as Windows, so a main suite hang plus two tensor hangs still fit. --- .github/workflows/check-onemath.yaml | 5 +++-- .github/workflows/conda-build-test.yml | 3 ++- .github/workflows/conda-package.yml | 7 ++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 1e02ec05b93..1454a1a278f 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -25,6 +25,7 @@ env: rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 + rerun-tensor-tests-max-attempts: 3 jobs: build_env_file: @@ -198,7 +199,7 @@ jobs: uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} + max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh @@ -332,7 +333,7 @@ jobs: uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} + max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index 16d9aac7045..6b123472fe1 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -36,6 +36,7 @@ env: rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 + rerun-tensor-tests-max-attempts: 3 # Healthy Windows tensor runs take up to ~19 min rerun-tensor-tests-timeout-win: 20 @@ -289,7 +290,7 @@ jobs: uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: ${{ runner.os == 'Windows' && env.rerun-tensor-tests-timeout-win || env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} + max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} retry_on: any shell: bash command: | diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index b25bd9caf94..e00453594fc 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -25,6 +25,7 @@ env: rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 + rerun-tensor-tests-max-attempts: 3 # Healthy Windows tensor runs take up to ~19 min rerun-tensor-tests-timeout-win: 20 @@ -156,7 +157,7 @@ jobs: needs: build runs-on: ${{ matrix.os }} - timeout-minutes: 100 + timeout-minutes: 120 defaults: run: @@ -312,7 +313,7 @@ jobs: uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} + max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh @@ -621,7 +622,7 @@ jobs: uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: ${{ env.rerun-tensor-tests-timeout-win }} - max_attempts: ${{ env.rerun-tests-max-attempts }} + max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} retry_on: any shell: pwsh command: |