From 6e5c5255b080098c1efa2ea386c1ea8a3170c4e1 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 4 Aug 2026 15:50:28 -0600 Subject: [PATCH 1/8] add multithreaded shutdown stress test --- .github/workflows/dynamic_arch.yml | 4 +- cpp_thread_test/CMakeLists.txt | 13 ++ cpp_thread_test/Makefile | 11 +- .../dgemm_thread_safety_shutdown.cpp | 165 ++++++++++++++++++ 4 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 cpp_thread_test/dgemm_thread_safety_shutdown.cpp diff --git a/.github/workflows/dynamic_arch.yml b/.github/workflows/dynamic_arch.yml index 67919abe49..f7d63d8c8d 100644 --- a/.github/workflows/dynamic_arch.yml +++ b/.github/workflows/dynamic_arch.yml @@ -593,7 +593,7 @@ jobs: - name: Build OpenBLAS run: | cd build - cmake --build . --target dgemm_thread_safety dgemm_thread_safety_mixed dgemv_thread_safety + cmake --build . --target dgemm_thread_safety dgemm_thread_safety_mixed dgemm_thread_safety_shutdown dgemv_thread_safety - name: Show ccache status continue-on-error: true @@ -611,7 +611,7 @@ jobs: run: | cd build export PATH="$PWD/lib:$PATH" - OPENBLAS_NUM_THREADS=8 OMP_NUM_THREADS=16 ctest -R 'dgemm_thread_safety|dgemm_thread_safety_mixed|dgemv_thread_safety' --output-on-failure + OPENBLAS_NUM_THREADS=8 OMP_NUM_THREADS=16 ctest -R 'dgemm_thread_safety|dgemm_thread_safety_mixed|dgemm_thread_safety_shutdown|dgemv_thread_safety' --output-on-failure cross_build: diff --git a/cpp_thread_test/CMakeLists.txt b/cpp_thread_test/CMakeLists.txt index 5271d4594c..0f140f25f8 100644 --- a/cpp_thread_test/CMakeLists.txt +++ b/cpp_thread_test/CMakeLists.txt @@ -19,6 +19,7 @@ endif() set(CPP_THREAD_SAFETY_DGEMM_ARGS "" CACHE STRING "Arguments passed to the DGEMM thread safety test") set(CPP_THREAD_SAFETY_DGEMM_MIXED_ARGS "" CACHE STRING "Arguments passed to the mixed DGEMM thread safety test") set(CPP_THREAD_SAFETY_DGEMV_ARGS "" CACHE STRING "Arguments passed to the DGEMV thread safety test") +set(CPP_THREAD_SAFETY_SHUTDOWN_ARGS "" CACHE STRING "Arguments passed to the DGEMM shutdown safety test") if (CPP_THREAD_SAFETY_TEST) message(STATUS "building thread safety test") @@ -29,6 +30,18 @@ if (CPP_THREAD_SAFETY_TEST) add_executable(dgemm_thread_safety_mixed dgemm_thread_safety_mixed.cpp) target_link_libraries(dgemm_thread_safety_mixed ${CPP_THREAD_SAFETY_LIBS}) add_test(NAME dgemm_thread_safety_mixed COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dgemm_thread_safety_mixed ${CPP_THREAD_SAFETY_DGEMM_MIXED_ARGS}) + + # The shutdown race is Windows-specific: on POSIX, exit() runs the library + # destructor while worker threads are still computing into OpenBLAS-owned + # buffers, which no amount of locking inside blas_shutdown can make safe. + if (WIN32) + add_executable(dgemm_thread_safety_shutdown dgemm_thread_safety_shutdown.cpp) + target_link_libraries(dgemm_thread_safety_shutdown ${CPP_THREAD_SAFETY_LIBS}) + add_test(NAME dgemm_thread_safety_shutdown COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dgemm_thread_safety_shutdown ${CPP_THREAD_SAFETY_SHUTDOWN_ARGS}) + # Bounded by the test itself: 40 children, each killed after 10s at worst. + # A passing run takes a few seconds; only a failing one approaches this. + set_tests_properties(dgemm_thread_safety_shutdown PROPERTIES TIMEOUT 600) + endif() endif() diff --git a/cpp_thread_test/Makefile b/cpp_thread_test/Makefile index fe7a286251..1aa04094b0 100644 --- a/cpp_thread_test/Makefile +++ b/cpp_thread_test/Makefile @@ -3,6 +3,11 @@ include $(TOPDIR)/Makefile.system all :: dgemv_tester dgemm_tester dgemm_mixed_tester +# The shutdown race is Windows-specific; see dgemm_thread_safety_shutdown.cpp. +ifeq ($(OSNAME), WINNT) +all :: dgemm_shutdown_tester +endif + dgemv_tester : $(CXX) $(COMMON_OPT) -Wall -Wextra -Wshadow -std=c++11 dgemv_thread_safety.cpp ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) -o dgemv_tester ./dgemv_tester @@ -15,5 +20,9 @@ dgemm_mixed_tester : dgemm_tester $(CXX) $(COMMON_OPT) -Wall -Wextra -Wshadow -std=c++11 dgemm_thread_safety_mixed.cpp ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) -o dgemm_mixed_tester ./dgemm_mixed_tester +dgemm_shutdown_tester : dgemm_mixed_tester + $(CXX) $(COMMON_OPT) -Wall -Wextra -Wshadow -std=c++11 dgemm_thread_safety_shutdown.cpp ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) -o dgemm_shutdown_tester + ./dgemm_shutdown_tester + clean :: - rm -f dgemv_tester dgemm_tester dgemm_mixed_tester + rm -f dgemv_tester dgemm_tester dgemm_mixed_tester dgemm_shutdown_tester diff --git a/cpp_thread_test/dgemm_thread_safety_shutdown.cpp b/cpp_thread_test/dgemm_thread_safety_shutdown.cpp new file mode 100644 index 0000000000..4afc7828f0 --- /dev/null +++ b/cpp_thread_test/dgemm_thread_safety_shutdown.cpp @@ -0,0 +1,165 @@ +/* Stress test for library shutdown racing with in-flight BLAS calls + * (https://github.com/OpenMathLib/OpenBLAS/issues/5954). + * + * Windows only. On POSIX, exit() runs the library destructor while worker + * threads are still computing into OpenBLAS-owned buffers, which no amount of + * locking inside blas_shutdown can make safe, so there is nothing to assert + * there; CMakeLists.txt only registers this test on WIN32. + * + * The parent re-executes itself as short-lived children and checks that each + * one terminates cleanly, turning shutdown-path crashes and deadlocks into + * ordinary test failures. Each child (--child-storm N) starts N callers that + * allocate their matrices and park on a gate, releases them so they all enter + * their first dgemm at once, and exits a millisecond later while that + * allocation storm is still in flight. + * + * N must exceed NUM_BUFFERS = MAX(50, NUM_THREADS * 2 * NUM_PARALLEL) for the + * build under test; below that every slot is already mapped and the race is + * unreachable. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef OPENBLAS_USE_GENERATED_CBLAS_H +#include "generated/cblas.h" +#else +#include "../cblas.h" +#endif + +#include + +namespace { + +const blasint stormM = 200, stormK = 120, stormN = 90; /* the gh-5954 shape */ +const blasint poolDim = 320; /* above the multithreading threshold, so the pool spins up */ +const uint32_t defaultStormCallers = 128; +const uint32_t stormDelayMs = 3; /* gate to sweep; at 0 the sweep beats the allocations */ +const int stormBlasThreads = 4; +const int stormTimeoutSec = 10; +const int numStormChildren = 40; + +std::atomic parked(0); /* callers built and waiting on the gate */ +std::atomic gate(false); + +void fillOperands(std::vector& A, std::vector& B) { + for (size_t i = 0; i < A.size(); i++) A[i] = (i % 1000) / 1000.0; + for (size_t i = 0; i < B.size(); i++) B[i] = (i % 997) / 997.0; +} + +void dgemmOnce(blasint m, blasint k, blasint n) { + std::vector A(m * k), B(k * n), C(m * n); + fillOperands(A, B); + cblas_dgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, + 1.0, A.data(), m, B.data(), k, 0.1, C.data(), m); +} + +/* Allocate before parking, so that when the gate opens nothing stands between + the thread and its first dgemm. */ +void gatedWorker(blasint m, blasint k, blasint n) { + std::vector A(m * k), B(k * n), C(m * n); + fillOperands(A, B); + parked.fetch_add(1, std::memory_order_release); + while (!gate.load(std::memory_order_acquire)) std::this_thread::yield(); + for (;;) + cblas_dgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, + 1.0, A.data(), m, B.data(), k, 0.1, C.data(), m); +} + +int ChildStorm(uint32_t nCallers) { + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + openblas_set_num_threads(stormBlasThreads); + + /* Build the OpenBLAS worker pool first, so the storm is buffer allocation + and not pool startup. */ + dgemmOnce(poolDim, poolDim, poolDim); + + for (uint32_t i = 0; i < nCallers; i++) + std::thread(gatedWorker, stormM, stormK, stormN).detach(); + for (int ms = 0; parked.load(std::memory_order_acquire) < nCallers && ms < 10000; ms++) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + gate.store(true, std::memory_order_release); + std::this_thread::sleep_for(std::chrono::milliseconds(stormDelayMs)); + std::exit(0); +} + +/* Returns 0 if the child exited cleanly, nonzero otherwise; fills outcome. */ +int RunChild(const std::string& args, int timeoutSec, std::string& outcome) { + char exe[MAX_PATH]; + if (GetModuleFileNameA(NULL, exe, MAX_PATH) == 0) { + outcome = "GetModuleFileName failed"; + return 1; + } + std::string cmd = "\"" + std::string(exe) + "\" " + args; + + STARTUPINFOA si; + PROCESS_INFORMATION pi; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + ZeroMemory(&pi, sizeof(pi)); + if (!CreateProcessA(NULL, &cmd[0], NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { + outcome = "CreateProcess failed"; + return 1; + } + + int ret = 1; + char buf[64]; + if (WaitForSingleObject(pi.hProcess, timeoutSec * 1000) != WAIT_OBJECT_0) { + TerminateProcess(pi.hProcess, 1); + WaitForSingleObject(pi.hProcess, 5000); + snprintf(buf, sizeof(buf), "HANG (killed after %ds)", timeoutSec); + } else { + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + if (code == 0) { + snprintf(buf, sizeof(buf), "clean exit"); + ret = 0; + } else { + snprintf(buf, sizeof(buf), "CRASH (exit code 0x%08lX)", (unsigned long)code); + } + } + outcome = buf; + + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return ret; +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc >= 3 && std::strcmp(argv[1], "--child-storm") == 0) + return ChildStorm(uint32_t(std::atoi(argv[2]))); + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + + uint32_t callers = defaultStormCallers; + if (argc >= 2) { + int n = std::atoi(argv[1]); + if (n > 0) callers = uint32_t(n); + } + + int failures = 0; + std::cout << "Testing process exit during an allocation storm (" << callers << " callers)" + << std::endl; + for (int i = 0; i < numStormChildren; i++) { + std::string outcome; + failures += RunChild("--child-storm " + std::to_string(callers), stormTimeoutSec, outcome); + std::cout << " storm child " << i << ": " << outcome << std::endl; + } + + if (failures) { + std::cout << "CBLAS DGEMM shutdown safety test FAILED! (" << failures + << " child processes)" << std::endl; + return 1; + } + std::cout << "CBLAS DGEMM shutdown safety test PASSED!" << std::endl; + return 0; +} From 95120da710e8ca28d7e6a896fd82cf1cdb0a07ad Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 08:10:30 -0600 Subject: [PATCH 2/8] Centralize buffer release bookkeeping --- driver/others/memory.c | 130 +++++++++++++---------------------------- 1 file changed, 40 insertions(+), 90 deletions(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index a28edeaa63..7cef0f8dc0 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -2123,6 +2123,37 @@ static pthread_spinlock_t alloc_lock = 0; static BLASULONG alloc_lock = 0UL; #endif +static void blas_release_register(void *address, void (*func)(struct release_t *), long attr) { + + struct release_t *release; + int rpos; + +#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP) + LOCK_COMMAND(&alloc_lock); +#endif +#if defined(HAVE_C11) && !defined(__cplusplus) + rpos = atomic_fetch_add(&release_pos, 1); +#elif defined(__GNUC__) + rpos = __sync_fetch_and_add(&release_pos, 1); +#elif defined(OS_WINDOWS) + rpos = InterlockedIncrement((LONG volatile *)&release_pos) - 1; +#else + rpos = release_pos++; +#endif + if (likely(rpos < NUM_BUFFERS)) { + release = &release_info[rpos]; + } else { + release = &new_release_info[rpos - NUM_BUFFERS]; + } + release->address = address; + release->attr = attr; + WMB; + release->func = func; +#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP) + UNLOCK_COMMAND(&alloc_lock); +#endif +} + #ifdef ALLOC_MMAP static void alloc_mmap_free(struct release_t *release){ @@ -2154,20 +2185,7 @@ static void *alloc_mmap(void *address){ } if (map_address != (void *)-1) { -#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP) - LOCK_COMMAND(&alloc_lock); -#endif - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].func = alloc_mmap_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].func = alloc_mmap_free; - } -#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP) - UNLOCK_COMMAND(&alloc_lock); -#endif + blas_release_register(map_address, alloc_mmap_free, 0); } else { #ifdef DEBUG int errsv=errno; @@ -2323,20 +2341,7 @@ static void *alloc_mmap(void *address){ #endif if (map_address != (void *)-1) { -#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP) - LOCK_COMMAND(&alloc_lock); -#endif - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].func = alloc_mmap_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].func = alloc_mmap_free; - } -#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP) - UNLOCK_COMMAND(&alloc_lock); -#endif + blas_release_register(map_address, alloc_mmap_free, 0); } return map_address; @@ -2364,14 +2369,7 @@ static void *alloc_malloc(void *address){ if (map_address == (void *)NULL) map_address = (void *)-1; if (map_address != (void *)-1) { - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].func = alloc_malloc_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].func = alloc_malloc_free; - } + blas_release_register(map_address, alloc_malloc_free, 0); } return map_address; @@ -2403,14 +2401,7 @@ static void *alloc_qalloc(void *address){ if (map_address == (void *)NULL) map_address = (void *)-1; if (map_address != (void *)-1) { - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].func = alloc_qalloc_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].func = alloc_qalloc_free; - } + blas_release_register(map_address, alloc_qalloc_free, 0); } return (void *)(((BLASULONG)map_address + FIXED_PAGESIZE - 1) & ~(FIXED_PAGESIZE - 1)); @@ -2437,14 +2428,7 @@ static void *alloc_windows(void *address){ if (map_address == (void *)NULL) map_address = (void *)-1; if (map_address != (void *)-1) { - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].func = alloc_windows_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].func = alloc_windows_free; - } + blas_release_register(map_address, alloc_windows_free, 0); } return map_address; @@ -2486,16 +2470,7 @@ static void *alloc_devicedirver(void *address){ fd, 0); if (map_address != (void *)-1) { - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].attr = fd; - release_info[rpos].func = alloc_devicedirver_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].attr = fd; - new_release_info[rpos-NUM_BUFFERS].func = alloc_devicedirver_free; - } + blas_release_register(map_address, alloc_devicedirver_free, fd); } return map_address; @@ -2530,16 +2505,7 @@ static void *alloc_shm(void *address){ shmctl(shmid, IPC_RMID, 0); - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].attr = shmid; - release_info[rpos].func = alloc_shm_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].attr = shmid; - new_release_info[rpos-NUM_BUFFERS].func = alloc_shm_free; - } + blas_release_register(map_address, alloc_shm_free, shmid); } return map_address; @@ -2647,14 +2613,7 @@ fprintf(stderr,"alloc_hugetlb got called\n"); #endif if (map_address != (void *)-1){ - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].func = alloc_hugetlb_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].func = alloc_hugetlb_free; - } + blas_release_register(map_address, alloc_hugetlb_free, 0); } return map_address; @@ -2699,16 +2658,7 @@ static void *alloc_hugetlbfile(void *address){ fd, 0); if (map_address != (void *)-1) { - int rpos = release_pos++; - if (likely(rpos < NUM_BUFFERS)) { - release_info[rpos].address = map_address; - release_info[rpos].attr = fd; - release_info[rpos].func = alloc_hugetlbfile_free; - } else { - new_release_info[rpos-NUM_BUFFERS].address = map_address; - new_release_info[rpos-NUM_BUFFERS].attr = fd; - new_release_info[rpos-NUM_BUFFERS].func = alloc_hugetlbfile_free; - } + blas_release_register(map_address, alloc_hugetlbfile_free, fd); } return map_address; From b8d83a72f4e12650e1c790bb43badcfb6e917875 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 08:10:43 -0600 Subject: [PATCH 3/8] Avoid possible hang on Windows by introspecting shutdown state --- driver/others/memory.c | 46 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index 7cef0f8dc0..0ec1782490 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -2099,7 +2099,7 @@ int openblas_get_num_threads(void) { struct release_t { void *address; - void (*func)(struct release_t *); + void (* _Atomic func)(struct release_t *); long attr; }; @@ -3006,7 +3006,9 @@ void *blas_memory_alloc(int procpos){ #endif memory_overflowed=1; MB; - new_release_info = (struct release_t*) malloc(NEW_BUFFERS * sizeof(struct release_t)); + /* zeroed so blas_shutdown sees NULL func in slots that were reserved but + never published */ + new_release_info = (struct release_t*) calloc(NEW_BUFFERS, sizeof(struct release_t)); newmemory = (struct newmemstruct*) malloc(NEW_BUFFERS * sizeof(struct newmemstruct)); for (i = 0; i < NEW_BUFFERS; i++) { newmemory[i].addr = (void *)0; @@ -3190,9 +3192,29 @@ void blas_memory_free_nolock(void * map_address) { free(map_address); } +#if defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT) +/* During process termination Windows has already killed every other thread, + possibly while one held alloc_lock or a blas server lock, so any cleanup + here can only deadlock or crash; the OS reclaims the memory anyway. + FreeLibrary-style unloads still clean up as before. */ +static int blas_process_is_terminating(void) { + typedef BOOLEAN (WINAPI *rtl_dll_shutdown_in_progress_t)(VOID); + rtl_dll_shutdown_in_progress_t shutdown_in_progress; + HMODULE ntdll = GetModuleHandleA("ntdll.dll"); + if (!ntdll) return 0; + shutdown_in_progress = (rtl_dll_shutdown_in_progress_t)(void *) + GetProcAddress(ntdll, "RtlDllShutdownInProgress"); + return shutdown_in_progress && shutdown_in_progress(); +} +#endif + void blas_shutdown(void){ - int pos; + int pos, release_count; + +#if defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT) + if (blas_process_is_terminating()) return; +#endif #ifdef SMP BLASFUNC(blas_thread_shutdown)(); @@ -3200,12 +3222,18 @@ void blas_shutdown(void){ LOCK_COMMAND(&alloc_lock); - for (pos = 0; pos < release_pos; pos ++) { - if (likely(pos < NUM_BUFFERS)) - release_info[pos].func(&release_info[pos]); - else - new_release_info[pos-NUM_BUFFERS].func(&new_release_info[pos-NUM_BUFFERS]); + release_count = release_pos; + for (pos = 0; pos < release_count; pos ++) { + struct release_t *release = likely(pos < NUM_BUFFERS) ? + &release_info[pos] : &new_release_info[pos-NUM_BUFFERS]; + void (*func)(struct release_t *) = release->func; + RMB; + if (func == NULL) continue; /* reserved but never published: owner died mid-allocation */ + func(release); + release->func = NULL; + release->address = NULL; } + release_pos = 0; #ifdef SEEK_ADDRESS base_address = 0UL; @@ -3232,6 +3260,8 @@ void blas_shutdown(void){ } free((void*)newmemory); newmemory = NULL; + free(new_release_info); + new_release_info = NULL; memory_overflowed = 0; } From 4ea40779bd8d7c37a0b3579a0fbc2f62bfad1ab3 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 08:37:44 -0600 Subject: [PATCH 4/8] Add explanatory note about RtlDllShutdownInProgress --- driver/others/memory.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index 0ec1782490..ca5948a197 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -3196,7 +3196,13 @@ void blas_memory_free_nolock(void * map_address) { /* During process termination Windows has already killed every other thread, possibly while one held alloc_lock or a blas server lock, so any cleanup here can only deadlock or crash; the OS reclaims the memory anyway. - FreeLibrary-style unloads still clean up as before. */ + + Nothing in the SDK reports this from a destructor. DllMain's lpReserved + distinguishes the two cases, but outside MSVC gotoblas_quit runs from the + CRT's fini array via __attribute__((destructor)) and never sees + it. RtlDllShutdownInProgress is documented under Win32 Dev Notes but + deliberately absent from the SDK headers, so callers declare it themselves + If it cannot be resolved we fall back to the previous behaviour. */ static int blas_process_is_terminating(void) { typedef BOOLEAN (WINAPI *rtl_dll_shutdown_in_progress_t)(VOID); rtl_dll_shutdown_in_progress_t shutdown_in_progress; From 2bb30e03ddc99fb895aa39597db6e4bea01e6aad Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 09:01:58 -0600 Subject: [PATCH 5/8] bump timeouts slightly --- cpp_thread_test/CMakeLists.txt | 2 +- cpp_thread_test/dgemm_thread_safety_shutdown.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp_thread_test/CMakeLists.txt b/cpp_thread_test/CMakeLists.txt index 154742d0ac..53bb3f75bf 100644 --- a/cpp_thread_test/CMakeLists.txt +++ b/cpp_thread_test/CMakeLists.txt @@ -35,7 +35,7 @@ if (CPP_THREAD_SAFETY_TEST) add_executable(dgemm_thread_safety_shutdown dgemm_thread_safety_shutdown.cpp) target_link_libraries(dgemm_thread_safety_shutdown ${CPP_THREAD_SAFETY_LIBS}) add_test(NAME dgemm_thread_safety_shutdown COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dgemm_thread_safety_shutdown ${CPP_THREAD_SAFETY_SHUTDOWN_ARGS}) - set_tests_properties(dgemm_thread_safety_shutdown PROPERTIES TIMEOUT 600) + set_tests_properties(dgemm_thread_safety_shutdown PROPERTIES TIMEOUT 900) if (USE_THREAD AND (USE_OPENMP OR (NOT WIN32 AND NOT CYGWIN))) add_test(NAME dgemm_thread_safety_mixed_callback COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dgemm_thread_safety_mixed ${CPP_THREAD_SAFETY_DGEMM_MIXED_ARGS} --callback) endif() diff --git a/cpp_thread_test/dgemm_thread_safety_shutdown.cpp b/cpp_thread_test/dgemm_thread_safety_shutdown.cpp index 4afc7828f0..2b24aa640a 100644 --- a/cpp_thread_test/dgemm_thread_safety_shutdown.cpp +++ b/cpp_thread_test/dgemm_thread_safety_shutdown.cpp @@ -43,7 +43,7 @@ const blasint poolDim = 320; /* above the multithreading threshold, so the pool const uint32_t defaultStormCallers = 128; const uint32_t stormDelayMs = 3; /* gate to sweep; at 0 the sweep beats the allocations */ const int stormBlasThreads = 4; -const int stormTimeoutSec = 10; +const int stormTimeoutSec = 15; const int numStormChildren = 40; std::atomic parked(0); /* callers built and waiting on the gate */ From 28da957ee114703c0b0712c75cfed2e9aa7a69a8 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 09:02:15 -0600 Subject: [PATCH 6/8] cache ntdll pointer at startup --- driver/others/memory.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index ca5948a197..8e24773841 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -3203,14 +3203,20 @@ void blas_memory_free_nolock(void * map_address) { it. RtlDllShutdownInProgress is documented under Win32 Dev Notes but deliberately absent from the SDK headers, so callers declare it themselves If it cannot be resolved we fall back to the previous behaviour. */ -static int blas_process_is_terminating(void) { - typedef BOOLEAN (WINAPI *rtl_dll_shutdown_in_progress_t)(VOID); - rtl_dll_shutdown_in_progress_t shutdown_in_progress; +typedef BOOLEAN (WINAPI *rtl_dll_shutdown_in_progress_t)(VOID); +static rtl_dll_shutdown_in_progress_t rtl_dll_shutdown_in_progress = NULL; + +/* Resolved at init, not on the way out: GetModuleHandle takes LdrpSnapsLock, + which ExitProcess does not release before it kills the other threads. */ +static void blas_shutdown_check_init(void) { HMODULE ntdll = GetModuleHandleA("ntdll.dll"); - if (!ntdll) return 0; - shutdown_in_progress = (rtl_dll_shutdown_in_progress_t)(void *) + if (!ntdll) return; + rtl_dll_shutdown_in_progress = (rtl_dll_shutdown_in_progress_t)(void *) GetProcAddress(ntdll, "RtlDllShutdownInProgress"); - return shutdown_in_progress && shutdown_in_progress(); +} + +static int blas_process_is_terminating(void) { + return rtl_dll_shutdown_in_progress && rtl_dll_shutdown_in_progress(); } #endif @@ -3392,6 +3398,10 @@ void CONSTRUCTOR gotoblas_init(void) { if (gotoblas_initialized) return; +#if defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT) + blas_shutdown_check_init(); +#endif + #ifdef SMP openblas_fork_handler(); #endif From 0041b780edda5747ca2e9fc06dc833ff4f39478a Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 09:28:41 -0600 Subject: [PATCH 7/8] fix broken cmake --- cpp_thread_test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp_thread_test/CMakeLists.txt b/cpp_thread_test/CMakeLists.txt index 53bb3f75bf..c81aacd836 100644 --- a/cpp_thread_test/CMakeLists.txt +++ b/cpp_thread_test/CMakeLists.txt @@ -36,6 +36,7 @@ if (CPP_THREAD_SAFETY_TEST) target_link_libraries(dgemm_thread_safety_shutdown ${CPP_THREAD_SAFETY_LIBS}) add_test(NAME dgemm_thread_safety_shutdown COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dgemm_thread_safety_shutdown ${CPP_THREAD_SAFETY_SHUTDOWN_ARGS}) set_tests_properties(dgemm_thread_safety_shutdown PROPERTIES TIMEOUT 900) + endif() if (USE_THREAD AND (USE_OPENMP OR (NOT WIN32 AND NOT CYGWIN))) add_test(NAME dgemm_thread_safety_mixed_callback COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dgemm_thread_safety_mixed ${CPP_THREAD_SAFETY_DGEMM_MIXED_ARGS} --callback) endif() From b14b82ce1780e5273a5e96b30d18e13a3214ba20 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 5 Aug 2026 15:12:14 -0600 Subject: [PATCH 8/8] Fix merge conflict resolution screwup --- cpp_thread_test/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp_thread_test/Makefile b/cpp_thread_test/Makefile index 72bf7ec50c..56c4759d0a 100644 --- a/cpp_thread_test/Makefile +++ b/cpp_thread_test/Makefile @@ -16,6 +16,7 @@ ifneq ($(OSNAME),$(filter $(OSNAME),WINNT CYGWIN_NT)) CALLBACK_THREAD_TEST = 1 endif endif +endif ifeq ($(CALLBACK_THREAD_TEST),1) .PHONY : dgemm_mixed_callback_tester