From e9c73e24c08993687d0eb6a4c1d4711c96432dcc Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 23 Oct 2026 22:26:00 +0200 Subject: [PATCH] Don't synchronize command lists and queues that were destroyed A task's stream can be collected in the same GC cycle as memory it last used. Julia doesn't order finalizers, so the stream's immediate command list is destroyed while the freed memory, returned to the pool, still names the stream as its owner. The next allocation of that memory from another task then waits for the stream, synchronizing the destroyed list, which segfaults in the driver. This crashed the `array` tests on Aurora. The finalizers only destroy a list or queue once its work is done, so treat a destroyed one as done. Mark them destroyed before destroying them, and do so for queues on every stack, not only on LTS. --- lib/level-zero/cmdlist.jl | 9 +++++---- lib/level-zero/cmdqueue.jl | 12 +++++------- lib/level-zero/synchronization.jl | 12 ++++++++++-- test/array.jl | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/level-zero/cmdlist.jl b/lib/level-zero/cmdlist.jl index acbf2dbd..a9d16978 100644 --- a/lib/level-zero/cmdlist.jl +++ b/lib/level-zero/cmdlist.jl @@ -91,11 +91,12 @@ mutable struct ZeImmediateCommandList <: AbstractZeCommandList @warn "Leaking an immediate command list still busy after $(FINALIZER_SYNC_TIMEOUT_NS รท 1_000_000_000)s to avoid blocking finalization" maxlog = 1 return end - zeCommandListDestroy(obj) - release(obj.context) - # mark destroyed: the stream registry can still reach this list, and - # synchronizing a destroyed handle crashes in the driver + # mark the list as destroyed before destroying it: it can still be reached and + # synchronized (see `destroyed`), which crashes in the driver on a destroyed handle + handle = obj.handle obj.handle = ze_command_list_handle_t(C_NULL) + zeCommandListDestroy(handle) + release(obj.context) end obj end diff --git a/lib/level-zero/cmdqueue.jl b/lib/level-zero/cmdqueue.jl index 41c72098..df65efb0 100644 --- a/lib/level-zero/cmdqueue.jl +++ b/lib/level-zero/cmdqueue.jl @@ -62,14 +62,12 @@ mutable struct ZeCommandQueue for (_, fence) in Iterators.flatten((obj.submitted, obj.spare, obj.failed)) zeFenceDestroy(fence) end - zeCommandQueueDestroy(obj) + # mark the queue as destroyed before destroying it: it can still be reached and + # synchronized (see `destroyed`), which crashes in the driver on a destroyed handle + handle = obj.handle + obj.handle = ze_command_queue_handle_t(C_NULL) + zeCommandQueueDestroy(handle) release(obj.context) - if LTS[] - # mark the queue as destroyed: it can still be weakly reachable (e.g. from - # the stream registry used by `synchronize_all_streams`), and synchronizing a - # destroyed handle crashes in the driver. - obj.handle = ze_command_queue_handle_t(C_NULL) - end end obj end diff --git a/lib/level-zero/synchronization.jl b/lib/level-zero/synchronization.jl index d69c9c01..e10ae908 100644 --- a/lib/level-zero/synchronization.jl +++ b/lib/level-zero/synchronization.jl @@ -22,15 +22,23 @@ function check_done(res::ze_result_t) throw_api_error(res) end end +# Lists and queues are only destroyed by their finalizers, which first wait for their work +# where the driver doesn't. They can still be waited for afterwards: an object finalized in the same GC cycle as one +# of them can make it reachable again (e.g. memory freed then recording its last user), and +# synchronizing a destroyed handle crashes in the driver. +destroyed(obj::Union{ZeImmediateCommandList, ZeCommandQueue}) = obj.handle == C_NULL + Base.isdone(list::ZeImmediateCommandList) = - check_done(unchecked_zeCommandListHostSynchronize(list, 0)) + destroyed(list) || check_done(unchecked_zeCommandListHostSynchronize(list, 0)) Base.isdone(queue::ZeCommandQueue) = - check_done(unchecked_zeCommandQueueSynchronize(queue, 0)) + destroyed(queue) || check_done(unchecked_zeCommandQueueSynchronize(queue, 0)) # the blocking synchronization (GC-safe, like all Level Zero calls) gcsafe_synchronize(list::ZeImmediateCommandList) = + destroyed(list) ? RESULT_SUCCESS : unchecked_zeCommandListHostSynchronize(list, typemax(UInt64)) gcsafe_synchronize(queue::ZeCommandQueue) = + destroyed(queue) ? RESULT_SUCCESS : unchecked_zeCommandQueueSynchronize(queue, typemax(UInt64)) gcsafe_synchronize(event::ZeEvent) = unchecked_zeEventHostSynchronize(event, typemax(UInt64)) diff --git a/test/array.jl b/test/array.jl index 7e38d019..b4c9aaa2 100644 --- a/test/array.jl +++ b/test/array.jl @@ -313,6 +313,20 @@ end synchronize() @test Array(b) == ones(Float32, n_chunk) + # memory last used by a task that's done is collected together with that task's stream, + # whose command list then gets destroyed, and can be reused regardless. (the task is + # created in a function, as the testset would keep it alive.) + upload_in_task() = fetch(@async oneAPI.device_pointer(oneArray(ones(Float32, n_chunk)))) + ptr = upload_in_task() + GC.gc(true) + ys = [] + for _ in 1:64 + push!(ys, oneArray(fill(2f0, n_chunk))) + oneAPI.device_pointer(last(ys)) == ptr && break + end + oneAPI.POOL[] && @test oneAPI.device_pointer(last(ys)) == ptr + @test all(y -> Array(y) == fill(2f0, n_chunk), ys) + # many small arrays keep their contents, also when unused memory is given back xs = [oneArray(fill(Float32(i), i)) for i in 1:300] for i in 1:2:300