diff --git a/lib/level-zero/cmdlist.jl b/lib/level-zero/cmdlist.jl index acbf2dbd..a9d16978 100644 --- a/lib/level-zero/cmdlist.jl +++ b/lib/level-zero/cmdlist.jl @@ -91,11 +91,12 @@ mutable struct ZeImmediateCommandList <: AbstractZeCommandList @warn "Leaking an immediate command list still busy after $(FINALIZER_SYNC_TIMEOUT_NS รท 1_000_000_000)s to avoid blocking finalization" maxlog = 1 return end - zeCommandListDestroy(obj) - release(obj.context) - # mark destroyed: the stream registry can still reach this list, and - # synchronizing a destroyed handle crashes in the driver + # mark the list as destroyed before destroying it: it can still be reached and + # synchronized (see `destroyed`), which crashes in the driver on a destroyed handle + handle = obj.handle obj.handle = ze_command_list_handle_t(C_NULL) + zeCommandListDestroy(handle) + release(obj.context) end obj end diff --git a/lib/level-zero/cmdqueue.jl b/lib/level-zero/cmdqueue.jl index 41c72098..df65efb0 100644 --- a/lib/level-zero/cmdqueue.jl +++ b/lib/level-zero/cmdqueue.jl @@ -62,14 +62,12 @@ mutable struct ZeCommandQueue for (_, fence) in Iterators.flatten((obj.submitted, obj.spare, obj.failed)) zeFenceDestroy(fence) end - zeCommandQueueDestroy(obj) + # mark the queue as destroyed before destroying it: it can still be reached and + # synchronized (see `destroyed`), which crashes in the driver on a destroyed handle + handle = obj.handle + obj.handle = ze_command_queue_handle_t(C_NULL) + zeCommandQueueDestroy(handle) release(obj.context) - if LTS[] - # mark the queue as destroyed: it can still be weakly reachable (e.g. from - # the stream registry used by `synchronize_all_streams`), and synchronizing a - # destroyed handle crashes in the driver. - obj.handle = ze_command_queue_handle_t(C_NULL) - end end obj end diff --git a/lib/level-zero/synchronization.jl b/lib/level-zero/synchronization.jl index d69c9c01..e10ae908 100644 --- a/lib/level-zero/synchronization.jl +++ b/lib/level-zero/synchronization.jl @@ -22,15 +22,23 @@ function check_done(res::ze_result_t) throw_api_error(res) end end +# Lists and queues are only destroyed by their finalizers, which first wait for their work +# where the driver doesn't. They can still be waited for afterwards: an object finalized in the same GC cycle as one +# of them can make it reachable again (e.g. memory freed then recording its last user), and +# synchronizing a destroyed handle crashes in the driver. +destroyed(obj::Union{ZeImmediateCommandList, ZeCommandQueue}) = obj.handle == C_NULL + Base.isdone(list::ZeImmediateCommandList) = - check_done(unchecked_zeCommandListHostSynchronize(list, 0)) + destroyed(list) || check_done(unchecked_zeCommandListHostSynchronize(list, 0)) Base.isdone(queue::ZeCommandQueue) = - check_done(unchecked_zeCommandQueueSynchronize(queue, 0)) + destroyed(queue) || check_done(unchecked_zeCommandQueueSynchronize(queue, 0)) # the blocking synchronization (GC-safe, like all Level Zero calls) gcsafe_synchronize(list::ZeImmediateCommandList) = + destroyed(list) ? RESULT_SUCCESS : unchecked_zeCommandListHostSynchronize(list, typemax(UInt64)) gcsafe_synchronize(queue::ZeCommandQueue) = + destroyed(queue) ? RESULT_SUCCESS : unchecked_zeCommandQueueSynchronize(queue, typemax(UInt64)) gcsafe_synchronize(event::ZeEvent) = unchecked_zeEventHostSynchronize(event, typemax(UInt64)) diff --git a/test/array.jl b/test/array.jl index 7e38d019..b4c9aaa2 100644 --- a/test/array.jl +++ b/test/array.jl @@ -313,6 +313,20 @@ end synchronize() @test Array(b) == ones(Float32, n_chunk) + # memory last used by a task that's done is collected together with that task's stream, + # whose command list then gets destroyed, and can be reused regardless. (the task is + # created in a function, as the testset would keep it alive.) + upload_in_task() = fetch(@async oneAPI.device_pointer(oneArray(ones(Float32, n_chunk)))) + ptr = upload_in_task() + GC.gc(true) + ys = [] + for _ in 1:64 + push!(ys, oneArray(fill(2f0, n_chunk))) + oneAPI.device_pointer(last(ys)) == ptr && break + end + oneAPI.POOL[] && @test oneAPI.device_pointer(last(ys)) == ptr + @test all(y -> Array(y) == fill(2f0, n_chunk), ys) + # many small arrays keep their contents, also when unused memory is given back xs = [oneArray(fill(Float32(i), i)) for i in 1:300] for i in 1:2:300