From 6e34cb0083b5fb5a28ebe0634c0391eb6dd2fb38 Mon Sep 17 00:00:00 2001 From: antono3 Date: Sat, 12 Sep 2026 17:39:48 +0200 Subject: [PATCH] Add class-safe image suballocation --- .github/workflows/test.yml | 9 +- CHANGELOG.md | 19 ++ README.md | 95 +++++-- block_pool.v | 112 ++++++--- block_pool_stress_test.v | 39 ++- block_pool_test.v | 46 ++++ diagnostics.v | 19 +- diagnostics_test.v | 15 +- examples/image_suballocation/main.v | 218 ++++++++++++++++ memory_policy_test.v | 2 +- v.mod | 2 +- vulkan_memory_allocator.v | 376 +++++++++++++++++++--------- vulkan_memory_allocator_test.v | 41 ++- 13 files changed, 786 insertions(+), 207 deletions(-) create mode 100644 examples/image_suballocation/main.v diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 798436c..357385a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,7 @@ jobs: env: VMODULES: ${{ github.workspace }}/source/modules VULKAN_SDK: /usr + VK_INSTANCE_LAYERS: VK_LAYER_KHRONOS_validation steps: - uses: actions/checkout@v7 with: @@ -33,7 +34,7 @@ jobs: with: version: 0.5.2 - name: Install Vulkan development library - run: sudo apt-get update && sudo apt-get install -y libvulkan-dev libvulkan-volk-dev mesa-vulkan-drivers + run: sudo apt-get update && sudo apt-get install -y libvulkan-dev libvulkan-volk-dev mesa-vulkan-drivers vulkan-validationlayers - name: Check formatting working-directory: source/modules/antono2/vkmemalloc run: v fmt -verify . @@ -50,6 +51,12 @@ jobs: lavapipe_icd=$(find /usr/share/vulkan/icd.d -name 'lvp_icd*.json' -print -quit) test -n "$lavapipe_icd" VK_ICD_FILENAMES="$lavapipe_icd" v run examples/buffer_suballocation + - name: Run Vulkan image suballocation smoke test + working-directory: source/modules/antono2/vkmemalloc + run: | + lavapipe_icd=$(find /usr/share/vulkan/icd.d -name 'lvp_icd*.json' -print -quit) + test -n "$lavapipe_icd" + VK_ICD_FILENAMES="$lavapipe_icd" v run examples/image_suballocation - name: Run sustained Vulkan allocation workload working-directory: source/modules/antono2/vkmemalloc run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index ab838bf..28194b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file. +## 2.6.0 - 2026-09-12 + +- Add explicit `create_suballocated_image()` and + `create_suballocated_image_with_options()` APIs while retaining dedicated + allocation in the existing image APIs. +- Separate reusable blocks by Vulkan memory type and resource class so buffers, + linear images, and optimal images never share a block or create a + `bufferImageGranularity` boundary. +- Query Vulkan 1.1 dedicated-allocation requirements and honor both required and + preferred dedicated memory; conservatively use dedicated memory on Vulkan + 1.0. +- Expose the resource class in allocations, events, and class-filtered + fragmentation statistics. +- Reject sparse, disjoint, and DRM-format-modifier images in the ordinary image + suballocation path, which cannot provide their specialized binding rules. +- Add CPU isolation/ownership tests and a lavapipe example proving real optimal + image sharing alongside separate linear-image and buffer blocks; run every + real Vulkan workload with the Khronos validation layer in CI. + ## 2.5.0 - 2026-09-12 - Add cumulative allocation, fallback, block-reuse, trim, and high-water-mark diff --git a/README.md b/README.md index ecdbcf4..1f97992 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,18 @@ enough for examples while avoiding one Vulkan allocation per resource. ## How it fits together -The allocator has three deliberately separate layers: +The allocator has four deliberately separate layers: 1. **Policy** filters the memory types allowed by Vulkan, applies required property flags, and ranks the remaining types for GPU-only, upload, or readback use. The selected `MemoryTypeChoice` explains the heap, flags, budget state, and score. 2. **Block planning** uses `antono2.memory.RangeAllocator` to place compatible - buffers into larger memory-type-specific blocks. This CPU-only layer is - deterministic and independently tested. + resources into larger blocks. Every block has both a Vulkan memory type and + a resource class (`buffer`, `linear_image`, or `optimal_image`), so classes + with a [`bufferImageGranularity`](https://docs.vulkan.org/spec/latest/chapters/limits.html#limits-bufferimagegranularity) + boundary rule never become adjacent. This + CPU-only layer is deterministic and independently tested. 3. **Vulkan ownership** creates, maps, binds, and frees `VkDeviceMemory` while `AllocationInfo` keeps the selected type, heap, properties, block size, and private ownership record together. @@ -28,9 +31,10 @@ The allocator has three deliberately separate layers: activity and an optional bounded event trace. Normal applications pay no trace-storage cost unless they opt in. -Images remain dedicated. That conservative rule avoids hiding the additional -tiling and buffer-image granularity rules that a safe image suballocator would -need to model. Existing `MemType` APIs remain available for short examples; +The default image APIs remain dedicated for compatibility. Explicit +`create_suballocated_image*` APIs safely share ordinary linear- or +optimal-tiling images within their own class and honor driver requests for +dedicated memory. Existing `MemType` APIs remain available for short examples; new applications should normally use `AllocationOptions`. ## Install @@ -75,11 +79,12 @@ mut allocator := vma.new(vma.AllocatorCreateInfo{ }) ``` -Buffer blocks are separated by Vulkan memory-type index. Small buffers share the -preferred block size; a buffer larger than that receives a large-enough block -of its own. Images use isolated dedicated blocks, avoiding buffer-image -granularity conflicts and satisfying Vulkan 1.1 dedicated-allocation metadata. -`max_memory_blocks` defaults to 256 and may be lowered in the create information. +Shared blocks are separated by Vulkan memory-type index and resource class. +Small compatible resources share the preferred block size; a resource larger +than that receives a large-enough block of its own. The allocator queries +Vulkan 1.1 dedicated-allocation metadata before suballocating an image and +falls back to dedicated memory on Vulkan 1.0. `max_memory_blocks` defaults to +256 and may be lowered in the create information. The lower-level `allocate()` method also uses an isolated block because raw `VkMemoryRequirements` do not identify whether the caller will bind a buffer or @@ -159,9 +164,10 @@ if !allocator.release(mut allocation) { } ``` -`release()` returns shared-buffer ranges to their existing block so future -buffers can reuse them. Dedicated allocations are freed immediately. Empty -shared blocks remain cached; reclaim them explicitly when appropriate: +`release()` returns shared ranges to their compatible block so future resources +of the same class and memory type can reuse them. Dedicated allocations are +freed immediately. Empty shared blocks remain cached; reclaim them explicitly +when appropriate: ```v println('released ${allocator.trim_empty_blocks()} empty memory blocks') @@ -170,6 +176,38 @@ println('released ${allocator.trim_empty_blocks()} empty memory blocks') Call `allocator.destroy()` only after destroying every buffer and image backed by it. This frees any allocations that were not individually released. +## Image allocation + +`create_image()` and `create_image_with_options()` keep every image in a +dedicated block. This remains the simplest default for low image counts and is +source-compatible with earlier releases. + +When an application creates many ordinary images, opt into class-safe sharing: + +```v +mut image := vk.Image(unsafe { nil }) +mut image_allocation := vma.AllocationInfo{} +result := allocator.create_suballocated_image_with_options(&image_info, + vma.AllocationOptions{ + usage: .gpu_only + }, &image, mut image_allocation) +if result != .success { + return error('could not create image: ${result}') +} +``` + +Linear images share only with linear images; optimal images share only with +optimal images; buffers share only with buffers. If Vulkan 1.1's +[`VkMemoryDedicatedRequirements`](https://docs.vulkan.org/refpages/latest/refpages/source/VkMemoryDedicatedRequirements.html) +query reports that dedicated memory is required or preferred, the allocator +honors it transparently. Vulkan 1.0 also uses the dedicated fallback because it +cannot make the core requirements query. + +Sparse images use sparse binding instead of `vkBindImageMemory`; disjoint images +bind planes separately; DRM format modifier images require additional layout +handling. The explicit suballocation APIs return `error_feature_not_present` +for these specialized paths rather than treating them as ordinary images. + ## Statistics ```v @@ -196,6 +234,17 @@ type_stats := allocator.stats_for_memory_type(allocation.mem_type) println('type ${allocation.mem_type}: free=${type_stats.free}, largest=${type_stats.largest_free_range}') ``` +Memory-type totals can still span incompatible resource classes. For the exact +set of blocks a similar request could reuse, include the class: + +```v +compatible := allocator.stats_for_memory_type_and_class(allocation.mem_type, + allocation.resource_class) +println('compatible free=${compatible.free}, largest=${compatible.largest_free_range}') +``` + +`stats_for_resource_class()` aggregates one class across memory types. + If total compatible free space is large enough but its largest range is too small, the existing blocks are externally fragmented. If an empty block is reported, `trim_empty_blocks()` can return it to Vulkan before retrying another @@ -228,7 +277,7 @@ appropriate to their application. ```v for event in allocator.recent_events() { - println('#${event.sequence} ${event.kind}: size=${event.requested_size}, type=${event.memory_type}, result=${event.result}') + println('#${event.sequence} ${event.kind}: size=${event.requested_size}, type=${event.memory_type}, class=${event.resource_class}, result=${event.result}') } ``` @@ -331,7 +380,8 @@ non-coherent memory correctly. the renderer, which normally already serializes Vulkan device-memory calls. - Concurrently mapped allocations in one shared block reuse a single underlying Vulkan mapping. Each successful `map()` must have a matching `unmap()`. -- The allocator does not relocate live resources or suballocate images. +- The allocator does not relocate live resources. Image sharing is opt-in and + excludes sparse, disjoint, and DRM-format-modifier images. - Heap budgets guide selection but cannot enforce a process-wide or system-wide limit because other allocators can change process usage and external system activity can change the budget concurrently. @@ -369,5 +419,14 @@ release/refill cycles, bounded-trace wraparound, and final coalescing checks: v run examples/stress ``` -CI executes both examples against Mesa's CPU Vulkan implementation, so it does -not depend on access to a hardware GPU. +The [image suballocation example](examples/image_suballocation/main.v) verifies +that two optimal images share one real block while a linear image and buffer use +separate class-compatible blocks: + +```sh +v run examples/image_suballocation +``` + +CI executes all examples against Mesa's CPU Vulkan implementation with the +Khronos validation layer enabled, so it does not depend on a hardware GPU and +binding-rule regressions remain visible. diff --git a/block_pool.v b/block_pool.v index 59362bc..b8b3bea 100644 --- a/block_pool.v +++ b/block_pool.v @@ -8,17 +8,19 @@ struct BlockReservation { block_id u64 allocation memory.RangeAllocation pub: - memory_type u32 - offset u64 - size u64 + memory_type u32 + resource_class ResourceClass + offset u64 + size u64 } struct MemoryBlock { - id u64 - memory_type u32 - capacity u64 - dedicated bool - ranges &memory.RangeAllocator @[required] + id u64 + memory_type u32 + resource_class ResourceClass + capacity u64 + dedicated bool + ranges &memory.RangeAllocator @[required] } struct BlockPoolStats { @@ -32,9 +34,9 @@ struct BlockPoolStats { empty_block_count int } -// MemoryBlockPool plans suballocations without owning Vulkan handles. Keeping -// this layer independent makes allocation policy deterministic and testable on -// systems without a Vulkan device. +// MemoryBlockPool plans memory-type- and resource-class-compatible +// suballocations without owning Vulkan handles. Keeping this layer independent +// makes allocation policy deterministic and testable without a Vulkan device. struct MemoryBlockPool { default_block_size u64 max_blocks int @@ -96,6 +98,15 @@ fn (pool &MemoryBlockPool) block_memory_type(block_id u64) ?u32 { return none } +fn (pool &MemoryBlockPool) block_resource_class(block_id u64) ?ResourceClass { + for block in pool.blocks { + if block.id == block_id { + return block.resource_class + } + } + return none +} + fn (pool &MemoryBlockPool) heap_stats(props &vk.PhysicalDeviceMemoryProperties, heap_index u32) (u64, u64) { mut committed := u64(0) mut used := u64(0) @@ -122,14 +133,22 @@ fn (pool &MemoryBlockPool) recommended_block_size(requested_size u64) !u64 { } fn (mut pool MemoryBlockPool) add_block(memory_type u32, capacity u64) !u64 { - return pool.add_block_with_policy(memory_type, capacity, false) + return pool.add_block_for_class(memory_type, .buffer, capacity) } fn (mut pool MemoryBlockPool) add_dedicated_block(memory_type u32, capacity u64) !u64 { - return pool.add_block_with_policy(memory_type, capacity, true) + return pool.add_dedicated_block_for_class(memory_type, .unknown, capacity) } -fn (mut pool MemoryBlockPool) add_block_with_policy(memory_type u32, capacity u64, dedicated bool) !u64 { +fn (mut pool MemoryBlockPool) add_block_for_class(memory_type u32, resource_class ResourceClass, capacity u64) !u64 { + return pool.add_block_with_policy(memory_type, resource_class, capacity, false) +} + +fn (mut pool MemoryBlockPool) add_dedicated_block_for_class(memory_type u32, resource_class ResourceClass, capacity u64) !u64 { + return pool.add_block_with_policy(memory_type, resource_class, capacity, true) +} + +fn (mut pool MemoryBlockPool) add_block_with_policy(memory_type u32, resource_class ResourceClass, capacity u64, dedicated bool) !u64 { if capacity == 0 { return error('memory block capacity must be greater than zero') } @@ -138,11 +157,12 @@ fn (mut pool MemoryBlockPool) add_block_with_policy(memory_type u32, capacity u6 } id := pool.next_block_id() pool.blocks << MemoryBlock{ - id: id - memory_type: memory_type - capacity: capacity - dedicated: dedicated - ranges: memory.new_range_allocator(capacity) + id: id + memory_type: memory_type + resource_class: resource_class + capacity: capacity + dedicated: dedicated + ranges: memory.new_range_allocator(capacity) } return id } @@ -151,6 +171,10 @@ fn (mut pool MemoryBlockPool) add_block_with_policy(memory_type u32, capacity u6 // block, allowing the Vulkan layer to allocate a real VkDeviceMemory object // before registering its matching planning block. fn (mut pool MemoryBlockPool) reserve(memory_type u32, size u64, alignment u64) !BlockReservation { + return pool.reserve_for_class(memory_type, .buffer, size, alignment) +} + +fn (mut pool MemoryBlockPool) reserve_for_class(memory_type u32, resource_class ResourceClass, size u64, alignment u64) !BlockReservation { if size == 0 { return error('allocation size must be greater than zero') } @@ -158,17 +182,19 @@ fn (mut pool MemoryBlockPool) reserve(memory_type u32, size u64, alignment u64) return error('allocation alignment must be greater than zero') } for mut block in pool.blocks { - if block.memory_type != memory_type || block.dedicated { + if block.memory_type != memory_type || block.resource_class != resource_class + || block.dedicated { continue } if allocation := block.ranges.allocate(size, alignment) { return BlockReservation{ - owner: pool - block_id: block.id - allocation: allocation - memory_type: memory_type - offset: allocation.offset - size: allocation.size + owner: pool + block_id: block.id + allocation: allocation + memory_type: memory_type + resource_class: resource_class + offset: allocation.offset + size: allocation.size } } } @@ -188,12 +214,13 @@ fn (mut pool MemoryBlockPool) reserve_from_block(block_id u64, size u64, alignme } allocation := block.ranges.allocate(size, alignment)! return BlockReservation{ - owner: pool - block_id: block.id - allocation: allocation - memory_type: block.memory_type - offset: allocation.offset - size: allocation.size + owner: pool + block_id: block.id + allocation: allocation + memory_type: block.memory_type + resource_class: block.resource_class + offset: allocation.offset + size: allocation.size } } return error('memory block does not exist') @@ -206,6 +233,7 @@ fn (pool &MemoryBlockPool) contains(reservation BlockReservation) bool { for block in pool.blocks { if block.id == reservation.block_id { return block.memory_type == reservation.memory_type + && block.resource_class == reservation.resource_class && reservation.offset == reservation.allocation.offset && reservation.size == reservation.allocation.size && block.ranges.contains(reservation.allocation) @@ -219,7 +247,8 @@ fn (mut pool MemoryBlockPool) release(reservation BlockReservation) bool { return false } for mut block in pool.blocks { - if block.id != reservation.block_id || block.memory_type != reservation.memory_type { + if block.id != reservation.block_id || block.memory_type != reservation.memory_type + || block.resource_class != reservation.resource_class { continue } if reservation.offset != reservation.allocation.offset @@ -248,14 +277,22 @@ fn (mut pool MemoryBlockPool) remove_empty_block(block_id u64) bool { } fn (pool &MemoryBlockPool) stats() BlockPoolStats { - return pool.collect_stats(0, false) + return pool.collect_stats(0, .unknown, false, false) } fn (pool &MemoryBlockPool) stats_for_memory_type(memory_type u32) BlockPoolStats { - return pool.collect_stats(memory_type, true) + return pool.collect_stats(memory_type, .unknown, true, false) +} + +fn (pool &MemoryBlockPool) stats_for_resource_class(resource_class ResourceClass) BlockPoolStats { + return pool.collect_stats(0, resource_class, false, true) } -fn (pool &MemoryBlockPool) collect_stats(memory_type u32, filter_by_memory_type bool) BlockPoolStats { +fn (pool &MemoryBlockPool) stats_for_memory_type_and_class(memory_type u32, resource_class ResourceClass) BlockPoolStats { + return pool.collect_stats(memory_type, resource_class, true, true) +} + +fn (pool &MemoryBlockPool) collect_stats(memory_type u32, resource_class ResourceClass, filter_by_memory_type bool, filter_by_resource_class bool) BlockPoolStats { mut block_count := 0 mut allocation_count := 0 mut committed := u64(0) @@ -267,6 +304,9 @@ fn (pool &MemoryBlockPool) collect_stats(memory_type u32, filter_by_memory_type if filter_by_memory_type && block.memory_type != memory_type { continue } + if filter_by_resource_class && block.resource_class != resource_class { + continue + } block_count++ range_stats := block.ranges.stats() allocation_count += range_stats.allocation_count diff --git a/block_pool_stress_test.v b/block_pool_stress_test.v index a4376f4..ab95cae 100644 --- a/block_pool_stress_test.v +++ b/block_pool_stress_test.v @@ -2,7 +2,7 @@ module vkmemalloc fn assert_block_pool_invariants(pool &MemoryBlockPool, active []BlockReservation, committed u64) { stats := pool.stats() - assert stats.block_count == 4 + assert stats.block_count == 6 assert stats.allocation_count == active.len assert stats.committed == committed assert stats.used + stats.free == committed @@ -21,21 +21,36 @@ fn assert_block_pool_invariants(pool &MemoryBlockPool, active []BlockReservation assert stats.used == used type_0 := pool.stats_for_memory_type(0) type_1 := pool.stats_for_memory_type(1) - assert type_0.block_count == 2 - assert type_1.block_count == 2 + assert type_0.block_count == 3 + assert type_1.block_count == 3 assert type_0.allocation_count + type_1.allocation_count == active.len assert type_0.committed + type_1.committed == committed assert type_0.used + type_1.used == used + mut class_allocations := 0 + mut class_committed := u64(0) + mut class_used := u64(0) + for resource_class in [ResourceClass.buffer, .linear_image, .optimal_image] { + class_stats := pool.stats_for_resource_class(resource_class) + assert class_stats.block_count == 2 + class_allocations += class_stats.allocation_count + class_committed += class_stats.committed + class_used += class_stats.used + } + assert class_allocations == active.len + assert class_committed == committed + assert class_used == used } fn test_block_pool_sustained_mixed_workload() { block_size := u64(4096) - mut pool := new_memory_block_pool(block_size, 4) or { panic(err) } - for _ in 0 .. 2 { - _ = pool.add_block(0, block_size) or { panic(err) } - _ = pool.add_block(1, block_size) or { panic(err) } + mut pool := new_memory_block_pool(block_size, 6) or { panic(err) } + resource_classes := [ResourceClass.buffer, .linear_image, .optimal_image] + for memory_type in u32(0) .. 2 { + for resource_class in resource_classes { + _ = pool.add_block_for_class(memory_type, resource_class, block_size) or { panic(err) } + } } - committed := block_size * 4 + committed := block_size * 6 alignments := [u64(1), 2, 3, 4, 8, 16, 31, 64, 128, 256] mut active := []BlockReservation{} mut state := u32(0xa110ca7e) @@ -49,10 +64,12 @@ fn test_block_pool_sustained_mixed_workload() { active.delete(index) } else { memory_type := (state >> 4) & 1 + resource_class := resource_classes[int((state >> 6) % u32(resource_classes.len))] size := u64(1 + (state >> 12) % 257) alignment := alignments[int((state >> 24) % u32(alignments.len))] - if reservation := pool.reserve(memory_type, size, alignment) { + if reservation := pool.reserve_for_class(memory_type, resource_class, size, alignment) { assert reservation.offset % alignment == 0 + assert reservation.resource_class == resource_class active << reservation successful_allocations++ } else if active.len > 0 { @@ -73,7 +90,7 @@ fn test_block_pool_sustained_mixed_workload() { assert_block_pool_invariants(pool, [], committed) final_stats := pool.stats() assert final_stats.used == 0 - assert final_stats.free_range_count == 4 + assert final_stats.free_range_count == 6 assert final_stats.largest_free_range == block_size - assert final_stats.empty_block_count == 4 + assert final_stats.empty_block_count == 6 } diff --git a/block_pool_test.v b/block_pool_test.v index 079a5ba..06e560e 100644 --- a/block_pool_test.v +++ b/block_pool_test.v @@ -92,6 +92,52 @@ fn test_block_pool_never_reuses_dedicated_blocks() { assert pool.contains(dedicated) } +fn test_block_pool_never_mixes_resource_classes() { + mut pool := new_memory_block_pool(64, 3) or { panic(err) } + buffer_block := pool.add_block_for_class(2, .buffer, 64) or { panic(err) } + image_block := pool.add_block_for_class(2, .optimal_image, 64) or { panic(err) } + + buffer := pool.reserve_for_class(2, .buffer, 8, 8) or { panic(err) } + first_image := pool.reserve_for_class(2, .optimal_image, 16, 16) or { panic(err) } + second_image := pool.reserve_for_class(2, .optimal_image, 8, 8) or { panic(err) } + + assert buffer.block_id == buffer_block + assert first_image.block_id == image_block + assert second_image.block_id == image_block + assert first_image.offset == 0 + assert second_image.offset == 16 + assert pool.block_resource_class(buffer_block) or { panic('buffer block should exist') } == .buffer + assert pool.block_resource_class(image_block) or { panic('image block should exist') } == .optimal_image + if _ := pool.reserve_for_class(2, .linear_image, 8, 1) { + assert false, 'linear images must not reuse buffer or optimal-image blocks' + } else { + assert err.msg().contains('compatible memory block') + } + + buffer_stats := pool.stats_for_memory_type_and_class(2, .buffer) + assert buffer_stats.block_count == 1 + assert buffer_stats.allocation_count == 1 + optimal_stats := pool.stats_for_resource_class(.optimal_image) + assert optimal_stats.block_count == 1 + assert optimal_stats.allocation_count == 2 + assert pool.stats().block_count == 2 + assert pool.stats().allocation_count == 3 +} + +fn test_block_pool_rejects_a_forged_resource_class() { + mut pool := new_memory_block_pool(32, 1) or { panic(err) } + _ = pool.add_block_for_class(0, .optimal_image, 32) or { panic(err) } + reservation := pool.reserve_for_class(0, .optimal_image, 8, 1) or { panic(err) } + forged := BlockReservation{ + ...reservation + resource_class: .buffer + } + assert !pool.contains(forged) + assert !pool.release(forged) + assert pool.contains(reservation) + assert pool.release(reservation) +} + fn test_block_pool_release_coalesces_and_allows_empty_removal() { mut pool := new_memory_block_pool(32, 2) or { panic(err) } block_id := pool.add_block(3, 32) or { panic(err) } diff --git a/diagnostics.v b/diagnostics.v index 6447a91..e769138 100644 --- a/diagnostics.v +++ b/diagnostics.v @@ -20,6 +20,7 @@ pub: result vk.Result memory_type u32 = max_u32 heap_index u32 = max_u32 + resource_class ResourceClass requested_size u64 allocation_offset u64 block_size u64 @@ -164,6 +165,7 @@ fn (mut a Allocator) note_allocation_success(alloc_info AllocationInfo, dedicate result: .success memory_type: alloc_info.mem_type heap_index: alloc_info.heap_index + resource_class: alloc_info.resource_class requested_size: alloc_info.size allocation_offset: alloc_info.offset block_size: alloc_info.block_size @@ -172,13 +174,14 @@ fn (mut a Allocator) note_allocation_success(alloc_info AllocationInfo, dedicate }) } -fn (mut a Allocator) note_allocation_failure(result vk.Result, requested_size u64, memory_type u32, heap_index u32, dedicated bool) { +fn (mut a Allocator) note_allocation_failure(result vk.Result, requested_size u64, memory_type u32, heap_index u32, dedicated bool, resource_class ResourceClass) { a.counters_.allocation_failures++ a.record_event(AllocatorEvent{ kind: .allocation_failed result: result memory_type: memory_type heap_index: heap_index + resource_class: resource_class requested_size: requested_size dedicated: dedicated }) @@ -200,6 +203,7 @@ fn (mut a Allocator) note_allocation_release(alloc_info AllocationInfo, dedicate result: .success memory_type: alloc_info.mem_type heap_index: alloc_info.heap_index + resource_class: alloc_info.resource_class requested_size: alloc_info.size allocation_offset: alloc_info.offset block_size: alloc_info.block_size @@ -208,18 +212,19 @@ fn (mut a Allocator) note_allocation_release(alloc_info AllocationInfo, dedicate }) } -fn (mut a Allocator) note_block_trimmed(memory_type u32, heap_index u32, block_size u64) { +fn (mut a Allocator) note_block_trimmed(memory_type u32, heap_index u32, resource_class ResourceClass, block_size u64) { a.counters_.block_frees++ a.counters_.trimmed_blocks++ if block_size <= a.diagnostic_committed { a.diagnostic_committed -= block_size } a.record_event(AllocatorEvent{ - kind: .block_trimmed - result: .success - memory_type: memory_type - heap_index: heap_index - block_size: block_size + kind: .block_trimmed + result: .success + memory_type: memory_type + heap_index: heap_index + resource_class: resource_class + block_size: block_size }) } diff --git a/diagnostics_test.v b/diagnostics_test.v index 1122a77..6181138 100644 --- a/diagnostics_test.v +++ b/diagnostics_test.v @@ -28,7 +28,7 @@ fn test_allocator_diagnostics_track_lifecycle_and_bound_the_trace() { }] mut allocation := AllocationInfo{} result := allocator.allocate_from_choices(mut requirements, choices, unsafe { nil }, false, - .ignore, mut allocation) + .buffer, .ignore, mut allocation) assert result == .success assert !allocation.created_block assert allocator.release(mut allocation) @@ -39,16 +39,16 @@ fn test_allocator_diagnostics_track_lifecycle_and_bound_the_trace() { memoryTypeBits: 1 } mut unavailable_allocation := AllocationInfo{} - assert allocator.allocate_from_choices(mut unavailable, [], unsafe { nil }, false, .ignore, mut - unavailable_allocation) == .error_feature_not_present + assert allocator.allocate_from_choices(mut unavailable, [], unsafe { nil }, false, .buffer, + .ignore, mut unavailable_allocation) == .error_feature_not_present mut invalid := vk.MemoryRequirements{ size: 4 memoryTypeBits: 1 } mut invalid_allocation := AllocationInfo{} - assert allocator.allocate_from_choices(mut invalid, choices, unsafe { nil }, false, .ignore, mut - invalid_allocation) == .error_initialization_failed + assert allocator.allocate_from_choices(mut invalid, choices, unsafe { nil }, false, .buffer, + .ignore, mut invalid_allocation) == .error_initialization_failed diagnostics := allocator.diagnostics() assert diagnostics.current.block_count == 1 @@ -74,6 +74,7 @@ fn test_allocator_diagnostics_track_lifecycle_and_bound_the_trace() { assert events.len == 3 assert events[0].sequence == 2 assert events[0].kind == .allocation_released + assert events[0].resource_class == .buffer assert events[1].sequence == 3 assert events[1].kind == .allocation_failed assert events[1].result == .error_feature_not_present @@ -96,8 +97,8 @@ fn test_allocator_diagnostics_track_lifecycle_and_bound_the_trace() { memoryTypeBits: 1 } mut after_reset_allocation := AllocationInfo{} - assert allocator.allocate_from_choices(mut after_reset, [], unsafe { nil }, false, .ignore, mut - after_reset_allocation) == .error_feature_not_present + assert allocator.allocate_from_choices(mut after_reset, [], unsafe { nil }, false, .buffer, + .ignore, mut after_reset_allocation) == .error_feature_not_present after_reset_events := allocator.recent_events() assert after_reset_events.len == 1 assert after_reset_events[0].sequence == 5 diff --git a/examples/image_suballocation/main.v b/examples/image_suballocation/main.v new file mode 100644 index 0000000..15cfe8b --- /dev/null +++ b/examples/image_suballocation/main.v @@ -0,0 +1,218 @@ +module main + +import antono2.vkmemalloc as vma +import antono2.vulkan as vk + +fn require_success(result vk.Result, operation string) ! { + if result != .success { + return error('${operation} failed: ${result}') + } +} + +fn first_physical_device(instance vk.Instance) !vk.PhysicalDevice { + mut count := u32(0) + require_success(vk.enumerate_physical_devices(instance, &count, unsafe { nil }), + 'enumerate physical device count')! + if count == 0 { + return error('no Vulkan physical device is available') + } + mut devices := unsafe { []vk.PhysicalDevice{len: int(count)} } + require_success(vk.enumerate_physical_devices(instance, &count, devices.data), + 'enumerate physical devices')! + return devices[0] +} + +fn run() ! { + require_success(vk.initialize_loader(), 'initialize Vulkan loader')! + application_info := vk.ApplicationInfo{ + pApplicationName: c'vkmemalloc image suballocation example' + applicationVersion: 1 + pEngineName: c'none' + apiVersion: vk.api_version_1_1 + } + instance_info := vk.InstanceCreateInfo{ + pApplicationInfo: &application_info + } + mut instance := vk.Instance(unsafe { nil }) + require_success(vk.create_instance(&instance_info, unsafe { nil }, &instance), + 'create Vulkan instance')! + defer { + vk.destroy_instance(instance, unsafe { nil }) + } + vk.load_instance_commands(instance) + + physical_device := first_physical_device(instance)! + mut priority := f32(1) + queue_info := vk.DeviceQueueCreateInfo{ + queueFamilyIndex: 0 + queueCount: 1 + pQueuePriorities: &priority + } + device_info := vk.DeviceCreateInfo{ + queueCreateInfoCount: 1 + pQueueCreateInfos: &queue_info + } + mut device := vk.Device(unsafe { nil }) + require_success(vk.create_device(physical_device, &device_info, unsafe { nil }, &device), + 'create Vulkan device')! + defer { + vk.destroy_device(device, unsafe { nil }) + } + vk.load_device_commands(device) + + mut allocator := vma.new(vma.AllocatorCreateInfo{ + physical_device: physical_device + device: device + preferred_block_size: 128 * 1024 + }) + defer { + allocator.destroy() + } + + image_info := vk.ImageCreateInfo{ + imageType: ._2d + format: .r8g8b8a8_unorm + extent: vk.Extent3D{ + width: 64 + height: 64 + depth: 1 + } + mipLevels: 1 + arrayLayers: 1 + samples: ._1 + tiling: .optimal + usage: u32(vk.ImageUsageFlagBits.transfer_src) | u32(vk.ImageUsageFlagBits.transfer_dst) + sharingMode: .exclusive + initialLayout: .undefined + } + mut first_image := vk.Image(unsafe { nil }) + mut first_allocation := vma.AllocationInfo{} + require_success(allocator.create_suballocated_image_with_options(&image_info, vma.AllocationOptions{ + usage: .gpu_only + }, &first_image, mut first_allocation), 'create first optimal image')! + defer { + if !isnil(first_image) { + vk.destroy_image(device, first_image, unsafe { nil }) + } + if !isnil(first_allocation.memory) { + _ = allocator.release(mut first_allocation) + } + } + + mut second_image := vk.Image(unsafe { nil }) + mut second_allocation := vma.AllocationInfo{} + require_success(allocator.create_suballocated_image_with_options(&image_info, vma.AllocationOptions{ + usage: .gpu_only + }, &second_image, mut second_allocation), 'create second optimal image')! + defer { + if !isnil(second_image) { + vk.destroy_image(device, second_image, unsafe { nil }) + } + if !isnil(second_allocation.memory) { + _ = allocator.release(mut second_allocation) + } + } + + if first_allocation.memory != second_allocation.memory { + return error('the driver requested dedicated memory; this smoke test requires shareable images') + } + assert first_allocation.resource_class == .optimal_image + assert second_allocation.resource_class == .optimal_image + assert first_allocation.offset != second_allocation.offset + mut first_requirements := vk.MemoryRequirements{} + mut second_requirements := vk.MemoryRequirements{} + vk.get_image_memory_requirements(device, first_image, mut first_requirements) + vk.get_image_memory_requirements(device, second_image, mut second_requirements) + assert first_allocation.offset % first_requirements.alignment == 0 + assert second_allocation.offset % second_requirements.alignment == 0 + assert second_allocation.offset >= first_allocation.offset + first_allocation.size + + linear_image_info := vk.ImageCreateInfo{ + imageType: ._2d + format: .r8g8b8a8_unorm + extent: vk.Extent3D{ + width: 64 + height: 64 + depth: 1 + } + mipLevels: 1 + arrayLayers: 1 + samples: ._1 + tiling: .linear + usage: u32(vk.ImageUsageFlagBits.transfer_src) | u32(vk.ImageUsageFlagBits.transfer_dst) + sharingMode: .exclusive + initialLayout: .undefined + } + mut linear_image := vk.Image(unsafe { nil }) + mut linear_allocation := vma.AllocationInfo{} + require_success(allocator.create_suballocated_image_with_options(&linear_image_info, vma.AllocationOptions{ + usage: .gpu_only + }, &linear_image, mut linear_allocation), 'create linear image')! + defer { + if !isnil(linear_image) { + vk.destroy_image(device, linear_image, unsafe { nil }) + } + if !isnil(linear_allocation.memory) { + _ = allocator.release(mut linear_allocation) + } + } + assert linear_allocation.resource_class == .linear_image + assert linear_allocation.memory != first_allocation.memory + + buffer_info := vk.BufferCreateInfo{ + size: 4096 + usage: u32(vk.BufferUsageFlagBits.transfer_dst) + sharingMode: .exclusive + } + mut buffer := vk.Buffer(unsafe { nil }) + mut buffer_allocation := vma.AllocationInfo{} + require_success(allocator.create_buffer_with_options(&buffer_info, vma.AllocationOptions{ + usage: .gpu_only + }, &buffer, mut buffer_allocation), 'create device buffer')! + defer { + if !isnil(buffer) { + vk.destroy_buffer(device, buffer, unsafe { nil }) + } + if !isnil(buffer_allocation.memory) { + _ = allocator.release(mut buffer_allocation) + } + } + assert buffer_allocation.resource_class == .buffer + assert buffer_allocation.memory != first_allocation.memory + + image_stats := allocator.stats_for_memory_type_and_class(first_allocation.mem_type, + .optimal_image) + assert image_stats.block_count == 1 + assert image_stats.allocation_count == 2 + assert image_stats.committed >= image_stats.used + buffer_stats := allocator.stats_for_memory_type_and_class(buffer_allocation.mem_type, .buffer) + assert buffer_stats.block_count == 1 + assert buffer_stats.allocation_count == 1 + linear_stats := allocator.stats_for_memory_type_and_class(linear_allocation.mem_type, + .linear_image) + assert linear_stats.block_count == 1 + assert linear_stats.allocation_count == 1 + assert allocator.stats().block_count == 3 + assert allocator.stats().allocation_count == 4 + println('two optimal images share one class-safe block; the linear image and buffer use separate blocks') + println('images: committed=${image_stats.committed}, used=${image_stats.used}, offsets=${first_allocation.offset}/${second_allocation.offset}') + + vk.destroy_buffer(device, buffer, unsafe { nil }) + buffer = vk.Buffer(unsafe { nil }) + assert allocator.release(mut buffer_allocation) + vk.destroy_image(device, linear_image, unsafe { nil }) + linear_image = vk.Image(unsafe { nil }) + assert allocator.release(mut linear_allocation) + vk.destroy_image(device, second_image, unsafe { nil }) + second_image = vk.Image(unsafe { nil }) + assert allocator.release(mut second_allocation) + vk.destroy_image(device, first_image, unsafe { nil }) + first_image = vk.Image(unsafe { nil }) + assert allocator.release(mut first_allocation) + assert allocator.trim_empty_blocks() == 3 + assert allocator.stats() == vma.AllocatorStats{} +} + +fn main() { + run() or { panic(err) } +} diff --git a/memory_policy_test.v b/memory_policy_test.v index 3e199bf..8a8fef2 100644 --- a/memory_policy_test.v +++ b/memory_policy_test.v @@ -166,7 +166,7 @@ fn test_require_within_can_reuse_an_over_budget_buffer_block() { } mut allocation := AllocationInfo{} result := allocator.allocate_from_choices(mut requirements, choices, unsafe { nil }, false, - options.budget_policy, mut allocation) + .buffer, options.budget_policy, mut allocation) assert result == .success assert allocation.memory == voidptr(policy_test_memory(1)) assert allocation.block_size == 256 diff --git a/v.mod b/v.mod index fecfbb4..241503c 100644 --- a/v.mod +++ b/v.mod @@ -2,7 +2,7 @@ Module { name: 'antono2.vkmemalloc' author: 'Anton Oreskin' description: 'Policy-driven Vulkan memory selection, suballocation, mapping, and diagnostics' - version: '2.5.0' + version: '2.6.0' license: 'MIT' repo_url: 'https://github.com/antono2/vulkan_memory_allocator' tags: ['V','vulkan','allocator'] diff --git a/vulkan_memory_allocator.v b/vulkan_memory_allocator.v index 39ba398..8d95f8a 100644 --- a/vulkan_memory_allocator.v +++ b/vulkan_memory_allocator.v @@ -113,6 +113,17 @@ pub enum MemType { first_available } +// ResourceClass identifies resources that may safely share one VkDeviceMemory +// block. The allocator never mixes these classes, so Vulkan's +// bufferImageGranularity boundary rule cannot be violated inside a block. +pub enum ResourceClass { + // Raw requirements do not identify the resource kind and remain isolated. + unknown + buffer + linear_image + optimal_image +} + pub struct AllocationInfo { pub mut: // The memory type index @@ -121,6 +132,8 @@ pub mut: heap_index u32 // Properties of the selected memory type. property_flags vk.MemoryPropertyFlags + // Compatibility class of the resource occupying the memory block. + resource_class ResourceClass // The memory handle (VkDeviceMemory) memory voidptr = unsafe { nil } // The offset in the memory block @@ -234,10 +247,10 @@ pub fn (mut a Allocator) get_memory_type(type_bits_param u32, mem_props vk.Memor // the resource class needed for safe Vulkan granularity decisions. Prefer // create_buffer() to enable compatible buffer suballocation. pub fn (mut a Allocator) allocate(mut req vk.MemoryRequirements, type MemType, mut alloc_info AllocationInfo) vk.Result { - return a.allocate_with_policy(mut req, type, unsafe { nil }, true, mut alloc_info) + return a.allocate_with_policy(mut req, type, unsafe { nil }, true, .unknown, mut alloc_info) } -fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type MemType, allocation_pnext voidptr, dedicated bool, mut alloc_info AllocationInfo) vk.Result { +fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type MemType, allocation_pnext voidptr, dedicated bool, resource_class ResourceClass, mut alloc_info AllocationInfo) vk.Result { mut mem_type := vk.MemoryPropertyFlags(0) match type { .staging { @@ -257,14 +270,14 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me // device-local memory after a staging allocation fails is invalid and // previously led to a null mapped pointer and a delayed segfault. eprintln('No compatible Vulkan memory type: type bits 0x${req.memoryTypeBits:08x}, required flags 0x${u32(mem_type):08x}') - return a.allocate_from_choices(mut req, [], allocation_pnext, dedicated, .ignore, mut - alloc_info) + return a.allocate_from_choices(mut req, [], allocation_pnext, dedicated, resource_class, + .ignore, mut alloc_info) } choices := ranked_memory_types(a.props, u32(1) << memory_type, req.size, AllocationOptions{ budget_policy: .ignore }, a.heap_budget_snapshot()) - return a.allocate_from_choices(mut req, choices, allocation_pnext, dedicated, .ignore, mut - alloc_info) + return a.allocate_from_choices(mut req, choices, allocation_pnext, dedicated, resource_class, + .ignore, mut alloc_info) } // allocate_with_options reserves isolated memory using the portable ranked @@ -272,28 +285,28 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me // desired. pub fn (mut a Allocator) allocate_with_options(mut req vk.MemoryRequirements, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { choices := a.rank_memory_types(req.memoryTypeBits, req.size, options) - return a.allocate_from_choices(mut req, choices, unsafe { nil }, true, options.budget_policy, mut - alloc_info) + return a.allocate_from_choices(mut req, choices, unsafe { nil }, true, .unknown, + options.budget_policy, mut alloc_info) } -fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choices []MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { +fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choices []MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, resource_class ResourceClass, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { // `alloc_info` is the caller's output record. Reset and populate that record // directly so callers always receive the actual tracked handle. alloc_info = AllocationInfo{} a.begin_allocation(req.size) if req.size == 0 || req.alignment == 0 { result := vk.Result.error_initialization_failed - a.note_allocation_failure(result, req.size, max_u32, max_u32, dedicated) + a.note_allocation_failure(result, req.size, max_u32, max_u32, dedicated, resource_class) return result } if isnil(a.planner) { result := vk.Result.error_initialization_failed - a.note_allocation_failure(result, req.size, max_u32, max_u32, dedicated) + a.note_allocation_failure(result, req.size, max_u32, max_u32, dedicated, resource_class) return result } if choices.len == 0 { result := vk.Result.error_feature_not_present - a.note_allocation_failure(result, req.size, max_u32, max_u32, dedicated) + a.note_allocation_failure(result, req.size, max_u32, max_u32, dedicated, resource_class) return result } mut last_result := vk.Result.error_out_of_device_memory @@ -305,7 +318,7 @@ fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choice allow_new_block := budget_policy != .require_within || choice.within_budget a.note_memory_type_attempt(choice_index > 0) mut result := a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, - allow_new_block, budget_policy, mut alloc_info) + resource_class, allow_new_block, budget_policy, mut alloc_info) if result == .success { a.note_allocation_success(alloc_info, dedicated) return .success @@ -313,7 +326,8 @@ fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choice last_result = result if result !in [.error_out_of_device_memory, .error_out_of_host_memory, .error_too_many_objects] { - a.note_allocation_failure(result, req.size, choice.index, choice.heap_index, dedicated) + a.note_allocation_failure(result, req.size, choice.index, choice.heap_index, dedicated, + resource_class) return result } mut trimmed := 0 @@ -328,8 +342,8 @@ fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choice if trimmed > 0 { a.counters_.trim_retry_attempts++ a.note_memory_type_attempt(choice_index > 0) - result = a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, true, - budget_policy, mut alloc_info) + result = a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, + resource_class, true, budget_policy, mut alloc_info) if result == .success { a.note_allocation_success(alloc_info, dedicated) return .success @@ -337,17 +351,21 @@ fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choice last_result = result } if result == .error_too_many_objects { - a.note_allocation_failure(result, req.size, choice.index, choice.heap_index, dedicated) + a.note_allocation_failure(result, req.size, choice.index, choice.heap_index, dedicated, + resource_class) return result } } - a.note_allocation_failure(last_result, req.size, last_memory_type, last_heap_index, dedicated) + a.note_allocation_failure(last_result, req.size, last_memory_type, last_heap_index, dedicated, + resource_class) return last_result } -fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, choice MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, allow_new_block bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { +fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, choice MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, resource_class ResourceClass, allow_new_block bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { if !dedicated { - if reservation := a.planner.reserve(choice.index, req.size, req.alignment) { + if reservation := a.planner.reserve_for_class(choice.index, resource_class, req.size, + req.alignment) + { memory := a.memory_for_block(reservation.block_id) or { _ = a.planner.release(reservation) return .error_initialization_failed @@ -384,13 +402,13 @@ fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, cho return result } block_id := if dedicated { - a.planner.add_dedicated_block(choice.index, block_size) or { + a.planner.add_dedicated_block_for_class(choice.index, resource_class, block_size) or { vk.free_memory(a.create_info.device, memory, unsafe { nil }) alloc_info = AllocationInfo{} return .error_too_many_objects } } else { - a.planner.add_block(choice.index, block_size) or { + a.planner.add_block_for_class(choice.index, resource_class, block_size) or { vk.free_memory(a.create_info.device, memory, unsafe { nil }) alloc_info = AllocationInfo{} return .error_too_many_objects @@ -418,6 +436,7 @@ fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, cho fn (a &Allocator) populate_allocation(mut alloc_info AllocationInfo, memory vk.DeviceMemory, reservation BlockReservation) { alloc_info.memory = voidptr(memory) alloc_info.mem_type = reservation.memory_type + alloc_info.resource_class = reservation.resource_class if reservation.memory_type < a.props.memoryTypeCount { memory_type := a.props.memoryTypes[reservation.memory_type] alloc_info.heap_index = memory_type.heapIndex @@ -441,7 +460,7 @@ fn (a &Allocator) owns_allocation(alloc_info AllocationInfo) bool { return false } } - return voidptr(memory) == alloc_info.memory && alloc_info.offset == alloc_info.reservation.offset && alloc_info.size == alloc_info.reservation.size && alloc_info.mem_type == alloc_info.reservation.memory_type && alloc_info.block_size == (a.planner.block_capacity(alloc_info.reservation.block_id) or { + return voidptr(memory) == alloc_info.memory && alloc_info.offset == alloc_info.reservation.offset && alloc_info.size == alloc_info.reservation.size && alloc_info.mem_type == alloc_info.reservation.memory_type && alloc_info.resource_class == alloc_info.reservation.resource_class && alloc_info.block_size == (a.planner.block_capacity(alloc_info.reservation.block_id) or { return false }) } @@ -450,8 +469,8 @@ fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, forc if a.api_version < vk.api_version_1_1 { mut requirements := vk.MemoryRequirements{} vk.get_buffer_memory_requirements(a.create_info.device, buffer, mut requirements) - return a.allocate_with_policy(mut requirements, type, unsafe { nil }, force_dedicated, mut - alloc_info) + return a.allocate_with_policy(mut requirements, type, unsafe { nil }, force_dedicated, + .buffer, mut alloc_info) } mut dedicated_requirements := vk.MemoryDedicatedRequirements{} mut requirements := vk.MemoryRequirements2{ @@ -465,13 +484,13 @@ fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, forc || dedicated_requirements.prefersDedicatedAllocation == vk._true if !dedicated { return a.allocate_with_policy(mut requirements.memoryRequirements, type, unsafe { nil }, - false, mut alloc_info) + false, .buffer, mut alloc_info) } dedicated_info := vk.MemoryDedicatedAllocateInfo{ buffer: buffer } return a.allocate_with_policy(mut requirements.memoryRequirements, type, - voidptr(&dedicated_info), true, mut alloc_info) + voidptr(&dedicated_info), true, .buffer, mut alloc_info) } fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, options AllocationOptions, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { @@ -484,7 +503,7 @@ fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, optio a.rank_buffer_memory_types(requirements.memoryTypeBits, requirements.size, options) } return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, force_dedicated, - options.budget_policy, mut alloc_info) + .buffer, options.budget_policy, mut alloc_info) } mut dedicated_requirements := vk.MemoryDedicatedRequirements{} mut requirements := vk.MemoryRequirements2{ @@ -505,54 +524,167 @@ fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, optio } if !dedicated { return a.allocate_from_choices(mut requirements.memoryRequirements, choices, - unsafe { nil }, false, options.budget_policy, mut alloc_info) + unsafe { nil }, false, .buffer, options.budget_policy, mut alloc_info) } dedicated_info := vk.MemoryDedicatedAllocateInfo{ buffer: buffer } return a.allocate_from_choices(mut requirements.memoryRequirements, choices, - voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) + voidptr(&dedicated_info), true, .buffer, options.budget_policy, mut alloc_info) +} + +fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, resource_class ResourceClass, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { + if a.api_version < vk.api_version_1_1 { + mut requirements := vk.MemoryRequirements{} + vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) + // Vulkan 1.0 cannot query the core dedicated-allocation requirements. + // Keep the explicit suballocation API conservative on these devices. + return a.allocate_with_policy(mut requirements, type, unsafe { nil }, true, resource_class, mut + alloc_info) + } + mut dedicated_requirements := vk.MemoryDedicatedRequirements{} + mut requirements := vk.MemoryRequirements2{ + pNext: &dedicated_requirements + } + info := vk.ImageMemoryRequirementsInfo2{ + image: image + } + vk.get_image_memory_requirements2(a.create_info.device, &info, mut requirements) + dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true + || dedicated_requirements.prefersDedicatedAllocation == vk._true + if !dedicated { + return a.allocate_with_policy(mut requirements.memoryRequirements, type, unsafe { nil }, + false, resource_class, mut alloc_info) + } + dedicated_info := vk.MemoryDedicatedAllocateInfo{ + image: image + } + return a.allocate_with_policy(mut requirements.memoryRequirements, type, + voidptr(&dedicated_info), true, resource_class, mut alloc_info) +} + +fn (mut a Allocator) allocate_image_memory_with_options(image vk.Image, options AllocationOptions, resource_class ResourceClass, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { + if a.api_version < vk.api_version_1_1 { + mut requirements := vk.MemoryRequirements{} + vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) + choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, true, + resource_class, options.budget_policy, mut alloc_info) + } + mut dedicated_requirements := vk.MemoryDedicatedRequirements{} + mut requirements := vk.MemoryRequirements2{ + pNext: &dedicated_requirements + } + info := vk.ImageMemoryRequirementsInfo2{ + image: image + } + vk.get_image_memory_requirements2(a.create_info.device, &info, mut requirements) + dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true + || dedicated_requirements.prefersDedicatedAllocation == vk._true + choices := a.rank_memory_types(requirements.memoryRequirements.memoryTypeBits, + requirements.memoryRequirements.size, options) + if !dedicated { + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, + unsafe { nil }, false, resource_class, options.budget_policy, mut alloc_info) + } + dedicated_info := vk.MemoryDedicatedAllocateInfo{ + image: image + } + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, + voidptr(&dedicated_info), true, resource_class, options.budget_policy, mut alloc_info) } -fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut alloc_info AllocationInfo) vk.Result { - mut requirements := vk.MemoryRequirements{} - if a.api_version >= vk.api_version_1_1 { - mut requirements2 := vk.MemoryRequirements2{} - info := vk.ImageMemoryRequirementsInfo2{ - image: image +fn image_resource_class(tiling vk.ImageTiling) ?ResourceClass { + return match tiling { + .linear { .linear_image } + .optimal { .optimal_image } + else { none } + } +} + +fn image_requires_specialized_binding(image_info &vk.ImageCreateInfo) bool { + sparse_flags := u32(vk.ImageCreateFlagBits.sparse_binding) | u32(vk.ImageCreateFlagBits.sparse_residency) | u32(vk.ImageCreateFlagBits.sparse_aliased) + return u32(image_info.flags) & sparse_flags != 0 + || u32(image_info.flags) & u32(vk.ImageCreateFlagBits.disjoint) != 0 +} + +fn (mut a Allocator) create_image_with_policy(image_info &vk.ImageCreateInfo, type MemType, dedicated bool, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + unsafe { + *image = nil + } + alloc_info = AllocationInfo{} + resource_class := image_resource_class(image_info.tiling) or { + if dedicated { + ResourceClass.unknown + } else { + return .error_feature_not_present } - vk.get_image_memory_requirements2(a.create_info.device, &info, mut requirements2) - requirements = requirements2.memoryRequirements - dedicated_info := vk.MemoryDedicatedAllocateInfo{ - image: image + } + if image_requires_specialized_binding(image_info) { + return .error_feature_not_present + } + mut result := vk.create_image(a.create_info.device, image_info, unsafe { nil }, image) + if result != .success { + return result + } + result = a.allocate_image_memory(*image, type, resource_class, dedicated, mut alloc_info) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil } - return a.allocate_with_policy(mut requirements, type, voidptr(&dedicated_info), true, mut - alloc_info) + return result } - vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) - return a.allocate_with_policy(mut requirements, type, unsafe { nil }, true, mut alloc_info) + result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, + alloc_info.offset) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil + } + _ = a.release(mut alloc_info) + } + return result } -fn (mut a Allocator) allocate_image_memory_with_options(image vk.Image, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { - mut requirements := vk.MemoryRequirements{} - if a.api_version >= vk.api_version_1_1 { - mut requirements2 := vk.MemoryRequirements2{} - info := vk.ImageMemoryRequirementsInfo2{ - image: image +fn (mut a Allocator) create_image_with_options_policy(image_info &vk.ImageCreateInfo, options AllocationOptions, dedicated bool, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + unsafe { + *image = nil + } + alloc_info = AllocationInfo{} + resource_class := image_resource_class(image_info.tiling) or { + if dedicated { + ResourceClass.unknown + } else { + return .error_feature_not_present } - vk.get_image_memory_requirements2(a.create_info.device, &info, mut requirements2) - requirements = requirements2.memoryRequirements - choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) - dedicated_info := vk.MemoryDedicatedAllocateInfo{ - image: image + } + if image_requires_specialized_binding(image_info) { + return .error_feature_not_present + } + mut result := vk.create_image(a.create_info.device, image_info, unsafe { nil }, image) + if result != .success { + return result + } + result = a.allocate_image_memory_with_options(*image, options, resource_class, dedicated, mut + alloc_info) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil } - return a.allocate_from_choices(mut requirements, choices, voidptr(&dedicated_info), true, - options.budget_policy, mut alloc_info) + return result } - vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) - choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) - return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, true, - options.budget_policy, mut alloc_info) + result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, + alloc_info.offset) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil + } + _ = a.release(mut alloc_info) + } + return result } // create_buffer creates a buffer, suballocates compatible memory, and binds it. @@ -642,72 +774,31 @@ fn (mut a Allocator) create_buffer_with_policy(buffer_info &vk.BufferCreateInfo, return vk.Result.success } -// create_image creates an image, suballocates compatible memory, and binds it. -pub fn (mut a Allocator) create_image(p_image_create_info &vk.ImageCreateInfo, type MemType, p_image &vk.Image, mut alloc_info AllocationInfo) vk.Result { - unsafe { - *p_image = nil - } - alloc_info = AllocationInfo{} - mut res := vk.create_image(a.create_info.device, p_image_create_info, unsafe { nil }, p_image) - if res != vk.Result.success { - eprintln('Could not create Vulkan image: ${res}') - return res - } - - res = a.allocate_image_memory(*p_image, type, mut alloc_info) - if res != vk.Result.success { - eprintln('Could not allocate Vulkan image memory: ${res}') - vk.destroy_image(a.create_info.device, *p_image, unsafe { nil }) - unsafe { - *p_image = nil - } - return res - } - - res = vk.bind_image_memory(a.create_info.device, *p_image, alloc_info.memory, alloc_info.offset) - if res != vk.Result.success { - eprintln('Could not bind Vulkan image memory: ${res}') - vk.destroy_image(a.create_info.device, *p_image, unsafe { nil }) - unsafe { - *p_image = nil - } - a.allocator_free(mut alloc_info) - return res - } - return vk.Result.success +// create_image creates an image with isolated dedicated memory and binds it. +// Use create_suballocated_image() to opt into compatible image sharing. +pub fn (mut a Allocator) create_image(image_info &vk.ImageCreateInfo, type MemType, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + return a.create_image_with_policy(image_info, type, true, image, mut alloc_info) } // create_image_with_options creates a dedicated image allocation using the -// ranked usage/property/budget policy. Image suballocation remains deliberately -// conservative because buffer-image granularity and tiling compatibility must -// be tracked together. +// ranked usage/property/budget policy. pub fn (mut a Allocator) create_image_with_options(image_info &vk.ImageCreateInfo, options AllocationOptions, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { - unsafe { - *image = nil - } - alloc_info = AllocationInfo{} - mut result := vk.create_image(a.create_info.device, image_info, unsafe { nil }, image) - if result != .success { - return result - } - result = a.allocate_image_memory_with_options(*image, options, mut alloc_info) - if result != .success { - vk.destroy_image(a.create_info.device, *image, unsafe { nil }) - unsafe { - *image = nil - } - return result - } - result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, - alloc_info.offset) - if result != .success { - vk.destroy_image(a.create_info.device, *image, unsafe { nil }) - unsafe { - *image = nil - } - _ = a.release(mut alloc_info) - } - return result + return a.create_image_with_options_policy(image_info, options, true, image, mut alloc_info) +} + +// create_suballocated_image creates an ordinary linear- or optimal-tiling +// image, shares memory only with images in the same resource class, and binds +// it. A Vulkan 1.1+ driver request for dedicated memory is always honored; +// Vulkan 1.0 conservatively falls back to a dedicated allocation. +pub fn (mut a Allocator) create_suballocated_image(image_info &vk.ImageCreateInfo, type MemType, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + return a.create_image_with_policy(image_info, type, false, image, mut alloc_info) +} + +// create_suballocated_image_with_options is the policy-based counterpart of +// create_suballocated_image(). Sparse, disjoint, and DRM-format-modifier images +// require specialized binding paths and return error_feature_not_present here. +pub fn (mut a Allocator) create_suballocated_image_with_options(image_info &vk.ImageCreateInfo, options AllocationOptions, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + return a.create_image_with_options_policy(image_info, options, false, image, mut alloc_info) } // map maps the allocation's byte range for host access. Compatible allocations @@ -840,6 +931,10 @@ fn (mut a Allocator) trim_empty_blocks_filtered(heap_index u32, filter_by_heap b index++ continue } + resource_class := a.planner.block_resource_class(block_id) or { + index++ + continue + } heap := if memory_type < a.props.memoryTypeCount { a.props.memoryTypes[memory_type].heapIndex } else { @@ -856,7 +951,7 @@ fn (mut a Allocator) trim_empty_blocks_filtered(heap_index u32, filter_by_heap b return removed } vk.free_memory(a.create_info.device, memory, unsafe { nil }) - a.note_block_trimmed(memory_type, heap, block_size) + a.note_block_trimmed(memory_type, heap, resource_class, block_size) removed++ } return removed @@ -915,6 +1010,39 @@ pub fn (a &Allocator) stats_for_memory_type(memory_type u32) AllocatorStats { } } +// stats_for_resource_class returns aggregate commitment and fragmentation for +// one compatibility class across every Vulkan memory type. +pub fn (a &Allocator) stats_for_resource_class(resource_class ResourceClass) AllocatorStats { + if isnil(a.planner) { + return AllocatorStats{} + } + stats := a.planner.stats_for_resource_class(resource_class) + return allocator_stats_from_block_pool(stats) +} + +// stats_for_memory_type_and_class narrows diagnostics to blocks that a future +// request with both the given type and resource class could actually reuse. +pub fn (a &Allocator) stats_for_memory_type_and_class(memory_type u32, resource_class ResourceClass) AllocatorStats { + if isnil(a.planner) { + return AllocatorStats{} + } + stats := a.planner.stats_for_memory_type_and_class(memory_type, resource_class) + return allocator_stats_from_block_pool(stats) +} + +fn allocator_stats_from_block_pool(stats BlockPoolStats) AllocatorStats { + return AllocatorStats{ + block_count: stats.block_count + allocation_count: stats.allocation_count + committed: stats.committed + used: stats.used + free: stats.free + free_range_count: stats.free_range_count + largest_free_range: stats.largest_free_range + empty_block_count: stats.empty_block_count + } +} + // destroy frees every Vulkan memory block owned by the allocator. pub fn (mut a Allocator) destroy() { for i in 0 .. a.pool_size { diff --git a/vulkan_memory_allocator_test.v b/vulkan_memory_allocator_test.v index 1fed4a6..488c728 100644 --- a/vulkan_memory_allocator_test.v +++ b/vulkan_memory_allocator_test.v @@ -62,6 +62,7 @@ fn test_allocator_release_returns_only_the_suballocated_range() { allocator.populate_allocation(mut second, fake_memory(42), second_reservation) assert first.memory == second.memory + assert first.resource_class == .buffer assert first.offset == 0 assert second.offset == 16 before := allocator.stats() @@ -73,6 +74,9 @@ fn test_allocator_release_returns_only_the_suballocated_range() { assert before.largest_free_range == 32 assert before.empty_block_count == 0 assert allocator.stats_for_memory_type(4) == before + assert allocator.stats_for_resource_class(.buffer) == before + assert allocator.stats_for_memory_type_and_class(4, .buffer) == before + assert allocator.stats_for_memory_type_and_class(4, .optimal_image) == AllocatorStats{} assert allocator.stats_for_memory_type(99) == AllocatorStats{} first_released := allocator.release(mut first) @@ -97,6 +101,34 @@ fn test_allocator_release_returns_only_the_suballocated_range() { assert final_stats.empty_block_count == 1 } +fn test_image_resource_classes_and_specialized_paths_are_explicit() { + assert image_resource_class(.linear) or { panic('linear tiling should be supported') } == .linear_image + assert image_resource_class(.optimal) or { panic('optimal tiling should be supported') } == .optimal_image + assert image_resource_class(.drm_format_modifier_ext) == none + + mut allocator := Allocator{} + mut image := vk.Image(unsafe { nil }) + mut allocation := AllocationInfo{} + sparse_info := vk.ImageCreateInfo{ + flags: u32(vk.ImageCreateFlagBits.sparse_binding) + tiling: .optimal + } + assert allocator.create_suballocated_image(&sparse_info, .gpu, &image, mut allocation) == .error_feature_not_present + assert isnil(image) + assert isnil(allocation.memory) + disjoint_info := vk.ImageCreateInfo{ + flags: u32(vk.ImageCreateFlagBits.disjoint) + tiling: .optimal + } + assert allocator.create_suballocated_image(&disjoint_info, .gpu, &image, mut allocation) == .error_feature_not_present + assert isnil(image) + modifier_info := vk.ImageCreateInfo{ + tiling: .drm_format_modifier_ext + } + assert allocator.create_suballocated_image(&modifier_info, .gpu, &image, mut allocation) == .error_feature_not_present + assert isnil(image) +} + fn test_allocator_rejects_forged_public_allocation_fields() { mut planner := new_memory_block_pool(32, 1) or { panic(err) } block_id := planner.add_block(1, 32) or { panic(err) } @@ -108,10 +140,17 @@ fn test_allocator_rejects_forged_public_allocation_fields() { assert remembered mut allocation := AllocationInfo{} allocator.populate_allocation(mut allocation, fake_memory(7), reservation) + original_offset := allocation.offset allocation.offset++ mut mapped := voidptr(unsafe { nil }) assert allocator.map(mut allocation, &mapped) == .error_memory_map_failed assert !allocator.release(mut allocation) - assert allocator.stats().allocation_count == 1 + allocation.offset = original_offset + allocation.resource_class = .optimal_image + assert allocator.map(mut allocation, &mapped) == .error_memory_map_failed + assert !allocator.release(mut allocation) + allocation.resource_class = .buffer + assert allocator.release(mut allocation) + assert allocator.stats().allocation_count == 0 }