diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc7c454..94c3fb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,8 @@ jobs: ref: v1.4.0 path: source/modules/antono2/memory - uses: prantlf/setup-v-action@v4 + with: + version: 0.5.2 - name: Install Vulkan development library run: sudo apt-get update && sudo apt-get install -y libvulkan-dev libvulkan-volk-dev mesa-vulkan-drivers - name: Check formatting @@ -38,7 +40,7 @@ jobs: - name: Vet allocator sources working-directory: source/modules/antono2/vkmemalloc # V 0.5.2's directory-mode vet parser does not accept aliased imports. - run: v vet block_pool.v block_pool_test.v upload_ring.v upload_ring_test.v vulkan_memory_allocator.v vulkan_memory_allocator_test.v + run: v vet block_pool.v block_pool_test.v mapped_memory.v mapped_memory_test.v memory_policy.v memory_policy_test.v upload_ring.v upload_ring_test.v vulkan_memory_allocator.v vulkan_memory_allocator_test.v - name: Run allocator tests working-directory: source/modules/antono2/vkmemalloc run: v run setup.vsh --check @@ -48,3 +50,31 @@ jobs: lavapipe_icd=$(find /usr/share/vulkan/icd.d -name 'lvp_icd*.json' -print -quit) test -n "$lavapipe_icd" VK_ICD_FILENAMES="$lavapipe_icd" v run examples/buffer_suballocation + + sanitizers: + runs-on: ubuntu-24.04 + env: + VMODULES: ${{ github.workspace }}/source/modules + VULKAN_SDK: /usr + ASAN_OPTIONS: detect_leaks=0 + steps: + - uses: actions/checkout@v7 + with: + path: source/modules/antono2/vkmemalloc + - uses: actions/checkout@v7 + with: + repository: antono2/vulkan + path: source/modules/antono2/vulkan + - uses: actions/checkout@v7 + with: + repository: antono2/memory + ref: v1.4.0 + path: source/modules/antono2/memory + - uses: prantlf/setup-v-action@v4 + with: + version: 0.5.2 + - name: Install native compiler and Vulkan development library + run: sudo apt-get update && sudo apt-get install -y clang libvulkan-dev libvulkan-volk-dev + - name: Run CPU policy tests with AddressSanitizer and UndefinedBehaviorSanitizer + working-directory: source/modules/antono2/vkmemalloc + run: v -cc clang -cflags -fsanitize=address,undefined -cflags -fno-omit-frame-pointer -ldflags -fsanitize=address,undefined test . diff --git a/CHANGELOG.md b/CHANGELOG.md index dc56f19..ae39f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file. +## 2.4.0 - 2026-09-12 + +- Add explainable, deterministic memory-type ranking for GPU-only, upload, + readback, and automatic usage, with required, preferred, and avoided flags. +- Add optional `VK_EXT_memory_budget` discovery, refresh, selection policy, and + per-heap diagnostics with a portable allocator-commitment fallback. +- Add policy-based raw, buffer, dedicated-buffer, and image allocation APIs + while preserving the existing `MemType` entry points. +- Retry allocation after trimming empty blocks and fall through to compatible + lower-ranked memory types on host/device out-of-memory results. +- Record the selected heap, property flags, and containing block size in every + allocation. +- Add checked `flush`, `flush_range`, `invalidate`, and `invalidate_range` + helpers aligned to the device's `nonCoherentAtomSize`. +- Add policy-selected upload rings and slice-level flush/invalidate helpers + while retaining the coherent default constructor. +- Document the allocator from a high-level policy/planning/ownership viewpoint + and exercise policy selection, live budgets, and flushing with lavapipe. + ## 2.3.2 - 2026-09-12 - Promote the allocation-policy dependency to the production-hardened diff --git a/README.md b/README.md index f3cc4b4..038244b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,26 @@ Despite the repository name, this is not a binding to AMD's Vulkan Memory Allocator. It is a compact V-native allocator intended to remain understandable enough for examples while avoiding one Vulkan allocation per resource. +## How it fits together + +The allocator has three deliberately separate layers: + +1. **Policy** filters the memory types allowed by Vulkan, applies required + property flags, and ranks the remaining types for GPU-only, upload, or + readback use. The selected `MemoryTypeChoice` explains the heap, flags, + budget state, and score. +2. **Block planning** uses `antono2.memory.RangeAllocator` to place compatible + buffers into larger memory-type-specific blocks. This CPU-only layer is + deterministic and independently tested. +3. **Vulkan ownership** creates, maps, binds, and frees `VkDeviceMemory` while + `AllocationInfo` keeps the selected type, heap, properties, block size, and + private ownership record together. + +Images remain dedicated. That conservative rule avoids hiding the additional +tiling and buffer-image granularity rules that a safe image suballocator would +need to model. Existing `MemType` APIs remain available for short examples; +new applications should normally use `AllocationOptions`. + ## Install ```sh v install antono2.vkmemalloc @@ -60,6 +80,40 @@ The lower-level `allocate()` method also uses an isolated block because raw `VkMemoryRequirements` do not identify whether the caller will bind a buffer or image. Use `create_buffer()` when automatic buffer suballocation is desired. +## Policy-based allocation + +The policy API expresses how the resource will be used and keeps hard +requirements distinct from preferences: + +```v +options := vma.AllocationOptions{ + usage: .upload + // These are optional refinements. Required flags are never dropped. + preferred_flags: vk.MemoryPropertyFlags(vk.MemoryPropertyFlagBits.host_cached) +} +result := allocator.create_buffer_with_options(&buffer_info, options, &buffer, + mut allocation) +``` + +- `.gpu_only` requires device-local memory. +- `.upload` requires host-visible memory and prefers coherent, device-local + types. +- `.readback` requires host-visible memory and prefers cached, coherent types. +- `.automatic` has no implicit hard requirement and prefers device-local + memory. + +`required_flags` is a hard filter. `preferred_flags` improves a candidate's +rank, while `avoided_flags` lowers it without making the type unusable. The +default `.prefer_within` budget policy moves a heap with enough estimated room +ahead of an otherwise better match. `.require_within` filters over-budget +heaps for new Vulkan blocks while still permitting reuse of compatible blocks +that are already committed. `.ignore` ranks without considering room. + +Use `allocator.select_memory_type(...)` when you need to inspect the choice +before creating a resource. Policy allocation tries compatible types in rank +order after reclaiming empty cached blocks on memory pressure. It never relaxes +required flags. + Allocate and bind a buffer, checking the returned Vulkan result: ```v @@ -79,9 +133,18 @@ if allocator.map(mut allocation, &mapped) != .success { return error('could not map buffer memory') } // Copy data to mapped here. +if allocator.flush(allocation) != .success { + return error('could not flush buffer memory') +} allocator.unmap(mut allocation) ``` +For host-coherent memory, `flush()` and `invalidate()` are checked no-ops. For +non-coherent memory they call Vulkan with ranges expanded to +`nonCoherentAtomSize`. The `_range` variants accept allocation-relative offsets +and sizes. Flush after host writes before device access; invalidate only after +device writes have completed and before reading them on the host. + Destroy the Vulkan buffer or image before freeing its memory: ```v @@ -134,6 +197,39 @@ reported, `trim_empty_blocks()` can return it to Vulkan before retrying another memory class. The allocator may still create a new compatible block when its configured block limit and the Vulkan device allow it. +### Heap budgets + +`VK_EXT_memory_budget` exposes driver estimates for current heap usage and the +amount the process can reasonably consume. The current integration uses Vulkan +1.1's properties query. Opt in only after confirming and enabling the device +extension: + +```v +budget_supported := vma.supports_memory_budget(physical_device) +// Add vk.ext_memory_budget_extension_name to VkDeviceCreateInfo when true. +mut allocator := vma.new(vma.AllocatorCreateInfo{ + physical_device: physical_device + device: device + memory_budget_enabled: budget_supported +}) + +_ = allocator.refresh_memory_budget() +for heap in allocator.memory_heaps() { + println('heap ${heap.heap_index}: ${heap.usage}/${heap.budget}') +} +``` + +The [runnable example](examples/buffer_suballocation/main.v) shows the complete +extension-name array and logical-device creation sequence. + +`new()` obtains an initial enabled budget snapshot. Call +`refresh_memory_budget()` periodically (for example, once per frame or every +few seconds); policy selection uses the latest snapshot without adding a driver +query to every allocation. Without the extension, the same APIs fall back to +physical heap sizes and this allocator's own committed blocks. Budgets are +changing estimates, not reservations; Vulkan allocation can still fail and the +returned `vk.Result` remains authoritative. + ## Persistent upload ring `UploadRing` owns one dedicated, persistently mapped, host-coherent staging @@ -149,6 +245,9 @@ slice := uploads.allocate(4096, 256) or { panic(err) } unsafe { copy(&u8(slice.data), source.data, source.len) } +if uploads.flush(slice) != .success { + return error('could not flush upload slice') +} // Record a copy from uploads.buffer at slice.offset, submit it, and keep slice. // After the protecting fence or timeline value has completed: @@ -163,7 +262,13 @@ must not retire a slice until the GPU has finished reading it. Call `uploads.stats()` returns `UploadRingStats`, keeping this module's public API independent of the internal allocation-policy type. -## Memory classes +`new_upload_ring()` deliberately requires host-coherent memory, so its +`flush()` calls are checked no-ops. Advanced callers can use +`new_upload_ring_with_options(..., AllocationOptions{ usage: .upload })` to +permit other host-visible types; flushing each written slice then handles +non-coherent memory correctly. + +## Legacy memory classes - `.staging` requires host-visible and host-coherent memory and may be mapped. - `.gpu` requires device-local memory and normally cannot be mapped. @@ -181,8 +286,10 @@ independent of the internal allocation-policy type. when multiple threads can allocate or free concurrently. - Concurrently mapped allocations in one shared block reuse a single underlying Vulkan mapping. Each successful `map()` must have a matching `unmap()`. -- The allocator does not relocate live resources, enforce heap budgets, choose - between equivalent heaps, or automatically flush non-coherent memory. +- The allocator does not relocate live resources or suballocate images. +- Heap budgets guide selection but cannot enforce a process-wide or system-wide + limit because other allocators can change process usage and external system + activity can change the budget concurrently. - Vulkan objects must not outlive the memory bound to them. All allocation and binding functions return `vk.Result`; callers should handle @@ -196,9 +303,11 @@ The bookkeeping tests do not require a Vulkan-capable GPU: v test . ``` -The runnable example creates two real buffers, verifies that they share a -memory block, maps one range, creates a dedicated image, then exercises a -persistently mapped upload ring through wraparound and FIFO retirement: +The runnable example enables live budgets when available, creates two real +policy-selected upload buffers, verifies that they share a memory block, maps +and flushes them, creates a dedicated GPU-only image, prints heap diagnostics, +then exercises a persistently mapped upload ring through wraparound and FIFO +retirement: ```sh v run examples/buffer_suballocation diff --git a/block_pool.v b/block_pool.v index f6a7597..59362bc 100644 --- a/block_pool.v +++ b/block_pool.v @@ -1,6 +1,7 @@ module vkmemalloc import antono2.memory +import antono2.vulkan as vk struct BlockReservation { owner voidptr @@ -77,6 +78,38 @@ fn (pool &MemoryBlockPool) block_is_dedicated(block_id u64) ?bool { return none } +fn (pool &MemoryBlockPool) block_capacity(block_id u64) ?u64 { + for block in pool.blocks { + if block.id == block_id { + return block.capacity + } + } + return none +} + +fn (pool &MemoryBlockPool) block_memory_type(block_id u64) ?u32 { + for block in pool.blocks { + if block.id == block_id { + return block.memory_type + } + } + return none +} + +fn (pool &MemoryBlockPool) heap_stats(props &vk.PhysicalDeviceMemoryProperties, heap_index u32) (u64, u64) { + mut committed := u64(0) + mut used := u64(0) + for block in pool.blocks { + if block.memory_type >= props.memoryTypeCount + || props.memoryTypes[block.memory_type].heapIndex != heap_index { + continue + } + committed += block.capacity + used += block.ranges.stats().used + } + return committed, used +} + fn (pool &MemoryBlockPool) recommended_block_size(requested_size u64) !u64 { if requested_size == 0 { return error('allocation size must be greater than zero') diff --git a/examples/buffer_suballocation/main.v b/examples/buffer_suballocation/main.v index e0bdb6e..42bc60f 100644 --- a/examples/buffer_suballocation/main.v +++ b/examples/buffer_suballocation/main.v @@ -42,6 +42,11 @@ fn run() ! { vk.load_instance_commands(instance) physical_device := first_physical_device(instance)! + memory_budget_supported := vma.supports_memory_budget(physical_device) + mut device_extensions := []&char{} + if memory_budget_supported { + device_extensions << vk.ext_memory_budget_extension_name + } mut priority := f32(1) queue_info := vk.DeviceQueueCreateInfo{ queueFamilyIndex: 0 @@ -49,8 +54,10 @@ fn run() ! { pQueuePriorities: &priority } device_info := vk.DeviceCreateInfo{ - queueCreateInfoCount: 1 - pQueueCreateInfos: &queue_info + queueCreateInfoCount: 1 + pQueueCreateInfos: &queue_info + enabledExtensionCount: u32(device_extensions.len) + ppEnabledExtensionNames: device_extensions.data } mut device := vk.Device(unsafe { nil }) require_success(vk.create_device(physical_device, &device_info, unsafe { nil }, &device), @@ -61,9 +68,10 @@ fn run() ! { vk.load_device_commands(device) mut allocator := vma.new(vma.AllocatorCreateInfo{ - physical_device: physical_device - device: device - preferred_block_size: 4096 + physical_device: physical_device + device: device + preferred_block_size: 4096 + memory_budget_enabled: memory_budget_supported }) defer { allocator.destroy() @@ -76,8 +84,9 @@ fn run() ! { } mut first_buffer := vk.Buffer(unsafe { nil }) mut first_allocation := vma.AllocationInfo{} - require_success(allocator.create_buffer(&buffer_info, .staging, &first_buffer, mut - first_allocation), 'create first staging buffer')! + require_success(allocator.create_buffer_with_options(&buffer_info, vma.AllocationOptions{ + usage: .upload + }, &first_buffer, mut first_allocation), 'create first upload buffer')! defer { if !isnil(first_buffer) { vk.destroy_buffer(device, first_buffer, unsafe { nil }) @@ -89,8 +98,9 @@ fn run() ! { mut second_buffer := vk.Buffer(unsafe { nil }) mut second_allocation := vma.AllocationInfo{} - require_success(allocator.create_buffer(&buffer_info, .staging, &second_buffer, mut - second_allocation), 'create second staging buffer')! + require_success(allocator.create_buffer_with_options(&buffer_info, vma.AllocationOptions{ + usage: .upload + }, &second_buffer, mut second_allocation), 'create second upload buffer')! defer { if !isnil(second_buffer) { vk.destroy_buffer(device, second_buffer, unsafe { nil }) @@ -118,6 +128,8 @@ fn run() ! { *(&u8(first_mapped)) = 21 *(&u8(second_mapped)) = 42 } + require_success(allocator.flush(first_allocation), 'flush first upload buffer')! + require_success(allocator.flush(second_allocation), 'flush second upload buffer')! assert usize(second_mapped) - usize(first_mapped) == second_allocation.offset - first_allocation.offset allocator.unmap(mut first_allocation) allocator.unmap(mut second_allocation) @@ -141,8 +153,9 @@ fn run() ! { } mut image := vk.Image(unsafe { nil }) mut image_allocation := vma.AllocationInfo{} - require_success(allocator.create_image(&image_info, .gpu, &image, mut image_allocation), - 'create dedicated image')! + require_success(allocator.create_image_with_options(&image_info, vma.AllocationOptions{ + usage: .gpu_only + }, &image, mut image_allocation), 'create dedicated image')! defer { if !isnil(image) { vk.destroy_image(device, image, unsafe { nil }) @@ -155,6 +168,11 @@ fn run() ! { assert image_allocation.offset == 0 assert allocator.stats().block_count == 2 println('optimal image uses an isolated dedicated block') + for heap in allocator.memory_heaps() { + assert heap.budget > 0 + assert heap.allocator_committed >= heap.allocator_used + println('heap ${heap.heap_index}: budget=${heap.budget}, usage=${heap.usage}, reported=${heap.budget_reported}') + } vk.destroy_image(device, image, unsafe { nil }) image = vk.Image(unsafe { nil }) @@ -172,7 +190,9 @@ fn run() ! { assert allocator.trim_empty_blocks() == 1 assert allocator.stats().block_count == 0 - mut uploads := vma.new_upload_ring(mut allocator, 1024)! + mut uploads := vma.new_upload_ring_with_options(mut allocator, 1024, vma.AllocationOptions{ + usage: .upload + })! defer { _ = uploads.destroy() } @@ -182,6 +202,8 @@ fn run() ! { *(&u8(first_upload.data)) = 21 *(&u8(second_upload.data)) = 22 } + require_success(uploads.flush(first_upload), 'flush first upload slice')! + require_success(uploads.flush(second_upload), 'flush second upload slice')! first_retired := uploads.retire(first_upload) assert first_retired wrapped_upload := uploads.allocate(300, 16)! @@ -189,6 +211,7 @@ fn run() ! { unsafe { *(&u8(wrapped_upload.data)) = 23 } + require_success(uploads.flush(wrapped_upload), 'flush wrapped upload slice')! assert !uploads.retire(wrapped_upload) second_retired := uploads.retire(second_upload) assert second_retired diff --git a/mapped_memory.v b/mapped_memory.v new file mode 100644 index 0000000..0693074 --- /dev/null +++ b/mapped_memory.v @@ -0,0 +1,87 @@ +module vkmemalloc + +import antono2.vulkan as vk + +struct NormalizedMappedRange { + offset u64 + size u64 +} + +fn normalize_mapped_range(allocation_offset u64, allocation_size u64, block_size u64, relative_offset u64, size u64, atom_size u64) ?NormalizedMappedRange { + if size == 0 || atom_size == 0 || relative_offset > allocation_size + || size > allocation_size - relative_offset { + return none + } + absolute_offset := allocation_offset + relative_offset + if absolute_offset < allocation_offset || absolute_offset > block_size + || size > block_size - absolute_offset { + return none + } + start := absolute_offset - absolute_offset % atom_size + end := absolute_offset + size + rounded_end := if end % atom_size == 0 { + end + } else if end > max_u64 - (atom_size - end % atom_size) { + return none + } else { + end + atom_size - end % atom_size + } + return NormalizedMappedRange{ + offset: start + size: if rounded_end >= block_size { + vk.whole_size + } else { + rounded_end - start + } + } +} + +fn (a &Allocator) mapped_range(alloc_info AllocationInfo, relative_offset u64, size u64) ?vk.MappedMemoryRange { + if !a.owns_allocation(alloc_info) || !alloc_info.mapped + || !has_memory_flags(alloc_info.property_flags, memory_flag(.host_visible)) { + return none + } + normalized := normalize_mapped_range(alloc_info.offset, alloc_info.size, alloc_info.block_size, + relative_offset, size, a.non_coherent_atom_size) or { return none } + return vk.MappedMemoryRange{ + memory: vk.DeviceMemory(alloc_info.memory) + offset: normalized.offset + size: normalized.size + } +} + +// flush makes host writes in the whole allocation available to the device. +// Host-coherent memory succeeds without issuing a Vulkan call. +pub fn (a &Allocator) flush(alloc_info AllocationInfo) vk.Result { + return a.flush_range(alloc_info, 0, alloc_info.size) +} + +// flush_range makes one allocation-relative host-written range available to +// the device. The Vulkan range is expanded to nonCoherentAtomSize boundaries. +pub fn (a &Allocator) flush_range(alloc_info AllocationInfo, relative_offset u64, size u64) vk.Result { + range := a.mapped_range(alloc_info, relative_offset, size) or { + return .error_memory_map_failed + } + if has_memory_flags(alloc_info.property_flags, memory_flag(.host_coherent)) { + return .success + } + return vk.flush_mapped_memory_ranges(a.create_info.device, 1, &range) +} + +// invalidate makes device writes in the whole allocation visible to the host. +// Synchronize device access before calling it. +pub fn (a &Allocator) invalidate(alloc_info AllocationInfo) vk.Result { + return a.invalidate_range(alloc_info, 0, alloc_info.size) +} + +// invalidate_range makes one allocation-relative device-written range visible +// to the host and expands it to nonCoherentAtomSize boundaries. +pub fn (a &Allocator) invalidate_range(alloc_info AllocationInfo, relative_offset u64, size u64) vk.Result { + range := a.mapped_range(alloc_info, relative_offset, size) or { + return .error_memory_map_failed + } + if has_memory_flags(alloc_info.property_flags, memory_flag(.host_coherent)) { + return .success + } + return vk.invalidate_mapped_memory_ranges(a.create_info.device, 1, &range) +} diff --git a/mapped_memory_test.v b/mapped_memory_test.v new file mode 100644 index 0000000..786aa2f --- /dev/null +++ b/mapped_memory_test.v @@ -0,0 +1,64 @@ +module vkmemalloc + +import antono2.vulkan as vk + +fn mapped_test_memory(value usize) vk.DeviceMemory { + return unsafe { voidptr(value) } +} + +fn test_mapped_range_aligns_both_ends_to_atom_size() { + range := normalize_mapped_range(128, 512, 1024, 3, 130, 64) or { + panic('range should be valid') + } + assert range.offset == 128 + assert range.size == 192 +} + +fn test_mapped_range_uses_whole_size_at_memory_end() { + range := normalize_mapped_range(768, 256, 1024, 240, 16, 64) or { + panic('range should be valid') + } + assert range.offset == 960 + assert range.size == vk.whole_size +} + +fn test_mapped_range_rejects_empty_or_out_of_allocation_ranges() { + if _ := normalize_mapped_range(128, 256, 1024, 0, 0, 64) { + assert false, 'empty ranges must be rejected' + } + if _ := normalize_mapped_range(128, 256, 1024, 250, 7, 64) { + assert false, 'ranges must remain inside the allocation' + } + if _ := normalize_mapped_range(900, 200, 1024, 0, 200, 64) { + assert false, 'allocations must remain inside their memory block' + } +} + +fn test_coherent_flush_validates_ownership_without_a_driver_call() { + mut props := vk.PhysicalDeviceMemoryProperties{} + props.memoryHeapCount = 1 + props.memoryHeaps[0].size = 256 + props.memoryTypeCount = 1 + props.memoryTypes[0] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) + heapIndex: 0 + } + mut planner := new_memory_block_pool(256, 1) or { panic(err) } + block_id := planner.add_block(0, 256) or { panic(err) } + reservation := planner.reserve(0, 64, 1) or { panic(err) } + mut allocator := Allocator{ + props: props + non_coherent_atom_size: 64 + planner: planner + } + assert allocator.remember_block(mapped_test_memory(1), block_id) + mut allocation := AllocationInfo{} + allocator.populate_allocation(mut allocation, mapped_test_memory(1), reservation) + allocation.mapped = true + assert allocator.flush_range(allocation, 1, 1) == .success + assert allocator.invalidate(allocation) == .success + assert allocator.flush_range(allocation, allocation.size, 1) == .error_memory_map_failed + + allocation.property_flags = 0 + assert allocator.flush(allocation) == .error_memory_map_failed +} diff --git a/memory_policy.v b/memory_policy.v new file mode 100644 index 0000000..222954f --- /dev/null +++ b/memory_policy.v @@ -0,0 +1,360 @@ +module vkmemalloc + +import antono2.vulkan as vk + +// MemoryUsage describes how an allocation is expected to move between the CPU +// and GPU. It is a policy hint; required_flags always remain mandatory. +pub enum MemoryUsage { + automatic + gpu_only + upload + readback +} + +// BudgetPolicy controls how reported or physical heap capacity affects memory +// type selection. +pub enum BudgetPolicy { + prefer_within + ignore + require_within +} + +// AllocationOptions describes required and preferred memory properties. The +// usage profile supplies sensible defaults, while the explicit flag sets let +// callers refine them for specialized resources. +pub struct AllocationOptions { +pub: + usage MemoryUsage + required_flags vk.MemoryPropertyFlags + preferred_flags vk.MemoryPropertyFlags + avoided_flags vk.MemoryPropertyFlags + budget_policy BudgetPolicy = .prefer_within +} + +// MemoryTypeChoice explains why a Vulkan memory type was selected. +pub struct MemoryTypeChoice { +pub: + index u32 + heap_index u32 + property_flags vk.MemoryPropertyFlags + heap_size u64 + heap_budget u64 + heap_usage u64 + remaining_budget u64 + within_budget bool + budget_reported bool + preference_score int +} + +// MemoryHeapStats combines Vulkan heap capacity/budget information with the +// blocks currently committed by this allocator. +pub struct MemoryHeapStats { +pub: + heap_index u32 + size u64 + budget u64 + usage u64 + remaining_budget u64 + allocator_committed u64 + allocator_used u64 + device_local bool + budget_reported bool +} + +struct HeapBudgetSnapshot { + reported bool + budgets []u64 + usages []u64 +} + +// supports_memory_budget reports whether a physical device exposes +// VK_EXT_memory_budget through this allocator's Vulkan 1.1 query path. Call it +// after the Vulkan loader and instance commands are initialized, and enable +// that device extension before opting the allocator into live budget queries. +pub fn supports_memory_budget(physical_device vk.PhysicalDevice) bool { + mut device_properties := vk.PhysicalDeviceProperties{} + vk.get_physical_device_properties(physical_device, mut &device_properties) + if device_properties.apiVersion < vk.api_version_1_1 { + return false + } + for { + mut count := u32(0) + mut no_properties := unsafe { nil } + if vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut no_properties) != .success + || count == 0 { + return false + } + mut properties := []vk.ExtensionProperties{len: int(count)} + result := vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut + properties[0]) + if result == .incomplete { + continue + } + if result != .success { + return false + } + for index in 0 .. int(count) { + name := unsafe { cstring_to_vstring(&properties[index].extensionName[0]) } + if name == 'VK_EXT_memory_budget' { + return true + } + } + return false + } + return false +} + +fn memory_flag(flag vk.MemoryPropertyFlagBits) vk.MemoryPropertyFlags { + return vk.MemoryPropertyFlags(u32(flag)) +} + +fn has_memory_flags(flags vk.MemoryPropertyFlags, required vk.MemoryPropertyFlags) bool { + return (flags & required) == required +} + +fn memory_flag_count(flags vk.MemoryPropertyFlags) int { + mut value := u32(flags) + mut count := 0 + for value != 0 { + count += int(value & 1) + value >>= 1 + } + return count +} + +fn usage_required_flags(usage MemoryUsage) vk.MemoryPropertyFlags { + return match usage { + .gpu_only { memory_flag(.device_local) } + .upload, .readback { memory_flag(.host_visible) } + .automatic { vk.MemoryPropertyFlags(0) } + } +} + +fn usage_preference_score(usage MemoryUsage, flags vk.MemoryPropertyFlags) int { + device_local := has_memory_flags(flags, memory_flag(.device_local)) + host_coherent := has_memory_flags(flags, memory_flag(.host_coherent)) + host_cached := has_memory_flags(flags, memory_flag(.host_cached)) + device_uncached := has_memory_flags(flags, memory_flag(.device_uncached_bit_amd)) + return match usage { + .automatic { + memory_score(device_local, 16) + } + .gpu_only { + memory_score(device_uncached, -4) + } + .upload { + memory_score(host_coherent, 16) + memory_score(device_local, 8) + + memory_score(host_cached, 2) + memory_score(device_uncached, -4) + } + .readback { + memory_score(host_cached, 16) + memory_score(host_coherent, 8) + + memory_score(device_local, 2) + memory_score(device_uncached, -4) + } + } +} + +fn memory_score(condition bool, points int) int { + if condition { + return points + } + return 0 +} + +fn memory_preference_score(options AllocationOptions, flags vk.MemoryPropertyFlags) int { + preferred := memory_flag_count(flags & options.preferred_flags) + avoided := memory_flag_count(flags & options.avoided_flags) + return usage_preference_score(options.usage, flags) + preferred * 4 - avoided * 32 +} + +fn heap_budget_values(props vk.PhysicalDeviceMemoryProperties, heap_index u32, snapshot HeapBudgetSnapshot) (u64, u64, bool) { + heap_size := u64(props.memoryHeaps[heap_index].size) + if int(heap_index) < snapshot.budgets.len && int(heap_index) < snapshot.usages.len { + budget := if snapshot.budgets[heap_index] > 0 { + snapshot.budgets[heap_index] + } else { + heap_size + } + return budget, snapshot.usages[heap_index], snapshot.reported + } + return heap_size, 0, false +} + +fn memory_choice_is_better(candidate MemoryTypeChoice, current MemoryTypeChoice, policy BudgetPolicy) bool { + if policy == .prefer_within && candidate.within_budget != current.within_budget { + return candidate.within_budget + } + if candidate.preference_score != current.preference_score { + return candidate.preference_score > current.preference_score + } + if policy != .ignore && candidate.remaining_budget != current.remaining_budget { + return candidate.remaining_budget > current.remaining_budget + } + return candidate.index < current.index +} + +fn ranked_memory_types(props vk.PhysicalDeviceMemoryProperties, type_bits u32, request_size u64, options AllocationOptions, snapshot HeapBudgetSnapshot) []MemoryTypeChoice { + required := usage_required_flags(options.usage) | options.required_flags + mut choices := []MemoryTypeChoice{} + for index in 0 .. int(props.memoryTypeCount) { + if index >= int(vk.max_memory_types) || (type_bits & (u32(1) << u32(index))) == 0 { + continue + } + memory_type := props.memoryTypes[index] + if !has_memory_flags(memory_type.propertyFlags, required) + || memory_type.heapIndex >= props.memoryHeapCount { + continue + } + heap_size := u64(props.memoryHeaps[memory_type.heapIndex].size) + budget, usage, reported := heap_budget_values(props, memory_type.heapIndex, snapshot) + remaining := if usage < budget { budget - usage } else { u64(0) } + within_budget := request_size <= remaining + if options.budget_policy == .require_within && !within_budget { + continue + } + choice := MemoryTypeChoice{ + index: u32(index) + heap_index: memory_type.heapIndex + property_flags: memory_type.propertyFlags + heap_size: heap_size + heap_budget: budget + heap_usage: usage + remaining_budget: remaining + within_budget: within_budget + budget_reported: reported + preference_score: memory_preference_score(options, memory_type.propertyFlags) + } + mut inserted := false + for position, existing in choices { + if memory_choice_is_better(choice, existing, options.budget_policy) { + choices.insert(position, choice) + inserted = true + break + } + } + if !inserted { + choices << choice + } + } + return choices +} + +// select_memory_type applies the portable usage/property policy using physical +// heap sizes. Allocator.select_memory_type additionally uses live heap budgets +// when VK_EXT_memory_budget integration was enabled at allocator creation. +pub fn select_memory_type(props vk.PhysicalDeviceMemoryProperties, type_bits u32, request_size u64, options AllocationOptions) ?MemoryTypeChoice { + choices := ranked_memory_types(props, type_bits, request_size, options, HeapBudgetSnapshot{}) + if choices.len == 0 { + return none + } + return choices[0] +} + +fn (a &Allocator) heap_budget_snapshot() HeapBudgetSnapshot { + mut budgets := []u64{len: int(a.props.memoryHeapCount)} + mut usages := []u64{len: int(a.props.memoryHeapCount)} + for heap_index in 0 .. int(a.props.memoryHeapCount) { + if a.memory_budget_reported && heap_index < a.heap_budgets.len + && heap_index < a.heap_usages.len { + budgets[heap_index] = a.heap_budgets[heap_index] + usages[heap_index] = a.heap_usages[heap_index] + continue + } + budgets[heap_index] = u64(a.props.memoryHeaps[heap_index].size) + if !isnil(a.planner) { + committed, _ := a.planner.heap_stats(&a.props, u32(heap_index)) + usages[heap_index] = committed + } + } + return HeapBudgetSnapshot{ + reported: a.memory_budget_reported + budgets: budgets + usages: usages + } +} + +// refresh_memory_budget refreshes VK_EXT_memory_budget estimates when the +// allocator was created with memory_budget_enabled. It returns false when the +// optional integration is unavailable; physical heap sizes remain usable. +pub fn (mut a Allocator) refresh_memory_budget() bool { + if !a.create_info.memory_budget_enabled || a.api_version < vk.api_version_1_1 { + return false + } + mut budget := vk.PhysicalDeviceMemoryBudgetPropertiesEXT{} + mut properties := vk.PhysicalDeviceMemoryProperties2{ + pNext: voidptr(&budget) + } + vk.get_physical_device_memory_properties2(a.create_info.physical_device, mut &properties) + a.props = properties.memoryProperties + heap_count := int(a.props.memoryHeapCount) + a.heap_budgets = []u64{len: heap_count} + a.heap_usages = []u64{len: heap_count} + mut reported := false + for heap_index in 0 .. heap_count { + a.heap_budgets[heap_index] = u64(budget.heapBudget[heap_index]) + a.heap_usages[heap_index] = u64(budget.heapUsage[heap_index]) + if a.heap_budgets[heap_index] > 0 { + reported = true + } + } + a.memory_budget_reported = reported + return reported +} + +// select_memory_type ranks every compatible type and returns an explainable +// choice. It uses the latest refreshed budget snapshot when that optional +// integration is enabled; otherwise allocator-owned commitment is used. +pub fn (mut a Allocator) select_memory_type(type_bits u32, request_size u64, options AllocationOptions) ?MemoryTypeChoice { + choices := a.rank_memory_types(type_bits, request_size, options) + if choices.len == 0 { + return none + } + return choices[0] +} + +fn (mut a Allocator) rank_memory_types(type_bits u32, request_size u64, options AllocationOptions) []MemoryTypeChoice { + return ranked_memory_types(a.props, type_bits, request_size, options, a.heap_budget_snapshot()) +} + +// A buffer can reuse a compatible block without increasing heap usage. Keep +// over-budget candidates available for that reuse even when new block creation +// is forbidden by require_within. +fn (mut a Allocator) rank_buffer_memory_types(type_bits u32, request_size u64, options AllocationOptions) []MemoryTypeChoice { + if options.budget_policy != .require_within { + return a.rank_memory_types(type_bits, request_size, options) + } + return a.rank_memory_types(type_bits, request_size, AllocationOptions{ + usage: options.usage + required_flags: options.required_flags + preferred_flags: options.preferred_flags + avoided_flags: options.avoided_flags + budget_policy: .prefer_within + }) +} + +// memory_heaps returns one diagnostics record per Vulkan memory heap. Reported +// budget/usage values come from VK_EXT_memory_budget when enabled; the portable +// fallback uses heap size and this allocator's own committed blocks. +pub fn (a &Allocator) memory_heaps() []MemoryHeapStats { + snapshot := a.heap_budget_snapshot() + mut heaps := []MemoryHeapStats{cap: int(a.props.memoryHeapCount)} + for heap_index in 0 .. int(a.props.memoryHeapCount) { + mut committed := u64(0) + mut used := u64(0) + if !isnil(a.planner) { + committed, used = a.planner.heap_stats(&a.props, u32(heap_index)) + } + budget, usage, reported := heap_budget_values(a.props, u32(heap_index), snapshot) + heaps << MemoryHeapStats{ + heap_index: u32(heap_index) + size: u64(a.props.memoryHeaps[heap_index].size) + budget: budget + usage: usage + remaining_budget: if usage < budget { budget - usage } else { u64(0) } + allocator_committed: committed + allocator_used: used + device_local: (a.props.memoryHeaps[heap_index].flags & u32(vk.MemoryHeapFlagBits.device_local)) != 0 + budget_reported: reported + } + } + return heaps +} diff --git a/memory_policy_test.v b/memory_policy_test.v new file mode 100644 index 0000000..3e199bf --- /dev/null +++ b/memory_policy_test.v @@ -0,0 +1,174 @@ +module vkmemalloc + +import antono2.vulkan as vk + +fn policy_test_memory(value usize) vk.DeviceMemory { + return unsafe { voidptr(value) } +} + +fn policy_test_properties() vk.PhysicalDeviceMemoryProperties { + mut props := vk.PhysicalDeviceMemoryProperties{} + props.memoryHeapCount = 2 + props.memoryHeaps[0] = vk.MemoryHeap{ + size: 256 + flags: u32(vk.MemoryHeapFlagBits.device_local) + } + props.memoryHeaps[1] = vk.MemoryHeap{ + size: 1024 + } + props.memoryTypeCount = 4 + props.memoryTypes[0] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.device_local) + heapIndex: 0 + } + props.memoryTypes[1] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) + heapIndex: 1 + } + props.memoryTypes[2] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_cached) + heapIndex: 1 + } + props.memoryTypes[3] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.device_local) | u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) + heapIndex: 0 + } + return props +} + +fn test_memory_policy_selects_usage_specific_properties() { + props := policy_test_properties() + all_types := u32(0b1111) + gpu := select_memory_type(props, all_types, 16, AllocationOptions{ + usage: .gpu_only + }) or { panic('GPU memory type should exist') } + assert gpu.index == 0 + assert gpu.heap_index == 0 + + upload := select_memory_type(props, all_types, 16, AllocationOptions{ + usage: .upload + }) or { panic('upload memory type should exist') } + assert upload.index == 3 + + readback := select_memory_type(props, all_types, 16, AllocationOptions{ + usage: .readback + }) or { panic('readback memory type should exist') } + assert readback.index == 2 + assert has_memory_flags(readback.property_flags, memory_flag(.host_cached)) +} + +fn test_memory_policy_honors_required_preferred_and_avoided_flags() { + props := policy_test_properties() + host_visible := memory_flag(.host_visible) + host_coherent := memory_flag(.host_coherent) + host_cached := memory_flag(.host_cached) + choice := select_memory_type(props, 0b1110, 16, AllocationOptions{ + required_flags: host_visible + preferred_flags: host_cached + avoided_flags: host_coherent + }) or { panic('host-visible memory type should exist') } + assert choice.index == 2 + + if _ := select_memory_type(props, 0b0001, 16, AllocationOptions{ + required_flags: host_visible + }) + { + assert false, 'required properties must never be dropped' + } +} + +fn test_memory_policy_prefers_or_requires_available_budget() { + props := policy_test_properties() + options := AllocationOptions{ + usage: .automatic + } + snapshot := HeapBudgetSnapshot{ + reported: true + budgets: [u64(128), 1024] + usages: [u64(120), 0] + } + preferred := ranked_memory_types(props, 0b1011, 16, options, snapshot) + assert preferred.len == 3 + assert preferred[0].index == 1 + assert preferred[0].within_budget + assert preferred[0].budget_reported + assert preferred[0].remaining_budget == 1024 + + ignored := ranked_memory_types(props, 0b1011, 16, AllocationOptions{ + usage: .automatic + budget_policy: .ignore + }, snapshot) + assert ignored[0].index == 0 + assert !ignored[0].within_budget + + required := ranked_memory_types(props, 0b1011, 16, AllocationOptions{ + usage: .automatic + budget_policy: .require_within + }, snapshot) + assert required.len == 1 + assert required[0].index == 1 +} + +fn test_memory_policy_is_deterministic_for_equal_candidates() { + mut props := policy_test_properties() + props.memoryTypes[1].propertyFlags = u32(vk.MemoryPropertyFlagBits.host_visible) + props.memoryTypes[2].propertyFlags = u32(vk.MemoryPropertyFlagBits.host_visible) + choice := select_memory_type(props, 0b0110, 1, AllocationOptions{}) or { + panic('memory type should exist') + } + assert choice.index == 1 +} + +fn test_allocator_policy_uses_owned_commitment_as_portable_budget_fallback() { + props := policy_test_properties() + mut planner := new_memory_block_pool(256, 4) or { panic(err) } + _ = planner.add_block(0, 256) or { panic(err) } + mut allocator := Allocator{ + props: props + planner: planner + } + choice := allocator.select_memory_type(0b0011, 16, AllocationOptions{}) or { + panic('a memory type should remain available') + } + assert choice.index == 1 + assert !choice.budget_reported + assert choice.within_budget + + heaps := allocator.memory_heaps() + assert heaps.len == 2 + assert heaps[0].budget == 256 + assert heaps[0].usage == 256 + assert heaps[0].remaining_budget == 0 + assert heaps[0].allocator_committed == 256 + assert heaps[0].allocator_used == 0 +} + +fn test_require_within_can_reuse_an_over_budget_buffer_block() { + props := policy_test_properties() + mut planner := new_memory_block_pool(256, 1) or { panic(err) } + block_id := planner.add_block(0, 256) or { panic(err) } + mut allocator := Allocator{ + props: props + planner: planner + } + assert allocator.remember_block(policy_test_memory(1), block_id) + options := AllocationOptions{ + usage: .gpu_only + budget_policy: .require_within + } + choices := allocator.rank_buffer_memory_types(0b0001, 16, options) + assert choices.len == 1 + assert !choices[0].within_budget + mut requirements := vk.MemoryRequirements{ + size: 16 + alignment: 8 + memoryTypeBits: 0b0001 + } + mut allocation := AllocationInfo{} + result := allocator.allocate_from_choices(mut requirements, choices, unsafe { nil }, false, + options.budget_policy, mut allocation) + assert result == .success + assert allocation.memory == voidptr(policy_test_memory(1)) + assert allocation.block_size == 256 + assert allocator.release(mut allocation) +} diff --git a/upload_ring.v b/upload_ring.v index 910337a..c41d1c5 100644 --- a/upload_ring.v +++ b/upload_ring.v @@ -64,6 +64,45 @@ pub fn new_upload_ring(mut allocator Allocator, capacity u64) !&UploadRing { if result != .success { return error('could not create upload buffer: ${result}') } + return finish_upload_ring(mut allocator, capacity, buffer, backing) +} + +// new_upload_ring_with_options creates an upload ring with policy-selected +// host-visible memory. Prefer usage .upload; callers using a non-coherent type +// must flush each written slice before device access. +pub fn new_upload_ring_with_options(mut allocator Allocator, capacity u64, options AllocationOptions) !&UploadRing { + if capacity == 0 { + return error('upload ring capacity must be greater than zero') + } + $if x32 { + if capacity > u64(max_u32) { + return error('upload ring capacity exceeds the host address space') + } + } + buffer_info := vk.BufferCreateInfo{ + size: capacity + usage: u32(vk.BufferUsageFlagBits.transfer_src) + sharingMode: .exclusive + } + effective_options := AllocationOptions{ + usage: options.usage + required_flags: options.required_flags | memory_flag(.host_visible) + preferred_flags: options.preferred_flags + avoided_flags: options.avoided_flags + budget_policy: options.budget_policy + } + mut buffer := vk.Buffer(unsafe { nil }) + mut backing := AllocationInfo{} + result := allocator.create_dedicated_buffer_with_options(&buffer_info, effective_options, + &buffer, mut backing) + if result != .success { + return error('could not create upload buffer: ${result}') + } + return finish_upload_ring(mut allocator, capacity, buffer, backing) +} + +fn finish_upload_ring(mut allocator Allocator, capacity u64, buffer vk.Buffer, initial_backing AllocationInfo) !&UploadRing { + mut backing := initial_backing mut mapped := voidptr(unsafe { nil }) map_result := allocator.map(mut backing, &mapped) if map_result != .success { @@ -117,6 +156,24 @@ pub fn (mut ring UploadRing) retire(slice UploadSlice) bool { return ring.ranges.release(slice.allocation) } +// flush makes host writes in a live slice available to the device. It is a +// no-op for the coherent memory used by new_upload_ring(), but keeps upload +// code correct if the backing policy changes. +pub fn (ring &UploadRing) flush(slice UploadSlice) vk.Result { + if !ring.contains(slice) { + return .error_memory_map_failed + } + return ring.allocator.flush_range(ring.backing, slice.offset, slice.size) +} + +// invalidate makes device writes in a live slice visible to the host. +pub fn (ring &UploadRing) invalidate(slice UploadSlice) vk.Result { + if !ring.contains(slice) { + return .error_memory_map_failed + } + return ring.allocator.invalidate_range(ring.backing, slice.offset, slice.size) +} + // stats returns current payload, padding, free-space, and peak ring occupancy. pub fn (ring &UploadRing) stats() UploadRingStats { if ring.destroyed { diff --git a/v.mod b/v.mod index 80110d6..2d089b9 100644 --- a/v.mod +++ b/v.mod @@ -1,8 +1,8 @@ Module { name: 'antono2.vkmemalloc' author: 'Anton Oreskin' - description: 'Vulkan block suballocation helpers for buffers and images' - version: '2.3.2' + description: 'Policy-driven Vulkan memory selection, suballocation, mapping, and diagnostics' + version: '2.4.0' license: 'MIT' repo_url: 'https://github.com/antono2/vulkan_memory_allocator' tags: ['V','vulkan','allocator'] diff --git a/vulkan_memory_allocator.v b/vulkan_memory_allocator.v index bb25be8..3923150 100644 --- a/vulkan_memory_allocator.v +++ b/vulkan_memory_allocator.v @@ -6,16 +6,20 @@ pub const max_pools = 256 pub const memory_block = 1024 * 1024 pub struct Allocator { - create_info AllocatorCreateInfo - props vk.PhysicalDeviceMemoryProperties - api_version u32 + create_info AllocatorCreateInfo + api_version u32 + non_coherent_atom_size u64 mut: - planner &MemoryBlockPool = unsafe { nil } - pools [max_pools]vk.DeviceMemory - block_ids [max_pools]u64 - mapped [max_pools]voidptr - map_refs [max_pools]u32 - pool_size u32 + props vk.PhysicalDeviceMemoryProperties + planner &MemoryBlockPool = unsafe { nil } + pools [max_pools]vk.DeviceMemory + block_ids [max_pools]u64 + mapped [max_pools]voidptr + map_refs [max_pools]u32 + pool_size u32 + memory_budget_reported bool + heap_budgets []u64 + heap_usages []u64 } fn (a &Allocator) has_free_slot() bool { @@ -101,15 +105,21 @@ pub enum MemType { } pub struct AllocationInfo { - // The memory type index pub mut: + // The memory type index mem_type u32 + // The Vulkan memory heap backing that type. + heap_index u32 + // Properties of the selected memory type. + property_flags vk.MemoryPropertyFlags // The memory handle (VkDeviceMemory) memory voidptr = unsafe { nil } // The offset in the memory block offset u64 // The size reserved for this resource inside the memory block size u64 + // Total size of the VkDeviceMemory block containing this allocation. + block_size u64 mut: reservation BlockReservation mapped bool @@ -130,6 +140,9 @@ pub mut: preferred_block_size u64 = memory_block // Maximum number of live VkDeviceMemory blocks, capped by max_pools. max_memory_blocks int = max_pools + // Enable VK_EXT_memory_budget property queries. Set this only when the + // physical device reports support and the device extension is enabled. + memory_budget_enabled bool } // new creates a Vulkan allocator with memory-type-specific shared blocks. @@ -151,15 +164,24 @@ pub fn new(create_info AllocatorCreateInfo) Allocator { } else { create_info.max_memory_blocks } - planner := new_memory_block_pool(block_size, block_limit) or { + mut planner := new_memory_block_pool(block_size, block_limit) or { panic('invalid Vulkan memory block configuration: ${err}') } - return Allocator{ - create_info: create_info - props: mem_props - api_version: device_props.apiVersion - planner: planner + mut allocator := Allocator{ + create_info: create_info + props: mem_props + api_version: device_props.apiVersion + non_coherent_atom_size: if device_props.limits.nonCoherentAtomSize > 0 { + u64(device_props.limits.nonCoherentAtomSize) + } else { + u64(1) + } + planner: planner + } + if create_info.memory_budget_enabled { + _ = allocator.refresh_memory_budget() } + return allocator } // get_memory_type selects a supported memory type containing every requested @@ -206,32 +228,83 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me } } - // `alloc_info` is the caller's output record. Rebinding it to a freshly - // allocated local pointer loses the allocation handle at every call site. - // Reset and populate that record directly instead. - alloc_info = AllocationInfo{} - - // Note: VK_NULL_HANDLE is "nullptr", "voidptr(0)" for C++ compatible compilers, or "0ULL" (Unsigned Long Long 0) for 64bit and "0" for 32 bit in C - alloc_info.memory = unsafe { nil } // vk.null_handle - alloc_info.size = req.size - alloc_info.offset = 0 - alloc_info.mem_type = a.get_memory_type(req.memoryTypeBits, mem_type) - if alloc_info.mem_type == max_u32 { + memory_type := a.get_memory_type(req.memoryTypeBits, mem_type) + if memory_type == max_u32 { // Never drop required properties. In particular, mapping arbitrary // device-local memory after a staging allocation fails is invalid and // previously led to a null mapped pointer and a delayed segfault. eprintln('No compatible Vulkan memory type: type bits 0x${req.memoryTypeBits:08x}, required flags 0x${u32(mem_type):08x}') return .error_feature_not_present } + choices := ranked_memory_types(a.props, u32(1) << memory_type, req.size, AllocationOptions{ + budget_policy: .ignore + }, a.heap_budget_snapshot()) + return a.allocate_from_choices(mut req, choices, allocation_pnext, dedicated, .ignore, mut + alloc_info) +} + +// allocate_with_options reserves isolated memory using the portable ranked +// policy. Prefer create_buffer_with_options() when safe buffer suballocation is +// desired. +pub fn (mut a Allocator) allocate_with_options(mut req vk.MemoryRequirements, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { + choices := a.rank_memory_types(req.memoryTypeBits, req.size, options) + return a.allocate_from_choices(mut req, choices, unsafe { nil }, true, options.budget_policy, mut + alloc_info) +} + +fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choices []MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { + // `alloc_info` is the caller's output record. Reset and populate that record + // directly so callers always receive the actual tracked handle. + alloc_info = AllocationInfo{} if req.size == 0 || req.alignment == 0 { return .error_initialization_failed } if isnil(a.planner) { return .error_initialization_failed } + if choices.len == 0 { + return .error_feature_not_present + } + mut last_result := vk.Result.error_out_of_device_memory + for choice in choices { + allow_new_block := budget_policy != .require_within || choice.within_budget + mut result := a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, + allow_new_block, budget_policy, mut alloc_info) + if result == .success { + return .success + } + last_result = result + if result !in [.error_out_of_device_memory, .error_out_of_host_memory, + .error_too_many_objects] { + return result + } + mut trimmed := 0 + if allow_new_block { + trimmed = a.trim_empty_blocks_for_heap(choice.heap_index) + if result == .error_too_many_objects { + // The block-count cap is allocator-wide, so an empty block in a + // different heap can also make room for this candidate. + trimmed += a.trim_empty_blocks() + } + } + if trimmed > 0 { + result = a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, true, + budget_policy, mut alloc_info) + if result == .success { + return .success + } + last_result = result + } + if result == .error_too_many_objects { + return result + } + } + return last_result +} +fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, choice MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, allow_new_block bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { if !dedicated { - if reservation := a.planner.reserve(alloc_info.mem_type, req.size, req.alignment) { + if reservation := a.planner.reserve(choice.index, req.size, req.alignment) { memory := a.memory_for_block(reservation.block_id) or { _ = a.planner.release(reservation) return .error_initialization_failed @@ -240,18 +313,24 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me return .success } } - + if !allow_new_block { + return .error_out_of_device_memory + } if !a.has_free_slot() { return .error_too_many_objects } - block_size := if dedicated { + mut block_size := if dedicated { req.size } else { a.planner.recommended_block_size(req.size) or { return .error_out_of_device_memory } } + if budget_policy != .ignore && choice.budget_reported && choice.within_budget + && choice.remaining_budget < block_size { + block_size = choice.remaining_budget + } vkalloc_info := vk.MemoryAllocateInfo{ allocationSize: block_size - memoryTypeIndex: alloc_info.mem_type + memoryTypeIndex: choice.index pNext: allocation_pnext } mut memory := vk.DeviceMemory(unsafe { nil }) @@ -260,13 +339,13 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me return result } block_id := if dedicated { - a.planner.add_dedicated_block(alloc_info.mem_type, block_size) or { + a.planner.add_dedicated_block(choice.index, block_size) or { vk.free_memory(a.create_info.device, memory, unsafe { nil }) alloc_info = AllocationInfo{} return .error_too_many_objects } } else { - a.planner.add_block(alloc_info.mem_type, block_size) or { + a.planner.add_block(choice.index, block_size) or { vk.free_memory(a.create_info.device, memory, unsafe { nil }) alloc_info = AllocationInfo{} return .error_too_many_objects @@ -292,8 +371,14 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me fn (a &Allocator) populate_allocation(mut alloc_info AllocationInfo, memory vk.DeviceMemory, reservation BlockReservation) { alloc_info.memory = voidptr(memory) alloc_info.mem_type = reservation.memory_type + if reservation.memory_type < a.props.memoryTypeCount { + memory_type := a.props.memoryTypes[reservation.memory_type] + alloc_info.heap_index = memory_type.heapIndex + alloc_info.property_flags = memory_type.propertyFlags + } alloc_info.offset = reservation.offset alloc_info.size = reservation.size + alloc_info.block_size = a.planner.block_capacity(reservation.block_id) or { 0 } alloc_info.reservation = reservation } @@ -302,10 +387,16 @@ fn (a &Allocator) owns_allocation(alloc_info AllocationInfo) bool { return false } memory := a.memory_for_block(alloc_info.reservation.block_id) or { return false } - return voidptr(memory) == alloc_info.memory - && alloc_info.offset == alloc_info.reservation.offset - && alloc_info.size == alloc_info.reservation.size - && alloc_info.mem_type == alloc_info.reservation.memory_type + if alloc_info.mem_type < a.props.memoryTypeCount { + memory_type := a.props.memoryTypes[alloc_info.mem_type] + if alloc_info.heap_index != memory_type.heapIndex + || alloc_info.property_flags != memory_type.propertyFlags { + return false + } + } + return voidptr(memory) == alloc_info.memory && alloc_info.offset == alloc_info.reservation.offset && alloc_info.size == alloc_info.reservation.size && alloc_info.mem_type == alloc_info.reservation.memory_type && alloc_info.block_size == (a.planner.block_capacity(alloc_info.reservation.block_id) or { + return false + }) } fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { @@ -336,6 +427,46 @@ fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, forc voidptr(&dedicated_info), true, mut alloc_info) } +fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, options AllocationOptions, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { + if a.api_version < vk.api_version_1_1 { + mut requirements := vk.MemoryRequirements{} + vk.get_buffer_memory_requirements(a.create_info.device, buffer, mut requirements) + choices := if force_dedicated { + a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + } else { + a.rank_buffer_memory_types(requirements.memoryTypeBits, requirements.size, options) + } + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, force_dedicated, + options.budget_policy, mut alloc_info) + } + mut dedicated_requirements := vk.MemoryDedicatedRequirements{} + mut requirements := vk.MemoryRequirements2{ + pNext: &dedicated_requirements + } + info := vk.BufferMemoryRequirementsInfo2{ + buffer: buffer + } + vk.get_buffer_memory_requirements2(a.create_info.device, &info, mut requirements) + dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true + || dedicated_requirements.prefersDedicatedAllocation == vk._true + choices := if dedicated { + a.rank_memory_types(requirements.memoryRequirements.memoryTypeBits, + requirements.memoryRequirements.size, options) + } else { + a.rank_buffer_memory_types(requirements.memoryRequirements.memoryTypeBits, + requirements.memoryRequirements.size, options) + } + if !dedicated { + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, + unsafe { nil }, false, options.budget_policy, mut alloc_info) + } + dedicated_info := vk.MemoryDedicatedAllocateInfo{ + buffer: buffer + } + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, + voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) +} + fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut alloc_info AllocationInfo) vk.Result { mut requirements := vk.MemoryRequirements{} if a.api_version >= vk.api_version_1_1 { @@ -355,6 +486,28 @@ fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut all return a.allocate_with_policy(mut requirements, type, unsafe { nil }, true, mut alloc_info) } +fn (mut a Allocator) allocate_image_memory_with_options(image vk.Image, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { + mut requirements := vk.MemoryRequirements{} + if a.api_version >= vk.api_version_1_1 { + mut requirements2 := vk.MemoryRequirements2{} + info := vk.ImageMemoryRequirementsInfo2{ + image: image + } + vk.get_image_memory_requirements2(a.create_info.device, &info, mut requirements2) + requirements = requirements2.memoryRequirements + choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + dedicated_info := vk.MemoryDedicatedAllocateInfo{ + image: image + } + return a.allocate_from_choices(mut requirements, choices, voidptr(&dedicated_info), true, + options.budget_policy, mut alloc_info) + } + vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) + choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, true, + options.budget_policy, mut alloc_info) +} + // create_buffer creates a buffer, suballocates compatible memory, and binds it. pub fn (mut a Allocator) create_buffer(buffer_info &vk.BufferCreateInfo, type MemType, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { return a.create_buffer_with_policy(buffer_info, type, false, buffer, mut alloc_info) @@ -366,6 +519,47 @@ pub fn (mut a Allocator) create_dedicated_buffer(buffer_info &vk.BufferCreateInf return a.create_buffer_with_policy(buffer_info, type, true, buffer, mut alloc_info) } +// create_buffer_with_options creates a buffer and selects memory using an +// explicit usage/property/budget policy. Compatible buffers share blocks. +pub fn (mut a Allocator) create_buffer_with_options(buffer_info &vk.BufferCreateInfo, options AllocationOptions, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { + return a.create_buffer_with_options_policy(buffer_info, options, false, buffer, mut alloc_info) +} + +// create_dedicated_buffer_with_options is the policy-based counterpart of +// create_dedicated_buffer(). +pub fn (mut a Allocator) create_dedicated_buffer_with_options(buffer_info &vk.BufferCreateInfo, options AllocationOptions, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { + return a.create_buffer_with_options_policy(buffer_info, options, true, buffer, mut alloc_info) +} + +fn (mut a Allocator) create_buffer_with_options_policy(buffer_info &vk.BufferCreateInfo, options AllocationOptions, dedicated bool, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { + unsafe { + *buffer = nil + } + alloc_info = AllocationInfo{} + mut result := vk.create_buffer(a.create_info.device, buffer_info, unsafe { nil }, buffer) + if result != .success { + return result + } + result = a.allocate_buffer_memory_with_options(*buffer, options, dedicated, mut alloc_info) + if result != .success { + vk.destroy_buffer(a.create_info.device, *buffer, unsafe { nil }) + unsafe { + *buffer = nil + } + return result + } + result = vk.bind_buffer_memory(a.create_info.device, *buffer, alloc_info.memory, + alloc_info.offset) + if result != .success { + vk.destroy_buffer(a.create_info.device, *buffer, unsafe { nil }) + unsafe { + *buffer = nil + } + _ = a.release(mut alloc_info) + } + return result +} + fn (mut a Allocator) create_buffer_with_policy(buffer_info &vk.BufferCreateInfo, type MemType, dedicated bool, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { unsafe { *buffer = nil @@ -436,6 +630,39 @@ pub fn (mut a Allocator) create_image(p_image_create_info &vk.ImageCreateInfo, t return vk.Result.success } +// create_image_with_options creates a dedicated image allocation using the +// ranked usage/property/budget policy. Image suballocation remains deliberately +// conservative because buffer-image granularity and tiling compatibility must +// be tracked together. +pub fn (mut a Allocator) create_image_with_options(image_info &vk.ImageCreateInfo, options AllocationOptions, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + unsafe { + *image = nil + } + alloc_info = AllocationInfo{} + mut result := vk.create_image(a.create_info.device, image_info, unsafe { nil }, image) + if result != .success { + return result + } + result = a.allocate_image_memory_with_options(*image, options, mut alloc_info) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil + } + return result + } + result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, + alloc_info.offset) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil + } + _ = a.release(mut alloc_info) + } + return result +} + // map maps the allocation's byte range for host access. Compatible allocations // sharing one VkDeviceMemory block share one Vulkan mapping internally. pub fn (mut a Allocator) map(mut alloc_info AllocationInfo, data &voidptr) vk.Result { @@ -443,6 +670,10 @@ pub fn (mut a Allocator) map(mut alloc_info AllocationInfo, data &voidptr) vk.Re eprintln('Cannot map an allocation not owned by this allocator') return .error_memory_map_failed } + if !has_memory_flags(alloc_info.property_flags, memory_flag(.host_visible)) { + eprintln('Cannot map memory without the host-visible property') + return .error_memory_map_failed + } if alloc_info.mapped { eprintln('Cannot map an allocation that is already mapped') return .error_memory_map_failed @@ -519,6 +750,14 @@ pub fn (mut a Allocator) allocator_free(mut alloc_info AllocationInfo) { // trim_empty_blocks frees cached VkDeviceMemory blocks with no live ranges. pub fn (mut a Allocator) trim_empty_blocks() int { + return a.trim_empty_blocks_filtered(0, false) +} + +fn (mut a Allocator) trim_empty_blocks_for_heap(heap_index u32) int { + return a.trim_empty_blocks_filtered(heap_index, true) +} + +fn (mut a Allocator) trim_empty_blocks_filtered(heap_index u32, filter_by_heap bool) int { if isnil(a.planner) { return 0 } @@ -526,6 +765,15 @@ pub fn (mut a Allocator) trim_empty_blocks() int { mut index := 0 for index < int(a.pool_size) { block_id := a.block_ids[index] + memory_type := a.planner.block_memory_type(block_id) or { + index++ + continue + } + if filter_by_heap && (memory_type >= a.props.memoryTypeCount + || a.props.memoryTypes[memory_type].heapIndex != heap_index) { + index++ + continue + } allocation_count := a.planner.block_allocation_count(block_id) or { index++ continue