From 9caddfac43f2dd7830b607a204d7c8e61b4756d7 Mon Sep 17 00:00:00 2001 From: antono2 Date: Sat, 12 Sep 2026 15:38:04 +0200 Subject: [PATCH 1/2] Add budget-aware Vulkan memory policies --- .github/workflows/test.yml | 32 ++- CHANGELOG.md | 19 ++ README.md | 121 ++++++++- block_pool.v | 77 ++++-- block_pool_test.v | 10 +- examples/buffer_suballocation/main.v | 92 ++++--- mapped_memory.v | 86 +++++++ mapped_memory_test.v | 64 +++++ memory_policy.v | 357 +++++++++++++++++++++++++++ memory_policy_test.v | 172 +++++++++++++ upload_ring.v | 94 +++++-- upload_ring_test.v | 20 +- v.mod | 4 +- vulkan_memory_allocator.v | 347 +++++++++++++++++++++----- 14 files changed, 1337 insertions(+), 158 deletions(-) create mode 100644 mapped_memory.v create mode 100644 mapped_memory_test.v create mode 100644 memory_policy.v create mode 100644 memory_policy_test.v diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc7c454..94c3fb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,8 @@ jobs: ref: v1.4.0 path: source/modules/antono2/memory - uses: prantlf/setup-v-action@v4 + with: + version: 0.5.2 - name: Install Vulkan development library run: sudo apt-get update && sudo apt-get install -y libvulkan-dev libvulkan-volk-dev mesa-vulkan-drivers - name: Check formatting @@ -38,7 +40,7 @@ jobs: - name: Vet allocator sources working-directory: source/modules/antono2/vkmemalloc # V 0.5.2's directory-mode vet parser does not accept aliased imports. - run: v vet block_pool.v block_pool_test.v upload_ring.v upload_ring_test.v vulkan_memory_allocator.v vulkan_memory_allocator_test.v + run: v vet block_pool.v block_pool_test.v mapped_memory.v mapped_memory_test.v memory_policy.v memory_policy_test.v upload_ring.v upload_ring_test.v vulkan_memory_allocator.v vulkan_memory_allocator_test.v - name: Run allocator tests working-directory: source/modules/antono2/vkmemalloc run: v run setup.vsh --check @@ -48,3 +50,31 @@ jobs: lavapipe_icd=$(find /usr/share/vulkan/icd.d -name 'lvp_icd*.json' -print -quit) test -n "$lavapipe_icd" VK_ICD_FILENAMES="$lavapipe_icd" v run examples/buffer_suballocation + + sanitizers: + runs-on: ubuntu-24.04 + env: + VMODULES: ${{ github.workspace }}/source/modules + VULKAN_SDK: /usr + ASAN_OPTIONS: detect_leaks=0 + steps: + - uses: actions/checkout@v7 + with: + path: source/modules/antono2/vkmemalloc + - uses: actions/checkout@v7 + with: + repository: antono2/vulkan + path: source/modules/antono2/vulkan + - uses: actions/checkout@v7 + with: + repository: antono2/memory + ref: v1.4.0 + path: source/modules/antono2/memory + - uses: prantlf/setup-v-action@v4 + with: + version: 0.5.2 + - name: Install native compiler and Vulkan development library + run: sudo apt-get update && sudo apt-get install -y clang libvulkan-dev libvulkan-volk-dev + - name: Run CPU policy tests with AddressSanitizer and UndefinedBehaviorSanitizer + working-directory: source/modules/antono2/vkmemalloc + run: v -cc clang -cflags -fsanitize=address,undefined -cflags -fno-omit-frame-pointer -ldflags -fsanitize=address,undefined test . diff --git a/CHANGELOG.md b/CHANGELOG.md index dc56f19..ae39f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file. +## 2.4.0 - 2026-09-12 + +- Add explainable, deterministic memory-type ranking for GPU-only, upload, + readback, and automatic usage, with required, preferred, and avoided flags. +- Add optional `VK_EXT_memory_budget` discovery, refresh, selection policy, and + per-heap diagnostics with a portable allocator-commitment fallback. +- Add policy-based raw, buffer, dedicated-buffer, and image allocation APIs + while preserving the existing `MemType` entry points. +- Retry allocation after trimming empty blocks and fall through to compatible + lower-ranked memory types on host/device out-of-memory results. +- Record the selected heap, property flags, and containing block size in every + allocation. +- Add checked `flush`, `flush_range`, `invalidate`, and `invalidate_range` + helpers aligned to the device's `nonCoherentAtomSize`. +- Add policy-selected upload rings and slice-level flush/invalidate helpers + while retaining the coherent default constructor. +- Document the allocator from a high-level policy/planning/ownership viewpoint + and exercise policy selection, live budgets, and flushing with lavapipe. + ## 2.3.2 - 2026-09-12 - Promote the allocation-policy dependency to the production-hardened diff --git a/README.md b/README.md index f3cc4b4..038244b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,26 @@ Despite the repository name, this is not a binding to AMD's Vulkan Memory Allocator. It is a compact V-native allocator intended to remain understandable enough for examples while avoiding one Vulkan allocation per resource. +## How it fits together + +The allocator has three deliberately separate layers: + +1. **Policy** filters the memory types allowed by Vulkan, applies required + property flags, and ranks the remaining types for GPU-only, upload, or + readback use. The selected `MemoryTypeChoice` explains the heap, flags, + budget state, and score. +2. **Block planning** uses `antono2.memory.RangeAllocator` to place compatible + buffers into larger memory-type-specific blocks. This CPU-only layer is + deterministic and independently tested. +3. **Vulkan ownership** creates, maps, binds, and frees `VkDeviceMemory` while + `AllocationInfo` keeps the selected type, heap, properties, block size, and + private ownership record together. + +Images remain dedicated. That conservative rule avoids hiding the additional +tiling and buffer-image granularity rules that a safe image suballocator would +need to model. Existing `MemType` APIs remain available for short examples; +new applications should normally use `AllocationOptions`. + ## Install ```sh v install antono2.vkmemalloc @@ -60,6 +80,40 @@ The lower-level `allocate()` method also uses an isolated block because raw `VkMemoryRequirements` do not identify whether the caller will bind a buffer or image. Use `create_buffer()` when automatic buffer suballocation is desired. +## Policy-based allocation + +The policy API expresses how the resource will be used and keeps hard +requirements distinct from preferences: + +```v +options := vma.AllocationOptions{ + usage: .upload + // These are optional refinements. Required flags are never dropped. + preferred_flags: vk.MemoryPropertyFlags(vk.MemoryPropertyFlagBits.host_cached) +} +result := allocator.create_buffer_with_options(&buffer_info, options, &buffer, + mut allocation) +``` + +- `.gpu_only` requires device-local memory. +- `.upload` requires host-visible memory and prefers coherent, device-local + types. +- `.readback` requires host-visible memory and prefers cached, coherent types. +- `.automatic` has no implicit hard requirement and prefers device-local + memory. + +`required_flags` is a hard filter. `preferred_flags` improves a candidate's +rank, while `avoided_flags` lowers it without making the type unusable. The +default `.prefer_within` budget policy moves a heap with enough estimated room +ahead of an otherwise better match. `.require_within` filters over-budget +heaps for new Vulkan blocks while still permitting reuse of compatible blocks +that are already committed. `.ignore` ranks without considering room. + +Use `allocator.select_memory_type(...)` when you need to inspect the choice +before creating a resource. Policy allocation tries compatible types in rank +order after reclaiming empty cached blocks on memory pressure. It never relaxes +required flags. + Allocate and bind a buffer, checking the returned Vulkan result: ```v @@ -79,9 +133,18 @@ if allocator.map(mut allocation, &mapped) != .success { return error('could not map buffer memory') } // Copy data to mapped here. +if allocator.flush(allocation) != .success { + return error('could not flush buffer memory') +} allocator.unmap(mut allocation) ``` +For host-coherent memory, `flush()` and `invalidate()` are checked no-ops. For +non-coherent memory they call Vulkan with ranges expanded to +`nonCoherentAtomSize`. The `_range` variants accept allocation-relative offsets +and sizes. Flush after host writes before device access; invalidate only after +device writes have completed and before reading them on the host. + Destroy the Vulkan buffer or image before freeing its memory: ```v @@ -134,6 +197,39 @@ reported, `trim_empty_blocks()` can return it to Vulkan before retrying another memory class. The allocator may still create a new compatible block when its configured block limit and the Vulkan device allow it. +### Heap budgets + +`VK_EXT_memory_budget` exposes driver estimates for current heap usage and the +amount the process can reasonably consume. The current integration uses Vulkan +1.1's properties query. Opt in only after confirming and enabling the device +extension: + +```v +budget_supported := vma.supports_memory_budget(physical_device) +// Add vk.ext_memory_budget_extension_name to VkDeviceCreateInfo when true. +mut allocator := vma.new(vma.AllocatorCreateInfo{ + physical_device: physical_device + device: device + memory_budget_enabled: budget_supported +}) + +_ = allocator.refresh_memory_budget() +for heap in allocator.memory_heaps() { + println('heap ${heap.heap_index}: ${heap.usage}/${heap.budget}') +} +``` + +The [runnable example](examples/buffer_suballocation/main.v) shows the complete +extension-name array and logical-device creation sequence. + +`new()` obtains an initial enabled budget snapshot. Call +`refresh_memory_budget()` periodically (for example, once per frame or every +few seconds); policy selection uses the latest snapshot without adding a driver +query to every allocation. Without the extension, the same APIs fall back to +physical heap sizes and this allocator's own committed blocks. Budgets are +changing estimates, not reservations; Vulkan allocation can still fail and the +returned `vk.Result` remains authoritative. + ## Persistent upload ring `UploadRing` owns one dedicated, persistently mapped, host-coherent staging @@ -149,6 +245,9 @@ slice := uploads.allocate(4096, 256) or { panic(err) } unsafe { copy(&u8(slice.data), source.data, source.len) } +if uploads.flush(slice) != .success { + return error('could not flush upload slice') +} // Record a copy from uploads.buffer at slice.offset, submit it, and keep slice. // After the protecting fence or timeline value has completed: @@ -163,7 +262,13 @@ must not retire a slice until the GPU has finished reading it. Call `uploads.stats()` returns `UploadRingStats`, keeping this module's public API independent of the internal allocation-policy type. -## Memory classes +`new_upload_ring()` deliberately requires host-coherent memory, so its +`flush()` calls are checked no-ops. Advanced callers can use +`new_upload_ring_with_options(..., AllocationOptions{ usage: .upload })` to +permit other host-visible types; flushing each written slice then handles +non-coherent memory correctly. + +## Legacy memory classes - `.staging` requires host-visible and host-coherent memory and may be mapped. - `.gpu` requires device-local memory and normally cannot be mapped. @@ -181,8 +286,10 @@ independent of the internal allocation-policy type. when multiple threads can allocate or free concurrently. - Concurrently mapped allocations in one shared block reuse a single underlying Vulkan mapping. Each successful `map()` must have a matching `unmap()`. -- The allocator does not relocate live resources, enforce heap budgets, choose - between equivalent heaps, or automatically flush non-coherent memory. +- The allocator does not relocate live resources or suballocate images. +- Heap budgets guide selection but cannot enforce a process-wide or system-wide + limit because other allocators can change process usage and external system + activity can change the budget concurrently. - Vulkan objects must not outlive the memory bound to them. All allocation and binding functions return `vk.Result`; callers should handle @@ -196,9 +303,11 @@ The bookkeeping tests do not require a Vulkan-capable GPU: v test . ``` -The runnable example creates two real buffers, verifies that they share a -memory block, maps one range, creates a dedicated image, then exercises a -persistently mapped upload ring through wraparound and FIFO retirement: +The runnable example enables live budgets when available, creates two real +policy-selected upload buffers, verifies that they share a memory block, maps +and flushes them, creates a dedicated GPU-only image, prints heap diagnostics, +then exercises a persistently mapped upload ring through wraparound and FIFO +retirement: ```sh v run examples/buffer_suballocation diff --git a/block_pool.v b/block_pool.v index f6a7597..4670022 100644 --- a/block_pool.v +++ b/block_pool.v @@ -1,6 +1,7 @@ module vkmemalloc import antono2.memory +import antono2.vulkan as vk struct BlockReservation { owner voidptr @@ -51,7 +52,7 @@ fn new_memory_block_pool(default_block_size u64, max_blocks int) !&MemoryBlockPo } return &MemoryBlockPool{ default_block_size: default_block_size - max_blocks: max_blocks + max_blocks: max_blocks } } @@ -77,6 +78,38 @@ fn (pool &MemoryBlockPool) block_is_dedicated(block_id u64) ?bool { return none } +fn (pool &MemoryBlockPool) block_capacity(block_id u64) ?u64 { + for block in pool.blocks { + if block.id == block_id { + return block.capacity + } + } + return none +} + +fn (pool &MemoryBlockPool) block_memory_type(block_id u64) ?u32 { + for block in pool.blocks { + if block.id == block_id { + return block.memory_type + } + } + return none +} + +fn (pool &MemoryBlockPool) heap_stats(props &vk.PhysicalDeviceMemoryProperties, heap_index u32) (u64, u64) { + mut committed := u64(0) + mut used := u64(0) + for block in pool.blocks { + if block.memory_type >= props.memoryTypeCount + || props.memoryTypes[block.memory_type].heapIndex != heap_index { + continue + } + committed += block.capacity + used += block.ranges.stats().used + } + return committed, used +} + fn (pool &MemoryBlockPool) recommended_block_size(requested_size u64) !u64 { if requested_size == 0 { return error('allocation size must be greater than zero') @@ -105,11 +138,11 @@ fn (mut pool MemoryBlockPool) add_block_with_policy(memory_type u32, capacity u6 } id := pool.next_block_id() pool.blocks << MemoryBlock{ - id: id + id: id memory_type: memory_type - capacity: capacity - dedicated: dedicated - ranges: memory.new_range_allocator(capacity) + capacity: capacity + dedicated: dedicated + ranges: memory.new_range_allocator(capacity) } return id } @@ -130,12 +163,12 @@ fn (mut pool MemoryBlockPool) reserve(memory_type u32, size u64, alignment u64) } if allocation := block.ranges.allocate(size, alignment) { return BlockReservation{ - owner: pool - block_id: block.id - allocation: allocation + owner: pool + block_id: block.id + allocation: allocation memory_type: memory_type - offset: allocation.offset - size: allocation.size + offset: allocation.offset + size: allocation.size } } } @@ -155,12 +188,12 @@ fn (mut pool MemoryBlockPool) reserve_from_block(block_id u64, size u64, alignme } allocation := block.ranges.allocate(size, alignment)! return BlockReservation{ - owner: pool - block_id: block.id - allocation: allocation + owner: pool + block_id: block.id + allocation: allocation memory_type: block.memory_type - offset: allocation.offset - size: allocation.size + offset: allocation.offset + size: allocation.size } } return error('memory block does not exist') @@ -248,14 +281,14 @@ fn (pool &MemoryBlockPool) collect_stats(memory_type u32, filter_by_memory_type } } return BlockPoolStats{ - block_count: block_count - allocation_count: allocation_count - committed: committed - used: used - free: committed - used - free_range_count: free_range_count + block_count: block_count + allocation_count: allocation_count + committed: committed + used: used + free: committed - used + free_range_count: free_range_count largest_free_range: largest_free_range - empty_block_count: empty_block_count + empty_block_count: empty_block_count } } diff --git a/block_pool_test.v b/block_pool_test.v index 079a5ba..7042874 100644 --- a/block_pool_test.v +++ b/block_pool_test.v @@ -120,12 +120,12 @@ fn test_block_pool_rejects_foreign_forged_and_stale_reservations() { assert !first_pool.contains(foreign) assert !first_pool.release(foreign) assert !first_pool.release(BlockReservation{ - owner: reservation.owner - block_id: reservation.block_id - allocation: reservation.allocation + owner: reservation.owner + block_id: reservation.block_id + allocation: reservation.allocation memory_type: reservation.memory_type - offset: reservation.offset + 1 - size: reservation.size + offset: reservation.offset + 1 + size: reservation.size }) released := first_pool.release(reservation) assert released diff --git a/examples/buffer_suballocation/main.v b/examples/buffer_suballocation/main.v index e0bdb6e..a100e4c 100644 --- a/examples/buffer_suballocation/main.v +++ b/examples/buffer_suballocation/main.v @@ -11,73 +11,78 @@ fn require_success(result vk.Result, operation string) ! { fn first_physical_device(instance vk.Instance) !vk.PhysicalDevice { mut count := u32(0) - require_success(vk.enumerate_physical_devices(instance, &count, unsafe { nil }), - 'enumerate physical device count')! + require_success(vk.enumerate_physical_devices(instance, &count, unsafe { nil }), 'enumerate physical device count')! if count == 0 { return error('no Vulkan physical device is available') } mut devices := unsafe { []vk.PhysicalDevice{len: int(count)} } - require_success(vk.enumerate_physical_devices(instance, &count, devices.data), - 'enumerate physical devices')! + require_success(vk.enumerate_physical_devices(instance, &count, devices.data), 'enumerate physical devices')! return devices[0] } fn run() ! { require_success(vk.initialize_loader(), 'initialize Vulkan loader')! application_info := vk.ApplicationInfo{ - pApplicationName: c'vkmemalloc suballocation example' + pApplicationName: c'vkmemalloc suballocation example' applicationVersion: 1 - pEngineName: c'none' - apiVersion: vk.api_version_1_1 + pEngineName: c'none' + apiVersion: vk.api_version_1_1 } instance_info := vk.InstanceCreateInfo{ pApplicationInfo: &application_info } mut instance := vk.Instance(unsafe { nil }) - require_success(vk.create_instance(&instance_info, unsafe { nil }, &instance), - 'create Vulkan instance')! + require_success(vk.create_instance(&instance_info, unsafe { nil }, &instance), 'create Vulkan instance')! defer { vk.destroy_instance(instance, unsafe { nil }) } vk.load_instance_commands(instance) physical_device := first_physical_device(instance)! + memory_budget_supported := vma.supports_memory_budget(physical_device) + mut device_extensions := []&char{} + if memory_budget_supported { + device_extensions << vk.ext_memory_budget_extension_name + } mut priority := f32(1) queue_info := vk.DeviceQueueCreateInfo{ queueFamilyIndex: 0 - queueCount: 1 + queueCount: 1 pQueuePriorities: &priority } device_info := vk.DeviceCreateInfo{ queueCreateInfoCount: 1 - pQueueCreateInfos: &queue_info + pQueueCreateInfos: &queue_info + enabledExtensionCount: u32(device_extensions.len) + ppEnabledExtensionNames: device_extensions.data } mut device := vk.Device(unsafe { nil }) - require_success(vk.create_device(physical_device, &device_info, unsafe { nil }, &device), - 'create Vulkan device')! + require_success(vk.create_device(physical_device, &device_info, unsafe { nil }, &device), 'create Vulkan device')! defer { vk.destroy_device(device, unsafe { nil }) } vk.load_device_commands(device) mut allocator := vma.new(vma.AllocatorCreateInfo{ - physical_device: physical_device - device: device + physical_device: physical_device + device: device preferred_block_size: 4096 + memory_budget_enabled: memory_budget_supported }) defer { allocator.destroy() } buffer_info := vk.BufferCreateInfo{ - size: 512 - usage: u32(vk.BufferUsageFlagBits.transfer_src) + size: 512 + usage: u32(vk.BufferUsageFlagBits.transfer_src) sharingMode: .exclusive } mut first_buffer := vk.Buffer(unsafe { nil }) mut first_allocation := vma.AllocationInfo{} - require_success(allocator.create_buffer(&buffer_info, .staging, &first_buffer, mut - first_allocation), 'create first staging buffer')! + require_success(allocator.create_buffer_with_options(&buffer_info, vma.AllocationOptions{ + usage: .upload + }, &first_buffer, mut first_allocation), 'create first upload buffer')! defer { if !isnil(first_buffer) { vk.destroy_buffer(device, first_buffer, unsafe { nil }) @@ -89,8 +94,9 @@ fn run() ! { mut second_buffer := vk.Buffer(unsafe { nil }) mut second_allocation := vma.AllocationInfo{} - require_success(allocator.create_buffer(&buffer_info, .staging, &second_buffer, mut - second_allocation), 'create second staging buffer')! + require_success(allocator.create_buffer_with_options(&buffer_info, vma.AllocationOptions{ + usage: .upload + }, &second_buffer, mut second_allocation), 'create second upload buffer')! defer { if !isnil(second_buffer) { vk.destroy_buffer(device, second_buffer, unsafe { nil }) @@ -112,37 +118,39 @@ fn run() ! { mut first_mapped := voidptr(unsafe { nil }) mut second_mapped := voidptr(unsafe { nil }) require_success(allocator.map(mut first_allocation, &first_mapped), 'map first staging buffer')! - require_success(allocator.map(mut second_allocation, &second_mapped), - 'map second staging buffer')! + require_success(allocator.map(mut second_allocation, &second_mapped), 'map second staging buffer')! unsafe { *(&u8(first_mapped)) = 21 *(&u8(second_mapped)) = 42 } + require_success(allocator.flush(first_allocation), 'flush first upload buffer')! + require_success(allocator.flush(second_allocation), 'flush second upload buffer')! assert usize(second_mapped) - usize(first_mapped) == second_allocation.offset - first_allocation.offset allocator.unmap(mut first_allocation) allocator.unmap(mut second_allocation) println('shared staging suballocations mapped concurrently') image_info := vk.ImageCreateInfo{ - imageType: ._2d - format: .r8g8b8a8_unorm - extent: vk.Extent3D{ - width: 16 + imageType: ._2d + format: .r8g8b8a8_unorm + extent: vk.Extent3D{ + width: 16 height: 16 - depth: 1 + depth: 1 } - mipLevels: 1 - arrayLayers: 1 - samples: ._1 - tiling: .optimal - usage: u32(vk.ImageUsageFlagBits.transfer_dst) - sharingMode: .exclusive + mipLevels: 1 + arrayLayers: 1 + samples: ._1 + tiling: .optimal + usage: u32(vk.ImageUsageFlagBits.transfer_dst) + sharingMode: .exclusive initialLayout: .undefined } mut image := vk.Image(unsafe { nil }) mut image_allocation := vma.AllocationInfo{} - require_success(allocator.create_image(&image_info, .gpu, &image, mut image_allocation), - 'create dedicated image')! + require_success(allocator.create_image_with_options(&image_info, vma.AllocationOptions{ + usage: .gpu_only + }, &image, mut image_allocation), 'create dedicated image')! defer { if !isnil(image) { vk.destroy_image(device, image, unsafe { nil }) @@ -155,6 +163,11 @@ fn run() ! { assert image_allocation.offset == 0 assert allocator.stats().block_count == 2 println('optimal image uses an isolated dedicated block') + for heap in allocator.memory_heaps() { + assert heap.budget > 0 + assert heap.allocator_committed >= heap.allocator_used + println('heap ${heap.heap_index}: budget=${heap.budget}, usage=${heap.usage}, reported=${heap.budget_reported}') + } vk.destroy_image(device, image, unsafe { nil }) image = vk.Image(unsafe { nil }) @@ -172,7 +185,9 @@ fn run() ! { assert allocator.trim_empty_blocks() == 1 assert allocator.stats().block_count == 0 - mut uploads := vma.new_upload_ring(mut allocator, 1024)! + mut uploads := vma.new_upload_ring_with_options(mut allocator, 1024, vma.AllocationOptions{ + usage: .upload + })! defer { _ = uploads.destroy() } @@ -182,6 +197,8 @@ fn run() ! { *(&u8(first_upload.data)) = 21 *(&u8(second_upload.data)) = 22 } + require_success(uploads.flush(first_upload), 'flush first upload slice')! + require_success(uploads.flush(second_upload), 'flush second upload slice')! first_retired := uploads.retire(first_upload) assert first_retired wrapped_upload := uploads.allocate(300, 16)! @@ -189,6 +206,7 @@ fn run() ! { unsafe { *(&u8(wrapped_upload.data)) = 23 } + require_success(uploads.flush(wrapped_upload), 'flush wrapped upload slice')! assert !uploads.retire(wrapped_upload) second_retired := uploads.retire(second_upload) assert second_retired diff --git a/mapped_memory.v b/mapped_memory.v new file mode 100644 index 0000000..24755ca --- /dev/null +++ b/mapped_memory.v @@ -0,0 +1,86 @@ +module vkmemalloc + +import antono2.vulkan as vk + +struct NormalizedMappedRange { + offset u64 + size u64 +} + +fn normalize_mapped_range(allocation_offset u64, allocation_size u64, block_size u64, relative_offset u64, size u64, atom_size u64) ?NormalizedMappedRange { + if size == 0 || atom_size == 0 || relative_offset > allocation_size + || size > allocation_size - relative_offset { + return none + } + absolute_offset := allocation_offset + relative_offset + if absolute_offset < allocation_offset || absolute_offset > block_size + || size > block_size - absolute_offset { + return none + } + start := absolute_offset - absolute_offset % atom_size + end := absolute_offset + size + rounded_end := if end % atom_size == 0 { + end + } else if end > max_u64 - (atom_size - end % atom_size) { + return none + } else { + end + atom_size - end % atom_size + } + return NormalizedMappedRange{ + offset: start + size: if rounded_end >= block_size { + vk.whole_size + } else { + rounded_end - start + } + } +} + +fn (a &Allocator) mapped_range(alloc_info AllocationInfo, relative_offset u64, size u64) ?vk.MappedMemoryRange { + if !a.owns_allocation(alloc_info) || !alloc_info.mapped + || !has_memory_flags(alloc_info.property_flags, memory_flag(.host_visible)) { + return none + } + normalized := normalize_mapped_range(alloc_info.offset, alloc_info.size, alloc_info.block_size, relative_offset, size, a.non_coherent_atom_size) or { return none } + return vk.MappedMemoryRange{ + memory: vk.DeviceMemory(alloc_info.memory) + offset: normalized.offset + size: normalized.size + } +} + +// flush makes host writes in the whole allocation available to the device. +// Host-coherent memory succeeds without issuing a Vulkan call. +pub fn (a &Allocator) flush(alloc_info AllocationInfo) vk.Result { + return a.flush_range(alloc_info, 0, alloc_info.size) +} + +// flush_range makes one allocation-relative host-written range available to +// the device. The Vulkan range is expanded to nonCoherentAtomSize boundaries. +pub fn (a &Allocator) flush_range(alloc_info AllocationInfo, relative_offset u64, size u64) vk.Result { + range := a.mapped_range(alloc_info, relative_offset, size) or { + return .error_memory_map_failed + } + if has_memory_flags(alloc_info.property_flags, memory_flag(.host_coherent)) { + return .success + } + return vk.flush_mapped_memory_ranges(a.create_info.device, 1, &range) +} + +// invalidate makes device writes in the whole allocation visible to the host. +// Synchronize device access before calling it. +pub fn (a &Allocator) invalidate(alloc_info AllocationInfo) vk.Result { + return a.invalidate_range(alloc_info, 0, alloc_info.size) +} + +// invalidate_range makes one allocation-relative device-written range visible +// to the host and expands it to nonCoherentAtomSize boundaries. +pub fn (a &Allocator) invalidate_range(alloc_info AllocationInfo, relative_offset u64, size u64) vk.Result { + range := a.mapped_range(alloc_info, relative_offset, size) or { + return .error_memory_map_failed + } + if has_memory_flags(alloc_info.property_flags, memory_flag(.host_coherent)) { + return .success + } + return vk.invalidate_mapped_memory_ranges(a.create_info.device, 1, &range) +} diff --git a/mapped_memory_test.v b/mapped_memory_test.v new file mode 100644 index 0000000..e3dc4e7 --- /dev/null +++ b/mapped_memory_test.v @@ -0,0 +1,64 @@ +module vkmemalloc + +import antono2.vulkan as vk + +fn mapped_test_memory(value usize) vk.DeviceMemory { + return unsafe { voidptr(value) } +} + +fn test_mapped_range_aligns_both_ends_to_atom_size() { + range := normalize_mapped_range(128, 512, 1024, 3, 130, 64) or { + panic('range should be valid') + } + assert range.offset == 128 + assert range.size == 192 +} + +fn test_mapped_range_uses_whole_size_at_memory_end() { + range := normalize_mapped_range(768, 256, 1024, 240, 16, 64) or { + panic('range should be valid') + } + assert range.offset == 960 + assert range.size == vk.whole_size +} + +fn test_mapped_range_rejects_empty_or_out_of_allocation_ranges() { + if _ := normalize_mapped_range(128, 256, 1024, 0, 0, 64) { + assert false, 'empty ranges must be rejected' + } + if _ := normalize_mapped_range(128, 256, 1024, 250, 7, 64) { + assert false, 'ranges must remain inside the allocation' + } + if _ := normalize_mapped_range(900, 200, 1024, 0, 200, 64) { + assert false, 'allocations must remain inside their memory block' + } +} + +fn test_coherent_flush_validates_ownership_without_a_driver_call() { + mut props := vk.PhysicalDeviceMemoryProperties{} + props.memoryHeapCount = 1 + props.memoryHeaps[0].size = 256 + props.memoryTypeCount = 1 + props.memoryTypes[0] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) + heapIndex: 0 + } + mut planner := new_memory_block_pool(256, 1) or { panic(err) } + block_id := planner.add_block(0, 256) or { panic(err) } + reservation := planner.reserve(0, 64, 1) or { panic(err) } + mut allocator := Allocator{ + props: props + non_coherent_atom_size: 64 + planner: planner + } + assert allocator.remember_block(mapped_test_memory(1), block_id) + mut allocation := AllocationInfo{} + allocator.populate_allocation(mut allocation, mapped_test_memory(1), reservation) + allocation.mapped = true + assert allocator.flush_range(allocation, 1, 1) == .success + assert allocator.invalidate(allocation) == .success + assert allocator.flush_range(allocation, allocation.size, 1) == .error_memory_map_failed + + allocation.property_flags = 0 + assert allocator.flush(allocation) == .error_memory_map_failed +} diff --git a/memory_policy.v b/memory_policy.v new file mode 100644 index 0000000..e137b5f --- /dev/null +++ b/memory_policy.v @@ -0,0 +1,357 @@ +module vkmemalloc + +import antono2.vulkan as vk + +// MemoryUsage describes how an allocation is expected to move between the CPU +// and GPU. It is a policy hint; required_flags always remain mandatory. +pub enum MemoryUsage { + automatic + gpu_only + upload + readback +} + +// BudgetPolicy controls how reported or physical heap capacity affects memory +// type selection. +pub enum BudgetPolicy { + prefer_within + ignore + require_within +} + +// AllocationOptions describes required and preferred memory properties. The +// usage profile supplies sensible defaults, while the explicit flag sets let +// callers refine them for specialized resources. +pub struct AllocationOptions { +pub: + usage MemoryUsage + required_flags vk.MemoryPropertyFlags + preferred_flags vk.MemoryPropertyFlags + avoided_flags vk.MemoryPropertyFlags + budget_policy BudgetPolicy = .prefer_within +} + +// MemoryTypeChoice explains why a Vulkan memory type was selected. +pub struct MemoryTypeChoice { +pub: + index u32 + heap_index u32 + property_flags vk.MemoryPropertyFlags + heap_size u64 + heap_budget u64 + heap_usage u64 + remaining_budget u64 + within_budget bool + budget_reported bool + preference_score int +} + +// MemoryHeapStats combines Vulkan heap capacity/budget information with the +// blocks currently committed by this allocator. +pub struct MemoryHeapStats { +pub: + heap_index u32 + size u64 + budget u64 + usage u64 + remaining_budget u64 + allocator_committed u64 + allocator_used u64 + device_local bool + budget_reported bool +} + +struct HeapBudgetSnapshot { + reported bool + budgets []u64 + usages []u64 +} + +// supports_memory_budget reports whether a physical device exposes +// VK_EXT_memory_budget through this allocator's Vulkan 1.1 query path. Call it +// after the Vulkan loader and instance commands are initialized, and enable +// that device extension before opting the allocator into live budget queries. +pub fn supports_memory_budget(physical_device vk.PhysicalDevice) bool { + mut device_properties := vk.PhysicalDeviceProperties{} + vk.get_physical_device_properties(physical_device, mut &device_properties) + if device_properties.apiVersion < vk.api_version_1_1 { + return false + } + for { + mut count := u32(0) + mut no_properties := unsafe { nil } + if vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut no_properties) != .success || count == 0 { + return false + } + mut properties := []vk.ExtensionProperties{len: int(count)} + result := vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut properties[0]) + if result == .incomplete { + continue + } + if result != .success { + return false + } + for index in 0 .. int(count) { + name := unsafe { cstring_to_vstring(&properties[index].extensionName[0]) } + if name == 'VK_EXT_memory_budget' { + return true + } + } + return false + } + return false +} + +fn memory_flag(flag vk.MemoryPropertyFlagBits) vk.MemoryPropertyFlags { + return vk.MemoryPropertyFlags(u32(flag)) +} + +fn has_memory_flags(flags vk.MemoryPropertyFlags, required vk.MemoryPropertyFlags) bool { + return (flags & required) == required +} + +fn memory_flag_count(flags vk.MemoryPropertyFlags) int { + mut value := u32(flags) + mut count := 0 + for value != 0 { + count += int(value & 1) + value >>= 1 + } + return count +} + +fn usage_required_flags(usage MemoryUsage) vk.MemoryPropertyFlags { + return match usage { + .gpu_only { memory_flag(.device_local) } + .upload, .readback { memory_flag(.host_visible) } + .automatic { vk.MemoryPropertyFlags(0) } + } +} + +fn usage_preference_score(usage MemoryUsage, flags vk.MemoryPropertyFlags) int { + device_local := has_memory_flags(flags, memory_flag(.device_local)) + host_coherent := has_memory_flags(flags, memory_flag(.host_coherent)) + host_cached := has_memory_flags(flags, memory_flag(.host_cached)) + device_uncached := has_memory_flags(flags, memory_flag(.device_uncached_bit_amd)) + return match usage { + .automatic { + if device_local { 16 } else { 0 } + } + .gpu_only { + if device_uncached { -4 } else { 0 } + } + .upload { + (if host_coherent { 16 } else { 0 }) + (if device_local { 8 } else { 0 }) + (if host_cached { + 2 + } else { + 0 + }) - (if device_uncached { 4 } else { 0 }) + } + .readback { + (if host_cached { 16 } else { 0 }) + (if host_coherent { 8 } else { 0 }) + (if device_local { + 2 + } else { + 0 + }) - (if device_uncached { 4 } else { 0 }) + } + } +} + +fn memory_preference_score(options AllocationOptions, flags vk.MemoryPropertyFlags) int { + preferred := memory_flag_count(flags & options.preferred_flags) + avoided := memory_flag_count(flags & options.avoided_flags) + return usage_preference_score(options.usage, flags) + preferred * 4 - avoided * 32 +} + +fn heap_budget_values(props vk.PhysicalDeviceMemoryProperties, heap_index u32, snapshot HeapBudgetSnapshot) (u64, u64, bool) { + heap_size := u64(props.memoryHeaps[heap_index].size) + if int(heap_index) < snapshot.budgets.len && int(heap_index) < snapshot.usages.len { + budget := if snapshot.budgets[heap_index] > 0 { + snapshot.budgets[heap_index] + } else { + heap_size + } + return budget, snapshot.usages[heap_index], snapshot.reported + } + return heap_size, 0, false +} + +fn memory_choice_is_better(candidate MemoryTypeChoice, current MemoryTypeChoice, policy BudgetPolicy) bool { + if policy == .prefer_within && candidate.within_budget != current.within_budget { + return candidate.within_budget + } + if candidate.preference_score != current.preference_score { + return candidate.preference_score > current.preference_score + } + if policy != .ignore && candidate.remaining_budget != current.remaining_budget { + return candidate.remaining_budget > current.remaining_budget + } + return candidate.index < current.index +} + +fn ranked_memory_types(props vk.PhysicalDeviceMemoryProperties, type_bits u32, request_size u64, options AllocationOptions, snapshot HeapBudgetSnapshot) []MemoryTypeChoice { + required := usage_required_flags(options.usage) | options.required_flags + mut choices := []MemoryTypeChoice{} + for index in 0 .. int(props.memoryTypeCount) { + if index >= int(vk.max_memory_types) || (type_bits & (u32(1) << u32(index))) == 0 { + continue + } + memory_type := props.memoryTypes[index] + if !has_memory_flags(memory_type.propertyFlags, required) + || memory_type.heapIndex >= props.memoryHeapCount { + continue + } + heap_size := u64(props.memoryHeaps[memory_type.heapIndex].size) + budget, usage, reported := heap_budget_values(props, memory_type.heapIndex, snapshot) + remaining := if usage < budget { budget - usage } else { u64(0) } + within_budget := request_size <= remaining + if options.budget_policy == .require_within && !within_budget { + continue + } + choice := MemoryTypeChoice{ + index: u32(index) + heap_index: memory_type.heapIndex + property_flags: memory_type.propertyFlags + heap_size: heap_size + heap_budget: budget + heap_usage: usage + remaining_budget: remaining + within_budget: within_budget + budget_reported: reported + preference_score: memory_preference_score(options, memory_type.propertyFlags) + } + mut inserted := false + for position, existing in choices { + if memory_choice_is_better(choice, existing, options.budget_policy) { + choices.insert(position, choice) + inserted = true + break + } + } + if !inserted { + choices << choice + } + } + return choices +} + +// select_memory_type applies the portable usage/property policy using physical +// heap sizes. Allocator.select_memory_type additionally uses live heap budgets +// when VK_EXT_memory_budget integration was enabled at allocator creation. +pub fn select_memory_type(props vk.PhysicalDeviceMemoryProperties, type_bits u32, request_size u64, options AllocationOptions) ?MemoryTypeChoice { + choices := ranked_memory_types(props, type_bits, request_size, options, HeapBudgetSnapshot{}) + if choices.len == 0 { + return none + } + return choices[0] +} + +fn (a &Allocator) heap_budget_snapshot() HeapBudgetSnapshot { + mut budgets := []u64{len: int(a.props.memoryHeapCount)} + mut usages := []u64{len: int(a.props.memoryHeapCount)} + for heap_index in 0 .. int(a.props.memoryHeapCount) { + if a.memory_budget_reported && heap_index < a.heap_budgets.len + && heap_index < a.heap_usages.len { + budgets[heap_index] = a.heap_budgets[heap_index] + usages[heap_index] = a.heap_usages[heap_index] + continue + } + budgets[heap_index] = u64(a.props.memoryHeaps[heap_index].size) + if !isnil(a.planner) { + committed, _ := a.planner.heap_stats(&a.props, u32(heap_index)) + usages[heap_index] = committed + } + } + return HeapBudgetSnapshot{ + reported: a.memory_budget_reported + budgets: budgets + usages: usages + } +} + +// refresh_memory_budget refreshes VK_EXT_memory_budget estimates when the +// allocator was created with memory_budget_enabled. It returns false when the +// optional integration is unavailable; physical heap sizes remain usable. +pub fn (mut a Allocator) refresh_memory_budget() bool { + if !a.create_info.memory_budget_enabled || a.api_version < vk.api_version_1_1 { + return false + } + mut budget := vk.PhysicalDeviceMemoryBudgetPropertiesEXT{} + mut properties := vk.PhysicalDeviceMemoryProperties2{ + pNext: voidptr(&budget) + } + vk.get_physical_device_memory_properties2(a.create_info.physical_device, mut &properties) + a.props = properties.memoryProperties + heap_count := int(a.props.memoryHeapCount) + a.heap_budgets = []u64{len: heap_count} + a.heap_usages = []u64{len: heap_count} + mut reported := false + for heap_index in 0 .. heap_count { + a.heap_budgets[heap_index] = u64(budget.heapBudget[heap_index]) + a.heap_usages[heap_index] = u64(budget.heapUsage[heap_index]) + if a.heap_budgets[heap_index] > 0 { + reported = true + } + } + a.memory_budget_reported = reported + return reported +} + +// select_memory_type ranks every compatible type and returns an explainable +// choice. It uses the latest refreshed budget snapshot when that optional +// integration is enabled; otherwise allocator-owned commitment is used. +pub fn (mut a Allocator) select_memory_type(type_bits u32, request_size u64, options AllocationOptions) ?MemoryTypeChoice { + choices := a.rank_memory_types(type_bits, request_size, options) + if choices.len == 0 { + return none + } + return choices[0] +} + +fn (mut a Allocator) rank_memory_types(type_bits u32, request_size u64, options AllocationOptions) []MemoryTypeChoice { + return ranked_memory_types(a.props, type_bits, request_size, options, a.heap_budget_snapshot()) +} + +// A buffer can reuse a compatible block without increasing heap usage. Keep +// over-budget candidates available for that reuse even when new block creation +// is forbidden by require_within. +fn (mut a Allocator) rank_buffer_memory_types(type_bits u32, request_size u64, options AllocationOptions) []MemoryTypeChoice { + if options.budget_policy != .require_within { + return a.rank_memory_types(type_bits, request_size, options) + } + return a.rank_memory_types(type_bits, request_size, AllocationOptions{ + usage: options.usage + required_flags: options.required_flags + preferred_flags: options.preferred_flags + avoided_flags: options.avoided_flags + budget_policy: .prefer_within + }) +} + +// memory_heaps returns one diagnostics record per Vulkan memory heap. Reported +// budget/usage values come from VK_EXT_memory_budget when enabled; the portable +// fallback uses heap size and this allocator's own committed blocks. +pub fn (a &Allocator) memory_heaps() []MemoryHeapStats { + snapshot := a.heap_budget_snapshot() + mut heaps := []MemoryHeapStats{cap: int(a.props.memoryHeapCount)} + for heap_index in 0 .. int(a.props.memoryHeapCount) { + committed, used := if isnil(a.planner) { + u64(0), u64(0) + } else { + a.planner.heap_stats(&a.props, u32(heap_index)) + } + budget, usage, reported := heap_budget_values(a.props, u32(heap_index), snapshot) + heaps << MemoryHeapStats{ + heap_index: u32(heap_index) + size: u64(a.props.memoryHeaps[heap_index].size) + budget: budget + usage: usage + remaining_budget: if usage < budget { budget - usage } else { u64(0) } + allocator_committed: committed + allocator_used: used + device_local: (a.props.memoryHeaps[heap_index].flags & u32(vk.MemoryHeapFlagBits.device_local)) != 0 + budget_reported: reported + } + } + return heaps +} diff --git a/memory_policy_test.v b/memory_policy_test.v new file mode 100644 index 0000000..aad7e0f --- /dev/null +++ b/memory_policy_test.v @@ -0,0 +1,172 @@ +module vkmemalloc + +import antono2.vulkan as vk + +fn policy_test_memory(value usize) vk.DeviceMemory { + return unsafe { voidptr(value) } +} + +fn policy_test_properties() vk.PhysicalDeviceMemoryProperties { + mut props := vk.PhysicalDeviceMemoryProperties{} + props.memoryHeapCount = 2 + props.memoryHeaps[0] = vk.MemoryHeap{ + size: 256 + flags: u32(vk.MemoryHeapFlagBits.device_local) + } + props.memoryHeaps[1] = vk.MemoryHeap{ + size: 1024 + } + props.memoryTypeCount = 4 + props.memoryTypes[0] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.device_local) + heapIndex: 0 + } + props.memoryTypes[1] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) + heapIndex: 1 + } + props.memoryTypes[2] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_cached) + heapIndex: 1 + } + props.memoryTypes[3] = vk.MemoryType{ + propertyFlags: u32(vk.MemoryPropertyFlagBits.device_local) | u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) + heapIndex: 0 + } + return props +} + +fn test_memory_policy_selects_usage_specific_properties() { + props := policy_test_properties() + all_types := u32(0b1111) + gpu := select_memory_type(props, all_types, 16, AllocationOptions{ + usage: .gpu_only + }) or { panic('GPU memory type should exist') } + assert gpu.index == 0 + assert gpu.heap_index == 0 + + upload := select_memory_type(props, all_types, 16, AllocationOptions{ + usage: .upload + }) or { panic('upload memory type should exist') } + assert upload.index == 3 + + readback := select_memory_type(props, all_types, 16, AllocationOptions{ + usage: .readback + }) or { panic('readback memory type should exist') } + assert readback.index == 2 + assert has_memory_flags(readback.property_flags, memory_flag(.host_cached)) +} + +fn test_memory_policy_honors_required_preferred_and_avoided_flags() { + props := policy_test_properties() + host_visible := memory_flag(.host_visible) + host_coherent := memory_flag(.host_coherent) + host_cached := memory_flag(.host_cached) + choice := select_memory_type(props, 0b1110, 16, AllocationOptions{ + required_flags: host_visible + preferred_flags: host_cached + avoided_flags: host_coherent + }) or { panic('host-visible memory type should exist') } + assert choice.index == 2 + + if _ := select_memory_type(props, 0b0001, 16, AllocationOptions{ + required_flags: host_visible + }) { + assert false, 'required properties must never be dropped' + } +} + +fn test_memory_policy_prefers_or_requires_available_budget() { + props := policy_test_properties() + options := AllocationOptions{ + usage: .automatic + } + snapshot := HeapBudgetSnapshot{ + reported: true + budgets: [u64(128), 1024] + usages: [u64(120), 0] + } + preferred := ranked_memory_types(props, 0b1011, 16, options, snapshot) + assert preferred.len == 3 + assert preferred[0].index == 1 + assert preferred[0].within_budget + assert preferred[0].budget_reported + assert preferred[0].remaining_budget == 1024 + + ignored := ranked_memory_types(props, 0b1011, 16, AllocationOptions{ + usage: .automatic + budget_policy: .ignore + }, snapshot) + assert ignored[0].index == 0 + assert !ignored[0].within_budget + + required := ranked_memory_types(props, 0b1011, 16, AllocationOptions{ + usage: .automatic + budget_policy: .require_within + }, snapshot) + assert required.len == 1 + assert required[0].index == 1 +} + +fn test_memory_policy_is_deterministic_for_equal_candidates() { + mut props := policy_test_properties() + props.memoryTypes[1].propertyFlags = u32(vk.MemoryPropertyFlagBits.host_visible) + props.memoryTypes[2].propertyFlags = u32(vk.MemoryPropertyFlagBits.host_visible) + choice := select_memory_type(props, 0b0110, 1, AllocationOptions{}) or { + panic('memory type should exist') + } + assert choice.index == 1 +} + +fn test_allocator_policy_uses_owned_commitment_as_portable_budget_fallback() { + props := policy_test_properties() + mut planner := new_memory_block_pool(256, 4) or { panic(err) } + _ = planner.add_block(0, 256) or { panic(err) } + mut allocator := Allocator{ + props: props + planner: planner + } + choice := allocator.select_memory_type(0b0011, 16, AllocationOptions{}) or { + panic('a memory type should remain available') + } + assert choice.index == 1 + assert !choice.budget_reported + assert choice.within_budget + + heaps := allocator.memory_heaps() + assert heaps.len == 2 + assert heaps[0].budget == 256 + assert heaps[0].usage == 256 + assert heaps[0].remaining_budget == 0 + assert heaps[0].allocator_committed == 256 + assert heaps[0].allocator_used == 0 +} + +fn test_require_within_can_reuse_an_over_budget_buffer_block() { + props := policy_test_properties() + mut planner := new_memory_block_pool(256, 1) or { panic(err) } + block_id := planner.add_block(0, 256) or { panic(err) } + mut allocator := Allocator{ + props: props + planner: planner + } + assert allocator.remember_block(policy_test_memory(1), block_id) + options := AllocationOptions{ + usage: .gpu_only + budget_policy: .require_within + } + choices := allocator.rank_buffer_memory_types(0b0001, 16, options) + assert choices.len == 1 + assert !choices[0].within_budget + mut requirements := vk.MemoryRequirements{ + size: 16 + alignment: 8 + memoryTypeBits: 0b0001 + } + mut allocation := AllocationInfo{} + result := allocator.allocate_from_choices(mut requirements, choices, unsafe { nil }, false, options.budget_policy, mut allocation) + assert result == .success + assert allocation.memory == voidptr(policy_test_memory(1)) + assert allocation.block_size == 256 + assert allocator.release(mut allocation) +} diff --git a/upload_ring.v b/upload_ring.v index 910337a..aed272a 100644 --- a/upload_ring.v +++ b/upload_ring.v @@ -37,7 +37,7 @@ pub: mut: allocator &Allocator = unsafe { nil } backing AllocationInfo - mapped voidptr = unsafe { nil } + mapped voidptr = unsafe { nil } ranges &memory.RingAllocator = unsafe { nil } destroyed bool } @@ -54,8 +54,8 @@ pub fn new_upload_ring(mut allocator Allocator, capacity u64) !&UploadRing { } } buffer_info := vk.BufferCreateInfo{ - size: capacity - usage: u32(vk.BufferUsageFlagBits.transfer_src) + size: capacity + usage: u32(vk.BufferUsageFlagBits.transfer_src) sharingMode: .exclusive } mut buffer := vk.Buffer(unsafe { nil }) @@ -64,6 +64,44 @@ pub fn new_upload_ring(mut allocator Allocator, capacity u64) !&UploadRing { if result != .success { return error('could not create upload buffer: ${result}') } + return finish_upload_ring(mut allocator, capacity, buffer, backing) +} + +// new_upload_ring_with_options creates an upload ring with policy-selected +// host-visible memory. Prefer usage .upload; callers using a non-coherent type +// must flush each written slice before device access. +pub fn new_upload_ring_with_options(mut allocator Allocator, capacity u64, options AllocationOptions) !&UploadRing { + if capacity == 0 { + return error('upload ring capacity must be greater than zero') + } + $if x32 { + if capacity > u64(max_u32) { + return error('upload ring capacity exceeds the host address space') + } + } + buffer_info := vk.BufferCreateInfo{ + size: capacity + usage: u32(vk.BufferUsageFlagBits.transfer_src) + sharingMode: .exclusive + } + effective_options := AllocationOptions{ + usage: options.usage + required_flags: options.required_flags | memory_flag(.host_visible) + preferred_flags: options.preferred_flags + avoided_flags: options.avoided_flags + budget_policy: options.budget_policy + } + mut buffer := vk.Buffer(unsafe { nil }) + mut backing := AllocationInfo{} + result := allocator.create_dedicated_buffer_with_options(&buffer_info, effective_options, &buffer, mut backing) + if result != .success { + return error('could not create upload buffer: ${result}') + } + return finish_upload_ring(mut allocator, capacity, buffer, backing) +} + +fn finish_upload_ring(mut allocator Allocator, capacity u64, buffer vk.Buffer, initial_backing AllocationInfo) !&UploadRing { + mut backing := initial_backing mut mapped := voidptr(unsafe { nil }) map_result := allocator.map(mut backing, &mapped) if map_result != .success { @@ -72,12 +110,12 @@ pub fn new_upload_ring(mut allocator Allocator, capacity u64) !&UploadRing { return error('could not map upload buffer: ${map_result}') } return &UploadRing{ - buffer: buffer - capacity: capacity + buffer: buffer + capacity: capacity allocator: allocator - backing: backing - mapped: mapped - ranges: memory.new_ring_allocator(capacity) + backing: backing + mapped: mapped + ranges: memory.new_ring_allocator(capacity) } } @@ -90,11 +128,11 @@ pub fn (mut ring UploadRing) allocate(size u64, alignment u64) !UploadSlice { allocation := ring.ranges.allocate(size, alignment)! data := unsafe { voidptr(usize(ring.mapped) + usize(allocation.offset)) } return UploadSlice{ - owner: ring + owner: ring allocation: allocation - offset: allocation.offset - size: allocation.size - data: data + offset: allocation.offset + size: allocation.size + data: data } } @@ -117,6 +155,24 @@ pub fn (mut ring UploadRing) retire(slice UploadSlice) bool { return ring.ranges.release(slice.allocation) } +// flush makes host writes in a live slice available to the device. It is a +// no-op for the coherent memory used by new_upload_ring(), but keeps upload +// code correct if the backing policy changes. +pub fn (ring &UploadRing) flush(slice UploadSlice) vk.Result { + if !ring.contains(slice) { + return .error_memory_map_failed + } + return ring.allocator.flush_range(ring.backing, slice.offset, slice.size) +} + +// invalidate makes device writes in a live slice visible to the host. +pub fn (ring &UploadRing) invalidate(slice UploadSlice) vk.Result { + if !ring.contains(slice) { + return .error_memory_map_failed + } + return ring.allocator.invalidate_range(ring.backing, slice.offset, slice.size) +} + // stats returns current payload, padding, free-space, and peak ring occupancy. pub fn (ring &UploadRing) stats() UploadRingStats { if ring.destroyed { @@ -124,13 +180,13 @@ pub fn (ring &UploadRing) stats() UploadRingStats { } stats := ring.ranges.stats() return UploadRingStats{ - capacity: stats.capacity - used: stats.used - payload: stats.payload - padding: stats.padding - free: stats.free - peak_used: stats.peak_used - allocation_count: stats.allocation_count + capacity: stats.capacity + used: stats.used + payload: stats.payload + padding: stats.padding + free: stats.free + peak_used: stats.peak_used + allocation_count: stats.allocation_count largest_contiguous_free: stats.largest_contiguous_free } } diff --git a/upload_ring_test.v b/upload_ring_test.v index 40230d7..6d20852 100644 --- a/upload_ring_test.v +++ b/upload_ring_test.v @@ -6,8 +6,8 @@ fn test_upload_ring_returns_aligned_host_pointers_and_wraps() { mut storage := []u8{len: 64} mut uploads := UploadRing{ capacity: 64 - mapped: storage.data - ranges: memory.new_ring_allocator(64) + mapped: storage.data + ranges: memory.new_ring_allocator(64) } first := uploads.allocate(24, 16) or { panic(err) } second := uploads.allocate(24, 16) or { panic(err) } @@ -36,24 +36,24 @@ fn test_upload_ring_rejects_foreign_forged_and_destroyed_slices() { mut second_storage := []u8{len: 32} mut first_ring := UploadRing{ capacity: 32 - mapped: first_storage.data - ranges: memory.new_ring_allocator(32) + mapped: first_storage.data + ranges: memory.new_ring_allocator(32) } mut second_ring := UploadRing{ capacity: 32 - mapped: second_storage.data - ranges: memory.new_ring_allocator(32) + mapped: second_storage.data + ranges: memory.new_ring_allocator(32) } allocation := first_ring.allocate(8, 1) or { panic(err) } foreign := second_ring.allocate(8, 1) or { panic(err) } assert !first_ring.contains(foreign) assert !first_ring.retire(foreign) forged := UploadSlice{ - owner: allocation.owner + owner: allocation.owner allocation: allocation.allocation - offset: allocation.offset - size: allocation.size - data: unsafe { voidptr(usize(allocation.data) + 1) } + offset: allocation.offset + size: allocation.size + data: unsafe { voidptr(usize(allocation.data) + 1) } } assert !first_ring.contains(forged) assert !first_ring.retire(forged) diff --git a/v.mod b/v.mod index 80110d6..2d089b9 100644 --- a/v.mod +++ b/v.mod @@ -1,8 +1,8 @@ Module { name: 'antono2.vkmemalloc' author: 'Anton Oreskin' - description: 'Vulkan block suballocation helpers for buffers and images' - version: '2.3.2' + description: 'Policy-driven Vulkan memory selection, suballocation, mapping, and diagnostics' + version: '2.4.0' license: 'MIT' repo_url: 'https://github.com/antono2/vulkan_memory_allocator' tags: ['V','vulkan','allocator'] diff --git a/vulkan_memory_allocator.v b/vulkan_memory_allocator.v index bb25be8..e319d6c 100644 --- a/vulkan_memory_allocator.v +++ b/vulkan_memory_allocator.v @@ -6,16 +6,20 @@ pub const max_pools = 256 pub const memory_block = 1024 * 1024 pub struct Allocator { - create_info AllocatorCreateInfo - props vk.PhysicalDeviceMemoryProperties - api_version u32 + create_info AllocatorCreateInfo + api_version u32 + non_coherent_atom_size u64 mut: - planner &MemoryBlockPool = unsafe { nil } - pools [max_pools]vk.DeviceMemory - block_ids [max_pools]u64 - mapped [max_pools]voidptr - map_refs [max_pools]u32 - pool_size u32 + props vk.PhysicalDeviceMemoryProperties + planner &MemoryBlockPool = unsafe { nil } + pools [max_pools]vk.DeviceMemory + block_ids [max_pools]u64 + mapped [max_pools]voidptr + map_refs [max_pools]u32 + pool_size u32 + memory_budget_reported bool + heap_budgets []u64 + heap_usages []u64 } fn (a &Allocator) has_free_slot() bool { @@ -101,15 +105,21 @@ pub enum MemType { } pub struct AllocationInfo { - // The memory type index pub mut: + // The memory type index mem_type u32 + // The Vulkan memory heap backing that type. + heap_index u32 + // Properties of the selected memory type. + property_flags vk.MemoryPropertyFlags // The memory handle (VkDeviceMemory) memory voidptr = unsafe { nil } // The offset in the memory block offset u64 // The size reserved for this resource inside the memory block size u64 + // Total size of the VkDeviceMemory block containing this allocation. + block_size u64 mut: reservation BlockReservation mapped bool @@ -118,7 +128,7 @@ mut: pub struct MemNode { pub mut: alloc_info &AllocationInfo = unsafe { nil } - next &MemNode = unsafe { nil } + next &MemNode = unsafe { nil } } pub struct AllocatorCreateInfo { @@ -130,6 +140,9 @@ pub mut: preferred_block_size u64 = memory_block // Maximum number of live VkDeviceMemory blocks, capped by max_pools. max_memory_blocks int = max_pools + // Enable VK_EXT_memory_budget property queries. Set this only when the + // physical device reports support and the device extension is enabled. + memory_budget_enabled bool } // new creates a Vulkan allocator with memory-type-specific shared blocks. @@ -151,15 +164,24 @@ pub fn new(create_info AllocatorCreateInfo) Allocator { } else { create_info.max_memory_blocks } - planner := new_memory_block_pool(block_size, block_limit) or { + mut planner := new_memory_block_pool(block_size, block_limit) or { panic('invalid Vulkan memory block configuration: ${err}') } - return Allocator{ + mut allocator := Allocator{ create_info: create_info - props: mem_props + props: mem_props api_version: device_props.apiVersion - planner: planner + non_coherent_atom_size: if device_props.limits.nonCoherentAtomSize > 0 { + u64(device_props.limits.nonCoherentAtomSize) + } else { + u64(1) + } + planner: planner + } + if create_info.memory_budget_enabled { + _ = allocator.refresh_memory_budget() } + return allocator } // get_memory_type selects a supported memory type containing every requested @@ -206,32 +228,79 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me } } - // `alloc_info` is the caller's output record. Rebinding it to a freshly - // allocated local pointer loses the allocation handle at every call site. - // Reset and populate that record directly instead. - alloc_info = AllocationInfo{} - - // Note: VK_NULL_HANDLE is "nullptr", "voidptr(0)" for C++ compatible compilers, or "0ULL" (Unsigned Long Long 0) for 64bit and "0" for 32 bit in C - alloc_info.memory = unsafe { nil } // vk.null_handle - alloc_info.size = req.size - alloc_info.offset = 0 - alloc_info.mem_type = a.get_memory_type(req.memoryTypeBits, mem_type) - if alloc_info.mem_type == max_u32 { + memory_type := a.get_memory_type(req.memoryTypeBits, mem_type) + if memory_type == max_u32 { // Never drop required properties. In particular, mapping arbitrary // device-local memory after a staging allocation fails is invalid and // previously led to a null mapped pointer and a delayed segfault. eprintln('No compatible Vulkan memory type: type bits 0x${req.memoryTypeBits:08x}, required flags 0x${u32(mem_type):08x}') return .error_feature_not_present } + choices := ranked_memory_types(a.props, u32(1) << memory_type, req.size, AllocationOptions{ + budget_policy: .ignore + }, a.heap_budget_snapshot()) + return a.allocate_from_choices(mut req, choices, allocation_pnext, dedicated, .ignore, mut alloc_info) +} + +// allocate_with_options reserves isolated memory using the portable ranked +// policy. Prefer create_buffer_with_options() when safe buffer suballocation is +// desired. +pub fn (mut a Allocator) allocate_with_options(mut req vk.MemoryRequirements, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { + choices := a.rank_memory_types(req.memoryTypeBits, req.size, options) + return a.allocate_from_choices(mut req, choices, unsafe { nil }, true, options.budget_policy, mut alloc_info) +} + +fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choices []MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { + // `alloc_info` is the caller's output record. Reset and populate that record + // directly so callers always receive the actual tracked handle. + alloc_info = AllocationInfo{} if req.size == 0 || req.alignment == 0 { return .error_initialization_failed } if isnil(a.planner) { return .error_initialization_failed } + if choices.len == 0 { + return .error_feature_not_present + } + mut last_result := vk.Result.error_out_of_device_memory + for choice in choices { + allow_new_block := budget_policy != .require_within || choice.within_budget + mut result := a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, allow_new_block, budget_policy, mut alloc_info) + if result == .success { + return .success + } + last_result = result + if result !in [.error_out_of_device_memory, .error_out_of_host_memory, + .error_too_many_objects] { + return result + } + mut trimmed := 0 + if allow_new_block { + trimmed = a.trim_empty_blocks_for_heap(choice.heap_index) + if result == .error_too_many_objects { + // The block-count cap is allocator-wide, so an empty block in a + // different heap can also make room for this candidate. + trimmed += a.trim_empty_blocks() + } + } + if trimmed > 0 { + result = a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, true, budget_policy, mut alloc_info) + if result == .success { + return .success + } + last_result = result + } + if result == .error_too_many_objects { + return result + } + } + return last_result +} +fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, choice MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, allow_new_block bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { if !dedicated { - if reservation := a.planner.reserve(alloc_info.mem_type, req.size, req.alignment) { + if reservation := a.planner.reserve(choice.index, req.size, req.alignment) { memory := a.memory_for_block(reservation.block_id) or { _ = a.planner.release(reservation) return .error_initialization_failed @@ -240,19 +309,25 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me return .success } } - + if !allow_new_block { + return .error_out_of_device_memory + } if !a.has_free_slot() { return .error_too_many_objects } - block_size := if dedicated { + mut block_size := if dedicated { req.size } else { a.planner.recommended_block_size(req.size) or { return .error_out_of_device_memory } } + if budget_policy != .ignore && choice.budget_reported && choice.within_budget + && choice.remaining_budget < block_size { + block_size = choice.remaining_budget + } vkalloc_info := vk.MemoryAllocateInfo{ - allocationSize: block_size - memoryTypeIndex: alloc_info.mem_type - pNext: allocation_pnext + allocationSize: block_size + memoryTypeIndex: choice.index + pNext: allocation_pnext } mut memory := vk.DeviceMemory(unsafe { nil }) result := vk.allocate_memory(a.create_info.device, &vkalloc_info, unsafe { nil }, &memory) @@ -260,13 +335,13 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me return result } block_id := if dedicated { - a.planner.add_dedicated_block(alloc_info.mem_type, block_size) or { + a.planner.add_dedicated_block(choice.index, block_size) or { vk.free_memory(a.create_info.device, memory, unsafe { nil }) alloc_info = AllocationInfo{} return .error_too_many_objects } } else { - a.planner.add_block(alloc_info.mem_type, block_size) or { + a.planner.add_block(choice.index, block_size) or { vk.free_memory(a.create_info.device, memory, unsafe { nil }) alloc_info = AllocationInfo{} return .error_too_many_objects @@ -292,8 +367,14 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me fn (a &Allocator) populate_allocation(mut alloc_info AllocationInfo, memory vk.DeviceMemory, reservation BlockReservation) { alloc_info.memory = voidptr(memory) alloc_info.mem_type = reservation.memory_type + if reservation.memory_type < a.props.memoryTypeCount { + memory_type := a.props.memoryTypes[reservation.memory_type] + alloc_info.heap_index = memory_type.heapIndex + alloc_info.property_flags = memory_type.propertyFlags + } alloc_info.offset = reservation.offset alloc_info.size = reservation.size + alloc_info.block_size = a.planner.block_capacity(reservation.block_id) or { 0 } alloc_info.reservation = reservation } @@ -302,18 +383,57 @@ fn (a &Allocator) owns_allocation(alloc_info AllocationInfo) bool { return false } memory := a.memory_for_block(alloc_info.reservation.block_id) or { return false } + if alloc_info.mem_type < a.props.memoryTypeCount { + memory_type := a.props.memoryTypes[alloc_info.mem_type] + if alloc_info.heap_index != memory_type.heapIndex + || alloc_info.property_flags != memory_type.propertyFlags { + return false + } + } return voidptr(memory) == alloc_info.memory && alloc_info.offset == alloc_info.reservation.offset && alloc_info.size == alloc_info.reservation.size && alloc_info.mem_type == alloc_info.reservation.memory_type + && alloc_info.block_size == (a.planner.block_capacity(alloc_info.reservation.block_id) or { + return false + }) } fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { if a.api_version < vk.api_version_1_1 { mut requirements := vk.MemoryRequirements{} vk.get_buffer_memory_requirements(a.create_info.device, buffer, mut requirements) - return a.allocate_with_policy(mut requirements, type, unsafe { nil }, force_dedicated, mut - alloc_info) + return a.allocate_with_policy(mut requirements, type, unsafe { nil }, force_dedicated, mut alloc_info) + } + mut dedicated_requirements := vk.MemoryDedicatedRequirements{} + mut requirements := vk.MemoryRequirements2{ + pNext: &dedicated_requirements + } + info := vk.BufferMemoryRequirementsInfo2{ + buffer: buffer + } + vk.get_buffer_memory_requirements2(a.create_info.device, &info, mut requirements) + dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true + || dedicated_requirements.prefersDedicatedAllocation == vk._true + if !dedicated { + return a.allocate_with_policy(mut requirements.memoryRequirements, type, unsafe { nil }, false, mut alloc_info) + } + dedicated_info := vk.MemoryDedicatedAllocateInfo{ + buffer: buffer + } + return a.allocate_with_policy(mut requirements.memoryRequirements, type, voidptr(&dedicated_info), true, mut alloc_info) +} + +fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, options AllocationOptions, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { + if a.api_version < vk.api_version_1_1 { + mut requirements := vk.MemoryRequirements{} + vk.get_buffer_memory_requirements(a.create_info.device, buffer, mut requirements) + choices := if force_dedicated { + a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + } else { + a.rank_buffer_memory_types(requirements.memoryTypeBits, requirements.size, options) + } + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, force_dedicated, options.budget_policy, mut alloc_info) } mut dedicated_requirements := vk.MemoryDedicatedRequirements{} mut requirements := vk.MemoryRequirements2{ @@ -325,15 +445,18 @@ fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, forc vk.get_buffer_memory_requirements2(a.create_info.device, &info, mut requirements) dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true || dedicated_requirements.prefersDedicatedAllocation == vk._true + choices := if dedicated { + a.rank_memory_types(requirements.memoryRequirements.memoryTypeBits, requirements.memoryRequirements.size, options) + } else { + a.rank_buffer_memory_types(requirements.memoryRequirements.memoryTypeBits, requirements.memoryRequirements.size, options) + } if !dedicated { - return a.allocate_with_policy(mut requirements.memoryRequirements, type, unsafe { nil }, - false, mut alloc_info) + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, unsafe { nil }, false, options.budget_policy, mut alloc_info) } dedicated_info := vk.MemoryDedicatedAllocateInfo{ buffer: buffer } - return a.allocate_with_policy(mut requirements.memoryRequirements, type, - voidptr(&dedicated_info), true, mut alloc_info) + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) } fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut alloc_info AllocationInfo) vk.Result { @@ -348,13 +471,32 @@ fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut all dedicated_info := vk.MemoryDedicatedAllocateInfo{ image: image } - return a.allocate_with_policy(mut requirements, type, voidptr(&dedicated_info), true, mut - alloc_info) + return a.allocate_with_policy(mut requirements, type, voidptr(&dedicated_info), true, mut alloc_info) } vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) return a.allocate_with_policy(mut requirements, type, unsafe { nil }, true, mut alloc_info) } +fn (mut a Allocator) allocate_image_memory_with_options(image vk.Image, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { + mut requirements := vk.MemoryRequirements{} + if a.api_version >= vk.api_version_1_1 { + mut requirements2 := vk.MemoryRequirements2{} + info := vk.ImageMemoryRequirementsInfo2{ + image: image + } + vk.get_image_memory_requirements2(a.create_info.device, &info, mut requirements2) + requirements = requirements2.memoryRequirements + choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + dedicated_info := vk.MemoryDedicatedAllocateInfo{ + image: image + } + return a.allocate_from_choices(mut requirements, choices, voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) + } + vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) + choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, true, options.budget_policy, mut alloc_info) +} + // create_buffer creates a buffer, suballocates compatible memory, and binds it. pub fn (mut a Allocator) create_buffer(buffer_info &vk.BufferCreateInfo, type MemType, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { return a.create_buffer_with_policy(buffer_info, type, false, buffer, mut alloc_info) @@ -366,6 +508,46 @@ pub fn (mut a Allocator) create_dedicated_buffer(buffer_info &vk.BufferCreateInf return a.create_buffer_with_policy(buffer_info, type, true, buffer, mut alloc_info) } +// create_buffer_with_options creates a buffer and selects memory using an +// explicit usage/property/budget policy. Compatible buffers share blocks. +pub fn (mut a Allocator) create_buffer_with_options(buffer_info &vk.BufferCreateInfo, options AllocationOptions, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { + return a.create_buffer_with_options_policy(buffer_info, options, false, buffer, mut alloc_info) +} + +// create_dedicated_buffer_with_options is the policy-based counterpart of +// create_dedicated_buffer(). +pub fn (mut a Allocator) create_dedicated_buffer_with_options(buffer_info &vk.BufferCreateInfo, options AllocationOptions, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { + return a.create_buffer_with_options_policy(buffer_info, options, true, buffer, mut alloc_info) +} + +fn (mut a Allocator) create_buffer_with_options_policy(buffer_info &vk.BufferCreateInfo, options AllocationOptions, dedicated bool, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { + unsafe { + *buffer = nil + } + alloc_info = AllocationInfo{} + mut result := vk.create_buffer(a.create_info.device, buffer_info, unsafe { nil }, buffer) + if result != .success { + return result + } + result = a.allocate_buffer_memory_with_options(*buffer, options, dedicated, mut alloc_info) + if result != .success { + vk.destroy_buffer(a.create_info.device, *buffer, unsafe { nil }) + unsafe { + *buffer = nil + } + return result + } + result = vk.bind_buffer_memory(a.create_info.device, *buffer, alloc_info.memory, alloc_info.offset) + if result != .success { + vk.destroy_buffer(a.create_info.device, *buffer, unsafe { nil }) + unsafe { + *buffer = nil + } + _ = a.release(mut alloc_info) + } + return result +} + fn (mut a Allocator) create_buffer_with_policy(buffer_info &vk.BufferCreateInfo, type MemType, dedicated bool, buffer &vk.Buffer, mut alloc_info AllocationInfo) vk.Result { unsafe { *buffer = nil @@ -436,6 +618,38 @@ pub fn (mut a Allocator) create_image(p_image_create_info &vk.ImageCreateInfo, t return vk.Result.success } +// create_image_with_options creates a dedicated image allocation using the +// ranked usage/property/budget policy. Image suballocation remains deliberately +// conservative because buffer-image granularity and tiling compatibility must +// be tracked together. +pub fn (mut a Allocator) create_image_with_options(image_info &vk.ImageCreateInfo, options AllocationOptions, image &vk.Image, mut alloc_info AllocationInfo) vk.Result { + unsafe { + *image = nil + } + alloc_info = AllocationInfo{} + mut result := vk.create_image(a.create_info.device, image_info, unsafe { nil }, image) + if result != .success { + return result + } + result = a.allocate_image_memory_with_options(*image, options, mut alloc_info) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil + } + return result + } + result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, alloc_info.offset) + if result != .success { + vk.destroy_image(a.create_info.device, *image, unsafe { nil }) + unsafe { + *image = nil + } + _ = a.release(mut alloc_info) + } + return result +} + // map maps the allocation's byte range for host access. Compatible allocations // sharing one VkDeviceMemory block share one Vulkan mapping internally. pub fn (mut a Allocator) map(mut alloc_info AllocationInfo, data &voidptr) vk.Result { @@ -443,6 +657,10 @@ pub fn (mut a Allocator) map(mut alloc_info AllocationInfo, data &voidptr) vk.Re eprintln('Cannot map an allocation not owned by this allocator') return .error_memory_map_failed } + if !has_memory_flags(alloc_info.property_flags, memory_flag(.host_visible)) { + eprintln('Cannot map memory without the host-visible property') + return .error_memory_map_failed + } if alloc_info.mapped { eprintln('Cannot map an allocation that is already mapped') return .error_memory_map_failed @@ -519,6 +737,14 @@ pub fn (mut a Allocator) allocator_free(mut alloc_info AllocationInfo) { // trim_empty_blocks frees cached VkDeviceMemory blocks with no live ranges. pub fn (mut a Allocator) trim_empty_blocks() int { + return a.trim_empty_blocks_filtered(0, false) +} + +fn (mut a Allocator) trim_empty_blocks_for_heap(heap_index u32) int { + return a.trim_empty_blocks_filtered(heap_index, true) +} + +fn (mut a Allocator) trim_empty_blocks_filtered(heap_index u32, filter_by_heap bool) int { if isnil(a.planner) { return 0 } @@ -526,6 +752,15 @@ pub fn (mut a Allocator) trim_empty_blocks() int { mut index := 0 for index < int(a.pool_size) { block_id := a.block_ids[index] + memory_type := a.planner.block_memory_type(block_id) or { + index++ + continue + } + if filter_by_heap && (memory_type >= a.props.memoryTypeCount + || a.props.memoryTypes[memory_type].heapIndex != heap_index) { + index++ + continue + } allocation_count := a.planner.block_allocation_count(block_id) or { index++ continue @@ -576,14 +811,14 @@ pub fn (a &Allocator) stats() AllocatorStats { } stats := a.planner.stats() return AllocatorStats{ - block_count: stats.block_count - allocation_count: stats.allocation_count - committed: stats.committed - used: stats.used - free: stats.free - free_range_count: stats.free_range_count + block_count: stats.block_count + allocation_count: stats.allocation_count + committed: stats.committed + used: stats.used + free: stats.free + free_range_count: stats.free_range_count largest_free_range: stats.largest_free_range - empty_block_count: stats.empty_block_count + empty_block_count: stats.empty_block_count } } @@ -596,14 +831,14 @@ pub fn (a &Allocator) stats_for_memory_type(memory_type u32) AllocatorStats { } stats := a.planner.stats_for_memory_type(memory_type) return AllocatorStats{ - block_count: stats.block_count - allocation_count: stats.allocation_count - committed: stats.committed - used: stats.used - free: stats.free - free_range_count: stats.free_range_count + block_count: stats.block_count + allocation_count: stats.allocation_count + committed: stats.committed + used: stats.used + free: stats.free + free_range_count: stats.free_range_count largest_free_range: stats.largest_free_range - empty_block_count: stats.empty_block_count + empty_block_count: stats.empty_block_count } } From 917f7447825163cf9b75b5bb72e5925b0452ed11 Mon Sep 17 00:00:00 2001 From: antono2 Date: Sat, 12 Sep 2026 15:40:46 +0200 Subject: [PATCH 2/2] Support released V 0.5.2 --- block_pool.v | 44 ++++++------ block_pool_test.v | 10 +-- examples/buffer_suballocation/main.v | 61 ++++++++-------- mapped_memory.v | 7 +- mapped_memory_test.v | 6 +- memory_policy.v | 83 ++++++++++----------- memory_policy_test.v | 38 +++++----- upload_ring.v | 53 +++++++------- upload_ring_test.v | 20 +++--- vulkan_memory_allocator.v | 103 +++++++++++++++------------ 10 files changed, 225 insertions(+), 200 deletions(-) diff --git a/block_pool.v b/block_pool.v index 4670022..59362bc 100644 --- a/block_pool.v +++ b/block_pool.v @@ -52,7 +52,7 @@ fn new_memory_block_pool(default_block_size u64, max_blocks int) !&MemoryBlockPo } return &MemoryBlockPool{ default_block_size: default_block_size - max_blocks: max_blocks + max_blocks: max_blocks } } @@ -138,11 +138,11 @@ fn (mut pool MemoryBlockPool) add_block_with_policy(memory_type u32, capacity u6 } id := pool.next_block_id() pool.blocks << MemoryBlock{ - id: id + id: id memory_type: memory_type - capacity: capacity - dedicated: dedicated - ranges: memory.new_range_allocator(capacity) + capacity: capacity + dedicated: dedicated + ranges: memory.new_range_allocator(capacity) } return id } @@ -163,12 +163,12 @@ fn (mut pool MemoryBlockPool) reserve(memory_type u32, size u64, alignment u64) } if allocation := block.ranges.allocate(size, alignment) { return BlockReservation{ - owner: pool - block_id: block.id - allocation: allocation + owner: pool + block_id: block.id + allocation: allocation memory_type: memory_type - offset: allocation.offset - size: allocation.size + offset: allocation.offset + size: allocation.size } } } @@ -188,12 +188,12 @@ fn (mut pool MemoryBlockPool) reserve_from_block(block_id u64, size u64, alignme } allocation := block.ranges.allocate(size, alignment)! return BlockReservation{ - owner: pool - block_id: block.id - allocation: allocation + owner: pool + block_id: block.id + allocation: allocation memory_type: block.memory_type - offset: allocation.offset - size: allocation.size + offset: allocation.offset + size: allocation.size } } return error('memory block does not exist') @@ -281,14 +281,14 @@ fn (pool &MemoryBlockPool) collect_stats(memory_type u32, filter_by_memory_type } } return BlockPoolStats{ - block_count: block_count - allocation_count: allocation_count - committed: committed - used: used - free: committed - used - free_range_count: free_range_count + block_count: block_count + allocation_count: allocation_count + committed: committed + used: used + free: committed - used + free_range_count: free_range_count largest_free_range: largest_free_range - empty_block_count: empty_block_count + empty_block_count: empty_block_count } } diff --git a/block_pool_test.v b/block_pool_test.v index 7042874..079a5ba 100644 --- a/block_pool_test.v +++ b/block_pool_test.v @@ -120,12 +120,12 @@ fn test_block_pool_rejects_foreign_forged_and_stale_reservations() { assert !first_pool.contains(foreign) assert !first_pool.release(foreign) assert !first_pool.release(BlockReservation{ - owner: reservation.owner - block_id: reservation.block_id - allocation: reservation.allocation + owner: reservation.owner + block_id: reservation.block_id + allocation: reservation.allocation memory_type: reservation.memory_type - offset: reservation.offset + 1 - size: reservation.size + offset: reservation.offset + 1 + size: reservation.size }) released := first_pool.release(reservation) assert released diff --git a/examples/buffer_suballocation/main.v b/examples/buffer_suballocation/main.v index a100e4c..42bc60f 100644 --- a/examples/buffer_suballocation/main.v +++ b/examples/buffer_suballocation/main.v @@ -11,28 +11,31 @@ fn require_success(result vk.Result, operation string) ! { fn first_physical_device(instance vk.Instance) !vk.PhysicalDevice { mut count := u32(0) - require_success(vk.enumerate_physical_devices(instance, &count, unsafe { nil }), 'enumerate physical device count')! + require_success(vk.enumerate_physical_devices(instance, &count, unsafe { nil }), + 'enumerate physical device count')! if count == 0 { return error('no Vulkan physical device is available') } mut devices := unsafe { []vk.PhysicalDevice{len: int(count)} } - require_success(vk.enumerate_physical_devices(instance, &count, devices.data), 'enumerate physical devices')! + require_success(vk.enumerate_physical_devices(instance, &count, devices.data), + 'enumerate physical devices')! return devices[0] } fn run() ! { require_success(vk.initialize_loader(), 'initialize Vulkan loader')! application_info := vk.ApplicationInfo{ - pApplicationName: c'vkmemalloc suballocation example' + pApplicationName: c'vkmemalloc suballocation example' applicationVersion: 1 - pEngineName: c'none' - apiVersion: vk.api_version_1_1 + pEngineName: c'none' + apiVersion: vk.api_version_1_1 } instance_info := vk.InstanceCreateInfo{ pApplicationInfo: &application_info } mut instance := vk.Instance(unsafe { nil }) - require_success(vk.create_instance(&instance_info, unsafe { nil }, &instance), 'create Vulkan instance')! + require_success(vk.create_instance(&instance_info, unsafe { nil }, &instance), + 'create Vulkan instance')! defer { vk.destroy_instance(instance, unsafe { nil }) } @@ -47,26 +50,27 @@ fn run() ! { mut priority := f32(1) queue_info := vk.DeviceQueueCreateInfo{ queueFamilyIndex: 0 - queueCount: 1 + queueCount: 1 pQueuePriorities: &priority } device_info := vk.DeviceCreateInfo{ - queueCreateInfoCount: 1 - pQueueCreateInfos: &queue_info - enabledExtensionCount: u32(device_extensions.len) + queueCreateInfoCount: 1 + pQueueCreateInfos: &queue_info + enabledExtensionCount: u32(device_extensions.len) ppEnabledExtensionNames: device_extensions.data } mut device := vk.Device(unsafe { nil }) - require_success(vk.create_device(physical_device, &device_info, unsafe { nil }, &device), 'create Vulkan device')! + require_success(vk.create_device(physical_device, &device_info, unsafe { nil }, &device), + 'create Vulkan device')! defer { vk.destroy_device(device, unsafe { nil }) } vk.load_device_commands(device) mut allocator := vma.new(vma.AllocatorCreateInfo{ - physical_device: physical_device - device: device - preferred_block_size: 4096 + physical_device: physical_device + device: device + preferred_block_size: 4096 memory_budget_enabled: memory_budget_supported }) defer { @@ -74,8 +78,8 @@ fn run() ! { } buffer_info := vk.BufferCreateInfo{ - size: 512 - usage: u32(vk.BufferUsageFlagBits.transfer_src) + size: 512 + usage: u32(vk.BufferUsageFlagBits.transfer_src) sharingMode: .exclusive } mut first_buffer := vk.Buffer(unsafe { nil }) @@ -118,7 +122,8 @@ fn run() ! { mut first_mapped := voidptr(unsafe { nil }) mut second_mapped := voidptr(unsafe { nil }) require_success(allocator.map(mut first_allocation, &first_mapped), 'map first staging buffer')! - require_success(allocator.map(mut second_allocation, &second_mapped), 'map second staging buffer')! + require_success(allocator.map(mut second_allocation, &second_mapped), + 'map second staging buffer')! unsafe { *(&u8(first_mapped)) = 21 *(&u8(second_mapped)) = 42 @@ -131,19 +136,19 @@ fn run() ! { println('shared staging suballocations mapped concurrently') image_info := vk.ImageCreateInfo{ - imageType: ._2d - format: .r8g8b8a8_unorm - extent: vk.Extent3D{ - width: 16 + imageType: ._2d + format: .r8g8b8a8_unorm + extent: vk.Extent3D{ + width: 16 height: 16 - depth: 1 + depth: 1 } - mipLevels: 1 - arrayLayers: 1 - samples: ._1 - tiling: .optimal - usage: u32(vk.ImageUsageFlagBits.transfer_dst) - sharingMode: .exclusive + mipLevels: 1 + arrayLayers: 1 + samples: ._1 + tiling: .optimal + usage: u32(vk.ImageUsageFlagBits.transfer_dst) + sharingMode: .exclusive initialLayout: .undefined } mut image := vk.Image(unsafe { nil }) diff --git a/mapped_memory.v b/mapped_memory.v index 24755ca..0693074 100644 --- a/mapped_memory.v +++ b/mapped_memory.v @@ -28,7 +28,7 @@ fn normalize_mapped_range(allocation_offset u64, allocation_size u64, block_size } return NormalizedMappedRange{ offset: start - size: if rounded_end >= block_size { + size: if rounded_end >= block_size { vk.whole_size } else { rounded_end - start @@ -41,11 +41,12 @@ fn (a &Allocator) mapped_range(alloc_info AllocationInfo, relative_offset u64, s || !has_memory_flags(alloc_info.property_flags, memory_flag(.host_visible)) { return none } - normalized := normalize_mapped_range(alloc_info.offset, alloc_info.size, alloc_info.block_size, relative_offset, size, a.non_coherent_atom_size) or { return none } + normalized := normalize_mapped_range(alloc_info.offset, alloc_info.size, alloc_info.block_size, + relative_offset, size, a.non_coherent_atom_size) or { return none } return vk.MappedMemoryRange{ memory: vk.DeviceMemory(alloc_info.memory) offset: normalized.offset - size: normalized.size + size: normalized.size } } diff --git a/mapped_memory_test.v b/mapped_memory_test.v index e3dc4e7..786aa2f 100644 --- a/mapped_memory_test.v +++ b/mapped_memory_test.v @@ -41,15 +41,15 @@ fn test_coherent_flush_validates_ownership_without_a_driver_call() { props.memoryTypeCount = 1 props.memoryTypes[0] = vk.MemoryType{ propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) - heapIndex: 0 + heapIndex: 0 } mut planner := new_memory_block_pool(256, 1) or { panic(err) } block_id := planner.add_block(0, 256) or { panic(err) } reservation := planner.reserve(0, 64, 1) or { panic(err) } mut allocator := Allocator{ - props: props + props: props non_coherent_atom_size: 64 - planner: planner + planner: planner } assert allocator.remember_block(mapped_test_memory(1), block_id) mut allocation := AllocationInfo{} diff --git a/memory_policy.v b/memory_policy.v index e137b5f..222954f 100644 --- a/memory_policy.v +++ b/memory_policy.v @@ -80,11 +80,13 @@ pub fn supports_memory_budget(physical_device vk.PhysicalDevice) bool { for { mut count := u32(0) mut no_properties := unsafe { nil } - if vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut no_properties) != .success || count == 0 { + if vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut no_properties) != .success + || count == 0 { return false } mut properties := []vk.ExtensionProperties{len: int(count)} - result := vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut properties[0]) + result := vk.enumerate_device_extension_properties(physical_device, unsafe { nil }, &count, mut + properties[0]) if result == .incomplete { continue } @@ -135,28 +137,29 @@ fn usage_preference_score(usage MemoryUsage, flags vk.MemoryPropertyFlags) int { device_uncached := has_memory_flags(flags, memory_flag(.device_uncached_bit_amd)) return match usage { .automatic { - if device_local { 16 } else { 0 } + memory_score(device_local, 16) } .gpu_only { - if device_uncached { -4 } else { 0 } + memory_score(device_uncached, -4) } .upload { - (if host_coherent { 16 } else { 0 }) + (if device_local { 8 } else { 0 }) + (if host_cached { - 2 - } else { - 0 - }) - (if device_uncached { 4 } else { 0 }) + memory_score(host_coherent, 16) + memory_score(device_local, 8) + + memory_score(host_cached, 2) + memory_score(device_uncached, -4) } .readback { - (if host_cached { 16 } else { 0 }) + (if host_coherent { 8 } else { 0 }) + (if device_local { - 2 - } else { - 0 - }) - (if device_uncached { 4 } else { 0 }) + memory_score(host_cached, 16) + memory_score(host_coherent, 8) + + memory_score(device_local, 2) + memory_score(device_uncached, -4) } } } +fn memory_score(condition bool, points int) int { + if condition { + return points + } + return 0 +} + fn memory_preference_score(options AllocationOptions, flags vk.MemoryPropertyFlags) int { preferred := memory_flag_count(flags & options.preferred_flags) avoided := memory_flag_count(flags & options.avoided_flags) @@ -209,15 +212,15 @@ fn ranked_memory_types(props vk.PhysicalDeviceMemoryProperties, type_bits u32, r continue } choice := MemoryTypeChoice{ - index: u32(index) - heap_index: memory_type.heapIndex - property_flags: memory_type.propertyFlags - heap_size: heap_size - heap_budget: budget - heap_usage: usage + index: u32(index) + heap_index: memory_type.heapIndex + property_flags: memory_type.propertyFlags + heap_size: heap_size + heap_budget: budget + heap_usage: usage remaining_budget: remaining - within_budget: within_budget - budget_reported: reported + within_budget: within_budget + budget_reported: reported preference_score: memory_preference_score(options, memory_type.propertyFlags) } mut inserted := false @@ -264,8 +267,8 @@ fn (a &Allocator) heap_budget_snapshot() HeapBudgetSnapshot { } return HeapBudgetSnapshot{ reported: a.memory_budget_reported - budgets: budgets - usages: usages + budgets: budgets + usages: usages } } @@ -320,11 +323,11 @@ fn (mut a Allocator) rank_buffer_memory_types(type_bits u32, request_size u64, o return a.rank_memory_types(type_bits, request_size, options) } return a.rank_memory_types(type_bits, request_size, AllocationOptions{ - usage: options.usage - required_flags: options.required_flags + usage: options.usage + required_flags: options.required_flags preferred_flags: options.preferred_flags - avoided_flags: options.avoided_flags - budget_policy: .prefer_within + avoided_flags: options.avoided_flags + budget_policy: .prefer_within }) } @@ -335,22 +338,22 @@ pub fn (a &Allocator) memory_heaps() []MemoryHeapStats { snapshot := a.heap_budget_snapshot() mut heaps := []MemoryHeapStats{cap: int(a.props.memoryHeapCount)} for heap_index in 0 .. int(a.props.memoryHeapCount) { - committed, used := if isnil(a.planner) { - u64(0), u64(0) - } else { - a.planner.heap_stats(&a.props, u32(heap_index)) + mut committed := u64(0) + mut used := u64(0) + if !isnil(a.planner) { + committed, used = a.planner.heap_stats(&a.props, u32(heap_index)) } budget, usage, reported := heap_budget_values(a.props, u32(heap_index), snapshot) heaps << MemoryHeapStats{ - heap_index: u32(heap_index) - size: u64(a.props.memoryHeaps[heap_index].size) - budget: budget - usage: usage - remaining_budget: if usage < budget { budget - usage } else { u64(0) } + heap_index: u32(heap_index) + size: u64(a.props.memoryHeaps[heap_index].size) + budget: budget + usage: usage + remaining_budget: if usage < budget { budget - usage } else { u64(0) } allocator_committed: committed - allocator_used: used - device_local: (a.props.memoryHeaps[heap_index].flags & u32(vk.MemoryHeapFlagBits.device_local)) != 0 - budget_reported: reported + allocator_used: used + device_local: (a.props.memoryHeaps[heap_index].flags & u32(vk.MemoryHeapFlagBits.device_local)) != 0 + budget_reported: reported } } return heaps diff --git a/memory_policy_test.v b/memory_policy_test.v index aad7e0f..3e199bf 100644 --- a/memory_policy_test.v +++ b/memory_policy_test.v @@ -10,7 +10,7 @@ fn policy_test_properties() vk.PhysicalDeviceMemoryProperties { mut props := vk.PhysicalDeviceMemoryProperties{} props.memoryHeapCount = 2 props.memoryHeaps[0] = vk.MemoryHeap{ - size: 256 + size: 256 flags: u32(vk.MemoryHeapFlagBits.device_local) } props.memoryHeaps[1] = vk.MemoryHeap{ @@ -19,19 +19,19 @@ fn policy_test_properties() vk.PhysicalDeviceMemoryProperties { props.memoryTypeCount = 4 props.memoryTypes[0] = vk.MemoryType{ propertyFlags: u32(vk.MemoryPropertyFlagBits.device_local) - heapIndex: 0 + heapIndex: 0 } props.memoryTypes[1] = vk.MemoryType{ propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) - heapIndex: 1 + heapIndex: 1 } props.memoryTypes[2] = vk.MemoryType{ propertyFlags: u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_cached) - heapIndex: 1 + heapIndex: 1 } props.memoryTypes[3] = vk.MemoryType{ propertyFlags: u32(vk.MemoryPropertyFlagBits.device_local) | u32(vk.MemoryPropertyFlagBits.host_visible) | u32(vk.MemoryPropertyFlagBits.host_coherent) - heapIndex: 0 + heapIndex: 0 } return props } @@ -63,15 +63,16 @@ fn test_memory_policy_honors_required_preferred_and_avoided_flags() { host_coherent := memory_flag(.host_coherent) host_cached := memory_flag(.host_cached) choice := select_memory_type(props, 0b1110, 16, AllocationOptions{ - required_flags: host_visible + required_flags: host_visible preferred_flags: host_cached - avoided_flags: host_coherent + avoided_flags: host_coherent }) or { panic('host-visible memory type should exist') } assert choice.index == 2 if _ := select_memory_type(props, 0b0001, 16, AllocationOptions{ required_flags: host_visible - }) { + }) + { assert false, 'required properties must never be dropped' } } @@ -83,8 +84,8 @@ fn test_memory_policy_prefers_or_requires_available_budget() { } snapshot := HeapBudgetSnapshot{ reported: true - budgets: [u64(128), 1024] - usages: [u64(120), 0] + budgets: [u64(128), 1024] + usages: [u64(120), 0] } preferred := ranked_memory_types(props, 0b1011, 16, options, snapshot) assert preferred.len == 3 @@ -94,14 +95,14 @@ fn test_memory_policy_prefers_or_requires_available_budget() { assert preferred[0].remaining_budget == 1024 ignored := ranked_memory_types(props, 0b1011, 16, AllocationOptions{ - usage: .automatic + usage: .automatic budget_policy: .ignore }, snapshot) assert ignored[0].index == 0 assert !ignored[0].within_budget required := ranked_memory_types(props, 0b1011, 16, AllocationOptions{ - usage: .automatic + usage: .automatic budget_policy: .require_within }, snapshot) assert required.len == 1 @@ -123,7 +124,7 @@ fn test_allocator_policy_uses_owned_commitment_as_portable_budget_fallback() { mut planner := new_memory_block_pool(256, 4) or { panic(err) } _ = planner.add_block(0, 256) or { panic(err) } mut allocator := Allocator{ - props: props + props: props planner: planner } choice := allocator.select_memory_type(0b0011, 16, AllocationOptions{}) or { @@ -147,24 +148,25 @@ fn test_require_within_can_reuse_an_over_budget_buffer_block() { mut planner := new_memory_block_pool(256, 1) or { panic(err) } block_id := planner.add_block(0, 256) or { panic(err) } mut allocator := Allocator{ - props: props + props: props planner: planner } assert allocator.remember_block(policy_test_memory(1), block_id) options := AllocationOptions{ - usage: .gpu_only + usage: .gpu_only budget_policy: .require_within } choices := allocator.rank_buffer_memory_types(0b0001, 16, options) assert choices.len == 1 assert !choices[0].within_budget mut requirements := vk.MemoryRequirements{ - size: 16 - alignment: 8 + size: 16 + alignment: 8 memoryTypeBits: 0b0001 } mut allocation := AllocationInfo{} - result := allocator.allocate_from_choices(mut requirements, choices, unsafe { nil }, false, options.budget_policy, mut allocation) + result := allocator.allocate_from_choices(mut requirements, choices, unsafe { nil }, false, + options.budget_policy, mut allocation) assert result == .success assert allocation.memory == voidptr(policy_test_memory(1)) assert allocation.block_size == 256 diff --git a/upload_ring.v b/upload_ring.v index aed272a..c41d1c5 100644 --- a/upload_ring.v +++ b/upload_ring.v @@ -37,7 +37,7 @@ pub: mut: allocator &Allocator = unsafe { nil } backing AllocationInfo - mapped voidptr = unsafe { nil } + mapped voidptr = unsafe { nil } ranges &memory.RingAllocator = unsafe { nil } destroyed bool } @@ -54,8 +54,8 @@ pub fn new_upload_ring(mut allocator Allocator, capacity u64) !&UploadRing { } } buffer_info := vk.BufferCreateInfo{ - size: capacity - usage: u32(vk.BufferUsageFlagBits.transfer_src) + size: capacity + usage: u32(vk.BufferUsageFlagBits.transfer_src) sharingMode: .exclusive } mut buffer := vk.Buffer(unsafe { nil }) @@ -80,20 +80,21 @@ pub fn new_upload_ring_with_options(mut allocator Allocator, capacity u64, optio } } buffer_info := vk.BufferCreateInfo{ - size: capacity - usage: u32(vk.BufferUsageFlagBits.transfer_src) + size: capacity + usage: u32(vk.BufferUsageFlagBits.transfer_src) sharingMode: .exclusive } effective_options := AllocationOptions{ - usage: options.usage - required_flags: options.required_flags | memory_flag(.host_visible) + usage: options.usage + required_flags: options.required_flags | memory_flag(.host_visible) preferred_flags: options.preferred_flags - avoided_flags: options.avoided_flags - budget_policy: options.budget_policy + avoided_flags: options.avoided_flags + budget_policy: options.budget_policy } mut buffer := vk.Buffer(unsafe { nil }) mut backing := AllocationInfo{} - result := allocator.create_dedicated_buffer_with_options(&buffer_info, effective_options, &buffer, mut backing) + result := allocator.create_dedicated_buffer_with_options(&buffer_info, effective_options, + &buffer, mut backing) if result != .success { return error('could not create upload buffer: ${result}') } @@ -110,12 +111,12 @@ fn finish_upload_ring(mut allocator Allocator, capacity u64, buffer vk.Buffer, i return error('could not map upload buffer: ${map_result}') } return &UploadRing{ - buffer: buffer - capacity: capacity + buffer: buffer + capacity: capacity allocator: allocator - backing: backing - mapped: mapped - ranges: memory.new_ring_allocator(capacity) + backing: backing + mapped: mapped + ranges: memory.new_ring_allocator(capacity) } } @@ -128,11 +129,11 @@ pub fn (mut ring UploadRing) allocate(size u64, alignment u64) !UploadSlice { allocation := ring.ranges.allocate(size, alignment)! data := unsafe { voidptr(usize(ring.mapped) + usize(allocation.offset)) } return UploadSlice{ - owner: ring + owner: ring allocation: allocation - offset: allocation.offset - size: allocation.size - data: data + offset: allocation.offset + size: allocation.size + data: data } } @@ -180,13 +181,13 @@ pub fn (ring &UploadRing) stats() UploadRingStats { } stats := ring.ranges.stats() return UploadRingStats{ - capacity: stats.capacity - used: stats.used - payload: stats.payload - padding: stats.padding - free: stats.free - peak_used: stats.peak_used - allocation_count: stats.allocation_count + capacity: stats.capacity + used: stats.used + payload: stats.payload + padding: stats.padding + free: stats.free + peak_used: stats.peak_used + allocation_count: stats.allocation_count largest_contiguous_free: stats.largest_contiguous_free } } diff --git a/upload_ring_test.v b/upload_ring_test.v index 6d20852..40230d7 100644 --- a/upload_ring_test.v +++ b/upload_ring_test.v @@ -6,8 +6,8 @@ fn test_upload_ring_returns_aligned_host_pointers_and_wraps() { mut storage := []u8{len: 64} mut uploads := UploadRing{ capacity: 64 - mapped: storage.data - ranges: memory.new_ring_allocator(64) + mapped: storage.data + ranges: memory.new_ring_allocator(64) } first := uploads.allocate(24, 16) or { panic(err) } second := uploads.allocate(24, 16) or { panic(err) } @@ -36,24 +36,24 @@ fn test_upload_ring_rejects_foreign_forged_and_destroyed_slices() { mut second_storage := []u8{len: 32} mut first_ring := UploadRing{ capacity: 32 - mapped: first_storage.data - ranges: memory.new_ring_allocator(32) + mapped: first_storage.data + ranges: memory.new_ring_allocator(32) } mut second_ring := UploadRing{ capacity: 32 - mapped: second_storage.data - ranges: memory.new_ring_allocator(32) + mapped: second_storage.data + ranges: memory.new_ring_allocator(32) } allocation := first_ring.allocate(8, 1) or { panic(err) } foreign := second_ring.allocate(8, 1) or { panic(err) } assert !first_ring.contains(foreign) assert !first_ring.retire(foreign) forged := UploadSlice{ - owner: allocation.owner + owner: allocation.owner allocation: allocation.allocation - offset: allocation.offset - size: allocation.size - data: unsafe { voidptr(usize(allocation.data) + 1) } + offset: allocation.offset + size: allocation.size + data: unsafe { voidptr(usize(allocation.data) + 1) } } assert !first_ring.contains(forged) assert !first_ring.retire(forged) diff --git a/vulkan_memory_allocator.v b/vulkan_memory_allocator.v index e319d6c..3923150 100644 --- a/vulkan_memory_allocator.v +++ b/vulkan_memory_allocator.v @@ -128,7 +128,7 @@ mut: pub struct MemNode { pub mut: alloc_info &AllocationInfo = unsafe { nil } - next &MemNode = unsafe { nil } + next &MemNode = unsafe { nil } } pub struct AllocatorCreateInfo { @@ -168,15 +168,15 @@ pub fn new(create_info AllocatorCreateInfo) Allocator { panic('invalid Vulkan memory block configuration: ${err}') } mut allocator := Allocator{ - create_info: create_info - props: mem_props - api_version: device_props.apiVersion + create_info: create_info + props: mem_props + api_version: device_props.apiVersion non_coherent_atom_size: if device_props.limits.nonCoherentAtomSize > 0 { u64(device_props.limits.nonCoherentAtomSize) } else { u64(1) } - planner: planner + planner: planner } if create_info.memory_budget_enabled { _ = allocator.refresh_memory_budget() @@ -239,7 +239,8 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me choices := ranked_memory_types(a.props, u32(1) << memory_type, req.size, AllocationOptions{ budget_policy: .ignore }, a.heap_budget_snapshot()) - return a.allocate_from_choices(mut req, choices, allocation_pnext, dedicated, .ignore, mut alloc_info) + return a.allocate_from_choices(mut req, choices, allocation_pnext, dedicated, .ignore, mut + alloc_info) } // allocate_with_options reserves isolated memory using the portable ranked @@ -247,7 +248,8 @@ fn (mut a Allocator) allocate_with_policy(mut req vk.MemoryRequirements, type Me // desired. pub fn (mut a Allocator) allocate_with_options(mut req vk.MemoryRequirements, options AllocationOptions, mut alloc_info AllocationInfo) vk.Result { choices := a.rank_memory_types(req.memoryTypeBits, req.size, options) - return a.allocate_from_choices(mut req, choices, unsafe { nil }, true, options.budget_policy, mut alloc_info) + return a.allocate_from_choices(mut req, choices, unsafe { nil }, true, options.budget_policy, mut + alloc_info) } fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choices []MemoryTypeChoice, allocation_pnext voidptr, dedicated bool, budget_policy BudgetPolicy, mut alloc_info AllocationInfo) vk.Result { @@ -266,7 +268,8 @@ fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choice mut last_result := vk.Result.error_out_of_device_memory for choice in choices { allow_new_block := budget_policy != .require_within || choice.within_budget - mut result := a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, allow_new_block, budget_policy, mut alloc_info) + mut result := a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, + allow_new_block, budget_policy, mut alloc_info) if result == .success { return .success } @@ -285,7 +288,8 @@ fn (mut a Allocator) allocate_from_choices(mut req vk.MemoryRequirements, choice } } if trimmed > 0 { - result = a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, true, budget_policy, mut alloc_info) + result = a.allocate_for_memory_type(mut req, choice, allocation_pnext, dedicated, true, + budget_policy, mut alloc_info) if result == .success { return .success } @@ -325,9 +329,9 @@ fn (mut a Allocator) allocate_for_memory_type(mut req vk.MemoryRequirements, cho block_size = choice.remaining_budget } vkalloc_info := vk.MemoryAllocateInfo{ - allocationSize: block_size + allocationSize: block_size memoryTypeIndex: choice.index - pNext: allocation_pnext + pNext: allocation_pnext } mut memory := vk.DeviceMemory(unsafe { nil }) result := vk.allocate_memory(a.create_info.device, &vkalloc_info, unsafe { nil }, &memory) @@ -390,20 +394,17 @@ fn (a &Allocator) owns_allocation(alloc_info AllocationInfo) bool { return false } } - return voidptr(memory) == alloc_info.memory - && alloc_info.offset == alloc_info.reservation.offset - && alloc_info.size == alloc_info.reservation.size - && alloc_info.mem_type == alloc_info.reservation.memory_type - && alloc_info.block_size == (a.planner.block_capacity(alloc_info.reservation.block_id) or { - return false - }) + return voidptr(memory) == alloc_info.memory && alloc_info.offset == alloc_info.reservation.offset && alloc_info.size == alloc_info.reservation.size && alloc_info.mem_type == alloc_info.reservation.memory_type && alloc_info.block_size == (a.planner.block_capacity(alloc_info.reservation.block_id) or { + return false + }) } fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { if a.api_version < vk.api_version_1_1 { mut requirements := vk.MemoryRequirements{} vk.get_buffer_memory_requirements(a.create_info.device, buffer, mut requirements) - return a.allocate_with_policy(mut requirements, type, unsafe { nil }, force_dedicated, mut alloc_info) + return a.allocate_with_policy(mut requirements, type, unsafe { nil }, force_dedicated, mut + alloc_info) } mut dedicated_requirements := vk.MemoryDedicatedRequirements{} mut requirements := vk.MemoryRequirements2{ @@ -416,12 +417,14 @@ fn (mut a Allocator) allocate_buffer_memory(buffer vk.Buffer, type MemType, forc dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true || dedicated_requirements.prefersDedicatedAllocation == vk._true if !dedicated { - return a.allocate_with_policy(mut requirements.memoryRequirements, type, unsafe { nil }, false, mut alloc_info) + return a.allocate_with_policy(mut requirements.memoryRequirements, type, unsafe { nil }, + false, mut alloc_info) } dedicated_info := vk.MemoryDedicatedAllocateInfo{ buffer: buffer } - return a.allocate_with_policy(mut requirements.memoryRequirements, type, voidptr(&dedicated_info), true, mut alloc_info) + return a.allocate_with_policy(mut requirements.memoryRequirements, type, + voidptr(&dedicated_info), true, mut alloc_info) } fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, options AllocationOptions, force_dedicated bool, mut alloc_info AllocationInfo) vk.Result { @@ -433,7 +436,8 @@ fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, optio } else { a.rank_buffer_memory_types(requirements.memoryTypeBits, requirements.size, options) } - return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, force_dedicated, options.budget_policy, mut alloc_info) + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, force_dedicated, + options.budget_policy, mut alloc_info) } mut dedicated_requirements := vk.MemoryDedicatedRequirements{} mut requirements := vk.MemoryRequirements2{ @@ -446,17 +450,21 @@ fn (mut a Allocator) allocate_buffer_memory_with_options(buffer vk.Buffer, optio dedicated := force_dedicated || dedicated_requirements.requiresDedicatedAllocation == vk._true || dedicated_requirements.prefersDedicatedAllocation == vk._true choices := if dedicated { - a.rank_memory_types(requirements.memoryRequirements.memoryTypeBits, requirements.memoryRequirements.size, options) + a.rank_memory_types(requirements.memoryRequirements.memoryTypeBits, + requirements.memoryRequirements.size, options) } else { - a.rank_buffer_memory_types(requirements.memoryRequirements.memoryTypeBits, requirements.memoryRequirements.size, options) + a.rank_buffer_memory_types(requirements.memoryRequirements.memoryTypeBits, + requirements.memoryRequirements.size, options) } if !dedicated { - return a.allocate_from_choices(mut requirements.memoryRequirements, choices, unsafe { nil }, false, options.budget_policy, mut alloc_info) + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, + unsafe { nil }, false, options.budget_policy, mut alloc_info) } dedicated_info := vk.MemoryDedicatedAllocateInfo{ buffer: buffer } - return a.allocate_from_choices(mut requirements.memoryRequirements, choices, voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) + return a.allocate_from_choices(mut requirements.memoryRequirements, choices, + voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) } fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut alloc_info AllocationInfo) vk.Result { @@ -471,7 +479,8 @@ fn (mut a Allocator) allocate_image_memory(image vk.Image, type MemType, mut all dedicated_info := vk.MemoryDedicatedAllocateInfo{ image: image } - return a.allocate_with_policy(mut requirements, type, voidptr(&dedicated_info), true, mut alloc_info) + return a.allocate_with_policy(mut requirements, type, voidptr(&dedicated_info), true, mut + alloc_info) } vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) return a.allocate_with_policy(mut requirements, type, unsafe { nil }, true, mut alloc_info) @@ -490,11 +499,13 @@ fn (mut a Allocator) allocate_image_memory_with_options(image vk.Image, options dedicated_info := vk.MemoryDedicatedAllocateInfo{ image: image } - return a.allocate_from_choices(mut requirements, choices, voidptr(&dedicated_info), true, options.budget_policy, mut alloc_info) + return a.allocate_from_choices(mut requirements, choices, voidptr(&dedicated_info), true, + options.budget_policy, mut alloc_info) } vk.get_image_memory_requirements(a.create_info.device, image, mut requirements) choices := a.rank_memory_types(requirements.memoryTypeBits, requirements.size, options) - return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, true, options.budget_policy, mut alloc_info) + return a.allocate_from_choices(mut requirements, choices, unsafe { nil }, true, + options.budget_policy, mut alloc_info) } // create_buffer creates a buffer, suballocates compatible memory, and binds it. @@ -537,7 +548,8 @@ fn (mut a Allocator) create_buffer_with_options_policy(buffer_info &vk.BufferCre } return result } - result = vk.bind_buffer_memory(a.create_info.device, *buffer, alloc_info.memory, alloc_info.offset) + result = vk.bind_buffer_memory(a.create_info.device, *buffer, alloc_info.memory, + alloc_info.offset) if result != .success { vk.destroy_buffer(a.create_info.device, *buffer, unsafe { nil }) unsafe { @@ -639,7 +651,8 @@ pub fn (mut a Allocator) create_image_with_options(image_info &vk.ImageCreateInf } return result } - result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, alloc_info.offset) + result = vk.bind_image_memory(a.create_info.device, *image, alloc_info.memory, + alloc_info.offset) if result != .success { vk.destroy_image(a.create_info.device, *image, unsafe { nil }) unsafe { @@ -811,14 +824,14 @@ pub fn (a &Allocator) stats() AllocatorStats { } stats := a.planner.stats() return AllocatorStats{ - block_count: stats.block_count - allocation_count: stats.allocation_count - committed: stats.committed - used: stats.used - free: stats.free - free_range_count: stats.free_range_count + block_count: stats.block_count + allocation_count: stats.allocation_count + committed: stats.committed + used: stats.used + free: stats.free + free_range_count: stats.free_range_count largest_free_range: stats.largest_free_range - empty_block_count: stats.empty_block_count + empty_block_count: stats.empty_block_count } } @@ -831,14 +844,14 @@ pub fn (a &Allocator) stats_for_memory_type(memory_type u32) AllocatorStats { } stats := a.planner.stats_for_memory_type(memory_type) return AllocatorStats{ - block_count: stats.block_count - allocation_count: stats.allocation_count - committed: stats.committed - used: stats.used - free: stats.free - free_range_count: stats.free_range_count + block_count: stats.block_count + allocation_count: stats.allocation_count + committed: stats.committed + used: stats.used + free: stats.free + free_range_count: stats.free_range_count largest_free_range: stats.largest_free_range - empty_block_count: stats.empty_block_count + empty_block_count: stats.empty_block_count } }