diff --git a/.github/img/mip_graphic.png b/.github/img/mip_graphics.png similarity index 100% rename from .github/img/mip_graphic.png rename to .github/img/mip_graphics.png diff --git a/.github/img/ray_trace.png b/.github/img/ray_tracing.png similarity index 100% rename from .github/img/ray_trace.png rename to .github/img/ray_tracing.png diff --git a/.gitignore b/.gitignore index f2190ab7..10331561 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ imgui.ini libdxcompiler.so **/*.pak **/*.profraw +fuzz/artifacts +fuzz/corpus +graphify-out diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ec9f22c..e05e9e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,69 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.14.2] - 2026-06-18 + +### Added + +- `Graph` replaces the old `RenderGraph` workflow with command-first graph construction and + `Submission` finalization. +- `stream` module with reusable `CommandStream` values and typed stream arguments. +- `submission` module for prepared submission data, dependency tracking, and graph execution state. +- `node` module with resource node types moved out of the old `graph::node` namespace. +- `driver::graphics` and `cmd::graphics` modules replace the old singular `graphic` modules. +- `driver::ray_tracing` and `cmd::ray_tracing` modules replace the old `ray_trace` modules. +- `vk-graph-window` `Graphchain` API replaces the old display/swapchain presentation wrapper. +- `Fence` driver type and additional swapchain presentation result/error types. +- `checked` feature for runtime graph and stream validation. + +### Changed + +- This release intentionally ignores semver-checks compatibility failures against the published + `0.14.0+alpha` crate. It includes removed modules, renamed types, renamed fields, changed method + signatures, and trait changes while remaining on the `0.14` release line. +- The old pass-oriented `graph` API was replaced by command builder APIs under `cmd` and root-level + `Graph` methods. +- `Display`/`DisplayInfo` presentation APIs moved out of the main crate; use `vk-graph-window` and + `Graphchain` for windowed presentation. +- `GraphicPipeline`, `GraphicPipelineInfo`, `BlendMode`, and `DepthStencilMode` were renamed to + `GraphicsPipeline`, `GraphicsPipelineInfo`, `BlendInfo`, and `DepthStencilInfo`. +- `RayTracePipeline`, `RayTracePipelineInfo`, and `RayTraceShaderGroup` were renamed to + `RayTracingPipeline`, `RayTracingPipelineInfo`, and `RayTracingShaderGroup`. +- `Device::physical_device` was renamed to `Device::physical`. +- Physical-device extension support now uses actual Vulkan extension names such as + `vk_khr_swapchain`, `vk_khr_synchronization2`, and grouped `Option` support structs for + KHR extensions with feature/property data. +- Ambiguous public `ty` fields were renamed to domain-specific names such as `image_type`, + `view_type`, `acceleration_structure_type`, and `shader_group_type`. +- `SpecializationInfo` was renamed to `SpecializationMap`. +- Builder conversion methods now consistently use `into_builder` instead of `to_builder`. +- Device, resource, pipeline, swapchain, and surface APIs were tightened around explicit creation, + ownership, and unsafe raw-handle construction. +- Workspace integration crates now target `vk-graph` `0.14.2` and are versioned as `0.1.1`. + +### Removed + +- Removed the old `graph`, `graph::pass_ref`, and `display` public APIs. +- Removed the old `prelude` re-export surface from the main crate. +- Removed `pool::alias` and `AliasPool`. +- Removed old `Device::create_headless`, `Device::create_display`, `Device::load`, + `Device::format_properties`, `Device::image_format_properties`, and `Device::instance` helpers. +- Removed old `Buffer::access`, `Image::access`, and `AccelerationStructure::access` helpers. +- Removed `Pool::lease`; implement `Pool::resource` for custom pools instead. + +### Fixed + +- Documentation examples now compile as `no_run` doctests instead of being ignored. +- Rustdoc tables and module docs were cleaned up across command, driver, shader, stream, and example + documentation. + ## Planned +- Support https://vulkan.gpuinfo.org/displayextensiondetail.php?extension=VK_EXT_full_screen_exclusive, possibly through extension crate - Fix a few scheduling and merging issues found with specific workloads -- Add optional new-style Vulkan render passes when not on mobile or requested, see if that improves performance by reducing some of the inter-renderpass optimization steps - possibly a build feature +- Add optional new-style Vulkan render passes when not on mobile, or when requested, to see if that improves performance by reducing some of the inter-renderpass optimization steps - possibly a build feature - Clean up graph pass API - the names of the methods are not all consistent or in line with the Vulkan-spec terminology -- Make all Info structs have the same properties: Copy, Exact parameter values sent to Vulkan not magically interpreted and changed by Screen-13 +- Make all Info structs have the same properties: Copy, exact parameter values sent to Vulkan are not magically interpreted and changed by vk-graph - New crate name to reflect the stability and functionality of this crate, retire "QBasic" references, sad face ## [0.13.0] - 2025-12-14 @@ -31,8 +88,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Issue where RenderDoc clears images between renderpasses ("undefined img") due to incorrect usage of image layout barriers -- Out-of-memory errors during image and buffer creation leaked vulkan resource handles -- Build error seen on error on Android and Raspberry Pi (_See [#105](https://github.com/attackgoat/screen-13/pull/105)_) +- Out-of-memory errors during image and buffer creation leaked Vulkan resource handles +- Build error seen on Android and Raspberry Pi (_See [#105](https://github.com/attackgoat/screen-13/pull/105)_) ## [0.12.6] - 2025-05-10 @@ -52,7 +109,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed -- Segmentation fault crash and flickering on MacOS when resizing the swapchain (_See [#99](https://github.com/attackgoat/screen-13/pull/99)_) +- Segmentation fault crash and flickering on macOS when resizing the swapchain (_See [#99](https://github.com/attackgoat/screen-13/pull/99)_) ## [0.12.4] - 2025-03-30 @@ -69,7 +126,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed -- Fix compilation issue on MacOS +- Fix compilation issue on macOS - Remove incorrect debug assertion for swapchain desired image count ## [0.12.2] - 2025-03-24 @@ -82,7 +139,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed -- Swapchain image displays nothing after Window presentaion on linux Mesa drivers +- Swapchain image displays nothing after Window presentation on Linux Mesa drivers ## [0.12.0] - 2025-03-13 @@ -268,7 +325,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed - Improved performance during render graph resolution: `vsm_omni` example now records frames 10% - faster (~100 μs) and complex render graphs may be signifcantly more performant + faster (~100 μs) and complex render graphs may be significantly more performant ## [0.9.3] - 2024-01-30 @@ -439,7 +496,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added -- Mutlisampled anti-aliasing example (MSAA) +- Multisampled anti-aliasing example (MSAA) - `attach_color` and `attach_depth_stencil` functions on `PipelinePassRef` when bound to a `GraphicPipeline` for attachments which would otherwise use `VK_ATTACHMENT_LOAD_OP_DONT_CARE` - `node_info` function on `PassRef` and `PipelinePassRef` which may be accessed while recording passes @@ -469,7 +526,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added -- Fullscreen demostration in `vsm_omni` example using F11 and F12 keys +- Fullscreen demonstration in `vsm_omni` example using F11 and F12 keys - Configurable frames-in-flight setting ## [0.6.1] - 2022-10-16 @@ -492,7 +549,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `clear_color` and `clear_depth_stencil` functions now take the image being cleared: it is now possible to clear and attach, but not store or resolve, an image - `record_`-* methods now also provide a `Bindings` parameter to the recording closure - `RayTracePipeline::group_handle` is now an associated function where previously it was a method -- Many types have been moved betwen modules in order to document things cleary +- Many types have been moved between modules in order to document things clearly ### Removed @@ -617,7 +674,7 @@ _See [#25](https://github.com/attackgoat/screen-13/pull/25) for migration detail ### Changed -- Driver now directly based on vulkan, having removed support for the deprecated Gfx-Hal library +- Driver now directly based on Vulkan, having removed support for the deprecated Gfx-Hal library - Lease/pool functionality simplified: leases are now obtained through a common interface using info - `Engine`/`Program` structures have been merged into a simpler EventLoop structure @@ -655,7 +712,8 @@ _See [#25](https://github.com/attackgoat/screen-13/pull/25) for migration detail platforms and require no bare-metal graphics API knowledge - "Hello, world!" example using a bitmapped font -[Unreleased]: https://github.com/attackgoat/screen-13/compare/v0.12.6...HEAD +[Unreleased]: https://github.com/attackgoat/vk-graph/compare/v0.14.2...HEAD +[0.14.2]: https://crates.io/crates/vk-graph/0.14.2 [0.1.0]: https://crates.io/crates/screen-13/0.1.0 [0.2.0]: https://crates.io/crates/screen-13/0.2.0 [0.3.0]: https://crates.io/crates/screen-13/0.3.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d46ce7fc..d973d53c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Thank you for taking the time to look over this document. You are encouraged to open issues, submit PRs, suggest changes, or anything else you feel might move this project forward. -If you have any questions or would like private coorespondence with the main author, John Wells, +If you have any questions or would like private correspondence with the main author, John Wells, please use john@attackgoat.com. ## Ground Rules @@ -16,9 +16,8 @@ access to make changes within this project and you may be banned entirely. All contributions, ideas, issues, or other efforts you expend on this project must be provided using the existing MIT or Apache 2.0 license agreements applied to this project. This means that -anything you do for this project will be provided to the public without any strings or conditions -attached. You must also have the right to provide any code or ideas under these licenses as you will -retain no ownership or control after contribution. +anything you do for this project will be provided to the public without any extra strings or +conditions attached. You must also have the right to provide any code or ideas under these licenses. ## Technical Requirements @@ -26,7 +25,7 @@ All code must: - Be modern Rust code (currently 2024 edition, latest stable compiler) _or_ GLSL. - Pass `cargo fmt` and `cargo clippy` (debug and release) with no warnings -- Support required platforms: Linux, Mac, Windows +- Support required platforms: Linux, macOS, Windows - Use only `crates.io`-published crates ### Recommendations @@ -37,7 +36,7 @@ All code should: - Provide useful documentation and comments, including private code - Support other platforms: Android, iOS -## How do I get started +## How do I get started? Download the source code and play around with the examples, see README.md for more. @@ -45,7 +44,7 @@ Any and all contributions are acceptable, including major changes to design or c change is big please ask or open an issue to make sure the rest of the community is on board with your new ideas. -### What if my idea is too radical +### What if my idea is too radical? If we cannot fit your change into the existing design without breaking API for other users or creating other headaches then we will add your code into a `contrib` directory or separate branch as @@ -61,4 +60,4 @@ needed. - Change log: Transfer unreleased details to the new version - Change log: Add diff link (they are at the bottom) - Run `cargo publish` -- Github: Add a tag for the version +- GitHub: Add a tag for the version diff --git a/Cargo.toml b/Cargo.toml index 9fe5d54e..49f916db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,17 +24,17 @@ readme = "README.md" log = "0.4" profiling = "1.0" read-only = "0.1" -vk-graph = { path = "", version = "0.14.1" } -vk-graph-egui = { path = "crates/vk-graph-egui", version = "0.1.0" } -vk-graph-fx = { path = "crates/vk-graph-fx", version = "0.1.0" } -vk-graph-hot = { path = "crates/vk-graph-hot", version = "0.1.0" } -vk-graph-imgui = { path = "crates/vk-graph-imgui", version = "0.1.0" } -vk-graph-window = { path = "crates/vk-graph-window", version = "0.1.0" } +vk-graph = { path = "", version = "0.14.2" } +vk-graph-egui = { path = "crates/vk-graph-egui", version = "0.1.1" } +vk-graph-fx = { path = "crates/vk-graph-fx", version = "0.1.1" } +vk-graph-hot = { path = "crates/vk-graph-hot", version = "0.1.1" } +vk-graph-imgui = { path = "crates/vk-graph-imgui", version = "0.1.1" } +vk-graph-window = { path = "crates/vk-graph-window", version = "0.1.1" } winit = "0.30" [package] name = "vk-graph" -version = "0.14.1+beta" +version = "0.14.2" authors = ["John Wells "] edition.workspace = true license.workspace = true @@ -62,6 +62,7 @@ profile-with-tracy = ["profiling/profile-with-tracy"] ash = { version = "0.38.0", default-features = false, features = ["debug", "std"] } ash-window = "0.13" derive_builder = "0.20" +fixedbitset = "0.5" gpu-allocator = "0.28" log.workspace = true ordered-float = "5.1" @@ -70,6 +71,7 @@ paste = "1.0" profiling.workspace = true raw-window-handle = "0.6" read-only.workspace = true +smallvec = "1.15" spirq = "1.2" vk-sync = { version = "0.5", package = "vk-sync-fork" } # // SEE: https://github.com/gwihlidal/vk-sync-rs/pull/4 -> https://github.com/expenses/vk-sync-rs @@ -81,6 +83,7 @@ anyhow = "1.0" bmfont = { version = "0.3", default-features = false } bytemuck = "1.25" clap = { version = "4.5", features = ["derive"] } +criterion = "0.5" glam = { version = "0.32", features = ["bytemuck"] } half = { version = "2.4", features = ["bytemuck"] } hassle-rs = "0.11" @@ -89,15 +92,23 @@ log = "0.4" meshopt = "0.2" polyhedron-ops = ">=0.2, <=0.2.4" pretty_env_logger = "0.5" -puffin = "0.19" -puffin_http = "0.16" +puffin = "0.20" +puffin_http = "0.17" rand = "0.9" reqwest = { version = "0.12", features = ["blocking"] } -vk-graph-fx.workspace = true -vk-graph-imgui.workspace = true -vk-graph-egui.workspace = true -vk-graph-window.workspace = true +vk-graph-fx = { path = "crates/vk-graph-fx" } +vk-graph-imgui = { path = "crates/vk-graph-imgui" } +vk-graph-egui = { path = "crates/vk-graph-egui" } +vk-graph-window = { path = "crates/vk-graph-window" } vk-shader-macros = "0.2" tobj = "4.0" winit.workspace = true winit_input_helper = "0.17" + +[[bench]] +name = "submission_reorder" +harness = false + +[[bench]] +name = "image_swap_access" +harness = false diff --git a/README.md b/README.md index 975ddb17..18eb7781 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ real-world use, and supports modern Vulkan commands[^modern]. ```toml [dependencies] -vk-graph = "0.14" +vk-graph = "0.14.2" ``` [*Changelog*](https://github.com/attackgoat/vk-graph/blob/main/CHANGELOG.md) @@ -37,12 +37,39 @@ fn main() -> Result<(), WindowError> { ## Usage -_vk-graph_ provides a fully-generic graph structure for statically typed access to resources used -while rendering. The `Graph` structure allows Vulkan smart pointer resources to be bound as -"nodes" which may be used by shader pipelines. The graph supports swapchain integration and may be -used to execute custom command streams. +_vk-graph_ centers frame work around a `Graph`. Bind Vulkan smart-pointer resources such as +buffers, images, acceleration structures, and swapchain images into the graph to get statically typed +node handles. Commands then reference those nodes instead of raw Vulkan handles. When recording is +complete, finalize the graph into a `Submission` and submit it with a pool. -Features of the graph: +The normal flow is: + +- Create a `Device`, resources, and pipelines. +- Start a fresh `Graph` for the frame or workload. +- Bind resources into typed nodes with `Graph::bind_resource` or receive nodes from a swapchain + integration crate. +- Record compute, graphics, transfer, or ray tracing commands with `Graph::begin_cmd`. +- Declare each resource access on the command so the graph can build synchronization and pass data. +- Finalize and submit the graph. + +`CommandStream` is available when part of the command graph is reusable. A stream records commands +once with typed arguments, then each frame binds those arguments to the frame's concrete graph nodes. +This is useful for overlays, post-processing, or other repeated command sequences that still need +per-frame resources such as the current swapchain image. + +Device extension support is exposed through the selected physical device: + +```rust +if let Some(ray_tracing) = &device.physical.vk_khr_ray_tracing_pipeline { + println!("max recursion depth: {}", ray_tracing.properties.max_ray_recursion_depth); +} + +if device.physical.vk_khr_synchronization2 { + println!("synchronization2 is available"); +} +``` + +## Features - Compute, graphics, and ray tracing pipelines - Automatic Vulkan management (render passes, subpasses, descriptors, pools, _etc._) @@ -64,7 +91,7 @@ graph .depth_stencil_attachment_image(depth_image, LoadOp::Load, StoreOp::DontCare) .record_cmd(move |cmd| { cmd - .push_constants(some_u8_slice) + .push_constants(0, some_u8_slice) .draw(6, 1, 0, 0); }); ``` @@ -79,9 +106,12 @@ available. - **`checked`** *(enabled by default)* — Runtime validation of common misuse patterns (missing access declarations, buffer bounds, image aspects) that the Vulkan Validation Layer - cannot catch. Disable for zero-overhead in validated releases. + cannot catch, including cross-graph node ownership checks. Disable for zero-overhead in + validated releases. - **`loaded`** *(enabled by default)* — Support searching for the Vulkan loader manually at runtime. - **`linked`** — Link the Vulkan loader at compile time. +- **`ash-molten`** — Enable `ash-molten` support for MoltenVK-based platforms. +- **`parking_lot`** *(enabled by default)* — Use `parking_lot` synchronization primitives. - **`profile-with-*`** — Use the specified profiling backend: `profile-with-puffin`, `profile-with-optick`, `profile-with-superluminal`, or `profile-with-tracy` @@ -101,7 +131,7 @@ To enable logging, set the `RUST_LOG` environment variable to `trace`, `debug`, _You may also filter messages, for example:_ ```bash -RUST_LOG=vk_graph::driver=trace,vk_graph=warn cargo run --example ray_trace +RUST_LOG=vk_graph::driver=trace,vk_graph=warn cargo run --example ray_tracing ``` ``` @@ -140,10 +170,10 @@ Included are some examples you might find helpful: - [`hello_world.rs`](crates/vk-graph-window/examples/hello_world.rs) — Displays a window on the screen. Please start here. - [`triangle.rs`](examples/triangle.rs) — Shaders and full setup of index/vertex buffers; < 100 LOC. -- [`shader-toy/`](examples/shader-toy) — Recreation of a two-pass shader toy using the original +- [`shader-toy/`](examples/shader-toy) — Recreation of a two-pass Shadertoy using the original shader code. -See the [example code](examples/README.md), +See the [example code](examples/README.md), [documentation](https://docs.rs/vk-graph/latest/vk_graph/), or helpful [guide book](https://attackgoat.github.io/vk-graph) for more information. diff --git a/benches/image_swap_access.rs b/benches/image_swap_access.rs new file mode 100644 index 00000000..3358e526 --- /dev/null +++ b/benches/image_swap_access.rs @@ -0,0 +1,290 @@ +use { + ash::vk, + criterion::{Criterion, black_box, criterion_group, criterion_main}, + vk_graph::driver::image::bench::SwapAccessBenchHarness, + vk_sync::AccessType, +}; + +fn full_range(layers: u32, mips: u32) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: 0, + layer_count: layers, + base_mip_level: 0, + level_count: mips, + } +} + +fn full_dual_range(layers: u32, mips: u32) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, + base_array_layer: 0, + layer_count: layers, + base_mip_level: 0, + level_count: mips, + } +} + +fn single_subresource(layer: u32, mip: u32) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: layer, + layer_count: 1, + base_mip_level: mip, + level_count: 1, + } +} + +fn dual_single_subresource( + aspect: vk::ImageAspectFlags, + layer: u32, + mip: u32, +) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: aspect, + base_array_layer: layer, + layer_count: 1, + base_mip_level: mip, + level_count: 1, + } +} + +fn subresource_block( + aspect: vk::ImageAspectFlags, + start_layer: u32, + start_mip: u32, + count_layers: u32, + count_mips: u32, +) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: aspect, + base_array_layer: start_layer, + layer_count: count_layers, + base_mip_level: start_mip, + level_count: count_mips, + } +} + +fn remaining_range() -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: 0, + layer_count: vk::REMAINING_ARRAY_LAYERS, + base_mip_level: 0, + level_count: vk::REMAINING_MIP_LEVELS, + } +} + +fn image_swap_access_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("image_swap_access"); + + // 1. Uniform, 1x1, full range + let h = SwapAccessBenchHarness::new(1, 1, vk::Format::R8G8B8A8_UNORM); + group.bench_function("uniform_1x1_full", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, full_range(1, 1)))); + }); + + // 2. Uniform, 1x1, remaining ranges + let h = SwapAccessBenchHarness::new(1, 1, vk::Format::R8G8B8A8_UNORM); + group.bench_function("uniform_1x1_remaining", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, remaining_range()))); + }); + + // 3. Dual-aspect, full range + let h = SwapAccessBenchHarness::new(1, 1, vk::Format::D32_SFLOAT_S8_UINT); + group.bench_function("dual_aspect_full", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::DepthStencilAttachmentWrite, + full_dual_range(1, 1), + )) + }); + }); + + // 4. Dual-aspect, single aspect + let h = SwapAccessBenchHarness::new(1, 1, vk::Format::D32_SFLOAT_S8_UINT); + group.bench_function("dual_aspect_single_aspect", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::DepthStencilAttachmentWrite, + dual_single_subresource(vk::ImageAspectFlags::DEPTH, 0, 0), + )) + }); + }); + + // 5. Dense, 1x2 mips, full range (promotes + uniform shortcut) + let h = SwapAccessBenchHarness::new(1, 2, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_2mip_full", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, full_range(1, 2)))); + }); + + // 6. Dense, 4x4, single subresource (promotes + dense iteration) + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_4x4_partial", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)))); + }); + + // 7. Dense, 4x4, many subresources (promotes + dense iteration over 2x2) + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_4x4_many", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::AnyShaderWrite, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 2, 2), + )) + }); + }); + + // 8. Dense, 4x4, steady-state partial + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::R8G8B8A8_UNORM); + h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)); + group.bench_function("dense_4x4_partial_steady", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)))); + }); + + // 9. Dense, 4x4, steady-state many subresources + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::R8G8B8A8_UNORM); + h.swap_access( + AccessType::AnyShaderWrite, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 2, 2), + ); + group.bench_function("dense_4x4_many_steady", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::AnyShaderWrite, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 2, 2), + )) + }); + }); + + // 10. Dense, 8x8, single subresource (large promote) + let h = SwapAccessBenchHarness::new(8, 8, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_8x8_partial", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)))); + }); + + // 11. Dense, 8x8, many subresources (large promote + 4x4 iteration) + let h = SwapAccessBenchHarness::new(8, 8, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_8x8_many", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::AnyShaderWrite, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 4, 4), + )) + }); + }); + + // 12. Dense, 8x8, steady-state partial + let h = SwapAccessBenchHarness::new(8, 8, vk::Format::R8G8B8A8_UNORM); + h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)); + group.bench_function("dense_8x8_partial_steady", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)))); + }); + + // 13. Dense, 8x8, steady-state many subresources + let h = SwapAccessBenchHarness::new(8, 8, vk::Format::R8G8B8A8_UNORM); + h.swap_access( + AccessType::AnyShaderWrite, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 4, 4), + ); + group.bench_function("dense_8x8_many_steady", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::AnyShaderWrite, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 4, 4), + )) + }); + }); + + // 14. Dense, 1x8, single subresource (single layer, many mips) + let h = SwapAccessBenchHarness::new(1, 8, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_1x8_partial", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)))); + }); + + // 15. Dense, 8x1, single subresource (many layers, single mip) + let h = SwapAccessBenchHarness::new(8, 1, vk::Format::R8G8B8A8_UNORM); + group.bench_function("dense_8x1_partial", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0)))); + }); + + // 16. Dual-aspect dense, 4x4, partial + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::D32_SFLOAT_S8_UINT); + group.bench_function("dual_aspect_dense_4x4_partial", |b| { + b.iter(|| { + black_box(h.swap_access( + AccessType::DepthStencilAttachmentWrite, + subresource_block( + vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, + 0, + 0, + 1, + 1, + ), + )) + }); + }); + + // 17. Uniform steady-state (pre-warmed) + let h = SwapAccessBenchHarness::new(1, 1, vk::Format::R8G8B8A8_UNORM); + h.swap_access(AccessType::AnyShaderWrite, full_range(1, 1)); + group.bench_function("uniform_steady", |b| { + b.iter(|| black_box(h.swap_access(AccessType::AnyShaderWrite, full_range(1, 1)))); + }); + + // 18. Batch 4 swaps on dense 4x4 — mix of single and multi-subresource ranges + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::R8G8B8A8_UNORM); + group.bench_function("batch_4_swaps", |b| { + b.iter(|| { + black_box(h.swap_access(AccessType::AnyShaderWrite, single_subresource(0, 0))); + black_box(h.swap_access( + AccessType::AnyShaderReadOther, + subresource_block(vk::ImageAspectFlags::COLOR, 1, 0, 2, 1), + )); + black_box(h.swap_access(AccessType::TransferWrite, single_subresource(0, 1))); + black_box(h.swap_access( + AccessType::TransferRead, + subresource_block(vk::ImageAspectFlags::COLOR, 0, 2, 1, 2), + )); + }); + }); + + // 19. Batch 16 swaps on dense 4x4 — mix of single and multi-subresource ranges + let h = SwapAccessBenchHarness::new(4, 4, vk::Format::R8G8B8A8_UNORM); + let access_types = [ + AccessType::AnyShaderWrite, + AccessType::AnyShaderReadOther, + AccessType::TransferWrite, + AccessType::TransferRead, + ]; + let ranges = [ + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 2, 2), + single_subresource(1, 2), + single_subresource(0, 3), + subresource_block(vk::ImageAspectFlags::COLOR, 2, 0, 2, 1), + single_subresource(2, 2), + single_subresource(3, 2), + single_subresource(2, 3), + subresource_block(vk::ImageAspectFlags::COLOR, 0, 1, 1, 2), + single_subresource(3, 0), + single_subresource(0, 2), + subresource_block(vk::ImageAspectFlags::COLOR, 1, 3, 2, 1), + single_subresource(1, 1), + single_subresource(2, 0), + single_subresource(3, 3), + subresource_block(vk::ImageAspectFlags::COLOR, 0, 0, 1, 1), + single_subresource(3, 1), + ]; + group.bench_function("batch_16_swaps", |b| { + b.iter(|| { + for i in 0..16 { + black_box(h.swap_access(access_types[i & 3], ranges[i])); + } + }); + }); + + group.finish(); +} + +criterion_group!(benches, image_swap_access_bench); +criterion_main!(benches); diff --git a/benches/submission_reorder.rs b/benches/submission_reorder.rs new file mode 100644 index 00000000..2b60993b --- /dev/null +++ b/benches/submission_reorder.rs @@ -0,0 +1,69 @@ +use { + criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}, + vk_graph::submission::bench::{ReorderBenchHarness, ReorderBenchSpec}, +}; + +fn submission_reorder_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("submission_reorder"); + + for (shape, spec) in [ + ( + "sparse", + ReorderBenchSpec { + cmd_count: 128, + resource_count: 64, + short_lived_uses: 3, + long_lived_resource_count: 0, + long_lived_uses: 0, + }, + ), + ( + "mixed", + ReorderBenchSpec { + cmd_count: 512, + resource_count: 128, + short_lived_uses: 3, + long_lived_resource_count: 4, + long_lived_uses: 96, + }, + ), + ( + "mixed", + ReorderBenchSpec { + cmd_count: 1024, + resource_count: 192, + short_lived_uses: 3, + long_lived_resource_count: 8, + long_lived_uses: 160, + }, + ), + ( + "mixed", + ReorderBenchSpec { + cmd_count: 2000, + resource_count: 256, + short_lived_uses: 2, + long_lived_resource_count: 12, + long_lived_uses: 220, + }, + ), + ] { + let mut harness = ReorderBenchHarness::new(spec); + group.throughput(Throughput::Elements(spec.cmd_count as u64)); + group.bench_with_input( + BenchmarkId::new( + shape, + format!("{}c_{}r", spec.cmd_count, spec.resource_count), + ), + &spec, + |b, _| { + b.iter(|| black_box(harness.reorder_once())); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, submission_reorder_bench); +criterion_main!(benches); diff --git a/bin/check b/bin/check index 0408ba2d..0ba18d8f 100755 --- a/bin/check +++ b/bin/check @@ -68,7 +68,7 @@ run "workspace clippy (no checked)" cargo clippy $PROFILE --all-targets --worksp run "workspace tests" env RUSTFLAGS="-D warnings" cargo test $PROFILE --all-targets --workspace --no-default-features --features loaded if [ "$RUN_VULKAN" -eq 1 ]; then - run "integration tests" env RUSTFLAGS="-D warnings" cargo test $PROFILE --workspace integration -- --ignored --test-threads=1 + run "vulkan tests" env RUSTFLAGS="-D warnings" cargo test $PROFILE --workspace vulkan -- --ignored --test-threads=1 fi run "workspace doctests" env RUSTFLAGS="-D warnings" cargo test $PROFILE --workspace --doc diff --git a/bin/run-all-examples b/bin/run-all-examples index c54741d7..8e67345f 100755 --- a/bin/run-all-examples +++ b/bin/run-all-examples @@ -2,42 +2,112 @@ set -e +TIMEOUT= + +if [ "$#" -gt 0 ]; then + if [ "$1" = "--timeout" ] && [ "$#" -ge 2 ]; then + TIMEOUT=$2 + shift 2 + else + printf 'usage: %s [--timeout SECONDS]\n' "$0" >&2 + exit 1 + fi +fi + +xdotool="" +if [ -n "$TIMEOUT" ]; then + if [ -n "$DISPLAY" ] \ + && command -v xdotool >/dev/null 2>&1 \ + && xdotool getdisplaygeometry >/dev/null 2>&1; then + xdotool="xdotool" + else + printf "run-all-examples: WARNING xdotool unavailable, using timeout fallback" >&2 + fi +fi + +close_windows() { + _children=$(pgrep -P "$1" 2>/dev/null || true) + + # Try to close any X11 windows owned by PID $1, ignoring failure + "$xdotool" search --pid "$1" windowclose >/dev/null 2>&1 || true + + for _ch in $_children; do + close_windows "$_ch" + done +} + +ok_or_timeout() { + if "$@"; then + return 0 + fi + + _ret=$? + [ "$_ret" -eq 124 ] && return 0 + return "$_ret" +} + +run() { + # If there is no timeout we just run the command + if [ -z "$TIMEOUT" ]; then + "$@" + return + fi + + if [ -n "$xdotool" ]; then + # Close the window with a window message after a timeout + "$@" & + _PID=$! + sleep "$TIMEOUT" + close_windows "$_PID" + ok_or_timeout wait "$_PID" + else + # Kill the process after a timeout + ok_or_timeout timeout "$TIMEOUT" "$@" + fi +} + +# Build all examples and bins first to get failures fast +cargo build --workspace --examples --bins + # Run the "test" example first cargo run --example fuzzer # Run all regular examples, in debug mode, next -cargo run -p vk-graph-window --example hello_world -- --debug -cargo run --example app -- --debug -cargo run --example aliasing -- --debug -cargo run --example cpu_readback -- --debug -cargo run --example subgroup_ops -- --debug -cargo run --example bindless -- --debug -cargo run --example image_sampler -- --debug -cargo run --example image_sampler -- --debug --hlsl -cargo run --example image_sampler -- --debug --separate -cargo run --example image_sampler -- --debug --hlsl --separate -cargo run --example vertex_layout -- --debug -cargo run --example font_bmp -- --debug -cargo run --example egui -- --debug -cargo run --example imgui -- --debug -cargo run --example min_max -- --debug -cargo run --example mip_compute -- --debug -cargo run --example mip_graphic -- --debug -cargo run --example multithread -- --debug -cargo run --example multipass -- --debug -cargo run --example msaa -- --debug -cargo run --example triangle -- --debug -cargo run --example rt_triangle -- --debug -cargo run --example ray_trace -- --debug -cargo run --example transitions -- --debug -cargo run --example vsm_omni -- --debug -cargo run --example vsm_omni -- --debug --geometry-shader -cargo run --example ray_omni -- --debug -cargo run -p skeletal-anim -- --debug +run cargo run -p vk-graph-window --example hello_world -- --debug +run cargo run --example app -- --debug +run cargo run --example aliasing -- --debug +run cargo run --example cpu_readback -- --debug +run cargo run --example subgroup_ops -- --debug +run cargo run --example bindless -- --debug +run cargo run --example image_sampler -- --debug +run cargo run --example image_sampler -- --debug --hlsl +run cargo run --example image_sampler -- --debug --separate +run cargo run --example image_sampler -- --debug --hlsl --separate +run cargo run --example vertex_layout -- --debug +run cargo run --example font_bmp -- --debug +run cargo run --example egui -- --debug +run cargo run --example imgui -- --debug +run cargo run --example min_max -- --debug +run cargo run --example mip_compute -- --debug +run cargo run --example mip_graphics -- --debug +run cargo run --example multithread -- --debug +run cargo run --example multithread -- --debug --concurrent +run cargo run --example multithread --release +run cargo run --example multithread --release -- --concurrent +run cargo run --example multipass -- --debug +run cargo run --example msaa -- --debug +run cargo run --example triangle -- --debug +run cargo run --example rt_triangle -- --debug +run cargo run --example ray_tracing -- --debug +run cargo run --example transitions -- --debug +run cargo run --example vsm_omni -- --debug +run cargo run --example vsm_omni -- --debug --geometry-shader +run cargo run --example ray_omni -- --debug +run cargo run -p skeletal-anim -- --debug # Hot-reload examples -cargo run -p vk-graph-hot --example glsl -- --debug -cargo run -p vk-graph-hot --example hlsl -- --debug +run cargo run -p vk-graph-hot --example glsl -- --debug +run cargo run -p vk-graph-hot --example hlsl -- --debug # Run this one in release mode -cargo run -p shader-toy --release -- --debug +run cargo run -p shader-toy --release -- --debug diff --git a/crates/vk-graph-egui/CHANGELOG.md b/crates/vk-graph-egui/CHANGELOG.md index 97d366ed..5ecea9ba 100644 --- a/crates/vk-graph-egui/CHANGELOG.md +++ b/crates/vk-graph-egui/CHANGELOG.md @@ -6,6 +6,10 @@ This crate is versioned independently from `vk-graph`. ## [Unreleased] +## [0.1.1] - 2026-06-18 + +- Supports `vk-graph` `0.14.2`. + ## [0.1.0] - 2026-05-30 - Initial release. diff --git a/crates/vk-graph-egui/Cargo.toml b/crates/vk-graph-egui/Cargo.toml index 27f12328..14cbaf07 100644 --- a/crates/vk-graph-egui/Cargo.toml +++ b/crates/vk-graph-egui/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vk-graph-egui" -version = "0.1.0" +version = "0.1.1" authors = ["Christian Döring "] description = "egui renderer integration for vk-graph" documentation = "https://docs.rs/vk-graph-egui" diff --git a/crates/vk-graph-egui/README.md b/crates/vk-graph-egui/README.md index d3de7428..fc1c2972 100644 --- a/crates/vk-graph-egui/README.md +++ b/crates/vk-graph-egui/README.md @@ -3,7 +3,7 @@ Renderer integration for [egui](https://github.com/emilk/egui) on top of `vk-graph`. This crate is versioned independently from `vk-graph`. -Preview +Preview See the [example code](../../examples/README.md) and the workspace [guide book](https://attackgoat.github.io/vk-graph). diff --git a/crates/vk-graph-egui/src/lib.rs b/crates/vk-graph-egui/src/lib.rs index a6e93629..c8cfde13 100644 --- a/crates/vk-graph-egui/src/lib.rs +++ b/crates/vk-graph-egui/src/lib.rs @@ -10,7 +10,7 @@ use vk_graph::{ ash::vk, buffer::BufferInfo, device::Device, - graphic::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::{Image, ImageInfo}, shader::Shader, sync::AccessType, @@ -70,7 +70,7 @@ impl Egui { let ctx = egui::Context::default(); let max_texture_side = Some( device - .physical_device + .physical .properties_v1_0 .limits .max_image_dimension2_d as usize, @@ -171,12 +171,12 @@ impl Egui { }) .collect::>(); - // Bind the rest of the textures. + // Bind the rest of the textures for (id, image) in self.textures.drain() { bound_tex.insert(id, AnyImageNode::from(graph.bind_resource(image))); } - // Add user textures. + // Add user textures for (id, node) in self.user_textures.drain() { bound_tex.insert(id, node); } @@ -192,14 +192,14 @@ impl Egui { ) { // Unbind textures for (id, tex) in bound_tex.iter() { - if let AnyImageNode::ImageLease(tex) = tex + if let AnyImageNode::Pooled(tex) = tex && let egui::TextureId::Managed(_) = *id { self.textures.insert(*id, graph.resource(*tex).clone()); } } - // Free textures. + // Free textures for id in deltas.free.iter() { self.textures.remove(id); } @@ -304,7 +304,7 @@ impl Egui { .draw_indexed(num_indices, 1, 0, 0, 0); }); } - _ => panic!("Primitiv callback not yet supported."), + _ => panic!("Primitive callback not yet supported."), } } } @@ -318,7 +318,7 @@ impl Egui { graph: &mut Graph, ui_fn: impl FnMut(&mut egui::Ui), ) { - // Update events and generate shapes and texture deltas. + // Update events and generate shapes and texture deltas for event in events { if let Event::WindowEvent { event, .. } = event { #[allow(unused_must_use)] diff --git a/crates/vk-graph-fx/CHANGELOG.md b/crates/vk-graph-fx/CHANGELOG.md index 97d366ed..5ecea9ba 100644 --- a/crates/vk-graph-fx/CHANGELOG.md +++ b/crates/vk-graph-fx/CHANGELOG.md @@ -6,6 +6,10 @@ This crate is versioned independently from `vk-graph`. ## [Unreleased] +## [0.1.1] - 2026-06-18 + +- Supports `vk-graph` `0.14.2`. + ## [0.1.0] - 2026-05-30 - Initial release. diff --git a/crates/vk-graph-fx/Cargo.toml b/crates/vk-graph-fx/Cargo.toml index 455a38bf..935354f7 100644 --- a/crates/vk-graph-fx/Cargo.toml +++ b/crates/vk-graph-fx/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vk-graph-fx" -version = "0.1.0" +version = "0.1.1" authors = ["John Wells "] documentation = "https://docs.rs/vk-graph-fx" homepage = "https://github.com/attackgoat/vk-graph/tree/main/crates/vk-graph-fx" diff --git a/crates/vk-graph-fx/res/shader/graphic/font.frag b/crates/vk-graph-fx/res/shader/graphics/font.frag similarity index 99% rename from crates/vk-graph-fx/res/shader/graphic/font.frag rename to crates/vk-graph-fx/res/shader/graphics/font.frag index 0682663b..645bfdac 100644 --- a/crates/vk-graph-fx/res/shader/graphic/font.frag +++ b/crates/vk-graph-fx/res/shader/graphics/font.frag @@ -18,4 +18,4 @@ void main() { vec2 page_colors = texture(pages_sampler_nnr[page_in], texcoord).rg; color = page_colors.r * push_constants.glyph_color + page_colors.g * push_constants.outline_color; -} \ No newline at end of file +} diff --git a/crates/vk-graph-fx/res/shader/graphic/font.vert b/crates/vk-graph-fx/res/shader/graphics/font.vert similarity index 99% rename from crates/vk-graph-fx/res/shader/graphic/font.vert rename to crates/vk-graph-fx/res/shader/graphics/font.vert index 62bf8e3c..3b530239 100644 --- a/crates/vk-graph-fx/res/shader/graphic/font.vert +++ b/crates/vk-graph-fx/res/shader/graphics/font.vert @@ -22,4 +22,4 @@ void main() { gl_Position = push_constants.view_proj * vec4(position_in * push_constants.framebuffer_extent_inverse, 0, 1); -} \ No newline at end of file +} diff --git a/crates/vk-graph-fx/res/shader/graphic/present.frag b/crates/vk-graph-fx/res/shader/graphics/present.frag similarity index 100% rename from crates/vk-graph-fx/res/shader/graphic/present.frag rename to crates/vk-graph-fx/res/shader/graphics/present.frag diff --git a/crates/vk-graph-fx/res/shader/graphic/present.vert b/crates/vk-graph-fx/res/shader/graphics/present.vert similarity index 100% rename from crates/vk-graph-fx/res/shader/graphic/present.vert rename to crates/vk-graph-fx/res/shader/graphics/present.vert diff --git a/crates/vk-graph-fx/res/shader/inc/catmull_rom.glsl b/crates/vk-graph-fx/res/shader/inc/catmull_rom.glsl index 20e6234c..a222f692 100644 --- a/crates/vk-graph-fx/res/shader/inc/catmull_rom.glsl +++ b/crates/vk-graph-fx/res/shader/inc/catmull_rom.glsl @@ -17,7 +17,7 @@ vec4 sample_catmull_rom(sampler2D image, vec2 uv) { vec2 image_size = vec2(textureSize(image, 0)); - // We're going to sample a a 4x4 grid of texels surrounding the target UV coordinate. We'll do this by rounding + // We're going to sample a 4x4 grid of texels surrounding the target UV coordinate. We'll do this by rounding // down the sample location to get the exact center of our "starting" texel. The starting texel will be at // location [1, 1] in the grid, where [0, 0] is the top left corner. vec2 samplePos = uv * image_size; @@ -93,4 +93,4 @@ void mainImage( out vec4 fragColor, in vec2 fragCoord ) fragColor -= step(abs(uv.x-0.5), 0.001); fragColor.rgb += (hash32n(uv+fract(iTime))+hash32n(uv+0.1337*fract(iTime))-1.0)/255.0; //dither output } -*/ \ No newline at end of file +*/ diff --git a/crates/vk-graph-fx/src/bitmap_font.rs b/crates/vk-graph-fx/src/bitmap_font.rs index 89b71754..a48418a4 100644 --- a/crates/vk-graph-fx/src/bitmap_font.rs +++ b/crates/vk-graph-fx/src/bitmap_font.rs @@ -11,7 +11,7 @@ use { ash::vk, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::Image, shader::{Shader, SpecializationMap}, sync::AccessType, @@ -56,8 +56,8 @@ impl BitmapFont { device, GraphicsPipelineInfo::builder().blend(BlendInfo::ALPHA), [ - Shader::new_vertex(include_glsl!("res/shader/graphic/font.vert").as_slice()), - Shader::new_fragment(include_glsl!("res/shader/graphic/font.frag").as_slice()) + Shader::new_vertex(include_glsl!("res/shader/graphics/font.vert").as_slice()), + Shader::new_fragment(include_glsl!("res/shader/graphics/font.frag").as_slice()) .specialization( SpecializationMap::new(num_pages.to_ne_bytes()).constant(0, 0, 4), ), @@ -264,18 +264,18 @@ pub enum BitmapGlyphColor { } impl BitmapGlyphColor { - const TRANSARENT: Color = [0, 0, 0, u8::MAX]; + const TRANSPARENT: Color = [0, 0, 0, u8::MAX]; fn outline(&self) -> Color { match self { Self::Outline(color) => *color, - _ => Self::TRANSARENT, + _ => Self::TRANSPARENT, } } fn solid(&self) -> Color { match self { - Self::Outline(_) => Self::TRANSARENT, + Self::Outline(_) => Self::TRANSPARENT, Self::Solid(color) => *color, Self::SolidOutline(color, _) => *color, } diff --git a/crates/vk-graph-fx/src/image_loader.rs b/crates/vk-graph-fx/src/image_loader.rs index 9269f5e7..bab9de25 100644 --- a/crates/vk-graph-fx/src/image_loader.rs +++ b/crates/vk-graph-fx/src/image_loader.rs @@ -250,7 +250,7 @@ impl ImageLoader { let image = graph.resource(image).clone(); graph - .into_submission() + .finalize() .queue_submit(&mut self.pool, queue_family_index, queue_index)?; Ok(image) diff --git a/crates/vk-graph-fx/src/presenter.rs b/crates/vk-graph-fx/src/presenter.rs index e0a0f903..e0bb95ba 100644 --- a/crates/vk-graph-fx/src/presenter.rs +++ b/crates/vk-graph-fx/src/presenter.rs @@ -8,7 +8,7 @@ use { DriverError, compute::{ComputePipeline, ComputePipelineInfo}, device::Device, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, shader::Shader, sync::AccessType, }, @@ -90,7 +90,7 @@ impl ComputePresenter { } } -/// Graphic-pipeline presenter for drawing an image into a swapchain image. +/// Graphics pipeline presenter for drawing an image into a swapchain image. pub struct GraphicPresenter { pipeline: GraphicsPipeline, } @@ -102,15 +102,15 @@ impl GraphicPresenter { device, GraphicsPipelineInfo::default(), [ - Shader::new_vertex(include_glsl!("res/shader/graphic/present.vert").as_slice()), - Shader::new_fragment(include_glsl!("res/shader/graphic/present.frag").as_slice()), + Shader::new_vertex(include_glsl!("res/shader/graphics/present.vert").as_slice()), + Shader::new_fragment(include_glsl!("res/shader/graphics/present.frag").as_slice()), ], )?; Ok(Self { pipeline }) } - /// Draws the given image into the swapchain image using a fullscreen graphic pass. + /// Draws the given image into the swapchain image using a fullscreen graphics pass. pub fn present_image( &self, graph: &mut Graph, @@ -134,7 +134,7 @@ impl GraphicPresenter { graph .begin_cmd() - .debug_name("present (from graphic)") + .debug_name("present (from graphics)") .bind_pipeline(&self.pipeline) .shader_resource_access( 0, diff --git a/crates/vk-graph-fx/src/transition.rs b/crates/vk-graph-fx/src/transition.rs index 09fe8c09..4d2137c2 100644 --- a/crates/vk-graph-fx/src/transition.rs +++ b/crates/vk-graph-fx/src/transition.rs @@ -1,6 +1,6 @@ // Adapted from https://github.com/gl-transitions/gl-transitions // NOTE: Some are rough or broken and need a bit of care - others should be optimized for production -// use. +// use use { log::trace, @@ -143,8 +143,8 @@ pub enum Transition { }, Fade, FadeGrayscale { - /// if 0.0, the image directly turn grayscale, if 0.9, the grayscale transition phase is - /// very important + /// If `0.0`, the image turns grayscale directly; if `0.9`, the grayscale transition phase + /// is very important. intensity: f32, }, FilmBurn { @@ -183,13 +183,13 @@ pub enum Transition { luma_map: AnyImageNode, }, LuminanceMelt { - /// Direction of movement : 0 : up, 1, down + /// Direction of movement: `false` for up, `true` for down. direction: bool, - /// Luminance threshold + /// Luminance threshold. threshold: f32, - /// Does the movement takes effect above or below luminance threshold ? + /// Whether movement takes effect above or below the luminance threshold. above: bool, }, Morph { @@ -400,7 +400,6 @@ impl Transition { /// Cache and lazily create transition compute pipelines on demand. pub struct TransitionPipeline { cache: HashPool, - device: Device, // TODO REMOVE pipelines: HashMap, } @@ -408,14 +407,9 @@ impl TransitionPipeline { /// Creates an empty transition pipeline cache for the given device. pub fn new(device: &Device) -> Self { let cache = HashPool::new(device); - let device = device.clone(); let pipelines = Default::default(); - Self { - cache, - device, - pipelines, - } + Self { cache, pipelines } } /// Applies a transition between two images and returns a pooled destination image. @@ -479,7 +473,6 @@ impl TransitionPipeline { extend_push_constants(transition, &mut push_consts); - // TODO: Handle displacement and luma in an if case, below graph .begin_cmd() .debug_name(format!("transition {transition_ty:?}")) @@ -498,7 +491,7 @@ impl TransitionPipeline { trace!("creating {transition_ty:?}"); ComputePipeline::create( - &self.device, + &self.cache.device, ComputePipelineInfo::default(), Shader::new_compute(match transition_ty { TransitionType::Angular => { diff --git a/crates/vk-graph-hot/CHANGELOG.md b/crates/vk-graph-hot/CHANGELOG.md index 97d366ed..5ecea9ba 100644 --- a/crates/vk-graph-hot/CHANGELOG.md +++ b/crates/vk-graph-hot/CHANGELOG.md @@ -6,6 +6,10 @@ This crate is versioned independently from `vk-graph`. ## [Unreleased] +## [0.1.1] - 2026-06-18 + +- Supports `vk-graph` `0.14.2`. + ## [0.1.0] - 2026-05-30 - Initial release. diff --git a/crates/vk-graph-hot/Cargo.toml b/crates/vk-graph-hot/Cargo.toml index b54fe295..055c88fa 100644 --- a/crates/vk-graph-hot/Cargo.toml +++ b/crates/vk-graph-hot/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "vk-graph-hot" -version = "0.1.0" +version = "0.1.1" authors = ["John Wells "] -homepage = "https://github.com/attackgoat/vk-graph/crates/vk-graph-hot" +homepage = "https://github.com/attackgoat/vk-graph/tree/main/crates/vk-graph-hot" documentation = "https://docs.rs/vk-graph-hot" keywords = ["gamedev", "vulkan"] categories = ["game-development", "multimedia::images", "rendering::engine"] diff --git a/crates/vk-graph-hot/examples/README.md b/crates/vk-graph-hot/examples/README.md index b87428e0..7878d992 100644 --- a/crates/vk-graph-hot/examples/README.md +++ b/crates/vk-graph-hot/examples/README.md @@ -2,14 +2,14 @@ ## Getting Started -Hot-reloading shader pipelines are drop-in replacements for regular shader pipelines. Reference the -below code to get started. +Hot-reloading shader pipelines are drop-in replacements for regular shader pipelines. Use the code +below to get started. See the [README](../README.md) for more information. ## Example Code -Example | Instructions | Preview - --- | --- | :---: -[glsl.rs](glsl.rs) | 
cargo run --example glsl
| Preview -[hlsl.rs](hlsl.rs) | 
cargo run --example hlsl
| Preview +| Example | Instructions | Preview | +| --- | --- | :---: | +| [glsl.rs](glsl.rs) | 
cargo run --example glsl
| Preview | +| [hlsl.rs](hlsl.rs) | 
cargo run --example hlsl
| Preview | diff --git a/crates/vk-graph-hot/examples/hlsl.rs b/crates/vk-graph-hot/examples/hlsl.rs index c0cc5c46..2c214450 100644 --- a/crates/vk-graph-hot/examples/hlsl.rs +++ b/crates/vk-graph-hot/examples/hlsl.rs @@ -3,7 +3,7 @@ use { std::path::PathBuf, vk_graph::{ cmd::{LoadOp, StoreOp}, - driver::graphic::GraphicsPipelineInfo, + driver::graphics::GraphicsPipelineInfo, }, vk_graph_hot::{HotGraphicsPipeline, HotShader}, vk_graph_window::{Window, WindowError}, diff --git a/crates/vk-graph-hot/src/graphic.rs b/crates/vk-graph-hot/src/graphics.rs similarity index 97% rename from crates/vk-graph-hot/src/graphic.rs rename to crates/vk-graph-hot/src/graphics.rs index 1092e8f7..f5438071 100644 --- a/crates/vk-graph-hot/src/graphic.rs +++ b/crates/vk-graph-hot/src/graphics.rs @@ -12,7 +12,7 @@ use { driver::{ DriverError, device::Device, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, }, }, }; diff --git a/crates/vk-graph-hot/src/lib.rs b/crates/vk-graph-hot/src/lib.rs index b8aed9b3..7cffd893 100644 --- a/crates/vk-graph-hot/src/lib.rs +++ b/crates/vk-graph-hot/src/lib.rs @@ -4,14 +4,14 @@ #![deny(rustdoc::broken_intra_doc_links)] mod compute; -mod graphic; -mod ray_trace; +mod graphics; +mod ray_tracing; mod shader; pub use self::{ compute::HotComputePipeline, - graphic::HotGraphicsPipeline, - ray_trace::HotRayTracingPipeline, + graphics::HotGraphicsPipeline, + ray_tracing::HotRayTracingPipeline, shader::{HotShader, HotShaderBuilder}, }; @@ -270,11 +270,6 @@ macro_rules! pipeline { } impl [] { - /// Gets the debugging name assigned to this pipeline, if one has been set. - pub fn debug_name(&self) -> Option { - self.cache().pipeline.debug_name().map(ToOwned::to_owned) - } - /// The device which owns this pipeline. pub fn device(&self) -> &Device { &self.device @@ -289,15 +284,15 @@ macro_rules! pipeline { /// /// _Note:_ The pipeline name may only be assigned once. /// Subsequent calls will not update the previously set name value. - pub fn set_debug_name(&mut self, name: impl Into) { - self.cache_mut().pipeline.set_debug_name(name); + pub fn set_debug_name(&self, name: impl AsRef) { + self.cache().pipeline.set_debug_name(name); } /// Sets the debugging name assigned to this pipeline. /// /// _Note:_ The pipeline name may only be assigned once. /// Subsequent calls will not update the previously set name value. - pub fn with_debug_name(mut self, name: impl Into) -> Self { + pub fn with_debug_name(self, name: impl AsRef) -> Self { self.set_debug_name(name); self diff --git a/crates/vk-graph-hot/src/ray_trace.rs b/crates/vk-graph-hot/src/ray_tracing.rs similarity index 96% rename from crates/vk-graph-hot/src/ray_trace.rs rename to crates/vk-graph-hot/src/ray_tracing.rs index d83e6a8e..95c99002 100644 --- a/crates/vk-graph-hot/src/ray_trace.rs +++ b/crates/vk-graph-hot/src/ray_tracing.rs @@ -15,7 +15,7 @@ use { driver::{ DriverError, device::Device, - ray_trace::{RayTracingPipeline, RayTracingPipelineInfo, RayTracingShaderGroup}, + ray_tracing::{RayTracingPipeline, RayTracingPipelineInfo, RayTracingShaderGroup}, }, }, }; diff --git a/crates/vk-graph-hot/src/shader.rs b/crates/vk-graph-hot/src/shader.rs index b718518b..981dcf89 100644 --- a/crates/vk-graph-hot/src/shader.rs +++ b/crates/vk-graph-hot/src/shader.rs @@ -102,127 +102,71 @@ impl HotShader { } /// Creates a new ray tracing shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_any_hit(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::ANY_HIT_KHR, path) } /// Creates a new ray tracing shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_callable(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::CALLABLE_KHR, path) } /// Creates a new ray tracing shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_closest_hit(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::CLOSEST_HIT_KHR, path) } /// Creates a new compute shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_compute(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::COMPUTE, path) } /// Creates a new fragment shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_fragment(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::FRAGMENT, path) } /// Creates a new geometry shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_geometry(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::GEOMETRY, path) } /// Creates a new ray tracing shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_intersection(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::INTERSECTION_KHR, path) } /// Creates a new mesh shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_mesh(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::MESH_EXT, path) } /// Creates a new ray tracing shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_miss(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::MISS_KHR, path) } /// Creates a new ray tracing shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_ray_gen(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::RAYGEN_KHR, path) } /// Creates a new mesh task shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_task(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::TASK_EXT, path) } /// Creates a new tessellation control shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_tessellation_ctrl(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::TESSELLATION_CONTROL, path) } /// Creates a new tessellation evaluation shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_tessellation_eval(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::TESSELLATION_EVALUATION, path) } /// Creates a new vertex shader. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn new_vertex(path: impl AsRef) -> HotShaderBuilder { Self::new(vk::ShaderStageFlags::VERTEX, path) } @@ -324,10 +268,6 @@ impl HotShader { } /// Creates a shader using a shader kind inferred from the source code. - /// - /// # Panics - /// - /// If the shader code is invalid. pub fn from_path(path: impl AsRef) -> HotShaderBuilder { HotShaderBuilder::default().path(path) } diff --git a/crates/vk-graph-imgui/CHANGELOG.md b/crates/vk-graph-imgui/CHANGELOG.md index 97d366ed..5ecea9ba 100644 --- a/crates/vk-graph-imgui/CHANGELOG.md +++ b/crates/vk-graph-imgui/CHANGELOG.md @@ -6,6 +6,10 @@ This crate is versioned independently from `vk-graph`. ## [Unreleased] +## [0.1.1] - 2026-06-18 + +- Supports `vk-graph` `0.14.2`. + ## [0.1.0] - 2026-05-30 - Initial release. diff --git a/crates/vk-graph-imgui/Cargo.toml b/crates/vk-graph-imgui/Cargo.toml index 1b7c8ee3..934a93f2 100644 --- a/crates/vk-graph-imgui/Cargo.toml +++ b/crates/vk-graph-imgui/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vk-graph-imgui" -version = "0.1.0" +version = "0.1.1" authors = ["John Wells "] description = "Dear ImGui renderer integration for vk-graph" documentation = "https://docs.rs/vk-graph-imgui" diff --git a/crates/vk-graph-imgui/README.md b/crates/vk-graph-imgui/README.md index 3bb590e8..f92dfdba 100644 --- a/crates/vk-graph-imgui/README.md +++ b/crates/vk-graph-imgui/README.md @@ -1,9 +1,9 @@ -# _vk-graph_ _Dear imGui_ Integration +# _vk-graph_ _Dear ImGui_ Integration Renderer integration for [Dear ImGui](https://github.com/imgui-rs/imgui-rs) on top of `vk-graph`. This crate is versioned independently from `vk-graph`. -Preview +Preview See the [example code](../../examples/README.md) and the workspace [guide book](https://attackgoat.github.io/vk-graph). diff --git a/crates/vk-graph-imgui/src/lib.rs b/crates/vk-graph-imgui/src/lib.rs index 94c11a80..7bbda44e 100644 --- a/crates/vk-graph-imgui/src/lib.rs +++ b/crates/vk-graph-imgui/src/lib.rs @@ -28,7 +28,7 @@ use { ash::vk, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::{Image, ImageInfo}, shader::Shader, sync::AccessType, @@ -96,8 +96,10 @@ impl ImGui { } } - // TODO: This produces an image which is RGBA8 UNORM and has STORAGE set. *We* don't need - // storage here and should instead ask the user what settings to give the output image..... + /* + TODO: This produces an image which is RGBA8 UNORM and has STORAGE set. *We* don't need storage + here and should instead ask the user what settings to give the output image. + */ /// Builds a frame, records the necessary draw commands, and returns the rendered image. pub fn draw

( &mut self, @@ -139,24 +141,21 @@ impl ImGui { self.platform.prepare_render(ui, window); let draw_data = self.context.render(); - let image = graph.bind_resource({ - let mut image = pool - .resource(ImageInfo::image_2d( - window.inner_size().width, - window.inner_size().height, - vk::Format::R8G8B8A8_UNORM, - vk::ImageUsageFlags::COLOR_ATTACHMENT - | vk::ImageUsageFlags::SAMPLED - | vk::ImageUsageFlags::STORAGE - | vk::ImageUsageFlags::TRANSFER_DST - | vk::ImageUsageFlags::TRANSFER_SRC, /* TODO: Make TRANSFER_SRC an - * "extra flags" */ - )) - .expect("missing imgui output image"); - image.name = Some("ImGui Output".to_string()); - - image - }); + let image = graph.bind_resource( + pool.resource(ImageInfo::image_2d( + window.inner_size().width, + window.inner_size().height, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::COLOR_ATTACHMENT + | vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::STORAGE + | vk::ImageUsageFlags::TRANSFER_DST + | vk::ImageUsageFlags::TRANSFER_SRC, /* TODO: Make TRANSFER_SRC an + * "extra flags" */ + )) + .expect("missing imgui output image") + .with_debug_name("ImGui Output"), + ); let font_atlas_image = graph.bind_resource( self.font_atlas_image .as_ref() @@ -317,8 +316,8 @@ impl ImGui { } let temp_buf = graph.bind_resource(temp_buf); - let image = graph.bind_resource( - pool.resource(ImageInfo::image_2d( + let image = pool + .resource(ImageInfo::image_2d( texture.width, texture.height, vk::Format::R8G8B8A8_UNORM, @@ -327,8 +326,9 @@ impl ImGui { | vk::ImageUsageFlags::TRANSFER_DST, )) .expect("missing imgui font atlas image") - .debug_name("ImGui Font Atlas"), - ); + .with_debug_name("ImGui Font Atlas"); + + let image = graph.bind_resource(image); graph.copy_buffer_to_image(temp_buf, image); diff --git a/crates/vk-graph-window/CHANGELOG.md b/crates/vk-graph-window/CHANGELOG.md index 97d366ed..5ecea9ba 100644 --- a/crates/vk-graph-window/CHANGELOG.md +++ b/crates/vk-graph-window/CHANGELOG.md @@ -6,6 +6,10 @@ This crate is versioned independently from `vk-graph`. ## [Unreleased] +## [0.1.1] - 2026-06-18 + +- Supports `vk-graph` `0.14.2`. + ## [0.1.0] - 2026-05-30 - Initial release. diff --git a/crates/vk-graph-window/Cargo.toml b/crates/vk-graph-window/Cargo.toml index 7a120ebc..d4cc8f1c 100644 --- a/crates/vk-graph-window/Cargo.toml +++ b/crates/vk-graph-window/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vk-graph-window" -version = "0.1.0" +version = "0.1.1" authors = ["John Wells "] description = "winit window and swapchain integration for vk-graph" documentation = "https://docs.rs/vk-graph-window" @@ -13,6 +13,7 @@ readme = "README.md" repository.workspace = true [dependencies] +ctrlc = { version = "3.4", features = ["termination"] } derive_builder = "0.20" log.workspace = true profiling.workspace = true diff --git a/crates/vk-graph-window/README.md b/crates/vk-graph-window/README.md index 26828f88..cacda59b 100644 --- a/crates/vk-graph-window/README.md +++ b/crates/vk-graph-window/README.md @@ -11,4 +11,4 @@ cargo run --example hello_world See the workspace [guide book](https://attackgoat.github.io/vk-graph) and [examples](../../examples/README.md) for end-to-end usage. -hello_world.rs +hello_world.rs diff --git a/crates/vk-graph-window/src/graphchain.rs b/crates/vk-graph-window/src/graphchain.rs new file mode 100644 index 00000000..a69fab55 --- /dev/null +++ b/crates/vk-graph-window/src/graphchain.rs @@ -0,0 +1,1371 @@ +//! Window graphchain creation, acquisition, and presentation helpers. + +use { + derive_builder::Builder, + log::{Level, log_enabled, trace, warn}, + std::{ + error::Error, + fmt::{Debug, Formatter}, + mem::take, + ops::Deref, + slice, + thread::panicking, + }, + vk_graph::{ + Graph, + driver::{ + DriverError, + ash::vk, + cmd_buf::{CommandBuffer, CommandBufferInfo}, + device::Device, + fence::Fence, + surface::Surface, + swapchain::{ + PresentError, PresentFailure, PresentInfo, PresentResult, Swapchain, + SwapchainError, SwapchainImage, SwapchainInfo, SwapchainInfoBuilder, + }, + }, + node::SwapchainImageNode, + pool::{Pool, SubmissionPool}, + submission::{QueueSubmitInfo, RecordSelection, SemaphoreSubmitInfo}, + }, +}; + +/// A physical display interface. +#[read_only::embed] +pub struct Graphchain { + frame_idx: usize, + frames: Box<[FrameSlot]>, + image_frames: Vec, + last_present_queue_index: u32, + queue_family_index: u32, + recreate_pending: bool, + requested_info: GraphchainInfo, + strategy: PresentRetirementStrategy, + + /// Effective runtime information for the live graphchain. + /// + /// Calls to [`set_info`](Self::set_info) update the requested configuration. This field is + /// updated lazily after the next successful acquire recreates the swapchain. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub info: EffectiveGraphchainInfo, + + /// The driver swapchain backing this graphchain. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub swapchain: Swapchain, +} + +impl Deref for ReadOnlyGraphchain { + type Target = Swapchain; + + fn deref(&self) -> &Self::Target { + &self.swapchain + } +} + +impl Graphchain { + /// Constructs a new `Graphchain` object. + pub fn new(surface: Surface, info: impl Into) -> Result { + let mut info = info.into(); + + // Frame capacity must always be able to cover at least the requested minimum image count. + info.frame_capacity = info + .frame_capacity + .max(info.min_image_count as usize) + .max(1); + + let swapchain_info: SwapchainInfo = info.into(); + let swapchain = Swapchain::create(surface, swapchain_info)?; + let physical_device = &swapchain.surface.device.physical; + let queue_family_index = physical_device + .queue_families + .iter() + .enumerate() + .find_map(|(idx, _)| { + swapchain + .surface + .physical_device_support(idx as u32) + .ok() + .and_then(|supported| supported.then_some(idx as u32)) + }) + .ok_or(DriverError::Unsupported)?; + + // Use the modern strategy only when both present-id and present-wait are supported. + let strategy = if swapchain + .surface + .device + .physical + .vk_khr_present_id + .is_some() + && swapchain + .surface + .device + .physical + .vk_khr_present_wait + .is_some() + { + PresentRetirementStrategy::PresentWait(PresentWait::default()) + } else { + PresentRetirementStrategy::PerImageSemaphore(PerImageSemaphore::default()) + }; + + let mut frames = Vec::with_capacity(info.frame_capacity); + for _ in 0..info.frame_capacity { + let cmd_buf = CommandBuffer::create( + &swapchain.surface.device, + CommandBufferInfo::new(queue_family_index), + )?; + let fence = Fence::create(&swapchain.surface.device, false)?; + let swapchain_acquired = Device::create_semaphore(&swapchain.surface.device)?; + Device::try_set_debug_utils_object_name( + &swapchain.surface.device, + swapchain_acquired, + "graphchain acquired semaphore", + ); + + frames.push(FrameSlot { + cmd_buf, + fence, + swapchain_acquired, + }); + } + + let effective_info = { + let GraphchainInfo { + acquire_timeout, + clipped, + composite_alpha, + frame_capacity, + height, + min_image_count, + present_mode, + surface, + width, + } = info; + + EffectiveGraphchainInfo { + acquire_timeout, + clipped, + composite_alpha, + frame_capacity, + frame_count: frame_capacity, + height, + image_count: swapchain.info.image_count as usize, + min_image_count, + present_mode, + surface, + width, + } + }; + + Ok(Self { + frame_idx: info.frame_capacity - 1, + frames: frames.into_boxed_slice(), + image_frames: Vec::new(), + last_present_queue_index: 0, + queue_family_index, + recreate_pending: false, + requested_info: info, + strategy, + read_only: ReadOnlyGraphchain { + info: effective_info, + swapchain, + }, + }) + } + + /// Acquires the next available swapchain image for rendering. + pub fn acquire_next_image(&mut self) -> Result, GraphchainError> { + if self.recreate_pending { + for frame in &mut self.frames { + if frame.fence.is_queued() { + frame.fence.wait_signaled()?.reset()?; + } + } + + self.strategy + .retire_pending(&mut self.read_only.swapchain)?; + + if self.read_only.swapchain.recreate_pending { + self.read_only.swapchain.recreate()?; + } + + self.strategy + .reset(&self.read_only.swapchain.surface.device); + self.image_frames.clear(); + self.recreate_pending = false; + } + + self.frame_idx += 1; + self.frame_idx %= self.frames.len(); + + let frame = &mut self.frames[self.frame_idx]; + + if frame.fence.is_queued() { + frame.fence.wait_signaled()?.reset()?; + } + + self.strategy.prepare_frame( + &frame.cmd_buf.device, + &mut self.read_only.swapchain, + self.frame_idx, + )?; + + let swapchain_image = match self + .read_only + .swapchain + .acquire_next_image(frame.swapchain_acquired) + { + Ok(acquired) => acquired, + Err(SwapchainError::DeviceLost) => return Err(GraphchainError::DeviceLost), + Err(SwapchainError::Driver(err)) => return Err(GraphchainError::Driver(err)), + Err( + SwapchainError::FullScreenExclusiveModeLost + | SwapchainError::NotReady + | SwapchainError::Timeout, + ) => return Ok(None), + Err(SwapchainError::SurfaceLost) => { + return Err(GraphchainError::SurfaceLost); + } + }; + + if swapchain_image.suboptimal { + self.recreate_pending = true; + } + + let swapchain_info = self.read_only.swapchain.info; + self.read_only.info.acquire_timeout = swapchain_info.acquire_timeout; + self.read_only.info.clipped = swapchain_info.clipped; + self.read_only.info.composite_alpha = swapchain_info.composite_alpha; + self.read_only.info.frame_capacity = self.requested_info.frame_capacity; + self.read_only.info.height = swapchain_info.height; + self.read_only.info.image_count = swapchain_info.image_count as usize; + self.read_only.info.present_mode = swapchain_info.present_mode; + self.read_only.info.surface = swapchain_info.surface; + self.read_only.info.width = swapchain_info.width; + + // Live frame count cannot be smaller than the live swapchain image count. + self.read_only.info.frame_count = self + .read_only + .info + .frame_capacity + .max(self.read_only.info.image_count); + + self.ensure_frame_count()?; + + while swapchain_image.index >= self.image_frames.len() as u32 { + self.image_frames.push(0); + } + + self.image_frames[swapchain_image.index as usize] = self.frame_idx; + self.strategy.acquire_image( + &self.read_only.swapchain.surface.device, + self.frame_idx, + swapchain_image.index, + )?; + + Ok(Some(swapchain_image)) + } + + /// Displays the given swapchain image using passes specified in `graph`, if possible. + #[profiling::function] + pub fn present_image

( + &mut self, + pool: &mut P, + graph: Graph, + image_node: SwapchainImageNode, + queue_index: u32, + ) -> Result<(), GraphchainError> + where + P: Pool + SubmissionPool, + { + trace!("present_image"); + + let submission = graph.finalize(); + let (image_idx, image_sync_info) = { + let image = submission.resource(image_node); + + (image.index as usize, image.sync_info()) + }; + let frame_idx = self.image_frames[image_idx]; + let frame = &mut self.frames[frame_idx]; + let rendered = + self.strategy + .rendered_semaphore(&frame.cmd_buf.device, frame_idx, image_idx as u32)?; + + frame.cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + + let wait_signal = image_sync_info.subresources.first().map(|_| { + let wait = SemaphoreSubmitInfo { + semaphore: frame.swapchain_acquired, + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + }; + let signal = SemaphoreSubmitInfo { + semaphore: rendered, + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + }; + + (wait, signal) + }); + + let (mut recording, wait, signal, record_remaining) = + if let Some((wait, signal)) = wait_signal { + ( + submission.record(pool, &frame.cmd_buf, image_node)?, + wait, + signal, + true, + ) + } else { + warn!("uninitialized swapchain image"); + + let wait = SemaphoreSubmitInfo { + semaphore: frame.swapchain_acquired, + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + }; + let signal = SemaphoreSubmitInfo { + semaphore: rendered, + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + }; + + ( + submission.record(pool, &frame.cmd_buf, RecordSelection::All)?, + wait, + signal, + false, + ) + }; + + let supports_synchronization2 = recording.cmd_buf.device.physical.vk_khr_synchronization2; + + let image = recording.resource(image_node); + let image_sync_info = image.sync_info(); + let image_handle = image.handle; + + if log_enabled!(Level::Trace) { + for sync_info in &image_sync_info.subresources { + trace!( + "image {:?} {:?}->{:?}", + image, + sync_info.layout, + vk::ImageLayout::PRESENT_SRC_KHR, + ); + } + } + + if supports_synchronization2 { + for sync_info in &image_sync_info.subresources { + Device::cmd_pipeline_barrier2( + &recording.cmd_buf.device, + recording.cmd_buf.handle, + &vk::DependencyInfo::default().image_memory_barriers(slice::from_ref( + &vk::ImageMemoryBarrier2::default() + .src_stage_mask(vk::PipelineStageFlags2::from_raw( + sync_info.stage_mask.as_raw() as u64, + )) + .src_access_mask(vk::AccessFlags2::from_raw( + sync_info.access_mask.as_raw() as u64, + )) + .dst_stage_mask(vk::PipelineStageFlags2::NONE) + .dst_access_mask(vk::AccessFlags2::empty()) + .old_layout(sync_info.layout.unwrap_or(vk::ImageLayout::UNDEFINED)) + .new_layout(vk::ImageLayout::PRESENT_SRC_KHR) + .src_queue_family_index( + sync_info + .queue_family_index + .unwrap_or(vk::QUEUE_FAMILY_IGNORED), + ) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(image_handle) + .subresource_range(sync_info.range), + )), + ); + } + } else { + for sync_info in &image_sync_info.subresources { + unsafe { + recording.cmd_buf.device.cmd_pipeline_barrier( + recording.cmd_buf.handle, + sync_info.stage_mask, + vk::PipelineStageFlags::BOTTOM_OF_PIPE, + vk::DependencyFlags::empty(), + &[], + &[], + slice::from_ref( + &vk::ImageMemoryBarrier::default() + .src_access_mask(sync_info.access_mask) + .dst_access_mask(vk::AccessFlags::empty()) + .old_layout(sync_info.layout.unwrap_or(vk::ImageLayout::UNDEFINED)) + .new_layout(vk::ImageLayout::PRESENT_SRC_KHR) + .src_queue_family_index( + sync_info + .queue_family_index + .unwrap_or(vk::QUEUE_FAMILY_IGNORED), + ) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(image_handle) + .subresource_range(sync_info.range), + ), + ) + }; + } + } + + if record_remaining { + recording.record(RecordSelection::All)?; + } + + recording.cmd_buf.end()?; + + let image = unsafe { recording.resource(image_node).to_detached() }; + + let mut recorded = recording.finish()?; + let waits = slice::from_ref(&wait); + let signals = slice::from_ref(&signal); + let submit_info = QueueSubmitInfo::queue_submit(waits, signals); + recorded.queue_submit(&mut frame.fence, queue_index, submit_info)?; + + // Only mark presentation as pending when the driver actually queued the present. + let present_info = PresentInfo { + image, + swapchain: &mut self.read_only.swapchain, + wait_semaphores: slice::from_ref(&rendered), + }; + + let present = + Swapchain::queue_present(self.queue_family_index, queue_index, [present_info]); + let present_result = match &present { + Ok(results) => Some(results.first().copied().expect("missing present result")), + Err(err) => err.results.first().copied().flatten(), + }; + + if present_result.is_some_and(PresentResult::suboptimal) { + self.recreate_pending = true; + } + + if let Some(PresentResult::NotQueued(err)) = present_result { + match err { + PresentError::DeviceLost | PresentError::SurfaceLost => {} + PresentError::FullScreenExclusiveModeLost | PresentError::OutOfDate => { + self.recreate_pending = true; + } + } + } + + if present_result.is_some_and(PresentResult::queued) { + self.last_present_queue_index = queue_index; + self.strategy.present_image(frame_idx, image_idx as u32); + } + + if let Err(err) = present { + match err.source { + PresentFailure::DeviceLost => return Err(GraphchainError::DeviceLost), + PresentFailure::SurfaceLost => return Err(GraphchainError::SurfaceLost), + PresentFailure::Driver(err) => return Err(GraphchainError::Driver(err)), + PresentFailure::FullScreenExclusiveModeLost | PresentFailure::OutOfDate => { + self.recreate_pending = true; + } + } + } + + Ok(()) + } + + /// Updates the requested information which controls the graphchain. + /// + /// Previously acquired swapchain images should be discarded after calling this function. + /// The live effective [`info`](Self::info) field updates [`acquire_timeout`](GraphchainInfo::acquire_timeout) + /// immediately. Other fields are updated lazily after the next successful acquire recreates the + /// swapchain and reflects the runtime values that were actually selected. + pub fn set_info(&mut self, info: impl Into) { + let info = info.into(); + let swapchain_info = info.into_swapchain_info(); + let recreate = self.requested_info.height != info.height + || self.requested_info.clipped != info.clipped + || self.requested_info.min_image_count != info.min_image_count + || self.requested_info.present_mode != info.present_mode + || self.requested_info.composite_alpha != info.composite_alpha + || self.requested_info.surface != info.surface + || self.requested_info.width != info.width; + + self.read_only.swapchain.set_info(swapchain_info); + self.requested_info = info; + self.read_only.info.acquire_timeout = info.acquire_timeout; + self.read_only.info.clipped = info.clipped; + self.read_only.info.min_image_count = info.min_image_count; + + if recreate { + self.recreate_pending = true; + } + } + + fn ensure_frame_count(&mut self) -> Result<(), GraphchainError> { + if self.frames.len() >= self.read_only.info.frame_count { + return Ok(()); + } + + let device = self.read_only.swapchain.surface.device.clone(); + let mut frames = take(&mut self.frames).into_vec(); + + for _ in frames.len()..self.read_only.info.frame_count { + let cmd_buf = + CommandBuffer::create(&device, CommandBufferInfo::new(self.queue_family_index))?; + let fence = Fence::create(&device, false)?; + let swapchain_acquired = Device::create_semaphore(&device)?; + Device::try_set_debug_utils_object_name( + &device, + swapchain_acquired, + "graphchain acquired semaphore", + ); + + frames.push(FrameSlot { + cmd_buf, + fence, + swapchain_acquired, + }); + } + + self.frames = frames.into_boxed_slice(); + + Ok(()) + } +} + +impl Debug for Graphchain { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("Graphchain") + } +} + +impl Drop for Graphchain { + fn drop(&mut self) { + if panicking() { + return; + } + + let Some(frame) = self.frames.first() else { + return; + }; + let device = frame.cmd_buf.device.clone(); + + for frame in &mut self.frames { + if frame.fence.is_queued() && frame.fence.wait_signaled().is_err() { + return; + } + } + + if self + .strategy + .retire_pending(&mut self.read_only.swapchain) + .is_err() + { + return; + } + + if Device::with_queue( + &device, + self.queue_family_index, + self.last_present_queue_index, + |queue| Device::queue_wait_idle(&device, queue), + ) + .is_err() + { + return; + } + + for frame in &mut self.frames { + unsafe { + frame + .cmd_buf + .device + .destroy_semaphore(frame.swapchain_acquired, None); + } + } + + self.strategy.destroy(&device); + } +} + +/// Describes error conditions relating to physical displays. +#[derive(Clone, Copy, Debug)] +pub enum GraphchainError { + /// Unrecoverable device error; must destroy this device and display and start a new one. + DeviceLost, + + /// The surface was lost and must be recreated. + SurfaceLost, + + /// Recoverable driver error. + Driver(DriverError), +} + +impl Error for GraphchainError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Driver(err) => Some(err), + Self::DeviceLost | Self::SurfaceLost => None, + } + } +} + +impl From<()> for GraphchainError { + fn from(_: ()) -> Self { + Self::DeviceLost + } +} + +impl From for GraphchainError { + fn from(err: DriverError) -> Self { + Self::Driver(err) + } +} + +impl From for GraphchainError { + fn from(err: SwapchainError) -> Self { + match err { + SwapchainError::DeviceLost => Self::DeviceLost, + SwapchainError::Driver(err) => Self::Driver(err), + SwapchainError::FullScreenExclusiveModeLost + | SwapchainError::NotReady + | SwapchainError::Timeout => Self::Driver(DriverError::Unsupported), + SwapchainError::SurfaceLost => Self::SurfaceLost, + } + } +} + +impl std::fmt::Display for GraphchainError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::DeviceLost => f.write_str("device lost"), + Self::SurfaceLost => f.write_str("surface lost"), + Self::Driver(err) => std::fmt::Display::fmt(err, f), + } + } +} + +/// Information used to create a [`Graphchain`] instance. +#[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[builder( + build_fn(private, name = "fallible_build"), + derive(Clone, Copy, Debug), + pattern = "owned" +)] +pub struct GraphchainInfo { + /// Timeout in nanoseconds used when acquiring the next image. + /// + /// Do not use `u64::MAX` on surfaces where acquire forward progress cannot be guaranteed. + #[builder(default = "u64::MAX")] + pub acquire_timeout: u64, + + /// Alpha compositing mode used by the presentation engine. + #[builder(default = vk::CompositeAlphaFlagsKHR::OPAQUE)] + pub composite_alpha: vk::CompositeAlphaFlagsKHR, + + /// Whether to clip pixels obscured by other windows on the native surface. + #[builder(default = "true")] + pub clipped: bool, + + /// The requested frame capacity used for in-flight graphchain work. + /// + /// The effective live capacity may be raised at runtime to match the swapchain image count. + /// That resolved value is reflected through [`EffectiveGraphchainInfo`]. + #[builder(default = "4")] + pub frame_capacity: usize, + + /// The initial height of the surface. + #[builder(default)] + pub height: u32, + + /// The minimum number of presentable images that the application needs. + #[builder(default = "2")] + pub min_image_count: u32, + + /// `vk::PresentModeKHR` determines timing and queueing with which frames are displayed. + #[builder(default = vk::PresentModeKHR::MAILBOX)] + pub present_mode: vk::PresentModeKHR, + + /// The format and color space of the surface. + #[builder(default)] + pub surface: vk::SurfaceFormatKHR, + + /// The initial width of the surface. + #[builder(default)] + pub width: u32, +} + +/// Effective runtime information for a live [`Graphchain`]. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct EffectiveGraphchainInfo { + /// Timeout in nanoseconds used when acquiring the next image. + pub acquire_timeout: u64, + + /// Active alpha compositing mode used by the presentation engine. + pub composite_alpha: vk::CompositeAlphaFlagsKHR, + + /// Whether pixels obscured by other windows on the native surface are clipped. + pub clipped: bool, + + frame_capacity: usize, + + /// The effective number of tracked frame slots. + pub frame_count: usize, + + /// The live height of the surface. + pub height: u32, + + /// The live swapchain image count. + pub image_count: usize, + + min_image_count: u32, + + /// `vk::PresentModeKHR` determines timing and queueing with which frames are displayed. + pub present_mode: vk::PresentModeKHR, + + /// The format and color space of the surface. + pub surface: vk::SurfaceFormatKHR, + + /// The live width of the surface. + pub width: u32, +} + +impl GraphchainInfo { + /// Specifies a default graphchain with the given `width`, `height` and `format` values. + #[inline(always)] + pub fn new(width: u32, height: u32, surface: vk::SurfaceFormatKHR) -> GraphchainInfo { + Self { + acquire_timeout: u64::MAX, + clipped: true, + composite_alpha: vk::CompositeAlphaFlagsKHR::OPAQUE, + frame_capacity: 4, + height, + min_image_count: 2, + present_mode: vk::PresentModeKHR::MAILBOX, + surface, + width, + } + } + + /// Creates a default `GraphchainInfoBuilder`. + pub fn builder() -> GraphchainInfoBuilder { + Default::default() + } + + /// Converts a `GraphchainInfo` into a `GraphchainInfoBuilder`. + pub fn into_builder(self) -> GraphchainInfoBuilder { + GraphchainInfoBuilder { + acquire_timeout: Some(self.acquire_timeout), + clipped: Some(self.clipped), + composite_alpha: Some(self.composite_alpha), + frame_capacity: Some(self.frame_capacity), + height: Some(self.height), + min_image_count: Some(self.min_image_count), + present_mode: Some(self.present_mode), + surface: Some(self.surface), + width: Some(self.width), + } + } + + /// Converts this graphchain info into low-level swapchain info. + pub fn into_swapchain_info(self) -> SwapchainInfo { + self.into() + } +} + +impl EffectiveGraphchainInfo { + /// Creates a default `GraphchainInfoBuilder`. + pub fn builder() -> GraphchainInfoBuilder { + Default::default() + } + + /// Converts an `EffectiveGraphchainInfo` into a `GraphchainInfoBuilder`. + pub fn into_builder(self) -> GraphchainInfoBuilder { + self.into_requested_info().into_builder() + } + + /// Converts this effective runtime info back into requested graphchain info. + pub fn into_requested_info(self) -> GraphchainInfo { + let Self { + acquire_timeout, + clipped, + composite_alpha, + frame_capacity, + frame_count: _, + height, + image_count: _, + min_image_count, + present_mode, + surface, + width, + } = self; + + GraphchainInfo { + acquire_timeout, + clipped, + composite_alpha, + frame_capacity, + height, + min_image_count, + present_mode, + surface, + width, + } + } +} + +impl From for GraphchainInfo { + fn from(info: EffectiveGraphchainInfo) -> Self { + info.into_requested_info() + } +} + +impl From for GraphchainInfo { + fn from(info: GraphchainInfoBuilder) -> Self { + info.build() + } +} + +impl From for SwapchainInfo { + fn from(val: GraphchainInfo) -> Self { + SwapchainInfoBuilder::default() + .acquire_timeout(val.acquire_timeout) + .clipped(val.clipped) + .composite_alpha(val.composite_alpha) + .height(val.height) + .min_image_count(val.min_image_count) + .present_mode(val.present_mode) + .surface(val.surface) + .width(val.width) + .build() + } +} + +impl From for SwapchainInfo { + fn from(info: GraphchainInfoBuilder) -> Self { + info.build().into_swapchain_info() + } +} + +impl GraphchainInfoBuilder { + /// Builds a new `GraphchainInfo`. + #[inline(always)] + pub fn build(self) -> GraphchainInfo { + self.fallible_build().expect("all fields have defaults") + } +} + +enum PresentRetirementStrategy { + PresentWait(PresentWait), + PerImageSemaphore(PerImageSemaphore), +} + +impl PresentRetirementStrategy { + fn prepare_frame( + &mut self, + device: &Device, + swapchain: &mut Swapchain, + frame_idx: usize, + ) -> Result<(), GraphchainError> { + match self { + Self::PresentWait(strategy) => strategy.prepare_frame(device, swapchain, frame_idx), + Self::PerImageSemaphore(strategy) => { + strategy.prepare_frame(device, swapchain, frame_idx) + } + } + } + + fn acquire_image( + &mut self, + device: &Device, + frame_idx: usize, + image_idx: u32, + ) -> Result<(), DriverError> { + match self { + Self::PresentWait(strategy) => strategy.acquire_image(device, frame_idx, image_idx), + Self::PerImageSemaphore(strategy) => { + strategy.acquire_image(device, frame_idx, image_idx) + } + } + } + + fn rendered_semaphore( + &mut self, + device: &Device, + frame_idx: usize, + image_idx: u32, + ) -> Result { + match self { + Self::PresentWait(strategy) => { + strategy.rendered_semaphore(device, frame_idx, image_idx) + } + Self::PerImageSemaphore(strategy) => { + strategy.rendered_semaphore(device, frame_idx, image_idx) + } + } + } + + fn present_image(&mut self, frame_idx: usize, image_idx: u32) { + match self { + Self::PresentWait(strategy) => strategy.present_image(frame_idx, image_idx), + Self::PerImageSemaphore(strategy) => strategy.present_image(frame_idx, image_idx), + } + } + + fn retire_pending(&mut self, swapchain: &mut Swapchain) -> Result<(), GraphchainError> { + match self { + Self::PresentWait(strategy) => strategy.retire_pending(swapchain), + Self::PerImageSemaphore(strategy) => strategy.retire_pending(swapchain), + } + } + + fn reset(&mut self, device: &Device) { + match self { + Self::PresentWait(strategy) => strategy.reset(device), + Self::PerImageSemaphore(strategy) => strategy.reset(device), + } + } + + fn destroy(&mut self, device: &Device) { + match self { + Self::PresentWait(strategy) => strategy.destroy(device), + Self::PerImageSemaphore(strategy) => strategy.destroy(device), + } + } +} + +#[derive(Default)] +struct PresentWait { + retired: Vec, + rendered: Vec>, + pending_images: Vec>, +} + +impl PresentWait { + fn prepare_frame( + &mut self, + _device: &Device, + swapchain: &mut Swapchain, + frame_idx: usize, + ) -> Result<(), GraphchainError> { + while self.pending_images.len() <= frame_idx { + self.pending_images.push(None); + } + + if let Some(image_idx) = self.pending_images[frame_idx].take() { + unsafe { + match swapchain.wait_for_present_image(image_idx, u64::MAX) { + Ok(()) => {} + Err(SwapchainError::DeviceLost) => { + return Err(GraphchainError::DeviceLost); + } + Err(SwapchainError::Driver(err)) => { + return Err(GraphchainError::Driver(err)); + } + Err(SwapchainError::FullScreenExclusiveModeLost) => {} + Err(SwapchainError::NotReady | SwapchainError::Timeout) => { + return Err(GraphchainError::Driver(DriverError::Unsupported)); + } + Err(SwapchainError::SurfaceLost) => { + return Err(GraphchainError::SurfaceLost); + } + } + } + } + + Ok(()) + } + + fn acquire_image( + &mut self, + device: &Device, + _frame_idx: usize, + image_idx: u32, + ) -> Result<(), DriverError> { + self.ensure_rendered(device, image_idx).map(|_| ()) + } + + fn rendered_semaphore( + &mut self, + device: &Device, + _frame_idx: usize, + image_idx: u32, + ) -> Result { + self.ensure_rendered(device, image_idx) + } + + fn ensure_rendered( + &mut self, + device: &Device, + image_idx: u32, + ) -> Result { + while self.rendered.len() <= image_idx as usize { + self.rendered.push(None); + } + + if self.rendered[image_idx as usize].is_none() { + let semaphore = Device::create_semaphore(device)?; + Device::try_set_debug_utils_object_name( + device, + semaphore, + "graphchain present-wait per-image rendered semaphore", + ); + + self.rendered[image_idx as usize] = Some(semaphore); + } + + Ok(self.rendered[image_idx as usize].expect("missing rendered semaphore")) + } + + fn present_image(&mut self, frame_idx: usize, image_idx: u32) { + while self.pending_images.len() <= frame_idx { + self.pending_images.push(None); + } + + self.pending_images[frame_idx] = Some(image_idx); + } + + fn retire_pending(&mut self, swapchain: &mut Swapchain) -> Result<(), GraphchainError> { + for image_idx in self.pending_images.drain(..).flatten() { + unsafe { + match swapchain.wait_for_present_image(image_idx, u64::MAX) { + Ok(()) | Err(SwapchainError::FullScreenExclusiveModeLost) => {} + Err(SwapchainError::DeviceLost) => return Err(GraphchainError::DeviceLost), + Err(SwapchainError::Driver(err)) => return Err(GraphchainError::Driver(err)), + Err(SwapchainError::NotReady | SwapchainError::Timeout) => { + return Err(GraphchainError::Driver(DriverError::Unsupported)); + } + Err(SwapchainError::SurfaceLost) => return Err(GraphchainError::SurfaceLost), + } + } + } + + Ok(()) + } + + fn reset(&mut self, device: &Device) { + self.destroy_retired(device); + self.pending_images.clear(); + self.retired.extend(self.rendered.drain(..).flatten()); + } + + fn destroy(&mut self, device: &Device) { + self.destroy_retired(device); + + for semaphore in self + .rendered + .drain(..) + .flatten() + .chain(self.retired.drain(..)) + { + unsafe { + device.destroy_semaphore(semaphore, None); + } + } + } + + fn destroy_retired(&mut self, device: &Device) { + for semaphore in self.retired.drain(..) { + unsafe { + device.destroy_semaphore(semaphore, None); + } + } + } +} + +#[derive(Default)] +struct PerImageSemaphore { + retired: Vec, + rendered: Vec>, +} + +impl PerImageSemaphore { + fn prepare_frame( + &mut self, + _device: &Device, + _swapchain: &mut Swapchain, + _frame_idx: usize, + ) -> Result<(), GraphchainError> { + Ok(()) + } + + fn acquire_image( + &mut self, + device: &Device, + _frame_idx: usize, + image_idx: u32, + ) -> Result<(), DriverError> { + self.ensure_rendered(device, image_idx).map(|_| ()) + } + + fn rendered_semaphore( + &mut self, + device: &Device, + _frame_idx: usize, + image_idx: u32, + ) -> Result { + self.ensure_rendered(device, image_idx) + } + + fn ensure_rendered( + &mut self, + device: &Device, + image_idx: u32, + ) -> Result { + while self.rendered.len() <= image_idx as usize { + self.rendered.push(None); + } + + if self.rendered[image_idx as usize].is_none() { + let semaphore = Device::create_semaphore(device)?; + Device::try_set_debug_utils_object_name( + device, + semaphore, + "graphchain per-image rendered semaphore", + ); + + self.rendered[image_idx as usize] = Some(semaphore); + } + + Ok(self.rendered[image_idx as usize].expect("missing rendered semaphore")) + } + + fn present_image(&mut self, _frame_idx: usize, _image_idx: u32) {} + + fn retire_pending(&mut self, _swapchain: &mut Swapchain) -> Result<(), GraphchainError> { + Ok(()) + } + + fn reset(&mut self, device: &Device) { + self.destroy_retired(device); + self.retired.extend(self.rendered.drain(..).flatten()); + } + + fn destroy(&mut self, device: &Device) { + self.destroy_retired(device); + + for semaphore in self + .rendered + .drain(..) + .flatten() + .chain(self.retired.drain(..)) + { + unsafe { + device.destroy_semaphore(semaphore, None); + } + } + } + + fn destroy_retired(&mut self, device: &Device) { + for semaphore in self.retired.drain(..) { + unsafe { + device.destroy_semaphore(semaphore, None); + } + } + } +} + +struct FrameSlot { + cmd_buf: CommandBuffer, + fence: Fence, + swapchain_acquired: vk::Semaphore, +} + +#[cfg(test)] +mod test { + use super::*; + + type Info = GraphchainInfo; + type Builder = GraphchainInfoBuilder; + type Effective = EffectiveGraphchainInfo; + + #[test] + fn graphchain_info_round_trips_through_builder() { + let info = Info { + acquire_timeout: 42, + clipped: false, + composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + frame_capacity: 6, + height: 123, + min_image_count: 3, + present_mode: vk::PresentModeKHR::FIFO, + surface: vk::SurfaceFormatKHR::default() + .format(vk::Format::B8G8R8A8_UNORM) + .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + width: 456, + }; + + assert_eq!(info, info.into_builder().build()); + } + + #[test] + fn graphchain_info_builder_defaults() { + assert_eq!( + Builder::default().build(), + Info { + acquire_timeout: u64::MAX, + clipped: true, + composite_alpha: vk::CompositeAlphaFlagsKHR::OPAQUE, + frame_capacity: 4, + height: 0, + min_image_count: 2, + present_mode: vk::PresentModeKHR::MAILBOX, + surface: vk::SurfaceFormatKHR::default(), + width: 0, + } + ); + } + + #[test] + fn effective_graphchain_info_into_requested_info() { + let info = Effective { + acquire_timeout: 42, + clipped: false, + composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + frame_capacity: 6, + frame_count: 8, + height: 123, + image_count: 7, + min_image_count: 3, + present_mode: vk::PresentModeKHR::FIFO, + surface: vk::SurfaceFormatKHR::default() + .format(vk::Format::B8G8R8A8_UNORM) + .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + width: 456, + }; + + assert_eq!( + info.into_requested_info(), + Info { + acquire_timeout: 42, + clipped: false, + composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + frame_capacity: 6, + height: 123, + min_image_count: 3, + present_mode: vk::PresentModeKHR::FIFO, + surface: vk::SurfaceFormatKHR::default() + .format(vk::Format::B8G8R8A8_UNORM) + .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + width: 456, + } + ); + } + + // #[test] + // fn effective_graphchain_info_from_requested_info() { + // let info = Info { + // acquire_timeout: 42, + // composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + // frame_capacity: 6, + // height: 123, + // min_image_count: 3, + // present_mode: vk::PresentModeKHR::FIFO, + // surface: vk::SurfaceFormatKHR::default() + // .format(vk::Format::B8G8R8A8_UNORM) + // .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + // width: 456, + // }; + + // assert_eq!( + // Effective::from(info), + // Effective { + // acquire_timeout: 42, + // composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + // frame_capacity: 6, + // frame_count: 6, + // height: 123, + // image_count: 0, + // present_mode: vk::PresentModeKHR::FIFO, + // surface: vk::SurfaceFormatKHR::default() + // .format(vk::Format::B8G8R8A8_UNORM) + // .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + // width: 456, + // } + // ); + // } + + // #[test] + // fn effective_graphchain_info_has_only_runtime_fields() { + // let info = Effective { + // acquire_timeout: 42, + // composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + // frame_capacity: 6, + // frame_count: 8, + // height: 123, + // image_count: 7, + // present_mode: vk::PresentModeKHR::FIFO, + // surface: vk::SurfaceFormatKHR::default() + // .format(vk::Format::B8G8R8A8_UNORM) + // .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + // width: 456, + // }; + + // assert_eq!(info.frame_count, 8); + // assert_eq!(info.image_count, 7); + // } + + // #[test] + // fn effective_graphchain_info_round_trips_through_builder() { + // let info = Effective { + // acquire_timeout: 42, + // composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + // frame_capacity: 6, + // frame_count: 8, + // height: 123, + // image_count: 7, + // present_mode: vk::PresentModeKHR::FIFO, + // surface: vk::SurfaceFormatKHR::default() + // .format(vk::Format::B8G8R8A8_UNORM) + // .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + // width: 456, + // }; + + // assert_eq!(info, info.into_builder().build()); + // } + + // #[test] + // fn effective_graphchain_info_builder_defaults() { + // assert_eq!( + // EffectiveBuilder::default().build(), + // Effective { + // acquire_timeout: u64::MAX, + // composite_alpha: vk::CompositeAlphaFlagsKHR::OPAQUE, + // frame_capacity: 4, + // frame_count: 4, + // height: 0, + // image_count: 0, + // present_mode: vk::PresentModeKHR::MAILBOX, + // surface: vk::SurfaceFormatKHR::default(), + // width: 0, + // } + // ); + // } + + // #[test] + // fn effective_graphchain_info_is_not_requested_info() { + // let info = Info { + // acquire_timeout: 42, + // composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + // frame_capacity: 6, + // height: 123, + // min_image_count: 3, + // present_mode: vk::PresentModeKHR::FIFO, + // surface: vk::SurfaceFormatKHR::default() + // .format(vk::Format::B8G8R8A8_UNORM) + // .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + // width: 456, + // }; + + // assert_eq!(Effective::from(info).image_count, 0); + // } +} diff --git a/crates/vk-graph-window/src/lib.rs b/crates/vk-graph-window/src/lib.rs index b8099460..1b939f94 100644 --- a/crates/vk-graph-window/src/lib.rs +++ b/crates/vk-graph-window/src/lib.rs @@ -4,12 +4,12 @@ #![deny(rustdoc::broken_intra_doc_links)] mod frame; -pub mod swapchain; +pub mod graphchain; pub use {self::frame::FrameContext, winit}; use { - self::swapchain::{Swapchain, SwapchainError, SwapchainInfo}, + self::graphchain::{Graphchain, GraphchainError, GraphchainInfo}, log::{error, info, trace, warn}, std::{error, fmt, ops::Deref}, vk_graph::{ @@ -36,6 +36,63 @@ use { /// A closure type for picking surface formats. pub type SurfaceFormatFn = dyn Fn(&[vk::SurfaceFormatKHR]) -> vk::SurfaceFormatKHR; +fn create_graphchain( + device: &Device, + data: &WindowData, + window: &winit::window::Window, +) -> Result { + let surface = Surface::create(device, window, window)?; + let surface_formats = Surface::formats(&surface)?; + let surface_format = data + .surface_format_fn + .as_ref() + .map(|f| f(&surface_formats)) + .unwrap_or_else(|| Surface::linear_or_default(&surface_formats)); + let window_size = window.inner_size(); + + let mut graphchain_info = + GraphchainInfo::new(window_size.width, window_size.height, surface_format) + .into_builder() + .frame_capacity(data.cmd_buf_count); + + if let Some(min_image_count) = data.min_image_count { + graphchain_info = graphchain_info.min_image_count(min_image_count); + } + + let v_sync = data.v_sync.unwrap_or_default(); + let present_modes = Surface::present_modes(&surface)?; + if !present_modes.is_empty() { + let best_modes = if v_sync { + [vk::PresentModeKHR::FIFO_RELAXED, vk::PresentModeKHR::FIFO].as_slice() + } else { + [vk::PresentModeKHR::MAILBOX, vk::PresentModeKHR::IMMEDIATE].as_slice() + }; + + graphchain_info = graphchain_info.present_mode( + best_modes + .iter() + .copied() + .find(|best| present_modes.contains(best)) + .or_else(|| { + warn!("requested present modes unsupported: {best_modes:?}"); + + present_modes.first().copied() + }) + .ok_or_else(|| { + error!("display does not support presentation"); + + DriverError::Unsupported + })?, + ); + } + + let graphchain = Graphchain::new(surface, graphchain_info)?; + + trace!("created graphchain"); + + Ok(graphchain) +} + /// Describes a screen mode for display. #[derive(Clone, Copy, Debug)] pub enum FullscreenMode { @@ -95,61 +152,11 @@ impl Window { } impl Application { - fn create_swapchain( + fn create_graphchain( &mut self, window: &winit::window::Window, - ) -> Result { - let surface = Surface::create(&self.device, window, window)?; - let surface_formats = Surface::formats(&surface)?; - let surface_format = self - .data - .surface_format_fn - .as_ref() - .map(|f| f(&surface_formats)) - .unwrap_or_else(|| Surface::linear_or_default(&surface_formats)); - let window_size = window.inner_size(); - - let mut swapchain_info = - SwapchainInfo::new(window_size.width, window_size.height, surface_format) - .into_builder() - .command_buffer_count(self.data.cmd_buf_count); - - if let Some(min_image_count) = self.data.min_image_count { - swapchain_info = swapchain_info.min_image_count(min_image_count); - } - - let v_sync = self.data.v_sync.unwrap_or_default(); - let present_modes = Surface::present_modes(&surface)?; - if !present_modes.is_empty() { - let best_modes = if v_sync { - [vk::PresentModeKHR::FIFO_RELAXED, vk::PresentModeKHR::FIFO].as_slice() - } else { - [vk::PresentModeKHR::MAILBOX, vk::PresentModeKHR::IMMEDIATE].as_slice() - }; - - swapchain_info = swapchain_info.present_mode( - best_modes - .iter() - .copied() - .find(|best| present_modes.contains(best)) - .or_else(|| { - warn!("requested present modes unsupported: {best_modes:?}"); - - present_modes.first().copied() - }) - .ok_or_else(|| { - error!("display does not support presentation"); - - DriverError::Unsupported - })?, - ); - } - - let swapchain = Swapchain::new(surface, swapchain_info)?; - - trace!("created swapchain"); - - Ok(swapchain) + ) -> Result { + create_graphchain(&self.device, &self.data, window) } fn window_mode_attributes( @@ -178,9 +185,10 @@ impl Window { monitor.video_modes().find(|mode| { let mode_size = mode.size(); - // Don't pick a mode with greater resolution - // than the monitor. - // It can panic on x11 in winit. + /* + Don't pick a mode with greater resolution than the + monitor; it can panic on X11 in winit. + */ mode_size.height <= monitor_size.height && mode_size.width <= monitor_size.width }) @@ -265,9 +273,9 @@ impl Window { } Ok(res) => res, }; - let swapchain = match self.create_swapchain(&window) { + let graphchain = match self.create_graphchain(&window) { Err(err) => { - warn!("unable to create swapchain: {err}"); + warn!("unable to create graphchain: {err}"); self.error = Some(err.into()); event_loop.exit(); @@ -279,7 +287,7 @@ impl Window { let swapchain_pool = HashPool::new(&self.device); self.active_window = Some(ActiveWindow { - swapchain, + graphchain: Some(graphchain), swapchain_pool, swapchain_resize: None, events: vec![], @@ -287,7 +295,11 @@ impl Window { }); } - fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: ()) { + fn user_event(&mut self, event_loop: &ActiveEventLoop, event: ()) { + info!("signal received, exiting event loop"); + + event_loop.exit(); + if let Some(ActiveWindow { events, .. }) = self.active_window.as_mut() { events.push(Event::UserEvent(event)); } @@ -299,50 +311,72 @@ impl Window { window_id: WindowId, event: WindowEvent, ) { - if let Some(active_window) = self.active_window.as_mut() { + if let Some(mut active_window) = self.active_window.take() { match &event { WindowEvent::CloseRequested => { if event_loop.exiting() { + self.active_window = Some(active_window); + return; } info!("close requested"); event_loop.exit(); + self.active_window = Some(active_window); } WindowEvent::RedrawRequested => { if event_loop.exiting() { + self.active_window = Some(active_window); + return; } - match active_window.draw(&self.device, &mut self.draw_fn) { - Ok(true) => {} - res => { - if let Err(err) = res { - self.error = Some(WindowError::Swapchain(err)); - } + // Surface loss is recoverable here; other graphchain errors are fatal. + match active_window.draw(&self.device, &self.data, &mut self.draw_fn) { + Err(GraphchainError::SurfaceLost) => { + warn!("surface lost; abandoning current frame"); + + let _ = active_window.graphchain.take(); + + active_window.window.request_redraw(); + self.active_window = Some(active_window); + profiling::finish_frame!(); + + return; + } + Err(err) => { + self.error = Some(WindowError::Graphchain(err)); event_loop.exit(); } + Ok(false) => { + event_loop.exit(); + } + Ok(true) => {} } profiling::finish_frame!(); + self.active_window = Some(active_window); } - WindowEvent::Resized(size) => { + WindowEvent::Resized(size) if size.width * size.height > 0 => { active_window.swapchain_resize = Some((size.width, size.height)); + self.active_window = Some(active_window); } - _ => (), + _ => self.active_window = Some(active_window), } - active_window - .events - .push(Event::WindowEvent { window_id, event }); + if let Some(active_window) = self.active_window.as_mut() { + active_window + .events + .push(Event::WindowEvent { window_id, event }); + } } } } struct ActiveWindow { - swapchain: Swapchain, + graphchain: Option, swapchain_pool: HashPool, swapchain_resize: Option<(u32, u32)>, events: Vec>, @@ -353,19 +387,33 @@ impl Window { fn draw( &mut self, device: &Device, + data: &WindowData, mut f: impl FnMut(FrameContext), - ) -> Result { + ) -> Result { + if self.graphchain.is_none() { + self.graphchain = Some(create_graphchain(device, data, &self.window)?); + } + + let graphchain = self.graphchain.as_mut().expect("missing graphchain"); + if let Some((width, height)) = self.swapchain_resize.take() { - let mut swapchain_info = self.swapchain.info; - swapchain_info.width = width; - swapchain_info.height = height; - self.swapchain.set_info(swapchain_info); + if width == 0 || height == 0 { + self.swapchain_resize = Some((width, height)); + self.events.clear(); + + return Ok(true); + } + + let mut graphchain_info = graphchain.info; + graphchain_info.width = width; + graphchain_info.height = height; + graphchain.set_info(graphchain_info); } - if let Some(swapchain_image) = self.swapchain.acquire_next_image()? { + if let Some(swapchain_image) = graphchain.acquire_next_image()? { let mut graph = Graph::default(); let swapchain_image = graph.bind_resource(swapchain_image); - let swapchain_info = self.swapchain.info; + let graphchain_info = graphchain.info; let mut will_exit = false; @@ -374,10 +422,10 @@ impl Window { f(FrameContext { device, events: &self.events, - height: swapchain_info.height, + height: graphchain_info.height, graph: &mut graph, swapchain_image, - width: swapchain_info.width, + width: graphchain_info.width, will_exit: &mut will_exit, window: &self.window, }); @@ -391,13 +439,13 @@ impl Window { } self.window.pre_present_notify(); - self.swapchain + graphchain .present_image(&mut self.swapchain_pool, graph, swapchain_image, 0) .inspect_err(|err| { - warn!("unable to present swapchain image: {err}"); + warn!("unable to present graphchain image: {err}"); })?; } else { - warn!("unable to acquire swapchain image"); + warn!("unable to acquire graphchain image"); } self.window.request_redraw(); @@ -415,13 +463,22 @@ impl Window { primary_monitor: None, }; + let proxy = self.read_only.event_loop.create_proxy(); + if let Err(e) = ctrlc::set_handler(move || { + trace!("received SIGINT/SIGTERM"); + + let _ = proxy.send_event(()); + }) { + warn!("failed to set Ctrl-C handler: {e}"); + } + self.read_only.event_loop.run_app(&mut app)?; if let Some(ActiveWindow { - swapchain, window, .. + graphchain, window, .. }) = app.active_window.take() { - drop(swapchain); + drop(graphchain); drop(window); } @@ -486,9 +543,8 @@ impl WindowBuilder { /// Enables Vulkan graphics debugging layers. /// - /// _NOTE:_ Any validation warnings or errors will cause the current thread to park itself after - /// describing the error using the `log` crate. This makes it easy to attach a debugger and see - /// what is causing the issue directly. + /// _NOTE:_ Validation errors will only park the current thread for debugger attach when the + /// process is attached to an interactive terminal. Otherwise they continue after logging. /// /// ## Platform-specific /// @@ -531,12 +587,11 @@ impl WindowBuilder { self } - /// When `true` specifies that the presentation engine does not wait for a vertical blanking - /// period to update the current image, meaning this mode may result in visible tearing. + /// When `true`, requests presentation synchronized to the display refresh. /// /// # Note /// - /// Applies only to exlcusive fullscreen mode. + /// Applies only to exclusive fullscreen mode. pub fn v_sync(mut self, enabled: bool) -> Self { self.v_sync = Some(enabled); self @@ -610,8 +665,8 @@ pub enum WindowError { Driver(DriverError), /// `winit` failed to create or run the event loop. EventLoop(EventLoopError), - /// An window system integration or swapchain presentation error occurred. - Swapchain(SwapchainError), + /// A window system integration or swapchain presentation error occurred. + Graphchain(GraphchainError), } impl error::Error for WindowError { @@ -619,7 +674,7 @@ impl error::Error for WindowError { Some(match self { Self::Driver(err) => err, Self::EventLoop(err) => err, - Self::Swapchain(err) => err, + Self::Graphchain(err) => err, }) } } @@ -629,7 +684,7 @@ impl fmt::Display for WindowError { match self { Self::Driver(err) => err.fmt(f), Self::EventLoop(err) => err.fmt(f), - Self::Swapchain(err) => err.fmt(f), + Self::Graphchain(err) => err.fmt(f), } } } diff --git a/crates/vk-graph-window/src/swapchain.rs b/crates/vk-graph-window/src/swapchain.rs deleted file mode 100644 index a31b14a0..00000000 --- a/crates/vk-graph-window/src/swapchain.rs +++ /dev/null @@ -1,623 +0,0 @@ -//! Window swapchain creation, acquisition, and presentation helpers. - -use { - derive_builder::Builder, - log::{trace, warn}, - std::{ - error::Error, - fmt::{Debug, Formatter}, - ops::Deref, - slice, - thread::panicking, - time::Instant, - }, - vk_graph::{ - Graph, - driver::{ - DriverError, - ash::{self, vk}, - cmd_buf::{CommandBuffer, CommandBufferInfo}, - descriptor_set::{DescriptorPool, DescriptorPoolInfo}, - device::Device, - image::Image, - render_pass::{RenderPass, RenderPassInfo}, - surface::Surface, - swapchain::{self, SwapchainImage}, - sync::{AccessType, ImageBarrier, ImageLayout, cmd::pipeline_barrier}, - }, - node::SwapchainImageNode, - pool::Pool, - }, -}; - -fn create_semaphore(device: &ash::Device) -> Result { - let create_info = vk::SemaphoreCreateInfo::default(); - let allocation_callbacks = None; - - unsafe { device.create_semaphore(&create_info, allocation_callbacks) }.map_err(|err| { - warn!("unable to create semaphore: {err}"); - - DriverError::OutOfMemory - }) -} - -const fn image_access_layout(access: AccessType) -> ImageLayout { - if matches!(access, AccessType::Present | AccessType::ComputeShaderWrite) { - ImageLayout::General - } else { - ImageLayout::Optimal - } -} - -/// A physical display interface. -#[read_only::embed] -pub struct Swapchain { - exec_idx: usize, - execs: Box<[Execution]>, - image_execs: Vec, - - /// Information used to create this resource. - /// - /// _Note:_ This field is read-only. - #[readonly] - pub info: SwapchainInfo, - - /// The swapchain which supports this display. - /// - /// _Note:_ This field is read-only. - #[readonly] - pub swapchain: swapchain::Swapchain, -} - -impl Deref for ReadOnlySwapchain { - type Target = swapchain::Swapchain; - - fn deref(&self) -> &Self::Target { - &self.swapchain - } -} - -impl Swapchain { - /// Constructs a new `Swapchain` object. - pub fn new(surface: Surface, info: impl Into) -> Result { - let info = info.into(); - - assert_ne!(info.command_buffer_count, 0); - - let swapchain_info: swapchain::SwapchainInfo = info.into(); - let swapchain = swapchain::Swapchain::new(surface, swapchain_info)?; - - let mut execs = Vec::with_capacity(info.command_buffer_count as _); - for _ in 0..info.command_buffer_count { - let cmd = CommandBuffer::create( - &swapchain.surface.device, - CommandBufferInfo::new(info.queue_family_index), - )?; - let swapchain_acquired = create_semaphore(&swapchain.surface.device)?; - let swapchain_rendered = create_semaphore(&swapchain.surface.device)?; - - execs.push(Execution { - cmd, - swapchain_acquired, - swapchain_rendered, - }); - } - let execs = execs.into_boxed_slice(); - - Ok(Self { - exec_idx: info.command_buffer_count, - execs, - image_execs: Default::default(), - read_only: ReadOnlySwapchain { info, swapchain }, - }) - } - - /// Acquires the next available swapchain image for rendering. - /// - /// This is a high-level wrapper around [`Swapchain::acquire_next_image`]. - /// It manages the internal execution slot, fence, and semaphore lifecycle automatically. - /// - /// The returned [`SwapchainImage`] (if `Some`) has already had its acquire semaphore - /// submitted to the associated queue for this frame. The caller should record and - /// submit rendering commands that depend on this image in the current frame's graph. - /// - /// # Returns - /// - /// * `Ok(Some(image))` — An image was acquired and may be rendered to. - /// * `Ok(None)` — The swapchain is suboptimal (e.g. the window was resized). Render nothing for - /// this frame and call [`acquire_next_image`][Self::acquire_next_image] again next frame. - /// * `Err(SwapchainError::DeviceLost)` — The Vulkan device has been lost. - /// * `Err(SwapchainError::Driver(_))` — The surface was lost or another unrecoverable driver - /// error occurred. - /// - /// # Internal behavior - /// - /// Each call advances to the next internal execution slot (round-robin across - /// a fixed number of submissions in flight). Before acquiring, the method waits - /// on the fence for the previous submission in that slot, ensuring at most one - /// in-flight submission per slot. It then calls the core - /// [`acquire_next_image`](vk_graph::driver::swapchain::Swapchain::acquire_next_image) - /// with an infinite timeout ([`u64::MAX`]) and the slot's acquire semaphore. - /// - /// After successful acquisition the slot index is recorded against the returned - /// image index so that future waits are correctly associated with the right - /// submission. - /// - /// # Errors - /// - /// Returns `Ok(None)` instead of surfacing a suboptimal-state error; the window-level - /// API collapses suboptimal into a non-error signal so the draw loop can skip - /// the frame gracefully. - pub fn acquire_next_image(&mut self) -> Result, SwapchainError> { - self.exec_idx += 1; - self.exec_idx %= self.execs.len(); - let exec = &mut self.execs[self.exec_idx]; - - exec.cmd.wait_until_executed().inspect_err(|err| { - warn!("unable to wait for swapchain fence: {err}"); - })?; - - Device::reset_fences(&exec.cmd.device, slice::from_ref(&exec.cmd.fence))?; - - let acquire_next_image = self - .read_only - .swapchain - .acquire_next_image(u64::MAX, exec.swapchain_acquired); - - if let Err(err) = acquire_next_image { - warn!("unable to acquire next swapchain image: {err:?}"); - } - - let swapchain_image = match acquire_next_image { - Err(swapchain::SwapchainError::DeviceLost) => Err(SwapchainError::DeviceLost), - Err(swapchain::SwapchainError::Suboptimal) => return Ok(None), - Err(swapchain::SwapchainError::SurfaceLost) => { - warn!("invalid swapchain surface state: surface lost"); - Err(SwapchainError::Driver(DriverError::InvalidData)) - } - Ok(swapchain_image) => Ok(swapchain_image), - }?; - - while swapchain_image.index >= self.image_execs.len() as u32 { - self.image_execs.push(0); - } - - self.image_execs[swapchain_image.index as usize] = self.exec_idx; - - Ok(Some(swapchain_image)) - } - - /// Displays the given swapchain image using passes specified in `graph`, if possible. - #[profiling::function] - pub fn present_image

( - &mut self, - pool: &mut P, - graph: Graph, - swapchain_image: SwapchainImageNode, - queue_index: u32, - ) -> Result<(), SwapchainError> - where - P: Pool + Pool, - { - trace!("present_image"); - - let mut submission = graph.into_submission(); - let wait_dst_stage_mask = submission.node_stages(swapchain_image); - - // The swapchain should have been written to, otherwise it would be noise and that's a panic - assert!( - !wait_dst_stage_mask.is_empty(), - "uninitialized swapchain image: write something each frame!", - ); - - let image_idx = submission.resource(swapchain_image).index; - let exec_idx = self.image_execs[image_idx as usize]; - let exec = &mut self.execs[exec_idx]; - - debug_assert!(!exec.cmd.has_executed().unwrap()); - - let started = Instant::now(); - - Device::begin_command_buffer( - &exec.cmd.device, - exec.cmd.handle, - &vk::CommandBufferBeginInfo::default() - .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), - )?; - - // submission.record_node_dependencies(&mut *self.pool, cmd, swapchain_image)?; - submission.queue_cmds_for_resource(&mut exec.cmd, swapchain_image, pool)?; - - { - let swapchain_image = submission.resource(swapchain_image); - for (access, range) in Image::access( - swapchain_image, - AccessType::Present, - vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_array_layer: 0, - base_mip_level: 0, - layer_count: 1, - level_count: 1, - }, - ) { - trace!( - "image {:?} {:?}->{:?}", - swapchain_image, - access, - AccessType::Present, - ); - - // Force a presentation layout transition - pipeline_barrier( - &exec.cmd.device, - exec.cmd.handle, - None, - &[], - slice::from_ref(&ImageBarrier { - previous_accesses: slice::from_ref(&access), - previous_layout: image_access_layout(access), - next_accesses: slice::from_ref(&AccessType::Present), - next_layout: image_access_layout(AccessType::Present), - discard_contents: false, - src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - image: swapchain_image.handle, - range, - }), - ); - } - } - - // We may have unresolved nodes; things like copies that happen after present or operations - // before present which use nodes that are unused in the remainder of the graph. - // These operations are still important, but they don't need to wait for any of the above - // things so we do them last - submission.submit_cmd_buf(pool, &mut exec.cmd)?; - - Device::with_queue( - &exec.cmd.device, - self.read_only.info.queue_family_index, - queue_index, - |queue| { - Device::end_command_buffer(&exec.cmd.device, exec.cmd.handle)?; - Device::queue_submit( - &exec.cmd.device, - queue, - slice::from_ref( - &vk::SubmitInfo::default() - .command_buffers(slice::from_ref(&exec.cmd.handle)) - .wait_semaphores(slice::from_ref(&exec.swapchain_acquired)) - .wait_dst_stage_mask(slice::from_ref(&wait_dst_stage_mask)) - .signal_semaphores(slice::from_ref(&exec.swapchain_rendered)), - ), - exec.cmd.fence, - ) - }, - )?; - - let elapsed = Instant::now() - started; - trace!("🔜🔜🔜 vkQueueSubmit took {} μs", elapsed.as_micros(),); - - let swapchain_image = submission.resource(swapchain_image).clone(); - - self.read_only.swapchain.present_image( - swapchain_image, - slice::from_ref(&exec.swapchain_rendered), - self.read_only.info.queue_family_index, - queue_index, - )?; - - // Store the resolved graph because it contains bindings, pooled resources, and other shared - // resources that need to be kept alive until the fence is waited upon. - exec.cmd.drop_after_executed(submission); - - Ok(()) - } - - /// Updates the information which controls the swapchain. - /// - /// Previously acquired swapchain images should be discarded after calling this function. - pub fn set_info(&mut self, info: impl Into) { - let info = info.into(); - - self.read_only.swapchain.set_info(info); - self.read_only.info.height = info.height; - self.read_only.info.min_image_count = info.min_image_count; - self.read_only.info.present_mode = info.present_mode; - self.read_only.info.surface = info.surface; - self.read_only.info.width = info.width; - } -} - -impl Debug for Swapchain { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str("Swapchain") - } -} - -impl Drop for Swapchain { - fn drop(&mut self) { - if panicking() { - return; - } - - let idle = unsafe { self.execs[0].cmd.device.device_wait_idle() }; - if idle.is_err() { - warn!("unable to wait for device"); - - return; - } - - for batch in &mut self.execs { - // Wait for presentation to stop - let present = batch.cmd.wait_until_executed(); - if present.is_err() { - warn!("unable to wait for queue"); - - continue; - } - - unsafe { - batch - .cmd - .device - .destroy_semaphore(batch.swapchain_acquired, None); - batch - .cmd - .device - .destroy_semaphore(batch.swapchain_rendered, None); - } - } - } -} - -/// Describes error conditions relating to physical displays. -#[derive(Clone, Copy, Debug)] -pub enum SwapchainError { - /// Unrecoverable device error; must destroy this device and display and start a new one. - DeviceLost, - - /// Recoverable driver error. - Driver(DriverError), -} - -impl Error for SwapchainError {} - -impl From<()> for SwapchainError { - fn from(_: ()) -> Self { - Self::DeviceLost - } -} - -impl From for SwapchainError { - fn from(err: DriverError) -> Self { - Self::Driver(err) - } -} - -impl std::fmt::Display for SwapchainError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} - -/// Information used to create a [`Swapchain`] instance. -#[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] -#[builder( - build_fn(private, name = "fallible_build"), - derive(Clone, Copy, Debug), - pattern = "owned" -)] -pub struct SwapchainInfo { - /// The number of command buffers to use for image submissions. - /// - /// Generally one more than the swapchain image count is best. - #[builder(default = "4")] - pub command_buffer_count: usize, - - /// The initial height of the surface. - #[builder(default)] - pub height: u32, - - /// The minimum number of presentable images that the application needs. The implementation - /// will either create the swapchain with at least that many images, or it will fail to - /// create the swapchain. - /// - /// More images introduce more display lag, but smoother animation. - #[builder(default = "2")] - pub min_image_count: u32, - - /// `vk::PresentModeKHR` determines timing and queueing with which frames are actually - /// displayed to the user. - /// - /// `vk::PresentModeKHR::FIFO` - Presentation frames are kept in a First-In-First-Out queue - /// approximately 3 frames long. Every vertical blanking period, the presentation engine - /// will pop a frame off the queue to display. If there is no frame to display, it will - /// present the same frame again until the next vblank. - /// - /// When a present command is executed on the GPU, the presented image is added on the queue. - /// - /// * **Tearing:** No tearing will be observed. - /// * **Also known as**: "Vsync On" - /// - /// `vk::PresentModeKHR::FIFO_RELAXED` - Presentation frames are kept in a First-In-First-Out - /// queue approximately 3 frames long. Every vertical blanking period, the presentation - /// engine will pop a frame off the queue to display. If there is no frame to display, it - /// will present the same frame until there is a frame in the queue. The moment there is a - /// frame in the queue, it will immediately pop the frame off the queue. - /// - /// When a present command is executed on the GPU, the presented image is added on the queue. - /// - /// * **Tearing**: Tearing will be observed if frames last more than one vblank as the front - /// buffer. - /// * **Also known as**: "Adaptive Vsync" - /// - /// `vk::PresentModeKHR::IMMEDIATE` - Presentation frames are not queued at all. The moment a - /// present command is executed on the GPU, the presented image is swapped onto the front - /// buffer immediately. - /// - /// * **Tearing**: Tearing can be observed. - /// * **Also known as**: "Vsync Off" - /// - /// `vk::PresentModeKHR::MAILBOX` - Presentation frames are kept in a single-frame queue. Every - /// vertical blanking period, the presentation engine will pop a frame from the queue. If - /// there is no frame to display, it will present the same frame again until the next - /// vblank. - /// - /// When a present command is executed on the GPU, the frame will be put into the queue. - /// If there was already a frame in the queue, the new frame will _replace_ the old frame. - /// on the queue. - /// - /// * **Tearing**: No tearing will be observed. - /// * **Also known as**: "Fast Vsync" - #[builder(default = vk::PresentModeKHR::MAILBOX)] - pub present_mode: vk::PresentModeKHR, - - /// The device queue family which will be used to submit and present images. - #[builder(default = "0")] - pub queue_family_index: u32, - - /// The format and color space of the surface. - #[builder(default)] - pub surface: vk::SurfaceFormatKHR, - - /// The initial width of the surface. - #[builder(default)] - pub width: u32, -} - -impl SwapchainInfo { - /// Specifies a default swapchain with the given `width`, `height` and `format` values. - #[inline(always)] - pub fn new(width: u32, height: u32, surface: vk::SurfaceFormatKHR) -> SwapchainInfo { - Self { - command_buffer_count: 4, - height, - min_image_count: 2, - present_mode: vk::PresentModeKHR::MAILBOX, - queue_family_index: 0, - surface, - width, - } - } - - /// Converts a `SwapchainInfo` into a `SwapchainInfoBuilder`. - pub fn into_builder(self) -> SwapchainInfoBuilder { - SwapchainInfoBuilder { - command_buffer_count: Some(self.command_buffer_count), - height: Some(self.height), - min_image_count: Some(self.min_image_count), - present_mode: Some(self.present_mode), - queue_family_index: Some(self.queue_family_index), - surface: Some(self.surface), - width: Some(self.width), - } - } -} - -impl From for SwapchainInfo { - fn from(info: SwapchainInfoBuilder) -> Self { - info.build() - } -} - -impl From for swapchain::SwapchainInfo { - fn from(val: SwapchainInfo) -> Self { - swapchain::SwapchainInfoBuilder::default() - .height(val.height) - .min_image_count(val.min_image_count) - .present_mode(val.present_mode) - .surface(val.surface) - .width(val.width) - .build() - } -} - -impl SwapchainInfoBuilder { - /// Builds a new `SwapchainInfo`. - #[inline(always)] - pub fn build(self) -> SwapchainInfo { - self.fallible_build().expect("all fields have defaults") - } -} - -struct Execution { - cmd: CommandBuffer, - swapchain_acquired: vk::Semaphore, - swapchain_rendered: vk::Semaphore, -} - -#[cfg(test)] -mod test { - use super::*; - - type Info = SwapchainInfo; - type Builder = SwapchainInfoBuilder; - - #[test] - pub fn swapchain_info() { - let info = Info { - command_buffer_count: 42, - height: 123, - min_image_count: 99, - present_mode: vk::PresentModeKHR::IMMEDIATE, - queue_family_index: 16, - surface: vk::SurfaceFormatKHR::default() - .format(vk::Format::R8G8B8A8_UNORM) - .color_space(vk::ColorSpaceKHR::PASS_THROUGH_EXT), - width: 88, - }; - let builder = info.into_builder().build(); - - assert_eq!(info, builder); - } - - #[test] - pub fn swapchain_info_builder() { - let info = Info { - command_buffer_count: 42, - height: 123, - min_image_count: 99, - present_mode: vk::PresentModeKHR::IMMEDIATE, - queue_family_index: 16, - surface: vk::SurfaceFormatKHR::default() - .format(vk::Format::R8G8B8A8_UNORM) - .color_space(vk::ColorSpaceKHR::PASS_THROUGH_EXT), - width: 88, - }; - let builder = Builder::default() - .command_buffer_count(42) - .height(123) - .min_image_count(99) - .present_mode(vk::PresentModeKHR::IMMEDIATE) - .queue_family_index(16) - .surface( - vk::SurfaceFormatKHR::default() - .format(vk::Format::R8G8B8A8_UNORM) - .color_space(vk::ColorSpaceKHR::PASS_THROUGH_EXT), - ) - .width(88) - .build(); - - assert_eq!(info, builder); - } - - #[test] - pub fn swapchain_info_builder_defaults() { - let info = Builder::default().build(); - - assert_eq!( - info, - Info { - command_buffer_count: 4, - height: 0, - min_image_count: 2, - present_mode: vk::PresentModeKHR::MAILBOX, - queue_family_index: 0, - surface: vk::SurfaceFormatKHR::default(), - width: 0, - } - ); - } -} diff --git a/examples/README.md b/examples/README.md index e115e3c5..2d8e4be7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,35 +9,36 @@ See the [README](../README.md) for more information. ## Example Code -Example | Instructions | Preview - --- | --- | :---: -[aliasing.rs](aliasing.rs) | 

cargo run --example aliasing
| _See console output_ -[cpu_readback.rs](cpu_readback.rs) | 
cargo run --example cpu_readback
| _See console output_ -[debugger.rs](debugger.rs) | 
cargo run --example debugger
| _See console output_ -[min_max.rs](min_max.rs) | 
cargo run --example min_max
| _See console output_ -[mip_compute.rs](mip_compute.rs) | 
cargo run --example mip_compute
| _See console output_ -[subgroup_ops.rs](subgroup_ops.rs) | 
cargo run --example subgroup_ops
| _See console output_ -[hello_world.rs](../crates/vk-graph-window/examples/hello_world.rs) | _See [vk-graph-window](../crates/vk-graph-window/README.md)_ | hello_world.rs -[app.rs](app.rs) | 
cargo run --example app
| app.rs -[triangle.rs](triangle.rs) | 
cargo run --example triangle
| triangle.rs -[vertex_layout.rs](vertex_layout.rs) | 
cargo run --example vertex_layout
| vertex_layout.rs -[bindless.rs](bindless.rs) | 
cargo run --example bindless
| bindless.rs -[image_sampler.rs](image_sampler.rs) | 
cargo run --example image_sampler
| image_sampler.rs -[egui.rs](egui.rs) | 
cargo run --example egui
| egui.rs -[imgui.rs](imgui.rs) | 
cargo run --example imgui
| imgui.rs -[font_bmp.rs](font_bmp.rs) | 
cargo run --example font_bmp
| font_bmp.rs -[mip_graphic.rs](mip_graphic.rs) | 
cargo run --example mip_graphic
| mip_graphic.rs -[multipass.rs](multipass.rs) | 
cargo run --example multipass
| multipass.rs -[multithread.rs](multithread.rs) | 
cargo run --example multithread --release
| multithread.rs -[msaa.rs](msaa.rs) | 
cargo run --example msaa
Multisample anti-aliasing | msaa.rs -[rt_triangle.rs](rt_triangle.rs) | 
cargo run --example rt_triangle
| rt_triangle.rs -[ray_trace.rs](ray_trace.rs) | 
cargo run --example ray_trace
| ray_trace.rs -[vsm_omni.rs](vsm_omni.rs) | 
cargo run --example vsm_omni
Variance shadow mapping for omni/point lights | vsm_omni.rs -[ray_omni.rs](ray_omni.rs) | 
cargo run --example ray_omni
Ray query for omni/point lights | ray_omni.rs -[transitions.rs](transitions.rs) | 
cargo run --example transitions
| transitions.rs -[skeletal-anim/](skeletal-anim/src/main.rs) | 
cargo run -p skeletal-anim
Skeletal mesh animation using GLTF | skeletal-anim -[shader-toy/](shader-toy/src/main.rs) | 
cargo run -p shader-toy
| shader-toy -[vr/](vr/src/main.rs) | 
cargo run -p vr
| vr +| Example | Instructions | Preview | +| --- | --- | :---: | +| [aliasing.rs](aliasing.rs) | 
cargo run --example aliasing
| _See console output_ | +| [cpu_readback.rs](cpu_readback.rs) | 
cargo run --example cpu_readback
| _See console output_ | +| [debugger.rs](debugger.rs) | 
cargo run --example debugger
| _See console output_ | +| [min_max.rs](min_max.rs) | 
cargo run --example min_max
| _See console output_ | +| [mip_compute.rs](mip_compute.rs) | 
cargo run --example mip_compute
| _See console output_ | +| [baked.rs](baked.rs) | 
cargo run --example baked
| _See console output_ | +| [subgroup_ops.rs](subgroup_ops.rs) | 
cargo run --example subgroup_ops
| _See console output_ | +| [hello_world.rs](../crates/vk-graph-window/examples/hello_world.rs) | _See [vk-graph-window](../crates/vk-graph-window/README.md)_ | hello_world.rs | +| [app.rs](app.rs) | 
cargo run --example app
| app.rs | +| [triangle.rs](triangle.rs) | 
cargo run --example triangle
| triangle.rs | +| [vertex_layout.rs](vertex_layout.rs) | 
cargo run --example vertex_layout
| vertex_layout.rs | +| [bindless.rs](bindless.rs) | 
cargo run --example bindless
| bindless.rs | +| [image_sampler.rs](image_sampler.rs) | 
cargo run --example image_sampler
| image_sampler.rs | +| [egui.rs](egui.rs) | 
cargo run --example egui
| egui.rs | +| [imgui.rs](imgui.rs) | 
cargo run --example imgui
| imgui.rs | +| [font_bmp.rs](font_bmp.rs) | 
cargo run --example font_bmp
| font_bmp.rs | +| [mip_graphics.rs](mip_graphics.rs) | 
cargo run --example mip_graphics
| mip_graphics.rs | +| [multipass.rs](multipass.rs) | 
cargo run --example multipass
| multipass.rs | +| [multithread.rs](multithread.rs) | 
cargo run --example multithread --release
| multithread.rs | +| [msaa.rs](msaa.rs) | 
cargo run --example msaa
Multisample anti-aliasing | msaa.rs | +| [rt_triangle.rs](rt_triangle.rs) | 
cargo run --example rt_triangle
| rt_triangle.rs | +| [ray_tracing.rs](ray_tracing.rs) | 
cargo run --example ray_tracing
| ray_tracing.rs | +| [vsm_omni.rs](vsm_omni.rs) | 
cargo run --example vsm_omni
Variance shadow mapping for omni/point lights | vsm_omni.rs | +| [ray_omni.rs](ray_omni.rs) | 
cargo run --example ray_omni
Ray query for omni/point lights | ray_omni.rs | +| [transitions.rs](transitions.rs) | 
cargo run --example transitions
| transitions.rs | +| [skeletal-anim/](skeletal-anim/src/main.rs) | 
cargo run -p skeletal-anim
Skeletal mesh animation using glTF | skeletal-anim | +| [shader-toy/](shader-toy/src/main.rs) | 
cargo run -p shader-toy
| shader-toy | +| [vr/](vr/src/main.rs) | 
cargo run -p vr
| vr | ## Additional Examples diff --git a/examples/aliasing.rs b/examples/aliasing.rs index e2ccf17e..b403d114 100644 --- a/examples/aliasing.rs +++ b/examples/aliasing.rs @@ -75,7 +75,7 @@ fn main() -> Result<(), DriverError> { let image_info = ImageInfo::image_2d( image_info.width, image_info.height, - image_info.fmt, + image_info.format, vk::ImageUsageFlags::TRANSFER_DST, ); @@ -83,7 +83,7 @@ fn main() -> Result<(), DriverError> { let image3_node = graph.bind_resource(cache.tag(CacheTag::Shadow).resource(image_info)?); assert_eq!(image1_node, image3_node); - // Using a different tag for the same request produces an entirely different image. + // Using a different tag for the same request produces an entirely different image let image4_node = graph.bind_resource(cache.tag(CacheTag::Ui).resource(image_info)?); assert_ne!(image1_node, image4_node); diff --git a/examples/app.rs b/examples/app.rs index 48947008..683cb54f 100644 --- a/examples/app.rs +++ b/examples/app.rs @@ -11,7 +11,7 @@ use { }, pool::hash::HashPool, }, - vk_graph_window::swapchain::{Swapchain, SwapchainError, SwapchainInfo}, + vk_graph_window::graphchain::{Graphchain, GraphchainError, GraphchainInfo}, winit::{ application::ApplicationHandler, error::EventLoopError, @@ -53,9 +53,9 @@ impl ApplicationHandler for Application { .first() .copied() .unwrap(); - let swapchain = Swapchain::new( + let graphchain = Graphchain::new( surface, - SwapchainInfo::new(window_size.width, window_size.height, surface_format) + GraphchainInfo::new(window_size.width, window_size.height, surface_format) .into_builder() .present_mode(present_mode), ) @@ -64,7 +64,7 @@ impl ApplicationHandler for Application { let swapchain_pool = HashPool::new(&device); self.0 = Some(Context { - swapchain, + graphchain, swapchain_pool, window, }); @@ -81,10 +81,10 @@ impl ApplicationHandler for Application { match event { WindowEvent::CloseRequested => event_loop.exit(), WindowEvent::Resized(size) => { - let mut swapchain_info = context.swapchain.info; - swapchain_info.width = size.width; - swapchain_info.height = size.height; - context.swapchain.set_info(swapchain_info); + let mut info = context.graphchain.info; + info.width = size.width; + info.height = size.height; + context.graphchain.set_info(info); } WindowEvent::RedrawRequested => { if let Err(err) = context.draw() { @@ -109,14 +109,14 @@ struct Args { } struct Context { - swapchain: Swapchain, + graphchain: Graphchain, swapchain_pool: HashPool, window: Window, } impl Context { - fn draw(&mut self) -> Result<(), SwapchainError> { - if let Some(swapchain_image) = self.swapchain.acquire_next_image()? { + fn draw(&mut self) -> Result<(), GraphchainError> { + if let Some(swapchain_image) = self.graphchain.acquire_next_image()? { let mut graph = Graph::default(); let swapchain_image = graph.bind_resource(swapchain_image); @@ -124,7 +124,7 @@ impl Context { graph.clear_color_image(swapchain_image, [1.0, 0.0, 1.0, 1.0]); self.window.pre_present_notify(); - self.swapchain + self.graphchain .present_image(&mut self.swapchain_pool, graph, swapchain_image, 0)?; } diff --git a/examples/baked.rs b/examples/baked.rs new file mode 100644 index 00000000..89cf9310 --- /dev/null +++ b/examples/baked.rs @@ -0,0 +1,538 @@ +//! Demonstrates a reusable [`CommandStream`](vk_graph::stream::CommandStream) pattern for a game-like +//! frame. +//! +//! The scene renders a spinning tetrahedron every frame, then invokes a prepared command stream for +//! static UI chrome. Press Enter to regenerate the stream with different pseudo-text geometry. +//! +//! This pattern is useful when part of the frame is expensive to build but cheap to run: static HUD +//! panels, editor chrome, minimap backgrounds, large text blocks, or cached vector/UI tessellation. +//! The parent graph stays dynamic while the prepared stream keeps its internal graph optimization +//! and leased recording resources. + +use { + bytemuck::{Pod, Zeroable, cast_slice}, + clap::Parser, + glam::{Mat4, Vec3, vec3}, + rand::{Rng, SeedableRng, rngs::SmallRng}, + std::{mem::size_of, sync::Arc, time::Instant}, + vk_graph::{ + cmd::{ClearColorValue, LoadOp, StoreOp}, + driver::{ + DriverError, + ash::vk, + buffer::{Buffer, BufferInfo}, + graphics::{ + BlendInfo, DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfoBuilder, + }, + image::ImageInfo, + shader::Shader, + sync::AccessType, + }, + pool::{Pool as _, hash::HashPool}, + stream::{CommandStream, ImageArg}, + }, + vk_graph_window::{Window, WindowError, winit}, + vk_shader_macros::glsl, + winit::{ + event::{ElementState, Event, WindowEvent}, + keyboard::{KeyCode, PhysicalKey}, + }, +}; + +const UI_DESIGN_WIDTH: f32 = 1280.0; +const UI_DESIGN_HEIGHT: f32 = 720.0; + +fn main() -> Result<(), WindowError> { + pretty_env_logger::init(); + + let args = Args::parse(); + let window = Window::builder() + .debug(args.debug) + .min_image_count(3) + .build()?; + + let pipelines = Pipelines::create(&window.device)?; + let tetrahedron = Tetrahedron::create(&window.device)?; + let mut pool = HashPool::new(&window.device); + let mut ui = UiLayer::new(); + let started_at = Instant::now(); + + window.run(move |frame| { + let swapchain_info = frame.graph.resource(frame.swapchain_image).info; + let ui_dirty = frame.events.iter().any(enter_pressed) + || ui.width != frame.width + || ui.height != frame.height; + + if ui_dirty { + ui.regenerate( + &mut pool, + &pipelines.ui, + frame.width, + frame.height, + swapchain_info, + ) + .unwrap(); + } + + let elapsed = started_at.elapsed().as_secs_f32(); + + // Bind the current immutable UI generation into this frame's transient graph + let tetrahedron_vtx_buf = frame.graph.bind_resource(&tetrahedron.vertex_buffer); + let tetrahedron_idx_buf = frame.graph.bind_resource(&tetrahedron.index_buffer); + let depth_image = frame.graph.bind_resource( + pool.resource(ImageInfo::image_2d( + frame.width, + frame.height, + vk::Format::D32_SFLOAT, + vk::ImageUsageFlags::DEPTH_STENCIL_ATTACHMENT, + )) + .unwrap(), + ); + + let model = Mat4::from_rotation_y(elapsed * 0.9) * Mat4::from_rotation_x(elapsed * 0.6); + let view = Mat4::look_at_rh(vec3(0.0, 0.0, 3.2), Vec3::ZERO, Vec3::Y); + + // Square projection plus X correction keeps the model from stretching with the window + let projection = Mat4::perspective_rh(60.0_f32.to_radians(), 1.0, 0.1, 32.0); + let aspect_correction = Mat4::from_scale(vec3( + frame.height as f32 / frame.width.max(1) as f32, + 1.0, + 1.0, + )); + let transform = aspect_correction * projection * view * model; + + frame + .graph + .begin_cmd() + .debug_name("spinning tetrahedron") + .bind_pipeline(&pipelines.tetrahedron) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) + // Declaring access lets the graph schedule barriers around these buffer reads + .resource_access(tetrahedron_vtx_buf, AccessType::VertexBuffer) + .resource_access(tetrahedron_idx_buf, AccessType::IndexBuffer) + .color_attachment_image( + 0, + frame.swapchain_image, + LoadOp::Clear(ClearColorValue::rgba(0.012, 0.012, 0.014, 1.0)), + StoreOp::Store, + ) + .depth_stencil_attachment_image( + depth_image, + LoadOp::CLEAR_ONE_STENCIL_ZERO, + StoreOp::DontCare, + ) + .record_cmd(move |cmd| { + cmd.bind_vertex_buffer(0, tetrahedron_vtx_buf, 0) + .bind_index_buffer(tetrahedron_idx_buf, 0, vk::IndexType::UINT16) + .push_constants(0, cast_slice(&transform.to_cols_array())) + .draw_indexed(12, 1, 0, 0, 0); + }); + + if let Some(ui) = ui.stream.as_ref() { + frame + .graph + .insert_cmd_stream(&ui.stream) + .with_arg(ui.stream.args.output, frame.swapchain_image) + .finish(); + } + }) +} + +fn enter_pressed(event: &Event<()>) -> bool { + let Event::WindowEvent { + event: WindowEvent::KeyboardInput { event, .. }, + .. + } = event + else { + return false; + }; + + event.state == ElementState::Pressed + && matches!(event.physical_key, PhysicalKey::Code(KeyCode::Enter)) +} + +struct Pipelines { + tetrahedron: GraphicsPipeline, + ui: GraphicsPipeline, +} + +impl Pipelines { + fn create(device: &vk_graph::driver::device::Device) -> Result { + let tetrahedron = GraphicsPipeline::create( + device, + GraphicsPipelineInfoBuilder::default(), + [ + Shader::new_vertex( + glsl!( + r#" + #version 460 core + #pragma shader_stage(vertex) + + layout(push_constant) uniform PushConstants { + mat4 transform; + } pc; + + layout(location = 0) in vec3 position; + layout(location = 1) in vec4 color; + layout(location = 0) out vec4 frag_color; + + void main() { + gl_Position = pc.transform * vec4(position, 1.0); + frag_color = color; + } + "# + ) + .as_slice(), + ), + Shader::new_fragment( + glsl!( + r#" + #version 460 core + #pragma shader_stage(fragment) + + layout(location = 0) in vec4 frag_color; + layout(location = 0) out vec4 out_color; + + void main() { + out_color = frag_color; + } + "# + ) + .as_slice(), + ), + ], + )?; + + let ui = GraphicsPipeline::create( + device, + GraphicsPipelineInfoBuilder::default() + .blend(BlendInfo::ALPHA) + .cull_mode(vk::CullModeFlags::NONE), + [ + Shader::new_vertex( + glsl!( + r#" + #version 460 core + #pragma shader_stage(vertex) + + layout(location = 0) in vec2 position; + layout(location = 1) in vec4 color; + layout(location = 0) out vec4 frag_color; + + void main() { + gl_Position = vec4(position, 0.0, 1.0); + frag_color = color; + } + "# + ) + .as_slice(), + ), + Shader::new_fragment( + glsl!( + r#" + #version 460 core + #pragma shader_stage(fragment) + + layout(location = 0) in vec4 frag_color; + layout(location = 0) out vec4 out_color; + + void main() { + out_color = frag_color; + } + "# + ) + .as_slice(), + ), + ], + )?; + + Ok(Self { tetrahedron, ui }) + } +} + +struct UiLayer { + stream: Option, + width: u32, + height: u32, + seed: u64, +} + +impl UiLayer { + fn new() -> Self { + Self { + stream: None, + width: 0, + height: 0, + seed: rand::rng().random(), + } + } + + fn regenerate( + &mut self, + pool: &mut HashPool, + pipeline: &GraphicsPipeline, + width: u32, + height: u32, + output_info: ImageInfo, + ) -> Result<(), DriverError> { + self.seed = self.seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + self.stream = Some(UiStream::prepare( + pipeline, + pool, + self.seed, + width, + height, + output_info, + )?); + self.width = width; + self.height = height; + + Ok(()) + } +} + +#[derive(Clone, Copy)] +struct UiArgs { + output: ImageArg, +} + +struct UiStream { + stream: CommandStream, +} + +impl UiStream { + fn prepare( + pipeline: &GraphicsPipeline, + pool: &mut HashPool, + seed: u64, + width: u32, + height: u32, + output_info: ImageInfo, + ) -> Result { + let vertices = build_ui_vertices(seed, width, height); + let vertex_count = vertices.len() as u32; + let vertex_data = cast_slice(vertices.as_slice()); + let vertex_buf_info = + BufferInfo::host_mem(vertex_data.len() as _, vk::BufferUsageFlags::VERTEX_BUFFER); + let mut vertex_buf = pool.resource(vertex_buf_info)?; + vertex_buf.copy_from_slice(0, vertex_data); + + let stream = CommandStream::prepare(pool, |stream| { + let output = stream.arg(output_info); + let vertex_node = stream.bind_resource(vertex_buf); + + stream + .begin_cmd() + .debug_name("cached UI stream") + .bind_pipeline(pipeline) + .resource_access(vertex_node, AccessType::VertexBuffer) + .color_attachment_image(0, output, LoadOp::Load, StoreOp::Store) + .record_cmd(move |cmd| { + cmd.bind_vertex_buffer(0, vertex_node, 0) + .draw(vertex_count, 1, 0, 0); + }); + + UiArgs { output } + })?; + + Ok(Self { stream }) + } +} + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct UiVertex { + position: [f32; 2], + color: [f32; 4], +} + +fn build_ui_vertices(seed: u64, target_width: u32, target_height: u32) -> Vec { + let mut rng = SmallRng::seed_from_u64(seed); + + // Use one scale factor for both axes; extra window space remains empty instead of stretching + let scale = + (target_width as f32 / UI_DESIGN_WIDTH).min(target_height as f32 / UI_DESIGN_HEIGHT); + let mut canvas = UiCanvas { + vertices: Vec::new(), + width: target_width, + height: target_height, + scale, + }; + + canvas.rect(Rect::new(0.0, 0.0, UI_DESIGN_WIDTH, 30.0), gray(0.22, 0.92)); + canvas.rect(Rect::new(0.0, 30.0, UI_DESIGN_WIDTH, 3.0), gray(0.55, 0.95)); + canvas.rect(Rect::new(0.0, 33.0, UI_DESIGN_WIDTH, 4.0), gray(0.06, 0.9)); + + let mut x = 16.0; + for width in [52.0, 38.0, 64.0, 45.0, 58.0] { + canvas.rect(Rect::new(x, 8.0, width, 12.0), gray(0.72, 0.95)); + x += width + 18.0; + } + + canvas.outline(Rect::new(1018.0, 6.0, 20.0, 18.0), 2.0, gray(0.78, 0.95)); + canvas.outline(Rect::new(1048.0, 6.0, 20.0, 18.0), 2.0, gray(0.78, 0.95)); + canvas.outline(Rect::new(1078.0, 6.0, 20.0, 18.0), 2.0, gray(0.78, 0.95)); + + canvas.rect(Rect::new(16.0, 56.0, 378.0, 164.0), gray(0.13, 0.84)); + canvas.outline(Rect::new(16.0, 56.0, 378.0, 164.0), 4.0, gray(0.62, 0.9)); + canvas.rect(Rect::new(22.0, 62.0, 366.0, 20.0), gray(0.3, 0.95)); + + for i in 0..4 { + let y = 99.0 + i as f32 * 28.0; + let mut x = 36.0; + let target = rng.random_range(245.0..330.0); + + while x < target { + let width = rng.random_range(7.0..18.0); + canvas.outline(Rect::new(x, y, width, 15.0), 2.0, [0.05, 0.42, 1.0, 0.94]); + x += width + rng.random_range(4.0..8.0); + + if rng.random_bool(0.18) { + x += rng.random_range(8.0..15.0); + } + } + } + + canvas.rect(Rect::new(420.0, 54.0, 132.0, 88.0), gray(0.18, 0.75)); + canvas.outline(Rect::new(420.0, 54.0, 132.0, 88.0), 3.0, gray(0.64, 0.88)); + canvas.rect(Rect::new(438.0, 74.0, 96.0, 10.0), gray(0.5, 0.88)); + canvas.rect(Rect::new(438.0, 94.0, 74.0, 10.0), gray(0.38, 0.88)); + canvas.rect(Rect::new(438.0, 114.0, 106.0, 10.0), gray(0.46, 0.88)); + + canvas.vertices +} + +fn gray(value: f32, alpha: f32) -> [f32; 4] { + [value, value, value, alpha] +} + +#[derive(Clone, Copy)] +struct Rect { + x: f32, + y: f32, + w: f32, + h: f32, +} + +impl Rect { + const fn new(x: f32, y: f32, w: f32, h: f32) -> Self { + Self { x, y, w, h } + } +} + +struct UiCanvas { + vertices: Vec, + width: u32, + height: u32, + scale: f32, +} + +impl UiCanvas { + fn rect(&mut self, rect: Rect, color: [f32; 4]) { + let rect = Rect::new( + rect.x * self.scale, + rect.y * self.scale, + rect.w * self.scale, + rect.h * self.scale, + ); + let l = rect.x / self.width as f32 * 2.0 - 1.0; + let r = (rect.x + rect.w) / self.width as f32 * 2.0 - 1.0; + let t = rect.y / self.height as f32 * 2.0 - 1.0; + let b = (rect.y + rect.h) / self.height as f32 * 2.0 - 1.0; + + self.vertices.extend_from_slice(&[ + UiVertex { + position: [l, t], + color, + }, + UiVertex { + position: [l, b], + color, + }, + UiVertex { + position: [r, b], + color, + }, + UiVertex { + position: [l, t], + color, + }, + UiVertex { + position: [r, b], + color, + }, + UiVertex { + position: [r, t], + color, + }, + ]); + } + + fn outline(&mut self, rect: Rect, thickness: f32, color: [f32; 4]) { + self.rect(Rect::new(rect.x, rect.y, rect.w, thickness), color); + self.rect( + Rect::new(rect.x, rect.y + rect.h - thickness, rect.w, thickness), + color, + ); + self.rect(Rect::new(rect.x, rect.y, thickness, rect.h), color); + self.rect( + Rect::new(rect.x + rect.w - thickness, rect.y, thickness, rect.h), + color, + ); + } +} + +struct Tetrahedron { + index_buffer: Arc, + vertex_buffer: Arc, +} + +impl Tetrahedron { + fn create(device: &vk_graph::driver::device::Device) -> Result { + let vertices = [ + TetrahedronVertex::new([1.0, 1.0, 1.0], [0.0, 0.9, 0.9, 1.0]), + TetrahedronVertex::new([-1.0, -1.0, 1.0], [0.9, 0.0, 0.78, 1.0]), + TetrahedronVertex::new([-1.0, 1.0, -1.0], [0.82, 0.58, 0.12, 1.0]), + TetrahedronVertex::new([1.0, -1.0, -1.0], [0.34, 0.2, 0.11, 1.0]), + ]; + let indices = [0_u16, 1, 2, 0, 3, 1, 0, 2, 3, 1, 3, 2]; + + Ok(Self { + index_buffer: Arc::new(Buffer::create_from_slice( + device, + vk::BufferUsageFlags::INDEX_BUFFER, + cast_slice(&indices), + )?), + vertex_buffer: Arc::new(Buffer::create_from_slice( + device, + vk::BufferUsageFlags::VERTEX_BUFFER, + cast_slice(&vertices), + )?), + }) + } +} + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct TetrahedronVertex { + position: [f32; 3], + color: [f32; 4], +} + +impl TetrahedronVertex { + const fn new(position: [f32; 3], color: [f32; 4]) -> Self { + Self { position, color } + } +} + +#[derive(Parser)] +struct Args { + /// Enable Vulkan SDK validation layers. + #[arg(long)] + debug: bool, +} + +const _: () = { + assert!(size_of::() == 24); + assert!(size_of::() == 28); +}; diff --git a/examples/bindless.rs b/examples/bindless.rs index 09fa75ab..e90d43eb 100644 --- a/examples/bindless.rs +++ b/examples/bindless.rs @@ -12,7 +12,7 @@ use { DriverError, buffer::Buffer, device::Device, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, image::{Image, ImageInfo}, }, pool::lazy::LazyPool, @@ -92,7 +92,7 @@ fn create_images(device: &Device) -> Result>, DriverError> { let mut pool = LazyPool::new(device); - graph.into_submission().queue_submit(&mut pool, 0, 0)?; + graph.finalize().queue_submit(&mut pool, 0, 0)?; Ok(textures) } diff --git a/examples/cpu_readback.rs b/examples/cpu_readback.rs index 06bf8d77..e796eccd 100644 --- a/examples/cpu_readback.rs +++ b/examples/cpu_readback.rs @@ -36,29 +36,33 @@ fn main() -> Result<(), DriverError> { &[0, 0, 0, 0], )?); - // We are using the GPU to copy data, but the same thing works if you're executing a pipeline - // such as a ComputePipeline to run some shader code which writes to a buffer or image. It is - // important to note that dst_buf does not contain the new data until we submit this render - // graph and wait on the result + /* + We are using the GPU to copy data, but the same thing works if you're executing a pipeline such + as a ComputePipeline to run some shader code which writes to a buffer or image. It is important + to note that dst_buf does not contain the new data until we submit this render graph and wait on + the result. + */ graph.copy_buffer(src_buf, dst_buf); // This line is optional - just bind a borrow of Arc or a leased buffer so you retain // the actual buffer for later use and you could then remove this line let dst_buf = graph.resource(dst_buf).clone(); - // Resolve and wait (or you can check has_executed without blocking) - alternatively you might - // use device.queue_wait_idle(0) or device.device_wait_idle() - but those block on larger scopes - let mut cmd = graph - .into_submission() + /* + Resolve and wait. You can check Fence::is_signaled without blocking, or use + device.queue_wait_idle(0) or device.device_wait_idle(), but those block on larger scopes. + */ + let mut fence = graph + .finalize() .queue_submit(&mut HashPool::new(&device), 0, 0)?; - println!("Has executed? {}", cmd.has_executed()?); + println!("Has executed? {}", fence.is_signaled()?); let started = Instant::now(); - cmd.wait_until_executed()?; + fence.wait_signaled()?; assert!( - cmd.has_executed()?, + fence.is_signaled()?, "We checked above - so this will always be true" ); println!("Waited {}μs", (Instant::now() - started).as_micros()); diff --git a/examples/debugger.rs b/examples/debugger.rs index 51bed29a..8a0273ec 100644 --- a/examples/debugger.rs +++ b/examples/debugger.rs @@ -12,8 +12,8 @@ use vk_sync::AccessType; I hope you enjoy this choose-your-own-debugger adventure! - First you will want to read this: - https://github.com/attackgoat/vk-graph/blob/master/examples/getting-started.md + First you may want to read the example overview: + https://github.com/attackgoat/vk-graph/blob/main/examples/README.md Enter your "name" to begin: cargo run --example debugger @@ -29,7 +29,7 @@ use vk_sync::AccessType; All programs must be tested with Vulkan validation layers enabled and look for validity and synchronization errors. - To continue, uncomment line 30. + To continue, uncomment the logger initialization below. */ fn main() -> Result<(), vk_graph_window::WindowError> { use {log::debug, vk_graph_window::Window}; @@ -41,7 +41,8 @@ fn main() -> Result<(), vk_graph_window::WindowError> { The code ahead is filled with a dangerous `panic` if you did not install the Vulkan SDK! You must now choose: - - If you did not install the SDK, you must goto line 8, above. + - If you did not install the SDK, install it before running this example with debug + validation enabled. - If you have a recent SDK installed, you may advance the function pointer. */ Window::builder().debug(true).build()?.run(|frame| { @@ -52,7 +53,7 @@ fn main() -> Result<(), vk_graph_window::WindowError> { Note: This callback runs each time the operating system requests a new window image and it expects you to render something to `frame.swapchain_image` using - `frame.graph`. Note that this scope is infalliable. You may create additional + `frame.graph`. Note that this scope is infallible. You may create additional images and graphs if you choose. You can resolve multiple render graphs per frame - but you only need to do that if you have a hot-section that is part of a VERY large graph. @@ -92,8 +93,7 @@ fn main() -> Result<(), vk_graph_window::WindowError> { - Enter the PID - In the call stack pane, select the first thread; pause it - You are now parked on a syscall - - Walk up about 12 stack frames by scrolling down and selecting: - `{closure#0} debugger.rs 110:21` + - Walk up about 12 stack frames by scrolling down and selecting this callback. - It's 🕓 to de-🐛! */ @@ -102,10 +102,10 @@ fn main() -> Result<(), vk_graph_window::WindowError> { This will cause a validation error now (because this image is created here). You have followed the above directions and now have an active debug session looking at - line 115. You try to step forward in vain. Comment out the second `image` binding to - continue. + the failing image creation. You try to step forward in vain. Comment out the second + `image` binding to continue. - It is left as an excerise to the reader to determine *what* might have gone wrong here. + It is left as an exercise to the reader to determine *what* might have gone wrong here. */ #[allow(unused_variables)] let image = frame.graph.bind_resource( @@ -164,12 +164,12 @@ fn main() -> Result<(), vk_graph_window::WindowError> { thread 'main' panicked at 'uninitialized swapchain image ...' This will cause a static assertion after this closure completes, but before it is - called again. It happens in display.rs. + called again. It happens while the window frame is finalized. Because this error does not cause validation layer messages, it does not hit the debug - "breakpoint" we setup on line 39. You have two choices: - - Read how pass_ref.rs lays out data which resolver.rs submits; debug it (you die) - - Goto line 180 and fix the bug + "breakpoint" set up by validation. You have two choices: + - Read how the graph records and submits work; debug it the long way + - Fix the swapchain image access below This is a valid thing to panic over because we expect that all frames will render something to the swapchain image. The operating system is nicely asking that we repaint @@ -180,7 +180,7 @@ fn main() -> Result<(), vk_graph_window::WindowError> { compute, copy, render, store, transfer or any number of other things and that will signal it's OK to proceed without panicking. - Here is a fixed line 180: + Here is the fixed access declaration: .shader_resource_access(42, frame.swapchain_image, AccessType::ComputeShaderWrite) */ frame @@ -193,7 +193,7 @@ fn main() -> Result<(), vk_graph_window::WindowError> { cmd.dispatch(1024, 1024, 1); }); - // Growing tired of your advenutes, you signal that it is time to close the window and exit + // Growing tired of your adventures, you signal that it is time to close the window and exit *frame.will_exit = true; })?; diff --git a/examples/font_bmp.rs b/examples/font_bmp.rs index 12add40c..e98a0134 100644 --- a/examples/font_bmp.rs +++ b/examples/font_bmp.rs @@ -56,7 +56,7 @@ fn main() -> anyhow::Result<()> { )?; // A neato smoke effect just for fun - let Vulkan11Properties { subgroup_size, .. } = window.device.physical_device.properties_v1_1; + let Vulkan11Properties { subgroup_size, .. } = window.device.physical.properties_v1_1; let start_time = Instant::now(); let smoke_pipeline = ComputePipeline::create( &window.device, diff --git a/examples/fuzzer.rs b/examples/fuzzer.rs index a5c3e31d..32815077 100644 --- a/examples/fuzzer.rs +++ b/examples/fuzzer.rs @@ -38,7 +38,7 @@ use { buffer::{Buffer, BufferInfo}, compute::{ComputePipeline, ComputePipelineInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo, StencilMode}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo, StencilMode}, image::{ImageInfo, SampleCount}, physical_device::Vulkan10Limits, render_pass::ResolveMode, @@ -200,10 +200,11 @@ fn record_accel_struct_builds(frame: &mut FrameContext, pool: &mut HashPool) { let accel_struct_scratch_offset_alignment = frame .device - .physical_device - .accel_struct_properties + .physical + .vk_khr_acceleration_structure .as_ref() .unwrap() + .properties .min_accel_struct_scratch_offset_alignment as vk::DeviceSize; @@ -600,7 +601,7 @@ fn record_graphic_bindless(frame: &mut FrameContext, pool: &mut HashPool) { .clear_color_image(images[3], [0f32; 4]) .clear_color_image(images[4], [0f32; 4]) .begin_cmd() - .debug_name("graphic-bindless") + .debug_name("graphics-bindless") .bind_pipeline(&pipeline) .shader_resource_access( (0, [0]), @@ -680,7 +681,7 @@ fn record_graphic_msaa_depth_stencil(frame: &mut FrameContext, pool: &mut HashPo framebuffer_depth_sample_counts, framebuffer_stencil_sample_counts, .. - } = frame.device.physical_device.properties_v1_0.limits; + } = frame.device.physical.properties_v1_0.limits; match framebuffer_color_sample_counts & framebuffer_depth_sample_counts & framebuffer_stencil_sample_counts @@ -702,7 +703,7 @@ fn record_graphic_msaa_depth_stencil(frame: &mut FrameContext, pool: &mut HashPo vk::Format::D16_UNORM_S8_UINT, vk::Format::D32_SFLOAT_S8_UINT, ] { - let format_props = frame.device.physical_device.image_format_properties( + let format_props = frame.device.physical.image_format_properties( format, vk::ImageType::TYPE_2D, vk::ImageTiling::OPTIMAL, @@ -727,7 +728,7 @@ fn record_graphic_msaa_depth_stencil(frame: &mut FrameContext, pool: &mut HashPo ] { if frame .device - .physical_device + .physical .depth_stencil_resolve_properties .supported_depth_resolve_modes .contains(resolve_flags) @@ -775,7 +776,7 @@ fn record_graphic_msaa_depth_stencil(frame: &mut FrameContext, pool: &mut HashPo .as_slice(), ); - let swapchain_format = frame.graph.resource(frame.swapchain_image).info.fmt; + let swapchain_format = frame.graph.resource(frame.swapchain_image).info.format; let msaa_color_image = frame.graph.bind_resource( pool.resource( ImageInfo::image_2d( @@ -1523,8 +1524,8 @@ fn record_transfer_graphic_multipass(frame: &mut FrameContext, pool: &mut HashPo frame.graph.clear_color_image(images[0], [0f32; 4]); frame.graph.clear_color_image(images[1], [0f32; 4]); - // a and b should merge into one renderpass with two subpasses; however the use of images[1] in - // b should have a pipeline barrier (on the clear we just did) before the pass starts. + // a and b should merge into one render pass with two subpasses; however the use of images[1] in + // b should have a pipeline barrier (on the clear we just did) before the pass starts frame .graph .begin_cmd() diff --git a/examples/image_sampler.rs b/examples/image_sampler.rs index 2cc60784..b11f8a6e 100644 --- a/examples/image_sampler.rs +++ b/examples/image_sampler.rs @@ -16,7 +16,7 @@ use { driver::{ buffer::Buffer, device::Device, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, image::{Image, ImageInfo}, shader::{SamplerInfo, Shader}, }, @@ -31,7 +31,7 @@ use { /// /// Note that manually specifying image samplers is completely optional, valid defaults will be used /// if they are not specified when creating the shader which uses them. Additionally, you could -/// instead use use name suffixes such as _llr or _nne for linear/linear repeat or nearest/nearest +/// instead use name suffixes such as _llr or _nne for linear/linear repeat or nearest/nearest /// clamp-to-edge. /// /// You may run this example program with either --hlsl or --separate arguments as follows: @@ -152,12 +152,14 @@ fn create_pipeline( ) } (true, false) => { - // HLSL combined image sampler: include_glsl uses shaderc which does not support this, - // so we are using hassle_rs which uses dxc. You must follow the - // instructions listed here to use hassle_rs: - // See: https://github.com/Traverse-Research/hassle-rs - // See: https://github.com/microsoft/DirectXShaderCompiler/wiki/Vulkan-combined-image-sampler-type - // See: https://github.com/google/shaderc/issues/1310 + /* + HLSL combined image sampler: include_glsl uses shaderc which does not support this, so + we are using hassle_rs which uses dxc. You must follow the instructions listed here to + use hassle_rs: + See: https://github.com/Traverse-Research/hassle-rs + See: https://github.com/microsoft/DirectXShaderCompiler/wiki/Vulkan-combined-image-sampler-type + See: https://github.com/google/shaderc/issues/1310 + */ Shader::new_fragment( compile_hlsl( "fragment.hlsl", @@ -231,7 +233,7 @@ fn create_pipeline( }; // Use the builder pattern to specify an image sampler at the combined binding index (0) or - // separate binding index (1). + // separate binding index (1) let sampler_binding = args.separate as u32; frag_shader = frag_shader.image_sampler(sampler_binding, sampler_info); @@ -289,7 +291,7 @@ fn read_image(device: &Device, path: impl AsRef) -> anyhow::Result Result<(), DriverError> { pretty_env_logger::init(); @@ -64,10 +66,10 @@ fn main() -> Result<(), DriverError> { let min_result_buf = copy_image_to_buffer(&device, &mut graph, min_reduced_image)?; let max_result_buf = copy_image_to_buffer(&device, &mut graph, max_reduced_image)?; - graph - .into_submission() - .queue_submit(&mut HashPool::new(&device), 0, 0)? - .wait_until_executed()?; + let mut fence = graph + .finalize() + .queue_submit(&mut HashPool::new(&device), 0, 0)?; + fence.wait_signaled()?; // For each image we have reduced each 2x2 pixel group into the min/max values of each group let min_result_data: &[f32] = cast_slice(Buffer::mapped_slice(&min_result_buf)); @@ -111,8 +113,8 @@ fn fill_depth_image( vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST, ); let ImageInfo { - fmt, - ty, + format, + image_type, tiling, usage, flags, @@ -121,13 +123,13 @@ fn fill_depth_image( // Sometimes required because support is not 100% common: Check min/max reduction support // https://vulkan.gpuinfo.org/listdevicescoverage.php?extension=VK_EXT_sampler_filter_minmax&platform=all - let fmt_props = device.physical_device.format_properties(fmt); + let fmt_props = device.physical.format_properties(format); if !fmt_props.optimal_tiling_features.contains( vk::FormatFeatureFlags::SAMPLED_IMAGE | vk::FormatFeatureFlags::SAMPLED_IMAGE_FILTER_LINEAR | vk::FormatFeatureFlags::SAMPLED_IMAGE_FILTER_MINMAX, ) { - // In this case you might just fall back to a compute shader algorthm + // In this case you might just fall back to a compute shader algorithm warn!("Requested image does not support min/max reduction"); return Err(DriverError::Unsupported); @@ -137,15 +139,15 @@ fn fill_depth_image( // we already performed above, it's just a different way to go about finding the answer) assert!( device - .physical_device + .physical .sampler_filter_minmax_properties .single_component_formats ); // Not required, but good practice: Check image format support let image_fmt_props = device - .physical_device - .image_format_properties(fmt, ty, tiling, usage, flags)? + .physical + .image_format_properties(format, image_type, tiling, usage, flags)? .ok_or(DriverError::Unsupported)?; if size > image_fmt_props.max_extent.width || size > image_fmt_props.max_extent.height { // In this case you might use a smaller image @@ -155,7 +157,7 @@ fn fill_depth_image( } // You could check this if you needed to reduce multiple channel images: - // device.physical_device.sampler_filter_minmax_properties.image_component_mapping + // device.physical.sampler_filter_minmax_properties.image_component_mapping let depth_data = (0..size.pow(2)).map(|x| x as f32).collect::>(); let depth_data = graph.bind_resource(Buffer::create_from_slice( diff --git a/examples/mip_compute.rs b/examples/mip_compute.rs index 8894f066..5d253f85 100644 --- a/examples/mip_compute.rs +++ b/examples/mip_compute.rs @@ -20,10 +20,10 @@ use { vk_sync::AccessType, }; -/// This program demonstrates a single render pass which uses multiple executions to record a chain -/// of image copies which reduce an input image from 4x4 into 2x2 and finally 1x1. This is useful -/// for GPU-based mesh instance culling, where the depth buffer is summarized into a mip chain that -/// can be queried to find the maximum depth for a given render area. +/// This program demonstrates a single compute command which uses multiple executions to record a +/// chain of image copies which reduce an input image from 4x4 into 2x2 and finally 1x1. This is +/// useful for GPU-based mesh instance culling, where the depth buffer is summarized into a mip chain +/// that can be queried to find the maximum depth for a given render area. /// /// This technique is also known as a depth pyramid. fn main() -> Result<(), DriverError> { @@ -160,10 +160,10 @@ fn main() -> Result<(), DriverError> { let depth_pixel = graph.resource(depth_pixel).clone(); - graph - .into_submission() - .queue_submit(&mut HashPool::new(&device), 0, 0)? - .wait_until_executed()?; + let mut fence = graph + .finalize() + .queue_submit(&mut HashPool::new(&device), 0, 0)?; + fence.wait_signaled()?; let depth_pixel = f32::from_ne_bytes(Buffer::mapped_slice(&depth_pixel).try_into().unwrap()); diff --git a/examples/mip_graphic.rs b/examples/mip_graphics.rs similarity index 95% rename from examples/mip_graphic.rs rename to examples/mip_graphics.rs index 9d970d04..5d63b08a 100644 --- a/examples/mip_graphic.rs +++ b/examples/mip_graphics.rs @@ -13,7 +13,7 @@ use { driver::{ DriverError, device::Device, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, image::{Image, ImageInfo}, shader::{SamplerInfoBuilder, Shader}, }, @@ -178,9 +178,11 @@ fn fill_mip_levels(device: &Device, image: &Arc) -> Result<(), DriverErro let image_info = image.info; let image = graph.bind_resource(image); - // NOTE: Each pass writes to a different mip level, and so although it's the same image they are - // unable to be used as a single pass so we must call begin_pass for each. Without starting a - // new pass for each level the Vulkan framebuffer would be set to the size of the first image. + /* + NOTE: Each command writes to a different mip level, and so although it's the same image they are + unable to be used as a single render pass. Without starting a new command for each level the + Vulkan framebuffer would be set to the size of the first image. + */ for mip_level in 0..image_info.mip_level_count { graph .begin_cmd() @@ -211,7 +213,7 @@ fn fill_mip_levels(device: &Device, image: &Arc) -> Result<(), DriverErro // This is the overly-complicated way of picking queue family 0 let queue_family_index = device - .physical_device + .physical .queue_families .iter() .enumerate() @@ -225,7 +227,7 @@ fn fill_mip_levels(device: &Device, image: &Arc) -> Result<(), DriverErro // Submits to the GPU but does not wait for anything to be finished graph - .into_submission() + .finalize() .queue_submit(&mut LazyPool::new(device), queue_family_index, 0) .map(|_| ()) } diff --git a/examples/msaa.rs b/examples/msaa.rs index a307d986..a86be4cb 100644 --- a/examples/msaa.rs +++ b/examples/msaa.rs @@ -13,7 +13,7 @@ use { DriverError, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::{ImageInfo, SampleCount}, physical_device::Vulkan10Limits, }, @@ -109,7 +109,7 @@ fn main() -> anyhow::Result<()> { } else { &mesh_noaa_pipeline }) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .resource_access(cube_vertex_buf, AccessType::VertexBuffer) .shader_resource_access(0, scene_uniform_buf, AccessType::AnyShaderReadUniformBuffer); @@ -119,7 +119,7 @@ fn main() -> anyhow::Result<()> { ImageInfo::image_2d( frame.width, frame.height, - cmd.resource(frame.swapchain_image).info.fmt, + cmd.resource(frame.swapchain_image).info.format, vk::ImageUsageFlags::COLOR_ATTACHMENT | vk::ImageUsageFlags::TRANSIENT_ATTACHMENT, ) @@ -194,7 +194,7 @@ fn main() -> anyhow::Result<()> { fn best_depth_format(device: &Device) -> vk::Format { for format in [vk::Format::D32_SFLOAT, vk::Format::D16_UNORM] { - let format_props = device.physical_device.image_format_properties( + let format_props = device.physical.image_format_properties( format, vk::ImageType::TYPE_2D, vk::ImageTiling::OPTIMAL, @@ -216,7 +216,7 @@ fn max_supported_sample_count(device: &Device) -> SampleCount { framebuffer_color_sample_counts, framebuffer_depth_sample_counts, .. - } = device.physical_device.properties_v1_0.limits; + } = device.physical.properties_v1_0.limits; match framebuffer_color_sample_counts & framebuffer_depth_sample_counts { s if s.contains(vk::SampleCountFlags::TYPE_64) => SampleCount::Type64, s if s.contains(vk::SampleCountFlags::TYPE_32) => SampleCount::Type32, diff --git a/examples/multipass.rs b/examples/multipass.rs index 74e0d6dd..14f66de0 100644 --- a/examples/multipass.rs +++ b/examples/multipass.rs @@ -13,7 +13,7 @@ use { DriverError, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::ImageInfo, shader::Shader, }, @@ -105,7 +105,7 @@ fn main() -> anyhow::Result<()> { let light_buf = bind_light_buf(frame.graph, &mut pool); let push_const_data = write_push_consts(obj_pos, material); - let mut write = DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL; + let mut write = DepthStencilInfo::DEPTH_WRITE_LESS; // Depth Prepass frame @@ -202,7 +202,7 @@ fn best_depth_stencil_format(device: &Device) -> vk::Format { vk::Format::D16_UNORM_S8_UINT, vk::Format::D32_SFLOAT_S8_UINT, ] { - let format_props = device.physical_device.image_format_properties( + let format_props = device.physical.image_format_properties( format, vk::ImageType::TYPE_2D, vk::ImageTiling::OPTIMAL, @@ -326,7 +326,7 @@ fn create_funky_shape(device: &Device, pool: &mut LazyPool) -> Result anyhow::Result<()> { let secondary_queue_family = window .device - .physical_device + .physical .queue_families .iter() .enumerate() @@ -108,17 +108,30 @@ fn main() -> anyhow::Result<()> { // Clear a new image to a cycling color let mut graph = Graph::default(); - let image = graph.bind_resource( - pool.resource(ImageInfo::image_2d( + let image = match pool.resource( + ImageInfo::image_2d( 10, 10, vk::Format::R8G8B8A8_UNORM, vk::ImageUsageFlags::TRANSFER_DST | vk::ImageUsageFlags::TRANSFER_SRC, - )) - .unwrap(), - ); + ) + .into_builder() + .sharing_mode(if args.concurrent { + vk::SharingMode::CONCURRENT + } else { + vk::SharingMode::EXCLUSIVE + }), + ) { + Ok(image) => Arc::new(image), + Err(err) => { + warn!("unable to allocate worker image: {err}"); + + break; + } + }; + let image_node = graph.bind_resource(&image); graph.clear_color_image( - image, + image_node, [ (t.sin() * 127.0 + 128.0) as u8, ((t + 2.0).sin() * 127.0 + 128.0) as u8, @@ -127,16 +140,31 @@ fn main() -> anyhow::Result<()> { ], ); - let image = graph.resource(image).clone(); - // Submit on a queue we are reserving for only this thread to use - graph - .into_submission() - .queue_submit(&mut pool, secondary_queue_family_index, queue_index) - .unwrap(); - - // After submit() is called we can safely use this image on another thread! - tx.send(image).unwrap(); + let mut fence = match graph.finalize().queue_submit( + &mut pool, + secondary_queue_family_index, + queue_index, + ) { + Ok(fence) => fence, + Err(err) => { + warn!("unable to submit worker graph: {err}"); + + break; + } + }; + + if let Err(err) = fence.wait_signaled() { + warn!("unable to wait for worker fence: {err}"); + + break; + } + + // After the fence signals, GPU work using this image has completed and the image + // can be used on another thread + if tx.send(image).is_err() { + break; + } } })); } @@ -145,7 +173,7 @@ fn main() -> anyhow::Result<()> { let mut images = VecDeque::new(); let mut previous_frame = Instant::now(); - window.run(|frame| { + let result = window.run(|frame| { let current_frame = Instant::now(); let elapsed = current_frame - previous_frame; previous_frame = current_frame; @@ -209,7 +237,7 @@ fn main() -> anyhow::Result<()> { message, 4.0, ); - })?; + }); info!("Stopping threads"); @@ -218,6 +246,8 @@ fn main() -> anyhow::Result<()> { thread.join().unwrap(); } + result?; + Ok(()) } @@ -264,7 +294,7 @@ fn load_font(device: &Device) -> anyhow::Result { // This copy happens in queue index 0! graph - .into_submission() + .finalize() .queue_submit(&mut HashPool::new(device), 0, 0)?; BitmapFont::new(device, font, [page_0]) @@ -275,4 +305,9 @@ struct Args { /// Enable Vulkan SDK validation layers #[arg(long)] debug: bool, + + /// Use concurrent sharing mode (Vulkan implementation automatically transfers ownership) + /// instead of the default exclusive (vk-graph automatically tracks and transfers ownership) + #[arg(long)] + concurrent: bool, } diff --git a/examples/profile_with_puffin/mod.rs b/examples/profile_with_puffin/mod.rs index 0ad58989..6d641512 100644 --- a/examples/profile_with_puffin/mod.rs +++ b/examples/profile_with_puffin/mod.rs @@ -8,7 +8,7 @@ //! ``` //! //! For more information see: -//! https://github.com/attackgoat/vk-graph/blob/master/examples/getting-started.md +//! https://github.com/attackgoat/vk-graph/blob/main/examples/README.md #[cfg(feature = "profile-with-puffin")] use { diff --git a/examples/ray_omni.rs b/examples/ray_omni.rs index 3abade24..2138481b 100644 --- a/examples/ray_omni.rs +++ b/examples/ray_omni.rs @@ -28,7 +28,7 @@ use { }, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::ImageInfo, shader::Shader, }, @@ -122,12 +122,8 @@ fn main() -> anyhow::Result<()> { .resource_access(model_mesh_index_buf, AccessType::IndexBuffer) .resource_access(model_mesh_vertex_buf, AccessType::VertexBuffer) .shader_resource_access(0, camera_buf, AccessType::AnyShaderReadUniformBuffer) - .shader_resource_access( - 1, - scene_tlas, - AccessType::RayTracingShaderReadAccelerationStructure, - ) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .shader_resource_access(1, scene_tlas, AccessType::FragmentShaderReadOther) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .depth_stencil_attachment_image( depth_image, LoadOp::CLEAR_ONE_STENCIL_ZERO, @@ -160,7 +156,7 @@ fn best_2d_optimal_format( flags: vk::ImageCreateFlags, ) -> vk::Format { for format in formats { - let format_props = device.physical_device.image_format_properties( + let format_props = device.physical.image_format_properties( *format, vk::ImageType::TYPE_2D, vk::ImageTiling::OPTIMAL, @@ -214,10 +210,11 @@ fn create_blas( )?); let accel_struct_scratch_offset_alignment = device - .physical_device - .accel_struct_properties + .physical + .vk_khr_acceleration_structure .as_ref() .unwrap() + .properties .min_accel_struct_scratch_offset_alignment as vk::DeviceSize; let scratch_buf = graph.bind_resource(Buffer::create( @@ -254,7 +251,7 @@ fn create_blas( let blas = graph.resource(blas).clone(); graph - .into_submission() + .finalize() .queue_submit(&mut LazyPool::new(device), 0, 0)?; Ok(blas) @@ -335,7 +332,7 @@ fn create_pipeline(device: &Device) -> Result { 1000.0 ); - // Traverse the acceleration structure and store the first intersection, if any. + // Traverse the acceleration structure and store the first intersection, if any rayQueryProceedEXT(rayQuery); // If the intersection has hit a triangle, the fragment is shadowed @@ -411,10 +408,11 @@ fn create_tlas( graph.bind_resource(pool.resource(AccelerationStructureInfo::tlas(size.create_size))?); let accel_struct_scratch_offset_alignment = device - .physical_device - .accel_struct_properties + .physical + .vk_khr_acceleration_structure .as_ref() .unwrap() + .properties .min_accel_struct_scratch_offset_alignment as vk::DeviceSize; let scratch_buf = graph.bind_resource( diff --git a/examples/ray_trace.rs b/examples/ray_tracing.rs similarity index 98% rename from examples/ray_trace.rs rename to examples/ray_tracing.rs index 99c7739d..33e95ab9 100644 --- a/examples/ray_trace.rs +++ b/examples/ray_tracing.rs @@ -20,8 +20,8 @@ use { buffer::{Buffer, BufferInfo}, device::Device, image::ImageInfo, - physical_device::RayTracingPipelineProperties, - ray_trace::{RayTracingPipeline, RayTracingPipelineInfo, RayTracingShaderGroup}, + physical_device::khr::RayTracingPipelineProperties, + ray_tracing::{RayTracingPipeline, RayTracingPipelineInfo, RayTracingShaderGroup}, shader::Shader, }, pool::{Pool as _, hash::HashPool}, @@ -515,17 +515,18 @@ fn main() -> anyhow::Result<()> { // Setup the ray tracing pipeline // ------------------------------------------------------------------------------------------ // - let &RayTracingPipelineProperties { + let RayTracingPipelineProperties { shader_group_base_alignment, shader_group_handle_alignment, shader_group_handle_size, .. } = window .device - .physical_device - .ray_tracing_pipeline_properties + .physical + .vk_khr_ray_tracing_pipeline .as_ref() - .unwrap(); + .unwrap() + .properties; let ray_tracing_pipeline = create_ray_tracing_pipeline(&window.device)?; // ------------------------------------------------------------------------------------------ // @@ -597,7 +598,7 @@ fn main() -> anyhow::Result<()> { load_scene_buffers(&window.device)?; // ------------------------------------------------------------------------------------------ // - // Create the bottom level acceleration structure + // Create the bottom-level acceleration structure // ------------------------------------------------------------------------------------------ // let blas_geometry_info = AccelerationStructureGeometryInfo::blas([( @@ -659,7 +660,7 @@ fn main() -> anyhow::Result<()> { }); // ------------------------------------------------------------------------------------------ // - // Create the top level acceleration structure + // Create the top-level acceleration structure // ------------------------------------------------------------------------------------------ // let tlas_geometry_info = AccelerationStructureGeometryInfo::tlas([( @@ -682,10 +683,11 @@ fn main() -> anyhow::Result<()> { { let accel_struct_scratch_offset_alignment = window .device - .physical_device - .accel_struct_properties + .physical + .vk_khr_acceleration_structure .as_ref() .unwrap() + .properties .min_accel_struct_scratch_offset_alignment as vk::DeviceSize; let mut graph = Graph::default(); @@ -753,7 +755,7 @@ fn main() -> anyhow::Result<()> { }); } - graph.into_submission().queue_submit(&mut cache, 0, 0)?; + graph.finalize().queue_submit(&mut cache, 0, 0)?; } // ------------------------------------------------------------------------------------------ // @@ -781,7 +783,7 @@ fn main() -> anyhow::Result<()> { .resource(ImageInfo::image_2d( frame.width, frame.height, - frame.graph.resource(frame.swapchain_image).info.fmt, + frame.graph.resource(frame.swapchain_image).info.format, vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::TRANSFER_DST | vk::ImageUsageFlags::TRANSFER_SRC, diff --git a/examples/rt_triangle.rs b/examples/rt_triangle.rs index 4f6664b2..fb61ec44 100644 --- a/examples/rt_triangle.rs +++ b/examples/rt_triangle.rs @@ -17,8 +17,8 @@ use { }, buffer::{Buffer, BufferInfo}, device::Device, - physical_device::RayTracingPipelineProperties, - ray_trace::{RayTracingPipeline, RayTracingPipelineInfo, RayTracingShaderGroup}, + physical_device::khr::RayTracingPipelineProperties, + ray_tracing::{RayTracingPipeline, RayTracingPipelineInfo, RayTracingShaderGroup}, shader::Shader, }, pool::hash::HashPool, @@ -137,16 +137,17 @@ fn main() -> anyhow::Result<()> { // Setup the ray tracing pipeline // ------------------------------------------------------------------------------------------ // - let &RayTracingPipelineProperties { + let RayTracingPipelineProperties { shader_group_base_alignment, shader_group_handle_size, .. } = window .device - .physical_device - .ray_tracing_pipeline_properties + .physical + .vk_khr_ray_tracing_pipeline .as_ref() - .unwrap(); + .unwrap() + .properties; let ray_tracing_pipeline = create_ray_tracing_pipeline(&window.device)?; // ------------------------------------------------------------------------------------------ // @@ -262,7 +263,7 @@ fn main() -> anyhow::Result<()> { }; // ------------------------------------------------------------------------------------------ // - // Create the bottom level acceleration structure + // Create the bottom-level acceleration structure // ------------------------------------------------------------------------------------------ // let blas_geometry_info = AccelerationStructureGeometryInfo::blas([( @@ -324,7 +325,7 @@ fn main() -> anyhow::Result<()> { }); // ------------------------------------------------------------------------------------------ // - // Create the top level acceleration structure + // Create the top-level acceleration structure // ------------------------------------------------------------------------------------------ // let tlas_geometry_info = AccelerationStructureGeometryInfo::tlas([( @@ -347,10 +348,11 @@ fn main() -> anyhow::Result<()> { { let accel_struct_scratch_offset_alignment = window .device - .physical_device - .accel_struct_properties + .physical + .vk_khr_acceleration_structure .as_ref() .unwrap() + .properties .min_accel_struct_scratch_offset_alignment as vk::DeviceSize; let mut graph = Graph::default(); @@ -418,7 +420,7 @@ fn main() -> anyhow::Result<()> { }); } - graph.into_submission().queue_submit(&mut pool, 0, 0)?; + graph.finalize().queue_submit(&mut pool, 0, 0)?; } // ------------------------------------------------------------------------------------------ // diff --git a/examples/shader-toy/Cargo.toml b/examples/shader-toy/Cargo.toml index 0e6a7941..583110b9 100644 --- a/examples/shader-toy/Cargo.toml +++ b/examples/shader-toy/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "shader-toy" authors = ["John Wells "] -description = "Example api usage" +description = "Example API usage" edition.workspace = true license.workspace = true publish = false diff --git a/examples/shader-toy/README.md b/examples/shader-toy/README.md index c6812088..5b47aae6 100644 --- a/examples/shader-toy/README.md +++ b/examples/shader-toy/README.md @@ -1,33 +1,31 @@ -Preview +Preview -# Shader Toy Example +# Shadertoy Example This example uses computational fluid dynamics to create an effect like spilled paint. The original shader code comes from [Florian Berger](https://www.shadertoy.com/view/MsGSRd) and is attached to a permissive [CC BY-NC-SA 3.0](https://creativecommons.org/licenses/by-nc-sa/3.0/) license. -The implementation is presented as close as possible to the original usage on Shader Toy, but it -would not be recommended to use it directly - you probably want to use compute pipelines for things -like this. Also there are numerous unused descriptor bindings and push constant ranges which could -be removed; but those are standard things all Shader Toys require. +The implementation stays close to the original Shadertoy usage. For production code, a compute +pipeline would usually be a better fit for this kind of effect. The example also keeps the unused +descriptor bindings and push constant ranges that standard Shadertoy-style pipelines expect. ## Details -See the `build.rs` script: it packs the hefty (they are actually not hefty) images into a `.pak` -file. This makes it easier for the example to be run in other places, but of course all this is -overkill for this actual example. It also pre-compiles the shader code from GLSL to SPIR-V. +See the `build.rs` script: it packs the example images into a `.pak` file. This makes the example +easier to run from different working directories. It also pre-compiles the shader code from GLSL to +SPIR-V. ### Adding/Changing files -The `pak.toml` file references the images used in this example using a glob see line 6 in that file. +The `pak.toml` file references the images used in this example using a glob; see line 6 in that file. -If we add a reference directly to that file the build script will pick up and pack the new file. If -we let the glob continue to reference files you might want to ask the build script to look again, -like so: +If you add a file reference directly to `pak.toml`, the build script will pick up and pack the new +file. If the glob should pick up new files instead, ask the build script to look again: ```bash touch res/pak.toml ``` Now try building again and the newly added files should be packed and have bindings generated in the -Rust code. If any of those files change, the build script will automatically re-pack things. \ No newline at end of file +Rust code. If any of those files change, the build script will automatically re-pack things. diff --git a/examples/shader-toy/build.rs b/examples/shader-toy/build.rs index 80a1c620..5fe3b051 100644 --- a/examples/shader-toy/build.rs +++ b/examples/shader-toy/build.rs @@ -116,7 +116,7 @@ fn create_pak_bindings(src: impl AsRef, dst: impl AsRef) -> anyhow:: bindings.push_str("\"#;\n"); } - write(dst, bindings).context("Unable to bindings text")?; + write(dst, bindings).context("Unable to write bindings text")?; Ok(()) } @@ -144,7 +144,7 @@ fn create_shader_bindings( bindings.push_str("\"));\n"); } - write(dst, bindings).context("Unable to bindings text")?; + write(dst, bindings).context("Unable to write bindings text")?; Ok(()) } diff --git a/examples/shader-toy/src/main.rs b/examples/shader-toy/src/main.rs index 4682952d..3e5072d5 100644 --- a/examples/shader-toy/src/main.rs +++ b/examples/shader-toy/src/main.rs @@ -1,9 +1,9 @@ pub mod data { use super::PakBuf; - // The ".pak" file is a data transport type with compression and other useful features - // It is used to hold the images used by this example, because they *could* be really - // big - anyways we generated some bindings to make accessing those less error-prone: + // The ".pak" file is a data transport type with compression and other useful features. + // It is used to hold the images used by this example because they *could* be really + // big, so we generate some bindings to make accessing those less error-prone: include!(concat!(env!("OUT_DIR"), "/pak_bindings.rs")); // This happens if you want the .pak bytes inside the executable itself @@ -43,7 +43,7 @@ use { cmd::{LoadOp, StoreOp}, driver::{ ash::vk, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, image::ImageInfo, shader::Shader, sync::AccessType, @@ -101,7 +101,7 @@ fn main() -> anyhow::Result<()> { .context("Unable to decode noise bitmap")? }); - // The shader toy example used two graphics pipelines with defaults: + // The Shadertoy example used two graphics pipelines with defaults: // no depth/stencil // 1x sample count // one-sided @@ -173,7 +173,7 @@ fn main() -> anyhow::Result<()> { let mut blank_image_binding = Some(graph.resource(blank_image).clone()); let mut temp_image_binding = Some(graph.resource(temp_image).clone()); - graph.into_submission().queue_submit(&mut cache, 0, 0)?; + graph.finalize().queue_submit(&mut cache, 0, 0)?; let started_at = Instant::now(); let mut prev_frame_at = started_at; @@ -187,7 +187,7 @@ fn main() -> anyhow::Result<()> { .run(|frame| { let now = Instant::now(); - // Update the stuff any shader toy shader would want to know each frame + // Update the stuff any Shadertoy shader would want to know each frame let dt = now - prev_frame_at; prev_frame_at = now; @@ -212,7 +212,7 @@ fn main() -> anyhow::Result<()> { .graph .bind_resource(temp_image_binding.take().unwrap()); - // We need to push a shader-toy defined set of constants to each pipeline - any copy + // We need to push a Shadertoy-defined set of constants to each pipeline - any copy // type will do but we are getting fancy here by defining a struct to be super precise // about what we're doing - but you may want to just send a bunch of f32's #[repr(C)] diff --git a/examples/skeletal-anim/README.md b/examples/skeletal-anim/README.md index f6f1be12..d989d285 100644 --- a/examples/skeletal-anim/README.md +++ b/examples/skeletal-anim/README.md @@ -1,4 +1,4 @@ -Preview +Preview # Animation Example diff --git a/examples/skeletal-anim/src/main.rs b/examples/skeletal-anim/src/main.rs index 2fb1ea4e..a599ca2a 100644 --- a/examples/skeletal-anim/src/main.rs +++ b/examples/skeletal-anim/src/main.rs @@ -24,7 +24,7 @@ use { ash::vk, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfoBuilder}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfoBuilder}, image::{Image, ImageInfo}, shader::Shader, sync::AccessType, @@ -137,7 +137,7 @@ fn main() -> Result<(), WindowError> { .begin_cmd() .debug_name("🦴") .bind_pipeline(&pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .resource_access(index_buf, AccessType::IndexBuffer) .resource_access(vertex_buf, AccessType::VertexBuffer) .shader_resource_access(0, camera_buf, AccessType::VertexShaderReadUniformBuffer) @@ -214,7 +214,7 @@ fn load_texture(device: &Device, pak: &mut PakBuf, key: &str) -> Result Result<(), DriverError> { subgroup_size, subgroup_supported_operations, .. - } = device.physical_device.properties_v1_1; + } = device.physical.properties_v1_1; assert!(subgroup_supported_operations.contains(vk::SubgroupFeatureFlags::ARITHMETIC)); assert!(subgroup_supported_operations.contains(vk::SubgroupFeatureFlags::BALLOT)); @@ -95,8 +95,7 @@ fn exclusive_sum( ), )?)); - let workgroup_count = - input_data.len() as u32 / device.physical_device.properties_v1_1.subgroup_size; + let workgroup_count = input_data.len() as u32 / device.physical.properties_v1_1.subgroup_size; let reduce_count = workgroup_count - 1; let workgroup_buf = graph.bind_resource(Buffer::create( device, @@ -130,12 +129,12 @@ fn exclusive_sum( }); let output_buf = graph.resource(output_buf).clone(); - let mut cmd = graph - .into_submission() + let mut fence = graph + .finalize() .queue_submit(&mut HashPool::new(device), 0, 0)?; let started = Instant::now(); - cmd.wait_until_executed()?; + fence.wait_signaled()?; println!( "Waited {}μs (len={})", @@ -200,14 +199,8 @@ fn create_reduce_pipeline(device: &Device) -> Result Result anyhow::Result<()> { let blend_image = transition_pipeline.apply(frame.graph, bart_image, gulf_image, transition, progress); - // Draw UI: TODO: Sliders and value setters? That would be fun. + // Draw UI: TODO: Sliders and value setters? That would be fun let gui_image = imgui.draw( dt.as_secs_f32(), frame.events, diff --git a/examples/triangle.rs b/examples/triangle.rs index 332f8e02..3eb3f8ed 100644 --- a/examples/triangle.rs +++ b/examples/triangle.rs @@ -9,7 +9,7 @@ use { cmd::{LoadOp, StoreOp}, driver::{ buffer::Buffer, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, }, }, vk_graph_window::{Window, WindowError}, @@ -17,7 +17,7 @@ use { vk_sync::AccessType, }; -// A Vulkan triangle using a graphics pipeline, vertex/fragment shaders, and index/vertex buffers. +// A Vulkan triangle using a graphics pipeline, vertex/fragment shaders, and index/vertex buffers fn main() -> Result<(), WindowError> { pretty_env_logger::init(); profile_with_puffin::init(); diff --git a/examples/vertex_layout.rs b/examples/vertex_layout.rs index fcf9b3b1..8a431f35 100644 --- a/examples/vertex_layout.rs +++ b/examples/vertex_layout.rs @@ -12,7 +12,7 @@ use { DriverError, buffer::Buffer, device::Device, - graphic::{GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, shader::{Shader, ShaderBuilder}, }, }, @@ -208,11 +208,13 @@ fn create_vertex_shader(is_double: bool) -> ShaderBuilder { // // To support a vec3 64-bit case this means color_in needs to be on location 2 - // This shader is compiled with a macro because we want to be able to switch the vec2 type to a - // dvec2 when using 64-bit positions; and for the purposes of this example we don't want to - // duplicate this shader code. You probably don't want to do this, or you may have different - // facilities for generating SPIR-V code - either way ignore the macro unless you're interested - // in the include_glsl! wizardry it contains which is unrelated to this example. + /* + This shader is compiled with a macro because we want to be able to switch the vec2 type to a + dvec2 when using 64-bit positions; and for the purposes of this example we don't want to + duplicate this shader code. You probably don't want to do this, or you may have different + facilities for generating SPIR-V code - either way ignore the macro unless you're interested in + the include_glsl! wizardry it contains which is unrelated to this example. + */ macro_rules! compile_vert { ($vec2_ty:literal) => { glsl!( @@ -276,7 +278,7 @@ fn create_pipeline( fn supports_vertex_buffer(device: &Device, format: vk::Format) -> bool { device - .physical_device + .physical .format_properties(format) .buffer_features .contains(vk::FormatFeatureFlags::VERTEX_BUFFER) diff --git a/examples/vr/README.md b/examples/vr/README.md index 415b3377..e6de9048 100644 --- a/examples/vr/README.md +++ b/examples/vr/README.md @@ -1,8 +1,8 @@ -Preview +Preview # Virtual Reality Example -_Exaimine the skeleton of a Woolly Mammoth using the hands of Abraham Lincoln_ +_Examine the skeleton of a Woolly Mammoth using the hands of Abraham Lincoln_ This example uses assets from the Smithsonian collection to create a real-life-size display in VR. Any OpenXR-supported VR headset should work. @@ -13,9 +13,9 @@ link against a system loader instead. - Asset loading - Basic VR handling/driver code -- Normal mapping using Mikktspace +- Normal mapping using MikkTSpace - Hot shader reloading ## Tested Platforms -- Windows/Valve Index on SteamVR @ 144hz +- Windows/Valve Index on SteamVR @ 144 Hz diff --git a/examples/vr/src/driver/instance.rs b/examples/vr/src/driver/instance.rs index de3aeb2a..8e960e25 100644 --- a/examples/vr/src/driver/instance.rs +++ b/examples/vr/src/driver/instance.rs @@ -235,8 +235,8 @@ impl XrInstance { this.instance.create_session::( this.system, &xr::vulkan::SessionCreateInfo { - instance: this.device.physical_device.instance.handle().as_raw() as _, - physical_device: this.device.physical_device.handle.as_raw() as _, + instance: this.device.physical.instance.handle().as_raw() as _, + physical_device: this.device.physical.handle.as_raw() as _, device: this.device.handle().as_raw() as _, queue_family_index, queue_index, diff --git a/examples/vr/src/driver/swapchain.rs b/examples/vr/src/driver/swapchain.rs index 58424fe8..e3c610d2 100644 --- a/examples/vr/src/driver/swapchain.rs +++ b/examples/vr/src/driver/swapchain.rs @@ -60,10 +60,12 @@ impl Swapchain { resolution.height, 2, vk::Format::R8G8B8A8_SRGB, - vk::ImageUsageFlags::SAMPLED, + vk::ImageUsageFlags::COLOR_ATTACHMENT | vk::ImageUsageFlags::SAMPLED, ); - Arc::new(Image::from_raw(device, image, info)) + // SAFETY: OpenXR returned this Vulkan image for the swapchain created from + // `device`, and OpenXR retains ownership of the image lifetime + Arc::new(unsafe { Image::from_raw(device, image, info) }) }) .collect(), resolution, diff --git a/examples/vr/src/main.rs b/examples/vr/src/main.rs index 9fc03f29..6f1c9e47 100644 --- a/examples/vr/src/main.rs +++ b/examples/vr/src/main.rs @@ -28,7 +28,7 @@ use { ash::vk::{self}, buffer::{Buffer, BufferInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipelineInfo}, + graphics::{DepthStencilInfo, GraphicsPipelineInfo}, image::{Image, ImageInfo}, sync::AccessType, }, @@ -214,9 +214,11 @@ fn main() -> anyhow::Result<()> { 'main_loop: loop { if !running.load(Ordering::Relaxed) { println!("requesting exit"); - // The OpenXR runtime may want to perform a smooth transition between scenes, so we - // can't necessarily exit instantly. Instead, we must notify the runtime of our - // intent and wait for it to tell us when we're actually done. + /* + The OpenXR runtime may want to perform a smooth transition between scenes, so we can't + necessarily exit instantly. Instead, we must notify the runtime of our intent and wait + for it to tell us when we're actually done. + */ match session.request_exit() { Ok(()) => {} Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break, @@ -362,7 +364,7 @@ fn main() -> anyhow::Result<()> { .begin_cmd() .debug_name("Left hand") .bind_pipeline(&hands_pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .multiview(VIEW_MASK, VIEW_MASK) .resource_access(index_buf, AccessType::IndexBuffer) .resource_access(vertex_buf, AccessType::VertexBuffer) @@ -410,7 +412,7 @@ fn main() -> anyhow::Result<()> { .begin_cmd() .debug_name("Right hand") .bind_pipeline(&hands_pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .multiview(VIEW_MASK, VIEW_MASK) .resource_access(index_buf, AccessType::IndexBuffer) .resource_access(vertex_buf, AccessType::VertexBuffer) @@ -472,7 +474,7 @@ fn main() -> anyhow::Result<()> { ) .multiview(VIEW_MASK, VIEW_MASK) .color_attachment_image(0, swapchain_image, LoadOp::DontCare, StoreOp::Store) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .depth_stencil_attachment_image( depth_image, LoadOp::CLEAR_ZERO_STENCIL_ZERO, @@ -488,10 +490,10 @@ fn main() -> anyhow::Result<()> { // Wait on the acquired swapchain image to be ready, submit rendering commands, and release // the image - afterwards we keep the submitted command buffer around (including all - // in-flight resources) so that nothing is dropped until that image is actually done. + // in-flight resources) so that nothing is dropped until that image is actually done swapchain.wait_image(xr::Duration::INFINITE).unwrap(); let swapchain_queue = graph - .into_submission() + .finalize() .queue_submit(&mut pool, queue_family_index as _, 0) .unwrap(); swapchain.release_image().unwrap(); @@ -564,7 +566,7 @@ fn arbitrary_perspective_rh( /// Helper to pick a queue family for submitting device commands. fn device_queue_family_index(device: &Device, flags: vk::QueueFlags) -> Option { device - .physical_device + .physical .queue_families .iter() .enumerate() @@ -791,10 +793,11 @@ fn load_texture( graph.copy_buffer_to_image(staging_buf, texture_image); let queue_family_index = device_queue_family_index(device, vk::QueueFlags::TRANSFER).unwrap(); - graph - .into_submission() - .queue_submit(&mut LazyPool::new(device), queue_family_index as _, 0)? - .wait_until_executed()?; + let mut fence = + graph + .finalize() + .queue_submit(&mut LazyPool::new(device), queue_family_index as _, 0)?; + fence.wait_signaled()?; Ok(texture) } diff --git a/examples/vsm_omni.rs b/examples/vsm_omni.rs index 082a8e0b..3c771eb2 100644 --- a/examples/vsm_omni.rs +++ b/examples/vsm_omni.rs @@ -22,7 +22,7 @@ use { buffer::{Buffer, BufferInfo}, compute::{ComputePipeline, ComputePipelineInfo}, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, image::ImageInfo, physical_device::Vulkan10Features, shader::{Shader, SpecializationMap}, @@ -73,7 +73,7 @@ fn main() -> anyhow::Result<()> { let use_geometry_shader = { let Vulkan10Features { geometry_shader, .. - } = window.device.physical_device.features_v1_0; + } = window.device.physical.features_v1_0; args.geometry_shader && geometry_shader }; @@ -262,7 +262,7 @@ fn main() -> anyhow::Result<()> { .begin_cmd() .debug_name("DEBUG") .bind_pipeline(&debug_pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .shader_resource_access( 0, camera_uniform_buf, @@ -306,7 +306,7 @@ fn main() -> anyhow::Result<()> { .begin_cmd() .debug_name("Shadow (Using geometry shader)") .bind_pipeline(&shadow_pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .shader_resource_access( 0, light_uniform_buf, @@ -362,7 +362,7 @@ fn main() -> anyhow::Result<()> { .begin_cmd() .debug_name("Shadow") .bind_pipeline(&shadow_pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .shader_resource_access( 0, light_uniform_buf, @@ -437,7 +437,7 @@ fn main() -> anyhow::Result<()> { .begin_cmd() .debug_name("Mesh objects") .bind_pipeline(&mesh_pipeline) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS_IGNORE_STENCIL) + .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .shader_resource_access( 0, camera_uniform_buf, @@ -453,7 +453,8 @@ fn main() -> anyhow::Result<()> { shadow_faces_node, shadow_faces_info .into_image_view() - .with_type(vk::ImageViewType::CUBE), + .into_builder() + .view_type(vk::ImageViewType::CUBE), AccessType::AnyShaderReadSampledImageOrUniformTexelBuffer, ) .resource_access(model_mesh_index_buf, AccessType::IndexBuffer) @@ -495,7 +496,7 @@ fn best_2d_optimal_format( flags: vk::ImageCreateFlags, ) -> vk::Format { for format in formats { - let format_props = device.physical_device.image_format_properties( + let format_props = device.physical.image_format_properties( *format, vk::ImageType::TYPE_2D, vk::ImageTiling::OPTIMAL, diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 00000000..bf9476ce --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "vk-graph-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +arbitrary = { version = "1.4", features = ["derive"] } +libfuzzer-sys = "0.4" +vk-graph = { path = ".." } + +[[bin]] +name = "submission_schedule" +path = "fuzz_targets/submission_schedule.rs" +test = false +doc = false +bench = false + +[workspace] +members = [] diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 00000000..4f13a641 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,55 @@ +# Fuzzing + +This directory contains standalone libFuzzer targets for `vk-graph`. + +Targets here should focus on small, high-value invariants in subsystems that benefit from broad, +generated input coverage. + +## Targets + +- `submission_schedule` + - Exercises `vk_graph::submission::fuzz::check_schedule_reordering(...)`. + - Generates synthetic pass/resource usage graphs. + - Verifies that `Schedule::reorder_passes(...)` matches the reference implementation. + - Also checks permutation and determinism invariants. + +Additional fuzz targets can be added under `fuzz_targets/` as other subsystems need coverage. + +## Layout + +- `Cargo.toml`: standalone fuzz crate manifest. +- `fuzz_targets/`: libFuzzer entrypoints and input generators. + +Fuzz-specific entrypoints live here, while invariant checkers may live in the main crate when that +lets fuzzing exercise real production logic without duplicating it in the harness. + +## Build + +Compile the target directly: + +```sh +cargo build --manifest-path fuzz/Cargo.toml --bin submission_schedule +``` + +Available fuzz targets are defined by filenames under `fuzz/fuzz_targets/`. + +## Run With cargo-fuzz + +If `cargo-fuzz` is installed, run: + +```sh +cargo fuzz run submission_schedule +``` + +Install it with: + +```sh +cargo install cargo-fuzz +``` + +## Notes + +- The fuzz crate is kept separate from the main crate to avoid pulling `libfuzzer-sys` into normal + library builds. +- Inputs should usually be intentionally bounded so fuzzing spends time exploring behavior instead + of growing huge synthetic structures. diff --git a/fuzz/fuzz_targets/submission_schedule.rs b/fuzz/fuzz_targets/submission_schedule.rs new file mode 100644 index 00000000..93ea6db0 --- /dev/null +++ b/fuzz/fuzz_targets/submission_schedule.rs @@ -0,0 +1,43 @@ +#![no_main] + +use { + arbitrary::{Arbitrary, Unstructured}, + libfuzzer_sys::fuzz_target, + vk_graph::submission::fuzz::{ResourceAccess, check_schedule_reordering}, +}; + +#[derive(Debug)] +struct FuzzCase { + pass_count: usize, + resource_accesses: Vec>, +} + +impl<'a> Arbitrary<'a> for FuzzCase { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + let pass_count = usize::from(u.int_in_range::(0..=64)?); + let resource_count = usize::from(u.int_in_range::(0..=48)?); + + let mut resource_accesses = Vec::with_capacity(resource_count); + for _ in 0..resource_count { + let use_count = usize::from(u.int_in_range::(0..=96)?); + let mut accesses = Vec::with_capacity(use_count); + for _ in 0..use_count { + accesses.push(ResourceAccess { + cmd_idx: usize::from(u.arbitrary::()?), + write: u.arbitrary()?, + }); + } + + resource_accesses.push(accesses); + } + + Ok(Self { + pass_count, + resource_accesses, + }) + } +} + +fuzz_target!(|case: FuzzCase| { + check_schedule_reordering(case.pass_count, &case.resource_accesses); +}); diff --git a/guide/doctest.rs b/guide/doctest.rs index b7fc3ca9..48691e2d 100644 --- a/guide/doctest.rs +++ b/guide/doctest.rs @@ -1,4 +1,4 @@ -// HACK: Test tooling for mbdook is lacking so this fix is applied: +// HACK: Test tooling for mdBook is lacking so this fix is applied: // https://github.com/BurntSushi/jiff/blob/985b4156c4dbaf2ed69150a4ec4e6d5352f1d47e/book/doctest.rs #[doc = include_str!("src/cmd.md")] @@ -7,12 +7,6 @@ pub mod cmd {} #[doc = include_str!("src/cmd_compute.md")] pub mod cmd_compute {} -#[doc = include_str!("src/cmd_graphic.md")] -pub mod cmd_graphic {} - -#[doc = include_str!("src/cmd_ray_trace.md")] -pub mod cmd_ray_trace {} - #[doc = include_str!("src/pipeline.md")] pub mod pipeline {} @@ -45,3 +39,6 @@ pub mod usage_device {} #[doc = include_str!("src/usage_shader.md")] pub mod usage_shader {} + +#[doc = include_str!("src/usage_window.md")] +pub mod usage_window {} diff --git a/guide/guide-helper/src/lib.rs b/guide/guide-helper/src/lib.rs index f0f2abb9..de103e48 100644 --- a/guide/guide-helper/src/lib.rs +++ b/guide/guide-helper/src/lib.rs @@ -67,7 +67,7 @@ impl Preprocessor for GuideHelper { insert_vulkan_sdk_version(&mut book); // mdBook renders the raw `#` lines literally, but compile-only example - // lines should still keep those lines available for doctests. + // lines should still keep those lines available for doctests if _ctx.renderer == "html" { hide_hidden_lines(&mut book); } @@ -231,7 +231,7 @@ fn should_hide_lines_in_fence(info: &str) -> bool { let lang = info.split(',').map(str::trim).next().unwrap_or_default(); - !matches!(lang, "text" | "plain" | "plaintext" | "md" | "markdown") + matches!(lang, "rust" | "rs") || info.split(',').map(str::trim).any(|part| part == "no_run") } fn vulkan_sdk_version_for_ash(major: u64, minor: u64) -> &'static str { @@ -267,6 +267,9 @@ mod test { "let graph = Graph::new();\n", "# let _ = graph;\n", "```\n\n", + "```rust,no_run\n", + "# let _ = Graph::new();\n", + "```\n\n", "```text\n", "# keep this line\n", "```\n\n", @@ -287,13 +290,17 @@ mod test { "```rust\n", "let graph = Graph::new();\n", "```\n\n", + "```rust,no_run\n", + "```\n\n", "```text\n", "# keep this line\n", "```\n\n", "```toml\n", + "# Cargo.toml\n", "[dependencies]\n", "```\n\n", "```bash\n", + "# See: \"Shader Compilation\"\n", "run-example\n", "```\n", ) diff --git a/guide/guide-helper/src/main.rs b/guide/guide-helper/src/main.rs index 52d02074..57d05e02 100644 --- a/guide/guide-helper/src/main.rs +++ b/guide/guide-helper/src/main.rs @@ -4,7 +4,7 @@ fn main() { let mut args = std::env::args().skip(1); match args.next().as_deref() { Some("supports") => { - // Supports all renderers. + // Supports all renderers return; } Some(arg) => { diff --git a/guide/src/README.md b/guide/src/README.md index c460fe9e..bc09e2a1 100644 --- a/guide/src/README.md +++ b/guide/src/README.md @@ -54,7 +54,7 @@ The driver is based off the popular `ash` crate and `vk-sync`; reasoned as follo - 2022 --- v0.2 released with an earlier graph API based on [_`Kajiya`_](https://github.com/EmbarkStudios/kajiya) -- 2026 --- Project renamed `vk-graph` (v0.14) +- 2026 --- Project renamed `vk-graph` (v0.14) and the graph API was redesigned (v0.14.2) [^modern]: Modern Vulkan usage means no pixel queries. Anything else unsupported is due to there being better options, no current need, or no interest. Please open an issue. diff --git a/guide/src/cmd.md b/guide/src/cmd.md index 1b1a9d9d..55c855f0 100644 --- a/guide/src/cmd.md +++ b/guide/src/cmd.md @@ -4,7 +4,7 @@ API docs: [`Graph::begin_cmd`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.begin_cmd), [`Command::record_cmd`](https://docs.rs/vk-graph/latest/vk_graph/cmd/struct.Command.html#method.record_cmd), -[`Graph::into_submission`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.into_submission). +[`Graph::finalize`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.finalize). - Built-in graph commands such as `copy_buffer`, `clear_color_image`, and `update_buffer` - Explicit command-buffer recording through `begin_cmd().record_cmd(...)` diff --git a/guide/src/install.md b/guide/src/install.md index 990d658f..0d40c2ef 100644 --- a/guide/src/install.md +++ b/guide/src/install.md @@ -15,8 +15,12 @@ _vk-graph_ puts a lot of functionality behind optional features in order to opti compile time for the most common use cases. The following features are available. +- **`checked`** *(enabled by default)* — Enable runtime validation for common misuse patterns that + Vulkan validation layers cannot always catch. - **`loaded`** *(enabled by default)* — Support searching for the Vulkan loader manually at runtime. - **`linked`** — Link the Vulkan loader at compile time. +- **`parking_lot`** *(enabled by default)* — Use `parking_lot` synchronization primitives internally. +- **`ash-molten`** — Enable MoltenVK loading support on macOS. - **`profile-with-*`** — Use the specified profiling backend: `profile-with-puffin`, `profile-with-optick`, `profile-with-superluminal`, or `profile-with-tracy` diff --git a/guide/src/pipeline.md b/guide/src/pipeline.md index a4e7f3ad..194ea116 100644 --- a/guide/src/pipeline.md +++ b/guide/src/pipeline.md @@ -77,7 +77,7 @@ graph # Ok(()) } ``` -A call to `Graph::end_cmd` is not requried. The _end-command_ method exists to support builder-style +A call to `Graph::end_cmd` is not required. The _end-command_ method exists to support builder-style function-chaining. In the above example two commands are built and added to the graph. ## Shaders diff --git a/guide/src/resource.md b/guide/src/resource.md index dcbf0bc0..927b6937 100644 --- a/guide/src/resource.md +++ b/guide/src/resource.md @@ -65,3 +65,15 @@ over any `Pool`. Cached resources let complex graphs reuse compatible resources while keeping the pooling strategy separate from the reuse policy. + +## Queue Sharing + +`BufferInfo` and `ImageInfo` both expose `sharing_mode`. + +- `vk::SharingMode::EXCLUSIVE` is the default and lets `vk-graph` track queue-family ownership. +- `vk::SharingMode::CONCURRENT` makes the resource usable by multiple queue families without + exclusive ownership transfer tracking. + +Exclusive buffer and image ownership is range-aware. Transfer planning only considers ranges touched +by submitted commands, and barrier emission intersects ownership transfers with the resource ranges +actually accessed by those commands. diff --git a/guide/src/resource_accel_struct.md b/guide/src/resource_accel_struct.md index c571feae..8f9ff670 100644 --- a/guide/src/resource_accel_struct.md +++ b/guide/src/resource_accel_struct.md @@ -42,7 +42,7 @@ let build_range = vk::AccelerationStructureBuildRangeInfoKHR { }; let ty = vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL; let geom_info = AccelerationStructureGeometryInfo { - ty, + acceleration_structure_type: ty, flags: vk::BuildAccelerationStructureFlagsKHR::ALLOW_UPDATE, geometries: vec![ (geom, build_range), @@ -57,7 +57,7 @@ let AccelerationStructureSize { // Create acceleration structure info multiple ways: let info = AccelerationStructureInfo { - ty, + acceleration_structure_type: ty, size: build_size, }; let other_info = AccelerationStructureInfo::blas(build_size); @@ -66,7 +66,7 @@ assert_eq!(info, other_info); // Builder pattern let same_info = AccelerationStructureInfoBuilder::default() - .ty(ty) + .acceleration_structure_type(ty) .size(build_size); // Create directly from info diff --git a/guide/src/resource_buffer.md b/guide/src/resource_buffer.md index 1f8c76d7..ebe91aa2 100644 --- a/guide/src/resource_buffer.md +++ b/guide/src/resource_buffer.md @@ -13,9 +13,10 @@ let usage = vk::BufferUsageFlags::STORAGE_BUFFER; // Create buffer info multiple ways: let info = BufferInfo { alignment: 1, - dedicated: false, - host_read: false, - host_write: false, + alloc_dedicated: false, + host_readable: false, + host_writable: false, + sharing_mode: vk::SharingMode::EXCLUSIVE, size, usage, }; diff --git a/guide/src/resource_image.md b/guide/src/resource_image.md index c81b0385..911757da 100644 --- a/guide/src/resource_image.md +++ b/guide/src/resource_image.md @@ -15,15 +15,18 @@ let fmt = vk::Format::R8G8B8A8_UNORM; // Create image info multiple ways let info = ImageInfo { array_layer_count: 1, - dedicated: false, + alloc_dedicated: false, depth: 1, flags: vk::ImageCreateFlags::empty(), - fmt, + format: fmt, height, + host_readable: false, + host_writable: false, mip_level_count: 1, sample_count: SampleCount::Type1, + sharing_mode: vk::SharingMode::EXCLUSIVE, tiling: vk::ImageTiling::OPTIMAL, - ty: vk::ImageType::TYPE_2D, + image_type: vk::ImageType::TYPE_2D, usage, width, }; @@ -38,9 +41,9 @@ let same_info = ImageInfoBuilder::default() .width(width) .height(height) .depth(1) - .fmt(fmt) + .format(fmt) .usage(usage) - .ty(vk::ImageType::TYPE_2D); + .image_type(vk::ImageType::TYPE_2D); // Info built from other info let array_info = cube_info @@ -52,7 +55,7 @@ let array_info = cube_info let image = Image::create(device, info)?; // For interop this may be handy: -let image = Image::from_raw(device, vk::Image::null(), info); +let image = unsafe { Image::from_raw(device, vk::Image::null(), info) }; // The provided fields are helpful: assert_eq!(image.device, *device); @@ -74,9 +77,9 @@ let image_view = ImageViewInfo { aspect_mask: vk::ImageAspectFlags::COLOR, base_array_layer: 0, base_mip_level: 0, - fmt, + format: fmt, mip_level_count: 1, - ty: vk::ImageViewType::TYPE_2D, + view_type: vk::ImageViewType::TYPE_2D, }; // Image views have the same builder functionality: diff --git a/guide/src/usage.md b/guide/src/usage.md index 2c4926e8..53fe2f92 100644 --- a/guide/src/usage.md +++ b/guide/src/usage.md @@ -7,7 +7,7 @@ API docs: [`Graph`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html), [`Graph::begin_cmd`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.begin_cmd), [`Graph::bind_resource`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.bind_resource), [`Graph::resource`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.resource), -[`Graph::into_submission`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.into_submission). +[`Graph::finalize`](https://docs.rs/vk-graph/latest/vk_graph/struct.Graph.html#method.finalize). Typical usage contains: @@ -56,7 +56,7 @@ allocations. The allocation strategy provides a large section of memory which is then sub-allocated for any resources which use it. This may lead to fragmentation and memory exhaustion in some scenarios. -Individual buffers or images may use dedicated memory allocations by setting their `dedicated` +Individual buffers or images may use dedicated memory allocations by setting their `alloc_dedicated` field: ```rust @@ -73,14 +73,14 @@ field: let uber_mesh_buf = Buffer::create( device, BufferInfo { - dedicated: true, + alloc_dedicated: true, ..buffer_info } )?; -// Builder functions are also availble -// (builder and info types are interchangable) -let dedicated_info = image_info.into_builder().dedicated(true); +// Builder functions are also available +// (builder and info types are interchangeable) +let dedicated_info = image_info.into_builder().alloc_dedicated(true); let important_image = Image::create(device, dedicated_info)?; # Ok(()) } ``` @@ -122,6 +122,10 @@ assert_eq!(shared_image.info.width, 320); # } ``` +Concrete node types return the exact stored handle type. For example, `ImageNode` returns +`&Arc`. Erased node types such as `AnyImageNode` instead return `&Image` so they can unify +owned, leased, and swapchain-backed resources behind one view. + ## Commands Nodes may be used with built-in graph commands: @@ -269,11 +273,11 @@ graph, but they may do so manually: # graph: Graph, # device: &Device, # ) -> Result<(), DriverError> { -// NOTE: This will stall! Use the async functions to check periodically instead -graph - .into_submission() - .queue_submit(&mut LazyPool::new(device), 0, 0)? - .wait_until_executed()?; +// NOTE: This will stall! Use Fence::is_signaled to check periodically instead. +let mut fence = graph + .finalize() + .queue_submit(&mut LazyPool::new(device), 0, 0)?; +fence.wait_signaled()?; # Ok(()) } ``` diff --git a/guide/src/usage_debugging.md b/guide/src/usage_debugging.md index fb4291be..1b62d008 100644 --- a/guide/src/usage_debugging.md +++ b/guide/src/usage_debugging.md @@ -1,8 +1,12 @@ # Debugging -Debug mode (setting the `debug` field of `DeviceInfo` or `InstanceInfo` to `true`) is supported only -when a compatible [_Vulkan SDK_](https://vulkan.lunarg.com/sdk/home) - is installed. +Debug mode is enabled by setting the `debug` field of `DeviceInfo` or `InstanceInfo` to `true`. +It requires `VK_EXT_debug_utils` and `VK_EXT_private_data` support. Construction fails in debug mode +when those requirements are unavailable. + +A compatible [_Vulkan SDK_](https://vulkan.lunarg.com/sdk/home) + is required for validation layers and +debugging tools. > [!IMPORTANT] > The installed Vulkan SDK version must be at least v{{ vulkan_sdk.version }}. @@ -11,6 +15,29 @@ While in debug mode `vk-graph` watches for errors, warnings, and certain perform emitted from any currently enabled Vulkan debug application layers. Emitted events will cause the active thread to be parked and log a message indicating how to attach a debugger. +## RenderDoc Labels + +When debug mode is active, `vk-graph` emits Vulkan debug-utils object names and command label regions +for RenderDoc and similar tools. + +Resources and pipelines expose a setter and builder-style helper: + +```rust +# use vk_graph::driver::{DriverError, ash::vk, buffer::{Buffer, BufferInfo}, device::Device}; +# fn test(device: &Device) -> Result<(), DriverError> { +let buffer = Buffer::create( + device, + BufferInfo::device_mem(1024, vk::BufferUsageFlags::STORAGE_BUFFER), +)?.with_debug_name("work buffer"); + +buffer.set_debug_name("renamed work buffer"); +# Ok(()) } +``` + +Pipeline debug names are propagated to internal Vulkan objects such as pipeline layouts and +descriptor set layouts. Command names become debug label regions during submission recording. +Swapchain images are also named by index. + ## Logging `vk-graph` uses `log` v{{ log.version }} for low-overhead logging. @@ -73,10 +100,14 @@ misuse patterns that the VVL cannot catch: - Missing `resource_access` / `shader_resource_access` declarations before using a resource - [`update_buffer`] and [`copy_buffer_region`](crate::Graph::copy_buffer_region) buffer bounds - Valid image aspect masks and subresource ranges +- Cross-graph node ownership checks The `checked` feature is **enabled by default** — it activates in both debug and release builds. Disable it for zero-overhead release builds that have been validated: +With `checked` disabled, `vk-graph` no longer fail-fast validates that a node handle belongs to the +graph it is used with. That remains invalid usage; the caller is responsible for avoiding it. + ```bash cargo run --no-default-features --features loaded,parking_lot --release ``` diff --git a/guide/src/usage_device.md b/guide/src/usage_device.md index 93aa3fd1..1909f5fb 100644 --- a/guide/src/usage_device.md +++ b/guide/src/usage_device.md @@ -4,7 +4,7 @@ Most Vulkan operations occur within the context of a logical device, provided by `Device` (_a smart pointer for `ash::Device`_). API docs: [`Device::create`](https://docs.rs/vk-graph/latest/vk_graph/driver/device/struct.Device.html#method.create), -[`Device::try_from_ash_device`](https://docs.rs/vk-graph/latest/vk_graph/driver/device/struct.Device.html#method.try_from_ash_device), +[`Device::try_from_ash`](https://docs.rs/vk-graph/latest/vk_graph/driver/device/struct.Device.html#method.try_from_ash), [`Device::try_from_display`](https://docs.rs/vk-graph/latest/vk_graph/driver/device/struct.Device.html#method.try_from_display). > [!WARNING] @@ -25,7 +25,7 @@ production-ready code used to create a device: let info = DeviceInfo::default(); let device = Device::create(info)?; -assert_eq!(device.physical_device.instance.info.debug, false); +assert_eq!(device.physical.instance.info.debug, false); # Ok(()) } ``` @@ -82,8 +82,12 @@ let physical_devices = Instance::physical_devices(&instance)?; for physical_device in physical_devices { // We are looking for a device with support for these features - if !physical_device.khr_swapchain - || !physical_device.ray_tracing_pipeline_features.ray_tracing_pipeline { + if !physical_device.vk_khr_swapchain + || !physical_device + .vk_khr_ray_tracing_pipeline + .as_ref() + .is_some_and(|ext| ext.features.ray_tracing_pipeline) + { continue; } @@ -112,7 +116,7 @@ let physical_device: vk::PhysicalDevice = todo!(); // vk-graph types let instance = Instance::try_from_entry(entry, instance)?; -let physical_device = PhysicalDevice::try_from_ash(&instance, physical_device)?; +let physical_device = unsafe { PhysicalDevice::try_from_ash(&instance, physical_device) }?; // Use our PhysicalDevice to create a native ash::Device (OpenXR requires this) let device: ash::Device = unsafe { @@ -130,7 +134,7 @@ let device: ash::Device = unsafe { }.unwrap(); // Create a Device from their native stuff -let device = Device::try_from_ash(device, physical_device)?; +let device = unsafe { Device::try_from_ash(device, physical_device) }?; # Ok(()) } ``` diff --git a/guide/src/usage_thread.md b/guide/src/usage_thread.md index 19472bf5..5e7301d3 100644 --- a/guide/src/usage_thread.md +++ b/guide/src/usage_thread.md @@ -5,9 +5,11 @@ Resources are externally synchronized, and mutable graph-building APIs such as ` require exclusive access to the `Graph` itself. API docs: [`Submission`](https://docs.rs/vk-graph/latest/vk_graph/struct.Submission.html), +[`RecordedSubmission`](https://docs.rs/vk-graph/latest/vk_graph/struct.RecordedSubmission.html), [`Submission::queue_submit`](https://docs.rs/vk-graph/latest/vk_graph/struct.Submission.html#method.queue_submit), -[`Submission::queue_cmds_for_resource`](https://docs.rs/vk-graph/latest/vk_graph/struct.Submission.html#method.queue_cmds_for_resource), -[`Submission::queue_cmds_for_resource_dependencies`](https://docs.rs/vk-graph/latest/vk_graph/struct.Submission.html#method.queue_cmds_for_resource_dependencies), +[`Submission::record_resource`](https://docs.rs/vk-graph/latest/vk_graph/struct.Submission.html#method.record_resource), +[`Submission::record_resource_dependencies`](https://docs.rs/vk-graph/latest/vk_graph/struct.Submission.html#method.record_resource_dependencies), +[`RecordedSubmission::queue_submit`](https://docs.rs/vk-graph/latest/vk_graph/struct.RecordedSubmission.html#method.queue_submit), [`CommandBuffer::has_executed`](https://docs.rs/vk-graph/latest/vk_graph/driver/cmd_buf/struct.CommandBuffer.html#method.has_executed). More precisely, `vk-graph` stores the most recent access type of each subresource of a resource. As @@ -17,8 +19,9 @@ updated. Resource state is updated during the following function calls: - `Submission::queue_submit` -- `Submission::queue_cmds_for_resource` -- `Submission::queue_cmds_for_resource_dependencies` +- `Submission::record_resource` +- `Submission::record_resource_dependencies` +- `RecordedSubmission::queue_submit` > [!CAUTION] > Do not call any `Submission` recording or queue function that accesses buffers, images, or acceleration @@ -27,12 +30,11 @@ Resource state is updated during the following function calls: ## Execution The provided `Submission` recording and queue functions are designed to support a typical -swapchain-based -workflow: -1. Queue all commands the swapchain depends on -1. Acquire swapchain -1. Queue swapchain commands -1. Present swapchain +frame-presentation workflow: +1. Queue all commands the presented frame depends on +1. Acquire the presentation image, usually through `Graphchain` +1. Queue frame commands that write the presentation image +1. Present the frame 1. Submit any final unrelated commands ## Safe Patterns @@ -45,7 +47,7 @@ For example, there is no race condition or thread contention caused by using the two threads.[^threads] In fact, there is no runtime overhead at all from this. Additionally, it is safe to build `Graph` instances, bind resources, record command buffers, and -call `Graph::into_submission` at *any* time on *any* thread, as long as each `Graph` instance is not +call `Graph::finalize` at *any* time on *any* thread, as long as each `Graph` instance is not mutably shared across threads at the same time. These patterns are safe: @@ -58,9 +60,9 @@ These patterns are safe: Host-mappable buffers require extra understanding to use properly. -The contents of a buffer are undefined from the time of submission until that `Submission` has been -fully executed, as indicated by `CommandBuffer::has_executed`. This means that you should not call -`Buffer::mapped_slice` during any submission or execution accessing that memory. +The contents of a buffer are undefined from the time of submission until the returned `Fence` is +signaled. Use `Fence::is_signaled` or `Fence::wait_signaled` before reading or writing host-mapped +memory touched by that submission. See: [_`examples/cpu_readback.rs`_](https://github.com/attackgoat/vk-graph/blob/main/examples/cpu_readback.rs) diff --git a/guide/src/usage_window.md b/guide/src/usage_window.md index 02185b2b..2c18a843 100644 --- a/guide/src/usage_window.md +++ b/guide/src/usage_window.md @@ -8,15 +8,27 @@ > provides additional documentation and > examples. -## Swapchain +## Presentation -The bifurcation of `vk-graph` along the window abstraction results in two `Swapchain` types, one in -each crate. +The core crate and window crate intentionally separate low-level Vulkan presentation from the +frame-oriented helper used by most windowed applications. Type | Usage -- | -- `vk_graph::driver::swapchain::Swapchain` | Vulkan swapchain smart pointer, contains "raw" functions -`vk_graph_window::swapchain::Swapchain` | High-level display interface for building window handlers +`vk_graph_window::graphchain::Graphchain` | High-level frame-presentation helper for window handlers + +`Graphchain` acquires the next swapchain image, exposes it through [`FrameContext`], submits the +frame graph, and presents it. Applications normally use [`Window::run`] and write to +`frame.swapchain_image` rather than calling low-level acquire/present functions directly. + +`GraphchainInfo` controls presentation policy such as `frame_capacity`, `min_image_count`, +`present_mode`, `acquire_timeout`, and `composite_alpha`. The effective runtime values are exposed +through `EffectiveGraphchainInfo` because surface capabilities may force different values at runtime. + +Resize and surface transitions may cause a frame to be skipped internally. If the surface is lost, +the window layer tears down the current surface/graphchain and recreates them on a later draw +request. ### OpenXR @@ -24,3 +36,6 @@ Virtual reality support via OpenXR is provided as [_an example_](https://github.com/attackgoat/vk-graph/tree/main/examples/vr) which also implements a swapchain. + +[`FrameContext`]: https://docs.rs/vk-graph-window/latest/vk_graph_window/struct.FrameContext.html +[`Window::run`]: https://docs.rs/vk-graph-window/latest/vk_graph_window/struct.Window.html#method.run diff --git a/src/cmd/cmd_ref.rs b/src/cmd/cmd_ref.rs index 15f7d79d..2e141578 100644 --- a/src/cmd/cmd_ref.rs +++ b/src/cmd/cmd_ref.rs @@ -1,6 +1,6 @@ use { crate::{ - AnyAccelerationStructureNode, AnyResource, Node, + AnyAccelerationStructureNode, AnyResource, Execution, Node, driver::{ accel_struct::{ AccelerationStructureGeometry, AccelerationStructureGeometryInfo, @@ -14,9 +14,6 @@ use { std::{cell::RefCell, ops::Deref}, }; -#[cfg(feature = "checked")] -use crate::Execution; - /// Recording interface for general Vulkan commands. /// /// This structure provides a strongly-typed set of methods which allow acceleration structures to @@ -48,6 +45,7 @@ pub struct CommandRef<'a> { #[cfg(feature = "checked")] graph_id: crate::GraphId, + node_map: Option<&'a [usize]>, resources: &'a [AnyResource], } @@ -55,16 +53,19 @@ impl<'a> CommandRef<'a> { pub(crate) fn new( cmd: &'a crate::driver::cmd_buf::CommandBuffer, resources: &'a [AnyResource], - #[cfg(feature = "checked")] exec: &'a Execution, + exec: &'a Execution, #[cfg(feature = "checked")] graph_id: crate::GraphId, ) -> Self { Self { cmd, + node_map: exec.node_map.as_deref(), + resources, + #[cfg(feature = "checked")] exec, + #[cfg(feature = "checked")] - graph_id, - resources, + graph_id: exec.stream_graph_id.unwrap_or(graph_id), } } @@ -78,7 +79,7 @@ impl<'a> CommandRef<'a> { /// - Flags must include [`vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS`] /// - Size must be equal to or greater than the `build_size` value returned by /// `AccelerationStructure::size_of`, aligned to `min_accel_struct_scratch_offset_alignment` - /// of `PhysicalDevice::accel_struct_properties`. + /// of `PhysicalDevice::vk_khr_acceleration_structure`. /// /// # Examples /// @@ -159,7 +160,7 @@ impl<'a> CommandRef<'a> { /// See also: /// /// - [`examples/ray_omni.rs`](/examples/ray_omni.rs) - /// - [`examples/ray_trace.rs`](/examples/ray_trace.rs) + /// - [`examples/ray_tracing.rs`](/examples/ray_tracing.rs) /// - [`examples/rt_triangle.rs`](/examples/rt_triangle.rs) pub fn build_accel_struct(&self, infos: &[BuildAccelerationStructureInfo]) -> &Self { #[derive(Default)] @@ -208,7 +209,7 @@ impl<'a> CommandRef<'a> { let end = start + info.build_data.geometries.len(); vk_infos.push( vk::AccelerationStructureBuildGeometryInfoKHR::default() - .ty(info.build_data.ty) + .ty(info.build_data.acceleration_structure_type) .flags(info.build_data.flags) .mode(vk::BuildAccelerationStructureModeKHR::BUILD) .dst_acceleration_structure(self.resource(info.accel_struct).handle) @@ -289,7 +290,7 @@ impl<'a> CommandRef<'a> { let end = start + info.build_data.geometries.len(); vk_infos.push( vk::AccelerationStructureBuildGeometryInfoKHR::default() - .ty(info.build_data.ty) + .ty(info.build_data.acceleration_structure_type) .flags(info.build_data.flags) .mode(vk::BuildAccelerationStructureModeKHR::BUILD) .dst_acceleration_structure(self.resource(info.accel_struct).handle) @@ -323,6 +324,10 @@ impl<'a> CommandRef<'a> { self } + pub(crate) fn clone_resource_at(&self, node_idx: usize) -> AnyResource { + self.resources[node_idx].clone() + } + pub(crate) fn cmd_push_constants( &self, layout: vk::PipelineLayout, @@ -365,7 +370,7 @@ impl<'a> CommandRef<'a> { /// - Flags must include [`vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS`] /// - Size must be equal to or greater than the `update_size` value returned by /// `AccelerationStructure::size_of`, aligned to `min_accel_struct_scratch_offset_alignment` - /// of `PhysicalDevice::accel_struct_properties`. + /// of `PhysicalDevice::vk_khr_acceleration_structure`. pub fn update_accel_struct(&self, infos: &[UpdateAccelerationStructureInfo]) -> &Self { #[derive(Default)] struct Tls { @@ -413,7 +418,7 @@ impl<'a> CommandRef<'a> { let end = start + info.update_data.geometries.len(); vk_infos.push( vk::AccelerationStructureBuildGeometryInfoKHR::default() - .ty(info.update_data.ty) + .ty(info.update_data.acceleration_structure_type) .flags(info.update_data.flags) .mode(vk::BuildAccelerationStructureModeKHR::UPDATE) .dst_acceleration_structure(self.resource(info.dst_accel_struct).handle) @@ -495,7 +500,7 @@ impl<'a> CommandRef<'a> { let end = start + info.update_data.geometries.len(); vk_infos.push( vk::AccelerationStructureBuildGeometryInfoKHR::default() - .ty(info.update_data.ty) + .ty(info.update_data.acceleration_structure_type) .flags(info.update_data.flags) .mode(vk::BuildAccelerationStructureModeKHR::UPDATE) .src_acceleration_structure(self.resource(info.src_accel_struct).handle) @@ -539,21 +544,28 @@ impl<'a> CommandRef<'a> { #[cfg(feature = "checked")] resource_node.assert_owner(self.graph_id); - // You must have called an access function for this node on this execution before borrowing - // the resource! - // - // Why: Code that attempts to access this function is attempting to get access to the Vulkan - // resource (buffer, image, or acceleration structure). In order to access any resources the - // access type must first be specified so the correct barriers may be added. - // - // See: https://attackgoat.github.io/vk-graph/pipeline_sync.html + let mut node_idx = resource_node.index(); + if let Some(node_map) = self.node_map { + node_idx = node_map[node_idx]; + } + + /* + You must have called an access function for this node on this execution before borrowing + the resource! + + Code that attempts to access this function is attempting to get access to the Vulkan + resource (buffer, image, or acceleration structure). In order to access any resources the + access type must first be specified so the correct barriers may be added. + + See: https://attackgoat.github.io/vk-graph/pipeline_sync.html + */ #[cfg(feature = "checked")] assert!( - self.exec.accesses.contains_key(&resource_node.index()), + self.exec.accesses.contains(node_idx), "unexpected node access: call an access function first" ); - resource_node.borrow(self.resources) + resource_node.borrow_at(self.resources, node_idx) } } @@ -617,7 +629,7 @@ pub struct BuildAccelerationStructureIndirectInfo { /// Specifies the geometry data to use when building the acceleration structure. pub build_data: AccelerationStructureGeometryInfo, - /// A buffer device addresses which points to `data.geometry.len()` + /// A buffer device address which points to `data.geometry.len()` /// [vk::AccelerationStructureBuildRangeInfoKHR] structures defining dynamic offsets to the /// addresses where geometry data is stored. pub range_base: vk::DeviceAddress, @@ -661,7 +673,7 @@ pub struct UpdateAccelerationStructureIndirectInfo { /// The acceleration structure to be written. pub dst_accel_struct: AnyAccelerationStructureNode, - /// A buffer device addresses which points to `data.geometry.len()` + /// A buffer device address which points to `data.geometry.len()` /// [vk::AccelerationStructureBuildRangeInfoKHR] structures defining dynamic offsets to the /// addresses where geometry data is stored. pub range_base: vk::DeviceAddress, diff --git a/src/cmd/compute.rs b/src/cmd/compute.rs index b6ba97c2..2df75560 100644 --- a/src/cmd/compute.rs +++ b/src/cmd/compute.rs @@ -6,13 +6,13 @@ use { }; impl PipelineCommand<'_, ComputePipeline> { - /// Begin recording a compute pipeline command buffer. + /// Begin recording compute pipeline work for this graph command. pub fn record_cmd(mut self, func: impl FnOnce(ComputeCommandRef<'_>) + Send + 'static) -> Self { self.record_cmd_mut(func); self } - /// Begin recording a compute pipeline command buffer. + /// Mutable-borrow form of [`Self::record_cmd`]. pub fn record_cmd_mut(&mut self, func: impl FnOnce(ComputeCommandRef<'_>) + Send + 'static) { let pipeline = self .cmd @@ -25,9 +25,28 @@ impl PipelineCommand<'_, ComputePipeline> { func(ComputeCommandRef { cmd, pipeline }); }); } + + pub(crate) fn record_stream_mut( + &mut self, + func: impl for<'r> Fn(ComputeCommandRef<'r>) + Send + Sync + 'static, + ) { + let pipeline = self + .cmd + .cmd() + .expect_last_pipeline() + .expect_compute() + .clone(); + + self.cmd.push_reusable_exec(move |cmd| { + func(ComputeCommandRef { + cmd, + pipeline: pipeline.clone(), + }); + }); + } } -/// Recording interface for computing commands. +/// Recording interface for compute commands. /// /// This structure provides a strongly-typed set of methods which allow compute shader code to be /// executed. An instance is provided to the closure argument of @@ -80,12 +99,12 @@ impl ComputeCommandRef<'_> { /// #version 450 /// #pragma shader_stage(compute) /// - /// layout(set = 0, binding = 0, std430) restrict writeonly buffer MyBufer { + /// layout(set = 0, binding = 0, std430) restrict writeonly buffer MyBuffer { /// uint my_buf[]; /// }; /// /// void main() { - /// // TODO + /// my_buf[0] = 1; /// } /// # "#); /// ``` @@ -246,7 +265,7 @@ impl ComputeCommandRef<'_> { /// Updates push constants. /// - /// Push constants represent a high speed path to modify constant data in pipelines that is + /// Push constants represent a high-speed path to modify constant data in pipelines that is /// expected to outperform memory-backed resource updates. /// /// Push constant values can be updated incrementally, causing shader stages to read the new @@ -255,9 +274,8 @@ impl ComputeCommandRef<'_> { /// /// # Device limitations /// - /// See - /// [`device.physical_device.props.limits.max_push_constants_size`](vk::PhysicalDeviceLimits) - /// for the limits of the current device. You may also check [gpuinfo.org] for a listing of + /// See [`VkPhysicalDeviceLimits::maxPushConstantsSize`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceLimits.html) + /// for the limit of the current device. You may also check [gpuinfo.org] for a listing of /// reported limits on other devices. /// /// # Examples @@ -275,7 +293,7 @@ impl ComputeCommandRef<'_> { /// /// void main() /// { - /// // TODO: Add bindings to read/write things! + /// uint value = push_constants.the_answer; /// } /// # "#); /// ``` diff --git a/src/cmd/graphic.rs b/src/cmd/graphics.rs similarity index 75% rename from src/cmd/graphic.rs rename to src/cmd/graphics.rs index 337e303c..4fbdfcc2 100644 --- a/src/cmd/graphic.rs +++ b/src/cmd/graphics.rs @@ -1,10 +1,11 @@ use { super::{AttachmentIndex, cmd_ref::CommandRef, pipeline::PipelineCommand}, crate::{ - Attachment, Node, SubresourceAccess, + Attachment, ColorAttachment, ColorResolve, DepthStencilAttachment, DepthStencilResolve, + LoadOp, Node, StoreOp, SubresourceAccess, cmd::SubresourceRange, driver::{ - graphic::{DepthStencilInfo, GraphicsPipeline}, + graphics::{DepthStencilInfo, GraphicsPipeline}, image::{ ImageInfo, ImageViewInfo, image_subresource_range_contains, image_subresource_range_intersects, @@ -27,7 +28,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// /// Note: The default view (the whole image) is used for `image`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn color_attachment_image( mut self, color_attachment: AttachmentIndex, @@ -42,7 +43,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// Sets the `color_attachment` attachment index of the following render pass to the given /// `image`, as interpreted by `image_view_info`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn color_attachment_image_view( mut self, color_attachment: AttachmentIndex, @@ -60,7 +61,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// /// Note: The default view (the whole image) is used for `image`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn color_attachment_resolve_image( mut self, msaa_attachment: AttachmentIndex, @@ -74,7 +75,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// Resolves a multi-sampled (MSAA) color image attachment into a single-sampled attachment /// using the given `image`, as interpreted by `image_view_info`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn color_attachment_resolve_image_view( mut self, msaa_attachment: AttachmentIndex, @@ -106,7 +107,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// /// Note: The default view (the whole image) is used for `image`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn depth_stencil_attachment_image( mut self, image: impl Into, @@ -125,7 +126,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// /// Note: The default view (the whole image) is used for `image`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn depth_stencil_attachment_image_view( mut self, image: impl Into, @@ -142,7 +143,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// /// Note: The default view (the whole image) is used for `image`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn depth_stencil_attachment_resolve_image( mut self, depth_stencil_attachment: AttachmentIndex, @@ -164,7 +165,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { /// /// Note: The default view (the whole image) is used for `image`. /// - /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html) + /// See [_Render Pass_](https://docs.vulkan.org/spec/latest/chapters/renderpass.html). pub fn depth_stencil_attachment_resolve_image_view( mut self, depth_stencil_attachment: AttachmentIndex, @@ -192,7 +193,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// Begin recording a graphics pipeline command buffer. + /// Begin recording graphics pipeline work for this graph command. pub fn record_cmd( mut self, func: impl FnOnce(GraphicsCommandRef<'_>) + Send + 'static, @@ -201,13 +202,13 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// Begin recording a graphics pipeline command buffer. + /// Mutable-borrow form of [`Self::record_cmd`]. pub fn record_cmd_mut(&mut self, func: impl FnOnce(GraphicsCommandRef<'_>) + Send + 'static) { let pipeline = self .cmd .cmd() .expect_last_pipeline() - .expect_graphic() + .expect_graphics() .clone(); self.cmd.push_exec(move |cmd| { @@ -215,9 +216,29 @@ impl PipelineCommand<'_, GraphicsPipeline> { }); } + pub(crate) fn record_stream_mut( + &mut self, + func: impl for<'r> Fn(GraphicsCommandRef<'r>) + Send + Sync + 'static, + ) { + let pipeline = self + .cmd + .cmd() + .expect_last_pipeline() + .expect_graphics() + .clone(); + + self.cmd.push_reusable_exec(move |cmd| { + func(GraphicsCommandRef { + cmd, + pipeline: pipeline.clone(), + }); + }); + } + + /// Sets the render area used when beginning the render pass for subsequent command buffer + /// recordings of the current graph command. + /// /// See [`VkRenderPassBeginInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkRenderPassBeginInfo.html). - /// field when beginning a render pass used by any subsequent command buffer recordings - /// of the current graph command. /// /// _NOTE:_ Setting this value will cause the viewport and scissor to be unset, which is not the /// default behavior. When this value is set you should call `set_viewport` and `set_scissor` on @@ -231,7 +252,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::color_attachment_image`] + /// Mutable-borrow form of [`Self::color_attachment_image`]. pub fn set_color_attachment_image( &mut self, color_attachment: AttachmentIndex, @@ -240,14 +261,16 @@ impl PipelineCommand<'_, GraphicsPipeline> { store: StoreOp, ) -> &mut Self { let image = image.into(); - let image_view = self.resource(image).info; + let image_view: ImageViewInfo = self.cmd.graph.resources[image.index()] + .expect_image_info() + .into(); self.set_color_attachment_image_view(color_attachment, image, image_view, load, store); self } - /// See [`Self::color_attachment_image_view`] + /// Mutable-borrow form of [`Self::color_attachment_image_view`]. pub fn set_color_attachment_image_view( &mut self, color_attachment: AttachmentIndex, @@ -278,7 +301,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::color_attachment_resolve_image`] + /// Mutable-borrow form of [`Self::color_attachment_resolve_image`]. pub fn set_color_attachment_resolve_image( &mut self, msaa_attachment: AttachmentIndex, @@ -286,7 +309,9 @@ impl PipelineCommand<'_, GraphicsPipeline> { image: impl Into, ) -> &mut Self { let image = image.into(); - let image_view = self.resource(image).info; + let image_view: ImageViewInfo = self.cmd.graph.resources[image.index()] + .expect_image_info() + .into(); self.set_color_attachment_resolve_image_view( msaa_attachment, @@ -298,7 +323,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::color_attachment_resolve_image_view`] + /// Mutable-borrow form of [`Self::color_attachment_resolve_image_view`]. pub fn set_color_attachment_resolve_image_view( &mut self, msaa_attachment: AttachmentIndex, @@ -314,7 +339,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::depth_stencil`] + /// Mutable-borrow form of [`Self::depth_stencil`]. pub fn set_depth_stencil(&mut self, depth_stencil: impl Into) -> &mut Self { let depth_stencil = depth_stencil.into(); let cmd = self.cmd.cmd_mut(); @@ -327,7 +352,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::depth_stencil_attachment_image`] + /// Mutable-borrow form of [`Self::depth_stencil_attachment_image`]. pub fn set_depth_stencil_attachment_image( &mut self, image: impl Into, @@ -335,14 +360,16 @@ impl PipelineCommand<'_, GraphicsPipeline> { store: StoreOp, ) -> &mut Self { let image = image.into(); - let image_view_info = self.resource(image).info; + let image_view_info: ImageViewInfo = self.cmd.graph.resources[image.index()] + .expect_image_info() + .into(); self.set_depth_stencil_attachment_image_view(image, image_view_info, load, store); self } - /// See [`Self::depth_stencil_attachment_image_view`] + /// Mutable-borrow form of [`Self::depth_stencil_attachment_image_view`]. pub fn set_depth_stencil_attachment_image_view( &mut self, image: impl Into, @@ -372,7 +399,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::depth_stencil_attachment_resolve_image`] + /// Mutable-borrow form of [`Self::depth_stencil_attachment_resolve_image`]. pub fn set_depth_stencil_attachment_resolve_image( &mut self, depth_stencil_attachment: AttachmentIndex, @@ -381,7 +408,9 @@ impl PipelineCommand<'_, GraphicsPipeline> { stencil_mode: Option, ) -> &mut Self { let image = image.into(); - let image_view = self.resource(image).info; + let image_view: ImageViewInfo = self.cmd.graph.resources[image.index()] + .expect_image_info() + .into(); self.set_depth_stencil_attachment_resolve_image_view( depth_stencil_attachment, @@ -394,7 +423,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::depth_stencil_attachment_resolve_image_view`] + /// Mutable-borrow form of [`Self::depth_stencil_attachment_resolve_image_view`]. pub fn set_depth_stencil_attachment_resolve_image_view( &mut self, depth_stencil_attachment: AttachmentIndex, @@ -417,7 +446,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::multiview`] + /// Mutable-borrow form of [`Self::multiview`]. pub fn set_multiview(&mut self, view_mask: u32, correlated_view_mask: u32) -> &mut Self { let cmd = self.cmd.cmd_mut(); let exec = cmd.expect_last_exec_mut(); @@ -428,7 +457,7 @@ impl PipelineCommand<'_, GraphicsPipeline> { self } - /// See [`Self::render_area`] + /// Mutable-borrow form of [`Self::render_area`]. pub fn set_render_area(&mut self, area: vk::Rect2D) -> &mut Self { self.cmd.cmd_mut().expect_last_exec_mut().render_area = Some(area); self @@ -436,6 +465,201 @@ impl PipelineCommand<'_, GraphicsPipeline> { } impl PipelineCommand<'_, GraphicsPipeline> { + fn is_input_attachment(&self, attachment_idx: AttachmentIndex) -> bool { + self.cmd + .cmd() + .expect_last_pipeline() + .expect_graphics() + .inner + .input_attachments + .contains(&attachment_idx) + } + + fn set_color_attachment( + &mut self, + attachment_idx: AttachmentIndex, + attachment: Attachment, + load: LoadOp<[f32; 4]>, + ) { + let is_input = self.is_input_attachment(attachment_idx); + let exec = self.cmd.cmd_mut().expect_last_exec_mut(); + + if let Some(state) = exec.attachments.color_attachment_mut(attachment_idx) { + #[cfg(feature = "checked")] + assert!( + Attachment::are_compatible(Some(attachment), Some(state.attachment)), + "incompatible with existing attachment" + ); + + state.attachment = attachment; + state.load = load; + state.is_input = is_input; + state.is_attachment = true; + } else { + exec.attachments.set_color_attachment( + attachment_idx, + ColorAttachment { + attachment, + load, + store: StoreOp::DontCare, + resolve: None, + is_input, + is_attachment: true, + }, + ); + } + } + + fn set_depth_stencil_attachment( + &mut self, + attachment: Attachment, + load: LoadOp, + ) { + let exec = self.cmd.cmd_mut().expect_last_exec_mut(); + + if let Some(state) = exec.attachments.depth_stencil_attachment_mut() { + #[cfg(feature = "checked")] + assert!( + Attachment::are_compatible(Some(attachment), Some(state.attachment)), + "incompatible with existing attachment" + ); + + state.attachment = attachment; + state.load = load; + state.is_attachment = true; + } else { + exec.attachments + .set_depth_stencil_attachment(DepthStencilAttachment { + attachment, + load, + store: StoreOp::DontCare, + resolve: None, + is_attachment: true, + }); + } + } + + fn set_color_store(&mut self, attachment_idx: AttachmentIndex, attachment: Attachment) { + let is_input = self.is_input_attachment(attachment_idx); + let exec = self.cmd.cmd_mut().expect_last_exec_mut(); + + if let Some(state) = exec.attachments.color_attachment_mut(attachment_idx) { + #[cfg(feature = "checked")] + assert!( + Attachment::are_compatible(Some(attachment), Some(state.attachment)), + "incompatible with existing attachment" + ); + + state.attachment = attachment; + state.store = StoreOp::Store; + state.is_attachment = true; + state.is_input = is_input; + } else { + exec.attachments.set_color_attachment( + attachment_idx, + ColorAttachment { + attachment, + load: LoadOp::DontCare, + store: StoreOp::Store, + resolve: None, + is_input, + is_attachment: true, + }, + ); + } + } + + fn set_depth_stencil_store(&mut self, attachment: Attachment) { + let exec = self.cmd.cmd_mut().expect_last_exec_mut(); + + if let Some(state) = exec.attachments.depth_stencil_attachment_mut() { + #[cfg(feature = "checked")] + assert!( + Attachment::are_compatible(Some(attachment), Some(state.attachment)), + "incompatible with existing attachment" + ); + + state.attachment = attachment; + state.store = StoreOp::Store; + state.is_attachment = true; + } else { + exec.attachments + .set_depth_stencil_attachment(DepthStencilAttachment { + attachment, + load: LoadOp::DontCare, + store: StoreOp::Store, + resolve: None, + is_attachment: true, + }); + } + } + + fn set_color_resolve( + &mut self, + src_attachment_idx: AttachmentIndex, + dst_attachment_idx: AttachmentIndex, + attachment: Attachment, + ) { + let is_input = self.is_input_attachment(dst_attachment_idx); + let exec = self.cmd.cmd_mut().expect_last_exec_mut(); + + if let Some(state) = exec.attachments.color_attachment_mut(dst_attachment_idx) { + state.resolve = Some(ColorResolve { + attachment, + src_attachment_idx, + }); + state.is_input = is_input; + } else { + exec.attachments.set_color_attachment( + dst_attachment_idx, + ColorAttachment { + attachment, + load: LoadOp::DontCare, + store: StoreOp::DontCare, + resolve: Some(ColorResolve { + attachment, + src_attachment_idx, + }), + is_input, + is_attachment: false, + }, + ); + } + } + + fn set_depth_stencil_resolve( + &mut self, + dst_attachment_idx: AttachmentIndex, + attachment: Attachment, + depth_mode: Option, + stencil_mode: Option, + ) { + let exec = self.cmd.cmd_mut().expect_last_exec_mut(); + + if let Some(state) = exec.attachments.depth_stencil_attachment_mut() { + state.resolve = Some(DepthStencilResolve { + attachment, + dst_attachment_idx, + depth_mode, + stencil_mode, + }); + } else { + exec.attachments + .set_depth_stencil_attachment(DepthStencilAttachment { + attachment, + load: LoadOp::DontCare, + store: StoreOp::DontCare, + resolve: Some(DepthStencilResolve { + attachment, + dst_attachment_idx, + depth_mode, + stencil_mode, + }), + is_attachment: false, + }); + } + } + fn attach_color( &mut self, image: impl Into, @@ -445,69 +669,13 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; - - debug_assert!( - !self - .cmd - .cmd() - .expect_last_exec() - .color_clears - .contains_key(&attachment_idx), - "color attachment {attachment_idx} already attached via clear" - ); - debug_assert!( - !self - .cmd - .cmd() - .expect_last_exec() - .color_loads - .contains_key(&attachment_idx), - "color attachment {attachment_idx} already attached via load" - ); + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .color_attachments - .insert( - attachment_idx, - Attachment::new(image_view_info, sample_count, node_idx), - ); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_resolves - .get(&attachment_idx) - .map(|(attachment, _)| *attachment), - self.cmd - .cmd() - .expect_last_exec() - .color_attachments - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} incompatible with existing resolve" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_stores - .get(&attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_attachments - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} incompatible with existing store" + self.set_color_attachment( + attachment_idx, + Attachment::new(image_view_info, sample_count, node_idx), + LoadOp::DontCare, ); self.cmd.push_subresource_access( @@ -527,48 +695,12 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; - - debug_assert!( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_clear - .is_none(), - "depth/stencil attachment already attached via clear" - ); - debug_assert!( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_load - .is_none(), - "depth/stencil attachment already attached via load" - ); + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .depth_stencil_attachment = - Some(Attachment::new(image_view_info, sample_count, node_idx)); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_resolve - .map(|(attachment, ..)| attachment), - self.cmd.cmd().expect_last_exec().depth_stencil_attachment - ), - "depth/stencil attachment incompatible with existing resolve" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd.cmd().expect_last_exec().depth_stencil_store, - self.cmd.cmd().expect_last_exec().depth_stencil_attachment - ), - "depth/stencil attachment incompatible with existing store" + self.set_depth_stencil_attachment( + Attachment::new(image_view_info, sample_count, node_idx), + LoadOp::DontCare, ); self.cmd.push_subresource_access( @@ -602,76 +734,17 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); let color = color.into(); let color: vk::ClearColorValue = color.into(); let color = unsafe { color.float32 }; - debug_assert!( - !self - .cmd - .cmd() - .expect_last_exec() - .color_attachments - .contains_key(&attachment_idx), - "color attachment {attachment_idx} already attached" - ); - debug_assert!( - !self - .cmd - .cmd() - .expect_last_exec() - .color_loads - .contains_key(&attachment_idx), - "color attachment {attachment_idx} already attached via load" - ); - - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .color_clears - .insert( - attachment_idx, - ( - Attachment::new(image_view_info, sample_count, node_idx), - color, - ), - ); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_resolves - .get(&attachment_idx) - .map(|(attachment, _)| *attachment), - self.cmd - .cmd() - .expect_last_exec() - .color_clears - .get(&attachment_idx) - .map(|(attachment, _)| *attachment) - ), - "color attachment {attachment_idx} clear incompatible with existing resolve" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_stores - .get(&attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_clears - .get(&attachment_idx) - .map(|(attachment, _)| *attachment) - ), - "color attachment {attachment_idx} clear incompatible with existing store" + self.set_color_attachment( + attachment_idx, + Attachment::new(image_view_info, sample_count, node_idx), + LoadOp::Clear(color), ); let mut image_access = AccessType::ColorAttachmentWrite; @@ -726,58 +799,12 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - debug_assert!( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_attachment - .is_none(), - "depth/stencil attachment already attached" - ); - debug_assert!( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_load - .is_none(), - "depth/stencil attachment already attached via load" - ); - - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .depth_stencil_clear = Some(( + self.set_depth_stencil_attachment( Attachment::new(image_view_info, sample_count, node_idx), - vk::ClearDepthStencilValue { depth, stencil }, - )); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_resolve - .map(|(attachment, ..)| attachment), - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_clear - .map(|(attachment, _)| attachment) - ), - "depth/stencil attachment clear incompatible with existing resolve" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd.cmd().expect_last_exec().depth_stencil_store, - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_clear - .map(|(attachment, _)| attachment) - ), - "depth/stencil attachment clear incompatible with existing store" + LoadOp::Clear(vk::ClearDepthStencilValue { depth, stencil }), ); let mut image_access = if image_view_info @@ -878,69 +905,13 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; - - debug_assert!( - !self - .cmd - .cmd() - .expect_last_exec() - .color_attachments - .contains_key(&attachment_idx), - "color attachment {attachment_idx} already attached" - ); - debug_assert!( - !self - .cmd - .cmd() - .expect_last_exec() - .color_clears - .contains_key(&attachment_idx), - "color attachment {attachment_idx} already attached via clear" - ); + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .color_loads - .insert( - attachment_idx, - Attachment::new(image_view_info, sample_count, node_idx), - ); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_resolves - .get(&attachment_idx) - .map(|(attachment, _)| *attachment), - self.cmd - .cmd() - .expect_last_exec() - .color_loads - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} load incompatible with existing resolve" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_stores - .get(&attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_loads - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} load incompatible with existing store" + self.set_color_attachment( + attachment_idx, + Attachment::new(image_view_info, sample_count, node_idx), + LoadOp::Load, ); let mut image_access = AccessType::ColorAttachmentRead; @@ -993,45 +964,12 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - debug_assert!( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_attachment - .is_none(), - "depth/stencil attachment already attached" - ); - debug_assert!( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_clear - .is_none(), - "depth/stencil attachment already attached via clear" - ); - - self.cmd.cmd_mut().expect_last_exec_mut().depth_stencil_load = - Some(Attachment::new(image_view_info, sample_count, node_idx)); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_resolve - .map(|(attachment, ..)| attachment), - self.cmd.cmd().expect_last_exec().depth_stencil_load - ), - "depth/stencil attachment load incompatible with existing resolve" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd.cmd().expect_last_exec().depth_stencil_store, - self.cmd.cmd().expect_last_exec().depth_stencil_load - ), - "depth/stencil attachment load incompatible with existing store" + self.set_depth_stencil_attachment( + Attachment::new(image_view_info, sample_count, node_idx), + LoadOp::Load, ); let mut image_access = AccessType::DepthStencilAttachmentRead; @@ -1093,67 +1031,12 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; - - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .color_stores - .insert( - attachment_idx, - Attachment::new(image_view_info, sample_count, node_idx), - ); + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_attachments - .get(&attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_stores - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} store incompatible with existing attachment" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_clears - .get(&attachment_idx) - .map(|(attachment, _)| *attachment), - self.cmd - .cmd() - .expect_last_exec() - .color_stores - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} store incompatible with existing clear" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_loads - .get(&attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_stores - .get(&attachment_idx) - .copied() - ), - "color attachment {attachment_idx} store incompatible with existing load" + self.set_color_store( + attachment_idx, + Attachment::new(image_view_info, sample_count, node_idx), ); let mut image_access = AccessType::ColorAttachmentWrite; @@ -1206,38 +1089,10 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .depth_stencil_store = Some(Attachment::new(image_view_info, sample_count, node_idx)); - - debug_assert!( - Attachment::are_compatible( - self.cmd.cmd().expect_last_exec().depth_stencil_attachment, - self.cmd.cmd().expect_last_exec().depth_stencil_store - ), - "depth/stencil attachment store incompatible with existing attachment" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .depth_stencil_clear - .map(|(attachment, _)| attachment), - self.cmd.cmd().expect_last_exec().depth_stencil_store - ), - "depth/stencil attachment store incompatible with existing clear" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd.cmd().expect_last_exec().depth_stencil_load, - self.cmd.cmd().expect_last_exec().depth_stencil_store - ), - "depth/stencil attachment store incompatible with existing load" - ); + self.set_depth_stencil_store(Attachment::new(image_view_info, sample_count, node_idx)); let mut image_access = if image_view_info .aspect_mask @@ -1338,70 +1193,13 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .color_resolves - .insert( - dst_attachment_idx, - ( - Attachment::new(image_view_info, sample_count, node_idx), - src_attachment_idx, - ), - ); - - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_attachments - .get(&dst_attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_resolves - .get(&dst_attachment_idx) - .map(|(attachment, _)| *attachment) - ), - "color attachment {dst_attachment_idx} resolve conflict", - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_clears - .get(&dst_attachment_idx) - .map(|(attachment, _)| *attachment), - self.cmd - .cmd() - .expect_last_exec() - .color_resolves - .get(&dst_attachment_idx) - .map(|(attachment, _)| *attachment) - ), - "color attachment {dst_attachment_idx} resolve incompatible with existing clear" - ); - debug_assert!( - Attachment::are_compatible( - self.cmd - .cmd() - .expect_last_exec() - .color_loads - .get(&dst_attachment_idx) - .copied(), - self.cmd - .cmd() - .expect_last_exec() - .color_resolves - .get(&dst_attachment_idx) - .map(|(attachment, _)| *attachment) - ), - "color attachment {dst_attachment_idx} resolve incompatible with existing load" + self.set_color_resolve( + src_attachment_idx, + dst_attachment_idx, + Attachment::new(image_view_info, sample_count, node_idx), ); let mut image_access = AccessType::ColorAttachmentWrite; @@ -1457,17 +1255,15 @@ impl PipelineCommand<'_, GraphicsPipeline> { let image = image.into(); let image_view_info = image_view_info.into(); let node_idx = image.index(); - let ImageInfo { sample_count, .. } = self.resource(image).info; + let ImageInfo { sample_count, .. } = + self.cmd.graph.resources[image.index()].expect_image_info(); - self.cmd - .cmd_mut() - .expect_last_exec_mut() - .depth_stencil_resolve = Some(( - Attachment::new(image_view_info, sample_count, node_idx), + self.set_depth_stencil_resolve( dst_attachment_idx, + Attachment::new(image_view_info, sample_count, node_idx), depth_mode, stencil_mode, - )); + ); let mut image_access = if image_view_info .aspect_mask @@ -1594,7 +1390,7 @@ impl ClearColorValue { /// All zeros. pub const BLACK_ALPHA_ZERO: Self = Self::Float32([0.0, 0.0, 0.0, 0.0]); - /// RGB zeros and alpha ones. + /// RGB ones and alpha ones. pub const WHITE_ALPHA_ONE: Self = Self::Float32([1.0, 1.0, 1.0, 1.0]); /// RGB ones and alpha zeros. @@ -1677,7 +1473,7 @@ impl From for vk::ClearColorValue { /// # use vk_graph::cmd::{LoadOp, StoreOp}; /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; -/// # use vk_graph::driver::graphic::{GraphicsPipeline, GraphicsPipelineInfo}; +/// # use vk_graph::driver::graphics::{GraphicsPipeline, GraphicsPipelineInfo}; /// # use vk_graph::driver::image::{Image, ImageInfo}; /// # use vk_graph::Graph; /// # use vk_graph::driver::shader::Shader; @@ -1730,7 +1526,7 @@ impl GraphicsCommandRef<'_> { /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; /// # use vk_graph::driver::buffer::{Buffer, BufferInfo}; - /// # use vk_graph::driver::graphic::{GraphicsPipeline, GraphicsPipelineInfo}; + /// # use vk_graph::driver::graphics::{GraphicsPipeline, GraphicsPipelineInfo}; /// # use vk_graph::driver::image::{Image, ImageInfo}; /// # use vk_graph::driver::shader::Shader; /// # use vk_graph::Graph; @@ -1758,7 +1554,7 @@ impl GraphicsCommandRef<'_> { /// # let my_vtx_buf = my_graph.bind_resource(my_vtx_buf); /// my_graph /// .begin_cmd() - /// .debug_name("my indexed geometry draw pass") + /// .debug_name("my indexed geometry draw command") /// .bind_pipeline(&my_graphic_pipeline) /// .color_attachment_image(0, swapchain_image, LoadOp::DontCare, StoreOp::Store) /// .resource_access(my_idx_buf, AccessType::IndexBuffer) @@ -1790,7 +1586,7 @@ impl GraphicsCommandRef<'_> { self } - /// Bind a vertex buffer to the current pass. + /// Binds a vertex buffer for the current graphics command. /// /// The vertex input binding is updated to start at `offset` from the start of `buffer`. /// @@ -1805,7 +1601,7 @@ impl GraphicsCommandRef<'_> { /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; /// # use vk_graph::driver::buffer::{Buffer, BufferInfo}; - /// # use vk_graph::driver::graphic::{GraphicsPipeline, GraphicsPipelineInfo}; + /// # use vk_graph::driver::graphics::{GraphicsPipeline, GraphicsPipelineInfo}; /// # use vk_graph::driver::image::{Image, ImageInfo}; /// # use vk_graph::driver::shader::Shader; /// # use vk_graph::Graph; @@ -1830,7 +1626,7 @@ impl GraphicsCommandRef<'_> { /// # let my_vtx_buf = my_graph.bind_resource(my_vtx_buf); /// my_graph /// .begin_cmd() - /// .debug_name("my unindexed geometry draw pass") + /// .debug_name("my unindexed geometry draw command") /// .bind_pipeline(&my_graphic_pipeline) /// .color_attachment_image(0, swapchain_image, LoadOp::DontCare, StoreOp::Store) /// .resource_access(my_vtx_buf, AccessType::VertexBuffer) @@ -1863,7 +1659,8 @@ impl GraphicsCommandRef<'_> { self } - /// Binds multiple vertex buffers to the current pass, starting at the given `first_binding`. + /// Binds multiple vertex buffers for the current graphics command, starting at the given + /// `first_binding`. /// /// Each vertex input binding in `buffers` specifies an offset from the start of the /// corresponding buffer. @@ -1992,7 +1789,7 @@ impl GraphicsCommandRef<'_> { /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; /// # use vk_graph::driver::buffer::{Buffer, BufferInfo}; - /// # use vk_graph::driver::graphic::{GraphicsPipeline, GraphicsPipelineInfo}; + /// # use vk_graph::driver::graphics::{GraphicsPipeline, GraphicsPipelineInfo}; /// # use vk_graph::driver::image::{Image, ImageInfo}; /// # use vk_graph::driver::shader::Shader; /// # use vk_graph::Graph; @@ -2181,7 +1978,7 @@ impl GraphicsCommandRef<'_> { /// Updates push constants. /// - /// Push constants represent a high speed path to modify constant data in pipelines that is + /// Push constants represent a high-speed path to modify constant data in pipelines that is /// expected to outperform memory-backed resource updates. /// /// Push constant values can be updated incrementally, causing shader stages to read the new @@ -2190,9 +1987,8 @@ impl GraphicsCommandRef<'_> { /// /// # Device limitations /// - /// See - /// [`device.physical_device.props.limits.max_push_constants_size`](vk::PhysicalDeviceLimits) - /// for the limits of the current device. You may also check [gpuinfo.org] for a listing of + /// See [`VkPhysicalDeviceLimits::maxPushConstantsSize`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceLimits.html) + /// for the limit of the current device. You may also check [gpuinfo.org] for a listing of /// reported limits on other devices. /// /// # Examples @@ -2202,14 +1998,14 @@ impl GraphicsCommandRef<'_> { /// ``` /// # vk_shader_macros::glsl!(r#" /// #version 450 - /// #pragma shader_stage(compute) + /// #pragma shader_stage(fragment) /// /// layout(push_constant) uniform PushConstants { /// layout(offset = 0) uint the_answer; /// } push_constants; /// /// void main() { - /// // TODO: Add code! + /// uint value = push_constants.the_answer; /// } /// # "#); /// ``` @@ -2219,7 +2015,7 @@ impl GraphicsCommandRef<'_> { /// # use vk_graph::cmd::{LoadOp, StoreOp}; /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::graphic::{GraphicsPipeline, GraphicsPipelineInfo}; + /// # use vk_graph::driver::graphics::{GraphicsPipeline, GraphicsPipelineInfo}; /// # use vk_graph::driver::image::{Image, ImageInfo}; /// # use vk_graph::Graph; /// # use vk_graph::driver::shader::Shader; @@ -2266,9 +2062,11 @@ impl GraphicsCommandRef<'_> { self } - /// Set scissor rectangle dynamically for the current command. + /// Sets scissor rectangles dynamically for the current command. /// /// The default scissor state is no-clip. + /// + /// See [`vkCmdSetScissor`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdSetScissor.html). #[profiling::function] pub fn set_scissor(&self, first_scissor: u32, scissors: &[vk::Rect2D]) -> &Self { unsafe { @@ -2280,10 +2078,12 @@ impl GraphicsCommandRef<'_> { self } - /// Set the viewport dynamically for the current command. + /// Sets viewports dynamically for the current command. /// /// The default viewport state is the entire render target as defined by all combined image /// attachments. + /// + /// See [`vkCmdSetViewport`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdSetViewport.html). #[profiling::function] pub fn set_viewport(&self, first_viewport: u32, viewports: &[vk::Viewport]) -> &Self { unsafe { @@ -2304,36 +2104,17 @@ impl<'a> Deref for GraphicsCommandRef<'a> { } } -/// Specifies the state of a color or combined depth and stencil attachment image during graphic -/// render pass framebuffer load operations. -/// -/// Use this to specify the desired contents of any image before use in a pipeline command buffer. -#[derive(Clone, Copy, Debug)] -pub enum LoadOp { - /// Clears the attachment. - /// - /// `T` will be [ClearColorValue] for color images or [vk::ClearDepthStencilValue] for - /// combined depth and stencil images. - Clear(T), - - /// The attachment will become undefined and reads will produce garbage data. - DontCare, - - /// The attachment will be preserved in memory. - Load, -} - impl LoadOp { - /// A load operation which results in a color attachment filled with rgb zeros and alpha ones. + /// A load operation which results in a color attachment filled with RGB zeros and alpha ones. pub const CLEAR_BLACK_ALPHA_ONE: Self = Self::Clear(ClearColorValue::BLACK_ALPHA_ONE); /// A load operation which results in a color attachment filled with zeros. pub const CLEAR_BLACK_ALPHA_ZERO: Self = Self::Clear(ClearColorValue::BLACK_ALPHA_ZERO); - /// A load operation which results in a color attachment filled with rgb zeros and alpha ones. + /// A load operation which results in a color attachment filled with RGB ones and alpha ones. pub const CLEAR_WHITE_ALPHA_ONE: Self = Self::Clear(ClearColorValue::WHITE_ALPHA_ONE); - /// A load operation which results in a color attachment filled with rgb ones and alpha zeros. + /// A load operation which results in a color attachment filled with RGB ones and alpha zeros. pub const CLEAR_WHITE_ALPHA_ZERO: Self = Self::Clear(ClearColorValue::WHITE_ALPHA_ZERO); /// Convenience constructor for clear color values. @@ -2361,16 +2142,3 @@ impl LoadOp { Self::Clear(vk::ClearDepthStencilValue { depth, stencil }) } } - -/// Specifies the state of a color or combined depth and stencil attachment image after graphic -/// render pass framebuffer store operations. -/// -/// Use this to specify the desired contents of any image after use in a pipeline command buffer. -#[derive(Clone, Copy, Debug)] -pub enum StoreOp { - /// The attachment will become undefined and reads will produce garbage data. - DontCare, - - /// The attachment will be preserved in memory. - Store, -} diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 08939d55..8213cc6c 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -10,7 +10,7 @@ //! 3. With a pipeline, declare shader bindings with [`PipelineCommand::shader_resource_access`]. //! 4. Record work with [`record_cmd`](Command::record_cmd) — available on both //! [`Command`] and [`PipelineCommand`]. -//! 5. The command auto-closes when dropped or when [`Graph::into_submission`] is called. +//! 5. The command auto-closes when dropped or when [`Graph::finalize`] is called. //! //! A single command can call `record_cmd` multiple times — each call creates a separate //! "execution" within that command. Executions within a command stay in the specified @@ -19,31 +19,38 @@ mod cmd_ref; mod compute; -mod graphic; +mod graphics; mod pipeline; -mod ray_trace; - -pub use self::{ - cmd_ref::{ - BuildAccelerationStructureIndirectInfo, BuildAccelerationStructureInfo, CommandRef, - UpdateAccelerationStructureIndirectInfo, UpdateAccelerationStructureInfo, +mod ray_tracing; + +pub use { + self::{ + cmd_ref::{ + BuildAccelerationStructureIndirectInfo, BuildAccelerationStructureInfo, CommandRef, + UpdateAccelerationStructureIndirectInfo, UpdateAccelerationStructureInfo, + }, + compute::ComputeCommandRef, + graphics::{ClearColorValue, GraphicsCommandRef}, + pipeline::{Pipeline, PipelineCommand}, + ray_tracing::RayTracingCommandRef, }, - compute::ComputeCommandRef, - graphic::{ClearColorValue, GraphicsCommandRef, LoadOp, StoreOp}, - pipeline::{Pipeline, PipelineCommand}, - ray_trace::RayTracingCommandRef, + super::{LoadOp, StoreOp}, }; use { super::{ AccelerationStructureLeaseNode, AccelerationStructureNode, AnyAccelerationStructureNode, AnyBufferNode, AnyImageNode, AnyResource, BufferLeaseNode, BufferNode, CommandData, - Execution, ExecutionFunction, Graph, ImageLeaseNode, ImageNode, Node, Resource, + CommandFunction, Execution, Graph, ImageLeaseNode, ImageNode, Node, Resource, SwapchainImageNode, }, - crate::driver::{buffer::BufferSubresourceRange, image::ImageViewInfo}, + crate::{ + NodeIndex, + driver::{buffer::BufferSubresourceRange, image::ImageViewInfo}, + stream::{AccelerationStructureArg, BufferArg, ImageArg}, + }, ash::vk, - std::ops::Range, + std::{ops::Range, sync::Arc}, vk_sync::AccessType, }; @@ -78,14 +85,15 @@ pub struct Command<'a> { pub(super) graph: &'a mut Graph, } +#[allow(private_bounds)] impl<'a> Command<'a> { pub(super) fn new(graph: &'a mut Graph) -> Self { let cmd_idx = graph.cmds.len(); graph.cmds.push(CommandData { execs: vec![Default::default()], // We start off with a default execution! - #[cfg(debug_assertions)] name: None, + stream_scope_id: None, }); Self { @@ -117,14 +125,11 @@ impl<'a> Command<'a> { /// Binds a shader pipeline to the current command, allowing for strongly typed access to the /// related functions. /// - /// `P`|`P::Command` - /// -|- - /// [`ComputePipeline`](crate::driver::compute::ComputePipeline)|[`PipelineCommand<'_, - /// ComputePipeline>`] - /// [`GraphicsPipeline`](crate::driver::graphic::GraphicsPipeline)|[`PipelineCommand<'_, - /// GraphicsPipeline>`] - /// [`RayTracingPipeline`](crate::driver::ray_trace::RayTracingPipeline)|[`PipelineCommand<'_, - /// RayTracingPipeline>`] + /// | `P` | `P::Command` | + /// | --- | --- | + /// | [`ComputePipeline`](crate::driver::compute::ComputePipeline) | [`PipelineCommand<'_, ComputePipeline>`] | + /// | [`GraphicsPipeline`](crate::driver::graphics::GraphicsPipeline) | [`PipelineCommand<'_, GraphicsPipeline>`] | + /// | [`RayTracingPipeline`](crate::driver::ray_tracing::RayTracingPipeline) | [`PipelineCommand<'_, RayTracingPipeline>`] | pub fn bind_pipeline

(self, pipeline: P) -> P::Command where P: Pipeline<'a>, @@ -132,7 +137,7 @@ impl<'a> Command<'a> { pipeline.bind_cmd(self) } - /// Sets a debugging name, but only in debug builds + /// Sets a debugging name, but only in debug builds. pub fn debug_name(mut self, name: impl Into) -> Self { self.set_debug_name(name); self @@ -141,7 +146,7 @@ impl<'a> Command<'a> { /// Finalize the recording of this command and return to the `Graph` where you may record /// additional commands. pub fn end_cmd(self) -> &'a mut Graph { - // If nothing was done in this pass we can just ignore it + // If nothing was done in this command we can just ignore it. if self.exec_idx == 0 { self.graph.cmds.pop(); } @@ -153,7 +158,26 @@ impl<'a> Command<'a> { let cmd = self.cmd_mut(); let exec = { let last_exec = cmd.expect_last_exec_mut(); - last_exec.func = Some(ExecutionFunction(Box::new(func))); + last_exec.func = Some(CommandFunction::Once(Box::new(func))); + + Execution { + pipeline: last_exec.pipeline.clone(), + ..Default::default() + } + }; + + cmd.execs.push(exec); + self.exec_idx += 1; + } + + pub(crate) fn push_reusable_exec( + &mut self, + func: impl for<'r> Fn(CommandRef<'r>) + Send + Sync + 'static, + ) { + let cmd = self.cmd_mut(); + let exec = { + let last_exec = cmd.expect_last_exec_mut(); + last_exec.func = Some(CommandFunction::Reusable(Arc::new(func))); Execution { pipeline: last_exec.pipeline.clone(), @@ -175,21 +199,27 @@ impl<'a> Command<'a> { let node_idx = resource_node.index(); + self.push_subresource_access_index(node_idx, subresource, access); + } + + pub(crate) fn push_subresource_access_index( + &mut self, + node_idx: NodeIndex, + subresource: SubresourceRange, + access: AccessType, + ) { debug_assert!(self.graph.resources.get(node_idx).is_some()); - let access = SubresourceAccess { - access, - subresource, - }; - self.cmd_mut() - .expect_last_exec_mut() - .accesses - .entry(node_idx) - .and_modify(|accesses| accesses.push(access)) - .or_insert(vec![access]); + self.cmd_mut().expect_last_exec_mut().accesses.push( + node_idx, + SubresourceAccess { + access, + subresource, + }, + ); } - /// Begin recording a general-purpose command buffer. + /// Begin recording general-purpose work for this graph command. /// /// This is the entry point for building and updating an /// [`AccelerationStructure`](crate::driver::accel_struct::AccelerationStructure) instance. @@ -201,19 +231,28 @@ impl<'a> Command<'a> { self } - /// Begin recording a general-purpose command buffer. - /// - /// This is the entry point for building and updating an - /// [`AccelerationStructure`](crate::driver::accel_struct::AccelerationStructure) instance. - /// - /// The provided closure allows you to run any Vulkan code, or interoperate with other Vulkan - /// code and interfaces. + /// Mutable-borrow form of [`Self::record_cmd`]. pub fn record_cmd_mut(&mut self, func: impl FnOnce(CommandRef<'_>) + Send + 'static) { self.push_exec(move |cmd| { func(cmd); }); } + pub(crate) fn record_stream( + mut self, + func: impl for<'r> Fn(CommandRef<'r>) + Send + Sync + 'static, + ) -> Self { + self.record_stream_mut(func); + self + } + + pub(crate) fn record_stream_mut( + &mut self, + func: impl for<'r> Fn(CommandRef<'r>) + Send + Sync + 'static, + ) { + self.push_reusable_exec(func); + } + /// Returns a borrow of the original Vulkan resource (buffer, image or acceleration structure) /// which the given bound resource node represents. pub fn resource(&self, resource_node: N) -> &N::Resource @@ -223,11 +262,11 @@ impl<'a> Command<'a> { self.graph.resource(resource_node) } - /// Informs the command that the next recorded command buffer will read or write `resource_node` + /// Informs the command that recorded work will read or write `resource_node` /// using `access`. /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// An access function must be called for `resource_node` before it is used within a recording + /// function. pub fn resource_access(mut self, resource_node: N, access: AccessType) -> Self where N: Node + Subresource, @@ -237,21 +276,22 @@ impl<'a> Command<'a> { self } - /// Sets a debugging name, but only in debug builds. - pub fn set_debug_name(&mut self, _name: impl Into) -> &mut Self { + /// Mutable-borrow form of [`Self::debug_name`]. + pub fn set_debug_name(&mut self, name: impl Into) -> &mut Self { #[cfg(debug_assertions)] { - self.cmd_mut().name = Some(_name.into()); + self.cmd_mut().name = Some(name.into()); + } + + #[cfg(not(debug_assertions))] + { + let _ = name; } self } - /// Informs the command that the next recorded command buffer will read or write `resource_node` - /// using `access`. - /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// Mutable-borrow form of [`Self::resource_access`]. pub fn set_resource_access(&mut self, resource_node: N, access: AccessType) where N: Node + Subresource, @@ -263,11 +303,11 @@ impl<'a> Command<'a> { self.push_subresource_access(resource_node, subresource, access); } - /// Informs the command that the next recorded command buffer will read or write the - /// `subresource` of `resource_node` using `access`. - /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + pub(crate) fn set_stream_scope_id(&mut self, stream_scope_id: u64) { + self.cmd_mut().stream_scope_id = Some(stream_scope_id); + } + + /// Mutable-borrow form of [`Self::subresource_access`]. pub fn set_subresource_access( &mut self, resource_node: N, @@ -283,11 +323,11 @@ impl<'a> Command<'a> { self.push_subresource_access(resource_node, subresource, access); } - /// Informs the command that the next recorded command buffer will read or write the - /// `subresource` of `resource` using `access`. + /// Informs the command that recorded work will read or write the `subresource` of + /// `resource_node` using `access`. /// - /// An access function must be called for `resource` before it is used within a - /// `record_`-function. + /// An access function must be called for `resource_node` before it is used within a recording + /// function. pub fn subresource_access( mut self, resource_node: N, @@ -386,96 +426,96 @@ impl From<(DescriptorSetIndex, BindingIndex, [BindingOffset; 1])> for Binding { } /// Allows for a resource to be reinterpreted as differently formatted data. -pub trait Subresource { +#[allow(private_bounds)] +pub trait Subresource: private::SubresourceSealed { /// The information about the subresource when bound directly to shader descriptors. type Info; /// The information about the subresource when used indirectly by any part of a graph. type Range; - - #[doc(hidden)] - fn info(&self, _: &[AnyResource]) -> Self::Info - where - Self: Node; - - #[doc(hidden)] - fn range(&self, _: &[AnyResource]) -> Self::Range - where - Self: Node; } macro_rules! view_accel_struct { - ($name:ident) => { + ($name:ty) => { impl Subresource for $name { type Info = Self::Range; type Range = (); + } - fn info(&self, _: &[AnyResource]) -> Self::Info + impl private::SubresourceSealed for $name { + fn info(&self, _: &[AnyResource]) -> ::Info where - Self: Node, + Self: Node + Subresource, { } - fn range(&self, _: &[AnyResource]) -> Self::Range + fn range(&self, resources: &[AnyResource]) -> ::Range where - Self: Node, + Self: Node + Subresource, { + resources[self.index()].expect_accel_struct_info(); } } }; } view_accel_struct!(AnyAccelerationStructureNode); +view_accel_struct!(AccelerationStructureArg); view_accel_struct!(AccelerationStructureLeaseNode); view_accel_struct!(AccelerationStructureNode); macro_rules! view_buffer { - ($name:ident) => { + ($name:ty) => { impl Subresource for $name { type Info = Self::Range; type Range = BufferSubresourceRange; + } - fn info(&self, resources: &[AnyResource]) -> Self::Info + impl private::SubresourceSealed for $name { + fn info(&self, resources: &[AnyResource]) -> ::Info where - Self: Node, + Self: Node + Subresource, { self.range(resources) } - fn range(&self, resources: &[AnyResource]) -> Self::Range + fn range(&self, resources: &[AnyResource]) -> ::Range where - Self: Node, + Self: Node + Subresource, { let idx = self.index(); - resources[idx].expect_buffer().info.into() + resources[idx].expect_buffer_info().into() } } }; } view_buffer!(AnyBufferNode); +view_buffer!(BufferArg); view_buffer!(BufferLeaseNode); view_buffer!(BufferNode); macro_rules! view_image { - ($name:ident) => { + ($name:ty) => { impl Subresource for $name { type Info = ImageViewInfo; type Range = vk::ImageSubresourceRange; + } - fn info(&self, resources: &[AnyResource]) -> Self::Info + impl private::SubresourceSealed for $name { + fn info(&self, resources: &[AnyResource]) -> ::Info where - Self: Node, + Self: Node + Subresource, { let idx = self.index(); - resources[idx].expect_image().info.into() + resources[idx].expect_image_info().into() } - fn range(&self, resources: &[AnyResource]) -> Self::Range + fn range(&self, resources: &[AnyResource]) -> ::Range where - Self: Node, + Self: Node + Subresource, { self.info(resources).into() } @@ -484,13 +524,13 @@ macro_rules! view_image { } view_image!(AnyImageNode); +view_image!(ImageArg); view_image!(ImageLeaseNode); view_image!(ImageNode); view_image!(SwapchainImageNode); #[derive(Clone, Copy, Debug)] -#[doc(hidden)] -pub enum SubresourceRange { +pub(crate) enum SubresourceRange { /// Acceleration structures are bound whole. AccelerationStructure, @@ -547,8 +587,7 @@ pub(super) struct SubresourceAccess { /// Describes the interpretation of a resource. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[doc(hidden)] -pub enum ViewInfo { +pub(crate) enum ViewInfo { /// Acceleration structures are always whole resources. AccelerationStructure, @@ -609,3 +648,17 @@ impl From> for ViewInfo { }) } } + +mod private { + use crate::{AnyResource, Node}; + + pub(crate) trait SubresourceSealed: Sized { + fn info(&self, resources: &[AnyResource]) -> ::Info + where + Self: Node + super::Subresource; + + fn range(&self, resources: &[AnyResource]) -> ::Range + where + Self: Node + super::Subresource; + } +} diff --git a/src/cmd/pipeline.rs b/src/cmd/pipeline.rs index 8d605e61..b92c3d31 100644 --- a/src/cmd/pipeline.rs +++ b/src/cmd/pipeline.rs @@ -6,7 +6,7 @@ use { crate::{ ExecutionPipeline, driver::{ - compute::ComputePipeline, graphic::GraphicsPipeline, ray_trace::RayTracingPipeline, + compute::ComputePipeline, graphics::GraphicsPipeline, ray_tracing::RayTracingPipeline, }, }, std::marker::PhantomData, @@ -91,8 +91,13 @@ macro_rules! pipeline { // Pipelines you can bind to a command ref pipeline!(Compute, ComputePipeline, is_compute, unwrap_compute); -pipeline!(Graphic, GraphicsPipeline, is_graphic, unwrap_graphic); -pipeline!(RayTrace, RayTracingPipeline, is_ray_trace, unwrap_ray_trace); +pipeline!(Graphics, GraphicsPipeline, is_graphics, unwrap_graphics); +pipeline!( + RayTracing, + RayTracingPipeline, + is_ray_tracing, + unwrap_ray_tracing +); /// A [`Command`] which has been bound to a particular compute, graphics, or ray tracing pipeline. pub struct PipelineCommand<'c, T> { @@ -100,19 +105,10 @@ pub struct PipelineCommand<'c, T> { pub(super) cmd: Command<'c>, } -// NOTE: There are specific implementations of T in the compute, graphics, and ray tracing modules. +// NOTE: There are specific implementations of T in the compute, graphics, and ray tracing modules +#[allow(private_bounds)] impl<'c, T> PipelineCommand<'c, T> { - /// Binds a shader pipeline to the current command, allowing for strongly typed access to the - /// related functions. - /// - /// `P`|`P::Command` - /// -|- - /// [`ComputePipeline`](crate::driver::compute::ComputePipeline)|[`PipelineCommand<'_, - /// ComputePipeline>`] - /// [`GraphicsPipeline`](crate::driver::graphic::GraphicsPipeline)|[`PipelineCommand<'_, - /// GraphicsPipeline>`] - /// [`RayTracingPipeline`](crate::driver::ray_trace::RayTracingPipeline)|[`PipelineCommand<'_, - /// RayTracingPipeline>`] + /// Equivalent to [`Command::bind_pipeline`] for a command that already has a bound pipeline. pub fn bind_pipeline

(self, pipeline: P) -> P::Command where P: Pipeline<'c>, @@ -120,10 +116,7 @@ impl<'c, T> PipelineCommand<'c, T> { pipeline.bind_cmd(self.cmd) } - /// Binds a Vulkan buffer, image, or acceleration structure resource to the graph associated - /// with this command. - /// - /// Bound nodes may be used in passes for pipeline and shader operations. + /// Equivalent to [`Command::bind_resource`] for a command that already has a bound pipeline. pub fn bind_resource(&mut self, resource: R) -> R::Node where R: Resource, @@ -131,13 +124,12 @@ impl<'c, T> PipelineCommand<'c, T> { self.cmd.bind_resource(resource) } - /// Finalizes a command and returns the graph so that additional commands may be added. + /// Equivalent to [`Command::end_cmd`] for a command that already has a bound pipeline. pub fn end_cmd(self) -> &'c mut Graph { self.cmd.end_cmd() } - /// Returns a borrow of the original Vulkan resource (buffer, image or acceleration structure) - /// which the given bound resource node represents. + /// Equivalent to [`Command::resource`] for a command that already has a bound pipeline. pub fn resource(&self, resource_node: N) -> &N::Resource where N: Node, @@ -145,11 +137,10 @@ impl<'c, T> PipelineCommand<'c, T> { self.cmd.resource(resource_node) } - /// Informs the command that the next recorded command buffer will read or write `resource_node` - /// using `access`. + /// Informs the command that recorded work will read or write `resource_node` using `access`. /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// An access function must be called for `resource_node` before it is used within a recording + /// function. pub fn resource_access(mut self, resource_node: N, access: AccessType) -> Self where N: Node + Subresource, @@ -159,11 +150,7 @@ impl<'c, T> PipelineCommand<'c, T> { self } - /// Informs the command that the next recorded command buffer will read or write `resource_node` - /// using `access`. - /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// Mutable-borrow form of [`Self::resource_access`]. pub fn set_resource_access(&mut self, resource_node: N, access: AccessType) -> &mut Self where N: Node + Subresource, @@ -173,11 +160,7 @@ impl<'c, T> PipelineCommand<'c, T> { self } - /// Informs the command that the next recorded command buffer will read or write the - /// `resource_node` at the specified shader `binding` using `access`. - /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// Mutable-borrow form of [`Self::shader_resource_access`]. pub fn set_shader_resource_access( &mut self, binding: impl Into, @@ -195,15 +178,7 @@ impl<'c, T> PipelineCommand<'c, T> { self.set_shader_subresource_access(binding, resource_node, subresource, access) } - /// Informs the command that the next recorded command buffer will read or write the - /// `resource_node` at the specified shader `binding` using `access`. The resource will be - /// interpreted using `view_info`. - /// - /// If the same `binding` slot is used more than once, the last call wins and the - /// previous binding is silently overwritten. - /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// Mutable-borrow form of [`Self::shader_subresource_access`]. pub fn set_shader_subresource_access( &mut self, binding: impl Into, @@ -247,11 +222,7 @@ impl<'c, T> PipelineCommand<'c, T> { self } - /// Informs the command that the next recorded command buffer will read or write the - /// `subresource` of `resource_node` using `access`. - /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// Mutable-borrow form of [`Self::subresource_access`]. pub fn set_subresource_access( &mut self, resource_node: N, @@ -267,14 +238,14 @@ impl<'c, T> PipelineCommand<'c, T> { self } - /// Informs the command that the next recorded command buffer will read or write the - /// `resource_node` at the specified shader `binding` using `access`. + /// Informs the command that recorded work will read or write the `resource_node` at the + /// specified shader `binding` using `access`. /// - /// If the same `binding` slot is used more than once, the last call wins and the - /// previous binding is silently overwritten. + /// If the same `binding` slot is used more than once, the last call wins and the previous + /// binding is silently overwritten. /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// An access function must be called for `resource_node` before it is used within a recording + /// function. pub fn shader_resource_access( mut self, binding: impl Into, @@ -291,12 +262,15 @@ impl<'c, T> PipelineCommand<'c, T> { self } - /// Informs the command that the next recorded command buffer will read or write the - /// `resource_node` at the specified shader `binding` using `access`. The resource will be - /// interpreted using `view_info`. + /// Informs the command that recorded work will read or write the `resource_node` at the + /// specified shader `binding` using `access`. The resource will be interpreted using + /// `view_info`. + /// + /// If the same `binding` slot is used more than once, the last call wins and the previous + /// binding is silently overwritten. /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// An access function must be called for `resource_node` before it is used within a recording + /// function. pub fn shader_subresource_access( mut self, binding: impl Into, @@ -314,11 +288,11 @@ impl<'c, T> PipelineCommand<'c, T> { self } - /// Informs the command that the next recorded command buffer will read or write the - /// `subresource` of `resource_node` using `access`. + /// Informs the command that recorded work will read or write the `subresource` of + /// `resource_node` using `access`. /// - /// An access function must be called for `resource_node` before it is used within a - /// `record_`-function. + /// An access function must be called for `resource_node` before it is used within a recording + /// function. pub fn subresource_access( mut self, resource_node: N, diff --git a/src/cmd/ray_trace.rs b/src/cmd/ray_tracing.rs similarity index 79% rename from src/cmd/ray_trace.rs rename to src/cmd/ray_tracing.rs index 6895eec8..898bc08a 100644 --- a/src/cmd/ray_trace.rs +++ b/src/cmd/ray_tracing.rs @@ -1,12 +1,12 @@ use { super::{PipelineCommand, cmd_ref::CommandRef}, - crate::driver::{device::Device, ray_trace::RayTracingPipeline}, + crate::driver::{device::Device, ray_tracing::RayTracingPipeline}, ash::vk, std::ops::Deref, }; impl PipelineCommand<'_, RayTracingPipeline> { - /// Begin recording a ray tracing pipeline command buffer. + /// Begin recording ray tracing pipeline work for this graph command. pub fn record_cmd( mut self, func: impl FnOnce(RayTracingCommandRef<'_>) + Send + 'static, @@ -15,19 +15,38 @@ impl PipelineCommand<'_, RayTracingPipeline> { self } - /// Begin recording a ray tracing pipeline command buffer. + /// Mutable-borrow form of [`Self::record_cmd`]. pub fn record_cmd_mut(&mut self, func: impl FnOnce(RayTracingCommandRef<'_>) + Send + 'static) { let pipeline = self .cmd .cmd() .expect_last_pipeline() - .expect_ray_trace() + .expect_ray_tracing() .clone(); self.cmd.push_exec(move |cmd| { func(RayTracingCommandRef { cmd, pipeline }); }); } + + pub(crate) fn record_stream_mut( + &mut self, + func: impl for<'r> Fn(RayTracingCommandRef<'r>) + Send + Sync + 'static, + ) { + let pipeline = self + .cmd + .cmd() + .expect_last_pipeline() + .expect_ray_tracing() + .clone(); + + self.cmd.push_reusable_exec(move |cmd| { + func(RayTracingCommandRef { + cmd, + pipeline: pipeline.clone(), + }); + }); + } } /// Recording interface for ray tracing commands. @@ -44,7 +63,7 @@ impl PipelineCommand<'_, RayTracingPipeline> { /// # use ash::vk; /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; -/// # use vk_graph::driver::ray_trace::{ +/// # use vk_graph::driver::ray_tracing::{ /// # RayTracingPipeline, /// # RayTracingPipelineInfo, /// # RayTracingShaderGroup, @@ -55,14 +74,14 @@ impl PipelineCommand<'_, RayTracingPipeline> { /// # let device = Device::create(DeviceInfo::default())?; /// # let info = RayTracingPipelineInfo::default(); /// # let my_miss_code = [0u8; 1]; -/// # let my_ray_trace_pipeline = RayTracingPipeline::create(&device, info, +/// # let my_ray_tracing_pipeline = RayTracingPipeline::create(&device, info, /// # [Shader::new_miss(my_miss_code.as_slice())], /// # [RayTracingShaderGroup::new_general(0)], /// # )?; /// # let mut my_graph = Graph::default(); /// my_graph.begin_cmd() /// .debug_name("my ray tracing command") -/// .bind_pipeline(&my_ray_trace_pipeline) +/// .bind_pipeline(&my_ray_tracing_pipeline) /// .record_cmd(move |cmd| { /// // During this closure we have access to the ray tracing functions! /// }); @@ -76,7 +95,7 @@ pub struct RayTracingCommandRef<'a> { impl RayTracingCommandRef<'_> { /// Updates push constants. /// - /// Push constants represent a high speed path to modify constant data in pipelines that is + /// Push constants represent a high-speed path to modify constant data in pipelines that is /// expected to outperform memory-backed resource updates. /// /// Push constant values can be updated incrementally, causing shader stages to read the new @@ -85,9 +104,8 @@ impl RayTracingCommandRef<'_> { /// /// # Device limitations /// - /// See - /// [`device.physical_device.props.limits.max_push_constants_size`](vk::PhysicalDeviceLimits) - /// for the limits of the current device. You may also check [gpuinfo.org] for a listing of + /// See [`VkPhysicalDeviceLimits::maxPushConstantsSize`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceLimits.html) + /// for the limit of the current device. You may also check [gpuinfo.org] for a listing of /// reported limits on other devices. /// /// # Examples @@ -104,7 +122,7 @@ impl RayTracingCommandRef<'_> { /// } push_constants; /// /// void main() { - /// // TODO: Add bindings to write things! + /// uint value = push_constants.some_val; /// } /// # "#); /// ``` @@ -114,7 +132,7 @@ impl RayTracingCommandRef<'_> { /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; /// # use vk_graph::driver::buffer::{Buffer, BufferInfo}; - /// # use vk_graph::driver::ray_trace::{ + /// # use vk_graph::driver::ray_tracing::{ /// # RayTracingPipeline, /// # RayTracingPipelineInfo, /// # RayTracingShaderGroup, @@ -126,7 +144,7 @@ impl RayTracingCommandRef<'_> { /// # let shader = [0u8; 1]; /// # let info = RayTracingPipelineInfo::default(); /// # let my_miss_code = [0u8; 1]; - /// # let my_ray_trace_pipeline = RayTracingPipeline::create(&device, info, + /// # let my_ray_tracing_pipeline = RayTracingPipeline::create(&device, info, /// # [Shader::new_miss(my_miss_code.as_slice())], /// # [RayTracingShaderGroup::new_general(0)], /// # )?; @@ -153,7 +171,7 @@ impl RayTracingCommandRef<'_> { /// # let mut my_graph = Graph::default(); /// my_graph.begin_cmd() /// .debug_name("draw a cornell box") - /// .bind_pipeline(&my_ray_trace_pipeline) + /// .bind_pipeline(&my_ray_tracing_pipeline) /// .record_cmd(move |cmd| { /// cmd.push_constants(0, &[0xcb]) /// .trace_rays(&rgen_sbt, &hit_sbt, &miss_sbt, &call_sbt, 320, 200, 1); @@ -174,10 +192,13 @@ impl RayTracingCommandRef<'_> { self } - /// Set the stack size dynamically for a ray tracing pipeline. + /// Sets the stack size dynamically for a ray tracing pipeline. /// - /// See - /// `RayTracingPipelineInfo::dynamic_stack_size` and see the Vulkan spec. + /// The pipeline must have been created with + /// [`RayTracingPipelineInfo::dynamic_stack_size`](crate::driver::ray_tracing::RayTracingPipelineInfo::dynamic_stack_size) + /// enabled. + /// + /// See [`vkCmdSetRayTracingPipelineStackSizeKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdSetRayTracingPipelineStackSizeKHR.html). #[profiling::function] pub fn set_stack_size(&self, pipeline_stack_size: u32) -> &Self { let khr_ray_tracing_pipeline = Device::expect_vk_khr_ray_tracing_pipeline(&self.cmd.device); @@ -189,8 +210,10 @@ impl RayTracingCommandRef<'_> { ); unsafe { - // Checked mode catches missing dynamic_stack_size enablement early. Other Vulkan - // validation remains the responsibility of the validation layer. + /* + Checked mode catches missing dynamic_stack_size enablement early. Other Vulkan + validation remains the responsibility of the validation layer. + */ khr_ray_tracing_pipeline .cmd_set_ray_tracing_pipeline_stack_size(self.cmd.handle, pipeline_stack_size); } @@ -198,15 +221,19 @@ impl RayTracingCommandRef<'_> { self } - // TODO: If the rayTraversalPrimitiveCulling or rayQuery features are enabled, the - // SkipTrianglesKHR and SkipAABBsKHR ray flags can be specified when tracing a ray. - // SkipTrianglesKHR and SkipAABBsKHR are mutually exclusive. + /* + TODO: If the rayTraversalPrimitiveCulling or rayQuery features are enabled, the SkipTrianglesKHR + and SkipAABBsKHR ray flags can be specified when tracing a ray. SkipTrianglesKHR and + SkipAABBsKHR are mutually exclusive. + */ /// Ray traces using the currently-bound [`RayTracingPipeline`] and the given shader binding /// tables. /// /// Shader binding tables must be constructed according to this [example]. /// + /// See [`vkCmdTraceRaysKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdTraceRaysKHR.html). + /// /// # Examples /// /// Basic usage: @@ -216,7 +243,7 @@ impl RayTracingCommandRef<'_> { /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; /// # use vk_graph::driver::buffer::{Buffer, BufferInfo}; - /// # use vk_graph::driver::ray_trace::{ + /// # use vk_graph::driver::ray_tracing::{ /// # RayTracingPipeline, /// # RayTracingPipelineInfo, /// # RayTracingShaderGroup, @@ -228,7 +255,7 @@ impl RayTracingCommandRef<'_> { /// # let shader = [0u8; 1]; /// # let info = RayTracingPipelineInfo::default(); /// # let my_miss_code = [0u8; 1]; - /// # let my_ray_trace_pipeline = RayTracingPipeline::create(&device, info, + /// # let my_ray_tracing_pipeline = RayTracingPipeline::create(&device, info, /// # [Shader::new_miss(my_miss_code.as_slice())], /// # [RayTracingShaderGroup::new_general(0)], /// # )?; @@ -255,14 +282,14 @@ impl RayTracingCommandRef<'_> { /// # let mut my_graph = Graph::default(); /// my_graph.begin_cmd() /// .debug_name("draw a cornell box") - /// .bind_pipeline(&my_ray_trace_pipeline) + /// .bind_pipeline(&my_ray_tracing_pipeline) /// .record_cmd(move |cmd| { /// cmd.trace_rays(&rgen_sbt, &hit_sbt, &miss_sbt, &call_sbt, 320, 200, 1); /// }); /// # Ok(()) } /// ``` /// - /// [example]: https://github.com/attackgoat/vk-graph/blob/master/examples/ray_trace.rs + /// [example]: https://github.com/attackgoat/vk-graph/blob/master/examples/ray_tracing.rs #[allow(clippy::too_many_arguments)] #[profiling::function] pub fn trace_rays( diff --git a/src/driver/accel_struct.rs b/src/driver/accel_struct.rs index 9e615209..9b91d80c 100644 --- a/src/driver/accel_struct.rs +++ b/src/driver/accel_struct.rs @@ -1,23 +1,72 @@ //! Acceleration structure resource types use { - super::{Buffer, BufferInfo, DriverError, device::Device}, + super::{Buffer, BufferInfo, DriverError, device::Device, pipeline_stage_access_flags}, ash::vk, derive_builder::Builder, log::warn, std::{ ffi::c_void, - mem::{replace, size_of_val}, + fmt::{Debug, Formatter}, + mem::size_of_val, thread::panicking, }, vk_sync::AccessType, }; #[cfg(feature = "parking_lot")] -use parking_lot::Mutex; +use parking_lot::{Mutex, MutexGuard}; #[cfg(not(feature = "parking_lot"))] -use std::sync::Mutex; +use std::sync::{Mutex, MutexGuard}; + +fn accel_struct_sync_flags_for_access( + access: AccessType, +) -> (vk::PipelineStageFlags, vk::AccessFlags) { + match access { + AccessType::VertexShaderReadOther => ( + vk::PipelineStageFlags::VERTEX_SHADER, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::TessellationControlShaderReadOther => ( + vk::PipelineStageFlags::TESSELLATION_CONTROL_SHADER, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::TessellationEvaluationShaderReadOther => ( + vk::PipelineStageFlags::TESSELLATION_EVALUATION_SHADER, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::GeometryShaderReadOther => ( + vk::PipelineStageFlags::GEOMETRY_SHADER, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::FragmentShaderReadOther => ( + vk::PipelineStageFlags::FRAGMENT_SHADER, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::ComputeShaderReadOther => ( + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::AnyShaderReadOther => ( + vk::PipelineStageFlags::ALL_COMMANDS, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::RayTracingShaderReadOther => ( + vk::PipelineStageFlags::RAY_TRACING_SHADER_KHR, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::MeshShaderReadOther => ( + vk::PipelineStageFlags::MESH_SHADER_EXT, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + AccessType::TaskShaderReadOther => ( + vk::PipelineStageFlags::TASK_SHADER_EXT, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + ), + _ => pipeline_stage_access_flags(access), + } +} /// Smart pointer handle to an [acceleration structure] object. /// @@ -40,10 +89,10 @@ use std::sync::Mutex; /// ``` /// /// See [`VkAccelerationStructureKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkAccelerationStructureKHR.html). -#[derive(Debug)] #[read_only::cast] pub struct AccelerationStructure { - access: Mutex, + // TODO: Replace with single atomicu8 + accesses: Mutex>, /// The native Vulkan resource handle of the buffer which supports this acceleration structure. /// @@ -62,9 +111,6 @@ pub struct AccelerationStructure { /// _Note:_ This field is read-only. #[readonly] pub info: AccelerationStructureInfo, - - /// A name for debugging purposes. - pub name: Option, } impl AccelerationStructure { @@ -95,7 +141,7 @@ impl AccelerationStructure { device: &Device, info: impl Into, ) -> Result { - debug_assert!(device.physical_device.accel_struct_properties.is_some()); + debug_assert!(device.physical.vk_khr_acceleration_structure.is_some()); let info = info.into(); @@ -110,7 +156,7 @@ impl AccelerationStructure { let handle = { let create_info = vk::AccelerationStructureCreateInfoKHR::default() - .ty(info.ty) + .ty(info.acceleration_structure_type) .buffer(buffer.handle) .size(info.size); @@ -140,71 +186,13 @@ impl AccelerationStructure { }; Ok(Self { - access: Mutex::new(AccessType::Nothing), + accesses: Mutex::new(vec![AccessType::Nothing]), buffer, handle, info, - name: None, }) } - /// Keeps track of some `next_access` which affects this object. - /// - /// Returns the previous access for which a pipeline barrier should be used to prevent data - /// corruption. - /// - /// # Note - /// - /// Used to maintain object state when passing a _vk-graph_-created - /// `vk::AccelerationStructureKHR` handle to external code such as [_Ash_] or [_Erupt_] - /// bindings. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ```no_run - /// # use std::sync::Arc; - /// # use ash::vk; - /// # use vk_sync::AccessType; - /// # use vk_graph::driver::DriverError; - /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::accel_struct::{AccelerationStructure, AccelerationStructureInfo}; - /// # fn main() -> Result<(), DriverError> { - /// # let device = Device::create(DeviceInfo::default())?; - /// # const SIZE: vk::DeviceSize = 1024; - /// # let info = AccelerationStructureInfo::blas(SIZE); - /// # let my_accel_struct = AccelerationStructure::create(&device, info)?; - /// // Initially we want to "Build Write" - /// let next = AccessType::AccelerationStructureBuildWrite; - /// let prev = AccelerationStructure::access(&my_accel_struct, next); - /// assert_eq!(prev, AccessType::Nothing); - /// - /// // External code may now "Build Write"; no barrier required - /// - /// // Subsequently we want to "Build Read" - /// let next = AccessType::AccelerationStructureBuildRead; - /// let prev = AccelerationStructure::access(&my_accel_struct, next); - /// assert_eq!(prev, AccessType::AccelerationStructureBuildWrite); - /// - /// // A barrier on "Build Write" before "Build Read" is required! - /// # Ok(()) } - /// ``` - /// - /// [_Ash_]: https://crates.io/crates/ash - /// [_Erupt_]: https://crates.io/crates/erupt - #[profiling::function] - pub fn access(&self, next_access: AccessType) -> AccessType { - self.with_access(|prev_access| replace(prev_access, next_access)) - } - - /// Sets the debugging name assigned to this acceleration structure. - pub fn debug_name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); - - self - } - /// Returns the device address of this object. /// /// # Examples @@ -248,6 +236,26 @@ impl AccelerationStructure { unsafe { from_raw_parts(instances.as_ptr() as *const _, size_of_val(instances)) } } + fn lock_accesses(&self) -> MutexGuard<'_, Vec> { + let accesses = self.accesses.lock(); + + #[cfg(not(feature = "parking_lot"))] + let accesses = accesses.expect("poisoned acceleration structure access lock"); + + accesses + } + + /// Sets the debugging name assigned to this acceleration structure. + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.buffer.device, self.handle, &name); + Device::try_set_private_data_object_name( + &self.buffer.device, + vk::ObjectType::ACCELERATION_STRUCTURE_KHR, + self.handle, + &name, + ); + } + /// Returns the size of some geometry info which is then used to create a new /// [AccelerationStructure] instance or update an existing instance. /// @@ -324,7 +332,7 @@ impl AccelerationStructure { } let info = vk::AccelerationStructureBuildGeometryInfoKHR::default() - .ty(info.ty) + .ty(info.acceleration_structure_type) .flags(info.flags) .geometries(&tls.geometries); let mut sizes = vk::AccelerationStructureBuildSizesInfoKHR::default(); @@ -347,15 +355,51 @@ impl AccelerationStructure { }) } - fn with_access(&self, f: impl FnOnce(&mut AccessType) -> R) -> R { - let access = self.access.lock(); + /// Keeps track of a `next_access` which affects this object. + /// + /// Returns previous accesses for which a pipeline barrier should be used to prevent data + /// corruption. + #[profiling::function] + pub(crate) fn swap_access( + &self, + next_access: AccessType, + ) -> impl Iterator + '_ { + AccessIter::one(self.lock_accesses(), next_access) + } + + pub(crate) fn swap_accesses( + &self, + next_accesses: &[AccessType], + ) -> impl Iterator + '_ { + AccessIter::new(self.lock_accesses(), next_accesses) + } - #[cfg(not(feature = "parking_lot"))] - let access = access.expect("poisoned acceleration structure access lock"); + /// Returns synchronization information for the acceleration structure's current accesses. + pub fn sync_info(&self) -> AccelerationStructureSyncInfo { + AccelerationStructureSyncInfo::from_accesses(self.lock_accesses().iter().copied()) + } + + /// Sets the debugging name assigned to this acceleration structure. + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); + + self + } +} + +impl Debug for AccelerationStructure { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(AccelerationStructure)); - let mut access = access; + if let Some(debug_name) = &Device::private_data_object_name( + &self.buffer.device, + vk::ObjectType::ACCELERATION_STRUCTURE_KHR, + self.handle, + ) { + res.field("debug_name", debug_name); + } - f(&mut access) + res.field("handle", &self.handle).finish_non_exhaustive() } } @@ -366,6 +410,12 @@ impl Drop for AccelerationStructure { return; } + Device::try_clear_private_data_object_name( + &self.buffer.device, + vk::ObjectType::ACCELERATION_STRUCTURE_KHR, + self.handle, + ); + let khr_acceleration_structure = Device::expect_vk_khr_acceleration_structure(&self.buffer.device); @@ -452,7 +502,6 @@ impl From for vk::AccelerationStructureGeometryKH pub enum AccelerationStructureGeometryData { /// Axis-aligned bounding box geometry in a bottom-level acceleration structure. /// - /// See /// See [`VkAccelerationStructureGeometryAabbsDataKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkAccelerationStructureGeometryAabbsDataKHR.html). AABBs { /// A device or host address to memory containing [vk::AabbPositionsKHR] structures @@ -469,14 +518,13 @@ pub enum AccelerationStructureGeometryData { /// /// See [`VkAccelerationStructureGeometryInstancesDataKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkAccelerationStructureGeometryInstancesDataKHR.html). Instances { - /// Either the address of an array of device referencing individual - /// VkAccelerationStructureInstanceKHR structures or packed motion instance information as - /// described in - /// See [`VkAccelerationStructureInstanceKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkAccelerationStructureInstanceKHR.html). - /// if `array_of_pointers` is `true`, or the address of an array of - /// VkAccelerationStructureInstanceKHR structures. + /// Either the address of an array of device addresses referencing individual + /// [`VkAccelerationStructureInstanceKHR`] values if `array_of_pointers` is `true`, or the + /// address of an array of [`VkAccelerationStructureInstanceKHR`] values. /// - /// Addresses and VkAccelerationStructureInstanceKHR structures are tightly packed. + /// Addresses and `VkAccelerationStructureInstanceKHR` values are tightly packed. + /// + /// [`VkAccelerationStructureInstanceKHR`]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkAccelerationStructureInstanceKHR.html addr: DeviceOrHostAddress, /// Specifies whether data is used as an array of addresses or just an array. @@ -490,9 +538,9 @@ pub enum AccelerationStructureGeometryData { /// A device or host address to memory containing index data for this geometry. index_addr: DeviceOrHostAddress, - /// The - /// See [`VkIndexType`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkIndexType.html). - /// of each index element. + /// The [`VkIndexType`] of each index element. + /// + /// [`VkIndexType`]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkIndexType.html index_type: vk::IndexType, /// The highest index of a vertex that will be addressed by a build command using this @@ -500,17 +548,19 @@ pub enum AccelerationStructureGeometryData { max_vertex: u32, /// A device or host address to memory containing an optional reference to a - /// See [`VkTransformMatrixKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkTransformMatrixKHR.html). - /// structure describing a transformation from the space in which the vertices in this - /// geometry are described to the space in which the acceleration structure is defined. + /// [`VkTransformMatrixKHR`] structure describing a transformation from the space in which + /// the vertices in this geometry are described to the space in which the acceleration + /// structure is defined. + /// + /// [`VkTransformMatrixKHR`]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkTransformMatrixKHR.html transform_addr: Option, /// A device or host address to memory containing vertex data for this geometry. vertex_addr: DeviceOrHostAddress, - /// The - /// See [`VkFormat`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkFormat.html). - /// of each vertex element. + /// The [`VkFormat`] of each vertex element. + /// + /// [`VkFormat`]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkFormat.html vertex_format: vk::Format, /// The stride in bytes between each vertex. @@ -624,7 +674,7 @@ impl From for vk::AccelerationStructureGeomet #[derive(Clone, Debug)] pub struct AccelerationStructureGeometryInfo { /// Type of acceleration structure. - pub ty: vk::AccelerationStructureTypeKHR, + pub acceleration_structure_type: vk::AccelerationStructureTypeKHR, /// Specifies additional parameters of the acceleration structure. pub flags: vk::BuildAccelerationStructureFlagsKHR, @@ -639,7 +689,7 @@ impl AccelerationStructureGeometryInfo { let geometries = geometries.into(); Self { - ty: vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL, + acceleration_structure_type: vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL, flags: Default::default(), geometries, } @@ -651,7 +701,7 @@ impl AccelerationStructureGeometryInfo { let geometries = geometries.into(); Self { - ty: vk::AccelerationStructureTypeKHR::TOP_LEVEL, + acceleration_structure_type: vk::AccelerationStructureTypeKHR::TOP_LEVEL, flags: Default::default(), geometries, } @@ -665,6 +715,8 @@ impl AccelerationStructureGeometryInfo { } /// Information used to create an [`AccelerationStructure`] instance. +/// +/// See [`VkAccelerationStructureCreateInfoKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkAccelerationStructureCreateInfoKHR.html). #[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build"), @@ -674,7 +726,7 @@ impl AccelerationStructureGeometryInfo { pub struct AccelerationStructureInfo { /// Type of acceleration structure. #[builder(default = "vk::AccelerationStructureTypeKHR::GENERIC")] - pub ty: vk::AccelerationStructureTypeKHR, + pub acceleration_structure_type: vk::AccelerationStructureTypeKHR, /// The size of the backing buffer that will store the acceleration structure. /// @@ -689,7 +741,7 @@ impl AccelerationStructureInfo { #[inline(always)] pub const fn blas(size: vk::DeviceSize) -> Self { Self { - ty: vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL, + acceleration_structure_type: vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL, size, } } @@ -704,7 +756,7 @@ impl AccelerationStructureInfo { #[inline(always)] pub const fn tlas(size: vk::DeviceSize) -> Self { Self { - ty: vk::AccelerationStructureTypeKHR::TOP_LEVEL, + acceleration_structure_type: vk::AccelerationStructureTypeKHR::TOP_LEVEL, size, } } @@ -712,7 +764,7 @@ impl AccelerationStructureInfo { /// Converts an `AccelerationStructureInfo` into an `AccelerationStructureInfoBuilder`. pub fn into_builder(self) -> AccelerationStructureInfoBuilder { AccelerationStructureInfoBuilder { - ty: Some(self.ty), + acceleration_structure_type: Some(self.acceleration_structure_type), size: Some(self.size), } } @@ -739,19 +791,123 @@ impl AccelerationStructureInfoBuilder { /// Holds the results of the [`AccelerationStructure::size_of`] function. #[derive(Clone, Copy, Debug)] pub struct AccelerationStructureSize { - /// The size of the scratch buffer required when building an acceleration structure using the - /// `Acceleration::build_structure` function. + /// The size of the scratch buffer required when building an acceleration structure using + /// [`CommandRef::build_accel_struct`](crate::cmd::CommandRef::build_accel_struct). pub build_size: vk::DeviceSize, /// The value of `size` parameter needed by [`AccelerationStructureInfo`] for use with the /// [`AccelerationStructure::create`] function. pub create_size: vk::DeviceSize, - /// The size of the scratch buffer required when updating an acceleration structure using the - /// `Acceleration::update_structure` function. + /// The size of the scratch buffer required when updating an acceleration structure using + /// [`CommandRef::update_accel_struct`](crate::cmd::CommandRef::update_accel_struct). pub update_size: vk::DeviceSize, } +/// Synchronization information for an acceleration structure. +#[derive(Clone, Copy, Debug)] +pub struct AccelerationStructureSyncInfo { + /// Pipeline stages that access the acceleration structure. + pub stage_mask: vk::PipelineStageFlags, + + /// Access types performed by those stages. + pub access_mask: vk::AccessFlags, + + /// Current exclusive queue-family ownership, when relevant. + pub queue_family_index: Option, +} + +impl AccelerationStructureSyncInfo { + fn from_accesses(accesses: impl IntoIterator) -> Self { + let mut stage_mask = vk::PipelineStageFlags::empty(); + let mut access_mask = vk::AccessFlags::empty(); + + for access in accesses { + let (stages, mask) = accel_struct_sync_flags_for_access(access); + stage_mask |= stages; + access_mask |= mask; + } + + Self { + stage_mask, + access_mask, + queue_family_index: None, + } + } +} + +struct AccessIter<'a> { + accesses: MutexGuard<'a, Vec>, + idx: usize, + previous_len: usize, +} + +impl<'a> AccessIter<'a> { + fn one(mut accesses: MutexGuard<'a, Vec>, next_access: AccessType) -> Self { + let previous_len = accesses.len(); + accesses.push(next_access); + + Self { + accesses, + idx: 0, + previous_len, + } + } + + fn new(mut accesses: MutexGuard<'a, Vec>, next_accesses: &[AccessType]) -> Self { + let previous_len = accesses.len(); + + if next_accesses.is_empty() { + accesses.push(AccessType::Nothing); + } else { + for &next_access in next_accesses { + if !accesses[previous_len..].contains(&next_access) { + accesses.push(next_access); + } + } + } + + Self { + accesses, + idx: 0, + previous_len, + } + } +} + +impl Iterator for AccessIter<'_> { + type Item = AccessType; + + fn next(&mut self) -> Option { + if self.idx == self.previous_len { + return None; + } + + let access = self.accesses[self.idx]; + self.idx += 1; + + Some(access) + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + + (len, Some(len)) + } +} + +impl ExactSizeIterator for AccessIter<'_> { + fn len(&self) -> usize { + self.previous_len - self.idx + } +} + +impl Drop for AccessIter<'_> { + fn drop(&mut self) { + self.accesses.drain(..self.previous_len); + } +} + /// Specifies a constant device or host address. /// /// See [`VkDeviceOrHostAddressKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDeviceOrHostAddressKHR.html). @@ -817,7 +973,7 @@ mod test { pub fn accel_struct_info_builder() { let info = Info { size: 32, - ty: vk::AccelerationStructureTypeKHR::GENERIC, + acceleration_structure_type: vk::AccelerationStructureTypeKHR::GENERIC, }; let builder = Builder::default().size(32).build(); @@ -828,7 +984,7 @@ mod test { pub fn accel_struct_info_builder_default_size() { let info = Info { size: 0, - ty: vk::AccelerationStructureTypeKHR::GENERIC, + acceleration_structure_type: vk::AccelerationStructureTypeKHR::GENERIC, }; assert_eq!(Builder::default().build(), info); diff --git a/src/driver/buffer.rs b/src/driver/buffer.rs index a5d8335b..8c55703a 100644 --- a/src/driver/buffer.rs +++ b/src/driver/buffer.rs @@ -1,7 +1,7 @@ //! Buffer resource types use { - super::{DriverError, device::Device}, + super::{DriverError, SharingMode, device::Device, pipeline_stage_access_flags}, ash::vk, derive_builder::Builder, gpu_allocator::{ @@ -10,10 +10,13 @@ use { }, log::trace, log::warn, + smallvec::{SmallVec, smallvec}, std::{ fmt::{Debug, Formatter}, - mem::ManuallyDrop, + iter::once, + mem::{ManuallyDrop, take}, ops::{DerefMut, Range}, + sync::atomic::{AtomicU8, AtomicU64, Ordering}, thread::panicking, }, vk_sync::AccessType, @@ -25,6 +28,8 @@ use parking_lot::{Mutex, MutexGuard}; #[cfg(not(feature = "parking_lot"))] use std::sync::{Mutex, MutexGuard}; +type AccessRuns = RunMap; + /// Smart pointer handle to a [buffer] object. /// /// Also contains information about the object. @@ -44,10 +49,10 @@ use std::sync::{Mutex, MutexGuard}; /// # Ok(()) } /// ``` /// -/// [buffer]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkBuffer.html +/// [buffer]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkBuffer.html #[read_only::cast] pub struct Buffer { - accesses: Mutex, + access_runs: Mutex, allocation: ManuallyDrop, /// The device which owns this buffer resource. @@ -68,8 +73,7 @@ pub struct Buffer { #[readonly] pub info: BufferInfo, - /// A name for debugging purposes. - pub name: Option, + sharing: Sharing, } impl Buffer { @@ -107,8 +111,13 @@ impl Buffer { let buffer_info = vk::BufferCreateInfo::default() .size(info.size) .usage(info.usage) - .sharing_mode(vk::SharingMode::CONCURRENT) - .queue_family_indices(&device.physical_device.queue_family_indices); + .sharing_mode(info.sharing_mode); + + let buffer_info = if info.sharing_mode == vk::SharingMode::CONCURRENT { + buffer_info.queue_family_indices(&device.physical.queue_family_indices) + } else { + buffer_info + }; let handle = unsafe { device.create_buffer(&buffer_info, None).map_err(|err| { warn!("unable to create buffer: {err}"); @@ -119,14 +128,14 @@ impl Buffer { let mut requirements = unsafe { device.get_buffer_memory_requirements(handle) }; requirements.alignment = requirements.alignment.max(info.alignment); - let allocation_scheme = if info.dedicated { + let allocation_scheme = if info.alloc_dedicated { AllocationScheme::DedicatedBuffer(handle) } else { AllocationScheme::GpuAllocatorManaged }; - let location = if info.host_write { + let location = if info.host_writable { MemoryLocation::CpuToGpu - } else if info.host_read { + } else if info.host_readable { MemoryLocation::GpuToCpu } else { MemoryLocation::GpuOnly @@ -181,12 +190,12 @@ impl Buffer { debug_assert_ne!(handle, vk::Buffer::null()); Ok(Self { - accesses: Mutex::new(BufferAccess::new(info.size)), + access_runs: Mutex::new(AccessRuns::new(info.size, AccessType::Nothing)), allocation: ManuallyDrop::new(allocation), device, handle, info, - name: None, + sharing: Sharing::new(info.size, info.sharing_mode), }) } @@ -226,76 +235,11 @@ impl Buffer { Ok(buffer) } - /// Keeps track of some `next_access` which affects this object. - /// - /// Returns the previous access for which a pipeline barrier should be used to prevent data - /// corruption. - /// - /// # Note - /// - /// Used to maintain object state when passing a _vk-graph_-created `vk::Buffer` handle to - /// external code such as [_Ash_] or [_Erupt_] bindings. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ```no_run - /// # use std::sync::Arc; - /// # use ash::vk; - /// # use vk_sync::AccessType; - /// # use vk_graph::driver::DriverError; - /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::buffer::{Buffer, BufferInfo, BufferSubresourceRange}; - /// # fn main() -> Result<(), DriverError> { - /// # let device = Device::create(DeviceInfo::default())?; - /// # const SIZE: vk::DeviceSize = 1024; - /// # let info = BufferInfo::device_mem(SIZE, vk::BufferUsageFlags::STORAGE_BUFFER); - /// # let my_buf = Buffer::create(&device, info)?; - /// // Initially we want to "write" - /// let access = AccessType::ComputeShaderWrite; - /// let access_range = BufferSubresourceRange { start: 0, end: SIZE }; - /// let mut accesses = Buffer::access(&my_buf, access, access_range); - /// - /// assert_eq!(accesses.next(), Some((AccessType::Nothing, access_range))); - /// assert!(accesses.next().is_none()); - /// - /// // External code may now "write"; no barrier required in this case - /// - /// // Subsequently we want to "read" - /// let access = AccessType::ComputeShaderReadOther; - /// let mut accesses = Buffer::access(&my_buf, access, access_range); - /// - /// assert_eq!(accesses.next(), Some((AccessType::ComputeShaderWrite, access_range))); - /// assert!(accesses.next().is_none()); - /// - /// // A barrier on "write" before "read" is required! A render graph will do this - /// // automatically when resovled, but manual access like this requires manual barriers - /// # Ok(()) } - /// ``` - /// - /// [_Ash_]: https://crates.io/crates/ash - /// [_Erupt_]: https://crates.io/crates/erupt - #[profiling::function] - pub fn access( - &self, - access: AccessType, - access_range: impl Into, - ) -> impl Iterator + '_ { - let mut access_range: BufferSubresourceRange = access_range.into(); - - if access_range.end == vk::WHOLE_SIZE { - access_range.end = self.info.size; - } - - BufferAccessIter::new(self.lock_accesses(), access, access_range) - } - /// Updates a mappable buffer starting at `offset` with the data in `slice`. /// /// # Panics /// - /// Panics if the buffer was not created with the `mappable` flag set to `true`. + /// Panics if the buffer was not created with host-writable memory. /// /// # Examples /// @@ -352,7 +296,8 @@ impl Buffer { /// ``` #[profiling::function] pub fn device_address(&self) -> vk::DeviceAddress { - debug_assert!( + #[cfg(feature = "checked")] + assert!( self.info .usage .contains(vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS) @@ -365,20 +310,20 @@ impl Buffer { } } - fn lock_accesses(&self) -> MutexGuard<'_, BufferAccess> { - let accesses = self.accesses.lock(); + fn lock_access_runs(&self) -> MutexGuard<'_, AccessRuns> { + let access_runs = self.access_runs.lock(); #[cfg(not(feature = "parking_lot"))] - let accesses = accesses.expect("poisoned buffer access lock"); + let access_runs = access_runs.expect("poisoned buffer access lock"); - accesses + access_runs } /// Returns a mapped slice. /// /// # Panics /// - /// Panics if the buffer was not created with the `mappable` flag set to `true`. + /// Panics if the buffer was not created with host-readable memory. /// /// # Examples /// @@ -403,9 +348,10 @@ impl Buffer { /// ``` #[profiling::function] pub fn mapped_slice(&self) -> &[u8] { - debug_assert!( - self.info.host_read, - "Buffer is not readable - create using host_read flag" + #[cfg(feature = "checked")] + assert!( + self.info.host_readable, + "Buffer is not readable - create using host_readable flag" ); &self @@ -418,7 +364,7 @@ impl Buffer { /// /// # Panics /// - /// Panics if the buffer was not created with the `mappable` flag set to `true`. + /// Panics if the buffer was not created with host-writable memory. /// /// # Examples /// @@ -448,9 +394,10 @@ impl Buffer { /// ``` #[profiling::function] pub fn mapped_slice_mut(&mut self) -> &mut [u8] { - debug_assert!( - self.info.host_write, - "Buffer is not writable - create using host_write flag" + #[cfg(feature = "checked")] + assert!( + self.info.host_writable, + "Buffer is not writable - create using host_writable flag" ); &mut self @@ -460,272 +407,309 @@ impl Buffer { } /// Sets the debugging name assigned to this buffer. - pub fn debug_name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); - - self + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.device, self.handle, &name); + Device::try_set_private_data_object_name( + &self.device, + vk::ObjectType::BUFFER, + self.handle, + &name, + ); } -} -impl Debug for Buffer { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - if let Some(name) = &self.name { - write!(f, "{} ({:?})", name, self.handle) - } else { - write!(f, "{:?}", self.handle) + pub(crate) fn set_sharing_ranges( + &self, + sharing: SharingMode, + sharing_ranges: &[BufferSubresourceRange], + ) { + if sharing_ranges + .iter() + .all(|range| range.end != vk::WHOLE_SIZE) + { + self.sharing + .set_ranges(self.info.size, sharing, sharing_ranges.iter().copied()); + + return; } + + self.sharing.set_ranges( + self.info.size, + sharing, + sharing_ranges + .iter() + .copied() + .map(|range| range.resolve_whole(self.info.size)), + ); } -} -impl Drop for Buffer { + /// Keeps track of some `next_access` which affects this object. + /// + /// Returns the previous access for which a pipeline barrier should be used to prevent data + /// corruption. #[profiling::function] - fn drop(&mut self) { - if panicking() { - return; + pub(crate) fn swap_access( + &self, + next_access: AccessType, + access_range: impl Into, + ) -> impl Iterator + '_ { + let mut access_range: BufferSubresourceRange = access_range.into(); + + if access_range.end == vk::WHOLE_SIZE { + access_range.end = self.info.size; } - { - profiling::scope!("deallocate"); + RunMapIter::new(self.lock_access_runs(), next_access, access_range) + } - Device::with_allocator(&self.device, |allocator| { - allocator.free(unsafe { ManuallyDrop::take(&mut self.allocation) }) - }) + pub(crate) fn swap_accesses<'a, I>( + &'a self, + accesses: I, + ) -> impl Iterator + 'a + where + I: IntoIterator, + I::IntoIter: 'a, + { + struct Iter<'a, I> + where + I: Iterator, + { + access_runs: MutexGuard<'a, AccessRuns>, + accesses: I, + current: Option<(AccessType, RunMapCursor)>, + size: vk::DeviceSize, } - .unwrap_or_else(|err| warn!("unable to free buffer allocation: {err}")); - unsafe { - self.device.destroy_buffer(self.handle, None); + impl<'a, I> Iter<'a, I> + where + I: Iterator, + { + fn new( + access_runs: MutexGuard<'a, AccessRuns>, + accesses: I, + size: vk::DeviceSize, + ) -> Self { + Self { + access_runs, + accesses, + current: None, + size, + } + } } - } -} -impl Eq for Buffer {} + impl Iterator for Iter<'_, I> + where + I: Iterator, + { + type Item = (AccessType, AccessType, BufferSubresourceRange); + + fn next(&mut self) -> Option { + loop { + if let Some((next_access, cursor)) = &mut self.current { + if let Some((prev_access, range)) = + cursor.next(&mut self.access_runs, *next_access) + { + return Some((*next_access, prev_access, range)); + } -impl PartialEq for Buffer { - fn eq(&self, other: &Self) -> bool { - self.handle == other.handle - } -} + self.current = None; + } -#[derive(Debug)] -struct BufferAccess { - accesses: Vec<(AccessType, vk::DeviceSize)>, - size: vk::DeviceSize, -} + let (next_access, mut access_range) = self.accesses.next()?; + if access_range.end == vk::WHOLE_SIZE { + access_range.end = self.size; + } -impl BufferAccess { - fn new(size: vk::DeviceSize) -> Self { - Self { - accesses: vec![(AccessType::Nothing, 0)], - size, + self.current = Some(( + next_access, + RunMapCursor::new(&self.access_runs, access_range), + )); + } + } } - } -} - -struct BufferAccessIter { - access: AccessType, - access_range: BufferSubresourceRange, - buffer: T, - idx: usize, -} -impl BufferAccessIter -where - T: DerefMut, -{ - fn new(buffer: T, access: AccessType, access_range: BufferSubresourceRange) -> Self { - debug_assert!(access_range.start < access_range.end); - debug_assert!(access_range.end <= buffer.size); - - #[cfg(feature = "checked")] + impl Drop for Iter<'_, I> + where + I: Iterator, { - let access_start = |(_, access_start): &(AccessType, vk::DeviceSize)| *access_start; + fn drop(&mut self) { + while self.next().is_some() {} + } + } - assert_eq!(buffer.accesses.first().map(access_start), Some(0)); - assert!(buffer.accesses.last().map(access_start).unwrap() < buffer.size); + let accesses = accesses.into_iter(); + let (min_accesses, _) = accesses.size_hint(); + let mut access_runs = self.lock_access_runs(); + access_runs.runs.reserve(min_accesses.saturating_mul(2)); - // Custom is-sorted-by key to additionally check that all access starts are unique - let (mut prev_access, mut prev_start) = buffer.accesses.first().copied().unwrap(); - for (next_access, next_start) in buffer.accesses.iter().skip(1).copied() { - debug_assert_ne!(prev_access, next_access); - debug_assert!(prev_start < next_start); + Iter::new(access_runs, accesses, self.info.size) + } - prev_access = next_access; - prev_start = next_start; - } - }; + /// Returns compact synchronization information for the buffer's current access ranges. + pub fn sync_info(&self) -> BufferSyncInfo { + let ranges = self + .sync_info_with_sharing() + .map(|(range, sharing)| range.into_public(sharing)) + .collect(); - // The needle will always be odd, and the probe always even, the result will always be err - let needle = (access_range.start << 1) | 1; - let idx = buffer - .accesses - .binary_search_by(|(_, probe)| (probe << 1).cmp(&needle)); + BufferSyncInfo { ranges } + } - debug_assert!(idx.is_err()); + pub(crate) fn sync_info_with_sharing( + &self, + ) -> impl Iterator + '_ { + self.sync_info_with_sharing_range(BufferSubresourceRange { + start: 0, + end: self.info.size, + }) + } - let mut idx = unsafe { idx.unwrap_err_unchecked() }; + pub(crate) fn sync_info_with_sharing_range( + &self, + query_range: BufferSubresourceRange, + ) -> impl Iterator + '_ { + struct SyncInfoIter<'a> { + access_runs: MutexGuard<'a, AccessRuns>, + access_run_idx: usize, + query_range: BufferSubresourceRange, + sharing_run: Option<(SharingMode, BufferSubresourceRange)>, + sharing_runs: SharingRunIter<'a>, + } - // The first access will always be at start == 0, which is even, so idx cannot be 0 - debug_assert_ne!(idx, 0); + impl Iterator for SyncInfoIter<'_> { + type Item = (BufferSubresourceSyncInfo, SharingMode); + + fn next(&mut self) -> Option { + while self.access_run_idx < self.access_runs.runs.len() { + let (access, start) = self.access_runs.runs[self.access_run_idx]; + let end = self + .access_runs + .runs + .get(self.access_run_idx + 1) + .map(|(_, next_start)| *next_start) + .unwrap_or(self.access_runs.size); + let access_range = BufferSubresourceRange { start, end }; + + let Some(access_range) = access_range.intersection(self.query_range) else { + if end <= self.query_range.start { + self.access_run_idx += 1; + + continue; + } - idx -= 1; + return None; + }; - Self { - access, - access_range, - buffer, - idx, - } - } -} + let (sharing, sharing_run_range) = self.sharing_run?; -impl Iterator for BufferAccessIter -where - T: DerefMut, -{ - type Item = (AccessType, BufferSubresourceRange); + let Some(range) = access_range.intersection(sharing_run_range) else { + if sharing_run_range.end <= access_range.start { + self.sharing_run = self.sharing_runs.next(); + } else { + self.access_run_idx += 1; + } - fn next(&mut self) -> Option { - debug_assert!(self.access_range.start <= self.access_range.end); - debug_assert!(self.access_range.end <= self.buffer.size); + continue; + }; - if self.access_range.start == self.access_range.end { - return None; - } + if sharing_run_range.end <= access_range.end { + self.sharing_run = self.sharing_runs.next(); + } - debug_assert!(self.buffer.accesses.get(self.idx).is_some()); - - let (access, access_start) = unsafe { *self.buffer.accesses.get_unchecked(self.idx) }; - let access_end = self - .buffer - .accesses - .get(self.idx + 1) - .map(|(_, access_start)| *access_start) - .unwrap_or(self.buffer.size); - let mut access_range = self.access_range; - - access_range.end = access_range.end.min(access_end); - self.access_range.start = access_range.end; - - if access == self.access { - self.idx += 1; - } else if access_start < access_range.start { - if let Some((_, access_start)) = self - .buffer - .accesses - .get_mut(self.idx + 1) - .filter(|(access, _)| *access == self.access && access_end == access_range.end) - { - *access_start = access_range.start; - self.idx += 1; - } else { - self.idx += 1; - self.buffer - .accesses - .insert(self.idx, (self.access, access_range.start)); - - if access_end > access_range.end { - self.buffer - .accesses - .insert(self.idx + 1, (access, access_range.end)); + if access_range.end <= sharing_run_range.end { + self.access_run_idx += 1; + } + + return Some(( + BufferSubresourceSyncInfo::from_access(access, range), + sharing, + )); } - self.idx += 1; + None } - } else if self.idx > 0 { - if self - .buffer - .accesses - .get(self.idx - 1) - .filter(|(access, _)| *access == self.access) - .is_some() - { - if access_end == access_range.end { - self.buffer.accesses.remove(self.idx); - - if self - .buffer - .accesses - .get(self.idx) - .filter(|(access, _)| *access == self.access) - .is_some() - { - self.buffer.accesses.remove(self.idx); - self.idx -= 1; - } - } else { - debug_assert!(self.buffer.accesses.get(self.idx).is_some()); + } - let (_, access_start) = - unsafe { self.buffer.accesses.get_unchecked_mut(self.idx) }; - *access_start = access_range.end; - } - } else if access_end == access_range.end { - debug_assert!(self.buffer.accesses.get(self.idx).is_some()); + let access_runs = self.access_runs.lock(); - let (access, _) = unsafe { self.buffer.accesses.get_unchecked_mut(self.idx) }; - *access = self.access; + #[cfg(not(feature = "parking_lot"))] + let access_runs = access_runs.expect("poisoned buffer access lock"); + + let query_range = query_range.resolve_whole(self.info.size); + let access_run_idx = access_runs.run_index_at(query_range.start); + let mut sharing_runs = self.sharing.ranges_in(query_range); + let sharing_run = sharing_runs.next(); + + SyncInfoIter { + access_runs, + access_run_idx, + query_range, + sharing_run, + sharing_runs, + } + } - if self - .buffer - .accesses - .get(self.idx + 1) - .filter(|(access, _)| *access == self.access) - .is_some() - { - self.buffer.accesses.remove(self.idx + 1); - } else { - self.idx += 1; - } - } else { - if let Some((_, access_start)) = self.buffer.accesses.get_mut(self.idx) { - *access_start = access_range.end; - } + /// Sets the debugging name assigned to this buffer. + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); - self.buffer - .accesses - .insert(self.idx, (self.access, access_range.start)); - self.idx += 2; - } - } else if let Some((_, access_start)) = self - .buffer - .accesses - .get_mut(1) - .filter(|(access, _)| *access == self.access && access_end == access_range.end) + self + } +} + +impl Debug for Buffer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(Buffer)); + + if let Some(debug_name) = + &Device::private_data_object_name(&self.device, vk::ObjectType::BUFFER, self.handle) { - *access_start = 0; - self.buffer.accesses.remove(0); - } else if access_end > access_range.end { - self.buffer.accesses.insert(0, (self.access, 0)); + res.field("debug_name", debug_name); + } + + res.field("handle", &self.handle).finish_non_exhaustive() + } +} - debug_assert!(self.buffer.accesses.get(1).is_some()); +impl Drop for Buffer { + #[profiling::function] + fn drop(&mut self) { + if panicking() { + return; + } - let (_, access_start) = unsafe { self.buffer.accesses.get_unchecked_mut(1) }; - *access_start = access_range.end; - } else { - debug_assert!(!self.buffer.accesses.is_empty()); + { + profiling::scope!("deallocate"); - let (access, _) = unsafe { self.buffer.accesses.get_unchecked_mut(0) }; - *access = self.access; + Device::with_allocator(&self.device, |allocator| { + allocator.free(unsafe { ManuallyDrop::take(&mut self.allocation) }) + }) + } + .unwrap_or_else(|err| warn!("unable to free buffer allocation: {err}")); - if self - .buffer - .accesses - .get(1) - .filter(|(access, _)| *access == self.access) - .is_some() - { - self.buffer.accesses.remove(1); - } else { - self.idx += 1; - } + Device::try_clear_private_data_object_name( + &self.device, + vk::ObjectType::BUFFER, + self.handle, + ); + + unsafe { + self.device.destroy_buffer(self.handle, None); } + } +} + +impl Eq for Buffer {} - Some((access, access_range)) +impl PartialEq for Buffer { + fn eq(&self, other: &Self) -> bool { + self.handle == other.handle } } /// Information used to create a [`Buffer`] instance. +/// +/// See [`VkBufferCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkBufferCreateInfo.html). #[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build"), @@ -744,31 +728,45 @@ pub struct BufferInfo { /// /// The driver may optimize access to dedicated buffers. #[builder(default)] - pub dedicated: bool, + pub alloc_dedicated: bool, - /// Specifies a buffer whose memory is host visible and may be mapped. + /// Specifies a buffer whose memory is host-visible and may be mapped for reads. /// /// Memory optimal for CPU readback of data may be used. + /// #[builder(default)] - pub host_read: bool, + pub host_readable: bool, - /// Specifies a buffer whose memory is host visible and may be mapped. + /// Specifies a buffer whose memory is host-visible and may be mapped for writes. /// /// Memory optimal for uploading data to the GPU and potentially for constant buffers may be /// used. + /// #[builder(default)] - pub host_write: bool, + pub host_writable: bool, + + /// Controls whether the buffer is accessible from a single queue family (`EXCLUSIVE`) or + /// from all queues (`CONCURRENT`). + #[builder(default = "vk::SharingMode::EXCLUSIVE")] + pub sharing_mode: vk::SharingMode, /// Size in bytes of the buffer to be created. #[builder(default)] pub size: vk::DeviceSize, - /// A bitmask of specifying allowed usages of the buffer. + /// A bitmask specifying the allowed usages of the buffer. + /// + /// See [`VkBufferUsageFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkBufferUsageFlagBits.html). #[builder(default)] pub usage: vk::BufferUsageFlags, } impl BufferInfo { + /// Creates a default `BufferInfoBuilder`. + pub fn builder() -> BufferInfoBuilder { + Default::default() + } + /// Specifies a non-mappable buffer with the given `size` and `usage` values. /// /// Device-local memory (located on the GPU) is used. @@ -776,9 +774,10 @@ impl BufferInfo { pub const fn device_mem(size: vk::DeviceSize, usage: vk::BufferUsageFlags) -> BufferInfo { BufferInfo { alignment: 1, - dedicated: false, - host_read: false, - host_write: false, + alloc_dedicated: false, + host_readable: false, + host_writable: false, + sharing_mode: vk::SharingMode::EXCLUSIVE, size, usage, } @@ -786,7 +785,7 @@ impl BufferInfo { /// Specifies a mappable buffer with the given `size` and `usage` values. /// - /// Host-local memory (located in CPU-accesible RAM) is used. + /// Host-local memory (located in CPU-accessible RAM) is used. /// /// # Note /// @@ -802,35 +801,32 @@ impl BufferInfo { BufferInfo { alignment: 1, - dedicated: false, - host_read: true, - host_write: true, + alloc_dedicated: false, + host_readable: true, + host_writable: true, + sharing_mode: vk::SharingMode::EXCLUSIVE, size, usage, } } - /// Creates a default `BufferInfoBuilder`. - pub fn builder() -> BufferInfoBuilder { - Default::default() - } - - /// Returns `true` if this information specifies host-accessible memory. - pub fn is_host_mem(&self) -> bool { - self.host_read | self.host_write - } - /// Converts a `BufferInfo` into a `BufferInfoBuilder`. pub fn into_builder(self) -> BufferInfoBuilder { BufferInfoBuilder { alignment: Some(self.alignment), - dedicated: Some(self.dedicated), - host_read: Some(self.host_read), - host_write: Some(self.host_write), + alloc_dedicated: Some(self.alloc_dedicated), + host_readable: Some(self.host_readable), + host_writable: Some(self.host_writable), + sharing_mode: Some(self.sharing_mode), size: Some(self.size), usage: Some(self.usage), } } + + /// Returns `true` if this information specifies host-visible memory. + pub fn is_host_visible(&self) -> bool { + self.host_readable | self.host_writable + } } impl From for BufferInfo { @@ -842,7 +838,7 @@ impl From for BufferInfo { impl BufferInfoBuilder { /// Builds a new `BufferInfo`. /// - /// If `alignment` is not a power to two and the `checked` feature is active this function will + /// If `alignment` is not a power of two and the `checked` feature is active this function will /// panic. #[inline(always)] pub fn build(self) -> BufferInfo { @@ -864,15 +860,34 @@ pub struct BufferSubresourceRange { /// The start of range. pub start: vk::DeviceSize, - /// The non-inclusive end of the range. + /// The exclusive end of the range. pub end: vk::DeviceSize, } impl BufferSubresourceRange { + pub(crate) fn contains(self, other: Self) -> bool { + self.start <= other.start && self.end >= other.end + } + + pub(crate) fn intersection(self, other: Self) -> Option { + let start = self.start.max(other.start); + let end = self.end.min(other.end); + + (start < end).then_some(Self { start, end }) + } + #[cfg(test)] pub(crate) fn intersects(self, other: Self) -> bool { self.start < other.end && self.end > other.start } + + pub(crate) fn resolve_whole(mut self, size: vk::DeviceSize) -> Self { + if self.end == vk::WHOLE_SIZE { + self.end = size; + } + + self + } } impl From for BufferSubresourceRange { @@ -894,201 +909,880 @@ impl From> for BufferSubresourceRange { } impl From for Range { - fn from(subresource: BufferSubresourceRange) -> Self { - subresource.start..subresource.end + fn from(range: BufferSubresourceRange) -> Self { + range.start..range.end } } -#[cfg(test)] -mod test { - use { - super::*, - rand::{Rng, SeedableRng, rngs::SmallRng}, - }; +/// Synchronization information for one accessed buffer range. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BufferSubresourceSyncInfo { + /// Access types performed by `stage_mask`. + pub access_mask: vk::AccessFlags, - type Info = BufferInfo; - type Builder = BufferInfoBuilder; + /// Queue-family ownership for this range, when exclusive ownership is known. + pub queue_family_index: Option, - const FUZZ_COUNT: usize = 100_000; + /// The tracked buffer range. + pub range: BufferSubresourceRange, - #[test] + /// Pipeline stages that access `range`. + pub stage_mask: vk::PipelineStageFlags, +} + +impl BufferSubresourceSyncInfo { + fn can_merge(self, other: Self) -> bool { + self.stage_mask == other.stage_mask + && self.access_mask == other.access_mask + && self.queue_family_index == other.queue_family_index + && self.range.end == other.range.start + } + + fn from_access(access: AccessType, range: BufferSubresourceRange) -> Self { + let (stage_mask, access_mask) = pipeline_stage_access_flags(access); + + Self { + access_mask, + queue_family_index: None, + range, + stage_mask, + } + } + + fn into_public(self, sharing: SharingMode) -> Self { + Self { + queue_family_index: match sharing { + SharingMode::Concurrent | SharingMode::Exclusive(None) => None, + SharingMode::Exclusive(Some((queue_family_index, _))) => Some(queue_family_index), + }, + ..self + } + } + + fn merge(&mut self, other: Self) { + self.range.end = other.range.end; + } +} + +/// Synchronization information for a buffer. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BufferSyncInfo { + /// Access state for the tracked buffer ranges. + pub ranges: Box<[BufferSubresourceSyncInfo]>, +} + +impl BufferSyncInfo { + /// Compacts adjacent ranges with identical synchronization requirements. + /// + /// Runs in linear time over `ranges`. The implementation reuses the existing range storage by + /// converting the boxed slice into a vector, compacting entries in place, and converting it back + /// into a boxed slice. + pub fn compact(&mut self) { + let ranges = take(&mut self.ranges); + let mut ranges = ranges.into_vec(); + let mut compacted_len = 0; + + for idx in 0..ranges.len() { + let sync_info = ranges[idx]; + + if compacted_len > 0 && ranges[compacted_len - 1].can_merge(sync_info) { + ranges[compacted_len - 1].merge(sync_info); + } else { + ranges[compacted_len] = sync_info; + compacted_len += 1; + } + } + + ranges.truncate(compacted_len); + self.ranges = ranges.into_boxed_slice(); + } + + /// Returns a compacted copy of this synchronization snapshot. + /// + /// This has the same linear-time and in-place storage characteristics as [`Self::compact`], but + /// consumes and returns the snapshot for use in iterator chains or expression-oriented code. + pub fn into_compacted(mut self) -> Self { + self.compact(); + self + } +} + +#[derive(Debug)] +struct ExclusiveSharing { + sharing_runs: Mutex>, + sharing_runs_state: AtomicU8, + uniform: AtomicU64, +} + +impl ExclusiveSharing { + fn new(size: vk::DeviceSize) -> Self { + let sharing = SharingMode::Exclusive(None); + + Self { + sharing_runs: Mutex::new(RunMap::new(size, sharing)), + sharing_runs_state: AtomicU8::new(RunTrackingState::Uniform as _), + uniform: AtomicU64::new(sharing.encode()), + } + } + + fn is_sharing_runs_active(&self) -> bool { + self.sharing_runs_state() == RunTrackingState::Dense + } + + fn is_promoting(&self) -> bool { + self.sharing_runs_state() == RunTrackingState::Promoting + } + + fn promote_and_set_ranges( + &self, + size: vk::DeviceSize, + sharing: SharingMode, + sharing_ranges: I, + ) where + I: Iterator, + { + let sharing_runs = self.sharing_runs.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock"); + + #[cfg(feature = "parking_lot")] + let mut sharing_runs = sharing_runs; + + let (min_ranges, _) = sharing_ranges.size_hint(); + sharing_runs.runs.reserve(min_ranges.saturating_mul(2)); + + if self.is_sharing_runs_active() { + for sharing_range in sharing_ranges { + RunMapIter::new(&mut *sharing_runs, sharing, sharing_range).finish(); + } + + return; + } + + self.set_promoting(); + let current = SharingMode::decode(self.uniform.load(Ordering::Acquire)); + *sharing_runs = RunMap::new(size, current); + sharing_runs.runs.reserve(min_ranges.saturating_mul(2)); + + for sharing_range in sharing_ranges { + RunMapIter::new(&mut *sharing_runs, sharing, sharing_range).finish(); + } + + self.set_dense(); + } + + fn ranges_in(&self, query_range: BufferSubresourceRange) -> SharingRunIter<'_> { + if !self.uses_sharing_runs() { + let sharing = SharingMode::decode(self.uniform.load(Ordering::Acquire)); + + return SharingRunIter::Constant(Some((sharing, query_range))); + } + + let sharing_runs = self.sharing_runs.lock(); + + #[cfg(not(feature = "parking_lot"))] + let sharing_runs = sharing_runs.expect("poisoned buffer sharing lock"); + + let run_idx = sharing_runs.run_index_at(query_range.start); + + SharingRunIter::Dense { + query_range, + run_idx, + sharing_runs, + } + } + + fn set_promoting(&self) { + self.sharing_runs_state + .store(RunTrackingState::Promoting as _, Ordering::Release); + } + + fn set_dense(&self) { + self.sharing_runs_state + .store(RunTrackingState::Dense as _, Ordering::Release); + } + + fn set_range( + &self, + size: vk::DeviceSize, + sharing: SharingMode, + sharing_range: BufferSubresourceRange, + ) { + if sharing_range.start == 0 && sharing_range.end == size { + self.set_uniform_or_dense(sharing, sharing_range); + return; + } + + let sharing_runs = self.sharing_runs.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock"); + + #[cfg(feature = "parking_lot")] + let mut sharing_runs = sharing_runs; + + if self.is_sharing_runs_active() { + RunMapIter::new(sharing_runs, sharing, sharing_range).finish(); + + return; + } + + self.set_promoting(); + let current = SharingMode::decode(self.uniform.load(Ordering::Acquire)); + *sharing_runs = RunMap::new(size, current); + RunMapIter::new(sharing_runs, sharing, sharing_range).finish(); + self.set_dense(); + } + + fn set_ranges(&self, size: vk::DeviceSize, sharing: SharingMode, sharing_ranges: I) + where + I: IntoIterator, + { + let mut sharing_ranges = sharing_ranges.into_iter(); + let Some(first) = sharing_ranges.next() else { + return; + }; + + let Some(second) = sharing_ranges.next() else { + self.set_range(size, sharing, first); + + return; + }; + + self.promote_and_set_ranges( + size, + sharing, + once(first).chain(once(second)).chain(sharing_ranges), + ); + } + + fn set_uniform_or_dense(&self, sharing: SharingMode, sharing_range: BufferSubresourceRange) { + let encoded_sharing = sharing.encode(); + + loop { + if self.uses_sharing_runs() { + let sharing_runs = self.sharing_runs.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock"); + + #[cfg(feature = "parking_lot")] + let sharing_runs = sharing_runs; + + RunMapIter::new(sharing_runs, sharing, sharing_range).finish(); + + return; + } + + let current = self.uniform.load(Ordering::Acquire); + if self + .uniform + .compare_exchange( + current, + encoded_sharing, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + if self.is_promoting() { + let sharing_runs = self.sharing_runs.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock"); + + #[cfg(feature = "parking_lot")] + let sharing_runs = sharing_runs; + + RunMapIter::new(sharing_runs, sharing, sharing_range).finish(); + } + + return; + } + } + } + + fn sharing_runs_state(&self) -> RunTrackingState { + match self.sharing_runs_state.load(Ordering::Acquire) { + 0 => RunTrackingState::Uniform, + 1 => RunTrackingState::Promoting, + 2 => RunTrackingState::Dense, + _ => unreachable!("invalid buffer sharing_runs_state"), + } + } + + fn uses_sharing_runs(&self) -> bool { + self.sharing_runs_state() != RunTrackingState::Uniform + } +} + +#[derive(Debug)] +struct RunMap { + runs: SmallVec<[(V, vk::DeviceSize); 4]>, + size: vk::DeviceSize, +} + +impl RunMap { + fn new(size: vk::DeviceSize, value: V) -> Self { + Self { + runs: smallvec![(value, 0)], + size, + } + } + + fn run_index_at(&self, offset: vk::DeviceSize) -> usize { + let needle = (offset << 1) | 1; + let run_idx = self + .runs + .binary_search_by(|(_, probe)| (probe << 1).cmp(&needle)); + + debug_assert!(run_idx.is_err()); + + let run_idx = { + #[cfg(feature = "checked")] + { + run_idx.unwrap_err() + } + + #[cfg(not(feature = "checked"))] + unsafe { + run_idx.unwrap_err_unchecked() + } + }; + + run_idx.saturating_sub(1) + } +} + +struct RunMapCursor { + run_idx: usize, + remaining_range: BufferSubresourceRange, +} + +impl RunMapCursor { + fn new(map: &RunMap, remaining_range: BufferSubresourceRange) -> Self + where + V: Copy + PartialEq + Debug, + { + debug_assert!(remaining_range.start < remaining_range.end); + debug_assert!(remaining_range.end <= map.size); + + #[cfg(feature = "checked")] + { + let run_start = |(_, start): &(V, vk::DeviceSize)| *start; + + assert_eq!(map.runs.first().map(run_start), Some(0)); + assert!(map.runs.last().map(run_start).unwrap() < map.size); + + // Custom is-sorted-by key to additionally check that all run starts are unique + let (mut prev_value, mut prev_start) = map.runs.first().copied().unwrap(); + for (next_value, next_start) in map.runs.iter().skip(1).copied() { + debug_assert_ne!(prev_value, next_value); + debug_assert!(prev_start < next_start); + + prev_value = next_value; + prev_start = next_start; + } + }; + + // The needle will always be odd, and the probe always even, the result will always be err + let needle = (remaining_range.start << 1) | 1; + let run_idx = map + .runs + .binary_search_by(|(_, probe)| (probe << 1).cmp(&needle)); + + debug_assert!(run_idx.is_err()); + + let mut run_idx = { + #[cfg(feature = "checked")] + { + run_idx.unwrap_err() + } + + #[cfg(not(feature = "checked"))] + unsafe { + run_idx.unwrap_err_unchecked() + } + }; + + // The first access will always be at start == 0, which is even, so run_idx cannot be 0 + debug_assert_ne!(run_idx, 0); + + run_idx -= 1; + + Self { + remaining_range, + run_idx, + } + } + + fn next(&mut self, map: &mut RunMap, new_value: V) -> Option<(V, BufferSubresourceRange)> + where + V: Copy + PartialEq + Debug, + { + debug_assert!(self.remaining_range.start <= self.remaining_range.end); + debug_assert!(self.remaining_range.end <= map.size); + + if self.remaining_range.start == self.remaining_range.end { + return None; + } + + debug_assert!(map.runs.get(self.run_idx).is_some()); + + let (old_value, old_start) = unsafe { *map.runs.get_unchecked(self.run_idx) }; + let old_end = map + .runs + .get(self.run_idx + 1) + .map(|(_, start)| *start) + .unwrap_or(map.size); + let mut remaining_range = self.remaining_range; + + remaining_range.end = remaining_range.end.min(old_end); + self.remaining_range.start = remaining_range.end; + + if old_value == new_value { + self.run_idx += 1; + } else if old_start < remaining_range.start { + if let Some((_, start)) = map + .runs + .get_mut(self.run_idx + 1) + .filter(|(value, _)| *value == new_value && old_end == remaining_range.end) + { + *start = remaining_range.start; + self.run_idx += 1; + } else { + self.run_idx += 1; + map.runs + .insert(self.run_idx, (new_value, remaining_range.start)); + + if old_end > remaining_range.end { + map.runs + .insert(self.run_idx + 1, (old_value, remaining_range.end)); + } + + self.run_idx += 1; + } + } else if self.run_idx > 0 { + if map + .runs + .get(self.run_idx - 1) + .filter(|(value, _)| *value == new_value) + .is_some() + { + if old_end == remaining_range.end { + map.runs.remove(self.run_idx); + + if map + .runs + .get(self.run_idx) + .filter(|(value, _)| *value == new_value) + .is_some() + { + map.runs.remove(self.run_idx); + self.run_idx -= 1; + } + } else { + debug_assert!(map.runs.get(self.run_idx).is_some()); + + let (_, start) = unsafe { map.runs.get_unchecked_mut(self.run_idx) }; + *start = remaining_range.end; + } + } else if old_end == remaining_range.end { + debug_assert!(map.runs.get(self.run_idx).is_some()); + + let (value, _) = unsafe { map.runs.get_unchecked_mut(self.run_idx) }; + *value = new_value; + + if map + .runs + .get(self.run_idx + 1) + .filter(|(value, _)| *value == new_value) + .is_some() + { + map.runs.remove(self.run_idx + 1); + } else { + self.run_idx += 1; + } + } else { + if let Some((_, start)) = map.runs.get_mut(self.run_idx) { + *start = remaining_range.end; + } + + map.runs + .insert(self.run_idx, (new_value, remaining_range.start)); + self.run_idx += 2; + } + } else if let Some((_, start)) = map + .runs + .get_mut(1) + .filter(|(value, _)| *value == new_value && old_end == remaining_range.end) + { + *start = 0; + map.runs.remove(0); + } else if old_end > remaining_range.end { + map.runs.insert(0, (new_value, 0)); + + debug_assert!(map.runs.get(1).is_some()); + + let (_, start) = unsafe { map.runs.get_unchecked_mut(1) }; + *start = remaining_range.end; + } else { + debug_assert!(!map.runs.is_empty()); + + let (value, _) = unsafe { map.runs.get_unchecked_mut(0) }; + *value = new_value; + + if map + .runs + .get(1) + .filter(|(value, _)| *value == new_value) + .is_some() + { + map.runs.remove(1); + } else { + self.run_idx += 1; + } + } + + Some((old_value, remaining_range)) + } +} + +struct RunMapIter +where + M: DerefMut>, + V: Copy + PartialEq + Debug, +{ + cursor: RunMapCursor, + map: M, + new_value: V, +} + +impl RunMapIter +where + M: DerefMut>, + V: Copy + PartialEq + Debug, +{ + fn new(map: M, new_value: V, remaining_range: BufferSubresourceRange) -> Self { + let cursor = RunMapCursor::new(&map, remaining_range); + + Self { + cursor, + map, + new_value, + } + } + + fn finish(self) {} +} + +impl Iterator for RunMapIter +where + M: DerefMut>, + V: Copy + PartialEq + Debug, +{ + type Item = (V, BufferSubresourceRange); + + fn next(&mut self) -> Option { + self.cursor.next(&mut self.map, self.new_value) + } +} + +impl Drop for RunMapIter +where + M: DerefMut>, + V: Copy + PartialEq + Debug, +{ + fn drop(&mut self) { + while self.next().is_some() {} + } +} + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RunTrackingState { + Uniform = 0, + Promoting = 1, + Dense = 2, +} + +#[derive(Debug)] +enum Sharing { + Concurrent, + Exclusive(ExclusiveSharing), +} + +impl Sharing { + fn new(size: vk::DeviceSize, sharing_mode: vk::SharingMode) -> Self { + if sharing_mode == vk::SharingMode::CONCURRENT { + Self::Concurrent + } else { + Self::Exclusive(ExclusiveSharing::new(size)) + } + } + + fn ranges_in(&self, range: BufferSubresourceRange) -> SharingRunIter<'_> { + match self { + Self::Concurrent => SharingRunIter::Constant(Some((SharingMode::Concurrent, range))), + Self::Exclusive(sharing) => sharing.ranges_in(range), + } + } + + fn set_ranges(&self, size: vk::DeviceSize, sharing: SharingMode, sharing_ranges: I) + where + I: IntoIterator, + { + if let Self::Exclusive(exclusive) = self { + exclusive.set_ranges(size, sharing, sharing_ranges); + } + } +} + +enum SharingRunIter<'a> { + Constant(Option<(SharingMode, BufferSubresourceRange)>), + Dense { + query_range: BufferSubresourceRange, + run_idx: usize, + sharing_runs: MutexGuard<'a, RunMap>, + }, +} + +impl Iterator for SharingRunIter<'_> { + type Item = (SharingMode, BufferSubresourceRange); + + fn next(&mut self) -> Option { + match self { + Self::Constant(range) => range.take(), + Self::Dense { + query_range, + run_idx, + sharing_runs, + } => { + let &(sharing, start) = sharing_runs.runs.get(*run_idx)?; + if start >= query_range.end { + return None; + } + + let end = sharing_runs + .runs + .get(*run_idx + 1) + .map(|(_, next_start)| *next_start) + .unwrap_or(sharing_runs.size); + + *run_idx += 1; + + let range = BufferSubresourceRange { start, end }.intersection(*query_range)?; + + Some((sharing, range)) + } + } + } +} + +#[cfg(test)] +mod test { + use { + super::*, + rand::{Rng, SeedableRng, rngs::SmallRng}, + }; + + type Info = BufferInfo; + type Builder = BufferInfoBuilder; + + const FUZZ_COUNT: usize = 100_000; + + fn buffer_sync_info(range: Range) -> BufferSubresourceSyncInfo { + BufferSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_READ, + queue_family_index: None, + range: buffer_subresource_range(range), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + } + } + + fn assert_access_runs_eq(access_runs: &AccessRuns, expected: &[(AccessType, vk::DeviceSize)]) { + assert_eq!(access_runs.runs.as_slice(), expected); + } + + #[test] pub fn buffer_access() { - let mut buffer = BufferAccess::new(100); + let mut access_runs = AccessRuns::new(100, AccessType::Nothing); { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::TransferWrite, buffer_subresource_range(0..10), ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::Nothing, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::Nothing, 0)]); assert_eq!( accesses.next().unwrap(), (AccessType::Nothing, buffer_subresource_range(0..10)) ); - assert_eq!( - accesses.buffer.accesses, - vec![(AccessType::TransferWrite, 0), (AccessType::Nothing, 10)] + assert_access_runs_eq( + accesses.map, + &[(AccessType::TransferWrite, 0), (AccessType::Nothing, 10)], ); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::TransferRead, buffer_subresource_range(5..15), ); - assert_eq!( - accesses.buffer.accesses, - vec![(AccessType::TransferWrite, 0), (AccessType::Nothing, 10)] + assert_access_runs_eq( + accesses.map, + &[(AccessType::TransferWrite, 0), (AccessType::Nothing, 10)], ); assert_eq!( accesses.next().unwrap(), (AccessType::TransferWrite, buffer_subresource_range(5..10)) ); - assert_eq!( - accesses.buffer.accesses, - vec![ + assert_access_runs_eq( + accesses.map, + &[ (AccessType::TransferWrite, 0), (AccessType::TransferRead, 5), - (AccessType::Nothing, 10) - ] + (AccessType::Nothing, 10), + ], ); assert_eq!( accesses.next().unwrap(), (AccessType::Nothing, buffer_subresource_range(10..15)) ); - assert_eq!( - accesses.buffer.accesses, - vec![ + assert_access_runs_eq( + accesses.map, + &[ (AccessType::TransferWrite, 0), (AccessType::TransferRead, 5), - (AccessType::Nothing, 15) - ] + (AccessType::Nothing, 15), + ], ); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::HostRead, buffer_subresource_range(0..100), ); - assert_eq!( - accesses.buffer.accesses, - vec![ + assert_access_runs_eq( + accesses.map, + &[ (AccessType::TransferWrite, 0), (AccessType::TransferRead, 5), - (AccessType::Nothing, 15) - ] + (AccessType::Nothing, 15), + ], ); assert_eq!( accesses.next().unwrap(), (AccessType::TransferWrite, buffer_subresource_range(0..5)) ); - assert_eq!( - accesses.buffer.accesses, - vec![ + assert_access_runs_eq( + accesses.map, + &[ (AccessType::HostRead, 0), (AccessType::TransferRead, 5), - (AccessType::Nothing, 15) - ] + (AccessType::Nothing, 15), + ], ); assert_eq!( accesses.next().unwrap(), (AccessType::TransferRead, buffer_subresource_range(5..15)) ); - assert_eq!( - accesses.buffer.accesses, - vec![(AccessType::HostRead, 0), (AccessType::Nothing, 15)] + assert_access_runs_eq( + accesses.map, + &[(AccessType::HostRead, 0), (AccessType::Nothing, 15)], ); assert_eq!( accesses.next().unwrap(), (AccessType::Nothing, buffer_subresource_range(15..100)) ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostRead, 0),]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostRead, 0)]); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::HostWrite, buffer_subresource_range(0..100), ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostRead, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostRead, 0)]); assert_eq!( accesses.next().unwrap(), (AccessType::HostRead, buffer_subresource_range(0..100)) ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostWrite, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::HostWrite, buffer_subresource_range(0..100), ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostWrite, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]); assert_eq!( accesses.next().unwrap(), (AccessType::HostWrite, buffer_subresource_range(0..100)) ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostWrite, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::HostWrite, buffer_subresource_range(1..99), ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostWrite, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]); assert_eq!( accesses.next().unwrap(), (AccessType::HostWrite, buffer_subresource_range(1..99)) ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostWrite, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::HostRead, buffer_subresource_range(1..99), ); - assert_eq!(accesses.buffer.accesses, vec![(AccessType::HostWrite, 0)]); + assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]); assert_eq!( accesses.next().unwrap(), (AccessType::HostWrite, buffer_subresource_range(1..99)) ); - assert_eq!( - accesses.buffer.accesses, - vec![ + assert_access_runs_eq( + accesses.map, + &[ (AccessType::HostWrite, 0), (AccessType::HostRead, 1), - (AccessType::HostWrite, 99) - ] + (AccessType::HostWrite, 99), + ], ); assert!(accesses.next().is_none()); } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::Nothing, buffer_subresource_range(0..100), ); @@ -1109,8 +1803,8 @@ mod test { } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::AnyShaderWrite, buffer_subresource_range(0..100), ); @@ -1123,8 +1817,8 @@ mod test { } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::AnyShaderReadOther, buffer_subresource_range(1..2), ); @@ -1137,8 +1831,8 @@ mod test { } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::AnyShaderReadOther, buffer_subresource_range(3..4), ); @@ -1151,8 +1845,8 @@ mod test { } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::Nothing, buffer_subresource_range(0..5), ); @@ -1189,16 +1883,16 @@ mod test { #[test] pub fn buffer_access_basic() { - let mut buffer = BufferAccess::new(5); + let mut access_runs = AccessRuns::new(5, AccessType::Nothing); - buffer.accesses = vec![ + access_runs.runs = smallvec![ (AccessType::ColorAttachmentRead, 0), (AccessType::AnyShaderWrite, 4), ]; { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::AnyShaderWrite, buffer_subresource_range(0..2), ); @@ -1214,8 +1908,8 @@ mod test { } { - let mut accesses = BufferAccessIter::new( - &mut buffer, + let mut accesses = RunMapIter::new( + &mut access_runs, AccessType::HostWrite, buffer_subresource_range(0..5), ); @@ -1240,6 +1934,29 @@ mod test { } } + #[test] + pub fn buffer_sharing_ranges_in_clips_dense_runs_to_query_range() { + let sharing = ExclusiveSharing::new(16); + let owner_a = SharingMode::Exclusive(Some((1, 0))); + let owner_b = SharingMode::Exclusive(Some((2, 0))); + let range_a = buffer_subresource_range(0..8); + let range_b = buffer_subresource_range(8..16); + let query_range = buffer_subresource_range(4..12); + + sharing.set_ranges(16, owner_a, [range_a]); + sharing.set_ranges(16, owner_b, [range_b]); + + let ranges = sharing.ranges_in(query_range).collect::>(); + + assert_eq!( + ranges, + vec![ + (owner_a, buffer_subresource_range(4..8)), + (owner_b, buffer_subresource_range(8..12)), + ] + ); + } + fn buffer_access_fuzz(buffer_size: vk::DeviceSize) { static ACCESS_TYPES: &[AccessType] = &[ AccessType::AnyShaderReadOther, @@ -1252,7 +1969,7 @@ mod test { ]; let mut rng = SmallRng::seed_from_u64(42); - let mut buffer = BufferAccess::new(buffer_size); + let mut access_runs = AccessRuns::new(buffer_size, AccessType::Nothing); let mut data = vec![AccessType::Nothing; buffer_size as usize]; for _ in 0..FUZZ_COUNT { @@ -1262,8 +1979,8 @@ mod test { // println!("{access:?} {access_start}..{access_end}"); - let accesses = BufferAccessIter::new( - &mut buffer, + let accesses = RunMapIter::new( + &mut access_runs, access, buffer_subresource_range(access_start..access_end), ); @@ -1300,6 +2017,62 @@ mod test { buffer_access_fuzz(10_000); } + #[test] + pub fn buffer_sync_info_compact_merges_adjacent_equal_ranges() { + let mut sync_info = BufferSyncInfo { + ranges: vec![ + buffer_sync_info(0..4), + buffer_sync_info(4..8), + BufferSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_WRITE, + queue_family_index: None, + range: buffer_subresource_range(8..12), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + }, + ] + .into_boxed_slice(), + }; + + sync_info.compact(); + + assert_eq!(sync_info.ranges.len(), 2); + assert_eq!(sync_info.ranges[0], buffer_sync_info(0..8)); + assert_eq!( + sync_info.ranges[1], + BufferSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_WRITE, + queue_family_index: None, + range: buffer_subresource_range(8..12), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + } + ); + } + + #[test] + pub fn buffer_sync_info_into_compacted_preserves_non_adjacent_ranges() { + let sync_info = BufferSyncInfo { + ranges: vec![ + BufferSubresourceSyncInfo { + queue_family_index: Some(3), + ..buffer_sync_info(0..4) + }, + BufferSubresourceSyncInfo { + queue_family_index: Some(3), + ..buffer_sync_info(5..9) + }, + ] + .into_boxed_slice(), + }; + + let sync_info = sync_info.into_compacted(); + + assert_eq!(sync_info.ranges.len(), 2); + assert_eq!(sync_info.ranges[0].queue_family_index, Some(3)); + assert_eq!(sync_info.ranges[1].queue_family_index, Some(3)); + assert_eq!(sync_info.ranges[0].range, buffer_subresource_range(0..4)); + assert_eq!(sync_info.ranges[1].range, buffer_subresource_range(5..9)); + } + #[test] pub fn buffer_info() { let info = Info::device_mem(0, vk::BufferUsageFlags::empty()); diff --git a/src/driver/cmd_buf.rs b/src/driver/cmd_buf.rs index a197da25..62df66c2 100644 --- a/src/driver/cmd_buf.rs +++ b/src/driver/cmd_buf.rs @@ -1,18 +1,23 @@ //! Command buffer types use { - super::{DriverError, device::Device}, - ash::vk, + super::{DriverError, device::Device, fence::Fence}, + ash::vk::{self, Handle as _}, derive_builder::Builder, - log::{error, trace, warn}, - std::{fmt::Debug, slice, thread::panicking}, + log::warn, + std::{ + fmt::{Debug, Formatter}, + slice, + thread::panicking, + }, }; // TODO: Expose command functions so the fence, device, waiting flags do not // need to be public -/// Represents a Vulkan command buffer to which some work has been submitted. -#[derive(Debug)] +/// Represents a Vulkan command buffer allocation. +/// +/// See [`VkCommandBuffer`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkCommandBuffer.html). #[read_only::cast] pub struct CommandBuffer { /// The device which owns this command buffer resource. @@ -21,14 +26,6 @@ pub struct CommandBuffer { #[readonly] pub device: Device, - droppables: Vec>, - - /// The native Vulkan fence handle of this command buffer. - /// - /// _Note:_ This field is read-only. - #[readonly] - pub fence: vk::Fence, - /// The native Vulkan resource handle of this command buffer. /// /// _Note:_ This field is read-only. @@ -40,10 +37,23 @@ pub struct CommandBuffer { pub info: CommandBufferInfo, pub(crate) pool: vk::CommandPool, + release_semaphore: Option, } impl CommandBuffer { + /// Begins recording this command buffer. + /// + /// This is a thin wrapper around [`ash::Device::begin_command_buffer`] that maps Vulkan errors + /// to [`DriverError`] variants. + /// + /// See [`vkBeginCommandBuffer`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkBeginCommandBuffer.html). + pub fn begin(&self, info: &vk::CommandBufferBeginInfo) -> Result<(), DriverError> { + Device::begin_command_buffer(&self.device, self.handle, info) + } + /// Creates a command buffer allocation backed by a transient resettable command pool. + /// + /// See [`vkAllocateCommandBuffers`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkAllocateCommandBuffers.html). #[profiling::function] pub fn create( device: &Device, @@ -91,74 +101,142 @@ impl CommandBuffer { } _ => DriverError::Unsupported, } - })?[0]; + })? + .into_iter() + .find(|handle| !handle.is_null()) + .ok_or_else(|| { + warn!("missing command buffer handle"); - let fence = Device::create_fence(&device, false)?; + DriverError::Unsupported + })?; Ok(Self { device, - droppables: vec![], - fence, handle, info, pool, + release_semaphore: None, }) } - /// Drops an item after execution has been completed. - pub fn drop_after_executed(&mut self, x: impl Debug + Send + 'static) { - self.droppables.push(Box::new(x)); + /// Ends recording this command buffer. + /// + /// This is a thin wrapper around [`ash::Device::end_command_buffer`] that maps Vulkan errors + /// to [`DriverError`] variants. + /// + /// See [`vkEndCommandBuffer`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkEndCommandBuffer.html). + pub fn end(&self) -> Result<(), DriverError> { + Device::end_command_buffer(&self.device, self.handle) } - /// Signals that execution has completed and it is time to drop anything we collected. - #[profiling::function] - fn drop_fenced(&mut self) { - if !self.droppables.is_empty() { - trace!("dropping {} shared references", self.droppables.len()); + /// Ends recording a render pass. + pub fn end_render_pass(&self) { + unsafe { + self.device.cmd_end_render_pass(self.handle); } - - self.droppables.clear(); } - /// Returns `true` after the GPU has executed the previous submission to this command buffer. + /// Submits command buffers to a queue using `fence`. /// - /// See [`Self::wait_until_executed`] to block while checking. - #[profiling::function] - pub fn has_executed(&self) -> Result { - let res = unsafe { self.device.get_fence_status(self.fence) }; + /// This method does not begin, end, or reset `self` or `fence`. Callers are expected to + /// submit only executable command buffers and to manage fence waits and resets as needed. + /// + /// Typical handling is: + /// + /// 1. Begin recording with [`Self::begin`]. + /// 2. Record commands. + /// 3. End recording with [`Self::end`]. + /// 4. Submit this command buffer with `queue_submit`. + /// 5. Later, wait for completion with [`Fence::is_signaled`] or [`Fence::wait_signaled`]. + /// 6. Before re-submitting this same command buffer, reset the fence with [`Fence::reset`], + /// then begin recording again. + /// + /// See [`vkQueueSubmit`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkQueueSubmit.html). + pub fn queue_submit( + &self, + queue: vk::Queue, + fence: &mut Fence, + submits: &[vk::SubmitInfo], + ) -> Result<(), DriverError> { + Device::queue_submit(&self.device, queue, submits, fence.handle)?; + fence.mark_queued(); - match res { - Ok(status) => Ok(status), - Err(err) if err == vk::Result::ERROR_DEVICE_LOST => { - error!("invalid device state: lost"); + Ok(()) + } - Err(DriverError::InvalidData) - } - Err(err) => { - // VK_SUCCESS and VK_NOT_READY handled by get_fence_status in ash - // VK_ERROR_DEVICE_LOST already handled above, so no idea what happened - error!("unable to get fence status: {err}"); + /// Submits command buffers to a queue using `vkQueueSubmit2` (Vulkan 1.3 core or + /// `VK_KHR_synchronization2`). + /// + /// See [`vkQueueSubmit2`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkQueueSubmit2.html) + /// and [`VK_KHR_synchronization2`](https://registry.khronos.org/vulkan/specs/latest/man/html/VK_KHR_synchronization2.html). + pub fn queue_submit2( + &self, + queue: vk::Queue, + fence: &mut Fence, + submits: &[vk::SubmitInfo2], + ) -> Result<(), DriverError> { + Device::queue_submit2(&self.device, queue, submits, fence.handle)?; + fence.mark_queued(); - Err(DriverError::InvalidData) - } - } + Ok(()) } - /// Stalls by blocking the current thread until the GPU has executed the previous submission to - /// this command buffer. + /// Returns a cached semaphore used to signal temporary queue-ownership release submissions. /// - /// See [`Self::has_executed`] to check without blocking. - #[profiling::function] - pub fn wait_until_executed(&mut self) -> Result<(), DriverError> { - if self.droppables.is_empty() { - return Ok(()); + /// The semaphore is created lazily on first use and then reused with this command buffer for + /// subsequent release submissions. + pub(crate) fn release_semaphore(&mut self) -> Result { + if let Some(semaphore) = self.release_semaphore { + return Ok(semaphore); } - Device::wait_for_fence(&self.device, &self.fence)?; + let semaphore = Device::create_semaphore(&self.device)?; - self.drop_fenced(); + Device::try_set_debug_utils_object_name(&self.device, semaphore, "queue ownership release"); - Ok(()) + self.release_semaphore = Some(semaphore); + + Ok(semaphore) + } + + /// Sets the debugging name assigned to this command buffer. + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.device, self.handle, &name); + Device::try_set_private_data_object_name( + &self.device, + vk::ObjectType::COMMAND_BUFFER, + self.handle, + &name, + ); + } + + /// Sets the debugging name assigned to this command buffer. + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); + + self + } +} + +impl AsRef for CommandBuffer { + fn as_ref(&self) -> &CommandBuffer { + self + } +} + +impl Debug for CommandBuffer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(CommandBuffer)); + + if let Some(debug_name) = &Device::private_data_object_name( + &self.device, + vk::ObjectType::COMMAND_BUFFER, + self.handle, + ) { + res.field("debug_name", debug_name); + } + + res.field("handle", &self.handle).finish_non_exhaustive() } } @@ -169,15 +247,20 @@ impl Drop for CommandBuffer { return; } - if self.wait_until_executed().is_err() { - return; - } + Device::try_clear_private_data_object_name( + &self.device, + vk::ObjectType::COMMAND_BUFFER, + self.handle, + ); unsafe { + if let Some(semaphore) = self.release_semaphore.take() { + self.device.destroy_semaphore(semaphore, None); + } + self.device .free_command_buffers(self.pool, slice::from_ref(&self.handle)); self.device.destroy_command_pool(self.pool, None); - self.device.destroy_fence(self.fence, None); } } } diff --git a/src/driver/compute.rs b/src/driver/compute.rs index 31d6ff29..568f40ec 100644 --- a/src/driver/compute.rs +++ b/src/driver/compute.rs @@ -1,4 +1,4 @@ -//! Computing pipeline types +//! Compute pipeline types. use { super::{ @@ -6,22 +6,26 @@ use { device::Device, shader::{DescriptorBindingMap, PipelineDescriptorInfo, Shader}, }, - ash::vk, + crate::lazy_str, + ash::vk::{self, Handle as _}, derive_builder::Builder, log::{trace, warn}, std::{ ffi::CString, + fmt::{Debug, Formatter}, hash::{Hash, Hasher}, slice, - sync::{Arc, OnceLock}, + sync::Arc, thread::panicking, }, }; -/// Smart pointer handle of a pipeline object. +/// Smart pointer handle of a compute pipeline object. /// /// Also contains information about the object. -#[derive(Clone, Debug)] +/// +/// See [`VkPipeline`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipeline.html). +#[derive(Clone)] pub struct ComputePipeline { pub(crate) inner: Arc, } @@ -29,9 +33,11 @@ pub struct ComputePipeline { impl ComputePipeline { /// Creates a new compute pipeline on the given device. /// - /// # Panics + /// `shader` may be a pre-built [`Shader`] or any input that can be converted into one. + /// Invalid shader data is returned as [`DriverError::InvalidData`] through the `Result` + /// instead of panicking. /// - /// If shader code is not a multiple of four bytes. + /// See [`VkComputePipelineCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkComputePipelineCreateInfo.html). /// /// # Examples /// @@ -142,7 +148,14 @@ impl ComputePipeline { device.destroy_shader_module(shader_module, None); DriverError::Unsupported - })?[0]; + })? + .into_iter() + .find(|handle| !handle.is_null()) + .ok_or_else(|| { + warn!("missing pipeline handle"); + + DriverError::Unsupported + })?; device.destroy_shader_module(shader_module, None); @@ -154,18 +167,12 @@ impl ComputePipeline { handle, info, layout, - name: Default::default(), push_constants, }), }) } } - /// Gets the debugging name assigned to this pipeline, if one has been set. - pub fn debug_name(&self) -> Option<&str> { - self.inner.name.get().map(String::as_str) - } - /// The device which owns this compute pipeline. pub fn device(&self) -> &Device { &self.inner.device @@ -182,29 +189,51 @@ impl ComputePipeline { } /// Sets the debugging name assigned to this pipeline. - /// - /// _Note:_ The pipeline name may only be assigned once. Subsequent calls will not update the - /// previously set name value. - pub fn set_debug_name(&mut self, name: impl Into) { - if !self.inner.device.physical_device.instance.info.debug { - return; + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.inner.device, self.inner.handle, &name); + Device::try_set_private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE, + self.inner.handle, + &name, + ); + + Device::try_set_debug_utils_object_name( + &self.inner.device, + self.inner.layout, + lazy_str!("{} (layout)", name.as_ref()), + ); + + for (set_idx, layout) in &self.inner.descriptor_info.layouts { + layout.set_debug_name(lazy_str!("{} (DS{set_idx})", name.as_ref())); } - - // Both Ok and Err are valid conditions - let _ = self.inner.name.set(name.into()); } /// Sets the debugging name assigned to this pipeline. - /// - /// _Note:_ The pipeline name may only be assigned once. Subsequent calls will not update the - /// previously set name value. - pub fn with_debug_name(mut self, name: impl Into) -> Self { + pub fn with_debug_name(self, name: impl AsRef) -> Self { self.set_debug_name(name); self } } +impl Debug for ComputePipeline { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(ComputePipeline)); + + if let Some(debug_name) = &Device::private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE, + self.inner.handle, + ) { + res.field("debug_name", debug_name); + } + + res.field("handle", &self.inner.handle) + .finish_non_exhaustive() + } +} + impl Eq for ComputePipeline {} impl Hash for ComputePipeline { @@ -220,6 +249,8 @@ impl PartialEq for ComputePipeline { } /// Information used to create a [`ComputePipeline`] instance. +/// +/// See [`VkComputePipelineCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkComputePipelineCreateInfo.html). #[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build"), @@ -299,7 +330,6 @@ pub(crate) struct ComputePipelineInner { pub handle: vk::Pipeline, pub info: ComputePipelineInfo, pub layout: vk::PipelineLayout, - pub name: OnceLock, pub push_constants: Option, } @@ -310,6 +340,12 @@ impl Drop for ComputePipelineInner { return; } + Device::try_clear_private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE, + self.handle, + ); + unsafe { self.device.destroy_pipeline(self.handle, None); self.device.destroy_pipeline_layout(self.layout, None); diff --git a/src/driver/descriptor_set.rs b/src/driver/descriptor_set.rs index 99da3d0e..209d3e25 100644 --- a/src/driver/descriptor_set.rs +++ b/src/driver/descriptor_set.rs @@ -8,9 +8,11 @@ use { }; /// Descriptor pool resource used to allocate descriptor sets for pipeline execution. +/// +/// See [`VkDescriptorPool`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDescriptorPool.html). #[derive(Debug)] #[read_only::cast] -pub struct DescriptorPool { +pub(crate) struct DescriptorPool { /// The device which owns this descriptor pool resource. /// /// _Note:_ This field is read-only. @@ -27,7 +29,7 @@ pub struct DescriptorPool { /// /// _Note:_ This field is read-only. #[readonly] - pub info: DescriptorPoolInfo, + pub(crate) info: DescriptorPoolInfo, } impl DescriptorPool { @@ -223,8 +225,11 @@ impl Drop for DescriptorPool { } /// Descriptor counts and limits used to create a [`DescriptorPool`]. +/// +/// See [`VkDescriptorPoolCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDescriptorPoolCreateInfo.html) +/// and [`VkDescriptorPoolSize`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDescriptorPoolSize.html). #[derive(Clone, Debug, Default, Eq, Hash, PartialEq)] -pub struct DescriptorPoolInfo { +pub(crate) struct DescriptorPoolInfo { pub(crate) acceleration_structure_count: u32, pub(crate) combined_image_sampler_count: u32, pub(crate) input_attachment_count: u32, diff --git a/src/driver/descriptor_set_layout.rs b/src/driver/descriptor_set_layout.rs index 5a7e16b8..c896f8cd 100644 --- a/src/driver/descriptor_set_layout.rs +++ b/src/driver/descriptor_set_layout.rs @@ -2,10 +2,12 @@ use { super::{DriverError, device::Device}, ash::vk, log::warn, - std::thread::panicking, + std::{ + fmt::{Debug, Formatter}, + thread::panicking, + }, }; -#[derive(Debug)] #[read_only::cast] pub struct DescriptorSetLayout { pub device: Device, @@ -31,6 +33,33 @@ impl DescriptorSetLayout { Ok(Self { device, handle }) } + + /// Sets the debugging name assigned to this descriptor set layout. + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.device, self.handle, &name); + Device::try_set_private_data_object_name( + &self.device, + vk::ObjectType::DESCRIPTOR_SET_LAYOUT, + self.handle, + &name, + ); + } +} + +impl Debug for DescriptorSetLayout { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(DescriptorSetLayout)); + + if let Some(debug_name) = &Device::private_data_object_name( + &self.device, + vk::ObjectType::DESCRIPTOR_SET_LAYOUT, + self.handle, + ) { + res.field("debug_name", debug_name); + } + + res.field("handle", &self.handle).finish() + } } impl Drop for DescriptorSetLayout { @@ -40,6 +69,12 @@ impl Drop for DescriptorSetLayout { return; } + Device::try_clear_private_data_object_name( + &self.device, + vk::ObjectType::DESCRIPTOR_SET_LAYOUT, + self.handle, + ); + unsafe { self.device.destroy_descriptor_set_layout(self.handle, None); } diff --git a/src/driver/device.rs b/src/driver/device.rs index 38912e69..60943b64 100644 --- a/src/driver/device.rs +++ b/src/driver/device.rs @@ -3,10 +3,10 @@ use { super::{ DriverError, - instance::{Instance, InstanceInfoBuilder}, + instance::{ApiVersion, Instance, InstanceInfoBuilder}, physical_device::PhysicalDevice, }, - ash::{khr, vk}, + ash::{ext, khr, vk}, derive_builder::Builder, gpu_allocator::{ AllocatorDebugSettings, @@ -15,11 +15,14 @@ use { log::{error, info, trace, warn}, raw_window_handle::HasDisplayHandle, std::{ + collections::HashMap, + ffi::CString, fmt::{Debug, Formatter}, mem::{ManuallyDrop, forget}, ops::Deref, slice, sync::Arc, + sync::atomic::{AtomicU64, Ordering}, thread::panicking, time::Instant, }, @@ -60,11 +63,16 @@ pub struct Device { #[readonly] pub(self) inner: Arc, - /// The physical device, which contains useful data about features, properties, and limits. + /// The selected physical device. + /// + /// This contains the physical device's advertised features, properties, limits, memory + /// properties, queue families, and Vulkan extension support. Extension support is exposed with + /// fields named after the Vulkan extension, for example + /// `device.physical.vk_khr_acceleration_structure.is_some()`. /// /// _Note:_ This field is read-only. #[readonly] - pub physical_device: Box, + pub physical: Box, } impl Device { @@ -90,6 +98,123 @@ impl Device { } } + /// Begins a Vulkan debug label region on `command_buffer` when debug labeling is enabled. + /// + /// Returns without doing any work if debug mode is `false`. + pub fn begin_debug_utils_label( + this: &Self, + command_buffer: vk::CommandBuffer, + label_name: impl AsRef, + ) -> Result<(), DriverError> { + if !this.physical.instance.info.debug { + return Ok(()); + } + + let Ok(label_name) = CString::new(label_name.as_ref()) else { + warn!("invalid label name"); + + return Err(DriverError::InvalidData); + }; + + let ext = Self::try_vk_ext_debug_utils(this)?; + + unsafe { + ext.cmd_begin_debug_utils_label( + command_buffer, + &vk::DebugUtilsLabelEXT::default().label_name(label_name.as_c_str()), + ); + } + + Ok(()) + } + + /// Clears Vulkan private-data metadata associated with `object_type` and `object_handle`. + pub(crate) fn clear_private_data_object_name( + this: &Self, + object_type: vk::ObjectType, + object_handle: T, + ) -> Result<(), DriverError> + where + T: vk::Handle + Copy, + { + if this.inner.private_data_slot.is_none() { + return Ok(()); + } + + if object_handle.is_null() { + warn!("invalid object handle"); + + return Err(DriverError::InvalidData); + } + + let object_key = (object_type, object_handle.as_raw()); + let previous_metadata_id = Self::with_object_metadata_ids(this, |object_to_metadata_id| { + object_to_metadata_id.remove(&object_key) + }); + + if previous_metadata_id.is_none() { + return Ok(()); + } + + let ext = Self::try_vk_ext_private_data(this)?; + let private_data_slot = this + .inner + .private_data_slot + .expect("missing private data slot"); + + if let Err(err) = unsafe { ext.set_private_data(object_handle, private_data_slot, 0) } { + Self::with_object_metadata_ids(this, |object_metadata_ids| { + if let Some(metadata_id) = previous_metadata_id { + object_metadata_ids.insert(object_key, metadata_id); + } + }); + + warn!("unable to clear private data object name: {err}"); + + return Err(match err { + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY | vk::Result::ERROR_OUT_OF_HOST_MEMORY => { + DriverError::OutOfMemory + } + _ => DriverError::Unsupported, + }); + } + + Self::with_private_data_metadata(this, |metadata| { + metadata + .names + .remove(&previous_metadata_id.expect("metadata id removed")); + }); + + Ok(()) + } + + /// Records a pipeline barrier using the `VK_KHR_synchronization2` extension or the Vulkan 1.3 + /// core `vkCmdPipelineBarrier2` path. + /// + /// See [`vkCmdPipelineBarrier2`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdPipelineBarrier2.html) + /// and [`VK_KHR_synchronization2`](https://registry.khronos.org/vulkan/specs/latest/man/html/VK_KHR_synchronization2.html). + pub fn cmd_pipeline_barrier2( + this: &Self, + command_buffer: vk::CommandBuffer, + dependency_info: &vk::DependencyInfo, + ) { + #[cfg(feature = "checked")] + assert!( + this.physical.vk_khr_synchronization2, + "missing synchronization2 feature" + ); + + unsafe { + if this.physical.instance.info.api_version >= ApiVersion::Vulkan13 { + this.cmd_pipeline_barrier2(command_buffer, dependency_info); + } else { + let khr_synchronization2 = Device::expect_vk_khr_synchronization2(this); + + khr_synchronization2.cmd_pipeline_barrier2(command_buffer, dependency_info); + } + } + } + /// Constructs a new device using the given configuration. /// /// This constructor is intended for headless or manually managed setups. It does not infer or @@ -108,7 +233,12 @@ impl Device { Self::try_from_physical_device(physical_device) } - pub(crate) fn create_fence(this: &Self, signaled: bool) -> Result { + /// Creates a Vulkan fence on this device. + /// + /// Pass `true` for `signaled` when the fence should begin in the signaled state. + /// + /// See [`vkCreateFence`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCreateFence.html). + pub fn create_fence(this: &Self, signaled: bool) -> Result { let mut flags = vk::FenceCreateFlags::empty(); if signaled { @@ -118,17 +248,39 @@ impl Device { let create_info = vk::FenceCreateInfo::default().flags(flags); let allocation_callbacks = None; - unsafe { this.create_fence(&create_info, allocation_callbacks) }.map_err(|err| { - warn!("unable to create fence: {err}"); + unsafe { + this.create_fence(&create_info, allocation_callbacks) + .map_err(|err| { + warn!("unable to create fence: {err}"); + + DriverError::OutOfMemory + }) + } + } - DriverError::OutOfMemory - }) + /// Creates a Vulkan binary semaphore on this device. + /// + /// See [`vkCreateSemaphore`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCreateSemaphore.html). + pub fn create_semaphore(this: &Self) -> Result { + let create_info = vk::SemaphoreCreateInfo::default(); + let allocation_callbacks = None; + + unsafe { + this.create_semaphore(&create_info, allocation_callbacks) + .map_err(|err| { + warn!("unable to create semaphore: {err}"); + + DriverError::OutOfMemory + }) + } } /// Ends recording a command buffer on this device. /// /// This is a thin wrapper around [`ash::Device::end_command_buffer`] that maps Vulkan errors /// to [`DriverError`] variants. + /// + /// See [`vkEndCommandBuffer`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkEndCommandBuffer.html). pub fn end_command_buffer(this: &Self, cmd: vk::CommandBuffer) -> Result<(), DriverError> { unsafe { this.end_command_buffer(cmd).map_err(|err| { @@ -144,12 +296,32 @@ impl Device { } } + /// Ends a Vulkan debug label region on `command_buffer` when debug labeling is enabled. + /// + /// Returns without doing any work if debug mode is `false`. + pub fn end_debug_utils_label( + this: &Self, + command_buffer: vk::CommandBuffer, + ) -> Result<(), DriverError> { + if !this.physical.instance.info.debug { + return Ok(()); + } + + let ext = Self::try_vk_ext_debug_utils(this)?; + + unsafe { + ext.cmd_end_debug_utils_label(command_buffer); + } + + Ok(()) + } + /// Helper for times when you already know that the device supports the acceleration /// structure extension. /// /// # Panics /// - /// Panics if [Self.physical_device.accel_struct_properties] is `None`. + /// Panics if acceleration structure support was not enabled for this device. pub(crate) fn expect_vk_khr_acceleration_structure( this: &Self, ) -> &khr::acceleration_structure::Device { @@ -159,12 +331,24 @@ impl Device { .expect("missing VK_KHR_acceleration_structure") } + /// Helper for times when you already know that present wait support is enabled. + /// + /// # Panics + /// + /// Panics if `VK_KHR_present_wait` support was not enabled for this device. + pub(crate) fn expect_vk_khr_present_wait(this: &Self) -> &khr::present_wait::Device { + this.inner + .vk_khr_present_wait + .as_ref() + .expect("missing VK_KHR_present_wait") + } + /// Helper for times when you already know that the device supports the ray tracing pipeline /// extension. /// /// # Panics /// - /// Panics if [Self.physical_device.ray_tracing_features] is `None`. + /// Panics if ray tracing pipeline support was not enabled for this device. pub(crate) fn expect_vk_khr_ray_tracing_pipeline( this: &Self, ) -> &khr::ray_tracing_pipeline::Device { @@ -186,6 +370,19 @@ impl Device { .expect("missing VK_KHR_surface") } + /// Helper for times when you already know that the device supports the synchronization2 + /// extension. + /// + /// # Panics + /// + /// Panics if `VK_KHR_synchronization2` is not available. + pub(crate) fn expect_vk_khr_synchronization2(this: &Self) -> &khr::synchronization2::Device { + this.inner + .vk_khr_synchronization2 + .as_ref() + .expect("missing VK_KHR_synchronization2") + } + /// Helper for times when you already know that the device supports the swapchain extension. /// /// # Panics @@ -198,11 +395,69 @@ impl Device { .expect("missing VK_KHR_swapchain") } + /// Removes local Vulkan private-data metadata without touching the Vulkan object. + pub(crate) fn forget_private_data_object_name( + this: &Self, + object_type: vk::ObjectType, + object_handle: T, + ) where + T: vk::Handle + Copy, + { + if this.inner.private_data_slot.is_none() || object_handle.is_null() { + return; + } + + let object_key = (object_type, object_handle.as_raw()); + let Some(metadata_id) = Self::with_object_metadata_ids(this, |object_metadata_ids| { + object_metadata_ids.remove(&object_key) + }) else { + return; + }; + + Self::with_private_data_metadata(this, |metadata| { + metadata.names.remove(&metadata_id); + }); + } + + /// Returns `true` if both handles refer to the same logical device allocation. + pub(crate) fn is_same(lhs: &Self, rhs: &Self) -> bool { + Arc::ptr_eq(&lhs.inner, &rhs.inner) + } + + /// Returns the device-owned pipeline cache handle. pub(crate) fn pipeline_cache(this: &Self) -> vk::PipelineCache { this.inner.pipeline_cache } + /// Retrieves a Vulkan private-data name associated with `handle`. + /// + /// Returns `None` when metadata is not available or when the `VK_EXT_private_data` extension is + /// not available. + pub(crate) fn private_data_object_name( + this: &Self, + object_type: vk::ObjectType, + object_handle: T, + ) -> Option + where + T: vk::Handle + Copy, + { + this.inner.private_data_slot?; + + if object_handle.is_null() { + return None; + } + + let object_key = (object_type, object_handle.as_raw()); + Self::with_private_data_metadata(this, |metadata| { + let metadata_id = metadata.object_metadata_ids.get(&object_key)?; + + metadata.names.get(metadata_id).cloned() + }) + } + /// Submits command buffers to a queue, optionally signaling a fence. + /// + /// See [`vkQueueSubmit`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkQueueSubmit.html). pub fn queue_submit( this: &Self, queue: vk::Queue, @@ -223,7 +478,64 @@ impl Device { } } + /// Submits command buffers to a queue using the `VK_KHR_synchronization2` extension or the + /// Vulkan 1.3 core `vkQueueSubmit2` path. + /// + /// See [`vkQueueSubmit2`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkQueueSubmit2.html) + /// and [`VK_KHR_synchronization2`](https://registry.khronos.org/vulkan/specs/latest/man/html/VK_KHR_synchronization2.html). + pub fn queue_submit2( + this: &Self, + queue: vk::Queue, + submits: &[vk::SubmitInfo2], + fence: vk::Fence, + ) -> Result<(), DriverError> { + #[cfg(feature = "checked")] + assert!(this.physical.vk_khr_synchronization2); + + unsafe { + if this.physical.instance.info.api_version >= ApiVersion::Vulkan13 { + // Support derived from Vulkan v1.3 implementation + this.queue_submit2(queue, submits, fence) + } else { + let khr_synchronization2 = Device::expect_vk_khr_synchronization2(this); + + // Support derived from Vulkan v1.2 implementation + extension + khr_synchronization2.queue_submit2(queue, submits, fence) + } + .map_err(|err| { + warn!("unable to queue submit2 submissions: {err}"); + + match err { + vk::Result::ERROR_DEVICE_LOST => DriverError::InvalidData, + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY + | vk::Result::ERROR_OUT_OF_HOST_MEMORY => DriverError::OutOfMemory, + _ => DriverError::Unsupported, + } + }) + } + } + + /// Waits for queue idle. + pub fn queue_wait_idle(this: &Self, queue: vk::Queue) -> Result<(), DriverError> { + unsafe { + this.queue_wait_idle(queue).map_err(|err| { + warn!("unable to wait for queue idle: {err}"); + + match err { + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY + | vk::Result::ERROR_OUT_OF_HOST_MEMORY => DriverError::OutOfMemory, + vk::Result::ERROR_DEVICE_LOST | vk::Result::ERROR_VALIDATION_FAILED_EXT => { + DriverError::InvalidData + } + _ => DriverError::Unsupported, + } + }) + } + } + /// Resets one or more fences to the unsignaled state. + /// + /// See [`vkResetFences`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkResetFences.html). pub fn reset_fences(this: &Self, fences: &[vk::Fence]) -> Result<(), DriverError> { unsafe { this.reset_fences(fences).map_err(|err| { @@ -237,13 +549,160 @@ impl Device { } } - /// Loads and existing `ash` Vulkan device that may have been created by other means. + /// Assigns a Vulkan debug-utils name to `handle` when debug labeling is enabled. + /// + /// Returns without doing any work if debug mode is `false`. + pub fn set_debug_utils_object_name( + this: &Self, + object_handle: T, + object_name: impl AsRef, + ) -> Result<(), DriverError> + where + T: vk::Handle + Copy, + { + if !this.physical.instance.info.debug { + return Ok(()); + } + + if object_handle.is_null() { + warn!("invalid object handle"); + + return Err(DriverError::InvalidData); + } + + let Ok(object_name) = CString::new(object_name.as_ref()) else { + warn!("invalid object name"); + + return Err(DriverError::InvalidData); + }; + + let ext = Self::try_vk_ext_debug_utils(this)?; + + unsafe { + match ext.set_debug_utils_object_name( + &vk::DebugUtilsObjectNameInfoEXT::default() + .object_handle(object_handle) + .object_name(object_name.as_c_str()), + ) { + Err( + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY | vk::Result::ERROR_OUT_OF_HOST_MEMORY, + ) => Err(DriverError::OutOfMemory), + Err(vk::Result::ERROR_VALIDATION_FAILED_EXT) => Err(DriverError::InvalidData), + Err(err) => { + warn!("unable to set debug utils object name: {err}"); + + Err(DriverError::Unsupported) + } + Ok(_) => Ok(()), + } + } + } + + /// Stores a Vulkan name in private-data metadata. + /// + /// Returns without doing any work when `VK_EXT_private_data` is not available. + pub(crate) fn set_private_data_object_name( + this: &Self, + object_type: vk::ObjectType, + object_handle: T, + object_name: impl AsRef, + ) -> Result<(), DriverError> + where + T: vk::Handle + Copy, + { + if this.inner.private_data_slot.is_none() { + return Ok(()); + } + + if object_handle.is_null() { + warn!("invalid object handle"); + + return Err(DriverError::InvalidData); + } + + let object_key = (object_type, object_handle.as_raw()); + let metadata_id = this + .inner + .private_data_name_id + .fetch_add(1, Ordering::Relaxed) + + 1; + + let (previous_metadata_id, previous_name) = + Self::with_private_data_metadata(this, |metadata| { + let previous_metadata_id = + metadata.object_metadata_ids.insert(object_key, metadata_id); + let previous_name = previous_metadata_id.and_then(|id| metadata.names.remove(&id)); + + metadata + .names + .insert(metadata_id, object_name.as_ref().to_owned()); + + (previous_metadata_id, previous_name) + }); + + let ext = Self::try_vk_ext_private_data(this)?; + let private_data_slot = this + .inner + .private_data_slot + .expect("missing private data slot"); + + if let Err(err) = + unsafe { ext.set_private_data(object_handle, private_data_slot, metadata_id) } + { + Self::with_private_data_metadata(this, |metadata| { + let _ = metadata.names.remove(&metadata_id); + match previous_metadata_id { + Some(id) => { + metadata.object_metadata_ids.insert(object_key, id); + if let Some(name) = previous_name { + metadata.names.insert(id, name); + } + } + None => { + metadata.object_metadata_ids.remove(&object_key); + } + } + }); + + warn!("unable to set private data object name: {err}"); + + return Err(match err { + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY | vk::Result::ERROR_OUT_OF_HOST_MEMORY => { + DriverError::OutOfMemory + } + _ => DriverError::Unsupported, + }); + } + + Ok(()) + } + + /// Loads an existing `ash` Vulkan device that may have been created by other means. + /// + /// # Safety + /// + /// `device` must have been created from `physical_device` and must have all queues, + /// extensions, and features expected by `vk-graph`. The device must not be destroyed outside + /// this wrapper while any cloned [`Device`] or resources created from it remain alive. #[profiling::function] - pub fn try_from_ash( + pub unsafe fn try_from_ash( device: ash::Device, physical_device: PhysicalDevice, ) -> Result { let debug = physical_device.instance.info.debug; + + if debug && !Instance::supports_debug_utils(&physical_device.instance) { + error!("unsupported VK_EXT_debug_utils"); + + return Err(DriverError::Unsupported); + } + + if debug && !physical_device.vk_ext_private_data { + error!("unsupported VK_EXT_private_data"); + + return Err(DriverError::Unsupported); + } + let mut debug_settings = AllocatorDebugSettings::default(); debug_settings.log_leaks_on_shutdown = debug; debug_settings.log_memory_information = debug; @@ -277,21 +736,52 @@ impl Device { queues.push(queue_family.into_boxed_slice()); } - let vk_khr_surface = physical_device.khr_swapchain.then(|| { + let vk_ext_debug_utils = Some(ext::debug_utils::Device::new( + &physical_device.instance, + &device, + )); + let vk_ext_private_data = physical_device + .vk_ext_private_data + .then(|| ext::private_data::Device::new(&physical_device.instance, &device)); + let vk_ext_private_data_slot = vk_ext_private_data + .as_ref() + .map(|vk_ext_private_data| unsafe { + vk_ext_private_data + .create_private_data_slot( + &vk::PrivateDataSlotCreateInfoEXT::default() + .flags(vk::PrivateDataSlotCreateFlagsEXT::empty()), + None, + ) + .map_err(|err| { + warn!("unable to create private data slot: {err}"); + + DriverError::Unsupported + }) + }) + .transpose()?; + let vk_khr_present_wait = physical_device + .vk_khr_present_wait + .is_some() + .then(|| khr::present_wait::Device::new(&physical_device.instance, &device)); + let vk_khr_surface = physical_device.vk_khr_swapchain.then(|| { let entry = Instance::entry(&physical_device.instance); khr::surface::Instance::new(entry, &physical_device.instance) }); let vk_khr_swapchain = physical_device - .khr_swapchain + .vk_khr_swapchain .then(|| khr::swapchain::Device::new(&physical_device.instance, &device)); let vk_khr_acceleration_structure = physical_device - .accel_struct_properties + .vk_khr_acceleration_structure .is_some() .then(|| khr::acceleration_structure::Device::new(&physical_device.instance, &device)); let vk_khr_ray_tracing_pipeline = physical_device - .ray_tracing_pipeline_features - .ray_tracing_pipeline + .vk_khr_ray_tracing_pipeline + .as_ref() + .is_some_and(|ext| ext.features.ray_tracing_pipeline) .then(|| khr::ray_tracing_pipeline::Device::new(&physical_device.instance, &device)); + let vk_khr_synchronization2 = physical_device + .vk_khr_synchronization2 + .then(|| khr::synchronization2::Device::new(&physical_device.instance, &device)); let pipeline_cache = unsafe { device.create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) } @@ -308,12 +798,19 @@ impl Device { device, pipeline_cache, queues: queues.into_boxed_slice(), + vk_ext_debug_utils, + vk_ext_private_data, vk_khr_acceleration_structure, + vk_khr_present_wait, vk_khr_ray_tracing_pipeline, vk_khr_surface, vk_khr_swapchain, + vk_khr_synchronization2, + private_data_slot: vk_ext_private_data_slot, + private_data_name_id: AtomicU64::new(0), + private_data_metadata: Mutex::new(Default::default()), }), - physical_device: Box::new(physical_device), + physical: Box::new(physical_device), }, }) } @@ -355,14 +852,70 @@ impl Device { info!("created {}", physical_device.properties_v1_0.device_name); - Self::try_from_ash(device, physical_device) + unsafe { Self::try_from_ash(device, physical_device) } + } + + pub(crate) fn try_clear_private_data_object_name( + this: &Self, + object_type: vk::ObjectType, + object_handle: T, + ) where + T: vk::Handle + Copy, + { + let _ = Self::clear_private_data_object_name(this, object_type, object_handle); + } + + /// Assigns a Vulkan debug-utils name to `handle` when debug labeling is enabled. + /// + /// Returns without doing any work if debug mode is `false`. + pub fn try_set_debug_utils_object_name( + this: &Self, + object_handle: T, + object_name: impl AsRef, + ) where + T: vk::Handle + Copy, + { + let _ = Self::set_debug_utils_object_name(this, object_handle, object_name); + } + + /// Stores a Vulkan name in private-data metadata. + /// + /// Returns without doing any work when `VK_EXT_private_data` is not available. + pub(crate) fn try_set_private_data_object_name( + this: &Self, + object_type: vk::ObjectType, + object_handle: T, + object_name: impl AsRef, + ) where + T: vk::Handle + Copy, + { + let _ = Self::set_private_data_object_name(this, object_type, object_handle, object_name); + } + + fn try_vk_ext_debug_utils(this: &Self) -> Result<&ext::debug_utils::Device, DriverError> { + this.inner + .vk_ext_debug_utils + .as_ref() + .ok_or(DriverError::Unsupported) + } + + fn try_vk_ext_private_data(this: &Self) -> Result<&ext::private_data::Device, DriverError> { + this.inner + .vk_ext_private_data + .as_ref() + .ok_or(DriverError::Unsupported) } + /// Waits for a single fence to signal. #[profiling::function] pub(crate) fn wait_for_fence(this: &Self, fence: &vk::Fence) -> Result<(), DriverError> { Device::wait_for_fences(this, slice::from_ref(fence)) } + /// Waits for all fences in `fences` to signal. + /// + /// This first performs a short poll so uncontended waits return quickly, then falls back to an + /// indefinite wait while logging unusually slow completions. #[profiling::function] pub(crate) fn wait_for_fences(this: &Self, fences: &[vk::Fence]) -> Result<(), DriverError> { unsafe { @@ -383,7 +936,7 @@ impl Device { } } - let started = Instant::now(); + let started = cfg!(debug_assertions).then(Instant::now); match this.wait_for_fences(fences, true, u64::MAX) { Ok(_) => (), @@ -399,17 +952,38 @@ impl Device { } } - let elapsed = Instant::now() - started; - let elapsed_millis = elapsed.as_millis(); + if let Some(started) = started { + let elapsed = Instant::now() - started; + let elapsed_millis = elapsed.as_millis(); - if elapsed_millis > 0 { - warn!("slow fence wait: {} ms", elapsed_millis); + if elapsed_millis > 0 { + warn!("slow fence wait: {} ms", elapsed_millis); + } } } Ok(()) } + /// Waits for device idle. + pub fn wait_idle(this: &Self) -> Result<(), DriverError> { + unsafe { + this.device_wait_idle().map_err(|err| { + warn!("unable to wait for device idle: {err}"); + + match err { + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY + | vk::Result::ERROR_OUT_OF_HOST_MEMORY => DriverError::OutOfMemory, + vk::Result::ERROR_DEVICE_LOST | vk::Result::ERROR_VALIDATION_FAILED_EXT => { + DriverError::InvalidData + } + _ => DriverError::Unsupported, + } + }) + } + } + + /// Provides mutable access to the device allocator under its internal lock. pub(crate) fn with_allocator(this: &Self, f: impl FnOnce(&mut Allocator) -> R) -> R { let allocator = this.inner.allocator.lock(); @@ -421,6 +995,25 @@ impl Device { f(&mut allocator) } + fn with_object_metadata_ids( + this: &Self, + f: impl FnOnce(&mut HashMap<(vk::ObjectType, u64), u64>) -> R, + ) -> R { + Self::with_private_data_metadata(this, |metadata| f(&mut metadata.object_metadata_ids)) + } + + fn with_private_data_metadata( + this: &Self, + f: impl FnOnce(&mut PrivateDataMetadata) -> R, + ) -> R { + let mut metadata = this.inner.private_data_metadata.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut metadata = metadata.expect("poisoned private data metadata"); + + f(&mut metadata) + } + /// Provides locked access to a device queue. /// /// Acquires the mutex for the queue at the given family and index, calls `f` with the @@ -455,7 +1048,10 @@ impl Device { impl Debug for Device { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str("Device") + f.debug_struct(stringify!(Device)) + .field("handle", &self.inner.device.handle()) + .field("physical", &self.physical) + .finish_non_exhaustive() } } @@ -484,19 +1080,24 @@ impl PartialEq for Device { pattern = "owned" )] pub struct DeviceInfo { - /// Enables Vulkan validation layers. + /// Enables the Vulkan validation layers. + /// + /// This requires a Vulkan SDK installation and will panic when validation errors happen. See + /// the LunarG [Vulkan Validation Layers] documentation for setup and behavior details. /// - /// This requires a Vulkan SDK installation and will cause validation errors to introduce - /// panics as they happen. + /// When `stderr` is attached to an interactive terminal, validation errors will park the + /// callback thread for debugger attach. /// /// _NOTE:_ Consider turning OFF debug if you discover an unknown issue. Often the validation - /// layers will throw an error before other layers can provide additional context such as the + /// layers will report an error before other layers can provide additional context such as the /// API dump info or other messages. You might find the "actual" issue is detailed in those /// subsequent details. /// /// ## Platform-specific /// /// **macOS:** Has no effect unless the `loaded` feature is enabled. + /// + /// [Vulkan Validation Layers]: https://vulkan.lunarg.com/doc/sdk/latest/windows/validation_layers.html #[builder(default)] pub debug: bool, @@ -540,10 +1141,23 @@ struct DeviceInner { device: ash::Device, pipeline_cache: vk::PipelineCache, queues: Box<[Box<[Mutex]>]>, + vk_ext_debug_utils: Option, + vk_ext_private_data: Option, vk_khr_acceleration_structure: Option, + vk_khr_present_wait: Option, vk_khr_ray_tracing_pipeline: Option, vk_khr_surface: Option, vk_khr_swapchain: Option, + vk_khr_synchronization2: Option, + private_data_slot: Option, + private_data_name_id: AtomicU64, + private_data_metadata: Mutex, +} + +#[derive(Default)] +struct PrivateDataMetadata { + object_metadata_ids: HashMap<(vk::ObjectType, u64), u64>, + names: HashMap, } impl Drop for DeviceInner { @@ -568,6 +1182,13 @@ impl Drop for DeviceInner { self.device .destroy_pipeline_cache(self.pipeline_cache, None); + if let (Some(vk_ext_private_data), Some(private_data_slot)) = ( + self.vk_ext_private_data.as_ref(), + self.private_data_slot.take(), + ) { + vk_ext_private_data.destroy_private_data_slot(private_data_slot, None); + } + ManuallyDrop::drop(&mut self.allocator); } @@ -582,7 +1203,7 @@ impl Clone for ReadOnlyDevice { fn clone(&self) -> Self { Self { inner: self.inner.clone(), - physical_device: self.physical_device.clone(), + physical: self.physical.clone(), } } } diff --git a/src/driver/fence.rs b/src/driver/fence.rs new file mode 100644 index 00000000..15c7a9fd --- /dev/null +++ b/src/driver/fence.rs @@ -0,0 +1,138 @@ +//! Fence types. + +use { + super::{DriverError, device::Device}, + ash::vk, + log::{error, trace}, + std::{fmt::Debug, thread::panicking}, +}; + +/// Represents a Vulkan fence used to track queue submission completion. +/// +/// See [`VkFence`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkFence.html). +#[derive(Debug)] +#[read_only::cast] +pub struct Fence { + /// The device which owns this fence resource. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub device: Device, + + /// The native Vulkan fence handle. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub handle: vk::Fence, + + pub(crate) queued: bool, + droppables: Vec>, +} + +impl Fence { + /// Creates a Vulkan fence owned by `device`. + /// + /// See [`vkCreateFence`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCreateFence.html). + pub fn create(device: &Device, signaled: bool) -> Result { + Ok(Self { + device: device.clone(), + handle: Device::create_fence(device, signaled)?, + queued: signaled, + droppables: Vec::new(), + }) + } + + /// Drops an item after this fence signals. + pub(crate) fn drop_when_signaled(&mut self, x: impl Debug + Send + 'static) { + self.droppables.push(Box::new(x)); + } + + #[profiling::function] + fn drop_signaled(&mut self) { + if !self.droppables.is_empty() { + trace!("dropping {} shared references", self.droppables.len()); + } + + self.droppables.clear(); + } + + /// Returns `true` if this fence is signaled. + /// + /// See [`vkGetFenceStatus`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetFenceStatus.html). + #[profiling::function] + pub fn is_signaled(&self) -> Result { + let res = unsafe { self.device.get_fence_status(self.handle) }; + + match res { + Ok(status) => Ok(status), + Err(err) if err == vk::Result::ERROR_DEVICE_LOST => { + error!("invalid device state: lost"); + + Err(DriverError::InvalidData) + } + Err(err) => { + error!("unable to get fence status: {err}"); + + Err(DriverError::InvalidData) + } + } + } + + /// Returns `true` if work has been queued against this fence. + pub fn is_queued(&self) -> bool { + self.queued + } + + /// Marks this fence as having work queued against it. + pub(crate) fn mark_queued(&mut self) { + self.queued = true; + } + + /// Resets this fence to the unsignaled state. + /// + /// See [`vkResetFences`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkResetFences.html). + pub fn reset(&mut self) -> Result<&mut Self, DriverError> { + #[cfg(feature = "checked")] + if !self.queued { + return Ok(self); + } + + Device::reset_fences(&self.device, std::slice::from_ref(&self.handle))?; + self.queued = false; + + Ok(self) + } + + /// Waits for this fence to signal, then drops any deferred payloads. + /// + /// See [`vkWaitForFences`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkWaitForFences.html). + #[profiling::function] + pub fn wait_signaled(&mut self) -> Result<&mut Self, DriverError> { + #[cfg(feature = "checked")] + if !self.queued { + return Ok(self); + } + + Device::wait_for_fence(&self.device, &self.handle)?; + self.drop_signaled(); + + Ok(self) + } +} + +impl Drop for Fence { + #[profiling::function] + fn drop(&mut self) { + if panicking() { + return; + } + + if self.queued && self.wait_signaled().is_err() { + return; + } + + unsafe { + self.device.destroy_fence(self.handle, None); + } + } +} diff --git a/src/driver/graphic.rs b/src/driver/graphics.rs similarity index 79% rename from src/driver/graphic.rs rename to src/driver/graphics.rs index debd2f4d..8799a022 100644 --- a/src/driver/graphic.rs +++ b/src/driver/graphics.rs @@ -8,6 +8,7 @@ use { merge_push_constant_ranges, shader::{DescriptorBindingMap, PipelineDescriptorInfo, Shader, SpecializationMap}, }, + crate::lazy_str, ash::vk, derive_builder::Builder, log::{Level::Trace, log_enabled, trace, warn}, @@ -15,8 +16,9 @@ use { std::{ collections::HashSet, ffi::CString, + fmt::{Debug, Formatter}, hash::{Hash, Hasher}, - sync::{Arc, OnceLock}, + sync::Arc, thread::panicking, }, }; @@ -72,13 +74,25 @@ pub struct BlendInfo { #[builder(default = "vk::BlendOp::ADD")] pub alpha_blend_op: vk::BlendOp, - /// A bitmask of specifying which of the R, G, B, and/or A components are enabled for writing, + /// A bitmask specifying which of the R, G, B, and/or A components are enabled for writing, /// as described for [`VkPipelineColorBlendAttachmentState`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineColorBlendAttachmentState.html). #[builder(default = "RGBA_COLOR_COMPONENTS")] pub color_write_mask: vk::ColorComponentFlags, } impl BlendInfo { + /// A commonly used blend mode for additive blending. + pub const ADDITIVE: Self = Self { + blend_enable: true, + src_color_blend_factor: vk::BlendFactor::ONE, + dst_color_blend_factor: vk::BlendFactor::ONE, + color_blend_op: vk::BlendOp::ADD, + src_alpha_blend_factor: vk::BlendFactor::ONE, + dst_alpha_blend_factor: vk::BlendFactor::ONE, + alpha_blend_op: vk::BlendOp::ADD, + color_write_mask: RGBA_COLOR_COMPONENTS, + }; + /// A commonly used blend mode for replacing color attachment values with new ones. pub const REPLACE: Self = Self { blend_enable: false, @@ -103,6 +117,21 @@ impl BlendInfo { color_write_mask: RGBA_COLOR_COMPONENTS, }; + /// A color attachment state that disables all color component writes. + /// + /// This is useful for passes that bind a color attachment only to satisfy pipeline or render + /// target layout requirements, while writing depth or stencil data without modifying color. + pub const COLOR_WRITE_DISABLED: Self = Self { + blend_enable: false, + src_color_blend_factor: vk::BlendFactor::SRC_COLOR, + dst_color_blend_factor: vk::BlendFactor::ONE_MINUS_DST_COLOR, + color_blend_op: vk::BlendOp::ADD, + src_alpha_blend_factor: vk::BlendFactor::ZERO, + dst_alpha_blend_factor: vk::BlendFactor::ZERO, + alpha_blend_op: vk::BlendOp::ADD, + color_write_mask: vk::ColorComponentFlags::empty(), + }; + /// A commonly used blend mode for blending color attachment values based on the alpha channel, /// where the color components have been pre-multiplied with the alpha component value. pub const PRE_MULTIPLIED_ALPHA: Self = Self { @@ -159,17 +188,16 @@ impl From for vk::PipelineColorBlendAttachmentState { } impl BlendInfoBuilder { - /// Builds a new `BlendMode`. + /// Builds a new `BlendInfo`. pub fn build(self) -> BlendInfo { self.fallible_build().expect("invalid blend info") } } -// TODO: This could be simplified (bounds_test controsl min/max etc) /// Specifies the [depth bounds tests], [stencil test], and [depth test] pipeline state. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). -#[derive(Builder, Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +#[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build"), derive(Clone, Copy, Debug), @@ -177,12 +205,15 @@ impl BlendInfoBuilder { )] pub struct DepthStencilInfo { /// Control parameters of the stencil test. + /// + /// Defaults to [`StencilMode::IGNORE`]. #[builder(default)] pub back: StencilMode, /// Controls whether [depth bounds testing] is enabled. /// - /// See [`VkPipelineMultisampleStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineMultisampleStateCreateInfo.html). + /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// #[builder(default)] pub bounds_test: bool, @@ -190,12 +221,15 @@ pub struct DepthStencilInfo { /// [depth test]. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// + /// Defaults to [`vk::CompareOp::NEVER`]. #[builder(default)] pub compare_op: vk::CompareOp, /// Controls whether [depth testing] is enabled. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// #[builder(default)] pub depth_test: bool, @@ -204,10 +238,13 @@ pub struct DepthStencilInfo { /// Depth writes are always disabled when `depth_test` is `false`. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// #[builder(default)] pub depth_write: bool, /// Control parameters of the stencil test. + /// + /// Defaults to [`StencilMode::IGNORE`]. #[builder(default)] pub front: StencilMode, @@ -215,6 +252,7 @@ pub struct DepthStencilInfo { /// Minimum depth bound used in the [depth bounds test]. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// #[builder(default, setter(into))] pub min: OrderedFloat, @@ -222,30 +260,19 @@ pub struct DepthStencilInfo { /// Maximum depth bound used in the [depth bounds test]. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// #[builder(default, setter(into))] pub max: OrderedFloat, /// Controls whether [stencil testing] is enabled. /// /// See [`VkPipelineDepthStencilStateCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineDepthStencilStateCreateInfo.html). + /// #[builder(default)] pub stencil_test: bool, } impl DepthStencilInfo { - /// A commonly used depth/stencil mode - pub const DEPTH_WRITE_LESS_IGNORE_STENCIL: Self = Self { - back: StencilMode::IGNORE, - bounds_test: true, - compare_op: vk::CompareOp::LESS, - depth_test: true, - depth_write: true, - front: StencilMode::IGNORE, - min: OrderedFloat(0.0), - max: OrderedFloat(1.0), - stencil_test: false, - }; - /// Specifies a no-depth/no-stencil mode. /// /// This is the default state. @@ -261,6 +288,50 @@ impl DepthStencilInfo { stencil_test: false, }; + /// Creates a depth-read mode with stencil ignored. + pub const fn depth_read(compare_op: vk::CompareOp) -> Self { + Self { + depth_test: true, + compare_op, + min: OrderedFloat(0.0), + max: OrderedFloat(1.0), + ..Self::IGNORE + } + } + + /// Creates a depth-write mode with stencil ignored. + pub const fn depth_write(compare_op: vk::CompareOp) -> Self { + Self { + depth_test: true, + depth_write: true, + compare_op, + min: OrderedFloat(0.0), + max: OrderedFloat(1.0), + ..Self::IGNORE + } + } + + /// Creates a depth-read/write mode with stencil ignored. + pub const fn depth_read_write(compare_op: vk::CompareOp) -> Self { + Self::depth_write(compare_op) + } + + /// Common depth-write mode for normal-Z depth buffers. + pub const DEPTH_WRITE_LESS: Self = Self::depth_write(vk::CompareOp::LESS); + + /// Common depth-write mode for normal-Z depth buffers when equal depth passes are accepted. + pub const DEPTH_WRITE_LESS_OR_EQUAL: Self = Self::depth_write(vk::CompareOp::LESS_OR_EQUAL); + + /// Common depth-write mode for reversed-Z depth buffers. + pub const DEPTH_WRITE_GREATER: Self = Self::depth_write(vk::CompareOp::GREATER); + + /// Common depth-write mode for reversed-Z depth buffers when equal depth passes are accepted. + pub const DEPTH_WRITE_GREATER_OR_EQUAL: Self = + Self::depth_write(vk::CompareOp::GREATER_OR_EQUAL); + + /// A commonly used normal-Z depth-write mode with stencil ignored. + pub const DEPTH_WRITE_LESS_IGNORE_STENCIL: Self = Self::DEPTH_WRITE_LESS; + /// Creates a default `DepthStencilInfoBuilder`. pub fn builder() -> DepthStencilInfoBuilder { Default::default() @@ -282,6 +353,12 @@ impl DepthStencilInfo { } } +impl Default for DepthStencilInfo { + fn default() -> Self { + Self::IGNORE + } +} + impl From for vk::PipelineDepthStencilStateCreateInfo<'_> { fn from(info: DepthStencilInfo) -> Self { Self::default() @@ -308,8 +385,8 @@ impl DepthStencilInfoBuilder { /// /// Also contains information about the object. /// -/// [pipeline]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPipeline.html -#[derive(Clone, Debug)] +/// See [`VkPipeline`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipeline.html). +#[derive(Clone)] #[read_only::cast] pub struct GraphicsPipeline { pub(crate) inner: Arc, @@ -318,12 +395,14 @@ pub struct GraphicsPipeline { impl GraphicsPipeline { /// Creates a new graphics pipeline on the given device. /// - /// The correct pipeline stages will be enabled based on the provided shaders. See [Shader] for - /// details on all available stages. + /// The correct pipeline stages will be enabled based on the provided shaders. See [`Shader`] + /// for details on all available stages. /// - /// # Panics + /// See [`VkGraphicsPipelineCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkGraphicsPipelineCreateInfo.html). /// - /// If shader code is not a multiple of four bytes. + /// `shaders` may contain pre-built [`Shader`] values or any inputs that can be converted into + /// them. Invalid shader data is returned as [`DriverError::InvalidData`] through the `Result` + /// instead of panicking. /// /// # Examples /// @@ -334,7 +413,7 @@ impl GraphicsPipeline { /// # use ash::vk; /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::graphic::{GraphicsPipeline, GraphicsPipelineInfo}; + /// # use vk_graph::driver::graphics::{GraphicsPipeline, GraphicsPipelineInfo}; /// # use vk_graph::driver::shader::Shader; /// # fn main() -> Result<(), DriverError> { /// # let device = Device::create(DeviceInfo::default())?; @@ -374,11 +453,11 @@ impl GraphicsPipeline { .ok_or(DriverError::InvalidData)? .try_vertex_input()?; - // Check for proper stages because vulkan may not complain but this is bad + // Check for proper stages because Vulkan may not complain but this is invalid. let has_fragment_stage = shaders .iter() .any(|shader| shader.stage.contains(vk::ShaderStageFlags::FRAGMENT)); - let has_tesselation_stage = shaders.iter().any(|shader| { + let has_tessellation_stage = shaders.iter().any(|shader| { shader .stage .contains(vk::ShaderStageFlags::TESSELLATION_CONTROL) @@ -392,7 +471,7 @@ impl GraphicsPipeline { .any(|shader| shader.stage.contains(vk::ShaderStageFlags::GEOMETRY)); debug_assert!( - has_fragment_stage || has_tesselation_stage || has_geometry_stage, + has_fragment_stage || has_tessellation_stage || has_geometry_stage, "invalid shader stage combination" ); @@ -496,15 +575,17 @@ impl GraphicsPipeline { if let Some(OrderedFloat(min_sample_shading)) = info.min_sample_shading { #[cfg(debug_assertions)] if info.samples.is_single() { - // This combination of a single-sampled pipeline and minimum sample shading - // does not make sense and should not be requested. In the future maybe this is - // part of the MSAA value so it can't be specified. + /* + This combination of a single-sampled pipeline and minimum sample shading does + not make sense and should not be requested. In the future maybe this is part of + the MSAA value so it can't be specified. + */ warn!("unsupported sample rate shading of single-sample pipeline"); } // Callers should check this before attempting to use the feature debug_assert!( - device.physical_device.features_v1_0.sample_rate_shading, + device.physical.features_v1_0.sample_rate_shading, "unsupported sample rate shading feature" ); @@ -523,7 +604,6 @@ impl GraphicsPipeline { input_attachments, layout, multisample, - name: Default::default(), push_constants, shader_stages, vertex_input, @@ -532,11 +612,6 @@ impl GraphicsPipeline { } } - /// Gets the debugging name assigned to this pipeline, if one has been set. - pub fn debug_name(&self) -> Option<&str> { - self.inner.name.get().map(String::as_str) - } - /// The device which owns this graphics pipeline. pub fn device(&self) -> &Device { &self.inner.device @@ -549,28 +624,83 @@ impl GraphicsPipeline { /// Sets the debugging name assigned to this pipeline. /// - /// _Note:_ The pipeline name may only be assigned once. Subsequent calls will not update the - /// previously set name value. - pub fn set_debug_name(&mut self, name: impl Into) { - if !self.inner.device.physical_device.instance.info.debug { - return; + /// _Note:_ The name of the underlying Vulkan pipeline is lazily updated as submissions are + /// recorded. + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name( + &self.inner.device, + self.inner.layout, + lazy_str!("{} (layout)", name.as_ref()), + ); + Device::try_set_private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE_LAYOUT, + self.inner.layout, + lazy_str!("{}", name.as_ref()), + ); + + for (set_idx, layout) in &self.inner.descriptor_info.layouts { + layout.set_debug_name(lazy_str!("{} (DS{set_idx})", name.as_ref())); } + } - // Both Ok and Err are valid conditions - let _ = self.inner.name.set(name.into()); + pub(crate) fn set_variant_debug_name( + &self, + pipeline_handle: vk::Pipeline, + render_pass: vk::RenderPass, + subpass_idx: u32, + name: impl AsRef, + ) { + Device::try_set_debug_utils_object_name( + &self.inner.device, + pipeline_handle, + lazy_str!( + "{} (render pass {:?}, subpass {subpass_idx})", + name.as_ref(), + render_pass + ), + ); + Device::try_set_private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE, + pipeline_handle, + name, + ); } /// Sets the debugging name assigned to this pipeline. - /// - /// _Note:_ The pipeline name may only be assigned once. Subsequent calls will not update the - /// previously set name value. - pub fn with_debug_name(mut self, name: impl Into) -> Self { + pub fn with_debug_name(self, name: impl AsRef) -> Self { self.set_debug_name(name); self } } +impl Debug for GraphicsPipeline { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(GraphicsPipeline)); + let shader_stages = self + .inner + .shader_stages + .iter() + .map(|stage| stage.flags) + .collect::>(); + + if let Some(debug_name) = &Device::private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE_LAYOUT, + self.inner.layout, + ) { + res.field("debug_name", debug_name); + } + + res.field("layout", &self.inner.layout) + .field("shader_stages", &shader_stages) + .field("input_attachments", &self.inner.input_attachments) + .finish_non_exhaustive() + } +} + impl Eq for GraphicsPipeline {} impl Hash for GraphicsPipeline { @@ -586,6 +716,8 @@ impl PartialEq for GraphicsPipeline { } /// Information used to create a [`GraphicsPipeline`] instance. +/// +/// See [`VkGraphicsPipelineCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkGraphicsPipelineCreateInfo.html). #[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build"), @@ -641,17 +773,17 @@ pub struct GraphicsPipelineInfo { #[builder(default = "vk::CullModeFlags::BACK")] pub cull_mode: vk::CullModeFlags, - /// Interpret polygon front-facing orientation. + /// Interprets polygon front-facing orientation. /// /// The default value is `vk::FrontFace::COUNTER_CLOCKWISE`. #[builder(default = "vk::FrontFace::COUNTER_CLOCKWISE")] pub front_face: vk::FrontFace, - /// Specify a fraction of the minimum number of unique samples to process for each fragment. + /// Specifies a fraction of the minimum number of unique samples to process for each fragment. #[builder(default, setter(into, strip_option))] pub min_sample_shading: Option>, - /// Control polygon rasterization mode. + /// Controls polygon rasterization mode. /// /// The default value is `vk::PolygonMode::FILL`. #[builder(default = "vk::PolygonMode::FILL")] @@ -736,7 +868,6 @@ pub(crate) struct GraphicsPipelineInner { pub input_attachments: Box<[u32]>, pub layout: vk::PipelineLayout, pub multisample: MultisampleState, - pub name: OnceLock, pub push_constants: Box<[vk::PushConstantRange]>, pub shader_stages: Box<[ShaderStage]>, pub vertex_input: VertexInputState, @@ -749,6 +880,12 @@ impl Drop for GraphicsPipelineInner { return; } + Device::try_clear_private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE_LAYOUT, + self.layout, + ); + unsafe { self.device.destroy_pipeline_layout(self.layout, None); } @@ -810,7 +947,7 @@ pub struct StencilMode { } impl StencilMode { - /// Specifes a stencil mode which is has no effect. + /// Specifies a stencil mode which has no effect. pub const IGNORE: Self = Self { fail_op: vk::StencilOp::KEEP, pass_op: vk::StencilOp::KEEP, diff --git a/src/driver/image.rs b/src/driver/image.rs index 714d85a5..d95081f7 100644 --- a/src/driver/image.rs +++ b/src/driver/image.rs @@ -2,7 +2,8 @@ use { super::{ - DriverError, access_type_from_u8, access_type_into_u8, device::Device, format_aspect_mask, + DriverError, SharingMode, access_type_from_u8, access_type_into_u8, device::Device, + format_aspect_mask, pipeline_stage_access_flags, }, ash::vk::{self, ImageCreateInfo}, derive_builder::Builder, @@ -14,9 +15,10 @@ use { std::{ collections::{HashMap, hash_map::Entry}, fmt::{Debug, Formatter}, + marker::PhantomData, mem::{replace, take}, - ops::DerefMut, - sync::atomic::{AtomicU8, Ordering}, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicU8, AtomicU16, AtomicU64, Ordering}, thread::panicking, }, vk_sync::AccessType, @@ -25,10 +27,91 @@ use { #[cfg(feature = "parking_lot")] use parking_lot::{Mutex, MutexGuard}; -use std::marker::PhantomData; #[cfg(not(feature = "parking_lot"))] use std::sync::{Mutex, MutexGuard}; +const fn access_type_to_layout(access: AccessType) -> Option { + match access { + AccessType::Nothing => None, + AccessType::ColorAttachmentRead + | AccessType::ColorAttachmentReadWrite + | AccessType::ColorAttachmentWrite => Some(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL), + AccessType::DepthStencilAttachmentRead => { + Some(vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL) + } + AccessType::DepthStencilAttachmentReadWrite | AccessType::DepthStencilAttachmentWrite => { + Some(vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL) + } + AccessType::DepthAttachmentWriteStencilReadOnly => { + Some(vk::ImageLayout::DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL) + } + AccessType::StencilAttachmentWriteDepthReadOnly => { + Some(vk::ImageLayout::DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL) + } + AccessType::TransferRead => Some(vk::ImageLayout::TRANSFER_SRC_OPTIMAL), + AccessType::TransferWrite => Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL), + AccessType::VertexShaderReadSampledImageOrUniformTexelBuffer + | AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer + | AccessType::FragmentShaderReadColorInputAttachment + | AccessType::ComputeShaderReadSampledImageOrUniformTexelBuffer + | AccessType::TessellationControlShaderReadSampledImageOrUniformTexelBuffer + | AccessType::TessellationEvaluationShaderReadSampledImageOrUniformTexelBuffer + | AccessType::GeometryShaderReadSampledImageOrUniformTexelBuffer + | AccessType::AnyShaderReadSampledImageOrUniformTexelBuffer + | AccessType::MeshShaderReadSampledImageOrUniformTexelBuffer + | AccessType::TaskShaderReadSampledImageOrUniformTexelBuffer => { + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + } + AccessType::FragmentShaderReadDepthStencilInputAttachment => { + Some(vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL) + } + AccessType::Present => Some(vk::ImageLayout::PRESENT_SRC_KHR), + _ => Some(vk::ImageLayout::GENERAL), + } +} + +const fn aspect_mask_at_ordinal( + aspect_mask: vk::ImageAspectFlags, + ordinal: u32, +) -> vk::ImageAspectFlags { + // Common cases: + // - COLOR with ordinal 0 -> COLOR + // - DEPTH | STENCIL with ordinal 0 -> DEPTH + // - DEPTH | STENCIL with ordinal 1 -> STENCIL + let mut bits = aspect_mask.as_raw(); + let mut idx = 0; + + while bits != 0 { + let bit = bits.trailing_zeros(); + if idx == ordinal { + return vk::ImageAspectFlags::from_raw(1 << bit); + } + + bits &= !(1 << bit); + idx += 1; + } + + vk::ImageAspectFlags::empty() +} + +const fn aspect_ordinal(aspect_mask: vk::ImageAspectFlags, aspect: vk::ImageAspectFlags) -> u8 { + let mut bits = aspect_mask.as_raw(); + let target = aspect.as_raw(); + let mut idx = 0; + + while bits != 0 { + let bit = bits.trailing_zeros(); + if target == (1 << bit) { + return idx; + } + + bits &= !(1 << bit); + idx += 1; + } + + 0 +} + #[cfg(feature = "checked")] fn assert_aspect_mask_supported(aspect_mask: vk::ImageAspectFlags) { use vk::ImageAspectFlags as A; @@ -55,6 +138,31 @@ pub(crate) fn image_subresource_range_contains( && lhs.base_mip_level + lhs.level_count >= rhs.base_mip_level + rhs.level_count } +pub(crate) fn image_subresource_range_intersection( + lhs: vk::ImageSubresourceRange, + rhs: vk::ImageSubresourceRange, +) -> Option { + if !image_subresource_range_intersects(lhs, rhs) { + return None; + } + + let aspect_mask = lhs.aspect_mask & rhs.aspect_mask; + let base_array_layer = lhs.base_array_layer.max(rhs.base_array_layer); + let end_array_layer = + (lhs.base_array_layer + lhs.layer_count).min(rhs.base_array_layer + rhs.layer_count); + let base_mip_level = lhs.base_mip_level.max(rhs.base_mip_level); + let end_mip_level = + (lhs.base_mip_level + lhs.level_count).min(rhs.base_mip_level + rhs.level_count); + + Some(vk::ImageSubresourceRange { + aspect_mask, + base_array_layer, + layer_count: end_array_layer - base_array_layer, + base_mip_level, + level_count: end_mip_level - base_mip_level, + }) +} + pub(crate) fn image_subresource_range_intersects( lhs: vk::ImageSubresourceRange, rhs: vk::ImageSubresourceRange, @@ -68,79 +176,240 @@ pub(crate) fn image_subresource_range_intersects( #[derive(Debug)] enum Access { + Dense(DenseAccess), + DualAspect(DualAspectAccess), Uniform(UniformAccess), - Dense(Mutex>), } impl Access { fn new(info: ImageInfo, access: AccessType) -> Self { - let aspect_count = format_aspect_mask(info.fmt).as_raw().count_ones() as u8; + let aspect_count = format_aspect_mask(info.format).as_raw().count_ones() as u8; if aspect_count == 1 && info.array_layer_count == 1 && info.mip_level_count == 1 { Self::Uniform(UniformAccess::new(access)) + } else if aspect_count == 2 && info.array_layer_count == 1 && info.mip_level_count == 1 { + Self::DualAspect(DualAspectAccess::new(access)) } else { - Self::Dense(Mutex::new(DenseAccess::new(info, access))) + Self::Dense(DenseAccess::new(access)) } } - fn swap( - &self, - access: AccessType, + fn swap<'a>( + &'a self, + dense: &'a Mutex>>, + info: ImageInfo, + next_access: AccessType, access_range: vk::ImageSubresourceRange, - ) -> ImageAccessIter<'_> { + ) -> AccessIter<'a> { match self { Self::Uniform(uniform) => { - ImageAccessIter::Uniform(Some(uniform.swap(access, access_range))) + AccessIter::Uniform(Some(uniform.swap(next_access, access_range))) } - Self::Dense(accesses) => { - let accesses = accesses.lock(); + Self::DualAspect(dual) => AccessIter::DualAspect(DualAspectAccessIter::new( + dual, + info, + next_access, + access_range, + )), + Self::Dense(access) => { + if !access.uses_dense() && info.is_full_subresource_range(access_range) { + return AccessIter::Uniform(Some(access.swap_range(next_access, access_range))); + } + + let mut dense = dense.lock(); #[cfg(not(feature = "parking_lot"))] - let accesses = accesses.expect("poisoned image access lock"); + let mut dense = dense.expect("poisoned image dense lock"); - ImageAccessIter::Dense(DenseAccessIter::new(accesses, access, access_range)) + access.ensure_dense(&mut dense, info); + + AccessIter::DenseMap(DenseMapIter::new( + DenseAccessMapGuard { access, dense }, + next_access, + access_range, + )) } } } } -#[derive(Debug)] -pub(crate) struct DenseAccess { - accesses: Box<[A]>, +enum AccessIter<'a> { + DenseMap(DenseMapIter<'a, DenseAccessMapGuard<'a>, AccessType>), + DualAspect(DualAspectAccessIter<'a>), + Uniform(Option<(AccessType, vk::ImageSubresourceRange)>), +} - #[cfg(feature = "checked")] - array_layer_count: u32, +impl Drop for AccessIter<'_> { + fn drop(&mut self) { + while self.next().is_some() {} + } +} - aspect_count: u8, - mip_level_count: u32, +impl Iterator for AccessIter<'_> { + type Item = (AccessType, vk::ImageSubresourceRange); + + fn next(&mut self) -> Option { + match self { + Self::DenseMap(iter) => iter.next(), + Self::DualAspect(iter) => iter.next(), + Self::Uniform(item) => item.take(), + } + } } -impl DenseAccess { - pub(crate) fn new(info: ImageInfo, access: A) -> Self { - let aspect_mask = format_aspect_mask(info.fmt); +#[derive(Debug)] +struct DenseAccess(AtomicU16); - #[cfg(feature = "checked")] - assert_aspect_mask_supported(aspect_mask); +impl DenseAccess { + const ACCESS_MASK: u16 = 0x00_FF; + const STATE_MASK: u16 = 0xFF_00; + const STATE_SHIFT: u16 = 8; - let aspect_count = aspect_mask.as_raw().count_ones() as u8; - let array_layer_count = info.array_layer_count; - let mip_level_count = info.mip_level_count; + fn new(access: AccessType) -> Self { + Self(AtomicU16::new( + (DenseAccessState::Uniform as u16) << Self::STATE_SHIFT + | access_type_into_u8(access) as u16, + )) + } - Self { - accesses: vec![ - access; - (aspect_count as u32 * array_layer_count * mip_level_count) as _ - ] - .into_boxed_slice(), + fn ensure_dense(&self, dense: &mut Option>, info: ImageInfo) { + if self.is_dense_active() { + debug_assert!(dense.is_some()); + return; + } - #[cfg(feature = "checked")] - array_layer_count, + self.set_promoting(); + let current = self.load(); + *dense = Some(DenseMap::new(info, current)); + self.set_dense(); + } - aspect_count, - mip_level_count, + fn is_dense_active(&self) -> bool { + self.state() == DenseAccessState::Dense + } + + fn load(&self) -> AccessType { + access_type_from_u8((self.0.load(Ordering::Acquire) & Self::ACCESS_MASK) as u8) + } + + fn set_dense(&self) { + let current = self.0.load(Ordering::Acquire); + self.0.store( + (current & !Self::STATE_MASK) | (DenseAccessState::Dense as u16) << Self::STATE_SHIFT, + Ordering::Release, + ); + } + + fn set_promoting(&self) { + let current = self.0.load(Ordering::Acquire); + self.0.store( + (current & !Self::STATE_MASK) + | (DenseAccessState::Promoting as u16) << Self::STATE_SHIFT, + Ordering::Release, + ); + } + + fn set_uniform(&self, next_access: AccessType) { + self.0.store( + (DenseAccessState::Uniform as u16) << Self::STATE_SHIFT + | access_type_into_u8(next_access) as u16, + Ordering::Release, + ); + } + + fn state(&self) -> DenseAccessState { + match (self.0.load(Ordering::Acquire) >> Self::STATE_SHIFT) as u8 { + 0 => DenseAccessState::Uniform, + 1 => DenseAccessState::Promoting, + 2 => DenseAccessState::Dense, + _ => unreachable!("invalid image dense access state"), } } + fn swap_range( + &self, + next_access: AccessType, + access_range: vk::ImageSubresourceRange, + ) -> (AccessType, vk::ImageSubresourceRange) { + let packed = (DenseAccessState::Uniform as u16) << Self::STATE_SHIFT + | access_type_into_u8(next_access) as u16; + let prev = self.0.swap(packed, Ordering::AcqRel); + + (access_type_from_u8(prev as u8), access_range) + } + + fn uses_dense(&self) -> bool { + self.state() != DenseAccessState::Uniform + } +} + +struct DenseAccessMapGuard<'a> { + access: &'a DenseAccess, + dense: MutexGuard<'a, Option>>, +} + +impl DenseAccessMapGuard<'_> { + fn try_demote_to_uniform(&mut self) { + let DenseAccessState::Dense = self.access.state() else { + return; + }; + + let dense_map = self.dense.as_ref().expect("missing dense access state"); + let Some(access) = dense_map.uniform_value() else { + return; + }; + + *self.dense = None; + self.access.set_uniform(access); + } +} + +impl Deref for DenseAccessMapGuard<'_> { + type Target = DenseMap; + + fn deref(&self) -> &Self::Target { + self.dense.as_ref().expect("missing dense access state") + } +} + +impl DerefMut for DenseAccessMapGuard<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.dense.as_mut().expect("missing dense access state") + } +} + +impl Drop for DenseAccessMapGuard<'_> { + fn drop(&mut self) { + self.try_demote_to_uniform(); + } +} + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DenseAccessState { + Uniform = 0, + Promoting = 1, + Dense = 2, +} + +#[derive(Debug)] +pub(crate) struct DenseMap { + #[cfg(feature = "checked")] + array_layer_count: u32, + + aspect_count: u8, + mip_level_count: u32, + values: Box<[V]>, +} + +impl DenseMap { + fn base_aspect_ordinal(&self, base_aspect_bit: u8) -> u8 { + let stencil_bit = vk::ImageAspectFlags::STENCIL.as_raw().trailing_zeros() as u8; + + // DenseMap stores depth/stencil aspects as compact ordinals: depth = 0, stencil = 1 + (self.aspect_count == 2 && base_aspect_bit == stencil_bit) as u8 + } + fn idx(&self, aspect: u8, array_layer: u32, mip_level: u32) -> usize { let idx = (array_layer * self.aspect_count as u32 * self.mip_level_count + mip_level * self.aspect_count as u32 @@ -148,292 +417,594 @@ impl DenseAccess { #[cfg(feature = "checked")] assert!( - idx < self.accesses.len(), + idx < self.values.len(), "idx={idx}, aspect={aspect}, layer={array_layer}, mip={mip_level}, aspect_count={}, mip_level_count={}, array_layer_count={}, len={}", self.aspect_count, self.mip_level_count, self.array_layer_count, - self.accesses.len(), + self.values.len(), ); idx } } -impl DenseAccess { +impl DenseMap { + pub(crate) fn new(info: ImageInfo, value: V) -> Self { + let aspect_mask = format_aspect_mask(info.format); + + #[cfg(feature = "checked")] + assert_aspect_mask_supported(aspect_mask); + + let aspect_count = aspect_mask.as_raw().count_ones() as u8; + let array_layer_count = info.array_layer_count; + let mip_level_count = info.mip_level_count; + + Self { + aspect_count, + mip_level_count, + values: vec![value; (aspect_count as u32 * array_layer_count * mip_level_count) as _] + .into_boxed_slice(), + + #[cfg(feature = "checked")] + array_layer_count, + } + } + + fn subresource(&self, aspect: u8, array_layer: u32, mip_level: u32) -> V { + self.values[self.idx(aspect, array_layer, mip_level)] + } +} + +impl DenseMap { pub(crate) fn swap( &mut self, - access: A, - access_range: vk::ImageSubresourceRange, - ) -> DenseAccessIter<'_, &mut Self, A> { - DenseAccessIter::new(self, access, access_range) + value: V, + range: vk::ImageSubresourceRange, + ) -> DenseMapIter<'_, &mut Self, V> { + DenseMapIter::new(self, value, range) + } + + fn uniform_value(&self) -> Option { + let mut iter = self.values.iter().copied(); + let first = iter.next()?; + + iter.all(|value| value == first).then_some(first) } } -pub(crate) struct DenseAccessIter<'a, I, A> { - __: PhantomData<&'a mut DenseAccess>, - access: A, - access_range: ImageAccessRange, +struct DenseMapCursor { + range: DenseMapRange, array_layer: u32, aspect: u8, - image: I, mip_level: u32, } -impl<'a, I, A: Copy> DenseAccessIter<'a, I, A> { - fn new(image: I, access: A, access_range: vk::ImageSubresourceRange) -> Self - where - I: DerefMut>, - { +impl DenseMapCursor { + fn new(map: &DenseMap, range: vk::ImageSubresourceRange) -> Self { #[cfg(feature = "checked")] - assert_aspect_mask_supported(access_range.aspect_mask); + assert_aspect_mask_supported(range.aspect_mask); #[cfg(feature = "checked")] - assert!(access_range.base_array_layer < image.array_layer_count); + assert!(range.base_array_layer < map.array_layer_count); - debug_assert!(access_range.base_mip_level < image.mip_level_count); - debug_assert_ne!(access_range.layer_count, 0); - debug_assert_ne!(access_range.level_count, 0); + debug_assert!(range.base_mip_level < map.mip_level_count); + debug_assert_ne!(range.layer_count, 0); + debug_assert_ne!(range.level_count, 0); - let aspect_count = access_range.aspect_mask.as_raw().count_ones() as _; + let aspect_count = range.aspect_mask.as_raw().count_ones() as _; - debug_assert!(aspect_count <= image.aspect_count); + debug_assert!(aspect_count <= map.aspect_count); - let base_aspect = access_range.aspect_mask.as_raw().trailing_zeros() as _; + let base_aspect_bit = range.aspect_mask.as_raw().trailing_zeros() as _; Self { - __: PhantomData, - access, array_layer: 0, aspect: 0, - image, mip_level: 0, - access_range: ImageAccessRange { + range: DenseMapRange { aspect_count, - base_array_layer: access_range.base_array_layer, - base_aspect, - base_mip_level: access_range.base_mip_level, - layer_count: access_range.layer_count, - level_count: access_range.level_count, + base_array_layer: range.base_array_layer, + base_aspect_bit, + base_mip_level: range.base_mip_level, + layer_count: range.layer_count, + level_count: range.level_count, }, } } -} -impl<'a, I, A: Copy + PartialEq> Iterator for DenseAccessIter<'a, I, A> -where - I: DerefMut>, -{ - type Item = (A, vk::ImageSubresourceRange); - - fn next(&mut self) -> Option { - if self.aspect == self.access_range.aspect_count { + fn next(&mut self, map: &mut DenseMap, value: V) -> Option<(V, vk::ImageSubresourceRange)> + where + V: Copy + PartialEq, + { + if self.aspect == self.range.aspect_count { return None; } - let mut res = vk::ImageSubresourceRange { + let mut range = vk::ImageSubresourceRange { aspect_mask: vk::ImageAspectFlags::from_raw( - (1 << (self.access_range.base_aspect + self.aspect)) as _, + (1 << (self.range.base_aspect_bit + self.aspect)) as _, ), - base_array_layer: self.access_range.base_array_layer + self.array_layer, - base_mip_level: self.access_range.base_mip_level + self.mip_level, + base_array_layer: self.range.base_array_layer + self.array_layer, + base_mip_level: self.range.base_mip_level + self.mip_level, layer_count: 1, level_count: 1, }; - let base_aspect = (self.image.aspect_count << self.access_range.base_aspect == 8) as u8; - let prev_access = replace( + let base_aspect_ordinal = map.base_aspect_ordinal(self.range.base_aspect_bit); + let prev_value = replace( { - let idx = self.image.idx( - base_aspect + self.aspect, - res.base_array_layer, - res.base_mip_level, + let idx = map.idx( + base_aspect_ordinal + self.aspect, + range.base_array_layer, + range.base_mip_level, ); - unsafe { self.image.accesses.get_unchecked_mut(idx) } + unsafe { map.values.get_unchecked_mut(idx) } }, - self.access, + value, ); loop { self.mip_level += 1; - self.mip_level %= self.access_range.level_count; + self.mip_level %= self.range.level_count; if self.mip_level == 0 { break; } - let idx = self.image.idx( - base_aspect + self.aspect, - self.access_range.base_array_layer + self.array_layer, - self.access_range.base_mip_level + self.mip_level, + let idx = map.idx( + base_aspect_ordinal + self.aspect, + self.range.base_array_layer + self.array_layer, + self.range.base_mip_level + self.mip_level, ); - let access = unsafe { self.image.accesses.get_unchecked_mut(idx) }; - if *access != prev_access { - return Some((prev_access, res)); + let next_value = unsafe { map.values.get_unchecked_mut(idx) }; + if *next_value != prev_value { + return Some((prev_value, range)); } - *access = self.access; - res.level_count += 1; + *next_value = value; + range.level_count += 1; } loop { self.array_layer += 1; - self.array_layer %= self.access_range.layer_count; + self.array_layer %= self.range.layer_count; if self.array_layer == 0 { break; } - if res.base_mip_level != self.access_range.base_mip_level { - return Some((prev_access, res)); + if range.base_mip_level != self.range.base_mip_level { + return Some((prev_value, range)); } - let array_layer = self.access_range.base_array_layer + self.array_layer; - let end_mip_level = self.access_range.base_mip_level + self.access_range.level_count; + let array_layer = self.range.base_array_layer + self.array_layer; + let end_mip_level = self.range.base_mip_level + self.range.level_count; - for mip_level in self.access_range.base_mip_level..end_mip_level { - let idx = self - .image - .idx(base_aspect + self.aspect, array_layer, mip_level); - let access = unsafe { *self.image.accesses.get_unchecked(idx) }; - if access != prev_access { - return Some((prev_access, res)); + for mip_level in self.range.base_mip_level..end_mip_level { + let idx = map.idx(base_aspect_ordinal + self.aspect, array_layer, mip_level); + let next_value = unsafe { *map.values.get_unchecked(idx) }; + if next_value != prev_value { + return Some((prev_value, range)); } } - for mip_level in self.access_range.base_mip_level..end_mip_level { - let idx = self - .image - .idx(base_aspect + self.aspect, array_layer, mip_level); - let access = unsafe { self.image.accesses.get_unchecked_mut(idx) }; - *access = self.access; + for mip_level in self.range.base_mip_level..end_mip_level { + let idx = map.idx(base_aspect_ordinal + self.aspect, array_layer, mip_level); + let next_value = unsafe { map.values.get_unchecked_mut(idx) }; + *next_value = value; } - res.layer_count += 1; + range.layer_count += 1; } loop { self.aspect += 1; - if self.aspect == self.access_range.aspect_count { - return Some((prev_access, res)); + if self.aspect == self.range.aspect_count { + return Some((prev_value, range)); } - let end_array_layer = - self.access_range.base_array_layer + self.access_range.layer_count; - let end_mip_level = self.access_range.base_mip_level + self.access_range.level_count; - - for array_layer in self.access_range.base_array_layer..end_array_layer { - for mip_level in self.access_range.base_mip_level..end_mip_level { - let idx = self - .image - .idx(base_aspect + self.aspect, array_layer, mip_level); - let access = unsafe { *self.image.accesses.get_unchecked(idx) }; - if access != prev_access { - return Some((prev_access, res)); + let end_array_layer = self.range.base_array_layer + self.range.layer_count; + let end_mip_level = self.range.base_mip_level + self.range.level_count; + + for array_layer in self.range.base_array_layer..end_array_layer { + for mip_level in self.range.base_mip_level..end_mip_level { + let idx = map.idx(base_aspect_ordinal + self.aspect, array_layer, mip_level); + let next_value = unsafe { *map.values.get_unchecked(idx) }; + if next_value != prev_value { + return Some((prev_value, range)); } } } - for array_layer in self.access_range.base_array_layer..end_array_layer { - for mip_level in self.access_range.base_mip_level..end_mip_level { - let idx = self - .image - .idx(base_aspect + self.aspect, array_layer, mip_level); - let access = unsafe { self.image.accesses.get_unchecked_mut(idx) }; - *access = self.access; + for array_layer in self.range.base_array_layer..end_array_layer { + for mip_level in self.range.base_mip_level..end_mip_level { + let idx = map.idx(base_aspect_ordinal + self.aspect, array_layer, mip_level); + let next_value = unsafe { map.values.get_unchecked_mut(idx) }; + *next_value = value; } } - res.aspect_mask = vk::ImageAspectFlags::from_raw( - res.aspect_mask.as_raw() | (1 << (self.access_range.base_aspect + self.aspect)), + range.aspect_mask = vk::ImageAspectFlags::from_raw( + range.aspect_mask.as_raw() | (1 << (self.range.base_aspect_bit + self.aspect)), ); } } } -/// Smart pointer handle to an [image] object. -/// -/// Also contains information about the object. -/// -/// ```no_run -/// # use ash::vk; -/// # use vk_sync::AccessType; -/// # use vk_graph::driver::DriverError; -/// # use vk_graph::driver::device::{Device, DeviceInfo}; -/// # use vk_graph::driver::image::{Image, ImageInfo}; -/// # fn main() -> Result<(), DriverError> { -/// # let device = Device::create(DeviceInfo::default())?; -/// let fmt = vk::Format::R8G8B8A8_UNORM; -/// let usage = vk::ImageUsageFlags::SAMPLED; -/// let info = ImageInfo::image_2d(320, 200, fmt, usage); -/// let my_img = Image::create(&device, info)?; -/// -/// assert_eq!(my_img.info, info); -/// assert_ne!(my_img.handle, vk::Image::null()); -/// # Ok(()) } -/// ``` -/// -/// [image]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkBuffer.html -#[read_only::cast] -pub struct Image { - access: Access, - allocation: Option, // None when we don't own the image (Swapchain images) +pub(crate) struct DenseMapIter<'a, M, V> +where + M: DerefMut>, + V: Copy + PartialEq, +{ + __: PhantomData<&'a mut DenseMap>, + cursor: DenseMapCursor, + map: M, + value: V, +} - /// The device which owns this image resource. - /// - /// _Note:_ This field is read-only. - #[readonly] - pub device: Device, +impl Drop for DenseMapIter<'_, M, V> +where + M: DerefMut>, + V: Copy + PartialEq, +{ + fn drop(&mut self) { + while self.next().is_some() {} + } +} - /// The native Vulkan resource handle of this image. - /// - /// _Note:_ This field is read-only. - #[readonly] - pub handle: vk::Image, +impl<'a, M, V: Copy + PartialEq> DenseMapIter<'a, M, V> +where + M: DerefMut>, +{ + fn new(map: M, value: V, range: vk::ImageSubresourceRange) -> Self { + let cursor = DenseMapCursor::new(&map, range); - #[allow(clippy::type_complexity)] - image_view_cache: Mutex>, + Self { + __: PhantomData, + cursor, + map, + value, + } + } +} - /// Information used to create this resource. - /// - /// _Note:_ This field is read-only. - #[readonly] - pub info: ImageInfo, +impl<'a, M, V: Copy + PartialEq> Iterator for DenseMapIter<'a, M, V> +where + M: DerefMut>, +{ + type Item = (V, vk::ImageSubresourceRange); - /// A name for debugging purposes. - pub name: Option, + fn next(&mut self) -> Option { + self.cursor.next(&mut self.map, self.value) + } } -impl Image { - /// Creates a new image on the given device. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ```no_run - /// # use std::sync::Arc; - /// # use ash::vk; - /// # use vk_graph::driver::DriverError; - /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::image::{Image, ImageInfo}; - /// # fn main() -> Result<(), DriverError> { - /// # let device = Device::create(DeviceInfo::default())?; - /// let info = ImageInfo::image_2d( - /// 32, - /// 32, - /// vk::Format::R8G8B8A8_UNORM, - /// vk::ImageUsageFlags::SAMPLED, - /// ); - /// let image = Image::create(&device, info)?; - /// - /// assert_ne!(image.handle, vk::Image::null()); - /// assert_eq!(image.info.width, 32); - /// assert_eq!(image.info.height, 32); - /// # Ok(()) } - /// ``` - #[profiling::function] - pub fn create(device: &Device, info: impl Into) -> Result { - let info = info.into(); +#[derive(Copy, Clone)] +struct DenseMapRange { + aspect_count: u8, + base_array_layer: u32, + base_aspect_bit: u8, + base_mip_level: u32, + layer_count: u32, + level_count: u32, +} + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DenseSharingState { + Idle = 0, + Promoting = 1, + Dense = 2, +} + +#[derive(Debug)] +struct DualAspectAccess([AtomicU8; 2]); + +impl DualAspectAccess { + fn new(access: AccessType) -> Self { + let access = access_type_into_u8(access); + + Self([AtomicU8::new(access), AtomicU8::new(access)]) + } + + fn load(&self, aspect_idx: usize) -> AccessType { + access_type_from_u8(self.0[aspect_idx].load(Ordering::Acquire)) + } +} + +struct DualAspectAccessIter<'a> { + dual: &'a DualAspectAccess, + format_aspect_mask: vk::ImageAspectFlags, + next_access: AccessType, + ranges: ImageSubresourceRangeIter, +} + +impl<'a> DualAspectAccessIter<'a> { + fn new( + dual: &'a DualAspectAccess, + info: ImageInfo, + next_access: AccessType, + access_range: vk::ImageSubresourceRange, + ) -> Self { + debug_assert_eq!(access_range.base_array_layer, 0); + debug_assert_eq!(access_range.base_mip_level, 0); + debug_assert_eq!(access_range.layer_count, 1); + debug_assert_eq!(access_range.level_count, 1); + + Self { + dual, + format_aspect_mask: format_aspect_mask(info.format), + next_access, + ranges: ImageSubresourceRangeIter::new(access_range), + } + } +} + +impl ExactSizeIterator for DualAspectAccessIter<'_> { + fn len(&self) -> usize { + self.ranges.len() + } +} + +impl Iterator for DualAspectAccessIter<'_> { + type Item = (AccessType, vk::ImageSubresourceRange); + + fn next(&mut self) -> Option { + let range = self.ranges.next()?; + let aspect_idx = aspect_ordinal(self.format_aspect_mask, range.aspect_mask) as usize; + let prev_access = access_type_from_u8( + self.dual.0[aspect_idx].swap(access_type_into_u8(self.next_access), Ordering::AcqRel), + ); + + Some((prev_access, range)) + } + + fn size_hint(&self) -> (usize, Option) { + self.ranges.size_hint() + } +} + +#[derive(Debug)] +struct ExclusiveSharing { + // `promoting` keeps whole-image updates on the dense path while a partial update is + // converting uniform tracking into subresource tracking + dense_sharing_state: AtomicU8, + uniform: AtomicU64, +} + +impl ExclusiveSharing { + fn new(_info: ImageInfo) -> Self { + let sharing = SharingMode::Exclusive(None); + + Self { + uniform: AtomicU64::new(sharing.encode()), + dense_sharing_state: AtomicU8::new(0), + } + } + + fn dense_sharing_state(&self) -> DenseSharingState { + match self.dense_sharing_state.load(Ordering::Acquire) { + 0 => DenseSharingState::Idle, + 1 => DenseSharingState::Promoting, + 2 => DenseSharingState::Dense, + _ => unreachable!("invalid image dense sharing state"), + } + } + + fn is_dense_sharing_active(&self) -> bool { + self.dense_sharing_state() == DenseSharingState::Dense + } + + fn is_promoting_dense_sharing(&self) -> bool { + self.dense_sharing_state() == DenseSharingState::Promoting + } + + fn uses_dense_sharing(&self) -> bool { + self.dense_sharing_state() != DenseSharingState::Idle + } + + fn set_promoting_dense_sharing(&self) { + self.dense_sharing_state + .store(DenseSharingState::Promoting as _, Ordering::Release); + } + + fn set_dense_sharing_active(&self) { + self.dense_sharing_state + .store(DenseSharingState::Dense as _, Ordering::Release); + } + + fn set_ranges( + &self, + dense: &Mutex>>, + info: ImageInfo, + sharing: SharingMode, + sharing_ranges: &[vk::ImageSubresourceRange], + ) { + if sharing_ranges.is_empty() { + return; + } + + if sharing_ranges.len() == 1 && info.is_full_subresource_range(sharing_ranges[0]) { + self.set_uniform_or_dense_sharing(dense, info, sharing, sharing_ranges[0]); + + return; + } + + self.promote_dense_sharing_and_set_ranges(dense, info, sharing, sharing_ranges); + } + + fn set_uniform_or_dense_sharing( + &self, + dense: &Mutex>>, + _info: ImageInfo, + sharing: SharingMode, + sharing_range: vk::ImageSubresourceRange, + ) { + let encoded_sharing = sharing.encode(); + + loop { + if self.uses_dense_sharing() { + let mut dense = dense.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut dense = dense.expect("poisoned image dense lock"); + + dense + .as_mut() + .expect("missing dense sharing state") + .swap(sharing, sharing_range); + + return; + } + + let current = self.uniform.load(Ordering::Acquire); + if self + .uniform + .compare_exchange( + current, + encoded_sharing, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + if self.is_promoting_dense_sharing() { + let mut dense = dense.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut dense = dense.expect("poisoned image dense lock"); + + dense + .as_mut() + .expect("missing dense sharing state") + .swap(sharing, sharing_range); + } + + return; + } + } + } + + fn promote_dense_sharing_and_set_ranges( + &self, + dense: &Mutex>>, + info: ImageInfo, + sharing: SharingMode, + sharing_ranges: &[vk::ImageSubresourceRange], + ) { + let mut dense = dense.lock(); + + #[cfg(not(feature = "parking_lot"))] + let mut dense = dense.expect("poisoned image dense lock"); + + if self.is_dense_sharing_active() { + let dense_sharing = dense.as_mut().expect("missing dense sharing state"); + for &sharing_range in sharing_ranges { + dense_sharing.swap(sharing, info.resolve_subresource_counts(sharing_range)); + } + + return; + } + + self.set_promoting_dense_sharing(); + + let current = SharingMode::decode(self.uniform.load(Ordering::Acquire)); + + *dense = Some(DenseMap::new(info, current)); + let sharing_state = dense.as_mut().expect("missing dense sharing state"); + for &sharing_range in sharing_ranges { + sharing_state.swap(sharing, info.resolve_subresource_counts(sharing_range)); + } + + self.set_dense_sharing_active(); + } +} + +/// Smart pointer handle to an [image] object. +/// +/// Also contains information about the object. +/// +/// ```no_run +/// # use ash::vk; +/// # use vk_sync::AccessType; +/// # use vk_graph::driver::DriverError; +/// # use vk_graph::driver::device::{Device, DeviceInfo}; +/// # use vk_graph::driver::image::{Image, ImageInfo}; +/// # fn main() -> Result<(), DriverError> { +/// # let device = Device::create(DeviceInfo::default())?; +/// let fmt = vk::Format::R8G8B8A8_UNORM; +/// let usage = vk::ImageUsageFlags::SAMPLED; +/// let info = ImageInfo::image_2d(320, 200, fmt, usage); +/// let my_img = Image::create(&device, info)?; +/// +/// assert_eq!(my_img.info, info); +/// assert_ne!(my_img.handle, vk::Image::null()); +/// # Ok(()) } +/// ``` +/// +/// [image]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkImage.html +#[read_only::cast] +pub struct Image { + access: Access, + allocation: Option, // None when we don't own the image (Swapchain images) + dense_access: Mutex>>, + dense_sharing: Mutex>>, + + /// The device which owns this image resource. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub device: Device, + + /// The native Vulkan resource handle of this image. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub handle: vk::Image, + + #[allow(clippy::type_complexity)] + image_view_cache: Mutex>, + + /// Information used to create this resource. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub info: ImageInfo, + + sharing: Sharing, +} + +impl Image { + /// Creates a new image on the given device. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use ash::vk; + /// # use vk_graph::driver::DriverError; + /// # use vk_graph::driver::device::{Device, DeviceInfo}; + /// # use vk_graph::driver::image::{Image, ImageInfo}; + /// # fn main() -> Result<(), DriverError> { + /// # let device = Device::create(DeviceInfo::default())?; + /// let info = ImageInfo::image_2d( + /// 32, + /// 32, + /// vk::Format::R8G8B8A8_UNORM, + /// vk::ImageUsageFlags::SAMPLED, + /// ); + /// let image = Image::create(&device, info)?; + /// + /// assert_ne!(image.handle, vk::Image::null()); + /// assert_eq!(image.info.width, 32); + /// assert_eq!(image.info.height, 32); + /// # Ok(()) } + /// ``` + #[profiling::function] + pub fn create(device: &Device, info: impl Into) -> Result { + let info = info.into(); //trace!("create: {:?}", &info); trace!("create"); @@ -446,8 +1017,11 @@ impl Image { let device = device.clone(); let create_info: ImageCreateInfo = info.into(); - let create_info = - create_info.queue_family_indices(&device.physical_device.queue_family_indices); + let create_info = if info.sharing_mode == vk::SharingMode::CONCURRENT { + create_info.queue_family_indices(&device.physical.queue_family_indices) + } else { + create_info + }; let handle = unsafe { device.create_image(&create_info, None).map_err(|err| { warn!("unable to create image: {err}"); @@ -456,7 +1030,7 @@ impl Image { })? }; let requirements = unsafe { device.get_image_memory_requirements(handle) }; - let allocation_scheme = if info.dedicated { + let allocation_scheme = if info.alloc_dedicated { AllocationScheme::DedicatedImage(handle) } else { AllocationScheme::GpuAllocatorManaged @@ -469,7 +1043,7 @@ impl Image { .allocate(&AllocationCreateDesc { name: "image", requirements, - location: MemoryLocation::GpuOnly, + location: info.memory_location(), linear: false, allocation_scheme, }) @@ -513,128 +1087,50 @@ impl Image { Ok(Self { access, allocation: Some(allocation), + dense_access: Mutex::new(None), + dense_sharing: Mutex::new(None), device, handle, image_view_cache: Mutex::new(Default::default()), info, - name: None, + sharing: Sharing::new(info, info.sharing_mode), }) } - /// Keeps track of some next `access` which affects a `range` this image. - /// - /// Returns the previous access for which a pipeline barrier should be used to prevent data - /// corruption. - /// - /// # Note - /// - /// Used to maintain object state when passing a _vk-graph_-created `vk::Image` handle to - /// external code such as [_Ash_] or [_Erupt_] bindings. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ```no_run - /// # use std::sync::Arc; - /// # use ash::vk; - /// # use vk_sync::AccessType; - /// # use vk_graph::driver::DriverError; - /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::image::{Image, ImageInfo}; - /// # fn main() -> Result<(), DriverError> { - /// # let device = Device::create(DeviceInfo::default())?; - /// # let info = ImageInfo::image_1d(1, vk::Format::R8_UINT, vk::ImageUsageFlags::STORAGE); - /// # let my_image = Image::create(&device, info)?; - /// # let my_subresource_range = vk::ImageSubresourceRange::default(); - /// // Initially we want to "Read Other" - /// let next = AccessType::AnyShaderReadOther; - /// let mut prev = Image::access(&my_image, next, my_subresource_range); - /// assert_eq!(prev.next().unwrap().0, AccessType::Nothing); - /// - /// // External code may now "Read Other"; no barrier required - /// - /// // Subsequently we want to "Write" - /// let next = AccessType::FragmentShaderWrite; - /// let mut prev = Image::access(&my_image, next, my_subresource_range); - /// assert_eq!(prev.next().unwrap().0, AccessType::AnyShaderReadOther); - /// - /// // A barrier on "Read Other" before "Write" is required! - /// # Ok(()) } - /// ``` - /// - /// [_Ash_]: https://crates.io/crates/ash - /// [_Erupt_]: https://crates.io/crates/erupt + /// Drops the given allocation, all views, and the handle. #[profiling::function] - pub fn access( - &self, - access: AccessType, - mut access_range: vk::ImageSubresourceRange, - ) -> impl Iterator + '_ { - #[cfg(feature = "checked")] + fn drop_allocation(&self, allocation: Allocation) { { - assert_aspect_mask_supported(access_range.aspect_mask); + profiling::scope!("views"); - assert!(format_aspect_mask(self.info.fmt).contains(access_range.aspect_mask)); + self.with_image_view_cache(|cache| cache.clear()); } - if access_range.layer_count == vk::REMAINING_ARRAY_LAYERS { - debug_assert!(access_range.base_array_layer < self.info.array_layer_count); - - access_range.layer_count = self.info.array_layer_count - access_range.base_array_layer + unsafe { + self.device.destroy_image(self.handle, None); } - debug_assert!( - access_range.base_array_layer + access_range.layer_count <= self.info.array_layer_count - ); - - if access_range.level_count == vk::REMAINING_MIP_LEVELS { - debug_assert!(access_range.base_mip_level < self.info.mip_level_count); + { + profiling::scope!("deallocate"); - access_range.level_count = self.info.mip_level_count - access_range.base_mip_level + Device::with_allocator(&self.device, |allocator| allocator.free(allocation)) } - - debug_assert!( - access_range.base_mip_level + access_range.level_count <= self.info.mip_level_count - ); - - self.access.swap(access, access_range) - } - - /// Sets the debugging name assigned to this image. - pub fn debug_name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); - - self - } - - /// Drops the given allocation, all views, and the handle. - #[profiling::function] - fn drop_allocation(&self, allocation: Allocation) { - { - profiling::scope!("views"); - - self.with_image_view_cache(|cache| cache.clear()); - } - - unsafe { - self.device.destroy_image(self.handle, None); - } - - { - profiling::scope!("deallocate"); - - Device::with_allocator(&self.device, |allocator| allocator.free(allocation)) - } - .unwrap_or_else(|err| warn!("unable to free image allocation: {err}")); + .unwrap_or_else(|err| warn!("unable to free image allocation: {err}")); } /// Consumes a Vulkan image created by some other library. /// /// The image is not destroyed automatically on drop, unlike images created through the /// [`Image::create`] function. + /// + /// # Safety + /// + /// `handle` must be a valid [`vk::Image`] created from `device`, and `info` must accurately + /// describe the image's format, extent, usage, sharing mode, and subresource counts. The caller + /// remains responsible for keeping the handle and its memory backing valid until all wrappers + /// created from this function are no longer used. #[profiling::function] - pub fn from_raw(device: &Device, handle: vk::Image, info: impl Into) -> Self { + pub unsafe fn from_raw(device: &Device, handle: vk::Image, info: impl Into) -> Self { let device = device.clone(); let info = info.into(); @@ -643,12 +1139,456 @@ impl Image { Self { access, allocation: None, + dense_access: Mutex::new(None), + dense_sharing: Mutex::new(None), device, handle, image_view_cache: Mutex::new(Default::default()), info, - name: None, + sharing: Sharing::new(info, info.sharing_mode), + } + } + + /// Sets the debugging name assigned to this image. + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.device, self.handle, &name); + Device::try_set_private_data_object_name( + &self.device, + vk::ObjectType::IMAGE, + self.handle, + &name, + ); + } + + pub(crate) fn set_sharing_ranges( + &self, + sharing: SharingMode, + sharing_ranges: &[vk::ImageSubresourceRange], + ) { + self.sharing + .set_ranges(&self.dense_sharing, self.info, sharing, sharing_ranges); + } + + /// Keeps track of some next `access` which affects a `range` of this image. + /// + /// Returns the previous access for which a pipeline barrier should be used to prevent data + /// corruption. + #[profiling::function] + pub(crate) fn swap_access( + &self, + next_access: AccessType, + mut access_range: vk::ImageSubresourceRange, + ) -> impl Iterator + '_ { + #[cfg(feature = "checked")] + { + assert_aspect_mask_supported(access_range.aspect_mask); + + assert!(format_aspect_mask(self.info.format).contains(access_range.aspect_mask)); + } + + if access_range.layer_count == vk::REMAINING_ARRAY_LAYERS { + debug_assert!(access_range.base_array_layer < self.info.array_layer_count); + + access_range.layer_count = self.info.array_layer_count - access_range.base_array_layer + } + + debug_assert!( + access_range.base_array_layer + access_range.layer_count <= self.info.array_layer_count + ); + + if access_range.level_count == vk::REMAINING_MIP_LEVELS { + debug_assert!(access_range.base_mip_level < self.info.mip_level_count); + + access_range.level_count = self.info.mip_level_count - access_range.base_mip_level + } + + debug_assert!( + access_range.base_mip_level + access_range.level_count <= self.info.mip_level_count + ); + + self.access + .swap(&self.dense_access, self.info, next_access, access_range) + } + + pub(crate) fn swap_accesses<'a, I>( + &'a self, + accesses: I, + ) -> impl Iterator + 'a + where + I: IntoIterator, + I::IntoIter: 'a, + { + let info = self.info; + let format_aspect_mask = format_aspect_mask(info.format); + let accesses = accesses + .into_iter() + .map(move |(next_access, access_range)| { + #[cfg(feature = "checked")] + { + assert_aspect_mask_supported(access_range.aspect_mask); + + assert!(format_aspect_mask.contains(access_range.aspect_mask)); + } + + (next_access, info.resolve_subresource_counts(access_range)) + }); + + struct Iter<'a, I> + where + I: Iterator, + { + access: &'a Access, + accesses: I, + dense_access: &'a Mutex>>, + info: ImageInfo, + current: Option<(AccessType, AccessIter<'a>)>, + } + + impl Iterator for Iter<'_, I> + where + I: Iterator, + { + type Item = (AccessType, AccessType, vk::ImageSubresourceRange); + + fn next(&mut self) -> Option { + loop { + if let Some((next_access, iter)) = self.current.as_mut() { + if let Some((prev_access, range)) = iter.next() { + return Some((*next_access, prev_access, range)); + } + + self.current = None; + } + + let (next_access, access_range) = self.accesses.next()?; + self.current = Some(( + next_access, + self.access + .swap(self.dense_access, self.info, next_access, access_range), + )); + } + } + } + + impl Drop for Iter<'_, I> + where + I: Iterator, + { + fn drop(&mut self) { + while self.next().is_some() {} + } + } + + Iter { + access: &self.access, + accesses, + dense_access: &self.dense_access, + info, + current: None, + } + } + + /// Returns compact synchronization information for the image's current subresource accesses. + pub fn sync_info(&self) -> ImageSyncInfo { + ImageSyncInfo { + subresources: ImageSyncInfo::compact_subresources( + self.sync_info_with_sharing() + .map(|(subresource, sharing)| subresource.into_public(sharing)), + ), + } + } + + pub(crate) fn sync_info_with_sharing( + &self, + ) -> impl Iterator { + self.sync_info_with_sharing_range(vk::ImageSubresourceRange { + aspect_mask: format_aspect_mask(self.info.format), + base_mip_level: 0, + level_count: self.info.mip_level_count, + base_array_layer: 0, + layer_count: self.info.array_layer_count, + }) + } + + pub(crate) fn sync_info_with_sharing_range( + &self, + query_range: vk::ImageSubresourceRange, + ) -> impl Iterator { + #[derive(Clone, Copy)] + enum SharingSource { + Concurrent, + Uniform(SharingMode), + Dense, + } + + let query_range = self.info.resolve_subresource_counts(query_range); + let subresource_ranges = ImageSubresourceRangeIter::new(query_range); + let format_aspect_mask = format_aspect_mask(self.info.format); + #[derive(Clone, Copy)] + enum AccessSource<'a> { + Uniform(AccessType), + DualAspect(&'a DualAspectAccess), + Dense, + } + + let access_source = match &self.access { + Access::Uniform(uniform) => AccessSource::Uniform(uniform.load()), + Access::DualAspect(dual) => AccessSource::DualAspect(dual), + Access::Dense(access) if access.uses_dense() => AccessSource::Dense, + Access::Dense(access) => AccessSource::Uniform(access.load()), + }; + let sharing_source = match &self.sharing { + Sharing::Concurrent => SharingSource::Concurrent, + Sharing::Exclusive(exclusive) if exclusive.uses_dense_sharing() => SharingSource::Dense, + Sharing::Exclusive(exclusive) => SharingSource::Uniform(SharingMode::decode( + exclusive.uniform.load(Ordering::Acquire), + )), + }; + + struct UniformSyncInfoIter { + access: AccessType, + sharing: SharingMode, + subresource_ranges: ImageSubresourceRangeIter, + } + + impl Iterator for UniformSyncInfoIter { + type Item = (ImageSubresourceSyncInfo, SharingMode); + + fn next(&mut self) -> Option { + self.subresource_ranges.next().map(|range| { + ( + ImageSubresourceSyncInfo::from_access(self.access, range), + self.sharing, + ) + }) + } + + fn size_hint(&self) -> (usize, Option) { + self.subresource_ranges.size_hint() + } + } + + impl ExactSizeIterator for UniformSyncInfoIter { + fn len(&self) -> usize { + self.subresource_ranges.len() + } + } + + struct DenseSyncInfoIter<'a> { + access_source: AccessSource<'a>, + format_aspect_mask: vk::ImageAspectFlags, + access_dense: Option>>>, + sharing_dense: Option>>>, + sharing_source: SharingSource, + subresource_ranges: ImageSubresourceRangeIter, + } + + impl Iterator for DenseSyncInfoIter<'_> { + type Item = (ImageSubresourceSyncInfo, SharingMode); + + fn next(&mut self) -> Option { + let range = self.subresource_ranges.next()?; + let aspect = aspect_ordinal(self.format_aspect_mask, range.aspect_mask); + let access = match self.access_source { + AccessSource::Uniform(access) => access, + AccessSource::DualAspect(dual) => dual.load(aspect as usize), + AccessSource::Dense => self + .access_dense + .as_ref() + .expect("missing dense access state") + .as_ref() + .expect("missing dense access map") + .subresource(aspect, range.base_array_layer, range.base_mip_level), + }; + let sharing = match self.sharing_source { + SharingSource::Concurrent => SharingMode::Concurrent, + SharingSource::Uniform(sharing) => sharing, + SharingSource::Dense => self + .sharing_dense + .as_ref() + .expect("missing dense sharing state") + .as_ref() + .expect("missing dense sharing map") + .subresource(aspect, range.base_array_layer, range.base_mip_level), + }; + + Some(( + ImageSubresourceSyncInfo::from_access(access, range), + sharing, + )) + } + + fn size_hint(&self) -> (usize, Option) { + self.subresource_ranges.size_hint() + } + } + + impl ExactSizeIterator for DenseSyncInfoIter<'_> { + fn len(&self) -> usize { + self.subresource_ranges.len() + } + } + + enum SyncInfoIter<'a> { + Uniform(UniformSyncInfoIter), + Dense(DenseSyncInfoIter<'a>), + } + + impl Iterator for SyncInfoIter<'_> { + type Item = (ImageSubresourceSyncInfo, SharingMode); + + fn next(&mut self) -> Option { + match self { + Self::Uniform(iter) => iter.next(), + Self::Dense(iter) => iter.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + + (len, Some(len)) + } + } + + impl ExactSizeIterator for SyncInfoIter<'_> { + fn len(&self) -> usize { + match self { + Self::Uniform(iter) => iter.len(), + Self::Dense(iter) => iter.len(), + } + } + } + + let uniform_sharing = match sharing_source { + SharingSource::Concurrent => Some(SharingMode::Concurrent), + SharingSource::Uniform(sharing) => Some(sharing), + SharingSource::Dense => None, + }; + + let sync_infos = if let (AccessSource::Uniform(access), Some(sharing)) = + (access_source, uniform_sharing) + { + SyncInfoIter::Uniform(UniformSyncInfoIter { + access, + sharing, + subresource_ranges, + }) + } else { + let access_dense = if matches!(access_source, AccessSource::Dense) { + let dense = self.dense_access.lock(); + + #[cfg(not(feature = "parking_lot"))] + let dense = dense.expect("poisoned image dense access lock"); + + Some(dense) + } else { + None + }; + let sharing_dense = if matches!(sharing_source, SharingSource::Dense) { + let dense = self.dense_sharing.lock(); + + #[cfg(not(feature = "parking_lot"))] + let dense = dense.expect("poisoned image dense sharing lock"); + + Some(dense) + } else { + None + }; + + SyncInfoIter::Dense(DenseSyncInfoIter { + access_source, + format_aspect_mask, + access_dense, + sharing_dense, + sharing_source, + subresource_ranges, + }) + }; + + struct CompactIter { + iter: I, + pending: Option<(ImageSubresourceSyncInfo, SharingMode)>, + can_merge: P, + merge: M, + } + + /* + Lazily compacts adjacent iterator entries. Each pass is linear in the number of source + entries and keeps only one pending entry, so it uses `O(1)` extra memory. The image sync + iterator applies this twice: first to merge mip levels, then to merge array layers. + */ + impl CompactIter + where + I: Iterator, + P: Fn( + (ImageSubresourceSyncInfo, SharingMode), + (ImageSubresourceSyncInfo, SharingMode), + ) -> bool, + M: Fn( + &mut (ImageSubresourceSyncInfo, SharingMode), + (ImageSubresourceSyncInfo, SharingMode), + ), + { + fn new(iter: I, can_merge: P, merge: M) -> Self { + Self { + iter, + pending: None, + can_merge, + merge, + } + } + } + + impl Iterator for CompactIter + where + I: Iterator, + P: Fn( + (ImageSubresourceSyncInfo, SharingMode), + (ImageSubresourceSyncInfo, SharingMode), + ) -> bool, + M: Fn( + &mut (ImageSubresourceSyncInfo, SharingMode), + (ImageSubresourceSyncInfo, SharingMode), + ), + { + type Item = (ImageSubresourceSyncInfo, SharingMode); + + fn next(&mut self) -> Option { + let mut pending = self.pending.take().or_else(|| self.iter.next())?; + + for next in self.iter.by_ref() { + if (self.can_merge)(pending, next) { + (self.merge)(&mut pending, next); + } else { + self.pending = Some(next); + return Some(pending); + } + } + + Some(pending) + } } + + let same_sync_and_sharing = + |lhs: (ImageSubresourceSyncInfo, SharingMode), + rhs: (ImageSubresourceSyncInfo, SharingMode)| { + lhs.0.same_sync(rhs.0) && lhs.1 == rhs.1 + }; + let merge_array_layers = + |lhs: &mut (ImageSubresourceSyncInfo, SharingMode), + rhs: (ImageSubresourceSyncInfo, SharingMode)| { + lhs.0.merge_array_layers(rhs.0); + }; + let merge_mip_levels = + |lhs: &mut (ImageSubresourceSyncInfo, SharingMode), + rhs: (ImageSubresourceSyncInfo, SharingMode)| { + lhs.0.merge_mip_levels(rhs.0); + }; + + let mip_levels = CompactIter::new(sync_infos, same_sync_and_sharing, merge_mip_levels); + + CompactIter::new(mip_levels, same_sync_and_sharing, merge_array_layers) } /// Produces a new `Image` sharing the same Vulkan handle with independent access tracking. @@ -676,11 +1616,13 @@ impl Image { Self { access: Access::new(info, AccessType::Nothing), allocation: None, + dense_access: Mutex::new(None), + dense_sharing: Mutex::new(None), device: self.device.clone(), handle, image_view_cache: Mutex::new(image_view_cache), info, - name: self.name.clone(), + sharing: Sharing::new(info, info.sharing_mode), } } @@ -698,6 +1640,13 @@ impl Image { }) } + /// Sets the debugging name assigned to this image. + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); + + self + } + fn with_image_view_cache( &self, f: impl FnOnce(&mut HashMap) -> R, @@ -715,25 +1664,44 @@ impl Image { impl Debug for Image { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - if let Some(name) = &self.name { - write!(f, "{} ({:?})", name, self.handle) - } else { - write!(f, "{:?}", self.handle) + let mut res = f.debug_struct(stringify!(Image)); + + if let Some(debug_name) = + &Device::private_data_object_name(&self.device, vk::ObjectType::IMAGE, self.handle) + { + res.field("debug_name", debug_name); } + + res.field("handle", &self.handle).finish_non_exhaustive() } } impl Drop for Image { - // This function is not profiled because drop_allocation is + // This function is not profiled because dropping the allocation may run during shutdown. fn drop(&mut self) { if panicking() { return; } - // When our allocation is some we allocated ourself; otherwise somebody - // else owns this image and we should not destroy it. Usually it's the swapchain... + /* + When allocation is Some, we allocated the image ourselves; otherwise somebody else owns this + image and we should not destroy it. Usually it's the swapchain. + */ if let Some(allocation) = self.allocation.take() { + Device::try_clear_private_data_object_name( + &self.device, + vk::ObjectType::IMAGE, + self.handle, + ); Self::drop_allocation(self, allocation); + } else { + // Non-owned handles may already be invalid when their owner, such as a swapchain, has + // been destroyed. Remove local metadata without issuing vkSetPrivateDataEXT. + Device::forget_private_data_object_name( + &self.device, + vk::ObjectType::IMAGE, + self.handle, + ); } } } @@ -746,33 +1714,9 @@ impl PartialEq for Image { } } -enum ImageAccessIter<'a> { - Uniform(Option<(AccessType, vk::ImageSubresourceRange)>), - Dense(DenseAccessIter<'a, MutexGuard<'a, DenseAccess>, AccessType>), -} - -impl Iterator for ImageAccessIter<'_> { - type Item = (AccessType, vk::ImageSubresourceRange); - - fn next(&mut self) -> Option { - match self { - Self::Uniform(item) => item.take(), - Self::Dense(iter) => iter.next(), - } - } -} - -#[derive(Copy, Clone)] -struct ImageAccessRange { - aspect_count: u8, - base_array_layer: u32, - base_aspect: u8, - base_mip_level: u32, - layer_count: u32, - level_count: u32, -} - /// Information used to create an [`Image`] instance. +/// +/// See [`VkImageCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkImageCreateInfo.html). #[derive(Builder, Clone, Copy, Debug, Hash, PartialEq, Eq)] #[builder( build_fn(private, name = "fallible_build"), @@ -780,33 +1724,49 @@ struct ImageAccessRange { pattern = "owned" )] pub struct ImageInfo { - /// The number of layers in the image. - #[builder(default = "1")] - pub array_layer_count: u32, - /// Specifies a dedicated memory allocation managed by the Vulkan driver and not by the /// internal memory allocation pool transient resources share. /// - /// The driver may optimize access to dedicated buffers. + /// The driver may optimize access to dedicated images. #[builder(default)] - pub dedicated: bool, + pub alloc_dedicated: bool, + + /// The number of layers in the image. + #[builder(default = "1")] + pub array_layer_count: u32, /// Image extent of the Z axis, when describing a three dimensional image. #[builder(default)] pub depth: u32, - /// A bitmask of describing additional parameters of the image. + /// A bitmask describing additional parameters of the image. + /// + /// See [`VkImageCreateFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkImageCreateFlagBits.html). #[builder(default)] pub flags: vk::ImageCreateFlags, /// The format and type of the texel blocks that will be contained in the image. #[builder(default = "vk::Format::UNDEFINED")] - pub fmt: vk::Format, + pub format: vk::Format, /// Image extent of the Y axis, when describing a two or three dimensional image. #[builder(default)] pub height: u32, + /// Specifies an image whose memory is host-visible and may be mapped for reads. + /// + /// Memory optimal for CPU readback of data may be used. + /// + #[builder(default)] + pub host_readable: bool, + + /// Specifies an image whose memory is host-visible and may be mapped for writes. + /// + /// Memory optimal for uploading data to the GPU may be used. + /// + #[builder(default)] + pub host_writable: bool, + /// The number of levels of detail available for minified sampling of the image. #[builder(default = "1")] pub mip_level_count: u32, @@ -817,6 +1777,22 @@ pub struct ImageInfo { #[builder(default = "SampleCount::Type1")] pub sample_count: SampleCount, + /// Controls whether the image is accessible from a single queue family (`EXCLUSIVE`) or from + /// multiple queue families concurrently (`CONCURRENT`). + /// + /// `EXCLUSIVE` (the default) restricts the image to a single queue family. This may enable + /// driver optimizations but requires ownership transfers to use the image on a different queue + /// family. + /// + /// Set to `CONCURRENT` when the image will be accessed from multiple queue families (e.g. + /// graphics and compute on separate queues). + /// + /// See + /// [`VkSharingMode`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSharingMode.html) + /// in the Vulkan specification. + #[builder(default = "vk::SharingMode::EXCLUSIVE")] + pub sharing_mode: vk::SharingMode, + /// Specifies the tiling arrangement of the texel blocks in memory. /// /// The default value is [`vk::ImageTiling::OPTIMAL`]. @@ -827,9 +1803,11 @@ pub struct ImageInfo { /// /// Layers in array textures do not count as a dimension for the purposes of the image type. #[builder(default = "vk::ImageType::TYPE_2D")] - pub ty: vk::ImageType, + pub image_type: vk::ImageType, - /// A bitmask of describing the intended usage of the image. + /// A bitmask describing the intended usage of the image. + /// + /// See [`VkImageUsageFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkImageUsageFlagBits.html). #[builder(default)] pub usage: vk::ImageUsageFlags, @@ -841,8 +1819,8 @@ pub struct ImageInfo { impl ImageInfo { /// Specifies a cube image. #[inline(always)] - pub const fn cube(size: u32, fmt: vk::Format, usage: vk::ImageUsageFlags) -> ImageInfo { - let mut res = Self::new(vk::ImageType::TYPE_2D, size, size, 1, 6, fmt, usage); + pub const fn cube(size: u32, format: vk::Format, usage: vk::ImageUsageFlags) -> ImageInfo { + let mut res = Self::new(vk::ImageType::TYPE_2D, size, size, 1, 6, format, usage); res.flags = vk::ImageCreateFlags::from_raw( vk::ImageCreateFlags::CUBE_COMPATIBLE.as_raw() | res.flags.as_raw(), ); @@ -852,8 +1830,8 @@ impl ImageInfo { /// Specifies a one-dimensional image. #[inline(always)] - pub const fn image_1d(size: u32, fmt: vk::Format, usage: vk::ImageUsageFlags) -> ImageInfo { - Self::new(vk::ImageType::TYPE_1D, size, 1, 1, 1, fmt, usage) + pub const fn image_1d(size: u32, format: vk::Format, usage: vk::ImageUsageFlags) -> ImageInfo { + Self::new(vk::ImageType::TYPE_1D, size, 1, 1, 1, format, usage) } /// Specifies a two-dimensional image. @@ -861,10 +1839,10 @@ impl ImageInfo { pub const fn image_2d( width: u32, height: u32, - fmt: vk::Format, + format: vk::Format, usage: vk::ImageUsageFlags, ) -> ImageInfo { - Self::new(vk::ImageType::TYPE_2D, width, height, 1, 1, fmt, usage) + Self::new(vk::ImageType::TYPE_2D, width, height, 1, 1, format, usage) } /// Specifies a two-dimensional image array. @@ -872,8 +1850,8 @@ impl ImageInfo { pub const fn image_2d_array( width: u32, height: u32, - array_elements: u32, - fmt: vk::Format, + array_layer_count: u32, + format: vk::Format, usage: vk::ImageUsageFlags, ) -> ImageInfo { Self::new( @@ -881,8 +1859,8 @@ impl ImageInfo { width, height, 1, - array_elements, - fmt, + array_layer_count, + format, usage, ) } @@ -893,32 +1871,43 @@ impl ImageInfo { width: u32, height: u32, depth: u32, - fmt: vk::Format, + format: vk::Format, usage: vk::ImageUsageFlags, ) -> ImageInfo { - Self::new(vk::ImageType::TYPE_3D, width, height, depth, 1, fmt, usage) - } - - #[inline(always)] + Self::new( + vk::ImageType::TYPE_3D, + width, + height, + depth, + 1, + format, + usage, + ) + } + + #[inline(always)] const fn new( - ty: vk::ImageType, + image_type: vk::ImageType, width: u32, height: u32, depth: u32, array_layer_count: u32, - fmt: vk::Format, + format: vk::Format, usage: vk::ImageUsageFlags, ) -> Self { Self { - dedicated: false, - ty, + alloc_dedicated: false, + image_type, width, height, depth, array_layer_count, - fmt, + format, usage, flags: vk::ImageCreateFlags::empty(), + host_readable: false, + host_writable: false, + sharing_mode: vk::SharingMode::EXCLUSIVE, tiling: vk::ImageTiling::OPTIMAL, mip_level_count: 1, sample_count: SampleCount::Type1, @@ -935,38 +1924,79 @@ impl ImageInfo { self.into() } - /// Returns `true` if this image is an array + pub(crate) fn resolve_subresource_counts( + self, + mut range: vk::ImageSubresourceRange, + ) -> vk::ImageSubresourceRange { + if range.layer_count == vk::REMAINING_ARRAY_LAYERS { + range.layer_count = self.array_layer_count - range.base_array_layer; + } + + if range.level_count == vk::REMAINING_MIP_LEVELS { + range.level_count = self.mip_level_count - range.base_mip_level; + } + + range + } + + fn is_full_subresource_range(self, range: vk::ImageSubresourceRange) -> bool { + range.aspect_mask == format_aspect_mask(self.format) + && range.base_array_layer == 0 + && range.layer_count == self.array_layer_count + && range.base_mip_level == 0 + && range.level_count == self.mip_level_count + } + + /// Returns `true` if this image is an array. pub fn is_array(self) -> bool { self.array_layer_count > 1 } - /// Returns `true` if this image is a cube or cube array + /// Returns `true` if this image is a cube or cube array. pub fn is_cube(self) -> bool { - self.ty == vk::ImageType::TYPE_2D + self.image_type == vk::ImageType::TYPE_2D && self.width == self.height && self.depth == 1 && self.array_layer_count >= 6 && self.flags.contains(vk::ImageCreateFlags::CUBE_COMPATIBLE) } - /// Returns `true` if this image is a cube array + /// Returns `true` if this image is a cube array. pub fn is_cube_array(self) -> bool { self.is_cube() && self.array_layer_count > 6 } + /// Returns `true` if this information specifies host-visible memory. + pub fn is_host_visible(self) -> bool { + self.host_readable | self.host_writable + } + + const fn memory_location(self) -> MemoryLocation { + if self.host_writable { + MemoryLocation::CpuToGpu + } else if self.host_readable { + MemoryLocation::GpuToCpu + } else { + MemoryLocation::GpuOnly + } + } + /// Converts an `ImageInfo` into an `ImageInfoBuilder`. pub fn into_builder(self) -> ImageInfoBuilder { ImageInfoBuilder { array_layer_count: Some(self.array_layer_count), - dedicated: Some(self.dedicated), + alloc_dedicated: Some(self.alloc_dedicated), depth: Some(self.depth), flags: Some(self.flags), - fmt: Some(self.fmt), + format: Some(self.format), height: Some(self.height), + host_readable: Some(self.host_readable), + host_writable: Some(self.host_writable), mip_level_count: Some(self.mip_level_count), sample_count: Some(self.sample_count), + sharing_mode: Some(self.sharing_mode), tiling: Some(self.tiling), - ty: Some(self.ty), + image_type: Some(self.image_type), usage: Some(self.usage), width: Some(self.width), } @@ -977,8 +2007,8 @@ impl From for vk::ImageCreateInfo<'_> { fn from(value: ImageInfo) -> Self { Self::default() .flags(value.flags) - .image_type(value.ty) - .format(value.fmt) + .image_type(value.image_type) + .format(value.format) .extent(vk::Extent3D { width: value.width, height: value.height, @@ -989,7 +2019,7 @@ impl From for vk::ImageCreateInfo<'_> { .samples(value.sample_count.into()) .tiling(value.tiling) .usage(value.usage) - .sharing_mode(vk::SharingMode::CONCURRENT) + .sharing_mode(value.sharing_mode) .initial_layout(vk::ImageLayout::UNDEFINED) } } @@ -1021,6 +2051,220 @@ impl ImageInfoBuilder { } } +struct ImageSubresourceRangeIter { + aspect_mask: vk::ImageAspectFlags, + aspect: u8, + aspect_count: u8, + array_layer: u32, + end_array_layer: u32, + base_array_layer: u32, + base_mip_level: u32, + mip_level: u32, + end_mip_level: u32, + remaining: usize, +} + +impl ImageSubresourceRangeIter { + fn new(range: vk::ImageSubresourceRange) -> Self { + let aspect_mask = range.aspect_mask; + let aspect_count = aspect_mask.as_raw().count_ones() as u8; + + Self { + aspect_mask, + aspect: 0, + aspect_count, + array_layer: range.base_array_layer, + end_array_layer: range.base_array_layer + range.layer_count, + base_array_layer: range.base_array_layer, + base_mip_level: range.base_mip_level, + mip_level: range.base_mip_level, + end_mip_level: range.base_mip_level + range.level_count, + remaining: aspect_count as usize + * range.layer_count as usize + * range.level_count as usize, + } + } +} + +impl ExactSizeIterator for ImageSubresourceRangeIter { + fn len(&self) -> usize { + self.remaining + } +} + +impl Iterator for ImageSubresourceRangeIter { + type Item = vk::ImageSubresourceRange; + + fn next(&mut self) -> Option { + if self.aspect >= self.aspect_count { + return None; + } + + let range = vk::ImageSubresourceRange { + aspect_mask: aspect_mask_at_ordinal(self.aspect_mask, self.aspect as u32), + base_array_layer: self.array_layer, + layer_count: 1, + base_mip_level: self.mip_level, + level_count: 1, + }; + + self.mip_level += 1; + if self.mip_level >= self.end_mip_level { + self.mip_level = self.base_mip_level; + self.array_layer += 1; + if self.array_layer >= self.end_array_layer { + self.array_layer = self.base_array_layer; + self.aspect += 1; + } + } + + self.remaining -= 1; + + Some(range) + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + + (len, Some(len)) + } +} + +/// Synchronization information for one accessed image subresource range. +#[derive(Clone, Copy, Debug)] +pub struct ImageSubresourceSyncInfo { + /// Access types performed by `stage_mask`. + pub access_mask: vk::AccessFlags, + + /// Required image layout for the next external use, when one is defined. + pub layout: Option, + + /// Queue-family ownership for this subresource, when exclusive ownership is known. + pub queue_family_index: Option, + + /// The tracked image subresource range. + pub range: vk::ImageSubresourceRange, + + /// Pipeline stages that access this `range`. + pub stage_mask: vk::PipelineStageFlags, +} + +impl ImageSubresourceSyncInfo { + fn can_merge_array_layers(self, other: Self) -> bool { + self.same_sync(other) + && self.range.aspect_mask == other.range.aspect_mask + && self.range.base_mip_level == other.range.base_mip_level + && self.range.level_count == other.range.level_count + && self.range.base_array_layer + self.range.layer_count == other.range.base_array_layer + } + + fn can_merge_mip_levels(self, other: Self) -> bool { + self.same_sync(other) + && self.range.aspect_mask == other.range.aspect_mask + && self.range.base_array_layer == other.range.base_array_layer + && self.range.layer_count == other.range.layer_count + && self.range.base_mip_level + self.range.level_count == other.range.base_mip_level + } + + fn from_access(access: AccessType, range: vk::ImageSubresourceRange) -> Self { + let (stage_mask, access_mask) = pipeline_stage_access_flags(access); + + Self { + access_mask, + layout: access_type_to_layout(access), + queue_family_index: None, + range, + stage_mask, + } + } + + fn into_public(self, sharing: SharingMode) -> Self { + Self { + queue_family_index: match sharing { + SharingMode::Concurrent | SharingMode::Exclusive(None) => None, + SharingMode::Exclusive(Some((queue_family_index, _))) => Some(queue_family_index), + }, + ..self + } + } + + fn merge_array_layers(&mut self, other: Self) { + self.range.layer_count += other.range.layer_count; + } + + fn merge_mip_levels(&mut self, other: Self) { + self.range.level_count += other.range.level_count; + } + + fn same_sync(self, other: Self) -> bool { + self.access_mask == other.access_mask + && self.layout == other.layout + && self.queue_family_index == other.queue_family_index + && self.stage_mask == other.stage_mask + } +} + +/// Synchronization information for an image. +#[derive(Clone, Debug)] +pub struct ImageSyncInfo { + /// Access state for the tracked image subresource ranges. + pub subresources: Box<[ImageSubresourceSyncInfo]>, +} + +impl ImageSyncInfo { + fn compact_subresources( + subresources: impl IntoIterator, + ) -> Box<[ImageSubresourceSyncInfo]> { + let mut mip_levels = Vec::new(); + + for sync_info in subresources { + if let Some(prev) = mip_levels.last_mut() + && ImageSubresourceSyncInfo::can_merge_mip_levels(*prev, sync_info) + { + prev.merge_mip_levels(sync_info); + } else { + mip_levels.push(sync_info); + } + } + + let mut array_layers = Vec::with_capacity(mip_levels.len()); + + for sync_info in mip_levels { + if let Some(prev) = array_layers.last_mut() + && ImageSubresourceSyncInfo::can_merge_array_layers(*prev, sync_info) + { + prev.merge_array_layers(sync_info); + } else { + array_layers.push(sync_info); + } + } + + array_layers.into_boxed_slice() + } + + /// Compacts adjacent subresource entries with identical synchronization requirements. + /// + /// This is opt-in because some callers may prefer the exact per-range snapshot produced by the + /// internal access tracker. + /// + /// Runs in linear time over `subresources`. Image compaction performs two passes, first merging + /// adjacent mip levels and then adjacent array layers, and uses temporary vector storage for + /// each pass. + pub fn compact(&mut self) { + let subresources = take(&mut self.subresources); + self.subresources = Self::compact_subresources(subresources); + } + + /// Returns a compacted copy of this synchronization snapshot. + /// + /// This has the same linear-time and temporary-storage characteristics as [`Self::compact`], + /// but consumes and returns the snapshot for use in iterator chains or expression-oriented code. + pub fn into_compacted(mut self) -> Self { + self.compact(); + self + } +} + struct ImageView { device: Device, image_view: vk::ImageView, @@ -1036,8 +2280,8 @@ impl ImageView { let info = info.into(); let device = device.clone(); let create_info = vk::ImageViewCreateInfo::default() - .view_type(info.ty) - .format(info.fmt) + .view_type(info.view_type) + .format(info.format) .components(vk::ComponentMapping { r: vk::ComponentSwizzle::R, g: vk::ComponentSwizzle::G, @@ -1078,6 +2322,8 @@ impl Drop for ImageView { } /// Information used to reinterpret an existing [`Image`] instance. +/// +/// See [`VkImageViewCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkImageViewCreateInfo.html). #[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build"), @@ -1105,7 +2351,7 @@ pub struct ImageViewInfo { /// The format and type of the texel blocks that will be contained in the view. #[builder(default = "vk::Format::UNDEFINED")] - pub fmt: vk::Format, + pub format: vk::Format, /// The number of mip levels that will be contained in the view. /// @@ -1115,46 +2361,40 @@ pub struct ImageViewInfo { /// The basic dimensionality of the view. #[builder(default = "vk::ImageViewType::TYPE_2D")] - pub ty: vk::ImageViewType, + pub view_type: vk::ImageViewType, } impl ImageViewInfo { - /// Specifies a default view with the given `fmt` and `ty` values. + /// Specifies a default view with the given `format` and `view_type` values. /// /// # Note /// /// Automatically sets [`aspect_mask`](Self::aspect_mask) to a suggested value. #[inline(always)] - pub const fn new(fmt: vk::Format, ty: vk::ImageViewType) -> ImageViewInfo { + pub const fn new(format: vk::Format, view_type: vk::ImageViewType) -> ImageViewInfo { Self { array_layer_count: vk::REMAINING_ARRAY_LAYERS, - aspect_mask: format_aspect_mask(fmt), + aspect_mask: format_aspect_mask(format), base_array_layer: 0, base_mip_level: 0, - fmt, + format, mip_level_count: vk::REMAINING_MIP_LEVELS, - ty, + view_type, } } - /// Converts a `ImageViewInfo` into a `ImageViewInfoBuilder`. + /// Converts an `ImageViewInfo` into an `ImageViewInfoBuilder`. pub fn into_builder(self) -> ImageViewInfoBuilder { ImageViewInfoBuilder { array_layer_count: Some(self.array_layer_count), aspect_mask: Some(self.aspect_mask), base_array_layer: Some(self.base_array_layer), base_mip_level: Some(self.base_mip_level), - fmt: Some(self.fmt), + format: Some(self.format), mip_level_count: Some(self.mip_level_count), - ty: Some(self.ty), + view_type: Some(self.view_type), } } - - /// Takes this instance and returns it with a newly specified `ImageViewType`. - pub fn with_type(mut self, ty: vk::ImageViewType) -> Self { - self.ty = ty; - self - } } impl From for ImageViewInfo { @@ -1168,12 +2408,12 @@ impl ImageViewInfo { pub fn from_image_info(info: ImageInfo) -> Result { Ok(Self { array_layer_count: info.array_layer_count, - aspect_mask: format_aspect_mask(info.fmt), + aspect_mask: format_aspect_mask(info.format), base_array_layer: 0, base_mip_level: 0, - fmt: info.fmt, + format: info.format, mip_level_count: info.mip_level_count, - ty: match (info.ty, info.array_layer_count) { + view_type: match (info.image_type, info.array_layer_count) { (vk::ImageType::TYPE_1D, 1) => vk::ImageViewType::TYPE_1D, (vk::ImageType::TYPE_1D, _) => vk::ImageViewType::TYPE_1D_ARRAY, (vk::ImageType::TYPE_2D, 1) => vk::ImageViewType::TYPE_2D, @@ -1193,7 +2433,7 @@ impl ImageViewInfo { _ => { warn!( "invalid image view source info: image type {:?} with {} array layers", - info.ty, info.array_layer_count + info.image_type, info.array_layer_count ); return Err(DriverError::InvalidData); @@ -1222,7 +2462,7 @@ impl From for vk::ImageSubresourceRange { } impl ImageViewInfoBuilder { - /// Builds a new 'ImageViewInfo'. + /// Builds a new `ImageViewInfo`. #[inline(always)] pub fn build(self) -> ImageViewInfo { self.fallible_build().expect("all fields have defaults") @@ -1231,10 +2471,14 @@ impl ImageViewInfoBuilder { /// Specifies sample counts supported for an image used for storage operation. /// -/// Values must not exceed the device limits specified by [Device.physical_device.props.limits]. +/// Values must not exceed the device limits specified by the physical device properties. +/// +/// See [`VkSampleCountFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSampleCountFlagBits.html). #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub enum SampleCount { /// Single image sample. This is the usual mode. + /// + /// This is the default value. #[default] Type1, @@ -1287,20 +2531,50 @@ impl From for vk::SampleCountFlags { } #[derive(Debug)] -struct UniformAccess { - access: AtomicU8, +enum Sharing { + Concurrent, + Exclusive(ExclusiveSharing), +} + +impl Sharing { + fn new(info: ImageInfo, sharing_mode: vk::SharingMode) -> Self { + if sharing_mode == vk::SharingMode::CONCURRENT { + Self::Concurrent + } else { + Self::Exclusive(ExclusiveSharing::new(info)) + } + } + + fn set_ranges( + &self, + dense: &Mutex>>, + info: ImageInfo, + sharing: SharingMode, + sharing_ranges: &[vk::ImageSubresourceRange], + ) { + let Self::Exclusive(exclusive) = self else { + return; + }; + + exclusive.set_ranges(dense, info, sharing, sharing_ranges); + } } +#[derive(Debug)] +struct UniformAccess(AtomicU8); + impl UniformAccess { fn new(access: AccessType) -> Self { - Self { - access: AtomicU8::new(access_type_into_u8(access)), - } + Self(AtomicU8::new(access_type_into_u8(access))) + } + + fn load(&self) -> AccessType { + access_type_from_u8(self.0.load(Ordering::Acquire)) } fn swap( &self, - access: AccessType, + next_access: AccessType, access_range: vk::ImageSubresourceRange, ) -> (AccessType, vk::ImageSubresourceRange) { debug_assert_eq!(access_range.base_array_layer, 0); @@ -1309,61 +2583,447 @@ impl UniformAccess { debug_assert_eq!(access_range.level_count, 1); debug_assert_eq!(access_range.aspect_mask.as_raw().count_ones(), 1); - let prev_access = access_type_from_u8( - self.access - .swap(access_type_into_u8(access), Ordering::AcqRel), - ); + self.swap_range(next_access, access_range) + } + + fn swap_range( + &self, + next_access: AccessType, + access_range: vk::ImageSubresourceRange, + ) -> (AccessType, vk::ImageSubresourceRange) { + let prev_access = access_type_from_u8( + self.0 + .swap(access_type_into_u8(next_access), Ordering::AcqRel), + ); + + (prev_access, access_range) + } +} + +#[doc(hidden)] +pub mod bench { + use super::*; + + pub struct SwapAccessBenchHarness { + access: Access, + dense_access: Mutex>>, + info: ImageInfo, + } + + impl SwapAccessBenchHarness { + pub fn new(layers: u32, mips: u32, format: vk::Format) -> Self { + let info = ImageInfo::image_2d(1, 1, format, vk::ImageUsageFlags::empty()) + .into_builder() + .array_layer_count(layers) + .mip_level_count(mips) + .build(); + Self { + access: Access::new(info, AccessType::Nothing), + dense_access: Mutex::new(None), + info, + } + } + + pub fn swap_access( + &self, + next_access: AccessType, + mut access_range: vk::ImageSubresourceRange, + ) -> Vec<(AccessType, vk::ImageSubresourceRange)> { + #[cfg(feature = "checked")] + { + assert_aspect_mask_supported(access_range.aspect_mask); + assert!(format_aspect_mask(self.info.format).contains(access_range.aspect_mask)); + } + + if access_range.layer_count == vk::REMAINING_ARRAY_LAYERS { + debug_assert!(access_range.base_array_layer < self.info.array_layer_count); + access_range.layer_count = + self.info.array_layer_count - access_range.base_array_layer; + } + + debug_assert!( + access_range.base_array_layer + access_range.layer_count + <= self.info.array_layer_count + ); + + if access_range.level_count == vk::REMAINING_MIP_LEVELS { + debug_assert!(access_range.base_mip_level < self.info.mip_level_count); + access_range.level_count = self.info.mip_level_count - access_range.base_mip_level; + } + + debug_assert!( + access_range.base_mip_level + access_range.level_count <= self.info.mip_level_count + ); + + self.access + .swap(&self.dense_access, self.info, next_access, access_range) + .collect() + } + } +} + +#[cfg(test)] +mod test { + use { + super::*, + rand::{Rng, SeedableRng, rngs::SmallRng}, + std::ops::Range, + }; + + // ImageSubresourceRange does not implement PartialEq + fn assert_access_ranges_eq( + lhs: (AccessType, vk::ImageSubresourceRange), + rhs: (AccessType, vk::ImageSubresourceRange), + ) { + assert_eq!( + ( + lhs.0, + lhs.1.aspect_mask, + lhs.1.base_array_layer, + lhs.1.layer_count, + lhs.1.base_mip_level, + lhs.1.level_count + ), + ( + rhs.0, + rhs.1.aspect_mask, + rhs.1.base_array_layer, + rhs.1.layer_count, + rhs.1.base_mip_level, + rhs.1.level_count + ) + ); + } + + fn image_sync_subresource( + aspect_mask: vk::ImageAspectFlags, + array_layers: Range, + mip_levels: Range, + ) -> ImageSubresourceSyncInfo { + ImageSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_READ, + layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), + queue_family_index: None, + range: image_subresource_range(aspect_mask, array_layers, mip_levels), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + } + } + + #[test] + pub fn image_access_basic() { + use vk::ImageAspectFlags as A; + + let mut image = DenseMap::new( + image_subresource(vk::Format::R8G8B8A8_UNORM, 1, 1), + AccessType::Nothing, + ); + + { + let mut accesses = DenseMapIter::new( + &mut image, + AccessType::AnyShaderWrite, + image_subresource_range(A::COLOR, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::COLOR, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } + + { + let mut accesses = DenseMapIter::new( + &mut image, + AccessType::AnyShaderReadOther, + image_subresource_range(A::COLOR, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::AnyShaderWrite, + image_subresource_range(A::COLOR, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } + } + + #[test] + pub fn image_access_uniform() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::R8G8B8A8_UNORM, 1, 1); + let image = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderWrite, + image_subresource_range(A::COLOR, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::COLOR, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } + + #[test] + pub fn image_access_dual_aspect_tracks_aspects_independently() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::D32_SFLOAT_S8_UINT, 1, 1); + let image = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + + let mut accesses = image.swap( + &dense, + info, + AccessType::DepthStencilAttachmentWrite, + image_subresource_range(A::DEPTH, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::DEPTH, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + + let mut accesses = image.swap( + &dense, + info, + AccessType::DepthStencilAttachmentRead, + image_subresource_range(A::STENCIL, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::STENCIL, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderReadOther, + image_subresource_range(A::DEPTH | A::STENCIL, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::DepthStencilAttachmentWrite, + image_subresource_range(A::DEPTH, 0..1, 0..1), + ), + ); + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::DepthStencilAttachmentRead, + image_subresource_range(A::STENCIL, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } + + #[test] + pub fn image_access_dense_promotes_only_on_partial_update() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::R8_UINT, 2, 2); + let image = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + + let Access::Dense(access) = &image else { + panic!("expected dense-capable access tracking"); + }; + + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderReadOther, + image_subresource_range(A::COLOR, 0..2, 0..2), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::COLOR, 0..2, 0..2), + ), + ); + assert!(accesses.next().is_none()); + assert!(!access.is_dense_active()); + + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderWrite, + image_subresource_range(A::COLOR, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::AnyShaderReadOther, + image_subresource_range(A::COLOR, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + assert!(access.is_dense_active()); + } + + #[test] + pub fn image_access_dense_collapses_to_uniform_after_equalizing_updates() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::R8_UINT, 2, 2); + let image = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + + let Access::Dense(access) = &image else { + panic!("expected dense-capable access tracking"); + }; + + { + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderReadOther, + image_subresource_range(A::COLOR, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::COLOR, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } + + assert!(access.is_dense_active()); + + { + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderReadOther, + image_subresource_range(A::COLOR, 0..2, 0..2), + ); + + assert!(accesses.next().is_some()); + while accesses.next().is_some() {} + } + + assert!(!access.is_dense_active()); + assert_eq!(access.load(), AccessType::AnyShaderReadOther); + + let dense = dense.lock(); + #[cfg(not(feature = "parking_lot"))] + let dense = dense.expect("poisoned image dense lock"); + + assert!(dense.is_none()); + } + + #[test] + pub fn image_access_dense_stays_active_for_mixed_updates() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::R8_UINT, 2, 2); + let image = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + + let Access::Dense(access) = &image else { + panic!("expected dense-capable access tracking"); + }; + + { + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderReadOther, + image_subresource_range(A::COLOR, 0..1, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::COLOR, 0..1, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } + + { + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderWrite, + image_subresource_range(A::COLOR, 1..2, 0..1), + ); + + assert_access_ranges_eq( + accesses.next().unwrap(), + ( + AccessType::Nothing, + image_subresource_range(A::COLOR, 1..2, 0..1), + ), + ); + assert!(accesses.next().is_none()); + } - (prev_access, access_range) - } -} + assert!(access.is_dense_active()); -#[cfg(test)] -mod test { - use { - super::*, - rand::{Rng, SeedableRng, rngs::SmallRng}, - std::ops::Range, - }; + let dense = dense.lock(); + #[cfg(not(feature = "parking_lot"))] + let dense = dense.expect("poisoned image dense lock"); - // ImageSubresourceRange does not implement PartialEq - fn assert_access_ranges_eq( - lhs: (AccessType, vk::ImageSubresourceRange), - rhs: (AccessType, vk::ImageSubresourceRange), - ) { + let dense_map = dense.as_ref().expect("missing dense access map"); assert_eq!( - ( - lhs.0, - lhs.1.aspect_mask, - lhs.1.base_array_layer, - lhs.1.layer_count, - lhs.1.base_mip_level, - lhs.1.level_count - ), - ( - rhs.0, - rhs.1.aspect_mask, - rhs.1.base_array_layer, - rhs.1.layer_count, - rhs.1.base_mip_level, - rhs.1.level_count - ) + dense_map.subresource(0, 0, 0), + AccessType::AnyShaderReadOther ); + assert_eq!(dense_map.subresource(0, 1, 0), AccessType::AnyShaderWrite); + assert_eq!(dense_map.subresource(0, 0, 1), AccessType::Nothing); + assert_eq!(dense_map.subresource(0, 1, 1), AccessType::Nothing); } #[test] - pub fn image_access_basic() { + pub fn image_access_dense_iter_drains_on_drop() { use vk::ImageAspectFlags as A; - let mut image = DenseAccess::new( - image_subresource(vk::Format::R8G8B8A8_UNORM, 1, 1), - AccessType::Nothing, - ); + let info = image_subresource(vk::Format::R8_UINT, 2, 2); + let image = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + + let Access::Dense(access) = &image else { + panic!("expected dense-capable access tracking"); + }; { - let mut accesses = DenseAccessIter::new( - &mut image, - AccessType::AnyShaderWrite, + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 0..1, 0..1), ); @@ -1378,58 +3038,54 @@ mod test { } { - let mut accesses = DenseAccessIter::new( - &mut image, - AccessType::AnyShaderReadOther, - image_subresource_range(A::COLOR, 0..1, 0..1), + let mut accesses = image.swap( + &dense, + info, + AccessType::AnyShaderWrite, + image_subresource_range(A::COLOR, 1..2, 0..1), ); assert_access_ranges_eq( accesses.next().unwrap(), ( - AccessType::AnyShaderWrite, - image_subresource_range(A::COLOR, 0..1, 0..1), + AccessType::Nothing, + image_subresource_range(A::COLOR, 1..2, 0..1), ), ); assert!(accesses.next().is_none()); } - } - - #[test] - pub fn image_access_uniform() { - use vk::ImageAspectFlags as A; - - let image = Access::new( - image_subresource(vk::Format::R8G8B8A8_UNORM, 1, 1), - AccessType::Nothing, - ); let mut accesses = image.swap( - AccessType::AnyShaderWrite, - image_subresource_range(A::COLOR, 0..1, 0..1), + &dense, + info, + AccessType::HostRead, + image_subresource_range(A::COLOR, 0..2, 0..2), ); - assert_access_ranges_eq( - accesses.next().unwrap(), - ( - AccessType::Nothing, - image_subresource_range(A::COLOR, 0..1, 0..1), - ), - ); - assert!(accesses.next().is_none()); + assert!(accesses.next().is_some()); + drop(accesses); + + assert!(!access.is_dense_active()); + assert_eq!(access.load(), AccessType::HostRead); + + let dense = dense.lock(); + #[cfg(not(feature = "parking_lot"))] + let dense = dense.expect("poisoned image dense lock"); + + assert!(dense.is_none()); } #[test] pub fn image_access_color() { use vk::ImageAspectFlags as A; - let mut image = DenseAccess::new( + let mut image = DenseMap::new( image_subresource(vk::Format::R8G8B8A8_UNORM, 3, 3), AccessType::Nothing, ); { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderWrite, image_subresource_range(A::COLOR, 0..3, 0..3), @@ -1446,7 +3102,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 0..1, 0..1), @@ -1463,7 +3119,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::ComputeShaderWrite, image_subresource_range(A::COLOR, 0..3, 0..3), @@ -1494,7 +3150,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostRead, image_subresource_range(A::COLOR, 0..3, 0..3), @@ -1511,7 +3167,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostWrite, image_subresource_range(A::COLOR, 1..2, 1..2), @@ -1528,7 +3184,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::GeometryShaderReadOther, image_subresource_range(A::COLOR, 0..3, 0..3), @@ -1573,7 +3229,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::VertexBuffer, image_subresource_range(A::COLOR, 0..3, 1..2), @@ -1590,7 +3246,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::ColorAttachmentRead, image_subresource_range(A::COLOR, 0..3, 0..3), @@ -1667,13 +3323,13 @@ mod test { pub fn image_access_layers() { use vk::ImageAspectFlags as A; - let mut image = DenseAccess::new( + let mut image = DenseMap::new( image_subresource(vk::Format::R8G8B8A8_UNORM, 3, 1), AccessType::Nothing, ); { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderWrite, image_subresource_range(A::COLOR, 0..3, 0..1), @@ -1690,7 +3346,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 2..3, 0..1), @@ -1707,7 +3363,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostRead, image_subresource_range(A::COLOR, 0..2, 0..1), @@ -1724,7 +3380,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 0..1, 0..1), @@ -1741,7 +3397,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 1..2, 0..1), @@ -1758,7 +3414,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostWrite, image_subresource_range(A::COLOR, 0..3, 0..1), @@ -1779,13 +3435,13 @@ mod test { pub fn image_access_levels() { use vk::ImageAspectFlags as A; - let mut image = DenseAccess::new( + let mut image = DenseMap::new( image_subresource(vk::Format::R8G8B8A8_UNORM, 1, 3), AccessType::Nothing, ); { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderWrite, image_subresource_range(A::COLOR, 0..1, 0..3), @@ -1802,7 +3458,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 0..1, 2..3), @@ -1819,7 +3475,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostRead, image_subresource_range(A::COLOR, 0..1, 0..2), @@ -1836,7 +3492,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 0..1, 0..1), @@ -1853,7 +3509,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::COLOR, 0..1, 1..2), @@ -1870,7 +3526,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostWrite, image_subresource_range(A::COLOR, 0..1, 0..3), @@ -1891,13 +3547,13 @@ mod test { pub fn image_access_depth_stencil() { use vk::ImageAspectFlags as A; - let mut image = DenseAccess::new( + let mut image = DenseMap::new( image_subresource(vk::Format::D24_UNORM_S8_UINT, 4, 3), AccessType::Nothing, ); { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderWrite, image_subresource_range(A::DEPTH, 0..4, 0..1), @@ -1914,7 +3570,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderWrite, image_subresource_range(A::STENCIL, 0..4, 1..2), @@ -1931,7 +3587,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::DEPTH | A::STENCIL, 0..4, 0..2), @@ -2053,7 +3709,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AccelerationStructureBuildWrite, image_subresource_range(A::DEPTH | A::STENCIL, 0..4, 0..2), @@ -2070,7 +3726,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AccelerationStructureBuildRead, image_subresource_range(A::DEPTH, 1..3, 0..2), @@ -2091,13 +3747,13 @@ mod test { pub fn image_access_stencil() { use vk::ImageAspectFlags as A; - let mut image = DenseAccess::new( + let mut image = DenseMap::new( image_subresource(vk::Format::S8_UINT, 2, 2), AccessType::Nothing, ); { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderWrite, image_subresource_range(A::STENCIL, 0..2, 0..1), @@ -2114,7 +3770,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::AnyShaderReadOther, image_subresource_range(A::STENCIL, 0..2, 1..2), @@ -2131,7 +3787,7 @@ mod test { } { - let mut accesses = DenseAccessIter::new( + let mut accesses = DenseMapIter::new( &mut image, AccessType::HostRead, image_subresource_range(A::STENCIL, 0..2, 0..2), @@ -2181,8 +3837,8 @@ mod test { pub fn image_info_cube_builder() { let info = ImageInfo::cube(42, vk::Format::R32_SFLOAT, vk::ImageUsageFlags::empty()); let builder = ImageInfoBuilder::default() - .ty(vk::ImageType::TYPE_2D) - .fmt(vk::Format::R32_SFLOAT) + .image_type(vk::ImageType::TYPE_2D) + .format(vk::Format::R32_SFLOAT) .width(42) .height(42) .depth(1) @@ -2205,8 +3861,8 @@ mod test { pub fn image_info_image_1d_builder() { let info = ImageInfo::image_1d(42, vk::Format::R32_SFLOAT, vk::ImageUsageFlags::empty()); let builder = ImageInfoBuilder::default() - .ty(vk::ImageType::TYPE_1D) - .fmt(vk::Format::R32_SFLOAT) + .image_type(vk::ImageType::TYPE_1D) + .format(vk::Format::R32_SFLOAT) .width(42) .height(1) .depth(1) @@ -2229,8 +3885,8 @@ mod test { let info = ImageInfo::image_2d(42, 84, vk::Format::R32_SFLOAT, vk::ImageUsageFlags::empty()); let builder = ImageInfoBuilder::default() - .ty(vk::ImageType::TYPE_2D) - .fmt(vk::Format::R32_SFLOAT) + .image_type(vk::ImageType::TYPE_2D) + .format(vk::Format::R32_SFLOAT) .width(42) .height(84) .depth(1) @@ -2263,8 +3919,8 @@ mod test { vk::ImageUsageFlags::empty(), ); let builder = ImageInfoBuilder::default() - .ty(vk::ImageType::TYPE_2D) - .fmt(vk::Format::R32_SFLOAT) + .image_type(vk::ImageType::TYPE_2D) + .format(vk::Format::R32_SFLOAT) .width(42) .height(84) .depth(1) @@ -2298,8 +3954,8 @@ mod test { vk::ImageUsageFlags::empty(), ); let builder = ImageInfoBuilder::default() - .ty(vk::ImageType::TYPE_3D) - .fmt(vk::Format::R32_SFLOAT) + .image_type(vk::ImageType::TYPE_3D) + .format(vk::Format::R32_SFLOAT) .width(42) .height(84) .depth(100) @@ -2312,15 +3968,18 @@ mod test { pub fn image_info_builder_defaults() { let info = ImageInfo { array_layer_count: 1, - dedicated: false, + alloc_dedicated: false, depth: 0, flags: vk::ImageCreateFlags::empty(), - fmt: vk::Format::UNDEFINED, + format: vk::Format::UNDEFINED, height: 0, + host_readable: false, + host_writable: false, mip_level_count: 1, sample_count: SampleCount::Type1, + sharing_mode: vk::SharingMode::EXCLUSIVE, tiling: vk::ImageTiling::OPTIMAL, - ty: vk::ImageType::TYPE_2D, + image_type: vk::ImageType::TYPE_2D, usage: vk::ImageUsageFlags::empty(), width: 0, }; @@ -2348,7 +4007,7 @@ mod test { let mut rng = SmallRng::seed_from_u64(42); let total = (aspect_count as u32 * array_layer_count * mip_level_count) as usize; - let mut access = DenseAccess::new( + let mut access_map = DenseMap::new( image_subresource(fmt, array_layer_count, mip_level_count), AccessType::Nothing, ); @@ -2376,7 +4035,7 @@ mod test { let range = image_subresource_range(aspect_mask, layer_start..layer_end, mip_start..mip_end); - for (prev, range) in access.swap(new_access, range) { + for (prev, range) in access_map.swap(new_access, range) { let range_mask = range.aspect_mask.as_raw(); for ai in 0..range_mask.count_ones() as u8 { let bit = range_mask.trailing_zeros() + ai as u32; @@ -2403,7 +4062,7 @@ mod test { } for l in layer_start..layer_end { for m in mip_start..mip_end { - let idx = access.idx(a, l, m); + let idx = access_map.idx(a, l, m); data[idx] = new_access; } } @@ -2426,12 +4085,306 @@ mod test { image_access_fuzz(1, 10, 10); } + fn image_access_fuzz_through_access( + aspect_count: u8, + array_layer_count: u32, + mip_level_count: u32, + ) { + const FUZZ_COUNT: usize = 10_000; + static ACCESS_TYPES: &[AccessType] = &[ + AccessType::AnyShaderReadOther, + AccessType::AnyShaderWrite, + AccessType::ColorAttachmentRead, + AccessType::ColorAttachmentWrite, + AccessType::HostRead, + AccessType::HostWrite, + AccessType::Nothing, + ]; + + let fmt = match aspect_count { + 1 => vk::Format::R8G8B8A8_UNORM, + 2 => vk::Format::D24_UNORM_S8_UINT, + _ => unreachable!(), + }; + + let mut rng = SmallRng::seed_from_u64(42); + let info = image_subresource(fmt, array_layer_count, mip_level_count); + let total = (aspect_count as u32 * array_layer_count * mip_level_count) as usize; + let access = Access::new(info, AccessType::Nothing); + let dense = Mutex::new(None); + let mut data = vec![AccessType::Nothing; total]; + + let aspect_bits = format_aspect_mask(fmt); + + for _ in 0..FUZZ_COUNT { + let new_access = ACCESS_TYPES[rng.random_range(..ACCESS_TYPES.len())]; + + let aspect_mask = if aspect_count == 2 && rng.random_bool(0.5) { + aspect_bits + } else { + let bit_index = + rng.random_range(..aspect_count) + aspect_bits.as_raw().trailing_zeros() as u8; + vk::ImageAspectFlags::from_raw(1 << bit_index) + }; + + let layer_start = rng.random_range(..array_layer_count); + let layer_end = rng.random_range(layer_start + 1..=array_layer_count); + let mip_start = rng.random_range(..mip_level_count); + let mip_end = rng.random_range(mip_start + 1..=mip_level_count); + + let range = + image_subresource_range(aspect_mask, layer_start..layer_end, mip_start..mip_end); + let resolved = info.resolve_subresource_counts(range); + + for (prev, returned_range) in access.swap(&dense, info, new_access, resolved) { + let range_mask = returned_range.aspect_mask.as_raw(); + for ai in 0..range_mask.count_ones() as u8 { + let bit = range_mask.trailing_zeros() + ai as u32; + let a = (aspect_bits.as_raw() & ((1 << bit) - 1)).count_ones() as u8; + for l in returned_range.base_array_layer + ..returned_range.base_array_layer + returned_range.layer_count + { + for m in returned_range.base_mip_level + ..returned_range.base_mip_level + returned_range.level_count + { + let idx = (l * aspect_count as u32 * mip_level_count + + m * aspect_count as u32 + + a as u32) as usize; + assert_eq!( + data[idx], prev, + "prev mismatch at aspect={a} layer={l} mip={m} idx={idx}: expected {prev:?}, got {:?}", + data[idx], + ); + } + } + } + } + + for a in 0..aspect_count { + let bit = aspect_bits.as_raw().trailing_zeros() as u8 + a; + if aspect_mask.as_raw() & (1 << bit) == 0 { + continue; + } + for l in layer_start..layer_end { + for m in mip_start..mip_end { + let idx = (l * aspect_count as u32 * mip_level_count + + m * aspect_count as u32 + + a as u32) as usize; + data[idx] = new_access; + } + } + } + } + } + + #[test] + pub fn image_access_fuzz_access_uniform() { + image_access_fuzz_through_access(1, 1, 1); + } + + #[test] + pub fn image_access_fuzz_access_dual_aspect() { + image_access_fuzz_through_access(2, 1, 1); + } + + #[test] + pub fn image_access_fuzz_access_dense_small() { + image_access_fuzz_through_access(1, 4, 4); + } + + #[test] + pub fn image_access_fuzz_access_dense_large() { + image_access_fuzz_through_access(1, 8, 8); + } + + #[test] + pub fn image_access_fuzz_access_dense_dual_aspect() { + image_access_fuzz_through_access(2, 3, 3); + } + + #[test] + pub fn image_sync_info_compact_merges_mips_then_layers() { + use vk::ImageAspectFlags as A; + + let mut sync_info = ImageSyncInfo { + subresources: vec![ + image_sync_subresource(A::COLOR, 0..1, 0..1), + image_sync_subresource(A::COLOR, 0..1, 1..2), + image_sync_subresource(A::COLOR, 1..2, 0..1), + image_sync_subresource(A::COLOR, 1..2, 1..2), + ] + .into_boxed_slice(), + }; + + sync_info.compact(); + + assert_eq!(sync_info.subresources.len(), 1); + let subresource = &sync_info.subresources[0]; + let range = image_subresource_range(A::COLOR, 0..2, 0..2); + assert_eq!(subresource.access_mask, vk::AccessFlags::SHADER_READ); + assert_eq!( + subresource.layout, + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + ); + assert_eq!(subresource.range.aspect_mask, range.aspect_mask); + assert_eq!(subresource.range.base_array_layer, range.base_array_layer); + assert_eq!(subresource.range.layer_count, range.layer_count); + assert_eq!(subresource.range.base_mip_level, range.base_mip_level); + assert_eq!(subresource.range.level_count, range.level_count); + assert_eq!( + subresource.stage_mask, + vk::PipelineStageFlags::COMPUTE_SHADER + ); + } + + #[test] + pub fn image_sync_info_compact_keeps_different_sync_separate() { + use vk::ImageAspectFlags as A; + + let sync_info = ImageSyncInfo { + subresources: vec![ + image_sync_subresource(A::COLOR, 0..1, 0..1), + ImageSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_WRITE, + layout: Some(vk::ImageLayout::GENERAL), + queue_family_index: None, + range: image_subresource_range(A::COLOR, 0..1, 1..2), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + }, + ] + .into_boxed_slice(), + }; + + let sync_info = sync_info.into_compacted(); + + assert_eq!(sync_info.subresources.len(), 2); + let subresource = &sync_info.subresources[0]; + let range = image_subresource_range(A::COLOR, 0..1, 0..1); + assert_eq!(subresource.access_mask, vk::AccessFlags::SHADER_READ); + assert_eq!( + subresource.layout, + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + ); + assert_eq!(subresource.range.aspect_mask, range.aspect_mask); + assert_eq!(subresource.range.base_array_layer, range.base_array_layer); + assert_eq!(subresource.range.layer_count, range.layer_count); + assert_eq!(subresource.range.base_mip_level, range.base_mip_level); + assert_eq!(subresource.range.level_count, range.level_count); + assert_eq!( + subresource.stage_mask, + vk::PipelineStageFlags::COMPUTE_SHADER + ); + assert_eq!( + sync_info.subresources[1].access_mask, + vk::AccessFlags::SHADER_WRITE + ); + assert_eq!( + sync_info.subresources[1].layout, + Some(vk::ImageLayout::GENERAL) + ); + } + + #[test] + pub fn image_sync_info_compact_keeps_different_queue_families_separate() { + use vk::ImageAspectFlags as A; + + let sync_info = ImageSyncInfo { + subresources: vec![ + ImageSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_READ, + layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), + queue_family_index: Some(1), + range: image_subresource_range(A::COLOR, 0..1, 0..1), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + }, + ImageSubresourceSyncInfo { + access_mask: vk::AccessFlags::SHADER_READ, + layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), + queue_family_index: Some(2), + range: image_subresource_range(A::COLOR, 0..1, 1..2), + stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER, + }, + ] + .into_boxed_slice(), + }; + + let sync_info = sync_info.into_compacted(); + + assert_eq!(sync_info.subresources.len(), 2); + assert_eq!(sync_info.subresources[0].queue_family_index, Some(1)); + assert_eq!(sync_info.subresources[1].queue_family_index, Some(2)); + } + + #[test] + pub fn image_ownership_set_promotes_dense_on_partial_update() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::R8_UINT, 2, 2); + let sharing = Sharing::new(info, vk::SharingMode::EXCLUSIVE); + let dense = Mutex::new(None); + + sharing.set_ranges( + &dense, + info, + SharingMode::Exclusive(Some((7, 3))), + &[image_subresource_range(A::COLOR, 0..1, 0..1)], + ); + + match &sharing { + Sharing::Exclusive(exclusive) => { + assert!(exclusive.is_dense_sharing_active()); + } + Sharing::Concurrent => panic!("expected exclusive ownership"), + } + + let dense = dense.lock(); + + #[cfg(not(feature = "parking_lot"))] + let dense = dense.expect("poisoned image dense lock"); + + let dense = dense.as_ref().expect("missing dense sharing state"); + assert_eq!( + dense.subresource(0, 0, 0), + SharingMode::Exclusive(Some((7, 3))) + ); + assert_eq!(dense.subresource(0, 1, 0), SharingMode::Exclusive(None)); + assert_eq!(dense.subresource(0, 0, 1), SharingMode::Exclusive(None)); + assert_eq!(dense.subresource(0, 1, 1), SharingMode::Exclusive(None)); + } + + #[test] + pub fn image_ownership_set_whole_image_stays_uniform() { + use vk::ImageAspectFlags as A; + + let info = image_subresource(vk::Format::R8_UINT, 2, 2); + let sharing = Sharing::new(info, vk::SharingMode::EXCLUSIVE); + let dense = Mutex::new(None); + + sharing.set_ranges( + &dense, + info, + SharingMode::Exclusive(Some((1, 2))), + &[image_subresource_range(A::COLOR, 0..2, 0..2)], + ); + + match &sharing { + Sharing::Exclusive(exclusive) => { + assert!(!exclusive.is_dense_sharing_active()); + assert_eq!( + SharingMode::decode(exclusive.uniform.load(Ordering::Acquire)), + SharingMode::Exclusive(Some((1, 2))) + ); + } + Sharing::Concurrent => panic!("expected exclusive ownership"), + } + } + fn image_subresource( - fmt: vk::Format, + format: vk::Format, array_layer_count: u32, mip_level_count: u32, ) -> ImageInfo { - ImageInfo::image_2d(1, 1, fmt, vk::ImageUsageFlags::empty()) + ImageInfo::image_2d(1, 1, format, vk::ImageUsageFlags::empty()) .into_builder() .array_layer_count(array_layer_count) .mip_level_count(mip_level_count) @@ -2503,6 +4456,25 @@ mod test { )); } + #[test] + pub fn image_subresource_range_normalize_remaining_counts() { + let info = image_subresource(vk::Format::R8_UINT, 4, 6); + let range = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: 1, + layer_count: vk::REMAINING_ARRAY_LAYERS, + base_mip_level: 2, + level_count: vk::REMAINING_MIP_LEVELS, + }; + + let range = info.resolve_subresource_counts(range); + + assert_eq!(range.base_array_layer, 1); + assert_eq!(range.layer_count, 3); + assert_eq!(range.base_mip_level, 2); + assert_eq!(range.level_count, 4); + } + #[test] pub fn image_view_info() { let info = ImageViewInfo::new(vk::Format::default(), vk::ImageViewType::TYPE_1D); @@ -2515,8 +4487,8 @@ mod test { pub fn image_view_info_builder() { let info = ImageViewInfo::new(vk::Format::default(), vk::ImageViewType::TYPE_1D); let builder = ImageViewInfoBuilder::default() - .fmt(vk::Format::default()) - .ty(vk::ImageViewType::TYPE_1D) + .format(vk::Format::default()) + .view_type(vk::ImageViewType::TYPE_1D) .aspect_mask(vk::ImageAspectFlags::COLOR) .build(); diff --git a/src/driver/instance.rs b/src/driver/instance.rs index 8d10a05f..59add8bd 100644 --- a/src/driver/instance.rs +++ b/src/driver/instance.rs @@ -1,12 +1,16 @@ -//! Vulkan initialization types +//! Vulkan initialization types. use { super::{DriverError, physical_device::PhysicalDevice}, - ash::{ext, khr, vk, vk::Handle}, + ash::{ + ext, khr, + vk::{self, Handle}, + }, derive_builder::Builder, log::{debug, error, trace, warn}, raw_window_handle::{HasDisplayHandle, RawDisplayHandle}, std::{ + collections::HashSet, error::Error, ffi::CStr, fmt::{Debug, Display, Formatter}, @@ -22,6 +26,7 @@ use { std::{ env::var, ffi::c_void, + io::{IsTerminal, stderr}, process::{abort, id}, thread::{current, park}, }, @@ -66,19 +71,6 @@ unsafe extern "system" fn debug_callback( let is_error = message_severity.contains(vk::DebugUtilsMessageSeverityFlagsEXT::ERROR); - // HACK: This is not production-quality - // TODO: This was debugged and the issue has not been found, so this may or may not be valid - // The validation layer reports `UNASSIGNED-Threading-MultipleThreads-Write` when two threads - // touch different VkQueue handles at the same time. Ignoring until the issue is found. - if is_error - && message.contains("THREADING ERROR") - && message.contains("VkQueue is simultaneously used") - { - info!("ignoring: {message}"); - - return vk::FALSE; - } - if is_error { error!("{message}"); } else if message_severity.contains(vk::DebugUtilsMessageSeverityFlagsEXT::WARNING) { @@ -114,7 +106,12 @@ unsafe extern "system" fn debug_callback( .unwrap_or(false) { warn!("validation callback park skipped; execution will continue"); - logger().flush(); + + return vk::FALSE; + } + + if !stderr().is_terminal() { + warn!("validation callback park skipped; stderr is not an interactive terminal"); return vk::FALSE; } @@ -131,27 +128,23 @@ unsafe extern "system" fn debug_callback( vk::FALSE } -#[cfg(any(not(target_os = "macos"), feature = "loaded"))] fn debug_extension_names() -> &'static [&'static CStr] { - &[ext::debug_utils::NAME] -} + #[cfg(any(not(target_os = "macos"), feature = "loaded"))] + return &[ext::debug_utils::NAME]; -#[cfg(all(target_os = "macos", not(feature = "loaded")))] -fn debug_extension_names() -> &'static [&'static CStr] { - &[] + #[cfg(all(target_os = "macos", not(feature = "loaded")))] + return &[]; } -#[cfg(any(not(target_os = "macos"), feature = "loaded"))] fn debug_layer_names() -> &'static [&'static CStr] { - &[c"VK_LAYER_KHRONOS_validation"] -} + #[cfg(any(not(target_os = "macos"), feature = "loaded"))] + return &[c"VK_LAYER_KHRONOS_validation"]; -#[cfg(all(target_os = "macos", not(feature = "loaded")))] -fn debug_layer_names() -> &'static [&'static CStr] { - &[] + #[cfg(all(target_os = "macos", not(feature = "loaded")))] + return &[]; } -// Copied from ash_window::enumerate_required_extensions to change the signature. +// Copied from ash_window::enumerate_required_extensions to change the signature fn display_extension_names( display_handle: RawDisplayHandle, ) -> Result<&'static [&'static CStr], DriverError> { @@ -174,10 +167,12 @@ fn display_extension_names( Ok(extensions) } -// Estimates surface extension support. -// -// Imported instances do not expose their enabled extension list, so we infer support by -// checking that the VK_KHR_surface entry points resolve for this instance handle. +/* +Estimates surface extension support. + +Imported instances do not expose their enabled extension list, so we infer support by checking that +the VK_KHR_surface entry points resolve for this instance handle. +*/ fn has_vk_khr_surface(entry: &ash::Entry, instance: vk::Instance) -> bool { [ c"vkGetPhysicalDeviceSurfaceCapabilitiesKHR", @@ -195,13 +190,17 @@ fn has_vk_khr_surface(entry: &ash::Entry, instance: vk::Instance) -> bool { } /// Vulkan API version. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)] +/// +/// See [`VkApplicationInfo::apiVersion`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkApplicationInfo.html). +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, PartialOrd, Ord)] pub enum ApiVersion { /// Version `1.2`. - #[default] Vulkan12, /// Version `1.3`. + /// + /// This is the default value. + #[default] Vulkan13, } @@ -226,7 +225,7 @@ impl ApiVersion { } } - /// Vulkan API minor version number component. Ex: `v0.0.X-0`. + /// Vulkan API patch version number component. Ex: `v0.0.X-0`. /// /// Always zero. pub fn patch(self) -> u32 { @@ -277,11 +276,13 @@ impl TryFrom for ApiVersion { } } -/// There is no global state in Vulkan and all per-application state is stored in a VkInstance +/// There is no global state in Vulkan and all per-application state is stored in a `VkInstance` /// object. /// -/// Creating an Instance initializes the Vulkan library and allows the application to pass +/// Creating an `Instance` initializes the Vulkan library and allows the application to pass /// information about itself to the implementation. +/// +/// See [`VkInstance`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkInstance.html). #[read_only::embed] #[allow(private_interfaces)] pub struct Instance { @@ -314,14 +315,16 @@ impl Clone for Instance { } impl Instance { - /// The most recent supported version of Vulkan. - pub const LATEST_API_VERSION: ApiVersion = ApiVersion::Vulkan13; + /// Default Vulkan API version requested when creating an instance. + pub const DEFAULT_API_VERSION: ApiVersion = ApiVersion::Vulkan13; /// Creates a new Vulkan instance. /// /// This constructor is intended for headless or manually managed setups. It does not infer or /// enable display platform surface extensions. Use [`Self::try_from_display`] when the /// resulting instance must be capable of later surface creation. + /// + /// See [`vkCreateInstance`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCreateInstance.html). #[profiling::function] pub fn create(info: impl Into) -> Result { Self::create_with_extension_names(info.into(), &[]) @@ -331,13 +334,19 @@ impl Instance { info: InstanceInfo, extra_extension_names: &[&CStr], ) -> Result { + if info.debug && debug_extension_names().is_empty() { + error!("debug mode requires VK_EXT_debug_utils support"); + + return Err(DriverError::Unsupported); + } + // Required to enable non-uniform descriptor indexing (bindless) #[cfg(target_os = "macos")] unsafe { set_var("MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS", "1"); } - // Link the Vulkan loader dynamically (default feature). + // Link the Vulkan loader dynamically (default feature) #[cfg(feature = "loaded")] let entry = unsafe { ash::Entry::load().map_err(|err| { @@ -353,7 +362,7 @@ impl Instance { #[cfg(not(target_os = "macos"))] let entry = ash::Entry::linked(); - // On MacOS, by default link molten-vk statically using ash-molten. + // On macOS, by default link molten-vk statically using ash-molten #[cfg(target_os = "macos")] let entry = ash_molten::load(); }; @@ -363,15 +372,17 @@ impl Instance { .iter() .chain(extra_extension_names) .copied() - .collect::>(); + .collect::>(); if info.debug { extension_names.extend(debug_extension_names()); } - // If linking dynamically on MacOS, we require a few additional extensions. - // Based on "Encountered VK_ERROR_INCOMPATIBLE_DRIVER" section in: - // https://vulkan.lunarg.com/doc/view/latest/mac/getting_started.html + /* + If linking dynamically on macOS, we require a few additional extensions. Based on + "Encountered VK_ERROR_INCOMPATIBLE_DRIVER" section in: + https://vulkan.lunarg.com/doc/view/latest/mac/getting_started.html + */ #[cfg(all(target_os = "macos", feature = "loaded"))] { extension_names.extend(&[ @@ -407,8 +418,10 @@ impl Instance { .enabled_layer_names(&layer_name_ptrs) .enabled_extension_names(&extension_name_ptrs); - // Molten-vk doesn't support the full Vulkan feature set, hence the portability flag needs - // to be set. + /* + MoltenVK doesn't support the full Vulkan feature set, hence the portability flag needs to be + set. + */ #[cfg(all(target_os = "macos", feature = "loaded"))] let instance_desc = instance_desc.flags(vk::InstanceCreateFlags::ENUMERATE_PORTABILITY_KHR); @@ -499,12 +512,18 @@ impl Instance { }) } - /// The ash entrypoint used to load Vulkan instance functions. + /// The ash entry point used to load Vulkan instance functions. pub fn entry(this: &Self) -> &ash::Entry { &this.inner.entry } + pub(crate) fn supports_debug_utils(this: &Self) -> bool { + this.inner.debug_utils.is_some() + } + /// Returns the available physical devices of this instance. + /// + /// See [`vkEnumeratePhysicalDevices`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkEnumeratePhysicalDevices.html). #[profiling::function] pub fn physical_devices( this: &Self, @@ -530,29 +549,44 @@ impl Instance { .into_iter() .enumerate() .filter_map(|(idx, physical_device)| { - let res = PhysicalDevice::try_from_ash(this, physical_device); + let physical_device = unsafe { + PhysicalDevice::try_from_ash(this, physical_device) + .inspect_err(|err| warn!("unsupported physical device #{idx}: {err}")) + .ok()? + }; + + let api_version = ApiVersion::try_parse_vk_api_version( + physical_device.properties_v1_0.api_version, + ) + .inspect_err(|err| { + warn!( + "unsupported physical device #{idx} {}: {err}", + physical_device.properties_v1_0.device_name + ); + }) + .ok()?; - if let Err(err) = &res { - warn!("unsupported physical device #{idx}: {err}"); + if api_version < this.info.api_version { + return None; } - res.ok().filter(|physical_device| { - ApiVersion::try_parse_vk_api_version( - physical_device.properties_v1_0.api_version, - ) - .inspect_err(|err| { - debug!( - "unsupported physical device `{}`: {err}", - physical_device.properties_v1_0.device_name - ); - }) - .is_ok() - }) + if this.info.debug && !physical_device.vk_ext_private_data { + warn!( + "unsupported physical device #{idx} {}: missing VK_EXT_private_data", + physical_device.properties_v1_0.device_name + ); + + return None; + } + + Some(physical_device) })) } /// Creates a new Vulkan instance with the platform surface extensions required by the provided /// display handle. + /// + /// See [`VK_KHR_surface`](https://registry.khronos.org/vulkan/specs/latest/man/html/VK_KHR_surface.html). #[profiling::function] pub fn try_from_display( display: impl HasDisplayHandle, @@ -577,6 +611,8 @@ impl Instance { /// /// This is useful when you want to use a Vulkan instance created by some other library, such /// as OpenXR. + /// + /// See [`VkInstance`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkInstance.html). #[profiling::function] pub fn try_from_entry(entry: ash::Entry, instance: vk::Instance) -> Result { if instance == vk::Instance::null() { @@ -596,8 +632,10 @@ impl Instance { } })? .unwrap_or_else(|| { - // The implementation *should* provide a version. If it does not we just send it. - Self::LATEST_API_VERSION.to_vk_api_version() + /* + The implementation *should* provide a version. If it does not, use the default. + */ + Self::DEFAULT_API_VERSION.to_vk_api_version() }) .try_into() .map_err(|err| { @@ -629,7 +667,12 @@ impl Instance { impl Debug for Instance { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str("Instance") + f.debug_struct(stringify!(Instance)) + .field("handle", &self.inner.instance.handle()) + .field("info", &self.info) + .field("khr_surface", &self.khr_surface) + .field("debug_utils", &self.inner.debug_utils.is_some()) + .finish_non_exhaustive() } } @@ -642,25 +685,30 @@ impl Debug for Instance { )] pub struct InstanceInfo { /// The Vulkan API version to target. - #[builder(default = "ApiVersion::Vulkan13")] + /// + /// Defaults to [`Instance::DEFAULT_API_VERSION`]. + #[builder(default = "Instance::DEFAULT_API_VERSION")] pub api_version: ApiVersion, /// Enables Vulkan validation layers. /// - /// This requires a Vulkan SDK installation and will cause validation errors to introduce - /// panics as they happen. + /// This requires a Vulkan SDK installation and will panic when validation errors happen. + /// Additionally, the device must support VK_EXT_private_data. /// - /// Set `VK_GRAPH_SKIP_VALIDATION_PARK=1` to keep logging validation errors without parking the + /// When `stderr` is attached to an interactive terminal, validation errors will park the /// callback thread for debugger attach. /// + /// Set `VK_GRAPH_SKIP_VALIDATION_PARK=1` to keep logging validation errors without parking. + /// /// _NOTE:_ Consider turning OFF debug if you discover an unknown issue. Often the validation - /// layers will throw an error before other layers can provide additional context such as the + /// layers will report an error before other layers can provide additional context such as the /// API dump info or other messages. You might find the "actual" issue is detailed in those /// subsequent details. /// /// ## Platform-specific /// /// **macOS:** Has no effect unless the `loaded` feature is enabled. + /// #[builder(default)] pub debug: bool, @@ -799,4 +847,27 @@ mod test { ApiVersion::Vulkan13 ); } + + #[test] + pub fn default_api_version_matches_instance_info_default() { + assert_eq!( + Instance::DEFAULT_API_VERSION, + InstanceInfo::default().api_version + ); + } + + #[test] + pub fn default_api_version_matches_api_version_default() { + assert_eq!(Instance::DEFAULT_API_VERSION, ApiVersion::default()); + } + + #[test] + pub fn invalid_api_versions_are_rejected() { + assert!(ApiVersion::try_parse_vk_api_version(vk::API_VERSION_1_1).is_err()); + assert!(ApiVersion::try_parse_vk_api_version(vk::make_api_version(0, 2, 0, 0)).is_err()); + assert!(ApiVersion::try_parse_vk_api_version(vk::make_api_version(1, 1, 9, 0)).is_err()); + assert!(ApiVersion::try_parse_vk_api_version(vk::make_api_version(1, 4, 2, 0)).is_err()); + assert!(ApiVersion::try_parse_vk_api_version(vk::make_api_version(1, 2, 0, 1)).is_err()); + assert!(ApiVersion::try_parse_vk_api_version(vk::make_api_version(1, 3, 0, 1)).is_err()); + } } diff --git a/src/driver/mod.rs b/src/driver/mod.rs index b8acfeb2..732e6ad2 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -14,11 +14,11 @@ //! Resources are logically mutable. All resource types contain useful read-only public fields, for //! example: //! -//! [`Buffer`] Field|`->` -//! -|- -//! [`device`](Buffer::device)|[`Device`](device::Device) -//! [`handle`](Buffer::handle)|[`vk::Buffer`] -//! [`info`](Buffer::info)|[`BufferInfo`] +//! | [`Buffer`] field | Type | +//! | --- | --- | +//! | [`device`](Buffer::device) | [`Device`](device::Device) | +//! | [`handle`](Buffer::handle) | [`vk::Buffer`] | +//! | [`info`](Buffer::info) | [`BufferInfo`] | //! //! Resources use atomic [`AccessType`](sync::AccessType) values to maintain consistency and track //! changes. @@ -31,39 +31,35 @@ //! The following pipelines are available: //! //! - [`ComputePipeline`](compute::ComputePipeline) -//! - [`GraphicsPipeline`](graphic::GraphicsPipeline) -//! - [`RayTracingPipeline`](ray_trace::RayTracingPipeline) +//! - [`GraphicsPipeline`](graphics::GraphicsPipeline) +//! - [`RayTracingPipeline`](ray_tracing::RayTracingPipeline) //! //! Pipelines are immutable. All pipeline types contain useful public methods, for //! example: //! -//! [`ComputePipeline`](compute::ComputePipeline) Method | `->` -//! -|- -//! [`device(&self)`](compute::ComputePipeline::device)|[`Device`](device::Device) -//! [`handle(&self)`](compute::ComputePipeline::handle)|[`vk::Pipeline`] -//! [`info(&self)`](compute::ComputePipeline::info) -//! | [`ComputePipelineInfo`](compute::ComputePipelineInfo) +//! | [`ComputePipeline`](compute::ComputePipeline) method | Type | +//! | --- | --- | +//! | [`device(&self)`](compute::ComputePipeline::device) | [`Device`](device::Device) | +//! | [`handle(&self)`](compute::ComputePipeline::handle) | [`vk::Pipeline`] | +//! | [`info(&self)`](compute::ComputePipeline::info) | [`ComputePipelineInfo`](compute::ComputePipelineInfo) | pub mod accel_struct; pub mod buffer; pub mod cmd_buf; pub mod compute; pub mod device; -pub mod graphic; +pub mod fence; +pub mod graphics; pub mod image; pub mod instance; pub mod physical_device; -pub mod ray_trace; +pub mod ray_tracing; pub mod render_pass; pub mod shader; pub mod surface; pub mod swapchain; -/// Descriptor pool allocation helpers used by pipeline execution. -/// -/// Part of the advanced driver API for callers that need direct control over descriptor -/// allocation. Most users should use the higher-level graph system instead. -pub mod descriptor_set; +pub(crate) mod descriptor_set; mod descriptor_set_layout; @@ -106,7 +102,7 @@ pub(crate) use self::{ use { self::{ buffer::{Buffer, BufferInfo}, - graphic::VertexInputState, + graphics::VertexInputState, }, ash::vk, gpu_allocator::AllocationError, @@ -481,8 +477,10 @@ pub const fn format_texel_block_size(fmt: vk::Format) -> u32 { vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 | vk::Format::G16_B16R16_2PLANE_444_UNORM => 6, - // Vulkan is extensible — new formats can appear at any time. The caller is expected to - // know the format is valid; no runtime assertion needed per the project's philosophy. + /* + Vulkan is extensible; new formats can appear at any time. The caller is expected to know the + format is valid; no runtime assertion needed per the project's philosophy. + */ _ => 0, } } @@ -743,8 +741,10 @@ pub const fn format_texel_block_extent(vk_format: vk::Format) -> (u32, u32) { | vk::Format::PVRTC1_4BPP_SRGB_BLOCK_IMG | vk::Format::PVRTC2_4BPP_UNORM_BLOCK_IMG | vk::Format::PVRTC2_4BPP_SRGB_BLOCK_IMG => (4, 4), - // Vulkan is extensible — new formats can appear at any time. The caller is expected to - // know the format is valid; no runtime assertion needed per the project's philosophy. + /* + Vulkan is extensible; new formats can appear at any time. The caller is expected to know the + format is valid; no runtime assertion needed per the project's philosophy. + */ _ => (1, 1), } } @@ -850,80 +850,6 @@ pub(super) const fn is_read_access(ty: self::sync::AccessType) -> bool { } } -pub(super) const fn is_write_access(ty: self::sync::AccessType) -> bool { - use self::sync::AccessType::*; - match ty { - Nothing - | CommandBufferReadNVX - | IndirectBuffer - | IndexBuffer - | VertexBuffer - | VertexShaderReadUniformBuffer - | VertexShaderReadSampledImageOrUniformTexelBuffer - | VertexShaderReadOther - | TessellationControlShaderReadUniformBuffer - | TessellationControlShaderReadSampledImageOrUniformTexelBuffer - | TessellationControlShaderReadOther - | TessellationEvaluationShaderReadUniformBuffer - | TessellationEvaluationShaderReadSampledImageOrUniformTexelBuffer - | TessellationEvaluationShaderReadOther - | GeometryShaderReadUniformBuffer - | GeometryShaderReadSampledImageOrUniformTexelBuffer - | GeometryShaderReadOther - | FragmentShaderReadUniformBuffer - | FragmentShaderReadSampledImageOrUniformTexelBuffer - | FragmentShaderReadColorInputAttachment - | FragmentShaderReadDepthStencilInputAttachment - | FragmentShaderReadOther - | ColorAttachmentRead - | DepthStencilAttachmentRead - | ComputeShaderReadUniformBuffer - | ComputeShaderReadSampledImageOrUniformTexelBuffer - | ComputeShaderReadOther - | AnyShaderReadUniformBuffer - | AnyShaderReadUniformBufferOrVertexBuffer - | AnyShaderReadSampledImageOrUniformTexelBuffer - | AnyShaderReadOther - | TransferRead - | HostRead - | Present - | RayTracingShaderReadSampledImageOrUniformTexelBuffer - | RayTracingShaderReadColorInputAttachment - | RayTracingShaderReadDepthStencilInputAttachment - | RayTracingShaderReadAccelerationStructure - | RayTracingShaderReadOther - | AccelerationStructureBuildRead - | MeshShaderReadUniformBuffer - | MeshShaderReadSampledImageOrUniformTexelBuffer - | MeshShaderReadOther - | TaskShaderReadUniformBuffer - | TaskShaderReadSampledImageOrUniformTexelBuffer - | TaskShaderReadOther => false, - CommandBufferWriteNVX - | VertexShaderWrite - | TessellationControlShaderWrite - | TessellationEvaluationShaderWrite - | GeometryShaderWrite - | FragmentShaderWrite - | ColorAttachmentWrite - | DepthStencilAttachmentWrite - | DepthStencilAttachmentReadWrite - | DepthAttachmentWriteStencilReadOnly - | StencilAttachmentWriteDepthReadOnly - | ComputeShaderWrite - | AnyShaderWrite - | TransferWrite - | HostWrite - | ColorAttachmentReadWrite - | General - | AccelerationStructureBuildWrite - | AccelerationStructureBufferWrite - | ComputeShaderReadWrite - | MeshShaderWrite - | TaskShaderWrite => true, - } -} - // Convert overlapping push constant regions such as this: // VERTEX 0..64 // FRAGMENT 0..80 @@ -1229,12 +1155,12 @@ pub(super) const fn pipeline_stage_access_flags( /// Describes the general category of all graphics driver failure cases. /// /// In the event of a failure you should follow the _vk-graph_ code to the responsible Vulkan API -/// and then to the `Ash` stub call; it will generally contain a link to the appropriate +/// and then to the `ash` function call; it will generally contain a link to the appropriate /// specification. The specifications provide a table of possible error conditions which can be a /// good starting point to debug the issue. /// -/// Feel free to open an issue on GitHub, [here](https://github.com/attackgoat/vk-graph/issues) for -/// help debugging the issue. +/// Feel free to open an [issue on GitHub](https://github.com/attackgoat/vk-graph/issues) for help +/// debugging the issue. #[derive(Clone, Copy, Debug)] pub enum DriverError { /// The input data, or referenced data, is not valid for the current state. @@ -1279,6 +1205,35 @@ impl Display for DriverError { impl Error for DriverError {} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SharingMode { + Concurrent, + Exclusive(Option<(u32, u32)>), +} + +impl SharingMode { + const MODE_CONCURRENT: u64 = u64::MAX; + const MODE_UNKNOWN: u64 = u64::MAX - 1; + + pub(super) const fn decode(val: u64) -> Self { + if val == Self::MODE_CONCURRENT { + Self::Concurrent + } else if val == Self::MODE_UNKNOWN { + Self::Exclusive(None) + } else { + Self::Exclusive(Some(((val >> 32) as u32, val as u32))) + } + } + + pub(super) const fn encode(self) -> u64 { + match self { + SharingMode::Concurrent => Self::MODE_CONCURRENT, + SharingMode::Exclusive(None) => Self::MODE_UNKNOWN, + SharingMode::Exclusive(Some((family, index))) => (family as u64) << 32 | index as u64, + } + } +} + #[cfg(test)] mod test { use {super::merge_push_constant_ranges, ash::vk}; diff --git a/src/driver/physical_device.rs b/src/driver/physical_device.rs index a988a755..5447cf91 100644 --- a/src/driver/physical_device.rs +++ b/src/driver/physical_device.rs @@ -1,9 +1,12 @@ -//! Physical device types +//! Physical device types. use { - super::{DriverError, instance::Instance}, + super::{ + DriverError, + instance::{ApiVersion, Instance}, + }, crate::driver::device::Device, - ash::{ext, khr, vk}, + ash::{ext, khr as ash_khr, vk}, log::{debug, error, warn}, std::{ collections::HashSet, @@ -13,61 +16,295 @@ use { }, }; -// TODO: There is a bunch of unsafe cstr handling here - does not check for null-termination +const UNKNOWN_C_STRING: &str = "unknown"; +const MAX_C_STRING_UTF8_BYTES: usize = 128; + +fn vk_cstr_to_utf8_string(cstr: &[c_char]) -> String { + let scan_len = cstr.len().min(MAX_C_STRING_UTF8_BYTES + 1); + let mut bytes = Vec::with_capacity(scan_len.min(MAX_C_STRING_UTF8_BYTES)); + + for (idx, &ch) in cstr.iter().take(scan_len).enumerate() { + let byte = ch as u8; + + if byte == 0 { + let Ok(res) = String::from_utf8(bytes) else { + break; + }; + + return res; + } + + if idx == MAX_C_STRING_UTF8_BYTES { + break; + } + + bytes.push(byte); + } -fn vk_cstr_to_string_lossy(cstr: &[c_char]) -> String { - unsafe { CStr::from_ptr(cstr.as_ptr()) } - .to_string_lossy() - .to_string() + UNKNOWN_C_STRING.to_owned() } -/// Properties of the physical device for acceleration structures. -/// -/// See [`VkPhysicalDeviceAccelerationStructurePropertiesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceAccelerationStructurePropertiesKHR.html). -#[derive(Clone, Copy, Debug)] -pub struct AccelerationStructureProperties { - /// The maximum number of geometries in a bottom level acceleration structure. - pub max_geometry_count: u64, +fn vk_extension_name(extension_name: &'static CStr) -> &'static str { + extension_name + .to_str() + .expect("Vulkan extension name should be UTF-8") +} - /// The maximum number of instances in a top level acceleration structure. - pub max_instance_count: u64, +/// Physical-device support types for `VK_KHR_*` extensions. +pub mod khr { + use ash::vk; + + /// Properties of the physical device for acceleration structures. + /// + /// See [`VkPhysicalDeviceAccelerationStructurePropertiesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceAccelerationStructurePropertiesKHR.html). + #[derive(Clone, Copy, Debug)] + pub struct AccelerationStructureProperties { + /// The maximum number of geometries in a bottom-level acceleration structure. + pub max_geometry_count: u64, + + /// The maximum number of instances in a top-level acceleration structure. + pub max_instance_count: u64, + + /// The maximum number of triangles or AABBs in all geometries in a bottom-level acceleration + /// structure. + pub max_primitive_count: u64, + + /// The maximum number of acceleration structure bindings that can be accessible to a single + /// shader stage in a pipeline layout. + /// + /// Descriptor bindings with a descriptor type of + /// `VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR` count against this limit. + pub max_per_stage_descriptor_accel_structs: u32, + + /// The maximum number of acceleration structure descriptors that can be included in descriptor + /// bindings in a pipeline layout across all pipeline shader stages and descriptor set numbers. + /// + /// Descriptor bindings with a descriptor type of + /// `VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR` count against this limit. + pub max_descriptor_set_accel_structs: u32, + + /// The minimum required alignment, in bytes, for scratch data passed in to an acceleration + /// structure build command. + pub min_accel_struct_scratch_offset_alignment: u32, + } - /// The maximum number of triangles or AABBs in all geometries in a bottom level acceleration - /// structure. - pub max_primitive_count: u64, + impl From> + for AccelerationStructureProperties + { + fn from(props: vk::PhysicalDeviceAccelerationStructurePropertiesKHR<'_>) -> Self { + Self { + max_geometry_count: props.max_geometry_count, + max_instance_count: props.max_instance_count, + max_primitive_count: props.max_primitive_count, + max_per_stage_descriptor_accel_structs: props + .max_per_stage_descriptor_acceleration_structures, + max_descriptor_set_accel_structs: props.max_descriptor_set_acceleration_structures, + min_accel_struct_scratch_offset_alignment: props + .min_acceleration_structure_scratch_offset_alignment, + } + } + } - /// The maximum number of acceleration structure bindings that can be accessible to a single - /// shader stage in a pipeline layout. - /// - /// Descriptor bindings with a descriptor type of - /// `VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR` count against this limit. - pub max_per_stage_descriptor_accel_structs: u32, + /// Features of the physical device for acceleration structures. + #[derive(Clone, Copy, Debug)] + pub struct AccelerationStructureFeatures { + /// Indicates whether the implementation supports acceleration structure functionality. + pub acceleration_structure: bool, + + /// Indicates whether acceleration structure capture and replay is supported. + pub acceleration_structure_capture_replay: bool, + + /// Indicates whether indirect acceleration structure build commands are supported. + pub acceleration_structure_indirect_build: bool, + + /// Indicates whether acceleration structures can be built on the host. + pub acceleration_structure_host_commands: bool, + + /// Indicates whether acceleration structure descriptors can be updated after bind. + pub descriptor_binding_acceleration_structure_update_after_bind: bool, + } + + impl From> + for AccelerationStructureFeatures + { + fn from(features: vk::PhysicalDeviceAccelerationStructureFeaturesKHR<'_>) -> Self { + Self { + acceleration_structure: features.acceleration_structure == vk::TRUE, + acceleration_structure_capture_replay: features + .acceleration_structure_capture_replay + == vk::TRUE, + acceleration_structure_indirect_build: features + .acceleration_structure_indirect_build + == vk::TRUE, + acceleration_structure_host_commands: features.acceleration_structure_host_commands + == vk::TRUE, + descriptor_binding_acceleration_structure_update_after_bind: features + .descriptor_binding_acceleration_structure_update_after_bind + == vk::TRUE, + } + } + } + + /// Features and properties advertised by `VK_KHR_acceleration_structure`. + #[derive(Clone, Copy, Debug)] + pub struct AccelerationStructure { + /// Features advertised by `VK_KHR_acceleration_structure`. + pub features: AccelerationStructureFeatures, + + /// Properties advertised by `VK_KHR_acceleration_structure`. + pub properties: AccelerationStructureProperties, + } + + /// Features of the physical device for ray tracing. + /// + /// See [`VkPhysicalDeviceRayTracingPipelineFeaturesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceRayTracingPipelineFeaturesKHR.html). + #[derive(Clone, Copy, Debug)] + pub struct RayTracingPipelineFeatures { + /// Indicates whether the implementation supports the ray tracing pipeline functionality. + /// + /// See the [ray tracing pipeline chapter](https://docs.vulkan.org/spec/latest/chapters/raytracing.html). + /// + pub ray_tracing_pipeline: bool, + + /// Indicates whether the implementation supports saving and reusing shader group handles, e.g. + /// for trace capture and replay. + pub ray_tracing_pipeline_shader_group_handle_capture_replay: bool, + + /// Indicates whether the implementation supports reuse of shader group handles being + /// arbitrarily mixed with creation of non-reused shader group handles. + /// + /// If this is `false`, all reused shader group handles must be specified before any non-reused + /// handles may be created. + pub ray_tracing_pipeline_shader_group_handle_capture_replay_mixed: bool, + + /// Indicates whether the implementation supports indirect ray tracing commands, e.g. + /// `vkCmdTraceRaysIndirectKHR`. + pub ray_tracing_pipeline_trace_rays_indirect: bool, + + /// Indicates whether the implementation supports primitive culling during ray traversal. + pub ray_traversal_primitive_culling: bool, + } + + impl From> for RayTracingPipelineFeatures { + fn from(features: vk::PhysicalDeviceRayTracingPipelineFeaturesKHR<'_>) -> Self { + Self { + ray_tracing_pipeline: features.ray_tracing_pipeline == vk::TRUE, + ray_tracing_pipeline_shader_group_handle_capture_replay: features + .ray_tracing_pipeline_shader_group_handle_capture_replay + == vk::TRUE, + ray_tracing_pipeline_shader_group_handle_capture_replay_mixed: features + .ray_tracing_pipeline_shader_group_handle_capture_replay_mixed + == vk::TRUE, + ray_tracing_pipeline_trace_rays_indirect: features + .ray_tracing_pipeline_trace_rays_indirect + == vk::TRUE, + ray_traversal_primitive_culling: features.ray_traversal_primitive_culling + == vk::TRUE, + } + } + } - /// The maximum number of acceleration structure descriptors that can be included in descriptor - /// bindings in a pipeline layout across all pipeline shader stages and descriptor set numbers. + /// Properties of the physical device for ray tracing. /// - /// Descriptor bindings with a descriptor type of - /// `VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR` count against this limit. - pub max_descriptor_set_accel_structs: u32, + /// See [`VkPhysicalDeviceRayTracingPipelinePropertiesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceRayTracingPipelinePropertiesKHR.html). + #[derive(Clone, Copy, Debug)] + pub struct RayTracingPipelineProperties { + /// The size in bytes of the shader header. + pub shader_group_handle_size: u32, - /// The minimum required alignment, in bytes, for scratch data passed in to an acceleration - /// structure build command. - pub min_accel_struct_scratch_offset_alignment: u32, -} + /// The maximum number of levels of ray recursion allowed in a trace command. + pub max_ray_recursion_depth: u32, -impl From> - for AccelerationStructureProperties -{ - fn from(props: vk::PhysicalDeviceAccelerationStructurePropertiesKHR<'_>) -> Self { - Self { - max_geometry_count: props.max_geometry_count, - max_instance_count: props.max_instance_count, - max_primitive_count: props.max_primitive_count, - max_per_stage_descriptor_accel_structs: props - .max_per_stage_descriptor_acceleration_structures, - max_descriptor_set_accel_structs: props.max_descriptor_set_acceleration_structures, - min_accel_struct_scratch_offset_alignment: props - .min_acceleration_structure_scratch_offset_alignment, + /// The maximum stride in bytes allowed between shader groups in the shader binding table. + pub max_shader_group_stride: u32, + + /// The required alignment in bytes for the base of the shader binding table. + pub shader_group_base_alignment: u32, + + /// The number of bytes for the information required to do capture and replay for shader group + /// handles. + pub shader_group_handle_capture_replay_size: u32, + + /// The maximum number of ray generation shader invocations which may be produced by a single + /// `vkCmdTraceRaysIndirectKHR` or `vkCmdTraceRaysKHR` command. + pub max_ray_dispatch_invocation_count: u32, + + /// The required alignment in bytes for each shader binding table entry. + /// + /// The value must be a power of two. + pub shader_group_handle_alignment: u32, + + /// The maximum size in bytes for a ray attribute structure. + pub max_ray_hit_attribute_size: u32, + } + + impl From> for RayTracingPipelineProperties { + fn from(props: vk::PhysicalDeviceRayTracingPipelinePropertiesKHR<'_>) -> Self { + Self { + shader_group_handle_size: props.shader_group_handle_size, + max_ray_recursion_depth: props.max_ray_recursion_depth, + max_shader_group_stride: props.max_shader_group_stride, + shader_group_base_alignment: props.shader_group_base_alignment, + shader_group_handle_capture_replay_size: props + .shader_group_handle_capture_replay_size, + max_ray_dispatch_invocation_count: props.max_ray_dispatch_invocation_count, + shader_group_handle_alignment: props.shader_group_handle_alignment, + max_ray_hit_attribute_size: props.max_ray_hit_attribute_size, + } + } + } + + /// Features and properties advertised by `VK_KHR_ray_tracing_pipeline`. + #[derive(Clone, Copy, Debug)] + pub struct RayTracingPipeline { + /// Features advertised by `VK_KHR_ray_tracing_pipeline`. + pub features: RayTracingPipelineFeatures, + + /// Properties advertised by `VK_KHR_ray_tracing_pipeline`. + pub properties: RayTracingPipelineProperties, + } + + /// Features advertised by `VK_KHR_present_id`. + #[derive(Clone, Copy, Debug)] + pub struct PresentId { + /// Features advertised by `VK_KHR_present_id`. + pub features: PresentIdFeatures, + } + + /// Features of the physical device for present IDs. + #[derive(Clone, Copy, Debug)] + pub struct PresentIdFeatures { + /// Indicates whether present IDs are supported. + pub present_id: bool, + } + + impl From> for PresentIdFeatures { + fn from(features: vk::PhysicalDevicePresentIdFeaturesKHR<'_>) -> Self { + Self { + present_id: features.present_id == vk::TRUE, + } + } + } + + /// Features advertised by `VK_KHR_present_wait`. + #[derive(Clone, Copy, Debug)] + pub struct PresentWait { + /// Features advertised by `VK_KHR_present_wait`. + pub features: PresentWaitFeatures, + } + + /// Features of the physical device for present wait. + #[derive(Clone, Copy, Debug)] + pub struct PresentWaitFeatures { + /// Indicates whether present wait is supported. + pub present_wait: bool, + } + + impl From> for PresentWaitFeatures { + fn from(features: vk::PhysicalDevicePresentWaitFeaturesKHR<'_>) -> Self { + Self { + present_wait: features.present_wait == vk::TRUE, + } } } } @@ -84,7 +321,7 @@ pub struct DepthStencilResolveProperties { /// support additional modes. pub supported_depth_resolve_modes: vk::ResolveModeFlags, - /// A bitmask of indicating the set of supported stencil resolve modes. + /// A bitmask indicating the set of supported stencil resolve modes. /// /// `VK_RESOLVE_MODE_SAMPLE_ZERO_BIT` must be included in the set but implementations may /// support additional modes. `VK_RESOLVE_MODE_AVERAGE_BIT` must not be included in the set. @@ -115,34 +352,20 @@ impl From> for DepthStencilR } } -/// Features of the physical device for vertex indexing. +/// Structure which holds data about a physical device. /// -/// See [`VkPhysicalDeviceIndexTypeUint8FeaturesEXT`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceIndexTypeUint8FeaturesEXT.html). -#[derive(Clone, Copy, Debug, Default)] -pub struct IndexTypeUint8Features { - /// Indicates that VK_INDEX_TYPE_UINT8_EXT can be used with vkCmdBindIndexBuffer2KHR and - /// vkCmdBindIndexBuffer. - pub index_type_uint8: bool, -} - -impl From> for IndexTypeUint8Features { - fn from(features: vk::PhysicalDeviceIndexTypeUint8FeaturesEXT<'_>) -> Self { - Self { - index_type_uint8: features.index_type_uint8 == vk::TRUE, - } - } -} - -/// Structure which holds data about the physical hardware selected by the current device. +/// Extension support is exposed through fields named after Vulkan extension names. Extensions that +/// only need a support check are booleans, such as [`Self::vk_ext_private_data`] and +/// [`Self::vk_khr_swapchain`]. Extensions with feature or property data are `Option` fields, such +/// as [`Self::vk_khr_acceleration_structure`] and [`Self::vk_khr_ray_tracing_pipeline`]. +/// +/// Use [`Device::physical`](super::device::Device::physical) for the selected physical device of a +/// logical device. +/// +/// See [`VkPhysicalDevice`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDevice.html). #[derive(Clone)] #[read_only::cast] pub struct PhysicalDevice { - /// Describes the properties of the device which relate to acceleration structures, if - /// available. - /// - /// _Note:_ This field is read-only. - pub accel_struct_properties: Option, - /// Describes the properties of the device which relate to depth/stencil resolve operations. /// /// _Note:_ This field is read-only. @@ -166,16 +389,11 @@ pub struct PhysicalDevice { /// _Note:_ This field is read-only. pub features_v1_2: Vulkan12Features, - /// The native Vulkan resource handle of this buffer. + /// The native Vulkan resource handle of this physical device. /// /// _Note:_ This field is read-only. pub handle: vk::PhysicalDevice, - /// Describes the features of the physical device which relate to vertex indexing. - /// - /// _Note:_ This field is read-only. - pub index_type_uint8_features: IndexTypeUint8Features, - /// The Vulkan instance which owns this device. /// /// _Note:_ This field is read-only. @@ -186,19 +404,19 @@ pub struct PhysicalDevice { /// _Note:_ This field is read-only. pub memory_properties: vk::PhysicalDeviceMemoryProperties, - /// Device properties of the physical device which are part of the Vulkan 1.0 base feature set. + /// Device properties of the physical device which are part of the Vulkan 1.0 base property set. /// /// _Note:_ This field is read-only. pub properties_v1_0: Vulkan10Properties, /// Describes the properties of the physical device which are part of the Vulkan 1.1 base - /// feature set. + /// property set. /// /// _Note:_ This field is read-only. pub properties_v1_1: Vulkan11Properties, /// Describes the properties of the physical device which are part of the Vulkan 1.2 base - /// feature set. + /// property set. /// /// _Note:_ This field is read-only. pub properties_v1_2: Vulkan12Properties, @@ -210,37 +428,63 @@ pub struct PhysicalDevice { pub(crate) queue_family_indices: Box<[u32]>, - /// Describes the features of the device which relate to ray query, if available. + /// `VK_KHR_ray_tracing_pipeline` features and properties, when supported. /// /// _Note:_ This field is read-only. - pub ray_query_features: RayQueryFeatures, + pub vk_khr_ray_tracing_pipeline: Option, - /// Describes the features of the device which relate to ray tracing, if available. + /// Describes the properties of the device which relate to min/max sampler filtering. /// /// _Note:_ This field is read-only. - pub ray_tracing_pipeline_features: RayTracingPipelineFeatures, + pub sampler_filter_minmax_properties: SamplerFilterMinmaxProperties, - /// Describes the properties of the device which relate to ray tracing, if available. + /// Whether `VK_EXT_index_type_uint8` support is available. /// /// _Note:_ This field is read-only. - pub ray_tracing_pipeline_properties: Option, + pub vk_ext_index_type_uint8: bool, - /// Describes the properties of the device which relate to min/max sampler filtering. + /// Whether `VK_EXT_private_data` support is available. /// /// _Note:_ This field is read-only. - pub sampler_filter_minmax_properties: SamplerFilterMinmaxProperties, + pub vk_ext_private_data: bool, - /// True if the device supports swapchain use. + /// `VK_KHR_acceleration_structure` features and properties, when supported. /// /// _Note:_ This field is read-only. - pub khr_swapchain: bool, + pub vk_khr_acceleration_structure: Option, + + /// `VK_KHR_present_id` features, when supported. + /// + /// _Note:_ This field is read-only. + pub vk_khr_present_id: Option, + + /// `VK_KHR_present_wait` features, when supported. + /// + /// _Note:_ This field is read-only. + pub vk_khr_present_wait: Option, + + /// Whether `VK_KHR_ray_query` support is available. + /// + /// _Note:_ This field is read-only. + pub vk_khr_ray_query: bool, + + /// Whether `VK_KHR_swapchain` support is available. + /// + /// _Note:_ This field is read-only. + pub vk_khr_swapchain: bool, + + /// Whether synchronization2 support is available through Vulkan 1.3 core or + /// `VK_KHR_synchronization2`. + /// + /// _Note:_ This field is read-only. + pub vk_khr_synchronization2: bool, } impl PhysicalDevice { /// Prepares device creation information and calls the provided callback to allow an application /// to control the device creation process. /// - /// _Note:_ This is only useful for interoperating with other libraries as device creation is + /// _Note:_ This is only useful for interoperating with other libraries as device creation is /// normally handled by the [`Device::try_from_display`] and [`Device::create`] /// functions. /// @@ -248,41 +492,65 @@ impl PhysicalDevice { /// /// This comes with all the caveats of using `ash` builder types, which are inherently /// dangerous. Use with extreme caution. + /// + /// See [`VkDeviceCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDeviceCreateInfo.html). #[profiling::function] pub unsafe fn create_ash_device(&self, create_fn: F) -> ash::prelude::VkResult where F: FnOnce(vk::DeviceCreateInfo) -> ash::prelude::VkResult, { - let mut enabled_ext_names = Vec::with_capacity(6); + let mut enabled_ext_names = Vec::with_capacity(11); - // The swapchain extension is required for presentation support, so we enable it whenever - // the physical device reports support. Imported instances may already carry the required - // instance extensions even though vk-graph did not create them. - if self.khr_swapchain { - enabled_ext_names.push(khr::swapchain::NAME.as_ptr()); + if self.vk_khr_acceleration_structure.is_some() { + enabled_ext_names.push(ash_khr::acceleration_structure::NAME.as_ptr()); + enabled_ext_names.push(ash_khr::deferred_host_operations::NAME.as_ptr()); } - if self.accel_struct_properties.is_some() { - enabled_ext_names.push(khr::acceleration_structure::NAME.as_ptr()); - enabled_ext_names.push(khr::deferred_host_operations::NAME.as_ptr()); + if self.vk_ext_index_type_uint8 { + enabled_ext_names.push(ext::index_type_uint8::NAME.as_ptr()); } - if self.ray_query_features.ray_query { - enabled_ext_names.push(khr::ray_query::NAME.as_ptr()); + if self.vk_khr_present_id.is_some() { + enabled_ext_names.push(ash_khr::present_id::NAME.as_ptr()); } - if self.ray_tracing_pipeline_features.ray_tracing_pipeline { - enabled_ext_names.push(khr::ray_tracing_pipeline::NAME.as_ptr()); + if self.vk_khr_present_wait.is_some() { + enabled_ext_names.push(ash_khr::present_wait::NAME.as_ptr()); } - if self.index_type_uint8_features.index_type_uint8 { - enabled_ext_names.push(ext::index_type_uint8::NAME.as_ptr()); + if self.vk_khr_synchronization2 && self.instance.info.api_version < ApiVersion::Vulkan13 { + enabled_ext_names.push(ash_khr::synchronization2::NAME.as_ptr()); + } + + if self.vk_ext_private_data { + enabled_ext_names.push(ext::private_data::NAME.as_ptr()); + } + + if self.vk_khr_ray_query { + enabled_ext_names.push(ash_khr::ray_query::NAME.as_ptr()); } - // Molten-vk doesn't support the full Vulkan feature set, hence the portability subset - // extension must be enabled. + if self + .vk_khr_ray_tracing_pipeline + .as_ref() + .is_some_and(|ext| ext.features.ray_tracing_pipeline) + { + enabled_ext_names.push(ash_khr::ray_tracing_pipeline::NAME.as_ptr()); + } + + /* + The swapchain extension is required for presentation support, so we enable it whenever the + physical device reports support. Imported instances may already carry the required instance + extensions even though vk-graph did not create them. + */ + if self.vk_khr_swapchain { + enabled_ext_names.push(ash_khr::swapchain::NAME.as_ptr()); + } + + // MoltenVK doesn't support the full Vulkan feature set, hence the portability subset + // extension must be enabled #[cfg(all(target_os = "macos", feature = "loaded"))] - enabled_ext_names.push(khr::portability_subset::NAME.as_ptr()); + enabled_ext_names.push(ash_khr::portability_subset::NAME.as_ptr()); let priorities = repeat_n( 1.0, @@ -317,27 +585,51 @@ impl PhysicalDevice { let mut acceleration_structure_features = vk::PhysicalDeviceAccelerationStructureFeaturesKHR::default(); let mut index_type_uint8_features = vk::PhysicalDeviceIndexTypeUint8FeaturesEXT::default(); + let mut present_id_features = vk::PhysicalDevicePresentIdFeaturesKHR::default(); + let mut present_wait_features = vk::PhysicalDevicePresentWaitFeaturesKHR::default(); let mut ray_query_features = vk::PhysicalDeviceRayQueryFeaturesKHR::default(); let mut ray_tracing_pipeline_features = vk::PhysicalDeviceRayTracingPipelineFeaturesKHR::default(); + let mut synchronization2_features = vk::PhysicalDeviceSynchronization2Features::default(); + let mut private_data_features = vk::PhysicalDevicePrivateDataFeaturesEXT::default(); let mut features = vk::PhysicalDeviceFeatures2::default() .push_next(&mut features_v1_1) .push_next(&mut features_v1_2); - if self.accel_struct_properties.is_some() { + if self.vk_khr_acceleration_structure.is_some() { features = features.push_next(&mut acceleration_structure_features); } - if self.ray_query_features.ray_query { + if self.vk_ext_index_type_uint8 { + features = features.push_next(&mut index_type_uint8_features); + } + + if self.vk_khr_present_id.is_some() { + features = features.push_next(&mut present_id_features); + } + + if self.vk_khr_present_wait.is_some() { + features = features.push_next(&mut present_wait_features); + } + + if self.vk_khr_ray_query { features = features.push_next(&mut ray_query_features); } - if self.ray_tracing_pipeline_features.ray_tracing_pipeline { + if self + .vk_khr_ray_tracing_pipeline + .as_ref() + .is_some_and(|ext| ext.features.ray_tracing_pipeline) + { features = features.push_next(&mut ray_tracing_pipeline_features); } - if self.index_type_uint8_features.index_type_uint8 { - features = features.push_next(&mut index_type_uint8_features); + if self.vk_khr_synchronization2 { + features = features.push_next(&mut synchronization2_features); + } + + if self.vk_ext_private_data { + features = features.push_next(&mut private_data_features); } unsafe { get_physical_device_features2(self.handle, &mut features) }; @@ -355,14 +647,19 @@ impl PhysicalDevice { /// /// # Panics /// - /// Panics if [Self.physical_device.ray_tracing_pipeline_properties] is `None`. - pub(crate) fn expect_ray_tracing_pipeline_properties(&self) -> &RayTracingPipelineProperties { - self.ray_tracing_pipeline_properties + /// Panics if [`Self::vk_khr_ray_tracing_pipeline`] is `None`. + pub(crate) fn expect_ray_tracing_pipeline_properties( + &self, + ) -> &khr::RayTracingPipelineProperties { + self.vk_khr_ray_tracing_pipeline .as_ref() + .map(|ext| &ext.properties) .expect("missing VK_KHR_ray_tracing_pipeline") } /// Lists the capabilities of a given format. + /// + /// See [`vkGetPhysicalDeviceFormatProperties`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetPhysicalDeviceFormatProperties.html). #[profiling::function] pub fn format_properties(&self, format: vk::Format) -> vk::FormatProperties { unsafe { @@ -374,11 +671,13 @@ impl PhysicalDevice { /// Lists the physical device's image format capabilities. /// /// A result of `None` indicates the format is not supported. + /// + /// See [`vkGetPhysicalDeviceImageFormatProperties`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetPhysicalDeviceImageFormatProperties.html). #[profiling::function] pub fn image_format_properties( &self, format: vk::Format, - ty: vk::ImageType, + image_type: vk::ImageType, tiling: vk::ImageTiling, usage: vk::ImageUsageFlags, flags: vk::ImageCreateFlags, @@ -387,7 +686,7 @@ impl PhysicalDevice { match self.instance.get_physical_device_image_format_properties( self.handle, format, - ty, + image_type, tiling, usage, flags, @@ -396,7 +695,7 @@ impl PhysicalDevice { Err(err) if err == vk::Result::ERROR_FORMAT_NOT_SUPPORTED => { // We don't log this condition because it is normal for unsupported // formats to be checked - we use the result to inform callers they - // cannot use those formats. + // cannot use those formats Ok(None) } @@ -410,7 +709,12 @@ impl PhysicalDevice { } /// Creates a physical device wrapper which reports features and properties. - pub fn try_from_ash( + /// + /// # Safety + /// + /// `physical_device` must be a valid handle enumerated from `instance`, and it must remain + /// valid for the lifetime of the returned wrapper. + pub unsafe fn try_from_ash( instance: &Instance, physical_device: vk::PhysicalDevice, ) -> Result { @@ -441,30 +745,111 @@ impl PhysicalDevice { .. } = instance.fp_v1_1(); - // Gather required features of the physical device + let extension_properties = unsafe { + instance + .enumerate_device_extension_properties(physical_device) + .map_err(|err| { + error!("unable to enumerate device extensions: {err}"); + + DriverError::Unsupported + })? + }; + + // Check for supported extensions + let extension_names = extension_properties + .iter() + .map(|prop| { + let extension_name = vk_cstr_to_utf8_string(&prop.extension_name); + + debug!("extension {:?} v{}", extension_name, prop.spec_version); + + extension_name + }) + .collect::>(); + let vk_khr_acceleration_structure = extension_names + .contains(vk_extension_name(ash_khr::acceleration_structure::NAME)) + && extension_names.contains(vk_extension_name(ash_khr::deferred_host_operations::NAME)); + let mut vk_ext_index_type_uint8 = + extension_names.contains(vk_extension_name(ext::index_type_uint8::NAME)); + let mut vk_ext_private_data = + extension_names.contains(vk_extension_name(ext::private_data::NAME)); + let mut vk_khr_present_id = + extension_names.contains(vk_extension_name(ash_khr::present_id::NAME)); + let mut vk_khr_present_wait = + extension_names.contains(vk_extension_name(ash_khr::present_wait::NAME)); + let mut vk_khr_ray_query = + extension_names.contains(vk_extension_name(ash_khr::ray_query::NAME)); + let vk_khr_ray_tracing_pipeline = + extension_names.contains(vk_extension_name(ash_khr::ray_tracing_pipeline::NAME)); + let vk_khr_swapchain = instance.khr_surface + && extension_names.contains(vk_extension_name(ash_khr::swapchain::NAME)); + let mut vk_khr_synchronization2 = extension_names + .contains(vk_extension_name(ash_khr::synchronization2::NAME)) + || instance.info.api_version >= ApiVersion::Vulkan13; + + // Gather advertised features of the physical device let mut features_v1_1 = vk::PhysicalDeviceVulkan11Features::default(); let mut features_v1_2 = vk::PhysicalDeviceVulkan12Features::default(); let mut acceleration_structure_features = vk::PhysicalDeviceAccelerationStructureFeaturesKHR::default(); let mut index_type_u8_features = vk::PhysicalDeviceIndexTypeUint8FeaturesEXT::default(); + let mut present_id_features = vk::PhysicalDevicePresentIdFeaturesKHR::default(); + let mut present_wait_features = vk::PhysicalDevicePresentWaitFeaturesKHR::default(); let mut ray_query_features = vk::PhysicalDeviceRayQueryFeaturesKHR::default(); let mut ray_tracing_pipeline_features = vk::PhysicalDeviceRayTracingPipelineFeaturesKHR::default(); + let mut synchronization2_features = vk::PhysicalDeviceSynchronization2Features::default(); + let mut private_data_features = vk::PhysicalDevicePrivateDataFeaturesEXT::default(); let mut features = vk::PhysicalDeviceFeatures2::default() .push_next(&mut features_v1_1) - .push_next(&mut features_v1_2) - .push_next(&mut acceleration_structure_features) - .push_next(&mut index_type_u8_features) - .push_next(&mut ray_query_features) - .push_next(&mut ray_tracing_pipeline_features); + .push_next(&mut features_v1_2); + + if vk_khr_acceleration_structure { + features = features.push_next(&mut acceleration_structure_features); + } + + if vk_ext_index_type_uint8 { + features = features.push_next(&mut index_type_u8_features); + } + + if vk_ext_private_data { + features = features.push_next(&mut private_data_features); + } + + if vk_khr_present_id { + features = features.push_next(&mut present_id_features); + } + + if vk_khr_present_wait { + features = features.push_next(&mut present_wait_features); + } + + if vk_khr_ray_query { + features = features.push_next(&mut ray_query_features); + } + + if vk_khr_ray_tracing_pipeline { + features = features.push_next(&mut ray_tracing_pipeline_features); + } + + if vk_khr_synchronization2 { + features = features.push_next(&mut synchronization2_features); + } + unsafe { get_physical_device_features2(physical_device, &mut features); } let features_v1_0 = features.features.into(); let features_v1_1 = features_v1_1.into(); let features_v1_2 = features_v1_2.into(); - - // Gather required properties of the physical device + vk_ext_index_type_uint8 &= index_type_u8_features.index_type_uint8 == vk::TRUE; + vk_khr_present_id &= present_id_features.present_id == vk::TRUE; + vk_khr_present_wait &= present_wait_features.present_wait == vk::TRUE && vk_khr_present_id; + vk_khr_ray_query &= ray_query_features.ray_query == vk::TRUE; + vk_khr_synchronization2 &= synchronization2_features.synchronization2 == vk::TRUE; + vk_ext_private_data &= private_data_features.private_data == vk::TRUE; + + // Gather advertised properties of the physical device let mut properties_v1_1 = vk::PhysicalDeviceVulkan11Properties::default(); let mut properties_v1_2 = vk::PhysicalDeviceVulkan12Properties::default(); let mut accel_struct_properties = @@ -491,86 +876,46 @@ impl PhysicalDevice { let depth_stencil_resolve_properties = depth_stencil_resolve_properties.into(); let sampler_filter_minmax_properties = sampler_filter_minmax_properties.into(); - let extension_properties = unsafe { - instance - .enumerate_device_extension_properties(physical_device) - .map_err(|err| { - error!("unable to enumerate device extensions: {err}"); - - DriverError::Unsupported - })? - }; - - debug!("physical device: {}", &properties_v1_0.device_name); - - for prop in &extension_properties { - let extension_name = prop.extension_name.as_ptr(); - - if extension_name.is_null() { - warn!("invalid device extension name pointer: null"); - - return Err(DriverError::InvalidData); - } - - let extension_name = unsafe { CStr::from_ptr(extension_name) }; - - debug!("extension {:?} v{}", extension_name, prop.spec_version); - } - - // Check for supported extensions - let extension_names = extension_properties - .iter() - .map(|prop| prop.extension_name.as_ptr()) - .filter(|extension_name| !extension_name.is_null()) - .map(|extension_name| unsafe { CStr::from_ptr(extension_name) }) - .collect::>(); - let accel_struct = extension_names.contains(khr::acceleration_structure::NAME) - && extension_names.contains(khr::deferred_host_operations::NAME); - let index_type_uint8 = extension_names.contains(ext::index_type_uint8::NAME); - let ray_query = extension_names.contains(khr::ray_query::NAME); - let ray_tracing_pipeline = extension_names.contains(khr::ray_tracing_pipeline::NAME); - let khr_swapchain = instance.khr_surface && extension_names.contains(khr::swapchain::NAME); - - // Gather optional features and properties of the physical device - let index_type_uint8_features = if index_type_uint8 { - index_type_u8_features.into() - } else { - Default::default() - }; - let ray_query_features = if ray_query { - ray_query_features.into() - } else { - Default::default() - }; - let ray_tracing_pipeline_features = if ray_tracing_pipeline { - ray_tracing_pipeline_features.into() - } else { - Default::default() - }; - let accel_struct_properties = accel_struct.then(|| accel_struct_properties.into()); - let ray_tracing_properties = - ray_tracing_pipeline.then(|| ray_tracing_pipeline_properties.into()); + let vk_khr_acceleration_structure = + vk_khr_acceleration_structure.then(|| khr::AccelerationStructure { + features: acceleration_structure_features.into(), + properties: accel_struct_properties.into(), + }); + let vk_khr_present_id = vk_khr_present_id.then(|| khr::PresentId { + features: present_id_features.into(), + }); + let vk_khr_present_wait = vk_khr_present_wait.then(|| khr::PresentWait { + features: present_wait_features.into(), + }); + let vk_khr_ray_tracing_pipeline = + vk_khr_ray_tracing_pipeline.then(|| khr::RayTracingPipeline { + features: ray_tracing_pipeline_features.into(), + properties: ray_tracing_pipeline_properties.into(), + }); Ok(Self { - accel_struct_properties, depth_stencil_resolve_properties, features_v1_0, features_v1_1, features_v1_2, handle: physical_device, - index_type_uint8_features, instance: instance.clone(), - khr_swapchain, memory_properties, properties_v1_0, properties_v1_1, properties_v1_2, queue_families, queue_family_indices, - ray_query_features, - ray_tracing_pipeline_features, - ray_tracing_pipeline_properties: ray_tracing_properties, sampler_filter_minmax_properties, + vk_ext_index_type_uint8, + vk_ext_private_data, + vk_khr_acceleration_structure, + vk_khr_present_id, + vk_khr_present_wait, + vk_khr_ray_query, + vk_khr_ray_tracing_pipeline, + vk_khr_swapchain, + vk_khr_synchronization2, }) } @@ -590,124 +935,9 @@ impl Debug for PhysicalDevice { } } -/// Features of the physical device for ray query. -/// -/// See [`VkPhysicalDeviceRayQueryFeaturesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceRayQueryFeaturesKHR.html). -#[derive(Clone, Copy, Debug, Default)] -pub struct RayQueryFeatures { - /// Indicates whether the implementation supports ray query (`OpRayQueryProceedKHR`) - /// functionality. - pub ray_query: bool, -} - -impl From> for RayQueryFeatures { - fn from(features: vk::PhysicalDeviceRayQueryFeaturesKHR<'_>) -> Self { - Self { - ray_query: features.ray_query == vk::TRUE, - } - } -} - -/// Features of the physical device for ray tracing. -/// -/// See [`VkPhysicalDeviceRayTracingPipelineFeaturesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceRayTracingPipelineFeaturesKHR.html). -#[derive(Clone, Copy, Debug, Default)] -pub struct RayTracingPipelineFeatures { - /// Indicates whether the implementation supports the ray tracing pipeline functionality. - /// - /// See - /// See the [ray tracing pipeline chapter](https://docs.vulkan.org/spec/latest/chapters/raytracing.html). - pub ray_tracing_pipeline: bool, - - /// Indicates whether the implementation supports saving and reusing shader group handles, e.g. - /// for trace capture and replay. - pub ray_tracing_pipeline_shader_group_handle_capture_replay: bool, - - /// Indicates whether the implementation supports reuse of shader group handles being - /// arbitrarily mixed with creation of non-reused shader group handles. - /// - /// If this is `false`, all reused shader group handles must be specified before any non-reused - /// handles may be created. - pub ray_tracing_pipeline_shader_group_handle_capture_replay_mixed: bool, - - /// Indicates whether the implementation supports indirect ray tracing commands, e.g. - /// `vkCmdTraceRaysIndirectKHR`. - pub ray_tracing_pipeline_trace_rays_indirect: bool, - - /// Indicates whether the implementation supports primitive culling during ray traversal. - pub ray_traversal_primitive_culling: bool, -} - -impl From> for RayTracingPipelineFeatures { - fn from(features: vk::PhysicalDeviceRayTracingPipelineFeaturesKHR<'_>) -> Self { - Self { - ray_tracing_pipeline: features.ray_tracing_pipeline == vk::TRUE, - ray_tracing_pipeline_shader_group_handle_capture_replay: features - .ray_tracing_pipeline_shader_group_handle_capture_replay - == vk::TRUE, - ray_tracing_pipeline_shader_group_handle_capture_replay_mixed: features - .ray_tracing_pipeline_shader_group_handle_capture_replay_mixed - == vk::TRUE, - ray_tracing_pipeline_trace_rays_indirect: features - .ray_tracing_pipeline_trace_rays_indirect - == vk::TRUE, - ray_traversal_primitive_culling: features.ray_traversal_primitive_culling == vk::TRUE, - } - } -} - -/// Properties of the physical device for ray tracing. -/// -/// See [`VkPhysicalDeviceRayTracingPipelinePropertiesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceRayTracingPipelinePropertiesKHR.html). -#[derive(Clone, Copy, Debug)] -pub struct RayTracingPipelineProperties { - /// The size in bytes of the shader header. - pub shader_group_handle_size: u32, - - /// The maximum number of levels of ray recursion allowed in a trace command. - pub max_ray_recursion_depth: u32, - - /// The maximum stride in bytes allowed between shader groups in the shader binding table. - pub max_shader_group_stride: u32, - - /// The required alignment in bytes for the base of the shader binding table. - pub shader_group_base_alignment: u32, - - /// The number of bytes for the information required to do capture and replay for shader group - /// handles. - pub shader_group_handle_capture_replay_size: u32, - - /// The maximum number of ray generation shader invocations which may be produced by a single - /// vkCmdTraceRaysIndirectKHR or vkCmdTraceRaysKHR command. - pub max_ray_dispatch_invocation_count: u32, - - /// The required alignment in bytes for each shader binding table entry. - /// - /// The value must be a power of two. - pub shader_group_handle_alignment: u32, - - /// The maximum size in bytes for a ray attribute structure. - pub max_ray_hit_attribute_size: u32, -} - -impl From> for RayTracingPipelineProperties { - fn from(props: vk::PhysicalDeviceRayTracingPipelinePropertiesKHR<'_>) -> Self { - Self { - shader_group_handle_size: props.shader_group_handle_size, - max_ray_recursion_depth: props.max_ray_recursion_depth, - max_shader_group_stride: props.max_shader_group_stride, - shader_group_base_alignment: props.shader_group_base_alignment, - shader_group_handle_capture_replay_size: props.shader_group_handle_capture_replay_size, - max_ray_dispatch_invocation_count: props.max_ray_dispatch_invocation_count, - shader_group_handle_alignment: props.shader_group_handle_alignment, - max_ray_hit_attribute_size: props.max_ray_hit_attribute_size, - } - } -} - /// Properties of the physical device for min/max sampler filtering. /// -/// See [`VkPhysicalDeviceLimits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceLimits.html). +/// See [`VkPhysicalDeviceSamplerFilterMinmaxProperties`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceSamplerFilterMinmaxProperties.html). #[derive(Clone, Copy, Debug)] pub struct SamplerFilterMinmaxProperties { /// When `false` the component mapping of the image view used with min/max filtering must have @@ -749,9 +979,9 @@ impl From> for SamplerFilter } } -/// Description of Vulkan features. +/// Description of Vulkan 1.0 features. /// -/// See [`VkPhysicalDeviceProperties`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceProperties.html). +/// See [`VkPhysicalDeviceFeatures`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceFeatures.html). #[derive(Clone, Copy, Debug)] pub struct Vulkan10Features { /// Specifies that accesses to buffers are bounds-checked against the range of the buffer @@ -762,11 +992,11 @@ pub struct Vulkan10Features { /// `VkIndexType` of `VK_INDEX_TYPE_UINT32`. /// /// `maxDrawIndexedIndexValue` is the maximum index value that may be used (aside from the - /// primitive restart index, which is always 2^32 - 1 when the VkIndexType is + /// primitive restart index, which is always 2^32 - 1 when the `VkIndexType` is /// `VK_INDEX_TYPE_UINT32`). /// /// If this feature is supported, `maxDrawIndexedIndexValue` must be 2^32 - 1; otherwise it - /// must be no smaller than 2^24 - 1. See maxDrawIndexedIndexValue. + /// must be no smaller than 2^24 - 1. pub full_draw_index_uint32: bool, /// Specifies whether image views with a `VkImageViewType` of `VK_IMAGE_VIEW_TYPE_CUBE_ARRAY` @@ -896,8 +1126,7 @@ pub struct Vulkan10Features { pub large_points: bool, /// Specifies whether the implementation is able to replace the alpha value of the fragment - /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). - /// fragment operation. + /// shader output in the multisample fragment operation. /// /// If this feature is not enabled, then the `alphaToOneEnable` member of the /// `VkPipelineMultisampleStateCreateInfo` structure must be set to `VK_FALSE`. Otherwise @@ -1082,17 +1311,16 @@ pub struct Vulkan10Features { /// - VK_FORMAT_R16_UINT /// - VK_FORMAT_R8_UINT /// - /// _Note:_ `shaderStorageImageExtendedFormats` feature only adds a guarantee of format + /// _Note:_ The `shaderStorageImageExtendedFormats` feature only adds a guarantee of format /// support, which is specified for the whole physical device. Therefore enabling or - /// disabling the feature via vkCreateDevice has no practical effect. + /// disabling the feature via `vkCreateDevice` has no practical effect. /// /// To query for additional properties, or if the feature is not supported, /// `vkGetPhysicalDeviceFormatProperties` and `vkGetPhysicalDeviceImageFormatProperties` can be /// used to check for supported properties of individual formats, as usual rules allow. /// - /// `VK_FORMAT_R32G32_UINT`, `VK_FORMAT_R32G32_SINT`, and `VK_FORMAT_R32G32_SFLOAT` from - /// `StorageImageExtendedFormats` SPIR-V capability, are already covered by core Vulkan - /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). + /// `VK_FORMAT_R32G32_UINT`, `VK_FORMAT_R32G32_SINT`, and `VK_FORMAT_R32G32_SFLOAT` from the + /// `StorageImageExtendedFormats` SPIR-V capability are already covered by core Vulkan. pub shader_storage_image_extended_formats: bool, /// Specifies whether multisampled storage images are supported. @@ -1107,17 +1335,15 @@ pub struct Vulkan10Features { /// Specifies whether storage images and storage texel buffers require a format qualifier to be /// specified when reading. /// - /// `shaderStorageImageReadWithoutFormat` applies only to formats listed in the - /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). - /// list. + /// `shaderStorageImageReadWithoutFormat` applies only to formats listed in the features + /// chapter's storage image read without format list. pub shader_storage_image_read_without_format: bool, /// Specifies whether storage images and storage texel buffers require a format qualifier to be /// specified when writing. /// - /// `shaderStorageImageWriteWithoutFormat` applies only to formats listed in the - /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). - /// list. + /// `shaderStorageImageWriteWithoutFormat` applies only to formats listed in the features + /// chapter's storage image write without format list. pub shader_storage_image_write_without_format: bool, /// Specifies whether arrays of uniform buffers can be indexed by dynamically uniform integer @@ -1315,9 +1541,9 @@ impl From for Vulkan10Features { } } -/// Description of Vulkan limitations. +/// Description of Vulkan 1.0 limits. /// -/// See [`VkPhysicalDeviceVulkan11Features`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceVulkan11Features.html). +/// See [`VkPhysicalDeviceLimits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceLimits.html). #[derive(Clone, Copy, Debug)] pub struct Vulkan10Limits { /// The largest dimension (width) that is guaranteed to be supported for all images created @@ -1795,27 +2021,22 @@ impl From for Vulkan10Limits { /// Description of Vulkan 1.0 properties. /// -/// See [`VkPhysicalDeviceVulkan11Properties`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceVulkan11Properties.html). +/// See [`VkPhysicalDeviceProperties`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceProperties.html). #[derive(Clone, Debug)] pub struct Vulkan10Properties { - /// The version of Vulkan supported by the device, encoded as described - /// See the [limits chapter](https://docs.vulkan.org/spec/latest/chapters/limits.html). + /// The version of Vulkan supported by the device. pub api_version: u32, /// The vendor-specified version of the driver. pub driver_version: u32, - /// A unique identifier for the vendor (see - /// See the [limits chapter](https://docs.vulkan.org/spec/latest/chapters/limits.html). - /// of the physical device. + /// A unique identifier for the vendor of the physical device. pub vendor_id: u32, /// A unique identifier for the physical device among devices available from the vendor. pub device_id: u32, - /// a - /// See the [limits chapter](https://docs.vulkan.org/spec/latest/chapters/limits.html). - /// specifying the type of device. + /// The type of physical device. pub device_type: vk::PhysicalDeviceType, /// A UTF-8 string which is the name of the device. @@ -1826,8 +2047,8 @@ pub struct Vulkan10Properties { pub pipeline_cache_uuid: [u8; vk::UUID_SIZE], /// The [`Vulkan10Limits`] structure specifying device-specific limits of the physical device. - /// See - /// [Limits](https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#limits) + /// + /// See the [limits chapter](https://registry.khronos.org/vulkan/specs/latest/html/vkspec.html#limits) /// for details. pub limits: Vulkan10Limits, // Unsupported (sparse residency): @@ -1842,7 +2063,7 @@ impl From for Vulkan10Properties { vendor_id: properties.vendor_id, device_id: properties.device_id, device_type: properties.device_type, - device_name: vk_cstr_to_string_lossy(&properties.device_name), + device_name: vk_cstr_to_utf8_string(&properties.device_name), pipeline_cache_uuid: properties.pipeline_cache_uuid, limits: properties.limits.into(), } @@ -1851,7 +2072,7 @@ impl From for Vulkan10Properties { /// Description of Vulkan 1.1 features. /// -/// See [`VkPhysicalDeviceVulkan12Features`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceVulkan12Features.html). +/// See [`VkPhysicalDeviceVulkan11Features`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceVulkan11Features.html). #[derive(Clone, Copy, Debug)] pub struct Vulkan11Features { /// Specifies whether objects in the StorageBuffer, ShaderRecordBufferKHR, or @@ -1958,7 +2179,7 @@ impl From> for Vulkan11Features { /// Description of Vulkan 1.1 properties. /// -/// See [`VkPhysicalDeviceVulkan12Properties`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceVulkan12Properties.html). +/// See [`VkPhysicalDeviceVulkan11Properties`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceVulkan11Properties.html). #[derive(Clone, Copy, Debug)] pub struct Vulkan11Properties { /// An array of `VK_UUID_SIZE` `u8` values representing a universally unique identifier for @@ -2023,8 +2244,8 @@ pub struct Vulkan11Properties { /// /// If this limit is `false`, applications must not perform these operations. /// - /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). - /// for more information. + /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html) for + /// more information. pub protected_no_fault: bool, /// A maximum number of descriptors (summed over all descriptor types) in a single descriptor @@ -2075,8 +2296,8 @@ pub struct Vulkan12Features { /// address mode must not be used. pub sampler_mirror_clamp_to_edge: bool, - /// Indicates whether the implementation supports the vkCmdDrawIndirectCount and - /// vkCmdDrawIndexedIndirectCount functions. + /// Indicates whether the implementation supports the `vkCmdDrawIndirectCount` and + /// `vkCmdDrawIndexedIndirectCount` functions. /// /// If this feature is not enabled, these functions must not be used. pub draw_indirect_count: bool, @@ -2134,8 +2355,8 @@ pub struct Vulkan12Features { pub shader_int8: bool, /// Indicates whether the implementation supports the minimum set of descriptor indexing - /// features as described in the [Feature Requirements] section. Enabling the - /// descriptorIndexing member when vkCreateDevice is called does not imply the other + /// features as described in the Vulkan feature requirements. Enabling the + /// `descriptorIndexing` member when `vkCreateDevice` is called does not imply the other /// minimum descriptor indexing features are also enabled. Those other descriptor indexing /// features must be enabled individually as needed by the application. /// @@ -2146,7 +2367,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT must be indexed only by constant integral expressions + /// `VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT` must be indexed only by constant integral expressions /// when aggregated into arrays in shader code. This also indicates whether shader modules can /// declare the InputAttachmentArrayDynamicIndexing capability. pub shader_input_attachment_array_dynamic_indexing: bool, @@ -2155,7 +2376,7 @@ pub struct Vulkan12Features { /// integer expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER must be indexed only by constant integral + /// `VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER` must be indexed only by constant integral /// expressions when aggregated into arrays in shader code. This also indicates whether shader /// modules can declare the UniformTexelBufferArrayDynamicIndexing capability. pub shader_uniform_texel_buffer_array_dynamic_indexing: bool, @@ -2164,7 +2385,7 @@ pub struct Vulkan12Features { /// integer expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER must be indexed only by constant integral + /// `VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER` must be indexed only by constant integral /// expressions when aggregated into arrays in shader code. This also indicates whether shader /// modules can declare the StorageTexelBufferArrayDynamicIndexing capability. pub shader_storage_texel_buffer_array_dynamic_indexing: bool, @@ -2173,7 +2394,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER or VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC must not be + /// `VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER` or `VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC` must not be /// indexed by non-uniform integer expressions when aggregated into arrays in shader code. This /// also indicates whether shader modules can declare the UniformBufferArrayNonUniformIndexing /// capability. @@ -2183,8 +2404,8 @@ pub struct Vulkan12Features { /// integer expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_SAMPLER, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, or - /// VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE must not be indexed by non-uniform integer expressions + /// `VK_DESCRIPTOR_TYPE_SAMPLER`, `VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER`, or + /// `VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE` must not be indexed by non-uniform integer expressions /// when aggregated into arrays in shader code. This also indicates whether shader modules /// can declare the SampledImageArrayNonUniformIndexing capability. pub shader_sampled_image_array_non_uniform_indexing: bool, @@ -2193,7 +2414,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_STORAGE_BUFFER or VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC must not be + /// `VK_DESCRIPTOR_TYPE_STORAGE_BUFFER` or `VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC` must not be /// indexed by non-uniform integer expressions when aggregated into arrays in shader code. This /// also indicates whether shader modules can declare the StorageBufferArrayNonUniformIndexing /// capability. @@ -2203,7 +2424,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_STORAGE_IMAGE must not be indexed by non-uniform integer expressions + /// `VK_DESCRIPTOR_TYPE_STORAGE_IMAGE` must not be indexed by non-uniform integer expressions /// when aggregated into arrays in shader code. This also indicates whether shader modules /// can declare the StorageImageArrayNonUniformIndexing capability. pub shader_storage_image_array_non_uniform_indexing: bool, @@ -2212,7 +2433,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT must not be indexed by non-uniform integer expressions + /// `VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT` must not be indexed by non-uniform integer expressions /// when aggregated into arrays in shader code. This also indicates whether shader modules can /// declare the InputAttachmentArrayNonUniformIndexing capability. pub shader_input_attachment_array_non_uniform_indexing: bool, @@ -2221,7 +2442,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER must not be indexed by non-uniform integer + /// `VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER` must not be indexed by non-uniform integer /// expressions when aggregated into arrays in shader code. This also indicates whether shader /// modules can declare the UniformTexelBufferArrayNonUniformIndexing capability. pub shader_uniform_texel_buffer_array_non_uniform_indexing: bool, @@ -2230,7 +2451,7 @@ pub struct Vulkan12Features { /// expressions in shader code. /// /// If this feature is not enabled, resources with a descriptor type of - /// VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER must not be indexed by non-uniform integer + /// `VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER` must not be indexed by non-uniform integer /// expressions when aggregated into arrays in shader code. This also indicates whether shader /// modules can declare the StorageTexelBufferArrayNonUniformIndexing capability. pub shader_storage_texel_buffer_array_non_uniform_indexing: bool, @@ -2303,8 +2524,8 @@ pub struct Vulkan12Features { /// min/max filtering as defined by the filterMinmaxSingleComponentFormats property minimum /// requirements. /// - /// If this feature is not enabled, then VkSamplerReductionModeCreateInfo must only use - /// VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE. + /// If this feature is not enabled, then `VkSamplerReductionModeCreateInfo` must only use + /// `VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE`. pub sampler_filter_minmax: bool, /// Indicates that the implementation supports the layout of resource blocks in shaders using @@ -2312,15 +2533,13 @@ pub struct Vulkan12Features { pub scalar_block_layout: bool, /// Indicates that the implementation supports specifying the image view for attachments at - /// render pass begin time via VkRenderPassAttachmentBeginInfo. + /// render pass begin time via `VkRenderPassAttachmentBeginInfo`. pub imageless_framebuffer: bool, /// Indicates that the implementation supports the same layouts for uniform buffers as for /// storage and other kinds of buffers. /// - /// See [Standard Buffer Layout]. - /// - /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). + /// See [standard buffer layout](https://docs.vulkan.org/spec/latest/chapters/interfaces.html#interfaces-resources-standard-layout). pub uniform_buffer_standard_layout: bool, /// A boolean specifying whether subgroup operations can use 8-bit integer, 16-bit integer, @@ -2328,23 +2547,23 @@ pub struct Vulkan12Features { /// subgroup scope, if the implementation supports the types. pub shader_subgroup_extended_types: bool, - /// Indicates whether the implementation supports a VkImageMemoryBarrier for a depth/stencil - /// image with only one of VK_IMAGE_ASPECT_DEPTH_BIT or VK_IMAGE_ASPECT_STENCIL_BIT set, and - /// whether VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_OPTIMAL, - /// VK_IMAGE_LAYOUT_STENCIL_ATTACHMENT_OPTIMAL, or VK_IMAGE_LAYOUT_STENCIL_READ_ONLY_OPTIMAL - /// can be used. + /// Indicates whether the implementation supports a `VkImageMemoryBarrier` for a depth/stencil + /// image with only one of `VK_IMAGE_ASPECT_DEPTH_BIT` or `VK_IMAGE_ASPECT_STENCIL_BIT` set, and + /// whether `VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_OPTIMAL`, + /// `VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_OPTIMAL`, `VK_IMAGE_LAYOUT_STENCIL_ATTACHMENT_OPTIMAL`, or + /// `VK_IMAGE_LAYOUT_STENCIL_READ_ONLY_OPTIMAL` can be used. pub separate_depth_stencil_layouts: bool, /// Indicates that the implementation supports resetting queries from the host with - /// vkResetQueryPool. + /// `vkResetQueryPool`. pub host_query_reset: bool, - /// Indicates whether semaphores created with a VkSemaphoreType of VK_SEMAPHORE_TYPE_TIMELINE - /// are supported. + /// Indicates whether semaphores created with a `VkSemaphoreType` of + /// `VK_SEMAPHORE_TYPE_TIMELINE` are supported. pub timeline_semaphore: bool, /// Indicates that the implementation supports accessing buffer memory in shaders as storage - /// buffers via an address queried from vkGetBufferDeviceAddress. + /// buffers via an address queried from `vkGetBufferDeviceAddress`. pub buffer_device_address: bool, /// Indicates that the implementation supports saving and reusing buffer and device addresses, @@ -2360,23 +2579,29 @@ pub struct Vulkan12Features { /// Indicates whether the [Vulkan Memory Model] is supported. /// - /// This also indicates whether shader modules can declare the VulkanMemoryModel capability. + /// This also indicates whether shader modules can declare the `VulkanMemoryModel` capability. /// /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). + /// + /// [Vulkan Memory Model]: https://docs.vulkan.org/spec/latest/chapters/memory.html#memory-model pub vulkan_memory_model: bool, /// Indicates whether the [Vulkan Memory Model] can use Device scope synchronization. /// - /// This also indicates whether shader modules can declare the VulkanMemoryModelDeviceScope + /// This also indicates whether shader modules can declare the `VulkanMemoryModelDeviceScope` /// capability. /// /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). + /// + /// [Vulkan Memory Model]: https://docs.vulkan.org/spec/latest/chapters/memory.html#memory-model pub vulkan_memory_model_device_scope: bool, /// Indicates whether the [Vulkan Memory Model] can use availability and visibility chains with /// more than one element. /// /// See the [features chapter](https://docs.vulkan.org/spec/latest/chapters/features.html). + /// + /// [Vulkan Memory Model]: https://docs.vulkan.org/spec/latest/chapters/memory.html#memory-model pub vulkan_memory_model_availability_visibility_chains: bool, /// Indicates whether the implementation supports the ShaderViewportIndex SPIR-V capability @@ -2515,8 +2740,7 @@ pub struct Vulkan12Properties { /// additional information about the driver. pub driver_info: String, - /// The version of the Vulkan conformance test this driver is conformant against (see - /// See the [limits chapter](https://docs.vulkan.org/spec/latest/chapters/limits.html). + /// The version of the Vulkan conformance test suite this driver is conformant against. pub conformance_version: vk::ConformanceVersion, /// A `vk::ShaderFloatControlsIndependence` value indicating whether, and how, denorm behavior @@ -2527,21 +2751,21 @@ pub struct Vulkan12Properties { /// can be set independently for different bit widths. pub rounding_mode_independence: vk::ShaderFloatControlsIndependence, - /// A `bool` value indicating whether sign of a zero, Nans and ±∞ can be preserved in 16-bit + /// A `bool` value indicating whether sign of a zero, NaNs and ±∞ can be preserved in 16-bit /// floating-point computations. /// /// It also indicates whether the SignedZeroInfNanPreserve execution mode can be used for /// 16-bit floating-point types. pub shader_signed_zero_inf_nan_preserve_float16: bool, - /// A `bool` value indicating whether sign of a zero, Nans and ±∞ can be preserved in 32-bit + /// A `bool` value indicating whether sign of a zero, NaNs and ±∞ can be preserved in 32-bit /// floating-point computations. /// /// It also indicates whether the SignedZeroInfNanPreserve execution mode can be used for /// 32-bit floating-point types. pub shader_signed_zero_inf_nan_preserve_float32: bool, - /// A `bool` value indicating whether sign of a zero, Nans and ±∞ can be preserved in 64-bit + /// A `bool` value indicating whether sign of a zero, NaNs and ±∞ can be preserved in 64-bit /// floating-point computations. /// /// It also indicates whether the SignedZeroInfNanPreserve execution mode can be used for @@ -2828,8 +3052,8 @@ impl From> for Vulkan12Properties { fn from(properties: vk::PhysicalDeviceVulkan12Properties<'_>) -> Self { Self { driver_id: properties.driver_id, - driver_name: vk_cstr_to_string_lossy(&properties.driver_name), - driver_info: vk_cstr_to_string_lossy(&properties.driver_info), + driver_name: vk_cstr_to_utf8_string(&properties.driver_name), + driver_info: vk_cstr_to_utf8_string(&properties.driver_info), conformance_version: properties.conformance_version, denorm_behavior_independence: properties.denorm_behavior_independence, rounding_mode_independence: properties.rounding_mode_independence, @@ -2930,3 +3154,61 @@ impl From> for Vulkan12Properties { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn c_chars(bytes: &[u8]) -> Vec { + bytes.iter().map(|&byte| byte as c_char).collect() + } + + #[test] + fn vk_cstr_to_utf8_string_reads_until_nul() { + let value = c_chars(b"hello\0ignored"); + + assert_eq!(vk_cstr_to_utf8_string(&value), "hello"); + } + + #[test] + fn vk_cstr_to_utf8_string_accepts_non_ascii_utf8() { + let value = c_chars("Møøse\0".as_bytes()); + + assert_eq!(vk_cstr_to_utf8_string(&value), "Møøse"); + } + + #[test] + fn vk_cstr_to_utf8_string_accepts_128_bytes_before_nul() { + let mut value = vec![b'a'; MAX_C_STRING_UTF8_BYTES]; + value.push(0); + let value = c_chars(&value); + + assert_eq!( + vk_cstr_to_utf8_string(&value), + "a".repeat(MAX_C_STRING_UTF8_BYTES) + ); + } + + #[test] + fn vk_cstr_to_utf8_string_rejects_missing_nul() { + let value = c_chars(b"unterminated"); + + assert_eq!(vk_cstr_to_utf8_string(&value), UNKNOWN_C_STRING); + } + + #[test] + fn vk_cstr_to_utf8_string_rejects_nul_after_max_len() { + let mut value = vec![b'a'; MAX_C_STRING_UTF8_BYTES + 1]; + value.push(0); + let value = c_chars(&value); + + assert_eq!(vk_cstr_to_utf8_string(&value), UNKNOWN_C_STRING); + } + + #[test] + fn vk_cstr_to_utf8_string_rejects_invalid_utf8() { + let value = c_chars(&[b'o', 0xff, 0]); + + assert_eq!(vk_cstr_to_utf8_string(&value), UNKNOWN_C_STRING); + } +} diff --git a/src/driver/ray_trace.rs b/src/driver/ray_tracing.rs similarity index 80% rename from src/driver/ray_trace.rs rename to src/driver/ray_tracing.rs index 670c20c0..fb990710 100644 --- a/src/driver/ray_trace.rs +++ b/src/driver/ray_tracing.rs @@ -5,24 +5,28 @@ use { DriverError, device::Device, merge_push_constant_ranges, - physical_device::RayTracingPipelineProperties, + physical_device::khr::RayTracingPipelineProperties, shader::{DescriptorBindingMap, PipelineDescriptorInfo, Shader}, }, - ash::vk, + crate::lazy_str, + ash::vk::{self, Handle}, derive_builder::Builder, log::warn, std::{ ffi::CString, + fmt::{Debug, Formatter}, hash::{Hash, Hasher}, - sync::{Arc, OnceLock}, + sync::Arc, thread::panicking, }, }; -/// Smart pointer handle of a pipeline object. +/// Smart pointer handle of a ray tracing pipeline object. /// /// Also contains information about the object. -#[derive(Clone, Debug)] +/// +/// See [`VkPipeline`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipeline.html). +#[derive(Clone)] #[read_only::cast] pub struct RayTracingPipeline { pub(crate) inner: Arc, @@ -31,15 +35,17 @@ pub struct RayTracingPipeline { impl RayTracingPipeline { /// Creates a new ray tracing pipeline on the given device. /// - /// The correct pipeline stages will be enabled based on the provided shaders. See [Shader] for - /// details on all available stages. + /// The correct pipeline stages will be enabled based on the provided shaders. See [`Shader`] + /// for details on all available stages. + /// + /// See [`VkRayTracingPipelineCreateInfoKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkRayTracingPipelineCreateInfoKHR.html). /// /// The number and composition of the `shader_groups` parameter must match the actual shaders /// provided. /// - /// # Panics - /// - /// If shader code is not a multiple of four bytes. + /// `shaders` may contain pre-built [`Shader`] values or any inputs that can be converted into + /// them. Invalid shader data is returned as [`DriverError::InvalidData`] through the `Result` + /// instead of panicking. /// /// # Examples /// @@ -50,7 +56,7 @@ impl RayTracingPipeline { /// # use ash::vk; /// # use vk_graph::driver::DriverError; /// # use vk_graph::driver::device::{Device, DeviceInfo}; - /// # use vk_graph::driver::ray_trace::{ + /// # use vk_graph::driver::ray_tracing::{ /// # RayTracingPipeline, /// # RayTracingPipelineInfo, /// # RayTracingShaderGroup, @@ -96,11 +102,7 @@ impl RayTracingPipeline { S: TryInto, S::Error: Into, { - if device - .physical_device - .ray_tracing_pipeline_properties - .is_none() - { + if device.physical.vk_khr_ray_tracing_pipeline.is_none() { warn!("unsupported ray tracing pipeline creation: missing ray tracing properties"); return Err(DriverError::Unsupported); @@ -218,7 +220,7 @@ impl RayTracingPipeline { .max_pipeline_ray_recursion_depth( info.max_ray_recursion_depth.min( device - .physical_device + .physical .expect_ray_tracing_pipeline_properties() .max_ray_recursion_depth, ), @@ -235,36 +237,43 @@ impl RayTracingPipeline { device.destroy_shader_module(shader_module, None); } - let handle = handle.map_err(|(pipelines, err)| { - warn!("unable to create ray tracing pipeline: {err}"); + let handle = handle + .map_err(|(pipelines, err)| { + warn!("unable to create ray tracing pipeline: {err}"); - for pipeline in pipelines { - device.destroy_pipeline(pipeline, None); - } + for pipeline in pipelines { + device.destroy_pipeline(pipeline, None); + } + + device.destroy_pipeline_layout(layout, None); - device.destroy_pipeline_layout(layout, None); + DriverError::Unsupported + })? + .into_iter() + .find(|handle| !handle.is_null()) + .ok_or_else(|| { + warn!("missing pipeline handle"); - DriverError::Unsupported - })?[0]; + DriverError::Unsupported + })?; let &RayTracingPipelineProperties { shader_group_handle_size, .. - } = device - .physical_device - .expect_ray_tracing_pipeline_properties(); + } = device.physical.expect_ray_tracing_pipeline_properties(); let push_constants = merge_push_constant_ranges(&push_constants).into_boxed_slice(); - // SAFETY: - // See [`vkGetRayTracingShaderGroupHandlesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetRayTracingShaderGroupHandlesKHR.html). - // Valid usage of this function requires: - // 1. pipeline must be raytracing pipeline. - // 2. first_group must be less than the number of shader groups in the pipeline. - // 3. the sum of first group and group_count must be less or equal to the number of - // shader modules in the pipeline. - // 4. data_size must be at least shader_group_handle_size * group_count. - // 5. pipeline must not have been created with VK_PIPELINE_CREATE_LIBRARY_BIT_KHR. - // + /* + SAFETY: + See [`vkGetRayTracingShaderGroupHandlesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetRayTracingShaderGroupHandlesKHR.html) + Valid usage of this function requires: + 1. pipeline must be a ray tracing pipeline + 2. first_group must be less than the number of shader groups in the pipeline + 3. the sum of first_group and group_count must be less than or equal to the number of + shader groups in the pipeline + 4. data_size must be at least shader_group_handle_size * group_count + 5. pipeline must not have been created with VK_PIPELINE_CREATE_LIBRARY_BIT_KHR + */ let shader_group_handles = { khr_ray_tracing_pipeline.get_ray_tracing_shader_group_handles( handle, @@ -284,7 +293,6 @@ impl RayTracingPipeline { handle, info, layout, - name: Default::default(), push_constants, shader_group_handles, }), @@ -292,24 +300,22 @@ impl RayTracingPipeline { } } - /// Gets the debugging name assigned to this pipeline, if one has been set. - pub fn debug_name(&self) -> Option<&str> { - self.inner.name.get().map(String::as_str) - } - /// The device which owns this ray tracing pipeline. pub fn device(&self) -> &Device { &self.inner.device } - /// Function returning a handle to a shader group of this pipeline. - /// This can be used to construct a sbt. + /// Returns a handle to a shader group of this pipeline. + /// + /// This can be used to construct a shader binding table. /// /// # Examples /// /// See - /// [ray_trace.rs](https://github.com/attackgoat/vk-graph/blob/master/examples/ray_trace.rs) - /// for a detail example which constructs a shader binding table buffer using this function. + /// [`ray_tracing.rs`](https://github.com/attackgoat/vk-graph/blob/master/examples/ray_tracing.rs) + /// for a detailed example that constructs a shader binding table buffer using this function. + /// + /// See [`vkGetRayTracingShaderGroupHandlesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetRayTracingShaderGroupHandlesKHR.html). pub fn group_handle(&self, idx: usize) -> &[u8] { let &RayTracingPipelineProperties { shader_group_handle_size, @@ -317,7 +323,7 @@ impl RayTracingPipeline { } = self .inner .device - .physical_device + .physical .expect_ray_tracing_pipeline_properties(); let start = idx * shader_group_handle_size as usize; let end = start + shader_group_handle_size as usize; @@ -325,18 +331,21 @@ impl RayTracingPipeline { &self.inner.shader_group_handles[start..end] } - /// Query ray tracing pipeline shader group shader stack size. + /// Queries ray tracing pipeline shader group shader stack size. /// /// The return value is the ray tracing pipeline stack size in bytes for the specified shader as /// called from the specified shader group. + /// + /// See [`vkGetRayTracingShaderGroupStackSizeKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetRayTracingShaderGroupStackSizeKHR.html). #[profiling::function] pub fn group_stack_size( &self, group: u32, group_shader: vk::ShaderGroupShaderKHR, ) -> vk::DeviceSize { - // Safely use unchecked because the ray tracing extension is checked during pipeline - // creation. + /* + Safely use unchecked because the ray tracing extension is checked during pipeline creation + */ let khr_ray_tracing_pipeline = Device::expect_vk_khr_ray_tracing_pipeline(&self.inner.device); @@ -359,35 +368,51 @@ impl RayTracingPipeline { self.inner.info } - /// Gets the debugging name assigned to this pipeline, if one has been set. - pub fn name(&self) -> Option<&str> { - self.inner.name.get().map(String::as_str) - } - /// Sets the debugging name assigned to this pipeline. - /// - /// _Note:_ The pipeline name may only be assigned once. Subsequent calls will not update the - /// previously set name value. - pub fn set_debug_name(&mut self, name: impl Into) { - if !self.inner.device.physical_device.instance.info.debug { - return; + pub fn set_debug_name(&self, name: impl AsRef) { + Device::try_set_debug_utils_object_name(&self.inner.device, self.inner.handle, &name); + Device::try_set_private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE, + self.inner.handle, + &name, + ); + Device::try_set_debug_utils_object_name( + &self.inner.device, + self.inner.layout, + lazy_str!("{} (layout)", name.as_ref()), + ); + + for (set_idx, layout) in &self.inner.descriptor_info.layouts { + layout.set_debug_name(lazy_str!("{} (DS{set_idx})", name.as_ref())); } - - // Both Ok and Err are valid conditions - let _ = self.inner.name.set(name.into()); } /// Sets the debugging name assigned to this pipeline. - /// - /// _Note:_ The pipeline name may only be assigned once. Subsequent calls will not update the - /// previously set name value. - pub fn with_debug_name(mut self, name: impl Into) -> Self { + pub fn with_debug_name(self, name: impl AsRef) -> Self { self.set_debug_name(name); self } } +impl Debug for RayTracingPipeline { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut res = f.debug_struct(stringify!(RayTracingPipeline)); + + if let Some(debug_name) = &Device::private_data_object_name( + &self.inner.device, + vk::ObjectType::PIPELINE, + self.inner.handle, + ) { + res.field("debug_name", debug_name); + } + + res.field("handle", &self.inner.handle) + .finish_non_exhaustive() + } +} + impl Eq for RayTracingPipeline {} impl Hash for RayTracingPipeline { @@ -437,7 +462,7 @@ pub struct RayTracingPipelineInfo { /// Allow [setting the stack size dynamically] for a ray tracing pipeline. /// - /// When set, you must manually set the stack size during ray tracing passes using + /// When set, you must manually set the stack size during ray tracing commands using /// [`RayTracingCommandRef::set_stack_size`](crate::cmd::RayTracingCommandRef::set_stack_size). /// /// See [`vkCmdSetRayTracingPipelineStackSizeKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdSetRayTracingPipelineStackSizeKHR.html). @@ -502,7 +527,6 @@ pub(crate) struct RayTracingPipelineInner { pub handle: vk::Pipeline, pub info: RayTracingPipelineInfo, pub layout: vk::PipelineLayout, - pub name: OnceLock, pub push_constants: Box<[vk::PushConstantRange]>, pub shader_group_handles: Box<[u8]>, } @@ -514,6 +538,9 @@ impl Drop for RayTracingPipelineInner { return; } + Device::clear_private_data_object_name(&self.device, vk::ObjectType::PIPELINE, self.handle) + .unwrap_or_else(|err| warn!("unable to clear private data object name: {err}")); + unsafe { self.device.destroy_pipeline(self.handle, None); self.device.destroy_pipeline_layout(self.layout, None); @@ -546,12 +573,12 @@ pub struct RayTracingShaderGroup { pub intersection_shader: Option, /// The type of hit group specified in this structure. - pub ty: RayTracingShaderGroupType, + pub shader_group_type: RayTracingShaderGroupType, } impl RayTracingShaderGroup { fn new( - ty: RayTracingShaderGroupType, + shader_group_type: RayTracingShaderGroupType, general_shader: impl Into>, intersection_shader: impl Into>, closest_hit_shader: impl Into>, @@ -567,7 +594,7 @@ impl RayTracingShaderGroup { closest_hit_shader, general_shader, intersection_shader, - ty, + shader_group_type, } } @@ -614,7 +641,7 @@ impl RayTracingShaderGroup { impl From for vk::RayTracingShaderGroupCreateInfoKHR<'static> { fn from(shader_group: RayTracingShaderGroup) -> Self { vk::RayTracingShaderGroupCreateInfoKHR::default() - .ty(shader_group.ty.into()) + .ty(shader_group.shader_group_type.into()) .any_hit_shader(shader_group.any_hit_shader.unwrap_or(vk::SHADER_UNUSED_KHR)) .closest_hit_shader( shader_group @@ -645,8 +672,8 @@ pub enum RayTracingShaderGroupType { } impl From for vk::RayTracingShaderGroupTypeKHR { - fn from(ty: RayTracingShaderGroupType) -> Self { - match ty { + fn from(shader_group_type: RayTracingShaderGroupType) -> Self { + match shader_group_type { RayTracingShaderGroupType::General => vk::RayTracingShaderGroupTypeKHR::GENERAL, RayTracingShaderGroupType::ProceduralHitGroup => { vk::RayTracingShaderGroupTypeKHR::PROCEDURAL_HIT_GROUP diff --git a/src/driver/render_pass.rs b/src/driver/render_pass.rs index 9aee79cc..4212b793 100644 --- a/src/driver/render_pass.rs +++ b/src/driver/render_pass.rs @@ -4,10 +4,10 @@ use { super::{ DriverError, device::Device, - graphic::{DepthStencilInfo, GraphicsPipeline}, + graphics::{DepthStencilInfo, GraphicsPipeline}, image::SampleCount, }, - ash::vk, + ash::vk::{self, Handle}, log::{trace, warn}, std::{ collections::{HashMap, hash_map::Entry}, @@ -19,7 +19,7 @@ use { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) struct AttachmentInfo { pub flags: vk::AttachmentDescriptionFlags, - pub fmt: vk::Format, + pub format: vk::Format, pub sample_count: SampleCount, pub load_op: vk::AttachmentLoadOp, pub store_op: vk::AttachmentStoreOp, @@ -33,7 +33,7 @@ impl From for vk::AttachmentDescription2<'_> { fn from(value: AttachmentInfo) -> Self { vk::AttachmentDescription2::default() .flags(value.flags) - .format(value.fmt) + .format(value.format) .samples(value.sample_count.into()) .load_op(value.load_op) .store_op(value.store_op) @@ -48,7 +48,7 @@ impl Default for AttachmentInfo { fn default() -> Self { AttachmentInfo { flags: vk::AttachmentDescriptionFlags::MAY_ALIAS, - fmt: vk::Format::UNDEFINED, + format: vk::Format::UNDEFINED, sample_count: SampleCount::Type1, initial_layout: vk::ImageLayout::UNDEFINED, load_op: vk::AttachmentLoadOp::DONT_CARE, @@ -99,9 +99,11 @@ struct GraphicPipelineKey { } /// Vulkan render pass state and cached framebuffer/pipeline objects for compatible attachments. +/// +/// See [`VkRenderPass`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkRenderPass.html). #[derive(Debug)] #[read_only::cast] -pub struct RenderPass { +pub(crate) struct RenderPass { /// The device which owns this render pass resource. /// /// _Note:_ This field is read-only. @@ -121,7 +123,7 @@ pub struct RenderPass { /// /// _Note:_ This field is read-only. #[readonly] - pub info: RenderPassInfo, + pub(crate) info: RenderPassInfo, } impl RenderPass { @@ -334,7 +336,24 @@ impl RenderPass { subpass_idx, }); if let Entry::Occupied(entry) = entry { - return Ok(*entry.get()); + let pipeline_handle = *entry.get(); + + if let Some(name) = Device::private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE_LAYOUT, + pipeline.inner.layout, + ) && match Device::private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE, + pipeline_handle, + ) { + None => true, + Some(previous) => previous != name, + } { + pipeline.set_variant_debug_name(pipeline_handle, self.handle, subpass_idx, &name); + } + + return Ok(pipeline_handle); } let entry = match entry { @@ -417,7 +436,7 @@ impl RenderPass { .vertex_input_state(&vertex_input_state) .viewport_state(&viewport_state); - let pipeline = unsafe { + let pipeline_handle = unsafe { self.device.create_graphics_pipelines( Device::pipeline_cache(&self.device), slice::from_ref(&create_info), @@ -428,11 +447,33 @@ impl RenderPass { warn!("create_graphics_pipelines: {err}\n{:#?}", create_info); DriverError::Unsupported - })?[0]; + })? + .into_iter() + .find(|handle| !handle.is_null()) + .ok_or_else(|| { + warn!("missing pipeline handle"); + + DriverError::Unsupported + })?; + + if let Some(name) = Device::private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE_LAYOUT, + pipeline.inner.layout, + ) && match Device::private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE, + pipeline_handle, + ) { + None => true, + Some(previous) => previous != name, + } { + pipeline.set_variant_debug_name(pipeline_handle, self.handle, subpass_idx, &name); + } - entry.insert(pipeline); + entry.insert(pipeline_handle); - Ok(pipeline) + Ok(pipeline_handle) } } @@ -450,6 +491,13 @@ impl Drop for RenderPass { } for (_, pipeline) in self.graphic_pipelines.drain() { + Device::clear_private_data_object_name( + &self.device, + vk::ObjectType::PIPELINE, + pipeline, + ) + .unwrap_or_else(|err| warn!("unable to clear private data object name: {err}")); + unsafe { self.device.destroy_pipeline(pipeline, None); } @@ -462,14 +510,18 @@ impl Drop for RenderPass { } /// Attachment, subpass, and dependency information used to create a [`RenderPass`]. +/// +/// See [`VkRenderPassCreateInfo2`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkRenderPassCreateInfo2.html). #[derive(Clone, Debug, Default, Eq, Hash, PartialEq)] -pub struct RenderPassInfo { +pub(crate) struct RenderPassInfo { pub(crate) attachments: Vec, pub(crate) subpasses: Vec, pub(crate) dependencies: Vec, } -/// Specifying depth and stencil resolve modes. +/// Specifies depth and stencil resolve modes. +/// +/// See [`VkResolveModeFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkResolveModeFlagBits.html). #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum ResolveMode { /// The result of the resolve operation is the average of the sample values. diff --git a/src/driver/shader.rs b/src/driver/shader.rs index f3b8dc3e..daf0c0e7 100644 --- a/src/driver/shader.rs +++ b/src/driver/shader.rs @@ -1,4 +1,97 @@ -//! Shader resource types +//! Shader reflection and pipeline-stage descriptions. +//! +//! This module contains the data used by compute, graphics, and ray tracing pipeline creation to +//! describe shader stages. +//! +//! # Construction +//! +//! The stage-specific constructors are the usual entry points: +//! +//! - [`Shader::new_compute`] for compute pipelines. +//! - [`Shader::new_vertex`] and [`Shader::new_fragment`] for graphics pipelines. +//! - [`Shader::new_ray_gen`], [`Shader::new_closest_hit`], [`Shader::new_miss`], and the other +//! ray tracing constructors for ray tracing pipelines. +//! +//! These constructors return a [`ShaderBuilder`]. Use [`ShaderBuilder::build`] when invalid SPIR-V +//! should panic during setup, or [`ShaderBuilder::try_build`] when invalid SPIR-V should become a +//! [`DriverError`]. [`Shader::try_new_compute`] and the other `try_new_*` constructors are shortcuts +//! for the fallible path. +//! +//! [`Shader::from_spirv`] is stage-neutral and is useful when the stage is selected separately. It +//! still needs a stage before the shader can be built. SPIR-V can be supplied as bytes, words, or a +//! value accepted by `spirq::parse::SpirvBinary`. +//! +//! # Entry Points +//! +//! The default entry point is `main`. Override it with [`ShaderBuilder::entry_name`] when a SPIR-V +//! module contains multiple entry points or when the exported function uses another name. +//! Reflection is performed for the selected entry point only, so descriptors, push constants, +//! attachments, and vertex inputs from other entry points are ignored. +//! +//! # Reflection +//! +//! Shader creation reflects the selected entry point immediately. Invalid SPIR-V, a missing entry +//! point, or unsupported reflection data makes the fallible builders return [`DriverError::InvalidData`] +//! or [`DriverError::Unsupported`], depending on where the problem is discovered. +//! +//! Reflection is used internally to collect: +//! +//! - descriptor set and binding numbers, descriptor types, array counts, and stage flags; +//! - push constant ranges for the stage; +//! - input attachments and color output locations for graphics render-pass setup; +//! - vertex input bindings, attributes, formats, offsets, strides, and input rates for vertex +//! shaders unless a manual vertex layout is supplied. +//! +//! When several shaders are combined into one pipeline, descriptor bindings with the same `(set, +//! binding)` must describe compatible descriptor types. Compatible bindings are merged and their +//! stage flags are ORed together. Conflicting descriptor declarations make pipeline creation fail. +//! +//! # Descriptor Bindings And Samplers +//! +//! [`Descriptor`] identifies a shader descriptor binding by set and binding number. It is a binding +//! location, not a live descriptor-set allocation. +//! +//! Combined image samplers and sampler descriptors carry immutable sampler information into the +//! generated descriptor set layout. You can provide explicit sampler state with +//! [`ShaderBuilder::image_sampler`]. If no sampler is specified, the shader binding name is inspected +//! for a compact sampler suffix of the form `_sampler_xyz`, where: +//! +//! - `x` selects texel filtering: `n` for nearest or `l` for linear; +//! - `y` selects mip filtering: `n` for nearest or `l` for linear; +//! - `z` selects address mode: `b` clamp-to-border, `e` clamp-to-edge, `m` mirrored-repeat, or `r` +//! repeat. +//! +//! For example, a binding whose reflected name ends with `_sampler_lle` uses linear texel filtering, +//! linear mip filtering, and clamp-to-edge addressing. Bindings without this suffix use a linear +//! repeat sampler by default. [`SamplerInfo`] and [`SamplerInfoBuilder`] expose the full Vulkan +//! sampler state when the inferred defaults are not appropriate. +//! +//! # Specialization Constants +//! +//! Specialization constants are supplied with [`ShaderBuilder::specialization`] using a +//! [`SpecializationMap`]. Reflection applies those values before descriptor information is collected, +//! so specialization constants can affect reflected array lengths and other specialization-dependent +//! layout data. The map stores raw bytes plus Vulkan specialization entries; use native endian byte +//! conversion such as `to_ne_bytes()` for scalar values. +//! +//! # Vertex Input Reflection +//! +//! Vertex shaders can provide graphics vertex input state automatically. Scalar and vector input +//! variables are mapped to Vulkan formats based on their reflected scalar type and component count. +//! Offsets and strides are inferred from the reflected byte sizes, grouped by binding. +//! +//! Binding numbers and input rates can be encoded in reflected input names. Names containing +//! `_vbindN` are assigned to vertex binding `N`; names containing `_ibindN` are assigned to instance +//! binding `N`. Inputs without either marker use binding `0` with vertex input rate. If reflection is +//! not expressive enough for a shader's vertex layout, provide a manual layout with +//! [`ShaderBuilder::vertex_input`]. +//! +//! # Pipeline Use +//! +//! Shaders are plain descriptions until a pipeline is created. Pipeline creation turns the SPIR-V +//! into Vulkan shader modules, uses the reflected layout to create compatible descriptor set layouts, +//! and then destroys the temporary shader modules after the pipeline has been created. Keep the +//! [`Shader`] values around only as long as they are useful to construct or rebuild pipelines. use { super::{DescriptorSetLayout, DriverError, VertexInputState, device::Device}, @@ -86,10 +179,10 @@ fn guess_immutable_sampler(binding_name: &str) -> SamplerInfo { /// bound descriptor reference. #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct Descriptor { - /// Descriptor set index + /// Descriptor set index. pub set: u32, - /// Descriptor binding index + /// Descriptor binding index. pub binding: u32, } @@ -283,12 +376,14 @@ impl PipelineDescriptorInfo { let mut create_info = vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings); - // The bindless flags have to be created for every descriptor set layout binding. - // See [`VkDescriptorSetLayoutBindingFlagsCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDescriptorSetLayoutBindingFlagsCreateInfo.html). - // Maybe using one vector and updating it would be more efficient. + /* + The bindless flags have to be created for every descriptor set layout binding. + See [`VkDescriptorSetLayoutBindingFlagsCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkDescriptorSetLayoutBindingFlagsCreateInfo.html) + Maybe using one vector and updating it would be more efficient. + */ let bindless_flags = vec![vk::DescriptorBindingFlags::PARTIALLY_BOUND; bindings.len()]; let mut bindless_flags = if device - .physical_device + .physical .features_v1_2 .descriptor_binding_partially_bound { @@ -381,7 +476,9 @@ impl Sampler { impl Debug for Sampler { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.sampler) + f.debug_struct(stringify!(Sampler)) + .field("handle", &self.sampler) + .finish_non_exhaustive() } } @@ -407,6 +504,8 @@ impl Drop for Sampler { } /// Information used to create a [`vk::Sampler`] instance. +/// +/// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). #[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[builder( build_fn(private, name = "fallible_build", error = "SamplerInfoBuilderError"), @@ -418,58 +517,54 @@ pub struct SamplerInfo { #[builder(default)] pub flags: vk::SamplerCreateFlags, - /// Specify the magnification filter to apply to texture lookups. + /// Specifies the magnification filter to apply to texture lookups. /// - /// The default value is [`vk::Filter::NEAREST`] + /// The default value is [`vk::Filter::NEAREST`]. #[builder(default)] pub mag_filter: vk::Filter, - /// Specify the minification filter to apply to texture lookups. + /// Specifies the minification filter to apply to texture lookups. /// - /// The default value is [`vk::Filter::NEAREST`] + /// The default value is [`vk::Filter::NEAREST`]. #[builder(default)] pub min_filter: vk::Filter, /// A value specifying the mipmap filter to apply to lookups. /// - /// The default value is [`vk::SamplerMipmapMode::NEAREST`] + /// The default value is [`vk::SamplerMipmapMode::NEAREST`]. #[builder(default)] pub mipmap_mode: vk::SamplerMipmapMode, /// A value specifying the addressing mode for U coordinates outside `[0, 1)`. /// - /// The default value is [`vk::SamplerAddressMode::REPEAT`] + /// The default value is [`vk::SamplerAddressMode::REPEAT`]. #[builder(default)] pub address_mode_u: vk::SamplerAddressMode, /// A value specifying the addressing mode for V coordinates outside `[0, 1)`. /// - /// The default value is [`vk::SamplerAddressMode::REPEAT`] + /// The default value is [`vk::SamplerAddressMode::REPEAT`]. #[builder(default)] pub address_mode_v: vk::SamplerAddressMode, /// A value specifying the addressing mode for W coordinates outside `[0, 1)`. /// - /// The default value is [`vk::SamplerAddressMode::REPEAT`] + /// The default value is [`vk::SamplerAddressMode::REPEAT`]. #[builder(default)] pub address_mode_w: vk::SamplerAddressMode, - /// The bias to be added to mipmap LOD calculation and bias provided by image sampling - /// functions in SPIR-V, as described in the - /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). - /// section. + /// Bias added to the mipmap LOD calculation and to bias provided by SPIR-V image sampling + /// instructions. #[builder(default, setter(into))] pub mip_lod_bias: OrderedFloat, - /// Enables anisotropic filtering, as described in the - /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). - /// section + /// Enables anisotropic filtering. #[builder(default)] pub anisotropy_enable: bool, /// The anisotropy value clamp used by the sampler when `anisotropy_enable` is `true`. /// - /// If `anisotropy_enable` is `false`, max_anisotropy is ignored. + /// If [`anisotropy_enable`](Self::anisotropy_enable) is `false`, `max_anisotropy` is ignored. #[builder(default, setter(into))] pub max_anisotropy: OrderedFloat, @@ -477,28 +572,23 @@ pub struct SamplerInfo { #[builder(default)] pub compare_enable: bool, - /// Specifies the comparison operator to apply to fetched data before filtering as described in - /// the - /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). - /// section. + /// Specifies the comparison operator to apply to fetched data before filtering. #[builder(default)] pub compare_op: vk::CompareOp, - /// Used to clamp the - /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). + /// Minimum LOD value used to clamp the computed level of detail. #[builder(default, setter(into))] pub min_lod: OrderedFloat, - /// Used to clamp the - /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). + /// Maximum LOD value used to clamp the computed level of detail. /// - /// To avoid clamping the maximum value, set maxLod to the constant `vk::LOD_CLAMP_NONE`. + /// To avoid clamping the maximum value, set `max_lod` to [`vk::LOD_CLAMP_NONE`]. #[builder(default, setter(into))] pub max_lod: OrderedFloat, - /// Secifies the predefined border color to use. + /// Specifies the predefined border color to use. /// - /// The default value is [`vk::BorderColor::FLOAT_TRANSPARENT_BLACK`] + /// The default value is [`vk::BorderColor::FLOAT_TRANSPARENT_BLACK`]. #[builder(default)] pub border_color: vk::BorderColor, @@ -510,7 +600,6 @@ pub struct SamplerInfo { /// /// When set to `false` the range of image coordinates is zero to one. /// - /// See /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). #[builder(default)] pub unnormalized_coordinates: bool, @@ -520,10 +609,9 @@ pub struct SamplerInfo { /// Setting magnification filter ([`mag_filter`](Self::mag_filter)) to [`vk::Filter::NEAREST`] /// disables sampler reduction mode. /// - /// The default value is [`vk::SamplerReductionMode::WEIGHTED_AVERAGE`] + /// The default value is [`vk::SamplerReductionMode::WEIGHTED_AVERAGE`]. /// - /// See - /// See [`VkSamplerCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerCreateInfo.html). + /// See [`VkSamplerReductionModeCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSamplerReductionModeCreateInfo.html). #[builder(default)] pub reduction_mode: vk::SamplerReductionMode, } @@ -648,7 +736,10 @@ impl From for SamplerInfoBuilderError { } } -/// Describes a shader program which runs on some pipeline stage. +/// Describes a shader program which runs on a pipeline stage. +/// +/// See [`VkShaderModuleCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkShaderModuleCreateInfo.html) +/// and [`VkPipelineShaderStageCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkPipelineShaderStageCreateInfo.html). #[allow(missing_docs)] #[derive(Builder, Clone)] #[builder( @@ -665,6 +756,8 @@ pub struct Shader { /// Data about Vulkan specialization constants. /// + /// See [`VkSpecializationInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSpecializationInfo.html). + /// /// # Examples /// /// Basic usage (GLSL): @@ -711,10 +804,14 @@ pub struct Shader { /// Although SPIR-V code is specified as `u32` values, this field uses `u8` in order to make /// loading from file simpler. You should always have a SPIR-V code length which is a multiple /// of four bytes, or an error will be returned during pipeline creation. + /// + /// See [`VkShaderModuleCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkShaderModuleCreateInfo.html). #[builder(setter(into))] pub spirv: SpirvBinary, /// The shader stage this structure applies to. + /// + /// See [`VkShaderStageFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkShaderStageFlagBits.html). pub stage: vk::ShaderStageFlags, #[builder(private)] @@ -757,6 +854,8 @@ macro_rules! shader_ctors { impl Shader { /// Specifies a shader with the given `stage` and shader code. + /// + /// See [`VkShaderModuleCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkShaderModuleCreateInfo.html). #[allow(clippy::new_ret_no_self)] pub fn new(stage: vk::ShaderStageFlags, spirv: impl Into) -> ShaderBuilder { ShaderBuilder::default().spirv(spirv).stage(stage) @@ -885,6 +984,8 @@ impl Shader { } /// Specifies a shader with the given shader code. + /// + /// See [`VkShaderModuleCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkShaderModuleCreateInfo.html). pub fn from_spirv(spirv: impl Into) -> ShaderBuilder { ShaderBuilder::default().spirv(spirv) } @@ -1071,8 +1172,10 @@ impl Shader { spirv: impl Into, specialization: Option<&SpecializationMap>, ) -> Result { - // spq-core can panic on malformed SPIR-V instead of returning Err, for example: - // `range start index 5 out of range for slice of length 0` from spq-core/src/parse/bin.rs. + /* + spirq can panic on malformed SPIR-V instead of returning Err, for example: + `range start index 5 out of range for slice of length 0` from spirq/src/parse/bin.rs + */ catch_unwind(AssertUnwindSafe(|| { Self::reflect_entry_point_unchecked(entry_name, spirv, specialization) })) @@ -1112,7 +1215,7 @@ impl Shader { } let entry_points = config.reflect().map_err(|err| { - error!("invalid spirv reflection data: {err}"); + error!("invalid SPIR-V reflection data: {err}"); DriverError::InvalidData })?; @@ -1286,7 +1389,7 @@ impl Shader { location, binding, format, - offset: byte_stride, // Figured out below - this data is iter'd in an unknown order + offset: byte_stride, }); } @@ -1345,9 +1448,13 @@ impl Shader { impl Debug for Shader { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - // We don't want the default formatter bc vec u8 - // TODO: Better output message - f.write_str("Shader") + // Avoid dumping the embedded SPIR-V bytes in debug output. + f.debug_struct(stringify!(Shader)) + .field("entry_name", &self.entry_name) + .field("stage", &self.stage) + .field("specialization", &self.specialization) + .field("reflected_vars", &self.entry_point.vars.len()) + .finish_non_exhaustive() } } @@ -1420,7 +1527,7 @@ impl ShaderBuilder { /// index `2`, or if the descriptor set index is `0` simply specify `2` for the same case. /// /// _NOTE:_ When defining image samplers which are used in multiple stages of a single pipeline - /// you must only call this function on one of the shader stages, it does not matter which one. + /// you must only call this function on one of the shader stages; it does not matter which one. /// /// # Panics /// diff --git a/src/driver/surface.rs b/src/driver/surface.rs index 6c2d30a8..31151963 100644 --- a/src/driver/surface.rs +++ b/src/driver/surface.rs @@ -13,6 +13,8 @@ use { }; /// Smart pointer handle to a [`vk::SurfaceKHR`] object. +/// +/// See [`VkSurfaceKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSurfaceKHR.html). #[read_only::cast] pub struct Surface { /// The device which owns this surface resource. @@ -27,15 +29,15 @@ pub struct Surface { } impl Surface { - /// Query surface capabilities + /// Queries surface capabilities. + /// + /// See [`vkGetPhysicalDeviceSurfaceCapabilitiesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetPhysicalDeviceSurfaceCapabilitiesKHR.html). pub fn capabilities(&self) -> Result { let khr_surface = Device::expect_vk_khr_surface(&self.device); unsafe { - khr_surface.get_physical_device_surface_capabilities( - self.device.physical_device.handle, - self.handle, - ) + khr_surface + .get_physical_device_surface_capabilities(self.device.physical.handle, self.handle) } .map_err(|err| { warn!("unable to get surface capabilities: {err}"); @@ -44,9 +46,11 @@ impl Surface { }) } - /// Create a surface from a raw window display handle. + /// Creates a surface from a raw window display handle. /// /// `device` must have been created with platform specific surface extensions enabled. + /// + /// See [`VkSurfaceKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSurfaceKHR.html). #[profiling::function] pub fn create( device: &Device, @@ -68,8 +72,8 @@ impl Surface { let handle = unsafe { create_surface( - Instance::entry(&device.physical_device.instance), - &device.physical_device.instance, + Instance::entry(&device.physical.instance), + &device.physical.instance, display_handle.as_raw(), window_handle.as_raw(), None, @@ -85,15 +89,15 @@ impl Surface { } /// Lists the supported surface formats. + /// + /// See [`vkGetPhysicalDeviceSurfaceFormatsKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetPhysicalDeviceSurfaceFormatsKHR.html). #[profiling::function] pub fn formats(&self) -> Result, DriverError> { let khr_surface = Device::expect_vk_khr_surface(&self.device); unsafe { - khr_surface.get_physical_device_surface_formats( - self.device.physical_device.handle, - self.handle, - ) + khr_surface + .get_physical_device_surface_formats(self.device.physical.handle, self.handle) } .map_err(|err| { warn!("unable to get surface formats: {err}"); @@ -125,14 +129,39 @@ impl Surface { Self::linear(formats).unwrap_or_else(|| formats.first().copied().unwrap_or_default()) } + /// Selects a supported composite alpha mode, preferring `requested` and then `OPAQUE`. + pub fn composite_alpha_or_default( + supported: vk::CompositeAlphaFlagsKHR, + requested: vk::CompositeAlphaFlagsKHR, + ) -> vk::CompositeAlphaFlagsKHR { + if supported.contains(requested) { + return requested; + } + + if supported.contains(vk::CompositeAlphaFlagsKHR::OPAQUE) { + return vk::CompositeAlphaFlagsKHR::OPAQUE; + } + + [ + vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + vk::CompositeAlphaFlagsKHR::POST_MULTIPLIED, + vk::CompositeAlphaFlagsKHR::INHERIT, + ] + .into_iter() + .find(|mode| supported.contains(*mode)) + .unwrap_or(requested) + } + /// Returns `true` if the given queue family supports presentation on this surface. + /// + /// See [`vkGetPhysicalDeviceSurfaceSupportKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetPhysicalDeviceSurfaceSupportKHR.html). pub fn physical_device_support(&self, queue_family_index: u32) -> Result { let khr_surface = Device::expect_vk_khr_surface(&self.device); unsafe { khr_surface .get_physical_device_surface_support( - self.device.physical_device.handle, + self.device.physical.handle, queue_family_index, self.handle, ) @@ -149,16 +178,15 @@ impl Surface { } } - /// Query supported presentation modes. + /// Queries supported presentation modes. + /// + /// See [`vkGetPhysicalDeviceSurfacePresentModesKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetPhysicalDeviceSurfacePresentModesKHR.html). pub fn present_modes(&self) -> Result, DriverError> { let khr_surface = Device::expect_vk_khr_surface(&self.device); unsafe { khr_surface - .get_physical_device_surface_present_modes( - self.device.physical_device.handle, - self.handle, - ) + .get_physical_device_surface_present_modes(self.device.physical.handle, self.handle) .map_err(|err| { warn!("unable to get present modes: {err}"); @@ -199,7 +227,10 @@ impl Surface { impl Debug for Surface { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str("Surface") + f.debug_struct(stringify!(Surface)) + .field("handle", &self.handle) + .field("device", &self.device) + .finish_non_exhaustive() } } @@ -294,4 +325,32 @@ mod test { vk::Format::R16G16B16A16_SFLOAT ); } + + #[test] + fn composite_alpha_prefers_requested_supported_mode() { + let supported = + vk::CompositeAlphaFlagsKHR::OPAQUE | vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED; + + assert_eq!( + Surface::composite_alpha_or_default( + supported, + vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + ), + vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED + ); + } + + #[test] + fn composite_alpha_falls_back_to_opaque() { + let supported = + vk::CompositeAlphaFlagsKHR::OPAQUE | vk::CompositeAlphaFlagsKHR::POST_MULTIPLIED; + + assert_eq!( + Surface::composite_alpha_or_default( + supported, + vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + ), + vk::CompositeAlphaFlagsKHR::OPAQUE + ); + } } diff --git a/src/driver/swapchain.rs b/src/driver/swapchain.rs index 53ccaca3..dfd3bf9c 100644 --- a/src/driver/swapchain.rs +++ b/src/driver/swapchain.rs @@ -4,75 +4,494 @@ use { super::{ DriverError, Surface, device::Device, + fence::Fence, image::{Image, ImageInfo}, }, - ash::vk::{self, Handle}, + ash::vk::{self, Handle as _}, derive_builder::Builder, - log::{debug, info, trace, warn}, - std::{mem::replace, ops::Deref, slice, thread::panicking}, + log::{Level, debug, error, info, log, trace, warn}, + std::{ + cell::RefCell, + error::Error, + fmt::{Display, Formatter}, + iter::FusedIterator, + marker::PhantomData, + mem::replace, + ops::{Deref, Index}, + slice, + thread::panicking, + }, }; -#[derive(Debug)] -struct QueueFamilySignal { - cmd_pool: vk::CommandPool, - queue_signals: Box<[QueueSignal]>, +#[cfg(feature = "checked")] +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Clone, Copy, Debug)] +enum AcquireImageError { + OutOfDate, + Swapchain(SwapchainError), } -#[derive(Debug)] -struct QueueSignal { - cmd: vk::CommandBuffer, - fence: vk::Fence, - queued: bool, +impl From for AcquireImageError { + fn from(err: SwapchainError) -> Self { + Self::Swapchain(err) + } } -#[derive(Debug, Default)] -struct QueueSwapchain { - handle: vk::SwapchainKHR, - queue_family_index: u32, - queue_index: u32, +/// Effective runtime information for a live [`Swapchain`]. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct EffectiveSwapchainInfo { + /// Timeout in nanoseconds used when acquiring the next image. + pub acquire_timeout: u64, + + /// Active alpha compositing mode used by the presentation engine. + pub composite_alpha: vk::CompositeAlphaFlagsKHR, + + /// Whether pixels obscured by other windows on the native surface are clipped. + pub clipped: bool, + + /// The live height of the surface. + pub height: u32, + + /// The active number of array layers for each swapchain image. + pub image_array_layers: u32, + + /// The number of images currently owned by the live swapchain. + pub image_count: u32, + + /// The active sharing mode used for swapchain images. + pub image_sharing_mode: vk::SharingMode, + + min_image_count: u32, + + /// The active transform applied to presented images. + pub pre_transform: vk::SurfaceTransformFlagsKHR, + + /// The active presentation mode. + pub present_mode: vk::PresentModeKHR, + + /// Timeout in nanoseconds used when presenting an image. + pub present_timeout: u64, + + /// The active format and color space of the surface. + pub surface: vk::SurfaceFormatKHR, + + /// The live width of the surface. + pub width: u32, +} + +impl EffectiveSwapchainInfo { + /// Creates a default `SwapchainInfoBuilder`. + pub fn builder() -> SwapchainInfoBuilder { + Default::default() + } + + /// Converts an `EffectiveSwapchainInfo` into an `EffectiveSwapchainInfoBuilder`. + pub fn into_builder(self) -> SwapchainInfoBuilder { + let Self { + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_count: _, + image_sharing_mode, + min_image_count, + pre_transform, + present_timeout, + present_mode, + surface, + width, + } = self; + + SwapchainInfo { + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + present_timeout, + surface, + width, + } + .into_builder() + } + + /// Converts this effective runtime info back into requested swapchain info. + pub fn into_requested_info(self) -> SwapchainInfo { + SwapchainInfo::from(self) + } } -/// Provides the ability to present rendering results to a [`Surface`]. #[derive(Debug)] -#[read_only::cast] -pub struct Swapchain { - /// The native Vulkan resource handle of this swapchain. - /// - /// _Note:_ This field is read-only. - pub handle: vk::SwapchainKHR, +struct Live; + +/// Describes one swapchain image to present. +#[derive(Debug)] +pub struct PresentInfo<'a> { + /// The acquired image to present. + pub image: SwapchainImage, - images: Box<[SwapchainImage]>, + /// The swapchain that acquired `image`. + pub swapchain: &'a mut Swapchain, - /// Information used to create this resource. + /// Semaphores to wait on before presentation. /// - /// _Note:_ This field is read-only. - pub info: SwapchainInfo, + /// Vulkan applies the flattened semaphore list to the whole present batch, not just this image. + pub wait_semaphores: &'a [vk::Semaphore], +} - prev_queue: (u32, u32), - prev_swapchain: QueueSwapchain, - queue_family_signals: Box<[QueueFamilySignal]>, - suboptimal: bool, +/// Result of one image in a queue presentation batch. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PresentResult { + /// The image was queued and the swapchain remained optimal. + QueuedOptimal, - /// The surface which supports this swapchain. + /// The image was queued, but the swapchain should be recreated soon. + QueuedSuboptimal, + + /// The image was not queued for presentation. + NotQueued(PresentError), +} + +impl PresentResult { + /// Returns `true` if this image was queued for presentation. + pub fn queued(self) -> bool { + matches!(self, Self::QueuedOptimal | Self::QueuedSuboptimal) + } + + /// Returns `true` if this present reported suboptimal state. + pub fn suboptimal(self) -> bool { + matches!(self, Self::QueuedSuboptimal) + } +} + +/// Ordered results for a queue presentation batch. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PresentBatch { + results: Box<[PresentResult]>, +} + +impl PresentBatch { + fn new(results: Box<[PresentResult]>) -> Self { + Self { results } + } + + /// Returns the per-input results in submission order. + pub fn as_slice(&self) -> &[PresentResult] { + &self.results + } + + /// Returns `true` if there are no results. + pub fn is_empty(&self) -> bool { + self.results.is_empty() + } + + /// Iterates over per-input results in submission order. + pub fn iter(&self) -> slice::Iter<'_, PresentResult> { + self.results.iter() + } + + /// Returns the number of results. + pub fn len(&self) -> usize { + self.results.len() + } +} + +impl AsRef<[PresentResult]> for PresentBatch { + fn as_ref(&self) -> &[PresentResult] { + self.as_slice() + } +} + +impl Deref for PresentBatch { + type Target = [PresentResult]; + + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl Index for PresentBatch { + type Output = PresentResult; + + fn index(&self, index: usize) -> &Self::Output { + &self.results[index] + } +} + +impl IntoIterator for PresentBatch { + type IntoIter = PresentBatchIntoIter; + type Item = PresentResult; + + fn into_iter(self) -> Self::IntoIter { + PresentBatchIntoIter { + inner: self.results.into_vec().into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a PresentBatch { + type IntoIter = slice::Iter<'a, PresentResult>; + type Item = &'a PresentResult; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// Error returned by a presentation batch. +#[derive(Clone, Debug)] +pub struct PresentBatchError { + /// Per-input results in the same order as the submitted [`PresentInfo`] values. /// - /// _Note:_ This field is read-only. - pub surface: Surface, + /// Entries are `None` when presentation did not run far enough to determine that input's + /// result. + pub results: Box<[Option]>, + + /// The batch-level failure. + pub source: PresentFailure, } -impl Swapchain { - /// Prepares a [`vk::SwapchainKHR`] object which is lazily created after calling - /// [`acquire_next_image`][Self::acquire_next_image]. - #[profiling::function] - pub fn new(surface: Surface, info: impl Into) -> Result { - let info = info.into(); +impl PresentBatchError { + fn new(source: PresentFailure, result_count: usize) -> Self { + Self { + results: vec![None; result_count].into_boxed_slice(), + source, + } + } + + fn with_results(source: PresentFailure, results: &[PresentResult]) -> Self { + Self { + results: results.iter().copied().map(Some).collect(), + source, + } + } +} + +impl Display for PresentBatchError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "unable to present batch: {}", self.source) + } +} + +impl Error for PresentBatchError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.source { + PresentFailure::Driver(err) => Some(err), + PresentFailure::DeviceLost + | PresentFailure::FullScreenExclusiveModeLost + | PresentFailure::OutOfDate + | PresentFailure::SurfaceLost => None, + } + } +} + +/// Owning iterator over [`PresentBatch`] results. +#[derive(Clone, Debug)] +pub struct PresentBatchIntoIter { + inner: std::vec::IntoIter, +} + +impl DoubleEndedIterator for PresentBatchIntoIter { + fn next_back(&mut self) -> Option { + self.inner.next_back() + } +} + +impl ExactSizeIterator for PresentBatchIntoIter { + fn len(&self) -> usize { + self.inner.len() + } +} + +impl FusedIterator for PresentBatchIntoIter {} + +impl Iterator for PresentBatchIntoIter { + type Item = PresentResult; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +/// Reason an image was not queued for presentation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PresentError { + /// The device was lost. + DeviceLost, + + /// Exclusive full-screen ownership was lost. + FullScreenExclusiveModeLost, + + /// The swapchain is out of date and should be recreated. + OutOfDate, + + /// The surface was lost and must be recreated. + SurfaceLost, +} + +/// Batch-level presentation failure. +#[derive(Clone, Copy, Debug)] +pub enum PresentFailure { + /// The device was lost. + DeviceLost, + + /// The driver rejected the request before item-level present results were available. + Driver(DriverError), + + /// Exclusive full-screen ownership was lost. + FullScreenExclusiveModeLost, + + /// The swapchain is out of date and should be recreated. + OutOfDate, + + /// The surface was lost and must be recreated. + SurfaceLost, +} + +impl Display for PresentFailure { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::DeviceLost => f.write_str("device lost"), + Self::Driver(err) => write!(f, "driver error: {err}"), + Self::FullScreenExclusiveModeLost => f.write_str("full-screen exclusive mode lost"), + Self::OutOfDate => f.write_str("out of date"), + Self::SurfaceLost => f.write_str("surface lost"), + } + } +} + +impl Error for PresentFailure { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Driver(err) => Some(err), + Self::DeviceLost + | Self::FullScreenExclusiveModeLost + | Self::OutOfDate + | Self::SurfaceLost => None, + } + } +} + +#[derive(Debug)] +struct PresentRetirement { + next_present_id: u64, + image_present_ids: Box<[Option]>, + present_pending: bool, + queue: Option, + use_present_wait: bool, +} + +impl PresentRetirement { + fn new(image_count: usize, use_present_wait: bool) -> Self { + Self { + // Present ID `0` means "no associated present ID". + next_present_id: 1, + image_present_ids: vec![None; image_count].into_boxed_slice(), + present_pending: false, + queue: None, + use_present_wait, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct QueueAddress { + family_index: u32, + queue_index: u32, +} + +#[derive(Debug, Default)] +struct QueueFamily { + cmd_pool: vk::CommandPool, + queue_signals: Box<[QueueSignal]>, +} + +#[derive(Debug)] +struct QueueSignals { + device: Device, + families: Box<[QueueFamily]>, +} + +impl QueueSignals { + fn create(surface: &Surface) -> Result { + struct PartialQueueSignals<'a> { + device: &'a Device, + families: Vec, + } + + impl Drop for PartialQueueSignals<'_> { + fn drop(&mut self) { + let mut queue_signals = QueueSignals { + device: self.device.clone(), + families: std::mem::take(&mut self.families).into_boxed_slice(), + }; + + queue_signals.destroy(); + } + } + + struct PartialQueueFamily<'a> { + device: &'a Device, + cmd_pool: vk::CommandPool, + queue_signals: Vec, + } + + impl PartialQueueFamily<'_> { + fn finish(mut self) -> QueueFamily { + let family = QueueFamily { + cmd_pool: self.cmd_pool, + queue_signals: std::mem::take(&mut self.queue_signals).into_boxed_slice(), + }; + self.cmd_pool = vk::CommandPool::null(); + + family + } + } + + impl Drop for PartialQueueFamily<'_> { + fn drop(&mut self) { + if self.cmd_pool.is_null() { + return; + } + + thread_local! { + static CMD_BUFS: RefCell> = Default::default(); + } + + CMD_BUFS.with_borrow_mut(|tls| { + tls.clear(); + tls.extend(self.queue_signals.iter().map(|queue| queue.cmd_buf)); + + unsafe { + self.device.free_command_buffers(self.cmd_pool, tls); + self.device.destroy_command_pool(self.cmd_pool, None); + } + }); + } + } let device = &surface.device; - let queue_families = surface.device.physical_device.queue_families.iter(); - let mut queue_family_signals = Vec::with_capacity(queue_families.len()); + let all_queue_families = &device.physical.queue_families; + let mut partial = PartialQueueSignals { + device, + families: Vec::with_capacity(all_queue_families.len()), + }; - for (idx, queue_family) in queue_families.enumerate() { + for (idx, queue_family) in all_queue_families.iter().enumerate() { let cmd_pool_info = vk::CommandPoolCreateInfo::default().queue_family_index(idx as _); if !surface.physical_device_support(cmd_pool_info.queue_family_index)? { + partial.families.push(Default::default()); + continue; } @@ -107,166 +526,260 @@ impl Swapchain { } })? }; + let mut partial_family = PartialQueueFamily { + device, + cmd_pool, + queue_signals: Vec::with_capacity(queue_family.queue_count as _), + }; - let mut queue_signals = Vec::with_capacity(queue_family.queue_count as _); - - for cmd in cmd_bufs { + for cmd_buf in cmd_bufs { Device::begin_command_buffer( device, - cmd, + cmd_buf, &vk::CommandBufferBeginInfo::default() .flags(vk::CommandBufferUsageFlags::SIMULTANEOUS_USE), )?; - Device::end_command_buffer(device, cmd)?; + Device::end_command_buffer(device, cmd_buf)?; + + let fence = Fence::create(device, false)?; + partial_family + .queue_signals + .push(QueueSignal { cmd_buf, fence }); + } + + partial.families.push(partial_family.finish()); + } + + let families = std::mem::take(&mut partial.families); + std::mem::forget(partial); + + Ok(Self { + device: device.clone(), + families: families.into_boxed_slice(), + }) + } + + fn destroy(&mut self) { + // Intentionally idempotent: callers may destroy early, then `Drop` will call this again. + let families = std::mem::take(&mut self.families); + + for queue_family in &families { + if queue_family.cmd_pool.is_null() { + continue; + } + + thread_local! { + static CMD_BUFS: RefCell> = Default::default(); + } + + CMD_BUFS.with_borrow_mut(|tls| { + tls.clear(); + tls.extend(queue_family.queue_signals.iter().map(|queue| queue.cmd_buf)); + unsafe { + self.device.free_command_buffers(queue_family.cmd_pool, tls); + } + }); + + unsafe { + self.device + .destroy_command_pool(queue_family.cmd_pool, None); + } + } + } + + fn reset_all(&mut self) -> Result<(), DriverError> { + for queue_family in &mut self.families { + for queue in &mut queue_family.queue_signals { + queue.fence.reset()?; + } + } + + Ok(()) + } + + fn submit( + &mut self, + device: &Device, + queue: vk::Queue, + queue_family_index: u32, + queue_index: u32, + ) -> Result<(), SwapchainError> { + let queue_signal = + &mut self.families[queue_family_index as usize].queue_signals[queue_index as usize]; + + if queue_signal.fence.is_queued() { + queue_signal + .fence + .wait_signaled() + .map_err(|_| SwapchainError::DeviceLost)? + .reset() + .map_err(|_| SwapchainError::DeviceLost)?; + } + + Device::queue_submit( + device, + queue, + slice::from_ref( + &vk::SubmitInfo::default().command_buffers(slice::from_ref(&queue_signal.cmd_buf)), + ), + queue_signal.fence.handle, + ) + .map_err(|_| SwapchainError::DeviceLost)?; + + queue_signal.fence.mark_queued(); + + Ok(()) + } + + fn wait_for_and_reset_all(&mut self, device: &Device) -> Result<(), DriverError> { + let mut fences = self.families.iter().flat_map(|queue_family| { + queue_family + .queue_signals + .iter() + .filter_map(|queue| queue.fence.is_queued().then_some(queue.fence.handle)) + }); + + let Some(first) = fences.next() else { + return Ok(()); + }; + + let Some(second) = fences.next() else { + Device::wait_for_fences(device, slice::from_ref(&first))?; + self.reset_all()?; + + return Ok(()); + }; + + thread_local! { + static FENCES: RefCell> = Default::default(); + } + + FENCES.with_borrow_mut(|tls| { + tls.clear(); + tls.extend([first, second]); + tls.extend(fences); + + Device::wait_for_fences(device, tls) + })?; + + self.reset_all()?; + + Ok(()) + } +} + +impl Drop for QueueSignals { + fn drop(&mut self) { + if panicking() { + return; + } + + self.destroy(); + } +} + +#[derive(Debug)] +struct QueueSignal { + cmd_buf: vk::CommandBuffer, + fence: Fence, +} + +#[derive(Debug)] +struct Retired; + +/// Provides the ability to present rendering results to a [`Surface`]. +/// +/// See [`VkSwapchainKHR`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSwapchainKHR.html). +#[derive(Debug)] +#[read_only::cast] +pub struct Swapchain { + /// The native Vulkan resource handle of this swapchain. + /// + /// _Note:_ This field is read-only. + pub handle: vk::SwapchainKHR, + + /// Effective runtime information for the live swapchain. + /// + /// Calls to [`set_info`](Self::set_info) update the requested configuration. This field is + /// updated lazily after the next successful acquire recreates the swapchain. The timeout fields + /// are updated immediately. + /// + /// _Note:_ This field is read-only. + pub info: EffectiveSwapchainInfo, + + live: SwapchainState, + queue_signals: QueueSignals, + + /// Whether swapchain recreation has been requested but not completed yet. + pub recreate_pending: bool, + + requested_info: SwapchainInfo, + retired: Vec>, - let queued = false; - let fence = Device::create_fence(device, queued)?; - queue_signals.push(QueueSignal { cmd, fence, queued }); - } + /// The surface which supports this swapchain. + /// + /// _Note:_ This field is read-only. + pub surface: Surface, - queue_family_signals.push(QueueFamilySignal { - cmd_pool, - queue_signals: queue_signals.into_boxed_slice(), - }); - } + #[cfg(feature = "checked")] + swapchain_id: SwapchainId, +} - Ok(Swapchain { - handle: Default::default(), - images: Default::default(), +impl Swapchain { + /// Creates a [`vk::SwapchainKHR`] object. + #[profiling::function] + pub fn create(surface: Surface, info: impl Into) -> Result { + let mut requested_info = info.into(); + let queue_signals = QueueSignals::create(&surface)?; + let (live, info) = SwapchainState::create(&surface, &mut requested_info, None)?; + let handle = live.handle(); + + Ok(Self { + handle, info, - prev_queue: Default::default(), - prev_swapchain: Default::default(), - queue_family_signals: queue_family_signals.into_boxed_slice(), - suboptimal: true, + queue_signals, + recreate_pending: false, + requested_info, + retired: Vec::new(), surface, + + #[cfg(feature = "checked")] + swapchain_id: live.swapchain_id, + + live, }) } /// Acquires the next available swapchain image for rendering. - /// - /// The returned [`SwapchainImage`] should be rendered to and then presented using - /// [`present_image`][Self::present_image]. Each call returns a unique image from the - /// swapchain's image ring buffer; the caller must not render to the same image concurrently. - /// - /// # Parameters - /// - /// * `timeout` — Maximum time to wait in nanoseconds before the operation times out. Pass - /// [`u64::MAX`] to wait indefinitely. A short timeout may return - /// [`SwapchainError::Suboptimal`] without acquiring an image. - /// * `acquired` — A semaphore that will be signaled when the acquired image is ready for use. - /// The caller must wait on this semaphore before submitting commands that write to the - /// returned image. - /// - /// # Errors - /// - /// Returns [`SwapchainError::Suboptimal`] if the swapchain no longer matches the - /// surface properties (e.g. after a window resize). The caller should typically - /// recreate any framebuffer-sized resources and try again next frame. - /// - /// Returns [`SwapchainError::SurfaceLost`] if the underlying surface has been - /// destroyed. The swapchain and surface must be recreated. - /// - /// Returns [`SwapchainError::DeviceLost`] if the Vulkan device has been lost. - /// The application must destroy and recreate the device. - /// - /// # Panics - /// - /// Panics if the acquired image index is out of bounds of the internal image array - /// (this indicates a driver or swapchain consistency bug). - /// - /// # Retry behavior - /// - /// Internally this method retries once on transient errors - /// (`TIMEOUT`, `NOT_READY`, `OUT_OF_DATE_KHR`, `FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT`). - /// Between retries the swapchain is recreated if the current state is suboptimal. #[profiling::function] pub fn acquire_next_image( &mut self, - timeout: u64, acquired: vk::Semaphore, ) -> Result { - for _ in 0..2 { - if self.suboptimal { - self.recreate().map_err(|err| { - if matches!(err, DriverError::Unsupported) { - SwapchainError::Suboptimal - } else { - SwapchainError::SurfaceLost - } - })?; - } + if self.recreate_pending { + self.recreate()?; + } - let ext = Device::expect_vk_khr_swapchain(&self.surface.device); + let image = match self.live.acquire_next_image(&self.surface.device, acquired) { + Ok(acquired) => acquired, + Err(AcquireImageError::OutOfDate) => { + self.recreate_pending = true; - let image_idx = unsafe { - ext.acquire_next_image(self.handle, timeout, acquired, vk::Fence::null()) + return Err(SwapchainError::NotReady); } - .map(|(idx, suboptimal)| { - if suboptimal { - debug!("acquired image is suboptimal"); - } - - self.suboptimal = suboptimal; - - idx - }); - - match image_idx { - Ok(image_idx) => { - let image_idx = image_idx as usize; - - assert!(image_idx < self.images.len()); - - let image = unsafe { self.images.get_unchecked(image_idx) }.clone(); - - return Ok(replace( - unsafe { self.images.get_unchecked_mut(image_idx) }, - image, - )); - } - Err(err) - if err == vk::Result::ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT - || err == vk::Result::ERROR_OUT_OF_DATE_KHR - || err == vk::Result::NOT_READY - || err == vk::Result::TIMEOUT => - { - warn!("unable to acquire image: {err}"); - - self.suboptimal = true; - - // Try again to see if we can acquire an image this frame - // (Makes redraw during resize look slightly better) - continue; - } - Err(err) if err == vk::Result::ERROR_DEVICE_LOST => { - warn!("unable to acquire image: {err}"); - - self.suboptimal = true; - - return Err(SwapchainError::DeviceLost); - } - Err(err) if err == vk::Result::ERROR_SURFACE_LOST_KHR => { - warn!("unable to acquire image: {err}"); - - self.suboptimal = true; - - return Err(SwapchainError::SurfaceLost); - } - Err(err) => { - // Probably: - // VK_ERROR_OUT_OF_HOST_MEMORY - // VK_ERROR_OUT_OF_DEVICE_MEMORY - - // TODO: Maybe handle timeout in here - - warn!("unable to acquire image: {err}"); + Err(AcquireImageError::Swapchain( + err @ SwapchainError::FullScreenExclusiveModeLost, + )) => { + self.recreate_pending = true; - return Err(SwapchainError::SurfaceLost); - } + return Err(err); } - } + Err(AcquireImageError::Swapchain(err)) => return Err(err), + }; + + self.recreate_pending = image.suboptimal; - Err(SwapchainError::Suboptimal) + Ok(image) } fn clamp_min_image_count(min_image_count: u32, surface: vk::SurfaceCapabilitiesKHR) -> u32 { @@ -279,342 +792,435 @@ impl Swapchain { min_image_count.min(surface.max_image_count) } - #[profiling::function] - fn destroy(device: &Device, handle: &mut vk::SwapchainKHR) { - if *handle == vk::SwapchainKHR::null() { - return; - } + fn destroy_retired(&mut self) -> Result<(), SwapchainError> { + let idx = 0; + let mut res = None; + + while idx < self.retired.len() { + if let Err(err) = self + .retired + .swap_remove(idx) + .destroy_when_idle(&self.surface.device) + && !matches!(res, Some(SwapchainError::Driver(DriverError::Unsupported))) + && res + .replace(err) + .map(|old| old.different(err)) + .unwrap_or_default() + { + warn!("unsupported multi-error: {err:?}"); - // wait for device to be finished with swapchain before destroying it. - // This avoid crashes when resizing windows - #[cfg(target_os = "macos")] - if let Err(err) = unsafe { device.device_wait_idle() } { - warn!("device_wait_idle() failed: {err}"); + res = Some(SwapchainError::Driver(DriverError::Unsupported)) + } } - let ext = Device::expect_vk_khr_swapchain(device); + res.map(Result::Err).unwrap_or(Ok(())) + } - unsafe { - ext.destroy_swapchain(*handle, None); + fn destroy_retired_on_drop(&mut self) { + for retired in &mut self.retired { + if let Err(err) = + retired.wait_until_retired(&self.surface.device, retired.present_timeout) + { + // Drop is best-effort shutdown. Preserve normal retryable semantics outside Drop, + // but do not silently lose retired Vulkan handles when the public owner is going away. + warn!("unable to wait for retired swapchain presents: {err:?}"); + } + + retired.handle.destroy(); } - *handle = vk::SwapchainKHR::null(); + self.retired.clear(); } - /// Presents an image which has been previously acquired using - /// [`acquire_next_image`][Self::acquire_next_image]. - #[profiling::function] - pub fn present_image( - &mut self, - image: SwapchainImage, - wait_semaphores: &[vk::Semaphore], - queue_family_index: u32, - queue_index: u32, - ) -> Result<(), DriverError> { - let device = &self.surface.device; - let queue_signal = &mut self.queue_family_signals[queue_family_index as usize] - .queue_signals[queue_index as usize]; - - let ext = Device::expect_vk_khr_swapchain(device); - - Device::with_queue(device, queue_family_index, queue_index, |queue| { - unsafe { - match ext.queue_present( - queue, - &vk::PresentInfoKHR::default() - .wait_semaphores(wait_semaphores) - .swapchains(slice::from_ref(&self.handle)) - .image_indices(slice::from_ref(&image.index)), - ) { - Ok(suboptimal) => self.suboptimal = suboptimal, - Err(err) - if err == vk::Result::ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT - || err == vk::Result::ERROR_OUT_OF_DATE_KHR - || err == vk::Result::SUBOPTIMAL_KHR => - { - self.suboptimal = true - } - Err(err) - if err == vk::Result::ERROR_DEVICE_LOST - || err == vk::Result::ERROR_SURFACE_LOST_KHR => - { - info!("skipping present: {err}"); + fn map_present_error(err: vk::Result) -> PresentError { + match err { + vk::Result::ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT => { + debug!("unable to present: {}", err); - self.suboptimal = true; - } - Err(err) => { - // Probably: - // VK_ERROR_OUT_OF_HOST_MEMORY - // VK_ERROR_OUT_OF_DEVICE_MEMORY - warn!("failed to present: {err}"); + PresentError::FullScreenExclusiveModeLost + } + vk::Result::ERROR_OUT_OF_DATE_KHR | vk::Result::SUBOPTIMAL_KHR => { + debug!("unable to present: {}", err); - self.suboptimal = true; - } - } + PresentError::OutOfDate } + vk::Result::ERROR_DEVICE_LOST => { + info!("device lost"); - if queue_signal.queued { - Device::wait_for_fence(device, &queue_signal.fence)?; - Device::reset_fences(device, slice::from_ref(&queue_signal.fence))?; + PresentError::DeviceLost } + vk::Result::ERROR_SURFACE_LOST_KHR => { + info!("surface lost"); - Device::queue_submit( - device, - queue, - &[vk::SubmitInfo::default().command_buffers(slice::from_ref(&queue_signal.cmd))], - queue_signal.fence, - )?; - queue_signal.queued = true; + PresentError::SurfaceLost + } + err => { + warn!("unable to present: {err}"); - Ok::<_, DriverError>(()) - })?; + PresentError::DeviceLost + } + } + } - if !self.prev_swapchain.handle.is_null() { - let QueueSignal { fence, queued, .. } = &mut self.queue_family_signals - [self.prev_swapchain.queue_family_index as usize] - .queue_signals[self.prev_swapchain.queue_index as usize]; - Device::wait_for_fence(device, &*fence)?; - Device::reset_fences(device, slice::from_ref(&*fence))?; - *queued = false; + fn present_failure_from_present_error(err: PresentError) -> PresentFailure { + match err { + PresentError::DeviceLost => PresentFailure::DeviceLost, + PresentError::FullScreenExclusiveModeLost => { + PresentFailure::FullScreenExclusiveModeLost + } + PresentError::OutOfDate => PresentFailure::OutOfDate, + PresentError::SurfaceLost => PresentFailure::SurfaceLost, + } + } - Self::destroy(device, &mut self.prev_swapchain.handle); + fn present_failure_from_swapchain_error(err: SwapchainError) -> PresentFailure { + match err { + SwapchainError::Driver(err) => PresentFailure::Driver(err), + SwapchainError::DeviceLost => PresentFailure::DeviceLost, + SwapchainError::FullScreenExclusiveModeLost => { + PresentFailure::FullScreenExclusiveModeLost + } + SwapchainError::NotReady | SwapchainError::Timeout => { + // TODO: Better errors + PresentFailure::Driver(DriverError::Unsupported) + } + SwapchainError::SurfaceLost => PresentFailure::SurfaceLost, } + } - let image_idx = image.index as usize; - self.images[image_idx] = image; + fn present_failure_requires_recreate(err: PresentFailure) -> bool { + matches!( + err, + PresentFailure::FullScreenExclusiveModeLost | PresentFailure::OutOfDate, + ) + } - self.prev_queue = (queue_family_index, queue_index); + fn present_result( + result: vk::Result, + aggregate_suboptimal: bool, + fallback_error: Option, + ) -> PresentResult { + match result { + vk::Result::SUCCESS if aggregate_suboptimal => PresentResult::QueuedSuboptimal, + vk::Result::SUCCESS => PresentResult::QueuedOptimal, + vk::Result::SUBOPTIMAL_KHR => PresentResult::QueuedSuboptimal, + vk::Result::ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT => { + PresentResult::NotQueued(PresentError::FullScreenExclusiveModeLost) + } + vk::Result::ERROR_OUT_OF_DATE_KHR => PresentResult::NotQueued(PresentError::OutOfDate), + vk::Result::ERROR_SURFACE_LOST_KHR => { + PresentResult::NotQueued(PresentError::SurfaceLost) + } + vk::Result::ERROR_DEVICE_LOST => PresentResult::NotQueued(PresentError::DeviceLost), + _ => PresentResult::NotQueued(fallback_error.unwrap_or(PresentError::DeviceLost)), + } + } - Ok(()) + fn present_result_requires_recreate(result: PresentResult) -> bool { + matches!( + result, + PresentResult::QueuedSuboptimal + | PresentResult::NotQueued( + PresentError::FullScreenExclusiveModeLost | PresentError::OutOfDate, + ) + ) } + /// Presents images previously acquired from their [`Swapchain`] instances. + /// + /// Panics with the same safety as [`Device::with_queue`]. #[profiling::function] - fn recreate(&mut self) -> Result<(), DriverError> { - self.wait_for_all_signals()?; - Self::destroy(&self.surface.device, &mut self.prev_swapchain.handle); - - let surface_caps = Surface::capabilities(&self.surface)?; - let min_image_count = Self::clamp_min_image_count(self.info.min_image_count, surface_caps); - let image_usage = self.supported_surface_usage(surface_caps.supported_usage_flags)?; - let (surface_width, surface_height) = match surface_caps.current_extent.width { - std::u32::MAX => ( - // TODO: Maybe handle this case with aspect-correct clamping? - self.info.width.clamp( - surface_caps.min_image_extent.width, - surface_caps.max_image_extent.width, - ), - self.info.height.clamp( - surface_caps.min_image_extent.height, - surface_caps.max_image_extent.height, - ), - ), - _ => ( - surface_caps.current_extent.width, - surface_caps.current_extent.height, - ), + pub fn queue_present<'a>( + queue_family_index: u32, + queue_index: u32, + present_info: impl IntoIterator>, + ) -> Result { + let mut present_info = present_info.into_iter().collect::>(); + let Some(first) = present_info.first() else { + return Ok(PresentBatch::new(Box::new([]))); + }; + let present_count = present_info.len(); + let device = first.swapchain.surface.device.clone(); + let queue = QueueAddress { + family_index: queue_family_index, + queue_index, }; - if surface_width * surface_height == 0 { - warn!( - "invalid surface extent: computed {}x{}", - surface_width, surface_height - ); + for info in &mut present_info { + if !Device::is_same(&device, &info.swapchain.surface.device) { + warn!("unable to present swapchains from different devices in one batch"); - return Err(DriverError::Unsupported); + return Err(PresentBatchError::new( + PresentFailure::Driver(DriverError::InvalidData), + present_count, + )); + } + + info.swapchain + .validate_present_info(&info.image, queue_family_index, queue_index); + info.swapchain + .live + .wait_for_present_idx( + &device, + info.image.index, + info.swapchain.info.present_timeout, + ) + .map_err(|err| { + PresentBatchError::new( + Self::present_failure_from_swapchain_error(err), + present_count, + ) + })?; } - let pre_transform = if surface_caps - .supported_transforms - .contains(vk::SurfaceTransformFlagsKHR::IDENTITY) - { - vk::SurfaceTransformFlagsKHR::IDENTITY - } else { - surface_caps.current_transform - }; + let mut image_indices = Vec::with_capacity(present_info.len()); + let mut present_ids = Vec::with_capacity(present_info.len()); + let mut results = vec![vk::Result::ERROR_UNKNOWN; present_info.len()]; + let mut swapchains = Vec::with_capacity(present_info.len()); + let mut wait_semaphores = Vec::new(); + + for info in &mut present_info { + let image_idx = info.image.index as usize; + + image_indices.push(info.image.index); + swapchains.push(info.swapchain.live.handle()); + wait_semaphores.extend_from_slice(info.wait_semaphores); + present_ids.push( + info.swapchain + .live + .next_present_id(image_idx) + .unwrap_or_default(), + ); + } - let ext = Device::expect_vk_khr_swapchain(&self.surface.device); - let swapchain_create_info = vk::SwapchainCreateInfoKHR::default() - .surface(self.surface.handle) - .min_image_count(min_image_count) - .image_color_space(self.info.surface.color_space) - .image_format(self.info.surface.format) - .image_extent(vk::Extent2D { - width: surface_width, - height: surface_height, - }) - .image_usage(image_usage) - .image_sharing_mode(vk::SharingMode::EXCLUSIVE) - .pre_transform(pre_transform) - .composite_alpha(vk::CompositeAlphaFlagsKHR::OPAQUE) - .present_mode(self.info.present_mode) - .clipped(true) - .old_swapchain(self.handle) - .image_array_layers(1); - let swapchain = - unsafe { ext.create_swapchain(&swapchain_create_info, None) }.map_err(|err| { - warn!("unable to create swapchain: {err}"); + let mut vk_present_info = vk::PresentInfoKHR::default() + .wait_semaphores(&wait_semaphores) + .swapchains(&swapchains) + .image_indices(&image_indices) + .results(&mut results); + + let mut present_id_info; + if present_ids.iter().any(|present_id| *present_id != 0) { + present_id_info = vk::PresentIdKHR::default().present_ids(&present_ids); + vk_present_info = vk_present_info.push_next(&mut present_id_info); + } - DriverError::Unsupported - })?; + let present = + Device::with_queue(&device, queue_family_index, queue_index, |queue| unsafe { + Device::expect_vk_khr_swapchain(&device) + .queue_present(queue, &vk_present_info) + .map_err(Self::map_present_error) + }); + let present_error = present.err(); + let aggregate_suboptimal = matches!(present, Ok(true)); + let present_results = results + .iter() + .map(|result| Self::present_result(*result, aggregate_suboptimal, present_error)) + .collect::>(); - let images = unsafe { ext.get_swapchain_images(swapchain) }.map_err(|err| match err { - vk::Result::INCOMPLETE => { - warn!("invalid swapchain image enumeration: incomplete"); + let present_failure = present_error.map(Self::present_failure_from_present_error); + let recreate_batch = present_failure.is_some_and(Self::present_failure_requires_recreate); - DriverError::InvalidData + for (idx, info) in present_info.iter_mut().enumerate() { + if recreate_batch || Self::present_result_requires_recreate(present_results[idx]) { + info.swapchain.recreate_pending = true; } - vk::Result::ERROR_OUT_OF_DEVICE_MEMORY | vk::Result::ERROR_OUT_OF_HOST_MEMORY => { - warn!("unable to get swapchain images: {err}"); - DriverError::OutOfMemory + if !present_results[idx].queued() { + info.swapchain + .live + .clear_present_id(info.image.index as usize); } - _ => { - warn!("unable to get swapchain images: {err}"); + } - DriverError::Unsupported + for (idx, info) in present_info.iter_mut().enumerate() { + if !present_results[idx].queued() { + continue; } - })?; - let images = images - .into_iter() - .enumerate() - .map(|(image_idx, image)| { - let mut image = Image::from_raw( - &self.surface.device, - image, - ImageInfo::image_2d( - surface_width, - surface_height, - self.info.surface.format, - image_usage, - ), - ); - let image_idx = image_idx as u32; - image.name = Some(format!("swapchain{image_idx}")); + let image = unsafe { info.image.to_detached() }; + info.swapchain.live.mark_present_queued(image, queue); + } - SwapchainImage { - read_only: ReadOnlySwapchainImage { - image, - index: image_idx, - }, + let signal = Device::with_queue(&device, queue_family_index, queue_index, |queue| { + for (idx, info) in present_info.iter_mut().enumerate() { + if !present_results[idx].queued() { + continue; } - }) - .collect::>(); - self.info.height = surface_height; - self.info.width = surface_width; - self.images = images; - self.prev_swapchain.handle = self.handle; - self.prev_swapchain.queue_family_index = self.prev_queue.0; - self.prev_swapchain.queue_index = self.prev_queue.1; - self.handle = swapchain; - self.suboptimal = false; + info.swapchain.queue_signals.submit( + &device, + queue, + queue_family_index, + queue_index, + )?; + } + + Ok::<_, SwapchainError>(()) + }); - info!( - "swapchain {}x{} {:?}x{} {:?} {image_usage:#?}", - self.info.width, - self.info.height, - self.info.present_mode, - self.images.len(), - self.info.surface.format, - ); + if let Err(err) = signal { + return Err(PresentBatchError::with_results( + Self::present_failure_from_swapchain_error(err), + &present_results, + )); + } + + if let Some(err) = present_error { + return Err(PresentBatchError::with_results( + Self::present_failure_from_present_error(err), + &present_results, + )); + } + + Ok(PresentBatch::new(present_results)) + } + + /// Recreates the swapchain using the currently requested configuration. + #[profiling::function] + pub fn recreate(&mut self) -> Result<(), SwapchainError> { + self.queue_signals + .wait_for_and_reset_all(&self.surface.device)?; + self.destroy_retired()?; + + let (new_live, info) = SwapchainState::create( + &self.surface, + &mut self.requested_info, + Some(self.live.handle()), + )?; + let old_live = replace(&mut self.live, new_live); + + self.retired.push(old_live.into_retired()); + self.handle = self.live.handle(); + self.info = info; + self.recreate_pending = false; + + #[cfg(feature = "checked")] + { + self.swapchain_id = self.live.swapchain_id; + } Ok(()) } - /// Updates the information which controls this swapchain. + /// Updates the requested information which controls this swapchain. /// - /// Previously acquired swapchain images should be discarded after calling this function. + /// The live effective [`info`](Self::info) field updates [`acquire_timeout`](SwapchainInfo::acquire_timeout) + /// immediately. Other fields are updated lazily after the next successful acquire recreates the + /// swapchain and reflects the runtime values that were actually selected. pub fn set_info(&mut self, info: impl Into) { let info: SwapchainInfo = info.into(); - if self.info != info { - // attempt to reducing flickering when resizing windows on mac - #[cfg(target_os = "macos")] - if let Err(err) = unsafe { self.surface.device.device_wait_idle() } { - warn!("device_wait_idle() failed: {err}"); - } - - self.info = info; - - trace!("info: {:?}", self.info); - - self.suboptimal = true; + let recreate = self.requested_info.height != info.height + || self.requested_info.clipped != info.clipped + || self.requested_info.image_array_layers != info.image_array_layers + || self.requested_info.image_sharing_mode != info.image_sharing_mode + || self.requested_info.min_image_count != info.min_image_count + || self.requested_info.pre_transform != info.pre_transform + || self.requested_info.present_mode != info.present_mode + || self.requested_info.composite_alpha != info.composite_alpha + || self.requested_info.surface != info.surface + || self.requested_info.width != info.width; + + self.requested_info = info; + self.info.acquire_timeout = info.acquire_timeout; + self.info.present_timeout = info.present_timeout; + self.live.acquire_timeout = info.acquire_timeout; + self.live.present_timeout = info.present_timeout; + + trace!("requested info: {:?}", self.requested_info); + + if recreate { + self.recreate_pending = true; } } fn supported_surface_usage( - &mut self, + device: &Device, + surface_format: vk::Format, surface_capabilities: vk::ImageUsageFlags, ) -> Result { - let mut res = vk::ImageUsageFlags::empty(); + let image_usage = surface_capabilities & !vk::ImageUsageFlags::ATTACHMENT_FEEDBACK_LOOP_EXT; - for bit in 0..u32::BITS { - let usage = vk::ImageUsageFlags::from_raw((1 << bit) & surface_capabilities.as_raw()); - if usage.is_empty() { - continue; - } + if image_usage.is_empty() { + warn!("surface reports no supported swapchain image usage"); + + return Err(DriverError::Unsupported); + } - if self - .surface - .device - .physical_device - .image_format_properties( - self.info.surface.format, - vk::ImageType::TYPE_2D, - vk::ImageTiling::OPTIMAL, - usage, - vk::ImageCreateFlags::empty(), + if device + .physical + .image_format_properties( + surface_format, + vk::ImageType::TYPE_2D, + vk::ImageTiling::OPTIMAL, + image_usage, + vk::ImageCreateFlags::empty(), + ) + .inspect_err(|err| { + warn!( + "unable to get image format properties: {:?} {:?} {err}", + surface_format, image_usage ) - .inspect_err(|err| { - warn!( - "unable to get image format properties: {:?} {:?} {err}", - self.info.surface.format, usage - ) - })? - .is_none() - { - continue; - } + })? + .is_none() + { + warn!( + "unsupported swapchain image usage combination: {:?} {:?}", + surface_format, image_usage + ); - res |= usage; + return Err(DriverError::Unsupported); } - // On mesa the device will return this usage flag as supported even when the extension - // that is needed for an image to have this flag isn't enabled - res &= !vk::ImageUsageFlags::ATTACHMENT_FEEDBACK_LOOP_EXT; - - Ok(res) + Ok(image_usage) } - /// Waits for all tracked per-queue fences to become signaled and resets their state. - fn wait_for_all_signals(&mut self) -> Result<(), DriverError> { - let fences = self - .queue_family_signals - .iter() - .flat_map(|queue_family| { - queue_family - .queue_signals - .iter() - .filter_map(|queue| queue.queued.then_some(queue.fence)) - }) - .collect::>(); - if fences.is_empty() { - return Ok(()); + fn validate_present_info( + &self, + image: &SwapchainImage, + queue_family_index: u32, + queue_index: u32, + ) { + #[cfg(feature = "checked")] + { + assert!( + image.index < self.live.images.len() as u32, + "swapchain image index out of bounds" + ); + assert!( + image.swapchain_id == self.swapchain_id, + "swapchain image belongs to a different swapchain" + ); } - let device = &self.surface.device; - - Device::wait_for_fences(device, &fences)?; - Device::reset_fences(device, &fences)?; - - for queue_family in &mut self.queue_family_signals { - for queue in &mut queue_family.queue_signals { - queue.queued = false; - } - } + #[cfg(feature = "checked")] + assert!( + self.queue_signals + .families + .get(queue_family_index as usize) + .and_then(|queue_family| queue_family.queue_signals.get(queue_index as usize)) + .is_some(), + "unsupported queue index" + ); + } - Ok(()) + /// Waits for a previously queued present on `image_idx` from the live swapchain to complete. + /// + /// This does not search retired swapchain handles after recreation. Callers that may span + /// recreation should retire their own pending image references before acquiring again. + /// + /// # Safety + /// + /// Callers must ensure that `image_idx` refers to an image acquired from this swapchain and + /// that waiting for the associated present is valid for the current lifetime of the swapchain. + pub unsafe fn wait_for_present_image( + &mut self, + image_idx: u32, + timeout: u64, + ) -> Result<(), SwapchainError> { + self.live + .wait_for_present_idx(&self.surface.device, image_idx, timeout) } } @@ -625,30 +1231,28 @@ impl Drop for Swapchain { return; } - if let Err(err) = self.wait_for_all_signals() { + if let Err(err) = self + .queue_signals + .wait_for_and_reset_all(&self.surface.device) + { warn!("unable to wait for swapchain signals: {err}"); } - let device = &self.surface.device; - - Self::destroy(device, &mut self.handle); - Self::destroy(device, &mut self.prev_swapchain.handle); - - for queue_family in &self.queue_family_signals { - let mut cmd_bufs = Vec::with_capacity(queue_family.queue_signals.len()); - for queue in &queue_family.queue_signals { - cmd_bufs.push(queue.cmd); - - unsafe { - device.destroy_fence(queue.fence, None); - } - } + self.destroy_retired_on_drop(); - unsafe { - device.free_command_buffers(queue_family.cmd_pool, &cmd_bufs); - device.destroy_command_pool(queue_family.cmd_pool, None); - } + if let Err(err) = self + .live + .wait_for_all_presents(&self.surface.device, self.info.present_timeout) + { + // Drop is best-effort shutdown. At this point there is no recoverable path for the + // public owner, so log the failed wait and still release the Vulkan handle. + warn!("unable to wait for live swapchain presents: {err:?}"); } + + // Destroy handles explicitly while queue signals are still alive; their field drop order is + // otherwise independent of the present-retirement fallback queues recorded by the states. + self.live.handle.destroy(); + self.handle = vk::SwapchainKHR::null(); } } @@ -661,33 +1265,151 @@ impl PartialEq for Swapchain { } /// Describes the condition of a swapchain. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug)] pub enum SwapchainError { /// This frame is lost but more may be acquired later. DeviceLost, - /// This frame is not lost but there may be a delay while the next frame is recreated. - Suboptimal, + /// Recoverable driver error. + Driver(DriverError), + + /// Exclusive full-screen ownership was lost and the swapchain should be recreated. + FullScreenExclusiveModeLost, + + /// No image is currently available, but a future acquire may succeed. + NotReady, /// The surface was lost and must be recreated, which includes any operating system window. SurfaceLost, + + /// No image was acquired before the configured acquire timeout elapsed. + Timeout, +} + +impl SwapchainError { + fn different(self, other: Self) -> bool { + !self.matches(other) + } + + fn matches(self, other: Self) -> bool { + use SwapchainError::*; + + matches!( + (self, other), + (DeviceLost, DeviceLost) + | ( + Driver(DriverError::InvalidData), + Driver(DriverError::InvalidData) + ) + | ( + Driver(DriverError::OutOfMemory), + Driver(DriverError::OutOfMemory) + ) + | ( + Driver(DriverError::Unsupported), + Driver(DriverError::Unsupported) + ) + | (FullScreenExclusiveModeLost, FullScreenExclusiveModeLost) + | (NotReady, NotReady) + | (SurfaceLost, SurfaceLost) + | (Timeout, Timeout) + ) + } +} + +impl From for SwapchainError { + fn from(err: DriverError) -> Self { + match err { + DriverError::InvalidData => Self::DeviceLost, + err => Self::Driver(err), + } + } +} + +#[derive(Debug)] +struct SwapchainHandle { + device: Device, + handle: vk::SwapchainKHR, +} + +impl SwapchainHandle { + fn new(device: &Device, handle: vk::SwapchainKHR) -> Self { + Self { + device: device.clone(), + handle, + } + } + + fn destroy(&mut self) { + Self::destroy_raw(&self.device, &mut self.handle); + } + + fn destroy_raw(device: &Device, handle: &mut vk::SwapchainKHR) { + if handle.is_null() { + return; + } + + unsafe { + Device::expect_vk_khr_swapchain(device).destroy_swapchain(*handle, None); + } + + *handle = vk::SwapchainKHR::null(); + } + + fn raw(&self) -> vk::SwapchainKHR { + self.handle + } +} + +impl Drop for SwapchainHandle { + fn drop(&mut self) { + if panicking() { + return; + } + + self.destroy(); + } +} + +/// Opaque swapchain ownership identifier used by checked builds. +#[cfg(feature = "checked")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct SwapchainId(u64); + +#[cfg(feature = "checked")] +impl SwapchainId { + fn next() -> Self { + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + + Self(NEXT_ID.fetch_add(1, Ordering::Relaxed)) + } } /// An opaque type representing a swapchain image. +/// +/// In checked builds, using a swapchain image with a different swapchain than the one that acquired +/// it will panic. #[derive(Debug)] #[read_only::embed] pub struct SwapchainImage { - /// The underlying image resource. + #[readonly] + pub(self) image: Image, + + /// The swapchain image index. /// /// _Note:_ This field is read-only. #[readonly] - image: Image, + pub index: u32, - /// The swapchain image index. + /// Whether the swapchain was suboptimal when this image was acquired. /// /// _Note:_ This field is read-only. #[readonly] - pub index: u32, + pub suboptimal: bool, + + #[cfg(feature = "checked")] + #[readonly] + pub(self) swapchain_id: SwapchainId, } impl SwapchainImage { @@ -700,19 +1422,32 @@ impl SwapchainImage { ) -> Self { Self { read_only: ReadOnlySwapchainImage { - image: Image::from_raw(device, handle, info), + image: unsafe { Image::from_raw(device, handle, info) }, index, + suboptimal: false, + + #[cfg(feature = "checked")] + swapchain_id: SwapchainId::next(), }, } } -} -impl Clone for SwapchainImage { - fn clone(&self) -> Self { + /// Creates an acquired wrapper for the same native swapchain image. + /// + /// # Safety + /// + /// The caller must ensure the Vulkan image handle remains valid for the lifetime of the + /// returned image. This should only be called for images owned by a live swapchain, and each + /// detached wrapper must be used according to the swapchain image acquire/present lifecycle. + pub unsafe fn to_detached(&self) -> Self { Self { read_only: ReadOnlySwapchainImage { image: unsafe { self.image.to_detached() }, index: self.index, + suboptimal: false, + + #[cfg(feature = "checked")] + swapchain_id: self.swapchain_id, }, } } @@ -734,64 +1469,48 @@ impl Deref for ReadOnlySwapchainImage { pattern = "owned" )] pub struct SwapchainInfo { + /// Timeout in nanoseconds used when acquiring the next image. + /// + /// Do not use `u64::MAX` on surfaces where acquire forward progress cannot be guaranteed. + #[builder(default = "u64::MAX")] + pub acquire_timeout: u64, + + /// Alpha compositing mode used by the presentation engine. + #[builder(default = vk::CompositeAlphaFlagsKHR::OPAQUE)] + pub composite_alpha: vk::CompositeAlphaFlagsKHR, + + /// Whether to clip pixels obscured by other windows on the native surface. + #[builder(default = "true")] + pub clipped: bool, + /// The initial height of the surface. #[builder(default)] pub height: u32, - /// The minimum number of presentable images that the application needs. The implementation - /// will either create the swapchain with at least that many images, or it will fail to - /// create the swapchain. - /// - /// More images introduce more display lag, but smoother animation. + /// The requested number of array layers for each swapchain image. + #[builder(default = "1")] + pub image_array_layers: u32, + + /// Sharing mode used for swapchain images. + #[builder(default = vk::SharingMode::EXCLUSIVE)] + pub image_sharing_mode: vk::SharingMode, + + /// The minimum number of presentable images that the application needs. #[builder(default = "2")] pub min_image_count: u32, - /// `vk::PresentModeKHR` determines timing and queueing with which frames are actually - /// displayed to the user. - /// - /// `vk::PresentModeKHR::FIFO` - Presentation frames are kept in a First-In-First-Out queue - /// approximately 3 frames long. Every vertical blanking period, the presentation engine - /// will pop a frame off the queue to display. If there is no frame to display, it will - /// present the same frame again until the next vblank. - /// - /// When a present command is executed on the GPU, the presented image is added on the queue. - /// - /// * **Tearing:** No tearing will be observed. - /// * **Also known as**: "Vsync On" - /// - /// `vk::PresentModeKHR::FIFO_RELAXED` - Presentation frames are kept in a First-In-First-Out - /// queue approximately 3 frames long. Every vertical blanking period, the presentation - /// engine will pop a frame off the queue to display. If there is no frame to display, it - /// will present the same frame until there is a frame in the queue. The moment there is a - /// frame in the queue, it will immediately pop the frame off the queue. - /// - /// When a present command is executed on the GPU, the presented image is added on the queue. - /// - /// * **Tearing**: Tearing will be observed if frames last more than one vblank as the front - /// buffer. - /// * **Also known as**: "Adaptive Vsync" - /// - /// `vk::PresentModeKHR::IMMEDIATE` - Presentation frames are not queued at all. The moment a - /// present command is executed on the GPU, the presented image is swapped onto the front - /// buffer immediately. - /// - /// * **Tearing**: Tearing can be observed. - /// * **Also known as**: "Vsync Off" - /// - /// `vk::PresentModeKHR::MAILBOX` - Presentation frames are kept in a single-frame queue. Every - /// vertical blanking period, the presentation engine will pop a frame from the queue. If - /// there is no frame to display, it will present the same frame again until the next - /// vblank. - /// - /// When a present command is executed on the GPU, the frame will be put into the queue. - /// If there was already a frame in the queue, the new frame will _replace_ the old frame. - /// on the queue. - /// - /// * **Tearing**: No tearing will be observed. - /// * **Also known as**: "Fast Vsync" + /// Transform applied to presented images. + #[builder(default = vk::SurfaceTransformFlagsKHR::IDENTITY)] + pub pre_transform: vk::SurfaceTransformFlagsKHR, + + /// `vk::PresentModeKHR` determines timing and queueing with which frames are displayed. #[builder(default = vk::PresentModeKHR::IMMEDIATE)] pub present_mode: vk::PresentModeKHR, + /// Timeout in nanoseconds used when presenting an image. + #[builder(default = "u64::MAX")] + pub present_timeout: u64, + /// The format and color space of the surface. #[builder(default)] pub surface: vk::SurfaceFormatKHR, @@ -806,9 +1525,16 @@ impl SwapchainInfo { #[inline(always)] pub fn new(width: u32, height: u32, surface: vk::SurfaceFormatKHR) -> SwapchainInfo { Self { + acquire_timeout: u64::MAX, + clipped: true, + composite_alpha: vk::CompositeAlphaFlagsKHR::OPAQUE, height, + image_array_layers: 1, + image_sharing_mode: vk::SharingMode::EXCLUSIVE, min_image_count: 2, + pre_transform: vk::SurfaceTransformFlagsKHR::IDENTITY, present_mode: vk::PresentModeKHR::IMMEDIATE, + present_timeout: u64::MAX, surface, width, } @@ -821,12 +1547,47 @@ impl SwapchainInfo { /// Converts a `SwapchainInfo` into a `SwapchainInfoBuilder`. pub fn into_builder(self) -> SwapchainInfoBuilder { + let Self { + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + present_timeout, + surface, + width, + } = self; + + let acquire_timeout = Some(acquire_timeout); + let clipped = Some(clipped); + let composite_alpha = Some(composite_alpha); + let height = Some(height); + let image_array_layers = Some(image_array_layers); + let image_sharing_mode = Some(image_sharing_mode); + let min_image_count = Some(min_image_count); + let pre_transform = Some(pre_transform); + let present_mode = Some(present_mode); + let present_timeout = Some(present_timeout); + let surface = Some(surface); + let width = Some(width); + SwapchainInfoBuilder { - height: Some(self.height), - min_image_count: Some(self.min_image_count), - present_mode: Some(self.present_mode), - surface: Some(self.surface), - width: Some(self.width), + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + present_timeout, + surface, + width, } } } @@ -837,6 +1598,41 @@ impl From for SwapchainInfo { } } +impl From for SwapchainInfo { + fn from(info: EffectiveSwapchainInfo) -> Self { + let EffectiveSwapchainInfo { + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_count: _, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + present_timeout, + surface, + width, + } = info; + + Self { + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + present_timeout, + surface, + width, + } + } +} + impl SwapchainInfoBuilder { /// Builds a new `SwapchainInfo`. #[inline(always)] @@ -845,12 +1641,558 @@ impl SwapchainInfoBuilder { } } +#[derive(Debug)] +struct SwapchainState { + __: PhantomData, + acquire_timeout: u64, + handle: SwapchainHandle, + images: Box<[Option]>, + present_timeout: u64, + retirement: PresentRetirement, + + #[cfg(feature = "checked")] + swapchain_id: SwapchainId, +} + +impl SwapchainState { + fn handle(&self) -> vk::SwapchainKHR { + self.handle.raw() + } + + fn wait_for_all_presents( + &mut self, + device: &Device, + timeout: u64, + ) -> Result<(), SwapchainError> { + if !self.retirement.present_pending { + return Ok(()); + } + + if !self.retirement.use_present_wait { + let queue = self.retirement.queue.expect("missing present queue"); + let queue_wait_idle = + Device::with_queue(device, queue.family_index, queue.queue_index, |queue| { + Device::queue_wait_idle(device, queue) + }); + + self.retirement.present_pending = false; + queue_wait_idle?; + + return Ok(()); + } + + let swapchain = self.handle(); + let mut res = None; + + for present_id in self + .retirement + .image_present_ids + .iter_mut() + .filter_map(|present_id| present_id.map(|_| present_id)) + { + let Some(id) = *present_id else { + unreachable!(); + }; + + let wait_for_present = + unsafe { Self::wait_for_present(device, swapchain, id, timeout) }; + + *present_id = None; + + if let Err(err) = wait_for_present + && !matches!(res, Some(SwapchainError::Driver(DriverError::Unsupported))) + && res + .replace(err) + .map(|old| old.different(err)) + .unwrap_or_default() + { + warn!("unsupported multi-error: {err:?}"); + + res = Some(SwapchainError::Driver(DriverError::Unsupported)) + } + } + + self.retirement.present_pending = false; + + res.map(Result::Err).unwrap_or(Ok(())) + } + + unsafe fn wait_for_present( + device: &Device, + swapchain: vk::SwapchainKHR, + present_id: u64, + timeout: u64, + ) -> Result<(), SwapchainError> { + unsafe { + Device::expect_vk_khr_present_wait(device) + .wait_for_present(swapchain, present_id, timeout) + .map_err(|err| { + warn!("unable to wait for present: {err}"); + + match err { + vk::Result::ERROR_DEVICE_LOST => SwapchainError::DeviceLost, + vk::Result::ERROR_SURFACE_LOST_KHR => SwapchainError::SurfaceLost, + vk::Result::ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT => { + SwapchainError::FullScreenExclusiveModeLost + } + vk::Result::ERROR_OUT_OF_DATE_KHR | vk::Result::SUBOPTIMAL_KHR => { + SwapchainError::NotReady + } + _ => SwapchainError::SurfaceLost, + } + }) + } + } +} + +impl SwapchainState { + fn create( + surface: &Surface, + requested_info: &mut SwapchainInfo, + old_swapchain: Option, + ) -> Result<(Self, EffectiveSwapchainInfo), DriverError> { + let device = &surface.device; + let surface_caps = Surface::capabilities(surface)?; + + { + let composite_alpha = Surface::composite_alpha_or_default( + surface_caps.supported_composite_alpha, + requested_info.composite_alpha, + ); + if requested_info.composite_alpha != composite_alpha { + warn!( + "requested composite alpha unsupported: {:?}; falling back to {:?}", + requested_info.composite_alpha, composite_alpha + ); + + requested_info.composite_alpha = composite_alpha; + } + } + + { + let image_array_layers = requested_info + .image_array_layers + .clamp(1, surface_caps.max_image_array_layers); + if requested_info.image_array_layers != image_array_layers { + warn!( + "requested image array layers unsupported: {}; falling back to {}", + requested_info.image_array_layers, image_array_layers + ); + + requested_info.image_array_layers = image_array_layers; + } + } + + { + if requested_info.image_sharing_mode == vk::SharingMode::CONCURRENT + && device.physical.queue_families.len() < 2 + { + warn!( + "requested concurrent image sharing with one queue family; falling back to exclusive" + ); + + requested_info.image_sharing_mode = vk::SharingMode::EXCLUSIVE; + } + } + + { + requested_info.min_image_count = + Swapchain::clamp_min_image_count(requested_info.min_image_count, surface_caps); + } + + { + if !surface_caps + .supported_transforms + .contains(requested_info.pre_transform) + { + warn!( + "requested pre-transform unsupported: {:?}; falling back to {:?}", + requested_info.pre_transform, surface_caps.current_transform + ); + + requested_info.pre_transform = surface_caps.current_transform; + } + } + + { + let present_modes = Surface::present_modes(surface)?; + if !present_modes.contains(&requested_info.present_mode) { + warn!( + "requested present mode unsupported: {:?}; falling back to FIFO", + requested_info.present_mode + ); + + requested_info.present_mode = vk::PresentModeKHR::FIFO; + } + } + + { + // TODO: There is special handling for the MAX case we're not doing (read specs) + if surface_caps.current_extent.width == u32::MAX + && surface_caps.current_extent.height == u32::MAX + { + requested_info.width = requested_info.width.clamp( + surface_caps.min_image_extent.width, + surface_caps.max_image_extent.width, + ); + requested_info.height = requested_info.height.clamp( + surface_caps.min_image_extent.height, + surface_caps.max_image_extent.height, + ); + } else { + requested_info.width = surface_caps.current_extent.width; + requested_info.height = surface_caps.current_extent.height; + } + + if requested_info.width == 0 || requested_info.height == 0 { + warn!( + "invalid surface extent: computed {}x{}", + requested_info.width, requested_info.height + ); + + return Err(DriverError::Unsupported); + } + } + + let image_extent = vk::Extent2D { + width: requested_info.width, + height: requested_info.height, + }; + let image_usage = Swapchain::supported_surface_usage( + device, + requested_info.surface.format, + surface_caps.supported_usage_flags, + )?; + + let mut swapchain_create_info = vk::SwapchainCreateInfoKHR::default() + .clipped(requested_info.clipped) + .composite_alpha(requested_info.composite_alpha) + .image_color_space(requested_info.surface.color_space) + .image_extent(image_extent) + .image_format(requested_info.surface.format) + .image_sharing_mode(requested_info.image_sharing_mode) + .image_usage(image_usage) + .min_image_count(requested_info.min_image_count) + .old_swapchain(old_swapchain.unwrap_or_default()) + .pre_transform(requested_info.pre_transform) + .present_mode(requested_info.present_mode) + .image_array_layers(requested_info.image_array_layers) + .surface(surface.handle); + + let queue_family_indices = matches!( + requested_info.image_sharing_mode, + vk::SharingMode::CONCURRENT + ) + .then(|| (0..device.physical.queue_families.len() as u32).collect::>()); + + if let Some(queue_family_indices) = &queue_family_indices { + swapchain_create_info = + swapchain_create_info.queue_family_indices(queue_family_indices); + } + + let vk_khr_swapchain = Device::expect_vk_khr_swapchain(device); + let mut handle = unsafe { + vk_khr_swapchain + .create_swapchain(&swapchain_create_info, None) + .map_err(|err| { + warn!("unable to create swapchain: {err}"); + + // TODO: Improve error handling + + DriverError::Unsupported + })? + }; + let images = unsafe { + vk_khr_swapchain + .get_swapchain_images(handle) + .map_err(|err| { + SwapchainHandle::destroy_raw(device, &mut handle); + + match err { + vk::Result::INCOMPLETE => { + warn!("invalid swapchain image enumeration: incomplete"); + + DriverError::InvalidData + } + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY + | vk::Result::ERROR_OUT_OF_HOST_MEMORY => { + warn!("unable to get swapchain images: {err}"); + + DriverError::OutOfMemory + } + _ => { + error!("unable to get swapchain images: {err}"); + + DriverError::Unsupported + } + } + })? + }; + + #[cfg(feature = "checked")] + let swapchain_id = SwapchainId::next(); + + let vk::Extent2D { height, width } = image_extent; + let vk::SwapchainCreateInfoKHR { + image_format, + image_array_layers, + .. + } = swapchain_create_info; + + let images = { + let image_info = ImageInfo::image_2d_array( + width, + height, + image_array_layers, + image_format, + image_usage, + ); + + images + .into_iter() + .enumerate() + .map(|(idx, image)| { + let index = idx as u32; + let image = unsafe { Image::from_raw(device, image, image_info) }; + + image.set_debug_name(format!("swapchain{index}")); + + Some(image) + }) + .collect::>() + }; + + let image_count = images.len() as _; + + let info = { + let SwapchainInfo { + acquire_timeout, + clipped, + image_array_layers, + present_timeout, + surface, + .. + } = *requested_info; + let vk::SwapchainCreateInfoKHR { + composite_alpha, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + .. + } = swapchain_create_info; + + EffectiveSwapchainInfo { + acquire_timeout, + clipped, + composite_alpha, + height, + image_array_layers, + image_count, + image_sharing_mode, + min_image_count, + pre_transform, + present_mode, + present_timeout, + surface, + width, + } + }; + + info!( + "swapchain {}x{} {:?}x{image_count} {:?} {image_usage:#?}", + info.width, info.height, info.present_mode, info.surface.format, + ); + + let use_present_wait = { + surface.device.physical.vk_khr_present_id.is_some() + && surface.device.physical.vk_khr_present_wait.is_some() + }; + + Ok(( + Self { + acquire_timeout: info.acquire_timeout, + handle: SwapchainHandle::new(device, handle), + images, + present_timeout: info.present_timeout, + retirement: PresentRetirement::new(image_count as _, use_present_wait), + + #[cfg(feature = "checked")] + swapchain_id, + + __: PhantomData, + }, + info, + )) + } + + fn acquire_next_image( + &mut self, + device: &Device, + acquired: vk::Semaphore, + ) -> Result { + let res = unsafe { + Device::expect_vk_khr_swapchain(device) + .acquire_next_image( + self.handle(), + self.acquire_timeout, + acquired, + vk::Fence::null(), + ) + .map_err(|err| match err { + vk::Result::ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT => { + SwapchainError::FullScreenExclusiveModeLost.into() + } + vk::Result::ERROR_OUT_OF_DATE_KHR => AcquireImageError::OutOfDate, + vk::Result::NOT_READY => SwapchainError::NotReady.into(), + vk::Result::TIMEOUT => SwapchainError::Timeout.into(), + vk::Result::ERROR_DEVICE_LOST => SwapchainError::DeviceLost.into(), + vk::Result::ERROR_SURFACE_LOST_KHR => SwapchainError::SurfaceLost.into(), + _ => AcquireImageError::Swapchain(DriverError::Unsupported.into()), + }) + }; + + if let Err(err) = res { + use {AcquireImageError::*, Level::*, SwapchainError::*}; + + let level = match err { + OutOfDate | Swapchain(NotReady | Timeout) => Debug, + Swapchain(DeviceLost | SurfaceLost) => Warn, + _ => Error, + }; + + log!(level, "unable to acquire image: {err:?}"); + } + + let (index, suboptimal) = res?; + + if suboptimal { + debug!("acquired image is suboptimal"); + } + + let image = self + .images + .get_mut(index as usize) + .map(|image| image.take().expect("expected available image")) + .expect("expected image"); + + let swapchain_image = SwapchainImage { + read_only: ReadOnlySwapchainImage { + image, + index, + suboptimal, + swapchain_id: self.swapchain_id, + }, + }; + + Ok(swapchain_image) + } + + fn clear_present_id(&mut self, image_idx: usize) { + let present_id = &mut self.retirement.image_present_ids[image_idx]; + + debug_assert!(present_id.is_some()); + + *present_id = None; + } + + fn into_retired(self) -> SwapchainState { + SwapchainState { + acquire_timeout: self.acquire_timeout, + handle: self.handle, + images: self.images, + present_timeout: self.present_timeout, + retirement: self.retirement, + + #[cfg(feature = "checked")] + swapchain_id: self.swapchain_id, + + __: PhantomData, + } + } + + fn mark_present_queued(&mut self, image: SwapchainImage, queue: QueueAddress) { + let ReadOnlySwapchainImage { image, index, .. } = image.read_only; + + self.images[index as usize] = Some(image); + self.retirement.present_pending = true; + self.retirement.queue = Some(queue); + } + + fn next_present_id(&mut self, image_idx: usize) -> Option { + if !self.retirement.use_present_wait { + return None; + } + + let next_present_id = self.retirement.next_present_id; + self.retirement.next_present_id += 1; + + let present_id = &mut self.retirement.image_present_ids[image_idx]; + + debug_assert!(present_id.is_none()); + + *present_id = Some(next_present_id); + + Some(next_present_id) + } + + fn wait_for_present_idx( + &mut self, + device: &Device, + image_idx: u32, + timeout: u64, + ) -> Result<(), SwapchainError> { + let swapchain = self.handle(); + let Some(image_present_id) = self + .retirement + .image_present_ids + .get_mut(image_idx as usize) + else { + return Ok(()); + }; + let Some(present_id) = *image_present_id else { + return Ok(()); + }; + + unsafe { + Self::wait_for_present(device, swapchain, present_id, timeout)?; + } + + *image_present_id = None; + + if self + .retirement + .image_present_ids + .iter() + .all(Option::is_none) + { + self.retirement.present_pending = false; + } + + Ok(()) + } +} + +impl SwapchainState { + fn destroy_when_idle(mut self, device: &Device) -> Result<(), SwapchainError> { + self.wait_until_retired(device, self.present_timeout)?; + self.handle.destroy(); + + Ok(()) + } + + fn wait_until_retired(&mut self, device: &Device, timeout: u64) -> Result<(), SwapchainError> { + self.wait_for_all_presents(device, timeout) + } +} + #[cfg(test)] mod test { use super::*; type Info = SwapchainInfo; type Builder = SwapchainInfoBuilder; + type EffectiveInfo = EffectiveSwapchainInfo; #[test] pub fn swapchain_info() { @@ -879,4 +2221,45 @@ mod test { Info::new(0, 0, vk::SurfaceFormatKHR::default()) ); } + + #[test] + pub fn effective_swapchain_info_round_trips_to_requested_info() { + let info = EffectiveInfo { + acquire_timeout: 42, + clipped: false, + composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + height: 123, + image_array_layers: 2, + image_count: 7, + image_sharing_mode: vk::SharingMode::CONCURRENT, + min_image_count: 3, + pre_transform: vk::SurfaceTransformFlagsKHR::ROTATE_90, + present_mode: vk::PresentModeKHR::FIFO, + present_timeout: 999, + surface: vk::SurfaceFormatKHR::default() + .format(vk::Format::B8G8R8A8_UNORM) + .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + width: 456, + }; + + assert_eq!( + info.into_requested_info(), + Info { + acquire_timeout: 42, + clipped: false, + composite_alpha: vk::CompositeAlphaFlagsKHR::PRE_MULTIPLIED, + height: 123, + image_array_layers: 2, + image_sharing_mode: vk::SharingMode::CONCURRENT, + min_image_count: 3, + pre_transform: vk::SurfaceTransformFlagsKHR::ROTATE_90, + present_mode: vk::PresentModeKHR::FIFO, + present_timeout: 999, + surface: vk::SurfaceFormatKHR::default() + .format(vk::Format::B8G8R8A8_UNORM) + .color_space(vk::ColorSpaceKHR::SRGB_NONLINEAR), + width: 456, + } + ); + } } diff --git a/src/lazy_str.rs b/src/lazy_str.rs new file mode 100644 index 00000000..68d63e04 --- /dev/null +++ b/src/lazy_str.rs @@ -0,0 +1,31 @@ +use std::{cell::OnceCell, fmt::Arguments}; + +/// Creates a [`LazyStr`] from `format_args!` without eagerly allocating a `String`. +#[macro_export] +macro_rules! lazy_str { + ($($arg:tt)*) => { + $crate::LazyStr::new(format_args!($($arg)*)) + }; +} + +/// Lazily formats string arguments on first string access. +pub struct LazyStr<'a> { + args: Arguments<'a>, + value: OnceCell, +} + +impl<'a> LazyStr<'a> { + /// Creates a lazily formatted string from prebuilt formatting arguments. + pub fn new(args: Arguments<'a>) -> Self { + Self { + args, + value: OnceCell::new(), + } + } +} + +impl AsRef for LazyStr<'_> { + fn as_ref(&self) -> &str { + self.value.get_or_init(|| self.args.to_string()).as_str() + } +} diff --git a/src/lib.rs b/src/lib.rs index 318a0c52..8f21d821 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,24 +18,12 @@ pub mod cmd; pub mod driver; pub mod node; pub mod pool; +pub mod stream; +pub mod submission; -mod submission; +mod lazy_str; -use std::sync::Arc; - -#[cfg(feature = "checked")] -use std::sync::atomic::{AtomicU64, Ordering}; - -use crate::{ - cmd::{ClearColorValue, CommandRef}, - driver::{ - accel_struct::AccelerationStructure, buffer::Buffer, image::Image, - swapchain::SwapchainImage, - }, - pool::Lease, -}; - -pub use self::submission::Submission; +pub use self::lazy_str::LazyStr; use { self::{ @@ -46,48 +34,100 @@ use { ImageLeaseNode, ImageNode, SwapchainImageNode, }, }, - crate::driver::{ - DescriptorBindingMap, - compute::ComputePipeline, - format_aspect_mask, format_texel_block_extent, format_texel_block_size, - graphic::{DepthStencilInfo, GraphicsPipeline}, - image::{ImageInfo, ImageViewInfo, SampleCount}, - image_subresource_range_from_layers, - ray_trace::RayTracingPipeline, - render_pass::ResolveMode, - shader::PipelineDescriptorInfo, + crate::{ + cmd::{ClearColorValue, CommandRef}, + driver::{ + DescriptorBindingMap, + accel_struct::AccelerationStructureInfo, + buffer::BufferInfo, + compute::ComputePipeline, + format_aspect_mask, format_texel_block_extent, format_texel_block_size, + graphics::{DepthStencilInfo, GraphicsPipeline}, + image::{ImageInfo, ImageViewInfo, SampleCount}, + image_subresource_range_from_layers, + ray_tracing::RayTracingPipeline, + render_pass::ResolveMode, + shader::PipelineDescriptorInfo, + }, + driver::{ + accel_struct::AccelerationStructure, buffer::Buffer, image::Image, + swapchain::SwapchainImage, + }, + pool::Lease, + submission::Submission, }, ash::vk, + smallvec::SmallVec, std::{ cmp::Ord, collections::{BTreeMap, HashMap}, fmt::{Debug, Formatter}, + mem, ops::Range, ops::{Deref, DerefMut}, + slice::Iter, + sync::Arc, }, vk_sync::AccessType, }; -type ExecFn = Box; +#[cfg(feature = "checked")] +use std::sync::atomic::{AtomicU64, Ordering}; + +type CommandFn = Arc Fn(CommandRef<'a>) + Send + Sync>; +type CommandFnOnce = Box; type NodeIndex = usize; -type GraphId = u64; +#[cfg(feature = "checked")] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct GraphId(u64); #[cfg(feature = "checked")] -static NEXT_GRAPH_ID: AtomicU64 = AtomicU64::new(1); +impl GraphId { + fn next() -> Self { + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + + Self(NEXT_ID.fetch_add(1, Ordering::Relaxed)) + } +} #[derive(Debug)] -#[doc(hidden)] -pub enum AnyResource { +enum AnyResource { AccelerationStructure(Arc), + AccelerationStructureArg(AccelerationStructureInfo), AccelerationStructureLease(Arc>), Buffer(Arc), + BufferArg(BufferInfo), BufferLease(Arc>), Image(Arc), + ImageArg(ImageInfo), ImageLease(Arc>), SwapchainImage(Box), } +impl Clone for AnyResource { + fn clone(&self) -> Self { + match self { + Self::AccelerationStructure(resource) => { + Self::AccelerationStructure(Arc::clone(resource)) + } + Self::AccelerationStructureArg(info) => Self::AccelerationStructureArg(*info), + Self::AccelerationStructureLease(resource) => { + Self::AccelerationStructureLease(Arc::clone(resource)) + } + Self::Buffer(resource) => Self::Buffer(Arc::clone(resource)), + Self::BufferArg(info) => Self::BufferArg(*info), + Self::BufferLease(resource) => Self::BufferLease(Arc::clone(resource)), + Self::Image(resource) => Self::Image(Arc::clone(resource)), + Self::ImageArg(info) => Self::ImageArg(*info), + Self::ImageLease(resource) => Self::ImageLease(Arc::clone(resource)), + Self::SwapchainImage(resource) => { + Self::SwapchainImage(Box::new(unsafe { resource.to_detached() })) + } + } + } +} + macro_rules! any_resource_from_arc { ($name:ident) => { paste::paste! { @@ -131,7 +171,7 @@ impl AnyResource { Some(match self { Self::Image(resource) => resource, Self::ImageLease(resource) => resource, - Self::SwapchainImage(resource) => &resource.image, + Self::SwapchainImage(resource) => resource, _ => return None, }) } @@ -141,13 +181,43 @@ impl AnyResource { .expect("missing acceleration structure resource") } + pub(crate) fn expect_accel_struct_info( + &self, + ) -> crate::driver::accel_struct::AccelerationStructureInfo { + match self { + Self::AccelerationStructure(resource) => resource.info, + Self::AccelerationStructureArg(info) => *info, + Self::AccelerationStructureLease(resource) => resource.info, + _ => panic!("missing acceleration structure resource"), + } + } + fn expect_buffer(&self) -> &Buffer { self.as_buffer().expect("missing buffer resource") } + pub(crate) fn expect_buffer_info(&self) -> crate::driver::buffer::BufferInfo { + match self { + Self::Buffer(resource) => resource.info, + Self::BufferArg(info) => *info, + Self::BufferLease(resource) => resource.info, + _ => panic!("missing buffer resource"), + } + } + fn expect_image(&self) -> &Image { self.as_image().expect("missing image resource") } + + pub(crate) fn expect_image_info(&self) -> ImageInfo { + match self { + Self::Image(resource) => resource.info, + Self::ImageArg(info) => *info, + Self::ImageLease(resource) => resource.info, + Self::SwapchainImage(resource) => resource.info, + _ => panic!("missing image resource"), + } + } } #[derive(Clone, Copy, Debug)] @@ -169,7 +239,7 @@ impl Attachment { aspect_mask: image_view_info.aspect_mask, base_array_layer: image_view_info.base_array_layer, base_mip_level: image_view_info.base_mip_level, - format: image_view_info.fmt, + format: image_view_info.format, mip_level_count: image_view_info.mip_level_count, sample_count, target, @@ -179,7 +249,7 @@ impl Attachment { fn are_compatible(lhs: Option, rhs: Option) -> bool { // Two attachment references are compatible if they have matching format and sample // count, or are both VK_ATTACHMENT_UNUSED or the pointer that would contain the - // reference is NULL. + // reference is NULL let (Some(lhs), Some(rhs)) = (lhs, rhs) else { return true; }; @@ -202,153 +272,144 @@ impl Attachment { .into_builder() .array_layer_count(self.array_layer_count) .mip_level_count(self.mip_level_count) - .fmt(self.format) + .format(self.format) .into_image_view() .aspect_mask(self.aspect_mask) .base_array_layer(self.base_array_layer) .base_mip_level(self.base_mip_level) .build() } -} - -#[derive(Default)] -struct Execution { - accesses: HashMap>, - bindings: BTreeMap, - correlated_view_mask: u32, - depth_stencil: Option, - render_area: Option, - view_mask: u32, + fn remap_nodes(&mut self, node_map: &[NodeIndex]) { + self.target = node_map[self.target]; + } +} - color_attachments: HashMap, - color_clears: HashMap, - color_loads: HashMap, - color_resolves: HashMap, - color_stores: HashMap, - depth_stencil_attachment: Option, - depth_stencil_clear: Option<(Attachment, vk::ClearDepthStencilValue)>, - depth_stencil_load: Option, - depth_stencil_resolve: Option<( - Attachment, - AttachmentIndex, - Option, - Option, - )>, - depth_stencil_store: Option, - - func: Option, - pipeline: Option, +#[derive(Clone, Copy, Debug)] +struct ColorAttachment { + attachment: Attachment, + load: LoadOp<[f32; 4]>, + store: StoreOp, + resolve: Option, + is_input: bool, + is_attachment: bool, } -impl Debug for Execution { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - // The only field missing is func which cannot easily be implemented because it is a - // FnOnce. - f.debug_struct("Execution") - .field("accesses", &self.accesses) - .field("bindings", &self.bindings) - .field("depth_stencil", &self.depth_stencil) - .field("color_attachments", &self.color_attachments) - .field("color_clears", &self.color_clears) - .field("color_loads", &self.color_loads) - .field("color_resolves", &self.color_resolves) - .field("color_stores", &self.color_stores) - .field("depth_stencil_attachment", &self.depth_stencil_attachment) - .field("depth_stencil_clear", &self.depth_stencil_clear) - .field("depth_stencil_load", &self.depth_stencil_load) - .field("depth_stencil_resolve", &self.depth_stencil_resolve) - .field("depth_stencil_store", &self.depth_stencil_store) - .field("pipeline", &self.pipeline) - .finish() - } +#[derive(Clone, Debug, Default)] +struct ExecutionAttachmentMap { + color: Vec>, + depth_stencil: Option, } -struct ExecutionFunction(ExecFn); +impl ExecutionAttachmentMap { + fn color_attachment(&self, attachment_idx: AttachmentIndex) -> Option<&ColorAttachment> { + self.color + .get(attachment_idx as usize) + .and_then(|slot| slot.as_ref()) + } -#[derive(Clone, Debug)] -enum ExecutionPipeline { - Compute(ComputePipeline), - Graphic(GraphicsPipeline), - RayTrace(RayTracingPipeline), -} + fn color_attachment_mut( + &mut self, + attachment_idx: AttachmentIndex, + ) -> Option<&mut ColorAttachment> { + self.color + .get_mut(attachment_idx as usize) + .and_then(|slot| slot.as_mut()) + } -impl ExecutionPipeline { - fn as_graphic(&self) -> Option<&GraphicsPipeline> { - if let Self::Graphic(pipeline) = self { - Some(pipeline) - } else { - None - } + fn color_attachments(&self) -> impl Iterator + '_ { + self.color + .iter() + .enumerate() + .filter_map(|(attachment_idx, slot)| { + Some((attachment_idx as AttachmentIndex, slot.as_ref()?)) + }) } - fn bind_point(&self) -> vk::PipelineBindPoint { - match self { - ExecutionPipeline::Compute(_) => vk::PipelineBindPoint::COMPUTE, - ExecutionPipeline::Graphic(_) => vk::PipelineBindPoint::GRAPHICS, - ExecutionPipeline::RayTrace(_) => vk::PipelineBindPoint::RAY_TRACING_KHR, - } + fn depth_stencil_attachment(&self) -> Option<&DepthStencilAttachment> { + self.depth_stencil.as_ref() } - fn descriptor_bindings(&self) -> &DescriptorBindingMap { - match self { - ExecutionPipeline::Compute(pipeline) => &pipeline.inner.descriptor_bindings, - ExecutionPipeline::Graphic(pipeline) => &pipeline.inner.descriptor_bindings, - ExecutionPipeline::RayTrace(pipeline) => &pipeline.inner.descriptor_bindings, - } + fn depth_stencil_attachment_mut(&mut self) -> Option<&mut DepthStencilAttachment> { + self.depth_stencil.as_mut() } - fn descriptor_info(&self) -> &PipelineDescriptorInfo { - match self { - ExecutionPipeline::Compute(pipeline) => &pipeline.inner.descriptor_info, - ExecutionPipeline::Graphic(pipeline) => &pipeline.inner.descriptor_info, - ExecutionPipeline::RayTrace(pipeline) => &pipeline.inner.descriptor_info, + fn set_color_attachment( + &mut self, + attachment_idx: AttachmentIndex, + attachment: ColorAttachment, + ) { + let attachment_idx = attachment_idx as usize; + + if self.color.len() <= attachment_idx { + self.color.resize(attachment_idx + 1, None); } - } - fn expect_compute(&self) -> &ComputePipeline { - if let Self::Compute(pipeline) = self { - pipeline - } else { - panic!("missing compute pipeline") + #[cfg(feature = "checked")] + { + let existing_attachment = self.color[attachment_idx] + .as_ref() + .map(|&color| color.attachment); + + assert!( + Attachment::are_compatible(existing_attachment, Some(attachment.attachment)), + "incompatible with existing attachment" + ); } - } - fn expect_graphic(&self) -> &GraphicsPipeline { - self.as_graphic().expect("missing graphics pipeline") + self.color[attachment_idx] = Some(attachment); } - fn expect_ray_trace(&self) -> &RayTracingPipeline { - if let Self::RayTrace(pipeline) = self { - pipeline - } else { - panic!("missing ray tracing pipeline") + fn set_depth_stencil_attachment(&mut self, attachment: DepthStencilAttachment) { + #[cfg(feature = "checked")] + { + let existing_attachment = self + .depth_stencil + .as_ref() + .map(|&depth_stencil| depth_stencil.attachment); + + assert!( + Attachment::are_compatible(existing_attachment, Some(attachment.attachment)), + "incompatible with existing attachment" + ); } + + self.depth_stencil = Some(attachment); } - fn layout(&self) -> vk::PipelineLayout { - match self { - ExecutionPipeline::Compute(pipeline) => pipeline.inner.layout, - ExecutionPipeline::Graphic(pipeline) => pipeline.inner.layout, - ExecutionPipeline::RayTrace(pipeline) => pipeline.inner.layout, + fn remap_nodes(&mut self, node_map: &[NodeIndex]) { + for attachment in self.color.iter_mut().flatten() { + attachment.attachment.remap_nodes(node_map); + + if let Some(resolve) = &mut attachment.resolve { + resolve.attachment.remap_nodes(node_map); + } } - } - fn stage(&self) -> vk::PipelineStageFlags { - match self { - ExecutionPipeline::Compute(_) => vk::PipelineStageFlags::COMPUTE_SHADER, - ExecutionPipeline::Graphic(_) => vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, - ExecutionPipeline::RayTrace(_) => vk::PipelineStageFlags::RAY_TRACING_SHADER_KHR, + if let Some(attachment) = &mut self.depth_stencil { + attachment.attachment.remap_nodes(node_map); + + if let Some(resolve) = &mut attachment.resolve { + resolve.attachment.remap_nodes(node_map); + } } } } -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] +struct ColorResolve { + attachment: Attachment, + src_attachment_idx: AttachmentIndex, +} + +#[derive(Clone, Debug)] struct CommandData { execs: Vec, #[cfg(debug_assertions)] name: Option, + + stream_scope_id: Option, } impl CommandData { @@ -395,19 +456,355 @@ impl CommandData { fn name(&self) -> &str { const DEFAULT: &str = "command"; + #[cfg(debug_assertions)] + { + self.name.as_deref().unwrap_or(DEFAULT) + } + #[cfg(not(debug_assertions))] { DEFAULT } + } - #[cfg(debug_assertions)] - self.name.as_deref().unwrap_or(DEFAULT) + fn remap_nodes(&mut self, node_map: &[NodeIndex]) { + for exec in &mut self.execs { + exec.remap_nodes(node_map); + } + } +} + +enum CommandFunction { + Once(CommandFnOnce), + Reusable(CommandFn), +} + +impl CommandFunction { + fn is_reusable(&self) -> bool { + matches!(self, Self::Reusable(_)) + } + + fn record(self, cmd: CommandRef<'_>) -> Option { + match self { + Self::Once(func) => { + func(cmd); + None + } + Self::Reusable(func) => { + func(cmd); + Some(Self::Reusable(func)) + } + } } } +impl Clone for CommandFunction { + fn clone(&self) -> Self { + match self { + Self::Once(_) => panic!("one-shot command callback cannot be cloned"), + Self::Reusable(func) => Self::Reusable(Arc::clone(func)), + } + } +} + +#[derive(Clone, Copy, Debug)] +struct DepthStencilAttachment { + attachment: Attachment, + load: LoadOp, + store: StoreOp, + resolve: Option, + is_attachment: bool, +} + +#[derive(Clone, Copy, Debug)] +struct DepthStencilResolve { + attachment: Attachment, + dst_attachment_idx: AttachmentIndex, + depth_mode: Option, + stencil_mode: Option, +} + +#[derive(Clone)] +enum ExecutionAccess { + Building(ExecutionAccessBuilder), + Frozen(FrozenExecutionAccess), +} + +impl ExecutionAccess { + fn contains(&self, node_idx: NodeIndex) -> bool { + match self { + Self::Building(builder) => builder.lookup.contains_key(&node_idx), + Self::Frozen(frozen) => frozen.lookup.contains_key(&node_idx), + } + } + + fn freeze(&mut self) { + let Self::Building(builder) = mem::take(self) else { + return; + }; + + let ExecutionAccessBuilder { entries, lookup } = builder; + let entries = entries + .into_iter() + .map(|entry| NodeAccess { + node_idx: entry.node_idx, + accesses: entry.accesses.into_vec().into_boxed_slice(), + }) + .collect(); + + *self = Self::Frozen(FrozenExecutionAccess { entries, lookup }); + } + + fn get_mut(&mut self, node_idx: &NodeIndex) -> Option<&mut [SubresourceAccess]> { + let Self::Building(builder) = self else { + panic!("execution accesses are frozen") + }; + + builder + .lookup + .get(node_idx) + .copied() + .map(|entry_idx| builder.entries[entry_idx].accesses.as_mut_slice()) + } + + fn iter(&self) -> ExecutionAccessIter<'_> { + match self { + Self::Building(builder) => ExecutionAccessIter::Building(builder.entries.iter()), + Self::Frozen(frozen) => ExecutionAccessIter::Frozen(frozen.entries.iter()), + } + } + + fn push(&mut self, node_idx: NodeIndex, access: SubresourceAccess) { + let Self::Building(builder) = self else { + panic!("execution accesses are frozen") + }; + + let idx = *builder.lookup.entry(node_idx).or_insert_with(|| { + let idx = builder.entries.len(); + builder.entries.push(NodeAccessBuilder { + node_idx, + accesses: Default::default(), + }); + + idx + }); + builder.entries[idx].accesses.push(access); + } + + fn remap_nodes(&mut self, node_map: &[NodeIndex]) { + match self { + Self::Building(builder) => { + for entry in &mut builder.entries { + entry.node_idx = node_map[entry.node_idx]; + } + + builder.lookup = builder + .entries + .iter() + .enumerate() + .map(|(idx, entry)| (entry.node_idx, idx)) + .collect(); + } + Self::Frozen(frozen) => { + for entry in frozen.entries.iter_mut() { + entry.node_idx = node_map[entry.node_idx]; + } + + frozen.lookup = frozen + .entries + .iter() + .enumerate() + .map(|(idx, entry)| (entry.node_idx, idx)) + .collect(); + } + } + } +} + +impl Debug for ExecutionAccess { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Building(builder) => builder.entries.fmt(f), + Self::Frozen(frozen) => frozen.entries.fmt(f), + } + } +} + +impl Default for ExecutionAccess { + fn default() -> Self { + Self::Building(Default::default()) + } +} + +#[derive(Clone, Debug, Default)] +struct ExecutionAccessBuilder { + entries: Vec, + lookup: HashMap, +} + +enum ExecutionAccessIter<'a> { + Building(Iter<'a, NodeAccessBuilder>), + Frozen(Iter<'a, NodeAccess>), +} + +impl<'a> Iterator for ExecutionAccessIter<'a> { + type Item = (NodeIndex, &'a [SubresourceAccess]); + + fn next(&mut self) -> Option { + match self { + ExecutionAccessIter::Building(iter) => iter + .next() + .map(|entry| (entry.node_idx, entry.accesses.as_slice())), + ExecutionAccessIter::Frozen(iter) => iter + .next() + .map(|entry| (entry.node_idx, entry.accesses.as_ref())), + } + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + + (len, Some(len)) + } +} + +impl ExactSizeIterator for ExecutionAccessIter<'_> { + fn len(&self) -> usize { + match self { + ExecutionAccessIter::Building(iter) => iter.len(), + ExecutionAccessIter::Frozen(iter) => iter.len(), + } + } +} + +#[derive(Clone, Default)] +struct Execution { + accesses: ExecutionAccess, + attachments: ExecutionAttachmentMap, + bindings: BTreeMap, + + correlated_view_mask: u32, + depth_stencil: Option, + render_area: Option, + view_mask: u32, + + func: Option, + node_map: Option>, + pipeline: Option, + + #[cfg(feature = "checked")] + stream_graph_id: Option, +} + +impl Execution { + fn remap_nodes(&mut self, node_map: &[NodeIndex]) { + let original_node_map = Arc::<[NodeIndex]>::from(node_map.to_vec()); + self.accesses.remap_nodes(node_map); + self.attachments.remap_nodes(node_map); + + self.bindings = mem::take(&mut self.bindings) + .into_iter() + .map(|(binding, (node_idx, view))| (binding, (node_map[node_idx], view))) + .collect(); + self.node_map = Some(original_node_map); + } +} + +impl Debug for Execution { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // The only field missing is func which cannot easily be implemented because it is a + // FnOnce + f.debug_struct("Execution") + .field("accesses", &self.accesses) + .field("attachments", &self.attachments) + .field("bindings", &self.bindings) + .field("correlated_view_mask", &self.correlated_view_mask) + .field("depth_stencil", &self.depth_stencil) + .field("render_area", &self.render_area) + .field("view_mask", &self.view_mask) + .field("pipeline", &self.pipeline) + .finish() + } +} + +#[derive(Clone, Debug)] +enum ExecutionPipeline { + Compute(ComputePipeline), + Graphics(GraphicsPipeline), + RayTracing(RayTracingPipeline), +} + +impl ExecutionPipeline { + fn as_graphics(&self) -> Option<&GraphicsPipeline> { + if let Self::Graphics(pipeline) = self { + Some(pipeline) + } else { + None + } + } + + fn bind_point(&self) -> vk::PipelineBindPoint { + match self { + ExecutionPipeline::Compute(_) => vk::PipelineBindPoint::COMPUTE, + ExecutionPipeline::Graphics(_) => vk::PipelineBindPoint::GRAPHICS, + ExecutionPipeline::RayTracing(_) => vk::PipelineBindPoint::RAY_TRACING_KHR, + } + } + + fn descriptor_bindings(&self) -> &DescriptorBindingMap { + match self { + ExecutionPipeline::Compute(pipeline) => &pipeline.inner.descriptor_bindings, + ExecutionPipeline::Graphics(pipeline) => &pipeline.inner.descriptor_bindings, + ExecutionPipeline::RayTracing(pipeline) => &pipeline.inner.descriptor_bindings, + } + } + + fn descriptor_info(&self) -> &PipelineDescriptorInfo { + match self { + ExecutionPipeline::Compute(pipeline) => &pipeline.inner.descriptor_info, + ExecutionPipeline::Graphics(pipeline) => &pipeline.inner.descriptor_info, + ExecutionPipeline::RayTracing(pipeline) => &pipeline.inner.descriptor_info, + } + } + + fn expect_compute(&self) -> &ComputePipeline { + if let Self::Compute(pipeline) = self { + pipeline + } else { + panic!("missing compute pipeline") + } + } + + fn expect_graphics(&self) -> &GraphicsPipeline { + self.as_graphics().expect("missing graphics pipeline") + } + + fn expect_ray_tracing(&self) -> &RayTracingPipeline { + if let Self::RayTracing(pipeline) = self { + pipeline + } else { + panic!("missing ray tracing pipeline") + } + } + + fn layout(&self) -> vk::PipelineLayout { + match self { + ExecutionPipeline::Compute(pipeline) => pipeline.inner.layout, + ExecutionPipeline::Graphics(pipeline) => pipeline.inner.layout, + ExecutionPipeline::RayTracing(pipeline) => pipeline.inner.layout, + } + } +} + +#[derive(Clone, Debug)] +struct FrozenExecutionAccess { + entries: Box<[NodeAccess]>, + lookup: HashMap, +} + /// A composable graph of Vulkan command buffer operations. /// -/// `Graph` instances are are intended for one-time use. +/// `Graph` instances are intended for one-time use. /// /// The design of this code originated with a combination of /// [`PassBuilder`](https://github.com/EmbarkStudios/kajiya/blob/main/crates/lib/kajiya-rg/src/pass_builder.rs) @@ -429,7 +826,7 @@ impl Default for Graph { resources: Default::default(), #[cfg(feature = "checked")] - graph_id: NEXT_GRAPH_ID.fetch_add(1, Ordering::Relaxed), + graph_id: GraphId::next(), } } } @@ -469,7 +866,16 @@ impl Graph { resource.bind_graph(self) } - /// Copy an image, potentially performing format conversion. + pub(crate) fn bind_stream_arg_resource(&mut self, resource: AnyResource) -> NodeIndex { + self.resources.bind(resource) + } + + /// Copies an image, potentially performing format conversion. + /// + /// Records a [`vkCmdBlitImage`] operation covering the full extent of the source and + /// destination images. + /// + /// [`vkCmdBlitImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdBlitImage.html pub fn blit_image( &mut self, src: impl Into, @@ -477,10 +883,10 @@ impl Graph { filter: vk::Filter, ) -> &mut Self { let src = src.into(); - let src_info = self.resource(src).info; + let src_info = self.resources[src.index()].expect_image_info(); let dst = dst.into(); - let dst_info = self.resource(dst).info; + let dst_info = self.resources[dst.index()].expect_image_info(); self.blit_image_region( src, @@ -488,7 +894,7 @@ impl Graph { filter, [vk::ImageBlit { src_subresource: vk::ImageSubresourceLayers { - aspect_mask: format_aspect_mask(src_info.fmt), + aspect_mask: format_aspect_mask(src_info.format), mip_level: 0, base_array_layer: 0, layer_count: 1, @@ -502,7 +908,7 @@ impl Graph { }, ], dst_subresource: vk::ImageSubresourceLayers { - aspect_mask: format_aspect_mask(dst_info.fmt), + aspect_mask: format_aspect_mask(dst_info.format), mip_level: 0, base_array_layer: 0, layer_count: 1, @@ -519,7 +925,12 @@ impl Graph { ) } - /// Copy regions of an image, potentially performing format conversion. + /// Copies regions of an image, potentially performing format conversion. + /// + /// Records a [`vkCmdBlitImage`] operation. The caller supplies the Vulkan blit regions and + /// filter exactly as they will be passed to Vulkan. + /// + /// [`vkCmdBlitImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdBlitImage.html #[profiling::function] pub fn blit_image_region( &mut self, @@ -530,6 +941,7 @@ impl Graph { ) -> &mut Self { let src = src.into(); let dst = dst.into(); + let regions = Arc::<[vk::ImageBlit]>::from(regions.as_ref()); let mut cmd = self.begin_cmd().debug_name("blit image"); @@ -541,7 +953,7 @@ impl Graph { cmd.set_subresource_access(dst, dst_region, AccessType::TransferWrite); } - cmd.record_cmd(move |cmd| { + cmd.record_stream(move |cmd| { let src_image = cmd.resource(src).handle; let dst_image = cmd.resource(dst).handle; @@ -560,7 +972,12 @@ impl Graph { .end_cmd() } - /// Clear a color image. + /// Clears a color image. + /// + /// Records a [`vkCmdClearColorImage`] operation for the full image subresource range described + /// by the image's [`ImageInfo`]. + /// + /// [`vkCmdClearColorImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdClearColorImage.html #[profiling::function] pub fn clear_color_image( &mut self, @@ -569,12 +986,12 @@ impl Graph { ) -> &mut Self { let color = color.into().into(); let image = image.into(); - let image_view = self.resource(image).info.into(); + let image_view = self.resources[image.index()].expect_image_info().into(); self.begin_cmd() .debug_name("clear color") .subresource_access(image, image_view, AccessType::TransferWrite) - .record_cmd(move |cmd| { + .record_stream(move |cmd| { let image = cmd.resource(image); unsafe { @@ -591,6 +1008,11 @@ impl Graph { } /// Clears a depth/stencil image. + /// + /// Records a [`vkCmdClearDepthStencilImage`] operation for the full image subresource range + /// described by the image's [`ImageInfo`]. + /// + /// [`vkCmdClearDepthStencilImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdClearDepthStencilImage.html #[profiling::function] pub fn clear_depth_stencil_image( &mut self, @@ -599,12 +1021,12 @@ impl Graph { stencil: u32, ) -> &mut Self { let image = image.into(); - let image_view = self.resource(image).info.into(); + let image_view = self.resources[image.index()].expect_image_info().into(); self.begin_cmd() .debug_name("clear depth/stencil") .subresource_access(image, image_view, AccessType::TransferWrite) - .record_cmd(move |cmd| { + .record_stream(move |cmd| { let image = cmd.resource(image); unsafe { @@ -620,7 +1042,12 @@ impl Graph { .end_cmd() } - /// Copy data between buffers + /// Copies data between buffers. + /// + /// Records a [`vkCmdCopyBuffer`] operation covering the common size of the source and + /// destination buffers. + /// + /// [`vkCmdCopyBuffer`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyBuffer.html pub fn copy_buffer( &mut self, src: impl Into, @@ -628,8 +1055,8 @@ impl Graph { ) -> &mut Self { let src = src.into(); let dst = dst.into(); - let src_info = self.resource(src).info; - let dst_info = self.resource(dst).info; + let src_info = self.resources[src.index()].expect_buffer_info(); + let dst_info = self.resources[dst.index()].expect_buffer_info(); self.copy_buffer_region( src, @@ -642,7 +1069,12 @@ impl Graph { ) } - /// Copy data between buffer regions. + /// Copies data between buffer regions. + /// + /// Records a [`vkCmdCopyBuffer`] operation. The caller supplies the Vulkan copy regions exactly + /// as they will be passed to Vulkan. + /// + /// [`vkCmdCopyBuffer`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyBuffer.html #[profiling::function] pub fn copy_buffer_region( &mut self, @@ -652,16 +1084,17 @@ impl Graph { ) -> &mut Self { let src = src.into(); let dst = dst.into(); + let regions = Arc::<[vk::BufferCopy]>::from(regions.as_ref()); #[cfg(feature = "checked")] - let src_size = self.resource(src).info.size; + let src_size = self.resources[src.index()].expect_buffer_info().size; #[cfg(feature = "checked")] - let dst_size = self.resource(dst).info.size; + let dst_size = self.resources[dst.index()].expect_buffer_info().size; let mut cmd = self.begin_cmd().debug_name("copy buffer"); - for region in regions.as_ref() { + for region in regions.iter() { #[cfg(feature = "checked")] { assert!( @@ -688,26 +1121,30 @@ impl Graph { ); } - cmd.record_cmd(move |cmd| { + cmd.record_stream(move |cmd| { let src = cmd.resource(src); let dst = cmd.resource(dst); unsafe { cmd.device - .cmd_copy_buffer(cmd.handle, src.handle, dst.handle, regions.as_ref()); + .cmd_copy_buffer(cmd.handle, src.handle, dst.handle, ®ions); } }) .end_cmd() } - /// Copy data from a buffer into an image. + /// Copies data from a buffer into an image. + /// + /// Records a [`vkCmdCopyBufferToImage`] operation covering the full destination image. + /// + /// [`vkCmdCopyBufferToImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyBufferToImage.html pub fn copy_buffer_to_image( &mut self, src: impl Into, dst: impl Into, ) -> &mut Self { let dst = dst.into(); - let dst_info = self.resource(dst).info; + let dst_info = self.resources[dst.index()].expect_image_info(); self.copy_buffer_to_image_region( src, @@ -717,7 +1154,7 @@ impl Graph { buffer_row_length: dst_info.width, buffer_image_height: dst_info.height, image_subresource: vk::ImageSubresourceLayers { - aspect_mask: format_aspect_mask(dst_info.fmt), + aspect_mask: format_aspect_mask(dst_info.format), mip_level: 0, base_array_layer: 0, layer_count: 1, @@ -732,7 +1169,12 @@ impl Graph { ) } - /// Copy data from a buffer into an image. + /// Copies data from a buffer into image regions. + /// + /// Records a [`vkCmdCopyBufferToImage`] operation. The caller supplies the Vulkan copy regions + /// exactly as they will be passed to Vulkan. + /// + /// [`vkCmdCopyBufferToImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyBufferToImage.html #[profiling::function] pub fn copy_buffer_to_image_region( &mut self, @@ -742,13 +1184,14 @@ impl Graph { ) -> &mut Self { let src = src.into(); let dst = dst.into(); - let dst_info = self.resource(dst).info; + let dst_info = self.resources[dst.index()].expect_image_info(); + let regions = Arc::<[vk::BufferImageCopy]>::from(regions.as_ref()); let mut cmd = self.begin_cmd().debug_name("copy buffer to image"); - for region in regions.as_ref() { - let block_bytes_size = format_texel_block_size(dst_info.fmt); - let (block_height, block_width) = format_texel_block_extent(dst_info.fmt); + for region in regions.iter() { + let block_bytes_size = format_texel_block_size(dst_info.format); + let (block_height, block_width) = format_texel_block_extent(dst_info.format); let data_size = block_bytes_size * (region.buffer_row_length / block_width) * (region.buffer_image_height / block_height); @@ -765,7 +1208,7 @@ impl Graph { ); } - cmd.record_cmd(move |cmd| { + cmd.record_stream(move |cmd| { let src = cmd.resource(src); let dst = cmd.resource(dst); @@ -775,38 +1218,43 @@ impl Graph { src.handle, dst.handle, vk::ImageLayout::TRANSFER_DST_OPTIMAL, - regions.as_ref(), + ®ions, ); } }) .end_cmd() } - /// Copy all layers of a source image to a destination image. + /// Copies all layers of a source image to a destination image. + /// + /// Records a [`vkCmdCopyImage`] operation covering the common extent of the source and + /// destination images. + /// + /// [`vkCmdCopyImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyImage.html pub fn copy_image( &mut self, src: impl Into, dst: impl Into, ) -> &mut Self { let src = src.into(); - let src_info = self.resource(src).info; + let src_info = self.resources[src.index()].expect_image_info(); let dst = dst.into(); - let dst_info = self.resource(dst).info; + let dst_info = self.resources[dst.index()].expect_image_info(); self.copy_image_region( src, dst, [vk::ImageCopy { src_subresource: vk::ImageSubresourceLayers { - aspect_mask: format_aspect_mask(src_info.fmt), + aspect_mask: format_aspect_mask(src_info.format), mip_level: 0, base_array_layer: 0, layer_count: src_info.array_layer_count, }, src_offset: vk::Offset3D { x: 0, y: 0, z: 0 }, dst_subresource: vk::ImageSubresourceLayers { - aspect_mask: format_aspect_mask(dst_info.fmt), + aspect_mask: format_aspect_mask(dst_info.format), mip_level: 0, base_array_layer: 0, layer_count: src_info.array_layer_count, @@ -821,7 +1269,12 @@ impl Graph { ) } - /// Copy data between images. + /// Copies data between image regions. + /// + /// Records a [`vkCmdCopyImage`] operation. The caller supplies the Vulkan copy regions exactly + /// as they will be passed to Vulkan. + /// + /// [`vkCmdCopyImage`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyImage.html #[profiling::function] pub fn copy_image_region( &mut self, @@ -831,10 +1284,11 @@ impl Graph { ) -> &mut Self { let src = src.into(); let dst = dst.into(); + let regions = Arc::<[vk::ImageCopy]>::from(regions.as_ref()); let mut cmd = self.begin_cmd().debug_name("copy image"); - for region in regions.as_ref() { + for region in regions.iter() { cmd.set_subresource_access( src, image_subresource_range_from_layers(region.src_subresource), @@ -847,7 +1301,7 @@ impl Graph { ); } - cmd.record_cmd(move |cmd| { + cmd.record_stream(move |cmd| { let src = cmd.resource(src); let dst = cmd.resource(dst); @@ -858,14 +1312,18 @@ impl Graph { vk::ImageLayout::TRANSFER_SRC_OPTIMAL, dst.handle, vk::ImageLayout::TRANSFER_DST_OPTIMAL, - regions.as_ref(), + ®ions, ); } }) .end_cmd() } - /// Copy image data into a buffer. + /// Copies image data into a buffer. + /// + /// Records a [`vkCmdCopyImageToBuffer`] operation covering the full source image. + /// + /// [`vkCmdCopyImageToBuffer`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyImageToBuffer.html pub fn copy_image_to_buffer( &mut self, src: impl Into, @@ -874,7 +1332,7 @@ impl Graph { let src = src.into(); let dst = dst.into(); - let src_info = self.resource(src).info; + let src_info = self.resources[src.index()].expect_image_info(); self.copy_image_to_buffer_region( src, @@ -884,7 +1342,7 @@ impl Graph { buffer_row_length: src_info.width, buffer_image_height: src_info.height, image_subresource: vk::ImageSubresourceLayers { - aspect_mask: format_aspect_mask(src_info.fmt), + aspect_mask: format_aspect_mask(src_info.format), mip_level: 0, base_array_layer: 0, layer_count: 1, @@ -899,7 +1357,12 @@ impl Graph { ) } - /// Copy image data into a buffer. + /// Copies image region data into a buffer. + /// + /// Records a [`vkCmdCopyImageToBuffer`] operation. The caller supplies the Vulkan copy regions + /// exactly as they will be passed to Vulkan. + /// + /// [`vkCmdCopyImageToBuffer`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdCopyImageToBuffer.html #[profiling::function] pub fn copy_image_to_buffer_region( &mut self, @@ -908,14 +1371,15 @@ impl Graph { regions: impl AsRef<[vk::BufferImageCopy]> + 'static + Send, ) -> &mut Self { let src = src.into(); - let src_info = self.resource(src).info; + let src_info = self.resources[src.index()].expect_image_info(); let dst = dst.into(); + let regions = Arc::<[vk::BufferImageCopy]>::from(regions.as_ref()); let mut cmd = self.begin_cmd().debug_name("copy image to buffer"); - for region in regions.as_ref() { - let block_bytes_size = format_texel_block_size(src_info.fmt); - let (block_height, block_width) = format_texel_block_extent(src_info.fmt); + for region in regions.iter() { + let block_bytes_size = format_texel_block_size(src_info.format); + let (block_height, block_width) = format_texel_block_extent(src_info.format); let data_size = block_bytes_size * (region.buffer_row_length / block_width) * (region.buffer_image_height / block_height); @@ -932,7 +1396,7 @@ impl Graph { ); } - cmd.record_cmd(move |cmd| { + cmd.record_stream(move |cmd| { let src = cmd.resource(src); let dst = cmd.resource(dst); @@ -942,14 +1406,18 @@ impl Graph { src.handle, vk::ImageLayout::TRANSFER_SRC_OPTIMAL, dst.handle, - regions.as_ref(), + ®ions, ); } }) .end_cmd() } - /// Fill a region of a buffer with a fixed value. + /// Fills a region of a buffer with a fixed value. + /// + /// Records a [`vkCmdFillBuffer`] operation. + /// + /// [`vkCmdFillBuffer`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkCmdFillBuffer.html pub fn fill_buffer( &mut self, buffer: impl Into, @@ -961,7 +1429,7 @@ impl Graph { self.begin_cmd() .debug_name("fill buffer") .subresource_access(buffer, region.clone(), AccessType::TransferWrite) - .record_cmd(move |cmd| { + .record_stream(move |cmd| { let buffer = cmd.resource(buffer); unsafe { @@ -977,7 +1445,7 @@ impl Graph { .end_cmd() } - /// Returns the index of the first pass which accesses a given node + /// Returns the index of the first command which accesses a given node. #[profiling::function] fn first_node_access_pass_index(&self, resource_node: impl Node) -> Option { self.assert_node_owner(&resource_node); @@ -986,7 +1454,7 @@ impl Graph { for (pass_idx, pass) in self.cmds.iter().enumerate() { for exec in pass.execs.iter() { - if exec.accesses.contains_key(&node_idx) { + if exec.accesses.contains(node_idx) { return Some(pass_idx); } } @@ -998,19 +1466,37 @@ impl Graph { /// Finalizes the graph and provides an object with functions for submitting the resulting /// commands. #[profiling::function] - pub fn into_submission(mut self) -> Submission { - // The final execution of each pass has no function + pub fn finalize(mut self) -> Submission { + // The final execution of each command has no function. for cmd in &mut self.cmds { debug_assert!(cmd.expect_last_exec().func.is_none()); cmd.execs.pop(); + + for exec in &mut cmd.execs { + exec.accesses.freeze(); + } } Submission::new(self) } - /// Returns a borrow of the original Vulkan resource (buffer, image or acceleration structure) - /// which the given bound resource node represents. + /// Returns a borrow of the Vulkan resource represented by `resource_node`. + /// + /// The exact return type depends on the node type: + /// + /// - Concrete nodes such as [`BufferNode`] and [`ImageNode`] return the exact stored handle + /// type, such as + /// `&Arc` or `&Arc`. + /// - Erased nodes such as [`AnyBufferNode`] and [`AnyImageNode`] return a borrow of the + /// underlying resource, + /// such as `&Buffer` or `&Image`. + /// + /// This distinction lets erased node enums unify owned, leased, and swapchain-backed resources + /// behind a single resource view. + /// + /// Node ownership is validated here when the `checked` feature is enabled. With `checked` + /// disabled, callers must ensure `resource_node` came from this graph. pub fn resource(&self, resource_node: N) -> &N::Resource where N: Node, @@ -1047,7 +1533,7 @@ impl Graph { data.as_ref().len() ); - let buffer_info = self.resource(buffer).info; + let buffer_info = self.resources[buffer.index()].expect_buffer_info(); assert!( data_end <= buffer_info.size, @@ -1056,43 +1542,97 @@ impl Graph { ); } + let data = Arc::<[u8]>::from(data.as_ref()); + self.begin_cmd() .debug_name("update buffer") .subresource_access(buffer, offset..data_end, AccessType::TransferWrite) - .record_cmd(move |cmd| { + .record_stream(move |cmd| { let buffer = cmd.resource(buffer); unsafe { cmd.device - .cmd_update_buffer(cmd.handle, buffer.handle, offset, data.as_ref()); + .cmd_update_buffer(cmd.handle, buffer.handle, offset, &data); } }) .end_cmd() } } +/// Specifies the state of a color or combined depth and stencil attachment image during graphics +/// render pass framebuffer load operations. +/// +/// Use this to specify the desired contents of any image before use in a pipeline command buffer. +#[derive(Clone, Copy, Debug)] +pub enum LoadOp { + /// Clears the attachment. + /// + /// `T` will be [ClearColorValue] for color images or [vk::ClearDepthStencilValue] for + /// combined depth and stencil images. + Clear(T), + + /// The attachment will become undefined and reads will produce garbage data. + DontCare, + + /// The attachment will be preserved in memory. + Load, +} + /// A Vulkan resource which has been bound to a [`Graph`]. /// /// See [`Graph::bind_resource`]. /// /// This trait is sealed and cannot be implemented outside of `vk-graph`. -pub trait Node: private::Sealed { - /// The Vulkan buffer, image, or acceleration struction type. +#[allow(private_bounds)] +pub trait Node: private::NodeSealed { + /// The Vulkan buffer, image, or acceleration structure type. type Resource; - #[doc(hidden)] - fn borrow(self, resources: &[AnyResource]) -> &Self::Resource; + /// Synchronization state snapshot returned for this node's resource type. + type SyncInfo; #[doc(hidden)] - fn index(&self) -> NodeIndex; + fn index(&self) -> usize; +} - #[doc(hidden)] - fn assert_owner(&self, _graph_id: GraphId) {} +#[derive(Clone, Debug)] +struct NodeAccess { + node_idx: NodeIndex, + accesses: Box<[SubresourceAccess]>, +} + +#[derive(Clone, Debug)] +struct NodeAccessBuilder { + node_idx: NodeIndex, + accesses: SmallVec<[SubresourceAccess; 2]>, } mod private { - /// Prevents external implementations of [`Node`] and [`Resource`]. - pub trait Sealed {} + use super::{AnyResource, Node}; + + #[cfg(feature = "checked")] + use super::GraphId; + + /// Prevents external implementations of [`Node`] and provides crate-private node internals. + pub(crate) trait NodeSealed: Sized { + fn borrow(self, resources: &[AnyResource]) -> &::Resource + where + Self: Node; + + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource + where + Self: Node, + { + debug_assert_eq!(self.index(), index); + self.borrow(resources) + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, _graph_id: GraphId) {} + } + + /// Prevents external implementations of [`Resource`](super::Resource). + pub(crate) trait ResourceSealed {} } /// A Vulkan resource which may be bound to a [`Graph`]. @@ -1101,7 +1641,8 @@ mod private { /// [`Command::bind_resource`](crate::cmd::Command::bind_resource). /// /// This trait is sealed and cannot be implemented outside of `vk-graph`. -pub trait Resource: private::Sealed { +#[allow(private_bounds)] +pub trait Resource: private::ResourceSealed { /// The resource handle type. type Node; @@ -1109,7 +1650,7 @@ pub trait Resource: private::Sealed { fn bind_graph(self, _: &mut Graph) -> Self::Node; } -impl private::Sealed for SwapchainImage {} +impl private::ResourceSealed for SwapchainImage {} impl Resource for SwapchainImage { type Node = SwapchainImageNode; @@ -1133,19 +1674,19 @@ impl Resource for SwapchainImage { macro_rules! resource { ($name:ident) => { paste::paste! { - impl private::Sealed for $name {} - impl private::Sealed for Arc<$name> {} - impl<'a> private::Sealed for &'a Arc<$name> {} - impl private::Sealed for Lease<$name> {} - impl private::Sealed for Arc> {} - impl<'a> private::Sealed for &'a Arc> {} + impl private::ResourceSealed for $name {} + impl private::ResourceSealed for Arc<$name> {} + impl<'a> private::ResourceSealed for &'a Arc<$name> {} + impl private::ResourceSealed for Lease<$name> {} + impl private::ResourceSealed for Arc> {} + impl<'a> private::ResourceSealed for &'a Arc> {} impl Resource for $name { type Node = [<$name Node>]; #[profiling::function] fn bind_graph(self, graph: &mut Graph) -> Self::Node { - // In this function we are resource a new item (Image or Buffer or etc) + // Bind a new owned resource, such as Image or Buffer. // We will return a new node Arc::new(self).bind_graph(graph) @@ -1157,8 +1698,7 @@ macro_rules! resource { #[profiling::function] fn bind_graph(self, graph: &mut Graph) -> Self::Node { - // In this function we are resource an existing resource (Arc or - // Arc or etc) + // Bind an existing shared resource, such as Arc or Arc. // We will return an existing node, if possible Self::Node::new( @@ -1173,8 +1713,7 @@ macro_rules! resource { type Node = [<$name Node>]; fn bind_graph(self, graph: &mut Graph) -> Self::Node { - // In this function we are resource a borrowed resource (&Arc or - // &Arc or etc) + // Bind a borrowed shared resource, such as &Arc or &Arc. Arc::clone(self).bind_graph(graph) } @@ -1185,8 +1724,7 @@ macro_rules! resource { #[profiling::function] fn bind_graph(self, graph: &mut Graph) -> Self::Node { - // In this function we are wrapping a newly pooled resource (Lease or - // Lease or etc) + // Bind a new pooled resource, such as Lease or Lease. // We will return a new node Arc::new(self).bind_graph(graph) @@ -1198,8 +1736,8 @@ macro_rules! resource { #[profiling::function] fn bind_graph(self, graph: &mut Graph) -> Self::Node { - // In this function we are wrapping an existing pooled resource - // (Arc> or Arc> or etc) + // Bind an existing shared pooled resource, such as Arc> or + // Arc>. // We will return an existing node, if possible Self::Node::new( @@ -1214,8 +1752,8 @@ macro_rules! resource { type Node = [<$name LeaseNode>]; fn bind_graph(self, graph: &mut Graph) -> Self::Node { - // In this function we are resource a borrowed resource (&Arc> or - // &Arc> or etc) + // Bind a borrowed shared pooled resource, such as &Arc> or + // &Arc>. Arc::clone(self).bind_graph(graph) } @@ -1235,6 +1773,13 @@ struct ResourceMap { } impl ResourceMap { + pub(crate) fn from_resources(resources: Vec) -> Self { + Self { + addr_index: HashMap::new(), + resources, + } + } + fn bind(&mut self, resource: AnyResource) -> NodeIndex { let node_idx = self.resources.len(); self.resources.push(resource); @@ -1271,6 +1816,19 @@ impl DerefMut for ResourceMap { } } +/// Specifies the state of a color or combined depth and stencil attachment image after graphics +/// render pass framebuffer store operations. +/// +/// Use this to specify the desired contents of any image after use in a pipeline command buffer. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum StoreOp { + /// The attachment will become undefined and reads will produce garbage data. + DontCare, + + /// The attachment will be preserved in memory. + Store, +} + #[cfg(test)] mod test { use std::sync::Arc; @@ -1434,7 +1992,7 @@ mod test { let mut shared_accels = SharedNodes::::default(); let mut shared_accel_leases = SharedNodes::>::default(); - let accel_supported = device.physical_device.accel_struct_properties.is_some(); + let accel_supported = device.physical.vk_khr_acceleration_structure.is_some(); let mut resource_kinds = vec![ ResourceKind::OwnedBuffer, diff --git a/src/node.rs b/src/node.rs index 8c79f49f..5d75efd4 100644 --- a/src/node.rs +++ b/src/node.rs @@ -2,7 +2,12 @@ //! //! When you bind a resource to a [`Graph`](crate::Graph), you get back a node handle: //! -//! ```ignore +//! ```no_run +//! # use std::sync::Arc; +//! # use vk_graph::{Graph, driver::{buffer::Buffer, image::Image}, node::{BufferNode, ImageNode}}; +//! # let mut graph = Graph::new(); +//! # let my_buffer: Arc = todo!(); +//! # let my_image: Arc = todo!(); //! let buf_node: BufferNode = graph.bind_resource(my_buffer); //! let img_node: ImageNode = graph.bind_resource(my_image); //! ``` @@ -24,63 +29,91 @@ //! //! For most users, [`BufferNode`] and [`ImageNode`] are all you need. The `Lease` and //! `Any*` variants exist for advanced pooling and dynamic dispatch scenarios. +//! +//! When borrowing resources back out of a graph with [`Graph::resource`](crate::Graph::resource), +//! concrete node types return the exact stored handle type, while `Any*` node types return a +//! borrow of the underlying resource. For example, `BufferNode` yields `&Arc`, but +//! `AnyBufferNode` yields `&Buffer`. use std::sync::Arc; use crate::{ - GraphId, Node, + Node, driver::{ - accel_struct::AccelerationStructure, buffer::Buffer, image::Image, + accel_struct::{AccelerationStructure, AccelerationStructureSyncInfo}, + buffer::{Buffer, BufferSyncInfo}, + image::{Image, ImageSyncInfo}, swapchain::SwapchainImage, }, pool::Lease, private, + stream::{AccelerationStructureArg, BufferArg, ImageArg}, }; +#[cfg(feature = "checked")] +use crate::GraphId; + use super::{AnyResource, NodeIndex}; /// Specifies either an owned acceleration structure or one obtained from a pool. #[derive(Clone, Copy, Debug)] pub enum AnyAccelerationStructureNode { + /// An acceleration structure supplied as a command stream argument. + Arg(AccelerationStructureArg), + /// An owned acceleration structure. - AccelerationStructure(AccelerationStructureNode), + Owned(AccelerationStructureNode), /// An acceleration structure obtained from a pool. - AccelerationStructureLease(AccelerationStructureLeaseNode), + Pooled(AccelerationStructureLeaseNode), } impl From for AnyAccelerationStructureNode { fn from(node: AccelerationStructureNode) -> Self { - Self::AccelerationStructure(node) + Self::Owned(node) + } +} + +impl From for AnyAccelerationStructureNode { + fn from(node: AccelerationStructureArg) -> Self { + Self::Arg(node) } } impl From for AnyAccelerationStructureNode { fn from(node: AccelerationStructureLeaseNode) -> Self { - Self::AccelerationStructureLease(node) + Self::Pooled(node) } } -impl private::Sealed for AnyAccelerationStructureNode {} - -impl Node for AnyAccelerationStructureNode { - type Resource = AccelerationStructure; - - fn borrow(self, resources: &[AnyResource]) -> &Self::Resource { +impl private::NodeSealed for AnyAccelerationStructureNode { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { resources[self.index()].expect_accel_struct() } - fn index(&self) -> NodeIndex { + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource { + resources[index].expect_accel_struct() + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, graph_id: GraphId) { match self { - Self::AccelerationStructure(node) => node.index(), - Self::AccelerationStructureLease(node) => node.index(), + Self::Arg(node) => node.assert_owner(graph_id), + Self::Owned(node) => node.assert_owner(graph_id), + Self::Pooled(node) => node.assert_owner(graph_id), } } +} - fn assert_owner(&self, graph_id: GraphId) { +impl Node for AnyAccelerationStructureNode { + type Resource = AccelerationStructure; + type SyncInfo = AccelerationStructureSyncInfo; + + fn index(&self) -> usize { match self { - Self::AccelerationStructure(node) => node.assert_owner(graph_id), - Self::AccelerationStructureLease(node) => node.assert_owner(graph_id), + Self::Arg(node) => node.index(), + Self::Owned(node) => node.index(), + Self::Pooled(node) => node.index(), } } } @@ -88,45 +121,62 @@ impl Node for AnyAccelerationStructureNode { /// Specifies either an owned buffer or one obtained from a pool. #[derive(Clone, Copy, Debug)] pub enum AnyBufferNode { + /// A buffer supplied as a command stream argument. + Arg(BufferArg), + /// An owned buffer. - Buffer(BufferNode), + Owned(BufferNode), /// A buffer obtained from a pool. - BufferLease(BufferLeaseNode), + Pooled(BufferLeaseNode), } impl From for AnyBufferNode { fn from(node: BufferNode) -> Self { - Self::Buffer(node) + Self::Owned(node) + } +} + +impl From for AnyBufferNode { + fn from(node: BufferArg) -> Self { + Self::Arg(node) } } impl From for AnyBufferNode { fn from(node: BufferLeaseNode) -> Self { - Self::BufferLease(node) + Self::Pooled(node) } } -impl private::Sealed for AnyBufferNode {} - -impl Node for AnyBufferNode { - type Resource = Buffer; - - fn borrow(self, resources: &[AnyResource]) -> &Self::Resource { +impl private::NodeSealed for AnyBufferNode { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { resources[self.index()].expect_buffer() } - fn index(&self) -> NodeIndex { + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource { + resources[index].expect_buffer() + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, graph_id: GraphId) { match self { - Self::Buffer(node) => node.index(), - Self::BufferLease(node) => node.index(), + Self::Arg(node) => node.assert_owner(graph_id), + Self::Owned(node) => node.assert_owner(graph_id), + Self::Pooled(node) => node.assert_owner(graph_id), } } +} - fn assert_owner(&self, graph_id: GraphId) { +impl Node for AnyBufferNode { + type Resource = Buffer; + type SyncInfo = BufferSyncInfo; + + fn index(&self) -> usize { match self { - Self::Buffer(node) => node.assert_owner(graph_id), - Self::BufferLease(node) => node.assert_owner(graph_id), + Self::Arg(node) => node.index(), + Self::Owned(node) => node.index(), + Self::Pooled(node) => node.index(), } } } @@ -136,62 +186,114 @@ impl Node for AnyBufferNode { /// The image may also be a special swapchain type of image. #[derive(Clone, Copy, Debug)] pub enum AnyImageNode { + /// An image supplied as a command stream argument. + Arg(ImageArg), + /// An owned image. - Image(ImageNode), + Owned(ImageNode), /// An image obtained from a pool. - ImageLease(ImageLeaseNode), + Pooled(ImageLeaseNode), /// A special swapchain image. - SwapchainImage(SwapchainImageNode), + Swapchain(SwapchainImageNode), } impl From for AnyImageNode { fn from(node: ImageNode) -> Self { - Self::Image(node) + Self::Owned(node) + } +} + +impl From for AnyImageNode { + fn from(node: ImageArg) -> Self { + Self::Arg(node) } } impl From for AnyImageNode { fn from(node: ImageLeaseNode) -> Self { - Self::ImageLease(node) + Self::Pooled(node) } } impl From for AnyImageNode { fn from(node: SwapchainImageNode) -> Self { - Self::SwapchainImage(node) + Self::Swapchain(node) } } -impl private::Sealed for AnyImageNode {} - -impl Node for AnyImageNode { - type Resource = Image; - - fn borrow(self, resources: &[AnyResource]) -> &Self::Resource { +impl private::NodeSealed for AnyImageNode { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { resources[self.index()].expect_image() } - fn index(&self) -> NodeIndex { + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource { + resources[index].expect_image() + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, graph_id: GraphId) { match self { - Self::Image(node) => node.index(), - Self::ImageLease(node) => node.index(), - Self::SwapchainImage(node) => node.index(), + Self::Arg(node) => node.assert_owner(graph_id), + Self::Owned(node) => node.assert_owner(graph_id), + Self::Pooled(node) => node.assert_owner(graph_id), + Self::Swapchain(node) => node.assert_owner(graph_id), } } +} - fn assert_owner(&self, graph_id: GraphId) { +impl Node for AnyImageNode { + type Resource = Image; + type SyncInfo = ImageSyncInfo; + + fn index(&self) -> usize { match self { - Self::Image(node) => node.assert_owner(graph_id), - Self::ImageLease(node) => node.assert_owner(graph_id), - Self::SwapchainImage(node) => node.assert_owner(graph_id), + Self::Arg(node) => node.index(), + Self::Owned(node) => node.index(), + Self::Pooled(node) => node.index(), + Self::Swapchain(node) => node.index(), } } } +/// A type-erased graph node for any buffer, image, or acceleration structure. +#[derive(Clone, Copy, Debug)] +pub enum AnyNode { + /// An acceleration-structure node. + AccelerationStructure(AnyAccelerationStructureNode), + + /// A buffer node. + Buffer(AnyBufferNode), + + /// An image node, including swapchain image nodes. + Image(AnyImageNode), +} + +macro_rules! any_node_from { + ($node:ty => $variant:ident) => { + impl From<$node> for AnyNode { + fn from(node: $node) -> Self { + Self::$variant(node.into()) + } + } + }; +} + +any_node_from!(AnyAccelerationStructureNode => AccelerationStructure); +any_node_from!(AnyBufferNode => Buffer); +any_node_from!(AnyImageNode => Image); + +any_node_from!(AccelerationStructureNode => AccelerationStructure); +any_node_from!(AccelerationStructureLeaseNode => AccelerationStructure); +any_node_from!(BufferNode => Buffer); +any_node_from!(BufferLeaseNode => Buffer); +any_node_from!(ImageNode => Image); +any_node_from!(ImageLeaseNode => Image); +any_node_from!(SwapchainImageNode => Image); + macro_rules! node { - ($name:ident, $resource:ty, $fn_name:ident) => { + ($name:ident, $resource:ty, $sync_info:ty, $fn_name:ident) => { paste::paste! { /// A graph-local handle for a bound resource. /// @@ -199,6 +301,9 @@ macro_rules! node { /// /// When the `checked` feature is enabled, using a node with a different graph will /// panic immediately. + /// + /// When `checked` is disabled, this ownership check is skipped for zero-overhead + /// builds, so cross-graph node misuse is invalid and may resolve to the wrong resource. #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct [<$name Node>] { index: NodeIndex, @@ -221,12 +326,8 @@ macro_rules! node { } } - impl private::Sealed for [<$name Node>] {} - - impl Node for [<$name Node>] { - type Resource = $resource; - - fn borrow(self, resources: &[AnyResource]) -> &Self::Resource { + impl private::NodeSealed for [<$name Node>] { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { let AnyResource::$name(res) = &resources[self.index] else { panic!("invalid resource node handle"); }; @@ -234,15 +335,33 @@ macro_rules! node { res } - fn index(&self) -> NodeIndex { - self.index + fn borrow_at( + self, + resources: &[AnyResource], + index: usize, + ) -> &::Resource { + let AnyResource::$name(res) = &resources[index] else { + panic!("invalid resource node handle"); + }; + + res } + #[cfg(feature = "checked")] fn assert_owner(&self, _graph_id: GraphId) { #[cfg(feature = "checked")] assert!(self.graph_id == _graph_id, "node belongs to a different graph"); } } + + impl Node for [<$name Node>] { + type Resource = $resource; + type SyncInfo = $sync_info; + + fn index(&self) -> usize { + self.index + } + } } }; } @@ -250,15 +369,22 @@ macro_rules! node { node!( AccelerationStructure, Arc, + AccelerationStructureSyncInfo, as_accel_struct ); node!( AccelerationStructureLease, Arc>, + AccelerationStructureSyncInfo, as_accel_struct ); -node!(Buffer, Arc, as_buffer); -node!(BufferLease, Arc>, as_buffer); -node!(Image, Arc, as_image); -node!(ImageLease, Arc>, as_image); -node!(SwapchainImage, SwapchainImage, as_swapchain_image); +node!(Buffer, Arc, BufferSyncInfo, as_buffer); +node!(BufferLease, Arc>, BufferSyncInfo, as_buffer); +node!(Image, Arc, ImageSyncInfo, as_image); +node!(ImageLease, Arc>, ImageSyncInfo, as_image); +node!( + SwapchainImage, + SwapchainImage, + ImageSyncInfo, + as_swapchain_image +); diff --git a/src/pool/cache.rs b/src/pool/cache.rs index 67693da9..98d35de6 100644 --- a/src/pool/cache.rs +++ b/src/pool/cache.rs @@ -1,7 +1,9 @@ //! Pool wrapper which enables memory-efficient resource caching. use { - super::{Lease, Pool}, + super::{ + BufferHostMappingCompatibility, Lease, Pool, compatible_buffer_info, compatible_image_info, + }, crate::driver::{ DriverError, accel_struct::{AccelerationStructure, AccelerationStructureInfo}, @@ -106,7 +108,7 @@ where profiling::scope!("check aliases"); for (item_info, item) in &state.accel_structs { - if item_info.ty == info.ty + if item_info.acceleration_structure_type == info.acceleration_structure_type && item_info.size >= info.size && let Some(item) = item.upgrade() { @@ -148,12 +150,7 @@ where profiling::scope!("check aliases"); for (item_info, item) in &state.buffers { - if (item_info.dedicated & info.dedicated) == info.dedicated - && item_info.host_read == info.host_read - && item_info.host_write == info.host_write - && item_info.alignment >= info.alignment - && item_info.size >= info.size - && item_info.usage.contains(info.usage) + if compatible_buffer_info(item_info, &info, BufferHostMappingCompatibility::Exact) && let Some(item) = item.upgrade() { result = Some(item); @@ -194,18 +191,7 @@ where profiling::scope!("check aliases"); for (item_info, item) in &state.images { - if item_info.array_layer_count == info.array_layer_count - && item_info.dedicated == info.dedicated - && item_info.depth == info.depth - && item_info.fmt == info.fmt - && item_info.height == info.height - && item_info.mip_level_count == info.mip_level_count - && item_info.sample_count == info.sample_count - && item_info.tiling == info.tiling - && item_info.ty == info.ty - && item_info.width == info.width - && item_info.flags.contains(info.flags) - && item_info.usage.contains(info.usage) + if compatible_image_info(item_info, &info) && let Some(item) = item.upgrade() { result = Some(item); @@ -268,8 +254,9 @@ where } } +#[allow(private_bounds)] #[doc(hidden)] -pub trait TaggedCacheResource: Sized { +pub trait TaggedCacheResource: cache_private::TaggedCacheResourceSealed + Sized { type Item; fn resource( @@ -282,6 +269,16 @@ pub trait TaggedCacheResource: Sized { T: Pool; } +mod cache_private { + pub trait TaggedCacheResourceSealed {} +} + +impl cache_private::TaggedCacheResourceSealed for AccelerationStructureInfo {} + +impl cache_private::TaggedCacheResourceSealed for BufferInfo {} + +impl cache_private::TaggedCacheResourceSealed for ImageInfo {} + impl TaggedCacheResource for AccelerationStructureInfo where Tag: Eq + Hash + Clone, diff --git a/src/pool/fifo.rs b/src/pool/fifo.rs index e3a10363..471c9e48 100644 --- a/src/pool/fifo.rs +++ b/src/pool/fifo.rs @@ -1,7 +1,10 @@ //! Pool which requests from a single bucket per resource type. use { - super::{Cache, Lease, Pool, PoolConfig, lease_command_buffer, with_cache}, + super::{ + BufferHostMappingCompatibility, Cache, Lease, Pool, PoolConfig, compatible_buffer_info, + with_cache, + }, crate::driver::{ DriverError, accel_struct::{AccelerationStructure, AccelerationStructureInfo}, @@ -128,7 +131,9 @@ impl Pool for FifoPool { // Look for a compatible acceleration structure (big enough and same type) for idx in 0..cache.len() { let item = unsafe { cache.get_unchecked(idx) }; - if item.info.size >= info.size && item.info.ty == info.ty { + if item.info.size >= info.size + && item.info.acceleration_structure_type == info.acceleration_structure_type + { let item = cache.swap_remove(idx); return Some(Lease::new(cache_ref.clone(), item)); @@ -162,13 +167,11 @@ impl Pool for FifoPool { // and superset of usage flags) for idx in 0..cache.len() { let item = unsafe { cache.get_unchecked(idx) }; - if (item.info.dedicated & info.dedicated) == info.dedicated - && item.info.host_read == info.host_read - && item.info.host_write == info.host_write - && item.info.alignment >= info.alignment - && item.info.size >= info.size - && item.info.usage.contains(info.usage) - { + if compatible_buffer_info( + &item.info, + &info, + BufferHostMappingCompatibility::Exact, + ) { let item = cache.swap_remove(idx); return Some(Lease::new(cache_ref.clone(), item)); @@ -197,13 +200,11 @@ impl Pool for FifoPool { .entry(info.queue_family_index) .or_insert_with(PoolConfig::default_cache); - let item = with_cache(cache_ref, lease_command_buffer) - .map(Ok) - .unwrap_or_else(|| { - debug!("Creating new {}", stringify!(CommandBuffer)); + let item = with_cache(cache_ref, Vec::pop).map(Ok).unwrap_or_else(|| { + debug!("Creating new {}", stringify!(CommandBuffer)); - CommandBuffer::create(&self.device, info) - })?; + CommandBuffer::create(&self.device, info) + })?; // Drop anything we were holding from the last submission //item.wait_until_executed()?; @@ -276,14 +277,15 @@ impl Pool for FifoPool { for idx in 0..cache.len() { let item = unsafe { cache.get_unchecked(idx) }; if item.info.array_layer_count == info.array_layer_count - && item.info.dedicated == info.dedicated + && item.info.alloc_dedicated == info.alloc_dedicated && item.info.depth == info.depth - && item.info.fmt == info.fmt + && item.info.format == info.format && item.info.height == info.height && item.info.mip_level_count == info.mip_level_count && item.info.sample_count == info.sample_count + && item.info.sharing_mode == info.sharing_mode && item.info.tiling == info.tiling - && item.info.ty == info.ty + && item.info.image_type == info.image_type && item.info.width == info.width && item.info.flags.contains(info.flags) && item.info.usage.contains(info.usage) diff --git a/src/pool/hash.rs b/src/pool/hash.rs index a4c14beb..e40da9dd 100644 --- a/src/pool/hash.rs +++ b/src/pool/hash.rs @@ -1,7 +1,7 @@ //! Pool which requests by exactly matching the information before creating new resources. use { - super::{Cache, Lease, Pool, PoolConfig, lease_command_buffer}, + super::{Cache, Lease, Pool, PoolConfig}, crate::driver::{ DriverError, accel_struct::{AccelerationStructure, AccelerationStructureInfo}, @@ -36,8 +36,8 @@ use std::sync::Mutex; /// /// # Memory Management /// -/// If requests for varying resources is common [`HashPool::clear_images_by_info`] and other memory -/// management functions are nessecery in order to avoid using all available device memory. +/// If requests for varying resources are common [`HashPool::clear_images_by_info`] and other +/// memory management functions are necessary in order to avoid using all available device memory. #[derive(Debug)] #[read_only::cast] pub struct HashPool { @@ -155,7 +155,7 @@ impl Pool for HashPool { #[cfg(not(feature = "parking_lot"))] let mut cache = cache.expect("poisoned cache lock"); - lease_command_buffer(&mut cache) + cache.pop() } .map(Ok) .unwrap_or_else(|| { diff --git a/src/pool/lazy.rs b/src/pool/lazy.rs index 002b46ea..846166dc 100644 --- a/src/pool/lazy.rs +++ b/src/pool/lazy.rs @@ -1,7 +1,9 @@ -//! Pool which requests by looking for compatibile information before creating new resources. +//! Pool which requests by looking for compatible information before creating new resources. use { - super::{Cache, Lease, Pool, PoolConfig, lease_command_buffer}, + super::{ + BufferHostMappingCompatibility, Cache, Lease, Pool, PoolConfig, compatible_buffer_info, + }, crate::driver::{ DriverError, accel_struct::{AccelerationStructure, AccelerationStructureInfo}, @@ -25,8 +27,9 @@ struct ImageKey { height: u32, mip_level_count: u32, sample_count: SampleCount, + sharing_mode: vk::SharingMode, tiling: vk::ImageTiling, - ty: vk::ImageType, + image_type: vk::ImageType, width: u32, } @@ -35,12 +38,13 @@ impl From for ImageKey { Self { array_layer_count: info.array_layer_count, depth: info.depth, - fmt: info.fmt, + fmt: info.format, height: info.height, mip_level_count: info.mip_level_count, sample_count: info.sample_count, + sharing_mode: info.sharing_mode, tiling: info.tiling, - ty: info.ty, + image_type: info.image_type, width: info.width, } } @@ -70,13 +74,13 @@ impl From for ImageKey { /// /// # Memory Management /// -/// If requests for varying resources is common [`LazyPool::clear_images_by_info`] and other memory -/// management functions are nessecery in order to avoid using all available device memory. +/// If requests for varying resources are common [`LazyPool::clear_images_by_info`] and other +/// memory management functions are necessary in order to avoid using all available device memory. #[derive(Debug)] #[read_only::cast] pub struct LazyPool { accel_struct_cache: HashMap>, - buffer_cache: HashMap<(bool, vk::DeviceSize), Cache>, + buffer_cache: HashMap<(bool, vk::DeviceSize, vk::SharingMode), Cache>, command_buffer_cache: HashMap>, descriptor_pool_cache: Cache, @@ -133,8 +137,11 @@ impl LazyPool { } /// Clears the pool of all acceleration structure resources matching the given type. - pub fn clear_accel_structs_by_ty(&mut self, ty: vk::AccelerationStructureTypeKHR) { - self.accel_struct_cache.remove(&ty); + pub fn clear_accel_structs_by_type( + &mut self, + accel_struct_ty: vk::AccelerationStructureTypeKHR, + ) { + self.accel_struct_cache.remove(&accel_struct_ty); } /// Clears the pool of buffer resources. @@ -167,7 +174,8 @@ impl LazyPool { where F: FnMut(vk::AccelerationStructureTypeKHR) -> bool, { - self.accel_struct_cache.retain(|&ty, _| f(ty)) + self.accel_struct_cache + .retain(|&accel_struct_ty, _| f(accel_struct_ty)) } } @@ -179,7 +187,7 @@ impl Pool for LazyPool { ) -> Result, DriverError> { let cache = self .accel_struct_cache - .entry(info.ty) + .entry(info.acceleration_structure_type) .or_insert_with(|| PoolConfig::explicit_cache(self.info.accel_struct_capacity)); let cache_ref = Arc::downgrade(cache); @@ -216,7 +224,11 @@ impl Pool for LazyPool { fn resource(&mut self, info: BufferInfo) -> Result, DriverError> { let cache = self .buffer_cache - .entry((info.host_read | info.host_write, info.alignment)) + .entry(( + info.host_readable | info.host_writable, + info.alignment, + info.sharing_mode, + )) .or_insert_with(|| PoolConfig::explicit_cache(self.info.buffer_capacity)); let cache_ref = Arc::downgrade(cache); @@ -232,13 +244,11 @@ impl Pool for LazyPool { // Look for a compatible buffer (big enough and superset of usage flags) for idx in 0..cache.len() { let item = unsafe { cache.get_unchecked(idx) }; - if (item.info.dedicated & info.dedicated) == info.dedicated - && (item.info.host_read & info.host_read) == info.host_read - && (item.info.host_write & info.host_write) == info.host_write - && item.info.alignment >= info.alignment - && item.info.size >= info.size - && item.info.usage.contains(info.usage) - { + if compatible_buffer_info( + &item.info, + &info, + BufferHostMappingCompatibility::Superset, + ) { let item = cache.swap_remove(idx); return Ok(Lease::new(cache_ref, item)); @@ -268,7 +278,7 @@ impl Pool for LazyPool { #[cfg(not(feature = "parking_lot"))] let mut cache = cache.expect("poisoned cache lock"); - lease_command_buffer(&mut cache) + cache.pop() } .map(Ok) .unwrap_or_else(|| { diff --git a/src/pool/mod.rs b/src/pool/mod.rs index 8b27941c..86d7db78 100644 --- a/src/pool/mod.rs +++ b/src/pool/mod.rs @@ -95,8 +95,9 @@ use { AccelerationStructure, AccelerationStructureInfo, AccelerationStructureInfoBuilder, }, buffer::{Buffer, BufferInfo, BufferInfoBuilder}, - cmd_buf::CommandBuffer, + descriptor_set::{DescriptorPool, DescriptorPoolInfo}, image::{Image, ImageInfo, ImageInfoBuilder}, + render_pass::{RenderPass, RenderPassInfo}, }, derive_builder::{Builder, UninitializedFieldError}, std::{ @@ -108,6 +109,61 @@ use { }, }; +#[derive(Clone, Copy)] +enum BufferHostMappingCompatibility { + Exact, + Superset, +} + +fn compatible_buffer_info( + item_info: &BufferInfo, + requested_info: &BufferInfo, + host_mapping: BufferHostMappingCompatibility, +) -> bool { + (item_info.alloc_dedicated & requested_info.alloc_dedicated) == requested_info.alloc_dedicated + && compatible_buffer_host_mapping(item_info, requested_info, host_mapping) + && item_info.alignment >= requested_info.alignment + && item_info.sharing_mode == requested_info.sharing_mode + && item_info.size >= requested_info.size + && item_info.usage.contains(requested_info.usage) +} + +fn compatible_buffer_host_mapping( + item_info: &BufferInfo, + requested_info: &BufferInfo, + compatibility: BufferHostMappingCompatibility, +) -> bool { + match compatibility { + BufferHostMappingCompatibility::Exact => { + item_info.host_readable == requested_info.host_readable + && item_info.host_writable == requested_info.host_writable + } + BufferHostMappingCompatibility::Superset => { + (item_info.host_readable & requested_info.host_readable) == requested_info.host_readable + && (item_info.host_writable & requested_info.host_writable) + == requested_info.host_writable + } + } +} + +fn compatible_image_info(item_info: &ImageInfo, requested_info: &ImageInfo) -> bool { + item_info.array_layer_count == requested_info.array_layer_count + && item_info.alloc_dedicated == requested_info.alloc_dedicated + && item_info.depth == requested_info.depth + && item_info.format == requested_info.format + && item_info.height == requested_info.height + && item_info.host_readable == requested_info.host_readable + && item_info.host_writable == requested_info.host_writable + && item_info.mip_level_count == requested_info.mip_level_count + && item_info.sample_count == requested_info.sample_count + && item_info.sharing_mode == requested_info.sharing_mode + && item_info.tiling == requested_info.tiling + && item_info.image_type == requested_info.image_type + && item_info.width == requested_info.width + && item_info.flags.contains(requested_info.flags) + && item_info.usage.contains(requested_info.usage) +} + #[cfg(feature = "parking_lot")] use parking_lot::Mutex; @@ -117,22 +173,6 @@ use std::sync::Mutex; type Cache = Arc>>; type CacheRef = Weak>>; -fn lease_command_buffer(cache: &mut Vec) -> Option { - for idx in 0..cache.len() { - if unsafe { - let cmd = cache.get_unchecked(idx); - - // Don't lease this command buffer if it is unsignalled; we'll create a new one - // and wait for this, and those behind it, to signal. - cmd.device.get_fence_status(cmd.fence).unwrap_or_default() - } { - return Some(cache.swap_remove(idx)); - } - } - - None -} - fn with_cache(cache: &Cache, f: impl FnOnce(&mut Vec) -> R) -> R { let cache = cache.lock(); @@ -155,13 +195,15 @@ pub struct Lease { item: ManuallyDrop, } -// The following debug_name functions take a self of Lease and return Self. -// This allows pooled resources to have the same `.debug_name("bugs")` chaining +/* +The following debug_name functions take a self of Lease and return Self. +This allows pooled resources to have the same `.debug_name("bugs")` chaining. +*/ impl Lease { /// Sets the debugging name assigned to this acceleration structure. - pub fn debug_name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); self } @@ -169,8 +211,8 @@ impl Lease { impl Lease { /// Sets the debugging name assigned to this buffer. - pub fn debug_name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); self } @@ -178,8 +220,8 @@ impl Lease { impl Lease { /// Sets the debugging name assigned to this image. - pub fn debug_name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); + pub fn with_debug_name(self, name: impl AsRef) -> Self { + self.set_debug_name(name); self } @@ -194,6 +236,12 @@ impl Lease { } } +impl AsRef for Lease { + fn as_ref(&self) -> &T { + self + } +} + impl Deref for Lease { type Target = T; @@ -216,7 +264,7 @@ impl Drop for Lease { } // If the pool cache has been dropped we must manually drop the item, otherwise it goes back - // into the pool. + // into the pool if let Some(cache) = self.cache_ref.upgrade() { with_cache(&cache, |cache| { if cache.len() >= cache.capacity() { @@ -239,6 +287,44 @@ pub trait Pool { fn resource(&mut self, info: I) -> Result, DriverError>; } +/// Pool capability required by graph submission scheduling. +/// +/// This sealed trait is implemented by the built-in pools. It covers internal descriptor-pool and +/// render-pass leases without exposing their cache-key types in public API bounds. +#[allow(private_bounds)] +pub trait SubmissionPool: submission_pool_private::SubmissionPoolSealed {} + +impl SubmissionPool for T where T: submission_pool_private::SubmissionPoolSealed {} + +pub(crate) mod submission_pool_private { + use super::*; + + pub(crate) trait SubmissionPoolSealed { + fn descriptor_pool( + &mut self, + info: DescriptorPoolInfo, + ) -> Result, DriverError>; + + fn render_pass(&mut self, info: RenderPassInfo) -> Result, DriverError>; + } + + impl SubmissionPoolSealed for T + where + T: Pool + Pool, + { + fn descriptor_pool( + &mut self, + info: DescriptorPoolInfo, + ) -> Result, DriverError> { + self.resource(info) + } + + fn render_pass(&mut self, info: RenderPassInfo) -> Result, DriverError> { + self.resource(info) + } + } +} + // Enable requesting items using their info builder type for convenience macro_rules! lease_builder { ($info:ident => $item:ident) => { @@ -385,6 +471,7 @@ impl PoolConfigBuilder { #[cfg(test)] mod test { use super::*; + use crate::driver::ash::vk; type Info = PoolConfig; type Builder = PoolConfigBuilder; @@ -412,4 +499,40 @@ mod test { assert_eq!(info, builder); } + + #[test] + fn buffer_info_compatibility_rejects_different_sharing_mode() { + let exclusive = BufferInfo::device_mem(64, vk::BufferUsageFlags::STORAGE_BUFFER); + let concurrent = BufferInfo { + sharing_mode: vk::SharingMode::CONCURRENT, + ..exclusive + }; + + assert!(!compatible_buffer_info( + &exclusive, + &concurrent, + BufferHostMappingCompatibility::Exact, + )); + assert!(!compatible_buffer_info( + &exclusive, + &concurrent, + BufferHostMappingCompatibility::Superset, + )); + } + + #[test] + fn image_info_compatibility_rejects_different_sharing_mode() { + let exclusive = ImageInfo::image_2d( + 16, + 16, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::STORAGE, + ); + let concurrent = ImageInfo { + sharing_mode: vk::SharingMode::CONCURRENT, + ..exclusive + }; + + assert!(!compatible_image_info(&exclusive, &concurrent)); + } } diff --git a/src/stream.rs b/src/stream.rs new file mode 100644 index 00000000..143de7b8 --- /dev/null +++ b/src/stream.rs @@ -0,0 +1,1556 @@ +//! Reusable command streams. +//! +//! A [`CommandStream`] is a prepared graph-like command sequence that can be inserted into a +//! per-frame [`Graph`] with typed arguments. +//! +//! Streams are useful when part of a frame is structurally the same across many frames but still +//! needs per-frame resources such as the current swapchain image. Declare those resources as stream +//! arguments, record reusable commands once, and bind concrete graph nodes when inserting the stream. +//! +//! ```no_run +//! # use ash::vk; +//! # use vk_graph::{Graph, node::{BufferNode, ImageNode}, pool::hash::HashPool}; +//! # use vk_graph::cmd::{LoadOp, StoreOp}; +//! # use vk_graph::driver::buffer::BufferInfo; +//! # use vk_graph::driver::graphics::GraphicsPipeline; +//! # use vk_graph::driver::image::ImageInfo; +//! # use vk_graph::stream::CommandStream; +//! # use vk_sync::AccessType; +//! # let mut pool: HashPool = todo!(); +//! # let pipeline: GraphicsPipeline = todo!(); +//! # let swapchain_image: ImageNode = todo!(); +//! # let vertex_buffer: BufferNode = todo!(); +//! let stream = CommandStream::prepare(&mut pool, |stream| { +//! let output = stream.arg(ImageInfo::image_2d( +//! 1280, +//! 720, +//! vk::Format::R8G8B8A8_UNORM, +//! vk::ImageUsageFlags::COLOR_ATTACHMENT, +//! )); +//! let vertices = stream.arg(BufferInfo::device_mem( +//! 4096, +//! vk::BufferUsageFlags::VERTEX_BUFFER, +//! )); +//! +//! stream +//! .begin_cmd() +//! .debug_name("reusable overlay") +//! .bind_pipeline(&pipeline) +//! .color_attachment_image(0, output, LoadOp::Load, StoreOp::Store) +//! .resource_access(vertices, AccessType::VertexBuffer) +//! .record_cmd(move |cmd| { +//! cmd.bind_vertex_buffer(0, vertices, 0).draw(3, 1, 0, 0); +//! }); +//! +//! (output, vertices) +//! })?; +//! +//! let mut graph = Graph::new(); +//! graph +//! .insert_cmd_stream(&stream) +//! .with_arg(stream.args.0, swapchain_image) +//! .with_arg(stream.args.1, vertex_buffer) +//! .finish(); +//! # Ok::<(), vk_graph::driver::DriverError>(()) +//! ``` + +use crate::private::NodeSealed; +use { + crate::{ + AnyResource, Graph, Node, Resource, ResourceMap, + cmd::{ + AttachmentIndex, ClearColorValue, Command, CommandRef, ComputeCommandRef, + GraphicsCommandRef, LoadOp, PipelineCommand, RayTracingCommandRef, StoreOp, + Subresource, SubresourceRange, + }, + driver::{ + DriverError, + accel_struct::{ + AccelerationStructure, AccelerationStructureInfo, AccelerationStructureInfoBuilder, + }, + buffer::{Buffer, BufferInfo, BufferInfoBuilder}, + compute::ComputePipeline, + graphics::{DepthStencilInfo, GraphicsPipeline}, + image::{Image, ImageInfo, ImageInfoBuilder, ImageViewInfo}, + ray_tracing::RayTracingPipeline, + }, + node::{ + AccelerationStructureLeaseNode, AccelerationStructureNode, + AnyAccelerationStructureNode, AnyBufferNode, AnyImageNode, BufferLeaseNode, BufferNode, + ImageLeaseNode, ImageNode, SwapchainImageNode, + }, + pool::SubmissionPool, + submission::Submission, + }, + ash::vk, + std::{ + collections::HashMap, + marker::PhantomData, + ops::Range, + sync::{Arc, Mutex}, + }, +}; + +#[cfg(feature = "checked")] +use crate::GraphId; + +use std::sync::atomic::{AtomicU64, Ordering}; + +fn next_stream_scope_id() -> u64 { + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + + NEXT_ID.fetch_add(1, Ordering::Relaxed) +} + +#[cfg(feature = "checked")] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct CommandStreamId(u64); + +#[cfg(feature = "checked")] +impl CommandStreamId { + fn next() -> Self { + Self(next_stream_scope_id()) + } +} + +/// A typed external argument for a [`CommandStream`]. +/// +/// `StreamArg` values are created with [`CommandStreamMut::arg`] while building a stream and are +/// later bound to parent-graph nodes with [`CommandStreamRun::with_arg`]. +/// +/// ```no_run +/// # use ash::vk; +/// # use vk_graph::{Graph, driver::image::ImageInfo, node::ImageNode, stream::CommandStream}; +/// # let swapchain_image: ImageNode = todo!(); +/// let stream = CommandStream::finalize(|stream| { +/// stream.arg(ImageInfo::image_2d( +/// 640, +/// 480, +/// vk::Format::R8G8B8A8_UNORM, +/// vk::ImageUsageFlags::COLOR_ATTACHMENT, +/// )) +/// }) +/// .into_stream(); +/// +/// let mut graph = Graph::new(); +/// graph +/// .insert_cmd_stream(&stream) +/// .with_arg(stream.args, swapchain_image) +/// .finish(); +/// ``` +#[derive(Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StreamArg { + pub(crate) arg_index: usize, + pub(crate) index: usize, + + #[cfg(feature = "checked")] + pub(crate) stream_id: CommandStreamId, + + #[cfg(feature = "checked")] + pub(crate) graph_id: GraphId, + + __: PhantomData T>, +} + +impl Clone for StreamArg { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for StreamArg {} + +impl StreamArg { + pub(crate) fn new( + arg_index: usize, + index: usize, + #[cfg(feature = "checked")] stream_id: CommandStreamId, + #[cfg(feature = "checked")] graph_id: GraphId, + ) -> Self { + Self { + arg_index, + index, + #[cfg(feature = "checked")] + stream_id, + #[cfg(feature = "checked")] + graph_id, + __: PhantomData, + } + } +} + +impl NodeSealed for StreamArg { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { + resources[self.index].expect_accel_struct() + } + + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource { + resources[index].expect_accel_struct() + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, _graph_id: GraphId) { + #[cfg(feature = "checked")] + assert!( + self.graph_id == _graph_id, + "node belongs to a different graph" + ); + } +} + +impl Node for StreamArg { + type Resource = AccelerationStructure; + type SyncInfo = crate::driver::accel_struct::AccelerationStructureSyncInfo; + + fn index(&self) -> usize { + self.index + } +} + +impl NodeSealed for StreamArg { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { + resources[self.index].expect_buffer() + } + + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource { + resources[index].expect_buffer() + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, _graph_id: GraphId) { + #[cfg(feature = "checked")] + assert!( + self.graph_id == _graph_id, + "node belongs to a different graph" + ); + } +} + +impl Node for StreamArg { + type Resource = Buffer; + type SyncInfo = crate::driver::buffer::BufferSyncInfo; + + fn index(&self) -> usize { + self.index + } +} + +impl NodeSealed for StreamArg { + fn borrow(self, resources: &[AnyResource]) -> &::Resource { + resources[self.index].expect_image() + } + + fn borrow_at(self, resources: &[AnyResource], index: usize) -> &::Resource { + resources[index].expect_image() + } + + #[cfg(feature = "checked")] + fn assert_owner(&self, _graph_id: GraphId) { + #[cfg(feature = "checked")] + assert!( + self.graph_id == _graph_id, + "node belongs to a different graph" + ); + } +} + +impl Node for StreamArg { + type Resource = Image; + type SyncInfo = crate::driver::image::ImageSyncInfo; + + fn index(&self) -> usize { + self.index + } +} + +/// A stream argument for an acceleration structure. +/// +/// ```no_run +/// # use vk_graph::driver::accel_struct::AccelerationStructureInfo; +/// # use vk_graph::stream::{AccelerationStructureArg, CommandStream}; +/// # let info: AccelerationStructureInfo = todo!(); +/// let stream = CommandStream::finalize(|stream| -> AccelerationStructureArg { +/// stream.arg(info) +/// }) +/// .into_stream(); +/// ``` +pub type AccelerationStructureArg = StreamArg; + +/// A stream argument for a buffer. +/// +/// ```no_run +/// # use ash::vk; +/// # use vk_graph::driver::buffer::BufferInfo; +/// # use vk_graph::stream::{BufferArg, CommandStream}; +/// let stream = CommandStream::finalize(|stream| -> BufferArg { +/// stream.arg(BufferInfo::device_mem( +/// 4096, +/// vk::BufferUsageFlags::STORAGE_BUFFER, +/// )) +/// }) +/// .into_stream(); +/// ``` +pub type BufferArg = StreamArg; + +/// A stream argument for an image. +/// +/// ```no_run +/// # use ash::vk; +/// # use vk_graph::driver::image::ImageInfo; +/// # use vk_graph::stream::{CommandStream, ImageArg}; +/// let stream = CommandStream::finalize(|stream| -> ImageArg { +/// stream.arg(ImageInfo::image_2d( +/// 128, +/// 128, +/// vk::Format::R8G8B8A8_UNORM, +/// vk::ImageUsageFlags::SAMPLED, +/// )) +/// }) +/// .into_stream(); +/// ``` +pub type ImageArg = StreamArg; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum StreamArgData { + AccelerationStructure(AccelerationStructureInfo), + Buffer(BufferInfo), + Image(ImageInfo), +} + +#[derive(Debug)] +pub(crate) struct CommandStreamInner { + pub(crate) arg_nodes: Box<[usize]>, + pub(crate) args: Box<[StreamArgData]>, + pub(crate) prepared: bool, + pub(crate) submission: Mutex, + + #[cfg(feature = "checked")] + pub(crate) stream_id: CommandStreamId, + + #[cfg(feature = "checked")] + pub(crate) graph_id: GraphId, +} + +/// A reusable command stream. +/// +/// Prepared streams reduce repeated CPU-side graph construction and preparation work by caching an +/// optimized schedule and static recording resources. Unprepared streams keep finalization cheaper +/// up front, but each insertion still has to reconcile arguments, dependencies, scheduling, and +/// recording with the parent graph. +/// +/// Inserting or concatenating many tiny streams is not free. Profile release builds before designing +/// around heavy stream composition. +/// +/// ```no_run +/// # use vk_graph::{Graph, pool::hash::HashPool, stream::CommandStream}; +/// # let mut pool: HashPool = todo!(); +/// let stream = CommandStream::prepare(&mut pool, |stream| { +/// stream.begin_cmd().debug_name("cached commands").record_cmd(|_| {}); +/// })?; +/// +/// let mut graph = Graph::new(); +/// graph.insert_cmd_stream(&stream).finish(); +/// # Ok::<(), vk_graph::driver::DriverError>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct CommandStream { + /// Typed handles returned by the preparation callback. + pub args: A, + pub(crate) inner: Arc, +} + +/// A finalized command stream definition that can be prepared later. +/// +/// Drafts are useful when construction should happen separately from preparation. Convert a draft +/// with [`CommandStreamDraft::into_stream`] for unprepared insertion or +/// [`CommandStreamDraft::prepare`] to cache preparation work. +/// +/// ```no_run +/// # use vk_graph::{Graph, pool::hash::HashPool, stream::CommandStream}; +/// # let mut pool: HashPool = todo!(); +/// let draft = CommandStream::finalize(|stream| { +/// stream.begin_cmd().record_cmd(|_| {}); +/// }); +/// +/// let prepared = draft.prepare(&mut pool)?; +/// let mut graph = Graph::new(); +/// graph.insert_cmd_stream(&prepared).finish(); +/// # Ok::<(), vk_graph::driver::DriverError>(()) +/// ``` +#[derive(Debug)] +pub struct CommandStreamDraft { + /// Typed handles returned by the finalization callback. + pub args: A, + inner: CommandStreamInner, +} + +/// A mutable graph-like command stream being prepared. +/// +/// `CommandStreamMut` is passed to [`CommandStream::finalize`] and [`CommandStream::prepare`] +/// callbacks. It provides graph-like methods plus [`CommandStreamMut::arg`] for typed stream +/// inputs. +/// +/// ```no_run +/// # use ash::vk; +/// # use vk_graph::{driver::buffer::BufferInfo, stream::CommandStream}; +/// let stream = CommandStream::finalize(|stream| { +/// let staging = stream.arg(BufferInfo::host_mem( +/// 1024, +/// vk::BufferUsageFlags::TRANSFER_SRC, +/// )); +/// stream.begin_cmd().resource_access(staging, vk_sync::AccessType::TransferRead); +/// staging +/// }) +/// .into_stream(); +/// ``` +pub struct CommandStreamMut { + pub(crate) arg_nodes: Vec, + pub(crate) args: Vec, + pub(crate) graph: Graph, + #[cfg(feature = "checked")] + pub(crate) stream_id: CommandStreamId, +} + +/// A command being recorded into a [`CommandStreamMut`]. +/// +/// ```no_run +/// # use vk_graph::stream::CommandStream; +/// let stream = CommandStream::finalize(|stream| { +/// stream +/// .begin_cmd() +/// .debug_name("stream command") +/// .record_cmd(|cmd| { +/// let _ = cmd; +/// }); +/// }) +/// .into_stream(); +/// ``` +pub struct StreamCommand<'a> { + inner: Command<'a>, +} + +/// A stream command with a bound pipeline. +/// +/// ```no_run +/// # use vk_graph::stream::CommandStream; +/// # use vk_graph::driver::compute::ComputePipeline; +/// # let pipeline: ComputePipeline = todo!(); +/// let stream = CommandStream::finalize(|stream| { +/// stream +/// .begin_cmd() +/// .bind_pipeline(&pipeline) +/// .record_cmd(|cmd| { +/// cmd.dispatch(1, 1, 1); +/// }); +/// }) +/// .into_stream(); +/// ``` +pub struct StreamPipelineCommand<'a, T> { + inner: PipelineCommand<'a, T>, +} + +/// A pipeline that can be bound to a stream command. +#[doc(hidden)] +pub trait StreamPipeline<'a>: stream_private::StreamPipelineSealed { + /// The stream command type returned after binding. + type Command; + + /// Stream equivalent of [`Pipeline::bind_cmd`]. + fn bind_stream_cmd(self, cmd: StreamCommand<'a>) -> Self::Command; +} + +macro_rules! stream_pipeline { + ($pipeline:ty) => { + impl<'a> StreamPipeline<'a> for $pipeline { + type Command = StreamPipelineCommand<'a, $pipeline>; + + fn bind_stream_cmd(self, cmd: StreamCommand<'a>) -> Self::Command { + StreamPipelineCommand { + inner: cmd.inner.bind_pipeline(self), + } + } + } + + impl stream_private::StreamPipelineSealed for $pipeline {} + + impl<'a> StreamPipeline<'a> for &'a $pipeline { + type Command = StreamPipelineCommand<'a, $pipeline>; + + fn bind_stream_cmd(self, cmd: StreamCommand<'a>) -> Self::Command { + StreamPipelineCommand { + inner: cmd.inner.bind_pipeline(self), + } + } + } + + impl<'a> stream_private::StreamPipelineSealed for &'a $pipeline {} + }; +} + +stream_pipeline!(ComputePipeline); +stream_pipeline!(GraphicsPipeline); +stream_pipeline!(RayTracingPipeline); + +#[allow(private_bounds)] +impl<'a> StreamCommand<'a> { + /// Stream equivalent of [`Command::bind_resource`]. + pub fn bind_resource(&mut self, resource: R) -> R::Node + where + R: Resource, + { + self.inner.bind_resource(resource) + } + + /// Stream equivalent of [`Command::bind_pipeline`]. + pub fn bind_pipeline

(self, pipeline: P) -> P::Command + where + P: StreamPipeline<'a>, + { + pipeline.bind_stream_cmd(self) + } + + /// Stream equivalent of [`Command::debug_name`]. + pub fn debug_name(mut self, name: impl Into) -> Self { + self.inner.set_debug_name(name); + self + } + + /// Stream equivalent of [`Command::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd( + mut self, + func: impl for<'r> Fn(CommandRef<'r>) + Send + Sync + 'static, + ) -> Self { + self.record_cmd_mut(func); + self + } + + /// Mutable-borrow stream equivalent of [`Command::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd_mut( + &mut self, + func: impl for<'r> Fn(CommandRef<'r>) + Send + Sync + 'static, + ) { + self.inner.record_stream_mut(func); + } + + /// Stream equivalent of [`Command::resource_access`]. + pub fn resource_access(mut self, resource_node: N, access: vk_sync::AccessType) -> Self + where + N: Node + Subresource, + SubresourceRange: From, + { + self.inner.set_resource_access(resource_node, access); + self + } + + /// Mutable-borrow stream equivalent of [`Command::resource_access`]. + pub fn set_resource_access(&mut self, resource_node: N, access: vk_sync::AccessType) + where + N: Node + Subresource, + SubresourceRange: From, + { + self.inner.set_resource_access(resource_node, access); + } +} + +#[allow(private_bounds)] +impl<'a, T> StreamPipelineCommand<'a, T> { + /// Stream equivalent of [`PipelineCommand::bind_resource`]. + pub fn bind_resource(&mut self, resource: R) -> R::Node + where + R: Resource, + { + self.inner.bind_resource(resource) + } + + /// Stream equivalent of [`PipelineCommand::resource_access`]. + pub fn resource_access(mut self, resource_node: N, access: vk_sync::AccessType) -> Self + where + N: Node + Subresource, + SubresourceRange: From, + { + self.inner.set_resource_access(resource_node, access); + self + } + + /// Mutable-borrow stream equivalent of [`PipelineCommand::resource_access`]. + pub fn set_resource_access( + &mut self, + resource_node: N, + access: vk_sync::AccessType, + ) -> &mut Self + where + N: Node + Subresource, + SubresourceRange: From, + { + self.inner.set_resource_access(resource_node, access); + self + } +} + +impl StreamPipelineCommand<'_, ComputePipeline> { + /// Stream equivalent of [`PipelineCommand::::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd( + mut self, + func: impl for<'r> Fn(ComputeCommandRef<'r>) + Send + Sync + 'static, + ) -> Self { + self.record_cmd_mut(func); + self + } + + /// Mutable-borrow stream equivalent of [`PipelineCommand::::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd_mut( + &mut self, + func: impl for<'r> Fn(ComputeCommandRef<'r>) + Send + Sync + 'static, + ) { + self.inner.record_stream_mut(func); + } +} + +impl StreamPipelineCommand<'_, GraphicsPipeline> { + /// Stream equivalent of [`PipelineCommand::::depth_stencil`]. + pub fn depth_stencil(mut self, depth_stencil: impl Into) -> Self { + self.inner.set_depth_stencil(depth_stencil); + self + } + + /// Stream equivalent of [`PipelineCommand::::color_attachment_image`]. + pub fn color_attachment_image( + mut self, + color_attachment: AttachmentIndex, + image: impl Into, + load: LoadOp, + store: StoreOp, + ) -> Self { + self.inner + .set_color_attachment_image(color_attachment, image, load, store); + self + } + + /// Stream equivalent of [`PipelineCommand::::color_attachment_image_view`]. + pub fn color_attachment_image_view( + mut self, + color_attachment: AttachmentIndex, + image: impl Into, + image_view_info: impl Into, + load: LoadOp, + store: StoreOp, + ) -> Self { + self.inner.set_color_attachment_image_view( + color_attachment, + image, + image_view_info, + load, + store, + ); + self + } + + /// Stream equivalent of [`PipelineCommand::::depth_stencil_attachment_image`]. + pub fn depth_stencil_attachment_image( + mut self, + image: impl Into, + load: LoadOp, + store: StoreOp, + ) -> Self { + self.inner + .set_depth_stencil_attachment_image(image, load, store); + self + } + + /// Stream equivalent of [`PipelineCommand::::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd( + mut self, + func: impl for<'r> Fn(GraphicsCommandRef<'r>) + Send + Sync + 'static, + ) -> Self { + self.record_cmd_mut(func); + self + } + + /// Mutable-borrow stream equivalent of [`PipelineCommand::::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd_mut( + &mut self, + func: impl for<'r> Fn(GraphicsCommandRef<'r>) + Send + Sync + 'static, + ) { + self.inner.record_stream_mut(func); + } +} + +impl StreamPipelineCommand<'_, RayTracingPipeline> { + /// Stream equivalent of [`PipelineCommand::::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd( + mut self, + func: impl for<'r> Fn(RayTracingCommandRef<'r>) + Send + Sync + 'static, + ) -> Self { + self.record_cmd_mut(func); + self + } + + /// Mutable-borrow stream equivalent of [`PipelineCommand::::record_cmd`]. + /// + /// Unlike graph commands, stream callbacks must be reusable and therefore implement + /// `Fn + Send + Sync + 'static`. + pub fn record_cmd_mut( + &mut self, + func: impl for<'r> Fn(RayTracingCommandRef<'r>) + Send + Sync + 'static, + ) { + self.inner.record_stream_mut(func); + } +} + +impl CommandStreamMut { + /// Declares a typed argument required by this command stream. + pub fn arg(&mut self, info: I) -> I::Arg + where + I: StreamArgInfo, + { + info.bind_stream_arg(self) + } + + /// Stream equivalent of [`Graph::begin_cmd`]. + pub fn begin_cmd(&mut self) -> StreamCommand<'_> { + StreamCommand { + inner: self.graph.begin_cmd(), + } + } + + /// Stream equivalent of [`Graph::bind_resource`]. + pub fn bind_resource(&mut self, resource: R) -> R::Node + where + R: Resource, + { + self.graph.bind_resource(resource) + } + + /// Stream equivalent of [`Graph::resource`]. + pub fn resource(&self, resource_node: N) -> &N::Resource + where + N: StreamResourceNode, + { + self.graph.resource(resource_node) + } + + /// Stream equivalent of [`Graph::blit_image`]. + pub fn blit_image( + &mut self, + src: impl Into, + dst: impl Into, + filter: vk::Filter, + ) -> &mut Self { + self.graph.blit_image(src, dst, filter); + self + } + + /// Stream equivalent of [`Graph::blit_image_region`]. + pub fn blit_image_region( + &mut self, + src: impl Into, + dst: impl Into, + filter: vk::Filter, + regions: impl AsRef<[vk::ImageBlit]> + 'static + Send, + ) -> &mut Self { + self.graph.blit_image_region(src, dst, filter, regions); + self + } + + /// Stream equivalent of [`Graph::clear_color_image`]. + pub fn clear_color_image( + &mut self, + image: impl Into, + color: impl Into, + ) -> &mut Self { + self.graph.clear_color_image(image, color); + self + } + + /// Stream equivalent of [`Graph::clear_depth_stencil_image`]. + pub fn clear_depth_stencil_image( + &mut self, + image: impl Into, + depth: f32, + stencil: u32, + ) -> &mut Self { + self.graph.clear_depth_stencil_image(image, depth, stencil); + self + } + + /// Stream equivalent of [`Graph::copy_buffer`]. + pub fn copy_buffer( + &mut self, + src: impl Into, + dst: impl Into, + ) -> &mut Self { + self.graph.copy_buffer(src, dst); + self + } + + /// Stream equivalent of [`Graph::copy_buffer_region`]. + pub fn copy_buffer_region( + &mut self, + src: impl Into, + dst: impl Into, + regions: impl AsRef<[vk::BufferCopy]> + 'static + Send, + ) -> &mut Self { + self.graph.copy_buffer_region(src, dst, regions); + self + } + + /// Stream equivalent of [`Graph::copy_buffer_to_image`]. + pub fn copy_buffer_to_image( + &mut self, + src: impl Into, + dst: impl Into, + ) -> &mut Self { + self.graph.copy_buffer_to_image(src, dst); + self + } + + /// Stream equivalent of [`Graph::copy_buffer_to_image_region`]. + pub fn copy_buffer_to_image_region( + &mut self, + src: impl Into, + dst: impl Into, + regions: impl AsRef<[vk::BufferImageCopy]> + 'static + Send, + ) -> &mut Self { + self.graph.copy_buffer_to_image_region(src, dst, regions); + self + } + + /// Stream equivalent of [`Graph::copy_image`]. + pub fn copy_image( + &mut self, + src: impl Into, + dst: impl Into, + ) -> &mut Self { + self.graph.copy_image(src, dst); + self + } + + /// Stream equivalent of [`Graph::copy_image_region`]. + pub fn copy_image_region( + &mut self, + src: impl Into, + dst: impl Into, + regions: impl AsRef<[vk::ImageCopy]> + 'static + Send, + ) -> &mut Self { + self.graph.copy_image_region(src, dst, regions); + self + } + + /// Stream equivalent of [`Graph::copy_image_to_buffer`]. + pub fn copy_image_to_buffer( + &mut self, + src: impl Into, + dst: impl Into, + ) -> &mut Self { + self.graph.copy_image_to_buffer(src, dst); + self + } + + /// Stream equivalent of [`Graph::copy_image_to_buffer_region`]. + pub fn copy_image_to_buffer_region( + &mut self, + src: impl Into, + dst: impl Into, + regions: impl AsRef<[vk::BufferImageCopy]> + 'static + Send, + ) -> &mut Self { + self.graph.copy_image_to_buffer_region(src, dst, regions); + self + } + + /// Stream equivalent of [`Graph::fill_buffer`]. + pub fn fill_buffer( + &mut self, + buffer: impl Into, + region: Range, + data: u32, + ) -> &mut Self { + self.graph.fill_buffer(buffer, region, data); + self + } + + /// Stream equivalent of [`Graph::update_buffer`]. + pub fn update_buffer( + &mut self, + buffer: impl Into, + offset: vk::DeviceSize, + data: impl AsRef<[u8]> + 'static + Send, + ) -> &mut Self { + self.graph.update_buffer(buffer, offset, data); + self + } + + fn push_arg(&mut self, data: StreamArgData) -> usize { + let index = self.args.len(); + self.args.push(data); + index + } + + fn bind_arg_resource(&mut self, data: StreamArgData) -> usize { + let resource = match data { + StreamArgData::AccelerationStructure(info) => { + AnyResource::AccelerationStructureArg(info) + } + StreamArgData::Buffer(info) => AnyResource::BufferArg(info), + StreamArgData::Image(info) => AnyResource::ImageArg(info), + }; + + self.graph.bind_stream_arg_resource(resource) + } +} + +impl CommandStream<()> { + /// Finalizes a reusable command stream without preparing optimizations. + /// + /// The returned draft can be inserted as an unprepared stream with [`CommandStreamDraft::into_stream`] + /// or prepared later with [`CommandStreamDraft::prepare`]. + pub fn finalize(build: impl FnOnce(&mut CommandStreamMut) -> A) -> CommandStreamDraft { + let mut stream = CommandStreamMut { + arg_nodes: Vec::new(), + args: Vec::new(), + graph: Graph::new(), + #[cfg(feature = "checked")] + stream_id: CommandStreamId::next(), + }; + let args = build(&mut stream); + + #[cfg(feature = "checked")] + let graph_id = stream.graph.graph_id(); + + let submission = stream.graph.finalize(); + submission.assert_reusable_commands(); + + CommandStreamDraft { + args, + inner: CommandStreamInner { + arg_nodes: stream.arg_nodes.into_boxed_slice(), + args: stream.args.into_boxed_slice(), + prepared: false, + submission: Mutex::new(submission), + + #[cfg(feature = "checked")] + stream_id: stream.stream_id, + + #[cfg(feature = "checked")] + graph_id, + }, + } + } + + /// Finalizes and prepares a reusable command stream. + /// + /// Prepared streams do more work up front so repeated insertions can reuse prepared scheduling + /// and static recording resources. + pub fn prepare( + pool: &mut P, + build: impl FnOnce(&mut CommandStreamMut) -> A, + ) -> Result, DriverError> + where + P: SubmissionPool, + { + Self::finalize(build).prepare(pool) + } +} + +impl CommandStreamDraft { + /// Converts this draft into a command stream without preparing optimizations. + /// + /// Unprepared streams avoid preparation cost until insertion, but they do not cache the prepared + /// schedule or static recording resources. + pub fn into_stream(self) -> CommandStream { + CommandStream { + args: self.args, + inner: Arc::new(self.inner), + } + } + + /// Prepares this stream by optimizing its finalized graph and leasing static recording + /// resources for the prepared schedule. + /// + /// This is most useful when the same stream is inserted many times with different arguments. + pub fn prepare

(mut self, pool: &mut P) -> Result, DriverError> + where + P: SubmissionPool, + { + self.inner + .submission + .get_mut() + .expect("poisoned command stream submission") + .prepare_command_stream(pool)?; + self.inner.prepared = true; + + Ok(self.into_stream()) + } +} + +/// An in-progress invocation of a [`CommandStream`] into a [`Graph`]. +/// +/// Bind every declared stream argument before calling [`CommandStreamRun::finish`]. +/// +/// ```no_run +/// # use ash::vk; +/// # use vk_graph::{Graph, driver::image::ImageInfo, node::ImageNode, stream::CommandStream}; +/// # let image: ImageNode = todo!(); +/// let stream = CommandStream::finalize(|stream| { +/// stream.arg(ImageInfo::image_2d( +/// 32, +/// 32, +/// vk::Format::R8G8B8A8_UNORM, +/// vk::ImageUsageFlags::TRANSFER_DST, +/// )) +/// }) +/// .into_stream(); +/// +/// let mut graph = Graph::new(); +/// graph +/// .insert_cmd_stream(&stream) +/// .with_arg(stream.args, image) +/// .finish(); +/// ``` +pub struct CommandStreamRun<'a, A> { + pub(crate) bindings: Vec>, + pub(crate) graph: &'a mut Graph, + pub(crate) stream: &'a CommandStream, +} + +impl<'a, A> CommandStreamRun<'a, A> { + /// Sets a stream argument to a graph node for this invocation. + pub fn with_arg(mut self, arg: StreamArg, node: N) -> Self + where + N: StreamArgBindable, + { + #[cfg(feature = "checked")] + assert!( + arg.stream_id == self.stream.inner.stream_id, + "argument belongs to a different command stream" + ); + node.assert_parent_node(); + self.graph.assert_node_owner(&node); + self.bindings[arg.arg_index] = Some(node.index()); + self + } + + /// Finishes this stream invocation and returns to the parent graph. + pub fn finish(self) -> &'a mut Graph { + #[cfg(feature = "checked")] + assert!( + self.bindings.iter().all(Option::is_some), + "missing command stream argument" + ); + + self.graph + .append_command_stream(self.stream, &self.bindings); + self.graph + } +} + +impl Graph { + /// Inserts a command stream into this graph. + /// + /// Prepared streams reduce repeated preparation work, but insertion still has argument binding, + /// dependency reconciliation, scheduling, and recording costs. + pub fn insert_cmd_stream<'a, A>( + &'a mut self, + stream: &'a CommandStream, + ) -> CommandStreamRun<'a, A> { + CommandStreamRun { + bindings: vec![None; stream.inner.args.len()], + graph: self, + stream, + } + } +} + +impl Graph { + pub(crate) fn append_command_stream( + &mut self, + stream: &CommandStream, + bindings: &[Option], + ) { + if stream.inner.prepared { + self.append_prepared_command_stream(stream, bindings); + } else { + self.append_unprepared_command_stream(stream, bindings); + } + } + + fn append_unprepared_command_stream( + &mut self, + stream: &CommandStream, + bindings: &[Option], + ) { + let submission = stream + .inner + .submission + .lock() + .expect("poisoned command stream submission"); + let stream_graph = submission.graph(); + let mut arg_by_node = HashMap::new(); + + for (arg_idx, &node_idx) in stream.inner.arg_nodes.iter().enumerate() { + arg_by_node.insert(node_idx, arg_idx); + } + + let mut node_map = Vec::with_capacity(stream_graph.resources.len()); + for (node_idx, resource) in stream_graph.resources.iter().enumerate() { + if let Some(&arg_idx) = arg_by_node.get(&node_idx) { + node_map.push(bindings[arg_idx].expect("missing command stream argument")); + } else { + node_map.push(self.resources.bind(resource.clone())); + } + } + + for cmd in &stream_graph.cmds { + let mut cmd = cmd.clone(); + cmd.remap_nodes(&node_map); + + #[cfg(feature = "checked")] + for exec in &mut cmd.execs { + exec.stream_graph_id = Some(stream.inner.graph_id); + } + + self.cmds.push(cmd); + } + } + + fn append_prepared_command_stream( + &mut self, + stream: &CommandStream, + bindings: &[Option], + ) { + let stream_scope_id = next_stream_scope_id(); + let submission = stream + .inner + .submission + .lock() + .expect("poisoned command stream submission"); + let stream_graph = submission.graph(); + let mut arg_by_node = HashMap::new(); + + for (arg_idx, &node_idx) in stream.inner.arg_nodes.iter().enumerate() { + arg_by_node.insert(node_idx, arg_idx); + } + + let mut cmd = self.begin_cmd().debug_name("command stream"); + cmd.set_stream_scope_id(stream_scope_id); + + for stream_cmd in &stream_graph.cmds { + for (node_idx, accesses) in stream_cmd + .execs + .iter() + .flat_map(|exec| exec.accesses.iter()) + { + let Some(&arg_idx) = arg_by_node.get(&node_idx) else { + continue; + }; + let parent_node_idx = bindings[arg_idx].expect("missing command stream argument"); + + for access in accesses { + cmd.push_subresource_access_index( + parent_node_idx, + access.subresource, + access.access, + ); + } + } + } + + drop(submission); + + let stream = Arc::clone(&stream.inner); + let bindings = bindings.to_vec(); + cmd.record_stream(move |cmd| { + let submission = stream + .submission + .lock() + .expect("poisoned command stream submission"); + let stream_graph = submission.graph(); + let mut arg_by_node = HashMap::new(); + + for (arg_idx, &node_idx) in stream.arg_nodes.iter().enumerate() { + arg_by_node.insert(node_idx, arg_idx); + } + + let resources = stream_graph + .resources + .iter() + .enumerate() + .map(|(node_idx, resource)| { + if let Some(&arg_idx) = arg_by_node.get(&node_idx) { + cmd.clone_resource_at( + bindings[arg_idx].expect("missing command stream argument"), + ) + } else { + resource.clone() + } + }) + .collect(); + drop(submission); + + stream + .submission + .lock() + .expect("poisoned command stream submission") + .record_prepared_command_stream(&cmd, ResourceMap::from_resources(resources)) + .expect("unable to record command stream"); + }); + } +} + +/// Information that can declare a typed [`CommandStream`] argument. +#[allow(private_bounds)] +#[doc(hidden)] +pub trait StreamArgInfo: stream_private::StreamArgInfoSealed { + /// The typed argument handle returned for this info. + type Arg; + + #[doc(hidden)] + fn bind_stream_arg(self, stream: &mut CommandStreamMut) -> Self::Arg; +} + +/// A graph node that can be supplied for a [`StreamArg`]. +#[allow(private_bounds)] +#[doc(hidden)] +pub trait StreamArgBindable: stream_private::StreamArgBindableSealed + Node { + #[doc(hidden)] + fn assert_parent_node(&self); +} + +/// A graph node that can be borrowed while building a [`CommandStream`]. +#[allow(private_bounds)] +#[doc(hidden)] +pub trait StreamResourceNode: stream_private::StreamResourceNodeSealed + Node {} + +mod stream_private { + pub trait StreamArgInfoSealed {} + + pub trait StreamArgBindableSealed {} + + pub trait StreamResourceNodeSealed {} + + pub trait StreamPipelineSealed {} +} + +macro_rules! stream_arg_info { + ($info:ty, $builder:ty, $variant:ident, $arg:ty) => { + impl stream_private::StreamArgInfoSealed for $info {} + + impl StreamArgInfo for $info { + type Arg = $arg; + + fn bind_stream_arg(self, stream: &mut CommandStreamMut) -> Self::Arg { + let data = StreamArgData::$variant(self); + let arg_index = stream.push_arg(data); + let node_index = stream.bind_arg_resource(data); + stream.arg_nodes.push(node_index); + StreamArg::new( + arg_index, + node_index, + #[cfg(feature = "checked")] + stream.stream_id, + #[cfg(feature = "checked")] + stream.graph.graph_id(), + ) + } + } + + impl stream_private::StreamArgInfoSealed for $builder {} + + impl StreamArgInfo for $builder { + type Arg = $arg; + + fn bind_stream_arg(self, stream: &mut CommandStreamMut) -> Self::Arg { + self.build().bind_stream_arg(stream) + } + } + }; +} + +stream_arg_info!( + AccelerationStructureInfo, + AccelerationStructureInfoBuilder, + AccelerationStructure, + AccelerationStructureArg +); +stream_arg_info!(BufferInfo, BufferInfoBuilder, Buffer, BufferArg); +stream_arg_info!(ImageInfo, ImageInfoBuilder, Image, ImageArg); + +macro_rules! stream_arg_bindable { + ($resource:ty => $($node:ty),+ $(,)?) => { + $( + impl stream_private::StreamArgBindableSealed<$resource> for $node {} + + impl StreamArgBindable<$resource> for $node { + fn assert_parent_node(&self) {} + } + )+ + }; +} + +stream_arg_bindable!( + AccelerationStructure => AccelerationStructureNode, + AccelerationStructureLeaseNode, +); +stream_arg_bindable!(Buffer => BufferNode, BufferLeaseNode); +stream_arg_bindable!(Image => ImageNode, ImageLeaseNode, SwapchainImageNode); + +impl stream_private::StreamArgBindableSealed + for AnyAccelerationStructureNode +{ +} + +impl StreamArgBindable for AnyAccelerationStructureNode { + fn assert_parent_node(&self) { + assert!( + !matches!(self, Self::Arg(_)), + "stream argument cannot be supplied as a parent graph node" + ); + } +} + +impl stream_private::StreamArgBindableSealed for AnyBufferNode {} + +impl StreamArgBindable for AnyBufferNode { + fn assert_parent_node(&self) { + assert!( + !matches!(self, Self::Arg(_)), + "stream argument cannot be supplied as a parent graph node" + ); + } +} + +impl stream_private::StreamArgBindableSealed for AnyImageNode {} + +impl StreamArgBindable for AnyImageNode { + fn assert_parent_node(&self) { + assert!( + !matches!(self, Self::Arg(_)), + "stream argument cannot be supplied as a parent graph node" + ); + } +} + +macro_rules! stream_resource_node { + ($($node:ty),+ $(,)?) => { + $( + impl stream_private::StreamResourceNodeSealed for $node {} + impl StreamResourceNode for $node {} + )+ + }; +} + +stream_resource_node!( + AccelerationStructureNode, + AccelerationStructureLeaseNode, + BufferNode, + BufferLeaseNode, + ImageNode, + ImageLeaseNode, + SwapchainImageNode, +); + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + driver::{ + descriptor_set::{DescriptorPool, DescriptorPoolInfo}, + render_pass::{RenderPass, RenderPassInfo}, + }, + pool::Pool, + }; + + struct NoopPool; + + impl Pool for NoopPool { + fn resource( + &mut self, + _: DescriptorPoolInfo, + ) -> Result, DriverError> { + unreachable!() + } + } + + impl Pool for NoopPool { + fn resource( + &mut self, + _: RenderPassInfo, + ) -> Result, DriverError> { + unreachable!() + } + } + + #[test] + fn empty_stream_can_be_inserted() { + let stream = CommandStream::finalize(|_| {}).into_stream(); + let mut graph = Graph::new(); + + graph.insert_cmd_stream(&stream).finish(); + } + + #[test] + fn reusable_callback_can_prepare_stream() { + let stream = CommandStream::finalize(|stream| { + stream.begin_cmd().record_cmd(|_| {}); + }) + .into_stream(); + let mut graph = Graph::new(); + + graph.insert_cmd_stream(&stream).finish(); + assert_eq!(graph.cmds.len(), 1); + } + + #[test] + fn reusable_callback_can_prepare_optimized_stream() { + let mut pool = NoopPool; + let stream = CommandStream::prepare(&mut pool, |stream| { + stream.begin_cmd().record_cmd(|_| {}); + }) + .expect("prepare stream"); + let mut graph = Graph::new(); + + graph.insert_cmd_stream(&stream).finish(); + assert_eq!(graph.cmds.len(), 1); + } + + #[test] + fn unprepared_stream_expands_commands() { + let stream = CommandStream::finalize(|stream| { + stream.begin_cmd().record_cmd(|_| {}); + stream.begin_cmd().record_cmd(|_| {}); + }) + .into_stream(); + let mut graph = Graph::new(); + + graph.insert_cmd_stream(&stream).finish(); + assert_eq!(graph.cmds.len(), 2); + } + + #[test] + fn prepared_stream_is_opaque_by_default() { + let mut pool = NoopPool; + let stream = CommandStream::prepare(&mut pool, |stream| { + stream.begin_cmd().record_cmd(|_| {}); + stream.begin_cmd().record_cmd(|_| {}); + }) + .expect("prepare stream"); + let mut graph = Graph::new(); + + graph.insert_cmd_stream(&stream).finish(); + assert_eq!(graph.cmds.len(), 1); + } + + #[test] + fn image_arg_can_use_info_based_helpers() { + let stream = CommandStream::finalize(|stream| { + let output = stream.arg(ImageInfo::image_2d( + 1, + 1, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::TRANSFER_DST, + )); + + stream.clear_color_image(output, [0.0, 0.0, 0.0, 0.0]); + + output + }) + .into_stream(); + + assert_eq!(stream.inner.args.len(), 1); + } + + #[test] + #[should_panic(expected = "missing command stream argument")] + fn missing_arg_panics_at_finish() { + let stream = CommandStream::finalize(|stream| { + stream.arg(ImageInfo::image_2d( + 1, + 1, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::SAMPLED, + )); + }) + .into_stream(); + let mut graph = Graph::new(); + + graph.insert_cmd_stream(&stream).finish(); + } + + #[test] + #[cfg(feature = "checked")] + #[should_panic(expected = "argument belongs to a different command stream")] + fn wrong_stream_arg_panics_at_with_arg() { + let stream_a = CommandStream::finalize(|stream| { + stream.arg(ImageInfo::image_2d( + 1, + 1, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::SAMPLED, + )) + }) + .into_stream(); + let stream_b = CommandStream::finalize(|stream| { + stream.arg(ImageInfo::image_2d( + 1, + 1, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::SAMPLED, + )) + }) + .into_stream(); + let mut graph = Graph::new(); + + graph + .insert_cmd_stream(&stream_a) + .with_arg(stream_b.args, AnyImageNode::from(stream_b.args)) + .finish(); + } + + #[test] + #[should_panic(expected = "stream argument cannot be supplied as a parent graph node")] + fn stream_arg_cannot_bind_as_parent_graph_node() { + let stream = CommandStream::finalize(|stream| { + stream.arg(ImageInfo::image_2d( + 1, + 1, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::SAMPLED, + )) + }) + .into_stream(); + let mut graph = Graph::new(); + + graph + .insert_cmd_stream(&stream) + .with_arg(stream.args, AnyImageNode::from(stream.args)) + .finish(); + } + + #[test] + #[should_panic(expected = "command stream contains a one-shot callback")] + fn one_shot_callback_cannot_prepare_stream() { + let _ = CommandStream::finalize(|stream| { + stream.graph.begin_cmd().record_cmd(|_| {}); + }); + } +} diff --git a/src/submission.rs b/src/submission.rs index a02921f3..312f9003 100644 --- a/src/submission.rs +++ b/src/submission.rs @@ -1,46 +1,244 @@ +//! Submission and recording types. +//! +//! This module contains the execution-facing types produced by [`Graph::finalize`]. +//! +//! Typical usage starts with a [`Submission`], which represents a finalized graph that has not yet +//! been bound to a command buffer: +//! +//! - Use [`Submission::queue_submit`] for the one-shot path that allocates, records, and submits a +//! command buffer internally. +//! - Use [`Submission::record`] with a [`RecordSelection`] to bind the submission to an existing +//! command buffer and obtain a [`Recording`]. +//! +//! A [`Recording`] keeps the remaining graph work paired with the command buffer it was +//! recorded into. This typestate prevents recording with one command buffer and accidentally +//! submitting with another. +//! +//! [`Graph::finalize`]: crate::Graph::finalize + use { super::{ - AnyResource, Attachment, CommandData, ExecutionPipeline, Graph, Node, NodeIndex, + AnyResource, Attachment, CommandData, ExecutionAccess, ExecutionPipeline, Graph, LoadOp, + Node, NodeIndex, cmd::{SubresourceAccess, SubresourceRange}, }, crate::{ + StoreOp, + cmd::CommandRef, driver::{ AttachmentInfo, AttachmentRef, Descriptor, DescriptorInfo, DescriptorSet, DriverError, - FramebufferAttachmentImageInfo, FramebufferInfo, SubpassDependency, SubpassInfo, + FramebufferAttachmentImageInfo, FramebufferInfo, SharingMode, SubpassDependency, + SubpassInfo, accel_struct::AccelerationStructure, - buffer::Buffer, + buffer::{Buffer, BufferSubresourceRange}, cmd_buf::{CommandBuffer, CommandBufferInfo}, descriptor_set::{DescriptorPool, DescriptorPoolInfo}, device::Device, + fence::Fence, format_aspect_mask, - graphic::{DepthStencilInfo, GraphicsPipeline}, - image::{DenseAccess, Image}, - initial_image_layout_access, is_read_access, is_write_access, - pipeline_stage_access_flags, + graphics::{DepthStencilInfo, GraphicsPipeline}, + image::{ + DenseMap, Image, image_subresource_range_contains, + image_subresource_range_intersection, + }, + initial_image_layout_access, is_read_access, pipeline_stage_access_flags, render_pass::{RenderPass, RenderPassInfo}, }, - pool::{Lease, Pool}, + lazy_str, + node::AnyNode, + pool::{Lease, Pool, SubmissionPool}, }, ash::vk, + fixedbitset::FixedBitSet, log::{ Level::{Debug, Trace}, debug, log_enabled, trace, warn, }, + smallvec::SmallVec, std::{ cell::RefCell, - collections::{BTreeMap, HashMap, VecDeque}, + collections::{BTreeMap, HashMap, HashSet, VecDeque}, + fmt::Debug as FmtDebug, iter::repeat_n, + mem::take, ops::Range, slice, + sync::{Arc, Mutex}, }, vk_sync::{ - AccessType, BufferBarrier, GlobalBarrier, ImageBarrier, ImageLayout, cmd::pipeline_barrier, + AccessType, BufferBarrier, GlobalBarrier, ImageBarrier, ImageLayout, + get_buffer_memory_barrier, get_image_memory_barrier, get_memory_barrier, }, }; #[cfg(not(feature = "checked"))] use std::hint::unreachable_unchecked; +thread_local! { + static SUBMIT: RefCell = Default::default(); +} + +fn aspect_mask_for_span(base_aspect: u32, start: u32, end: u32) -> vk::ImageAspectFlags { + let mut mask = vk::ImageAspectFlags::empty(); + + for ordinal in start..end { + mask |= vk::ImageAspectFlags::from_raw(1 << (base_aspect + ordinal)); + } + + mask +} + +fn buffer_barriers_from_transfers<'a>( + buffer: vk::Buffer, + prev_access: &'a AccessType, + next_access: &'a AccessType, + range: BufferSubresourceRange, + transfers: &'a [BufferQueueOwnershipTransfer], +) -> impl Iterator> + 'a { + struct BufferBarrierIter<'a> { + buffer: vk::Buffer, + cuts: SmallVec<[vk::DeviceSize; 4]>, + cut_idx: usize, + next_access: &'a AccessType, + prev_access: &'a AccessType, + transfers: &'a [BufferQueueOwnershipTransfer], + } + + impl<'a> Iterator for BufferBarrierIter<'a> { + type Item = BufferBarrier<'a>; + + fn next(&mut self) -> Option { + while self.cut_idx + 1 < self.cuts.len() { + let range = BufferSubresourceRange { + start: self.cuts[self.cut_idx], + end: self.cuts[self.cut_idx + 1], + }; + self.cut_idx += 1; + + if range.start == range.end { + continue; + } + + let transfer = self + .transfers + .iter() + .find(|transfer| transfer.range.contains(range)); + + trace!( + " buffer {:?} {:?} {:?}->{:?}", + self.buffer, + range.start..range.end, + self.prev_access, + self.next_access, + ); + + return Some(BufferBarrier { + next_accesses: slice::from_ref(self.next_access), + previous_accesses: slice::from_ref(self.prev_access), + src_queue_family_index: transfer.map_or(vk::QUEUE_FAMILY_IGNORED, |transfer| { + transfer.src_queue_family_index + }), + dst_queue_family_index: transfer.map_or(vk::QUEUE_FAMILY_IGNORED, |transfer| { + transfer.dst_queue_family_index + }), + buffer: self.buffer, + offset: range.start as _, + size: (range.end - range.start) as _, + }); + } + + None + } + } + + let mut cuts = SmallVec::<[vk::DeviceSize; 4]>::with_capacity( + transfers.len().saturating_mul(2).saturating_add(2), + ); + cuts.extend([range.start, range.end]); + + for transfer in transfers { + if let Some(overlap) = range.intersection(transfer.range) { + cuts.push(overlap.start); + cuts.push(overlap.end); + } + } + + cuts.sort_unstable(); + cuts.dedup(); + + BufferBarrierIter { + buffer, + cuts, + cut_idx: 0, + next_access, + prev_access, + transfers, + } +} + +fn buffer_subresource_range_intersects( + lhs: BufferSubresourceRange, + rhs: BufferSubresourceRange, +) -> bool { + lhs.start < rhs.end && lhs.end > rhs.start +} + +fn check_queue_submit_args( + waits: &[SemaphoreSubmitInfo], + signals: &[SemaphoreSubmitInfo], +) -> Result<(), DriverError> { + waits + .iter() + .chain(signals.iter()) + .all(SemaphoreSubmitInfo::is_supported_legacy_submit) + .then_some(()) + .ok_or(DriverError::Unsupported) +} + +fn check_queue_submit2_args( + device: &Device, + waits: &[SemaphoreSubmit2Info], + signals: &[SemaphoreSubmit2Info], +) -> Result<(), DriverError> { + if !device.physical.vk_khr_synchronization2 { + return Err(DriverError::Unsupported); + } + + if (waits.iter().any(|wait| wait.value != 0) || signals.iter().any(|signal| signal.value != 0)) + && !supports_timeline_semaphores(device) + { + return Err(DriverError::Unsupported); + } + + Ok(()) +} + +fn consume_pending_buffer_transfers( + transfers: &mut Vec, + range: BufferSubresourceRange, +) -> bool { + transfers.retain(|transfer| !buffer_subresource_range_intersects(transfer.range, range)); + transfers.is_empty() +} + +fn consume_pending_image_transfers( + transfers: &mut Vec, + range: vk::ImageSubresourceRange, +) -> bool { + transfers + .retain(|transfer| image_subresource_range_intersection(transfer.range, range).is_none()); + transfers.is_empty() +} + +fn exclusive_transfer_source(sharing: SharingMode, queue_family_index: u32) -> Option<(u32, u32)> { + let SharingMode::Exclusive(Some((src_queue_family_index, src_queue_index))) = sharing else { + return None; + }; + + (src_queue_family_index != queue_family_index) + .then_some((src_queue_family_index, src_queue_index)) +} + const fn image_access_layout(access: AccessType) -> ImageLayout { if matches!(access, AccessType::Present | AccessType::ComputeShaderWrite) { ImageLayout::General @@ -49,3320 +247,8022 @@ const fn image_access_layout(access: AccessType) -> ImageLayout { } } -#[derive(Default)] -struct AccessCache { - accesses: Vec, - binding_count: usize, - read_count: Vec, - reads: Vec, +fn image_barriers_from_transfers<'a>( + image: vk::Image, + prev_access: &'a AccessType, + next_access: &'a AccessType, + range: vk::ImageSubresourceRange, + transfers: &'a [ImageQueueOwnershipTransfer], + discard_contents: bool, +) -> impl Iterator> + 'a { + image_barrier_transfer_ranges(transfers, range).map(move |(range, transfer)| { + trace!( + " image {:?} {:?} {:?}->{:?}", + image, + ImageSubresourceRangeDebug(range), + prev_access, + next_access, + ); + + ImageBarrier { + next_accesses: slice::from_ref(next_access), + next_layout: image_access_layout(*next_access), + previous_accesses: slice::from_ref(prev_access), + previous_layout: image_access_layout(*prev_access), + discard_contents, + src_queue_family_index: transfer.map_or(vk::QUEUE_FAMILY_IGNORED, |transfer| { + transfer.src_queue_family_index + }), + dst_queue_family_index: transfer.map_or(vk::QUEUE_FAMILY_IGNORED, |transfer| { + transfer.dst_queue_family_index + }), + image, + range, + } + }) } -impl AccessCache { - /// Finds the unique indexes of the node bindings which a given pass reads. Results are - /// returned in the opposite order the dependencies must be resolved in. - /// - /// Dependent upon means that the node is read from the pass. - #[profiling::function] - fn dependent_nodes(&self, pass_idx: usize) -> impl ExactSizeIterator + '_ { - let pass_start = pass_idx * self.binding_count; - let pass_end = pass_start + self.read_count[pass_idx]; - self.reads[pass_start..pass_end].iter().copied() +fn image_barrier_transfer_ranges<'a>( + transfers: &'a [ImageQueueOwnershipTransfer], + range: vk::ImageSubresourceRange, +) -> impl Iterator< + Item = ( + vk::ImageSubresourceRange, + Option<&'a ImageQueueOwnershipTransfer>, + ), +> + 'a { + thread_local! { + static IMAGE_TRANSFER: RefCell = Default::default(); } - /// Finds the unique indexes of the passes which write to a given node; with the restriction - /// to not inspect later passes. Results are returned in the opposite order the dependencies - /// must be resolved in. - /// - /// Dependent upon means that the pass writes to the node. - #[profiling::function] - fn dependent_passes( - &self, - node_idx: usize, - end_pass_idx: usize, - ) -> impl Iterator + '_ { - self.accesses[node_idx..end_pass_idx * self.binding_count] - .iter() - .step_by(self.binding_count) - .enumerate() - .rev() - .filter_map(|(pass_idx, write)| write.then_some(pass_idx)) + #[derive(Default)] + struct ImageTransferScratch { + overlaps: Vec<(usize, vk::ImageSubresourceRange)>, + aspect_cuts: Vec, + layer_cuts: Vec, + mip_cuts: Vec, } - /// Returns the unique indexes of the passes which are dependent on the given pass. - #[profiling::function] - fn interdependent_passes( - &self, - pass_idx: usize, - end_pass_idx: usize, - ) -> impl Iterator + '_ { - self.dependent_nodes(pass_idx) - .flat_map(move |node_idx| self.dependent_passes(node_idx, end_pass_idx)) + struct ImageBarrierTransferIter<'a> { + transfers: &'a [ImageQueueOwnershipTransfer], + overlaps: Vec<(usize, vk::ImageSubresourceRange)>, + aspect_cuts: Vec, + layer_cuts: Vec, + mip_cuts: Vec, + base_aspect: u32, + range: vk::ImageSubresourceRange, + aspect_idx: usize, + layer_idx: usize, + mip_idx: usize, + yielded_empty: bool, } - fn update(&mut self, graph: &Graph, end_pass_idx: usize) { - self.binding_count = graph.resources.len(); - - let cache_len = self.binding_count * end_pass_idx; + impl<'a> Iterator for ImageBarrierTransferIter<'a> { + type Item = ( + vk::ImageSubresourceRange, + Option<&'a ImageQueueOwnershipTransfer>, + ); - self.accesses.truncate(cache_len); - self.accesses.fill(false); - self.accesses.resize(cache_len, false); + fn next(&mut self) -> Option { + if self.overlaps.is_empty() { + return if self.yielded_empty { + None + } else { + self.yielded_empty = true; + Some((self.range, None)) + }; + } - self.read_count.clear(); + let aspect_windows = self.aspect_cuts.len().saturating_sub(1); + let layer_windows = self.layer_cuts.len().saturating_sub(1); + let mip_windows = self.mip_cuts.len().saturating_sub(1); + + while self.aspect_idx < aspect_windows { + let aspect_start = self.aspect_cuts[self.aspect_idx]; + let aspect_end = self.aspect_cuts[self.aspect_idx + 1]; + if aspect_start == aspect_end { + self.aspect_idx += 1; + self.layer_idx = 0; + self.mip_idx = 0; + continue; + } - self.reads.truncate(cache_len); - self.reads.fill(usize::MAX); - self.reads.resize(cache_len, usize::MAX); + let aspect_mask = aspect_mask_for_span(self.base_aspect, aspect_start, aspect_end); - thread_local! { - static NODES: RefCell> = Default::default(); - } + while self.layer_idx < layer_windows { + let layer_start = self.layer_cuts[self.layer_idx]; + let layer_end = self.layer_cuts[self.layer_idx + 1]; + if layer_start == layer_end { + self.layer_idx += 1; + self.mip_idx = 0; + continue; + } - NODES.with_borrow_mut(|nodes| { - nodes.truncate(self.binding_count); - nodes.fill(true); - nodes.resize(self.binding_count, true); + while self.mip_idx < mip_windows { + let mip_start = self.mip_cuts[self.mip_idx]; + let mip_end = self.mip_cuts[self.mip_idx + 1]; + self.mip_idx += 1; + if mip_start == mip_end { + continue; + } - for (pass_idx, pass) in graph.cmds[0..end_pass_idx].iter().enumerate() { - let pass_start = pass_idx * self.binding_count; - let mut read_count = 0; + let subrange = vk::ImageSubresourceRange { + aspect_mask, + base_array_layer: self.range.base_array_layer + layer_start, + layer_count: layer_end - layer_start, + base_mip_level: self.range.base_mip_level + mip_start, + level_count: mip_end - mip_start, + }; - for (&node_idx, accesses) in pass.execs.iter().flat_map(|exec| exec.accesses.iter()) - { - self.accesses[pass_start + node_idx] = true; + let transfer = self + .overlaps + .iter() + .find(|(_, overlap)| { + image_subresource_range_contains(*overlap, subrange) + }) + .map(|(transfer_idx, _)| &self.transfers[*transfer_idx]); - if nodes[node_idx] - && is_read_access(accesses.first().expect("missing resource access").access) - { - self.reads[pass_start + read_count] = node_idx; - nodes[node_idx] = false; - read_count += 1; + return Some((subrange, transfer)); } - } - if pass_idx + 1 < end_pass_idx { - nodes.fill(true); + self.layer_idx += 1; + self.mip_idx = 0; } - self.read_count.push(read_count); + self.aspect_idx += 1; + self.layer_idx = 0; + self.mip_idx = 0; } - }); + + None + } } -} -struct ImageSubresourceRangeDebug(vk::ImageSubresourceRange); + impl Drop for ImageBarrierTransferIter<'_> { + fn drop(&mut self) { + IMAGE_TRANSFER.with_borrow_mut(|tls| { + tls.overlaps = take(&mut self.overlaps); + tls.aspect_cuts = take(&mut self.aspect_cuts); + tls.layer_cuts = take(&mut self.layer_cuts); + tls.mip_cuts = take(&mut self.mip_cuts); + }); + } + } -impl std::fmt::Debug for ImageSubresourceRangeDebug { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.aspect_mask.fmt(f)?; + IMAGE_TRANSFER.with_borrow_mut(|tls| { + let mut overlaps = take(&mut tls.overlaps); + let mut aspect_cuts = take(&mut tls.aspect_cuts); + let mut layer_cuts = take(&mut tls.layer_cuts); + let mut mip_cuts = take(&mut tls.mip_cuts); - f.write_str(" array: ")?; + overlaps.clear(); + aspect_cuts.clear(); + layer_cuts.clear(); + mip_cuts.clear(); - let array_layers = self.0.base_array_layer..self.0.base_array_layer + self.0.layer_count; - array_layers.fmt(f)?; + overlaps.extend( + transfers + .iter() + .enumerate() + .filter_map(|(transfer_idx, transfer)| { + image_subresource_range_intersection(transfer.range, range) + .map(|intersection| (transfer_idx, intersection)) + }), + ); - f.write_str(" mip: ")?; + let base_aspect = range.aspect_mask.as_raw().trailing_zeros(); - let mip_levels = self.0.base_mip_level..self.0.base_mip_level + self.0.level_count; - mip_levels.fmt(f) - } -} + if overlaps.is_empty() { + // Yield the whole range once when there is no overlapping transfer + } else { + let aspect_count = range.aspect_mask.as_raw().count_ones(); + + aspect_cuts.extend([0, aspect_count]); + layer_cuts.extend([0, range.layer_count]); + mip_cuts.extend([0, range.level_count]); + + for (_, overlap) in &overlaps { + let aspect_start = overlap.aspect_mask.as_raw().trailing_zeros() - base_aspect; + let aspect_end = aspect_start + overlap.aspect_mask.as_raw().count_ones(); + aspect_cuts.push(aspect_start); + aspect_cuts.push(aspect_end); + + let layer_start = overlap.base_array_layer - range.base_array_layer; + let layer_end = layer_start + overlap.layer_count; + layer_cuts.push(layer_start); + layer_cuts.push(layer_end); + + let mip_start = overlap.base_mip_level - range.base_mip_level; + let mip_end = mip_start + overlap.level_count; + mip_cuts.push(mip_start); + mip_cuts.push(mip_end); + } -#[derive(Debug)] -struct PhysicalPass { - descriptor_pool: Option>, - exec_descriptor_sets: HashMap>, - render_pass: Option>, + aspect_cuts.sort_unstable(); + aspect_cuts.dedup(); + layer_cuts.sort_unstable(); + layer_cuts.dedup(); + mip_cuts.sort_unstable(); + mip_cuts.dedup(); + } + + ImageBarrierTransferIter { + transfers, + overlaps, + aspect_cuts, + layer_cuts, + mip_cuts, + base_aspect, + range, + aspect_idx: 0, + layer_idx: 0, + mip_idx: 0, + yielded_empty: false, + } + }) } -impl PhysicalPass { - /// # Panics - /// - /// Panics if the physical pass has no render pass. - fn expect_render_pass_mut(&mut self) -> &mut Lease { - self.render_pass.as_mut().expect("missing render pass") - } +fn image_execution_discard_contents(prev_access: AccessType) -> bool { + prev_access == AccessType::Nothing } -impl Drop for PhysicalPass { - fn drop(&mut self) { - self.exec_descriptor_sets.clear(); - self.descriptor_pool = None; - } +fn image_layout_transition_discard_contents( + prev_access: AccessType, + next_access: AccessType, +) -> bool { + // Read/modify/write accesses must preserve the existing image contents + // Check for "not-read" here because some accesses both read and write + // Color Attachment Read/Write (blending) will prevent discarding contents + prev_access == AccessType::Nothing || !is_read_access(next_access) } -/// A structure which can optimize and submit [`Graph`] instances. -/// -/// This pattern was derived from: -/// -/// -/// -#[derive(Debug)] -pub struct Submission { - graph: Graph, - physical_passes: Vec, +fn image_subresource_range_eq( + lhs: vk::ImageSubresourceRange, + rhs: vk::ImageSubresourceRange, +) -> bool { + lhs.aspect_mask == rhs.aspect_mask + && lhs.base_array_layer == rhs.base_array_layer + && lhs.layer_count == rhs.layer_count + && lhs.base_mip_level == rhs.base_mip_level + && lhs.level_count == rhs.level_count } -impl Submission { - pub(super) fn new(graph: Graph) -> Self { - let physical_passes = Vec::with_capacity(graph.cmds.len()); +// Added because vk-sync requires allocation to record barriers, see that impl for reference +fn pipeline_barrier_from_iters<'a>( + device: &Device, + command_buffer: vk::CommandBuffer, + global_barrier: Option>, + buffer_barriers: impl IntoIterator>, + image_barriers: impl IntoIterator>, +) { + #[derive(Default)] + struct BarrierScratch { + memory_barriers: Vec>, + buffer_barriers: Vec>, + image_barriers: Vec>, + } - Self { - graph, - physical_passes, - } + thread_local! { + static BARRIER: RefCell = Default::default(); } - #[profiling::function] - fn allow_merge_passes(lhs: &CommandData, rhs: &CommandData) -> bool { - fn first_graphic_pipeline(pass: &CommandData) -> Option<&GraphicsPipeline> { - pass.execs - .first() - .and_then(|exec| exec.pipeline.as_ref().map(ExecutionPipeline::as_graphic)) - .flatten() + BARRIER.with_borrow_mut(|tls| { + tls.memory_barriers.clear(); + tls.buffer_barriers.clear(); + tls.image_barriers.clear(); + + let mut src_stage_mask = vk::PipelineStageFlags::TOP_OF_PIPE; + let mut dst_stage_mask = vk::PipelineStageFlags::BOTTOM_OF_PIPE; + + if let Some(ref barrier) = global_barrier { + let (src_mask, dst_mask, barrier) = get_memory_barrier(barrier); + src_stage_mask |= src_mask; + dst_stage_mask |= dst_mask; + tls.memory_barriers.push(vk::MemoryBarrier { + src_access_mask: barrier.src_access_mask, + dst_access_mask: barrier.dst_access_mask, + ..Default::default() + }); } - fn is_multiview(view_mask: u32) -> bool { - view_mask != 0 + for buffer_barrier in buffer_barriers { + let (src_mask, dst_mask, barrier) = get_buffer_memory_barrier(&buffer_barrier); + src_stage_mask |= src_mask; + dst_stage_mask |= dst_mask; + tls.buffer_barriers.push(vk::BufferMemoryBarrier { + src_access_mask: barrier.src_access_mask, + dst_access_mask: barrier.dst_access_mask, + src_queue_family_index: barrier.src_queue_family_index, + dst_queue_family_index: barrier.dst_queue_family_index, + buffer: barrier.buffer, + offset: barrier.offset, + size: barrier.size, + ..Default::default() + }); } - let lhs_pipeline = first_graphic_pipeline(lhs); - if lhs_pipeline.is_none() { - trace!(" {} is not graphic", lhs.name()); - - return false; + for image_barrier in image_barriers { + let (src_mask, dst_mask, barrier) = get_image_memory_barrier(&image_barrier); + src_stage_mask |= src_mask; + dst_stage_mask |= dst_mask; + tls.image_barriers.push(vk::ImageMemoryBarrier { + src_access_mask: barrier.src_access_mask, + dst_access_mask: barrier.dst_access_mask, + old_layout: barrier.old_layout, + new_layout: barrier.new_layout, + src_queue_family_index: barrier.src_queue_family_index, + dst_queue_family_index: barrier.dst_queue_family_index, + image: barrier.image, + subresource_range: barrier.subresource_range, + ..Default::default() + }); } - let rhs_pipeline = first_graphic_pipeline(rhs); - if rhs_pipeline.is_none() { - trace!(" {} is not graphic", rhs.name()); + unsafe { + device.cmd_pipeline_barrier( + command_buffer, + src_stage_mask, + dst_stage_mask, + vk::DependencyFlags::empty(), + tls.memory_barriers.as_slice(), + tls.buffer_barriers.as_slice(), + tls.image_barriers.as_slice(), + ); + } + }); +} - return false; +fn schedule_dependency_cmds_before_target_access( + access_index: &CommandAccessIndex, + target_node_idx: usize, + first_target_cmd_idx: usize, + schedule: &mut Vec, +) { + let mut pending_nodes = VecDeque::new(); + let mut scheduled = FixedBitSet::with_capacity(first_target_cmd_idx); + let mut queued_nodes = FixedBitSet::with_capacity(access_index.cmds_by_node.len()); + + for node_idx in access_index.read_nodes_for_cmd(first_target_cmd_idx) { + if node_idx != target_node_idx { + pending_nodes.push_back((node_idx, first_target_cmd_idx)); + queued_nodes.insert(node_idx); } + } - let lhs_pipeline = unsafe { lhs_pipeline.unwrap_unchecked() }; - let rhs_pipeline = unsafe { rhs_pipeline.unwrap_unchecked() }; + while let Some((node_idx, end_cmd_idx)) = pending_nodes.pop_front() { + for cmd_idx in access_index.prior_cmds_for_node(node_idx, end_cmd_idx) { + if scheduled.put(cmd_idx) { + continue; + } - // Must be same general rasterization modes - let lhs_info = lhs_pipeline.inner.info; - let rhs_info = rhs_pipeline.inner.info; - if lhs_info.blend != rhs_info.blend - || lhs_info.cull_mode != rhs_info.cull_mode - || lhs_info.front_face != rhs_info.front_face - || lhs_info.polygon_mode != rhs_info.polygon_mode - || lhs_info.samples != rhs_info.samples - { - trace!(" different rasterization modes",); + schedule.push(cmd_idx); - return false; - } + for read_node_idx in access_index.read_nodes_for_cmd(cmd_idx) { + if queued_nodes.put(read_node_idx) { + continue; + } - let rhs = rhs.execs.first(); + pending_nodes.push_back((read_node_idx, cmd_idx)); + } + } + } - // PassRef makes sure this never happens - debug_assert!(rhs.is_some()); + schedule.sort_unstable(); +} - let rhs = unsafe { rhs.unwrap_unchecked() }; +fn submit_stage_mask_legacy(stage_mask: vk::PipelineStageFlags2) -> vk::PipelineStageFlags { + match stage_mask { + vk::PipelineStageFlags2::NONE => vk::PipelineStageFlags::ALL_COMMANDS, + vk::PipelineStageFlags2::ALL_COMMANDS => vk::PipelineStageFlags::ALL_COMMANDS, + _ => { + #[cfg(feature = "checked")] + panic!("invalid legacy submit wait stage mask: {stage_mask:?}"); - let mut common_color_attachment = false; - let mut common_depth_attachment = false; + #[cfg(not(feature = "checked"))] + { + vk::PipelineStageFlags::ALL_COMMANDS + } + } + } +} - // Now we need to know what the subpasses (we may have prior merges) wrote - for lhs in lhs.execs.iter().rev() { - // Multiview subpasses cannot be combined with non-multiview subpasses - if is_multiview(lhs.view_mask) != is_multiview(rhs.view_mask) { - trace!(" incompatible multiview"); +fn supports_timeline_semaphores(device: &Device) -> bool { + device.physical.features_v1_2.timeline_semaphore +} - return false; +/// Builds and submits a release barrier command buffer for each release group, calling +/// `submit_release` to perform the final queue submission. +fn submit_queue_ownership_releases

( + pool: &mut P, + release_groups: &[QueueOwnershipReleaseGroup], + target_queue_family_index: u32, + submit_release: impl Fn( + &Device, + vk::Queue, + vk::CommandBuffer, + vk::Fence, + vk::Semaphore, + ) -> Result<(), DriverError>, +) -> Result, DriverError> +where + P: Pool, +{ + let mut releases = Vec::new(); + + if !release_groups.is_empty() { + for group in release_groups { + let mut release_cmd = + pool.resource(CommandBufferInfo::new(group.src_queue_family_index as _))?; + let mut release_fence = Fence::create(&release_cmd.device, false)?; + + #[cfg(feature = "checked")] + { + release_fence.wait_signaled()?; + release_fence.reset()?; } - // Compare individual color attachments for compatibility - for (attachment_idx, lhs_attachment) in lhs - .color_attachments - .iter() - .chain(lhs.color_loads.iter()) - .chain(lhs.color_stores.iter()) - .chain( - lhs.color_clears - .iter() - .map(|(attachment_idx, (attachment, _))| (attachment_idx, attachment)), - ) - .chain( - lhs.color_resolves - .iter() - .map(|(attachment_idx, (attachment, _))| (attachment_idx, attachment)), - ) - { - let rhs_attachment = rhs - .color_attachments - .get(attachment_idx) - .or_else(|| rhs.color_loads.get(attachment_idx)) - .or_else(|| rhs.color_stores.get(attachment_idx)) - .or_else(|| { - rhs.color_clears - .get(attachment_idx) - .map(|(attachment, _)| attachment) - }) - .or_else(|| { - rhs.color_resolves - .get(attachment_idx) - .map(|(attachment, _)| attachment) - }); + let semaphore = release_cmd.release_semaphore()?; - if !Attachment::are_compatible(Some(*lhs_attachment), rhs_attachment.copied()) { - trace!(" incompatible color attachments"); + release_cmd.set_debug_name(lazy_str!( + "queue ownership release qf{}:{} -> qf{}", + group.src_queue_family_index, + group.src_queue_index, + target_queue_family_index + )); - return false; - } + Device::begin_command_buffer( + &release_cmd.device, + release_cmd.handle, + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; - common_color_attachment = true; - } + { + let _ = CommandBufferDebugLabel::begin( + &release_cmd, + lazy_str!( + "queue ownership release qf{}:{} -> qf{}", + group.src_queue_family_index, + group.src_queue_index, + target_queue_family_index + ), + ); - // Compare depth/stencil attachments for compatibility - let lhs_depth_stencil = lhs - .depth_stencil_attachment - .or(lhs.depth_stencil_load) - .or(lhs.depth_stencil_store) - .or_else(|| lhs.depth_stencil_resolve.map(|(attachment, ..)| attachment)) - .or_else(|| lhs.depth_stencil_clear.map(|(attachment, _)| attachment)); + SUBMIT.with_borrow_mut(|tls| { + let _ = CommandBufferDebugLabel::begin(&release_cmd, "queue ownership barrier"); + + tls.release_image_barriers.clear(); + tls.release_buffer_barriers.clear(); + tls.release_buffer_barriers.reserve(group.buffers.len()); + tls.release_image_barriers.reserve(group.images.len()); + + tls.release_buffer_barriers.extend(group.buffers.iter().map( + |&(handle, range)| { + vk::BufferMemoryBarrier::default() + .src_access_mask(vk::AccessFlags::MEMORY_WRITE) + .dst_access_mask(vk::AccessFlags::empty()) + .src_queue_family_index(group.src_queue_family_index) + .dst_queue_family_index(target_queue_family_index) + .buffer(handle) + .offset(range.start) + .size(range.end - range.start) + }, + )); - let rhs_depth_stencil = rhs - .depth_stencil_attachment - .or(rhs.depth_stencil_load) - .or(rhs.depth_stencil_store) - .or_else(|| rhs.depth_stencil_resolve.map(|(attachment, ..)| attachment)) - .or_else(|| rhs.depth_stencil_clear.map(|(attachment, _)| attachment)); + tls.release_image_barriers.extend(group.images.iter().map( + |&(handle, current_layout, subresource_range)| { + vk::ImageMemoryBarrier::default() + .src_access_mask(vk::AccessFlags::MEMORY_WRITE) + .dst_access_mask(vk::AccessFlags::empty()) + .old_layout(current_layout) + .new_layout(current_layout) + .src_queue_family_index(group.src_queue_family_index) + .dst_queue_family_index(target_queue_family_index) + .image(handle) + .subresource_range(subresource_range) + }, + )); - if !Attachment::are_compatible(lhs_depth_stencil, rhs_depth_stencil) { - trace!(" incompatible depth/stencil attachments"); + unsafe { + release_cmd.device.cmd_pipeline_barrier( + release_cmd.handle, + vk::PipelineStageFlags::ALL_COMMANDS, + vk::PipelineStageFlags::ALL_COMMANDS, + vk::DependencyFlags::empty(), + &[], + tls.release_buffer_barriers.as_slice(), + tls.release_image_barriers.as_slice(), + ); + } + }); - return false; + Device::with_queue( + &release_cmd.device, + group.src_queue_family_index, + group.src_queue_index, + |queue| { + Device::end_command_buffer(&release_cmd.device, release_cmd.handle)?; + submit_release( + &release_cmd.device, + queue, + release_cmd.handle, + release_fence.handle, + semaphore, + )?; + + release_fence.mark_queued(); + + Ok::<_, DriverError>(()) + }, + )?; } - common_depth_attachment |= lhs_depth_stencil.is_some() && rhs_depth_stencil.is_some(); + releases.push(QueueOwnershipRelease { + _cmd_buf: release_cmd, + _fence: release_fence, + semaphore, + }); } + } - // Keep color and depth on tile. - if common_color_attachment || common_depth_attachment { - trace!(" merging due to common image"); + Ok(releases) +} - return true; - } +#[derive(Clone, Copy, Debug)] +struct BufferQueueOwnershipTransfer { + range: BufferSubresourceRange, + dst_queue_family_index: u32, + src_queue_family_index: u32, + src_queue_index: u32, +} - // Keep input on tile - if !rhs_pipeline.inner.input_attachments.is_empty() { - trace!(" merging due to subpass input"); +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct BufferSubresourceRangeKey { + start: vk::DeviceSize, + end: vk::DeviceSize, +} - return true; +impl BufferSubresourceRangeKey { + fn from_range(BufferSubresourceRange { start, end }: BufferSubresourceRange) -> Self { + Self { start, end } + } + + fn into_range(self) -> BufferSubresourceRange { + BufferSubresourceRange { + start: self.start, + end: self.end, } + } +} - trace!(" not merging"); +#[derive(Clone, Default)] +struct CommandAccessIndex { + cmds_by_node: Vec>, + accessed_nodes_by_cmd: Vec>, +} - // No reason to merge, so don't. - false +impl CommandAccessIndex { + #[profiling::function] + fn read_nodes_for_cmd(&self, cmd_idx: usize) -> impl ExactSizeIterator + '_ { + self.accessed_nodes_by_cmd[cmd_idx].iter().copied() } - fn attachment_layout( - aspect_mask: vk::ImageAspectFlags, - is_random_access: bool, - is_input: bool, - ) -> vk::ImageLayout { - if aspect_mask.contains(vk::ImageAspectFlags::COLOR) { - if is_input { - vk::ImageLayout::GENERAL - } else { - vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL - } - } else if aspect_mask.contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) - { - if is_random_access { - if is_input { - vk::ImageLayout::GENERAL - } else { - vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL - } - } else { - vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL - } - } else if aspect_mask.contains(vk::ImageAspectFlags::DEPTH) { - if is_random_access { - if is_input { - vk::ImageLayout::GENERAL - } else { - vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL - } - } else { - vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL - } - } else if aspect_mask.contains(vk::ImageAspectFlags::STENCIL) { - if is_random_access { - if is_input { - vk::ImageLayout::GENERAL - } else { - vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL - } - } else { - vk::ImageLayout::STENCIL_READ_ONLY_OPTIMAL - } - } else { - vk::ImageLayout::UNDEFINED - } - } + #[profiling::function] + fn prior_cmds_for_node( + &self, + node_idx: usize, + end_cmd_idx: usize, + ) -> impl Iterator + '_ { + let cmds = &self.cmds_by_node[node_idx]; + let end_idx = cmds.partition_point(|&cmd_idx| cmd_idx < end_cmd_idx); - fn expect_attachment_image<'a>( - bindings: &'a [AnyResource], - attachment: &Attachment, - ) -> &'a Image { - bindings[attachment.target] - .as_image() - .expect("invalid attachment target image") + cmds[..end_idx].iter().rev().copied() } #[profiling::function] - fn begin_render_pass( - cmd_buf: &CommandBuffer, - bindings: &[AnyResource], - pass: &CommandData, - physical_pass: &mut PhysicalPass, - render_area: vk::Rect2D, - ) -> Result<(), DriverError> { - trace!(" begin render pass"); + fn prior_read_dependency_cmds( + &self, + cmd_idx: usize, + end_cmd_idx: usize, + ) -> impl Iterator + '_ { + self.read_nodes_for_cmd(cmd_idx) + .flat_map(move |node_idx| self.prior_cmds_for_node(node_idx, end_cmd_idx)) + } - let render_pass = physical_pass.expect_render_pass_mut(); - let attachment_count = render_pass.info.attachments.len(); + fn update(&mut self, graph: &Graph, end_cmd_idx: usize) { + let binding_count = graph.resources.len(); + let cmds = &graph.cmds[0..end_cmd_idx]; + self.update_from_cmds(cmds, binding_count); + } - let mut attachments = Vec::with_capacity(attachment_count); - attachments.resize( - attachment_count, - FramebufferAttachmentImageInfo { - flags: vk::ImageCreateFlags::empty(), - usage: vk::ImageUsageFlags::empty(), - width: 0, - height: 0, - layer_count: 0, - view_formats: vec![], - }, - ); + fn update_from_cmds(&mut self, cmds: &[CommandData], binding_count: usize) { + self.cmds_by_node.clear(); + self.cmds_by_node.resize_with(binding_count, Vec::new); + + self.accessed_nodes_by_cmd.clear(); + self.accessed_nodes_by_cmd.resize_with(cmds.len(), Vec::new); thread_local! { - static CLEARS_VIEWS: RefCell<( - Vec, - Vec, - )> = Default::default(); + static SEEN_NODES: RefCell<(FixedBitSet, FixedBitSet)> = Default::default(); } - CLEARS_VIEWS.with_borrow_mut(|(clear_values, image_views)| { - clear_values.resize_with(attachment_count, vk::ClearValue::default); - image_views.resize(attachment_count, vk::ImageView::null()); + SEEN_NODES.with_borrow_mut(|(seen_nodes, seen_accesses)| { + seen_nodes.clear(); + seen_nodes.grow(binding_count); - for exec in &pass.execs { - for (attachment_idx, (attachment, clear_value)) in &exec.color_clears { - let attachment_image = &mut attachments[*attachment_idx as usize]; - if let Err(idx) = attachment_image - .view_formats - .binary_search(&attachment.format) - { - clear_values[*attachment_idx as usize] = vk::ClearValue { - color: vk::ClearColorValue { - float32: *clear_value, - }, - }; + seen_accesses.clear(); + seen_accesses.grow(binding_count); - let image = Self::expect_attachment_image(bindings, attachment); + for (cmd_idx, cmd) in cmds.iter().enumerate() { + let accessed_nodes = &mut self.accessed_nodes_by_cmd[cmd_idx]; - attachment_image.flags = image.info.flags; - attachment_image.usage = image.info.usage; - attachment_image.width = image.info.width >> attachment.base_mip_level; - attachment_image.height = image.info.height >> attachment.base_mip_level; - attachment_image.layer_count = attachment.array_layer_count; - attachment_image.view_formats.insert(idx, attachment.format); + for (node_idx, _) in cmd.execs.iter().flat_map(|exec| exec.accesses.iter()) { + if !seen_nodes.put(node_idx) { + self.cmds_by_node[node_idx].push(cmd_idx); + } - image_views[*attachment_idx as usize] = - Image::view(image, attachment.image_view_info(image.info))?; + if !seen_accesses.put(node_idx) { + accessed_nodes.push(node_idx); } } - for (attachment_idx, attachment) in exec - .color_attachments - .iter() - .chain(&exec.color_loads) - .chain(&exec.color_stores) - .chain(exec.color_resolves.iter().map( - |(dst_attachment_idx, (attachment, _))| (dst_attachment_idx, attachment), - )) - { - let attachment_image = &mut attachments[*attachment_idx as usize]; - if let Err(idx) = attachment_image - .view_formats - .binary_search(&attachment.format) - { - let image = Self::expect_attachment_image(bindings, attachment); - - attachment_image.flags = image.info.flags; - attachment_image.usage = image.info.usage; - attachment_image.width = image.info.width >> attachment.base_mip_level; - attachment_image.height = image.info.height >> attachment.base_mip_level; - attachment_image.layer_count = attachment.array_layer_count; - attachment_image.view_formats.insert(idx, attachment.format); + seen_nodes.clear(); + seen_nodes.grow(binding_count); + seen_accesses.clear(); + seen_accesses.grow(binding_count); + } + }); + } +} - image_views[*attachment_idx as usize] = - Image::view(image, attachment.image_view_info(image.info))?; - } - } +struct CommandBufferDebugLabel<'a> { + cmd_buf: &'a CommandBuffer, +} - if let Some((attachment, clear_value)) = &exec.depth_stencil_clear { - let attachment_idx = - attachments.len() - 1 - exec.depth_stencil_resolve.is_some() as usize; - let attachment_image = &mut attachments[attachment_idx]; - if let Err(idx) = attachment_image - .view_formats - .binary_search(&attachment.format) - { - clear_values[attachment_idx] = vk::ClearValue { - depth_stencil: *clear_value, - }; +impl<'a> CommandBufferDebugLabel<'a> { + fn begin(cmd_buf: &'a CommandBuffer, name: impl AsRef) -> Option { + Device::begin_debug_utils_label(&cmd_buf.device, cmd_buf.handle, name) + .ok() + .map(|_| Self { cmd_buf }) + } +} - let image = Self::expect_attachment_image(bindings, attachment); +impl Drop for CommandBufferDebugLabel<'_> { + fn drop(&mut self) { + let _ = Device::end_debug_utils_label(&self.cmd_buf.device, self.cmd_buf.handle); + } +} - attachment_image.flags = image.info.flags; - attachment_image.usage = image.info.usage; - attachment_image.width = image.info.width >> attachment.base_mip_level; - attachment_image.height = image.info.height >> attachment.base_mip_level; - attachment_image.layer_count = attachment.array_layer_count; - attachment_image.view_formats.insert(idx, attachment.format); +#[derive(Default)] +struct ExternalRenderPassAccessHistory { + accesses_by_node: Vec>, +} - image_views[attachment_idx] = - Image::view(image, attachment.image_view_info(image.info))?; - } - } +impl ExternalRenderPassAccessHistory { + fn new(node_count: usize) -> Self { + let mut accesses_by_node = Vec::with_capacity(node_count); + accesses_by_node.resize_with(node_count, Vec::new); - if let Some(attachment) = exec - .depth_stencil_attachment - .or(exec.depth_stencil_load) - .or(exec.depth_stencil_store) - { - let attachment_idx = - attachments.len() - 1 - exec.depth_stencil_resolve.is_some() as usize; - let attachment_image = &mut attachments[attachment_idx]; - if let Err(idx) = attachment_image - .view_formats - .binary_search(&attachment.format) - { - let image = Self::expect_attachment_image(bindings, &attachment); + Self { accesses_by_node } + } - attachment_image.flags = image.info.flags; - attachment_image.usage = image.info.usage; - attachment_image.width = image.info.width >> attachment.base_mip_level; - attachment_image.height = image.info.height >> attachment.base_mip_level; - attachment_image.layer_count = attachment.array_layer_count; - attachment_image.view_formats.insert(idx, attachment.format); + fn accesses(&self, node_idx: usize) -> &[PipelineStageAccessFlags] { + &self.accesses_by_node[node_idx] + } - image_views[attachment_idx] = - Image::view(image, attachment.image_view_info(image.info))?; - } - } + fn record_cmd(&mut self, cmd: &CommandData) { + for exec in &cmd.execs { + for (node_idx, accesses) in exec.accesses.iter() { + self.accesses_by_node[node_idx].extend( + accesses + .iter() + .map(|access| PipelineStageAccessFlags::new(access.access)), + ); + } + } + } +} - if let Some(attachment) = exec - .depth_stencil_resolve - .map(|(attachment, ..)| attachment) - { - let attachment_idx = attachments.len() - 1; - let attachment_image = &mut attachments[attachment_idx]; - if let Err(idx) = attachment_image - .view_formats - .binary_search(&attachment.format) - { - let image = Self::expect_attachment_image(bindings, &attachment); +#[derive(Clone, Copy, Debug)] +struct QueueOwnershipReleaseWait { + semaphore: vk::Semaphore, + stage_mask: vk::PipelineStageFlags2, + value: u64, + device_index: u32, +} - attachment_image.flags = image.info.flags; - attachment_image.usage = image.info.usage; - attachment_image.width = image.info.width >> attachment.base_mip_level; - attachment_image.height = image.info.height >> attachment.base_mip_level; - attachment_image.layer_count = attachment.array_layer_count; - attachment_image.view_formats.insert(idx, attachment.format); +#[derive(Debug)] +struct CommandRecordingResources { + descriptor_pool: Option>, + descriptor_sets: Vec>, + render_pass: Option>, +} - image_views[attachment_idx] = - Image::view(image, attachment.image_view_info(image.info))?; - } - } - } +impl CommandRecordingResources { + /// # Panics + /// + /// Panics if the physical pass has no render pass. + fn expect_render_pass_mut(&mut self) -> &mut Lease { + self.render_pass.as_mut().expect("missing render pass") + } +} - let framebuffer = - RenderPass::framebuffer(render_pass, FramebufferInfo { attachments })?; +impl Drop for CommandRecordingResources { + fn drop(&mut self) { + self.descriptor_sets.clear(); + self.descriptor_pool = None; + } +} - unsafe { - cmd_buf.device.cmd_begin_render_pass( - cmd_buf.handle, - &vk::RenderPassBeginInfo::default() - .render_pass(render_pass.handle) - .framebuffer(framebuffer) - .render_area(render_area) - .clear_values(clear_values) - .push_next( - &mut vk::RenderPassAttachmentBeginInfoKHR::default() - .attachments(image_views), - ), - vk::SubpassContents::INLINE, - ); - } +#[derive(Clone, Copy, Debug)] +struct ImageQueueOwnershipTransfer { + dst_queue_family_index: u32, + layout: vk::ImageLayout, + range: vk::ImageSubresourceRange, + src_queue_family_index: u32, + src_queue_index: u32, +} - Ok(()) - }) +impl PartialEq for ImageQueueOwnershipTransfer { + fn eq(&self, other: &Self) -> bool { + self.dst_queue_family_index == other.dst_queue_family_index + && self.layout == other.layout + && self.src_queue_family_index == other.src_queue_family_index + && self.src_queue_index == other.src_queue_index + && image_subresource_range_eq(self.range, other.range) } +} - #[profiling::function] - fn bind_descriptor_sets( - cmd_buf: &CommandBuffer, - pipeline: &ExecutionPipeline, - physical_pass: &PhysicalPass, - exec_idx: usize, - ) { - if let Some(exec_descriptor_sets) = physical_pass.exec_descriptor_sets.get(&exec_idx) { - thread_local! { - static DESCRIPTOR_SETS: RefCell> = Default::default(); - } +#[derive(Debug)] +struct ImageRangeSet { + image: vk::Image, + range_keys: HashSet, +} - if exec_descriptor_sets.is_empty() { - return; - } +struct ImageSubresourceRangeDebug(vk::ImageSubresourceRange); - DESCRIPTOR_SETS.with_borrow_mut(|descriptor_sets| { - descriptor_sets.clear(); - descriptor_sets.extend( - exec_descriptor_sets - .iter() - .map(|descriptor_set| **descriptor_set), - ); +impl std::fmt::Debug for ImageSubresourceRangeDebug { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.aspect_mask.fmt(f)?; - trace!(" bind descriptor sets {:?}", descriptor_sets); + f.write_str(" array: ")?; - unsafe { - cmd_buf.device.cmd_bind_descriptor_sets( - cmd_buf.handle, - pipeline.bind_point(), - pipeline.layout(), - 0, - descriptor_sets, - &[], - ); - } - }); + let array_layers = self.0.base_array_layer..self.0.base_array_layer + self.0.layer_count; + array_layers.fmt(f)?; + + f.write_str(" mip: ")?; + + let mip_levels = self.0.base_mip_level..self.0.base_mip_level + self.0.level_count; + mip_levels.fmt(f) + } +} + +/// This is needed because vk::ImageSubresourceRange doesn't currently support Hash +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct ImageSubresourceRangeKey { + aspect_mask: vk::ImageAspectFlags, + base_array_layer: u32, + layer_count: u32, + base_mip_level: u32, + level_count: u32, +} + +impl ImageSubresourceRangeKey { + fn from_range( + vk::ImageSubresourceRange { + aspect_mask, + base_array_layer, + layer_count, + base_mip_level, + level_count, + }: vk::ImageSubresourceRange, + ) -> Self { + Self { + aspect_mask, + base_array_layer, + layer_count, + base_mip_level, + level_count, } } - #[profiling::function] - fn bind_pipeline( - cmd_buf: &mut CommandBuffer, - physical_pass: &mut PhysicalPass, - exec_idx: usize, - pipeline: &mut ExecutionPipeline, - depth_stencil: Option, - ) -> Result<(), DriverError> { - if log_enabled!(Trace) { - let (ty, name, vk_pipeline) = match pipeline { - ExecutionPipeline::Compute(pipeline) => { - ("compute", pipeline.debug_name(), pipeline.handle()) - } - ExecutionPipeline::Graphic(pipeline) => { - ("graphic", pipeline.debug_name(), vk::Pipeline::null()) - } - ExecutionPipeline::RayTrace(pipeline) => { - ("ray tracing", pipeline.debug_name(), pipeline.handle()) - } - }; - if let Some(name) = name { - trace!(" bind {} pipeline {} ({:?})", ty, name, vk_pipeline); - } else { - trace!(" bind {} pipeline {:?}", ty, vk_pipeline); - } + fn into_range( + Self { + aspect_mask, + base_array_layer, + layer_count, + base_mip_level, + level_count, + }: Self, + ) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask, + base_array_layer, + layer_count, + base_mip_level, + level_count, } + } +} - // We store a shared reference to this pipeline inside the command buffer! - let pipeline_bind_point = pipeline.bind_point(); - let pipeline = match pipeline { - ExecutionPipeline::Compute(pipeline) => pipeline.handle(), - ExecutionPipeline::Graphic(pipeline) => RenderPass::pipeline_handle( - physical_pass.expect_render_pass_mut(), - pipeline, - depth_stencil, - exec_idx as _, - )?, - ExecutionPipeline::RayTrace(pipeline) => pipeline.handle(), - }; +#[derive(Debug)] +struct NodeIndexedScratch { + entries: Vec>, + indices: Vec, +} - unsafe { - cmd_buf - .device - .cmd_bind_pipeline(cmd_buf.handle, pipeline_bind_point, pipeline); +impl NodeIndexedScratch { + fn clear(&mut self) { + for &node_idx in self.indices.iter() { + let Some(entry) = self.entries.get_mut(node_idx) else { + continue; + }; + + entry.occupied = false; + entry.values.clear(); } - Ok(()) + self.indices.clear(); } - fn end_render_pass(&mut self, cmd: &CommandBuffer) { - trace!(" end render pass"); + fn get(&self, node_idx: NodeIndex) -> &[T] { + self.entries + .get(node_idx) + .filter(|entry| entry.occupied) + .map_or_else(Default::default, |entry| entry.values.as_slice()) + } - unsafe { - cmd.device.cmd_end_render_pass(cmd.handle); + fn push(&mut self, node_idx: NodeIndex, value: T) { + if self.entries.len() <= node_idx { + self.entries + .resize_with(node_idx.saturating_add(1), Default::default); + } + + let entry = &mut self.entries[node_idx]; + + if !entry.occupied { + entry.occupied = true; + self.indices.push(node_idx); } + + entry.values.push(value); } +} - /// Returns `true` when all recorded passes have been submitted to a driver command buffer. - /// - /// A fully-resolved graph contains no additional work and may be discarded, although doing so - /// will stall the GPU while the fences are waited on. It is preferrable to wait a few frame so - /// that the fences will have already been signalled. - pub fn is_submitted(&self) -> bool { - self.graph.cmds.is_empty() +impl Default for NodeIndexedScratch { + fn default() -> Self { + Self { + entries: Default::default(), + indices: Default::default(), + } } +} - #[allow(clippy::type_complexity)] - #[profiling::function] - fn lease_descriptor_pool

( - pool: &mut P, - pass: &CommandData, - ) -> Result>, DriverError> - where - P: Pool, - { - let max_set_idx = pass - .execs - .iter() - .flat_map(|exec| exec.bindings.keys()) - .map(|descriptor| descriptor.set()) - .max() - .unwrap_or_default(); - let max_sets = pass.execs.len() as u32 * (max_set_idx + 1); - let mut info = DescriptorPoolInfo { - max_sets, - ..Default::default() - }; +#[derive(Debug)] +struct NodeIndexedScratchEntry { + occupied: bool, + values: Vec, +} - // Find the total count of descriptors per type (there may be multiple pipelines!) - for pool_size in pass.descriptor_pools_sizes() { - for (&descriptor_ty, &descriptor_count) in pool_size { - debug_assert_ne!(descriptor_count, 0); +impl Default for NodeIndexedScratchEntry { + fn default() -> Self { + Self { + occupied: false, + values: Default::default(), + } + } +} - match descriptor_ty { - vk::DescriptorType::ACCELERATION_STRUCTURE_KHR => { - info.acceleration_structure_count += descriptor_count; - } - vk::DescriptorType::COMBINED_IMAGE_SAMPLER => { - info.combined_image_sampler_count += descriptor_count; - } - vk::DescriptorType::INPUT_ATTACHMENT => { - info.input_attachment_count += descriptor_count; - } - vk::DescriptorType::SAMPLED_IMAGE => { - info.sampled_image_count += descriptor_count; - } - vk::DescriptorType::SAMPLER => { - info.sampler_count += descriptor_count; - } - vk::DescriptorType::STORAGE_BUFFER => { - info.storage_buffer_count += descriptor_count; - } - vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => { - info.storage_buffer_dynamic_count += descriptor_count; - } - vk::DescriptorType::STORAGE_IMAGE => { - info.storage_image_count += descriptor_count; - } - vk::DescriptorType::STORAGE_TEXEL_BUFFER => { - info.storage_texel_buffer_count += descriptor_count; - } - vk::DescriptorType::UNIFORM_BUFFER => { - info.uniform_buffer_count += descriptor_count; - } - vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC => { - info.uniform_buffer_dynamic_count += descriptor_count; - } - vk::DescriptorType::UNIFORM_TEXEL_BUFFER => { - info.uniform_texel_buffer_count += descriptor_count; - } - _ => { - warn!( - "unsupported descriptor type {:?} for pass {}", - descriptor_ty, - pass.name(), - ); +#[derive(Debug)] +struct PendingTransferNode { + handle: H, + transfers: Vec, +} - return Err(DriverError::Unsupported); - } - }; - } - } +#[derive(Debug)] +struct PendingTransferNodes { + entries: Vec>>, + indices: Vec, +} - // It's possible to execute a command-only pipeline - if info.is_empty() { - return Ok(None); +impl PendingTransferNodes +where + H: Copy, +{ + fn new(node_count: usize) -> Self { + let mut entries = Vec::with_capacity(node_count); + entries.resize_with(node_count, || None); + + Self { + entries, + indices: Vec::new(), } + } - // Trivially round up the descriptor counts to increase cache coherence - const ATOM: u32 = 1 << 5; - info.acceleration_structure_count = - info.acceleration_structure_count.next_multiple_of(ATOM); - info.combined_image_sampler_count = - info.combined_image_sampler_count.next_multiple_of(ATOM); - info.input_attachment_count = info.input_attachment_count.next_multiple_of(ATOM); - info.sampled_image_count = info.sampled_image_count.next_multiple_of(ATOM); - info.sampler_count = info.sampler_count.next_multiple_of(ATOM); - info.storage_buffer_count = info.storage_buffer_count.next_multiple_of(ATOM); - info.storage_buffer_dynamic_count = - info.storage_buffer_dynamic_count.next_multiple_of(ATOM); - info.storage_image_count = info.storage_image_count.next_multiple_of(ATOM); - info.storage_texel_buffer_count = info.storage_texel_buffer_count.next_multiple_of(ATOM); - info.uniform_buffer_count = info.uniform_buffer_count.next_multiple_of(ATOM); - info.uniform_buffer_dynamic_count = - info.uniform_buffer_dynamic_count.next_multiple_of(ATOM); - info.uniform_texel_buffer_count = info.uniform_texel_buffer_count.next_multiple_of(ATOM); + fn contains(&self, node_idx: NodeIndex) -> bool { + self.entries[node_idx].is_some() + } - // Notice how all sets are big enough for any other set; TODO: efficiently dont + fn is_empty(&self) -> bool { + self.indices.is_empty() + } - // debug!("{:#?}", info); + fn iter(&self) -> impl Iterator + '_ { + self.indices.iter().filter_map(|&node_idx| { + self.entries[node_idx] + .as_ref() + .map(|entry| (node_idx, entry.handle, entry.transfers.as_slice())) + }) + } + + fn push_transfer(&mut self, node_idx: NodeIndex, handle: H, transfer: T) -> bool { + let inserted = self.entries[node_idx].is_none(); + + if inserted { + self.indices.push(node_idx); + self.entries[node_idx] = Some(PendingTransferNode { + handle, + transfers: vec![transfer], + }); + } else { + let entry = self.entries[node_idx] + .as_mut() + .expect("missing pending transfer node"); + + entry.handle = handle; + entry.transfers.push(transfer); + } - Ok(Some(pool.resource(info)?)) + inserted } - #[profiling::function] - fn lease_render_pass

( - &self, - pool: &mut P, - pass_idx: usize, - ) -> Result, DriverError> + fn remove_where(&mut self, mut remove: F) where - P: Pool, + F: FnMut(NodeIndex, H, &mut Vec) -> bool, { - let pass = &self.graph.cmds[pass_idx]; - let (mut color_attachment_count, mut depth_stencil_attachment_count) = (0, 0); - for exec in &pass.execs { - color_attachment_count = color_attachment_count - .max( - exec.color_attachments - .keys() - .max() - .map(|attachment_idx| attachment_idx + 1) - .unwrap_or_default() as usize, - ) - .max( - exec.color_clears - .keys() - .max() - .map(|attachment_idx| attachment_idx + 1) - .unwrap_or_default() as usize, - ) - .max( - exec.color_loads - .keys() - .max() - .map(|attachment_idx| attachment_idx + 1) - .unwrap_or_default() as usize, - ) - .max( - exec.color_resolves - .keys() - .max() - .map(|attachment_idx| attachment_idx + 1) - .unwrap_or_default() as usize, - ) - .max( - exec.color_stores - .keys() - .max() - .map(|attachment_idx| attachment_idx + 1) - .unwrap_or_default() as usize, - ); - let has_depth_stencil_attachment = exec.depth_stencil_attachment.is_some() - || exec.depth_stencil_clear.is_some() - || exec.depth_stencil_load.is_some() - || exec.depth_stencil_store.is_some(); - let has_depth_stencil_resolve = exec.depth_stencil_resolve.is_some(); + let mut pending_idx = 0; - depth_stencil_attachment_count = depth_stencil_attachment_count - .max(has_depth_stencil_attachment as usize + has_depth_stencil_resolve as usize); + while pending_idx < self.indices.len() { + let node_idx = self.indices[pending_idx]; + + let Some(entry) = self.entries[node_idx].as_mut() else { + self.indices.swap_remove(pending_idx); + continue; + }; + + if remove(node_idx, entry.handle, &mut entry.transfers) { + self.entries[node_idx] = None; + self.indices.swap_remove(pending_idx); + } else { + pending_idx += 1; + } } + } +} - let attachment_count = color_attachment_count + depth_stencil_attachment_count; - let mut attachments = Vec::with_capacity(attachment_count); - attachments.resize_with(attachment_count, AttachmentInfo::default); +#[derive(Clone, Copy)] +struct PipelineStageAccessFlags { + access_flags: vk::AccessFlags, + stage_flags: vk::PipelineStageFlags, +} - let mut subpasses = Vec::::with_capacity(pass.execs.len()); +impl PipelineStageAccessFlags { + fn new(access: AccessType) -> Self { + let (mut stage_flags, access_flags) = pipeline_stage_access_flags(access); + if stage_flags.contains(vk::PipelineStageFlags::ALL_COMMANDS) { + stage_flags |= vk::PipelineStageFlags::ALL_GRAPHICS; + stage_flags &= !vk::PipelineStageFlags::ALL_COMMANDS; + } - { - let mut color_set = vec![false; attachment_count]; - let mut depth_stencil_set = false; + Self { + access_flags, + stage_flags, + } + } +} - // Add load op attachments using the first executions - for exec in &pass.execs { - // Cleared color attachments - for (attachment_idx, (cleared_attachment, _)) in &exec.color_clears { - let color_set = &mut color_set[*attachment_idx as usize]; - if *color_set { - continue; - } +#[derive(Debug)] +struct QueueOwnershipRelease { + _cmd_buf: Lease, + _fence: Fence, + semaphore: vk::Semaphore, +} - let attachment = &mut attachments[*attachment_idx as usize]; - attachment.fmt = cleared_attachment.format; - attachment.sample_count = cleared_attachment.sample_count; - attachment.load_op = vk::AttachmentLoadOp::CLEAR; - attachment.initial_layout = vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL; - *color_set = true; - } +#[derive(Debug)] +struct QueueOwnershipReleaseGroup { + buffers: Vec<(vk::Buffer, BufferSubresourceRange)>, + images: Vec<(vk::Image, vk::ImageLayout, vk::ImageSubresourceRange)>, + src_queue_family_index: u32, + src_queue_index: u32, +} - // Loaded color attachments - for (attachment_idx, loaded_attachment) in &exec.color_loads { - let color_set = &mut color_set[*attachment_idx as usize]; - if *color_set { - continue; - } +/// Submission payload for [`RecordedSubmission::queue_submit`]. +#[derive(Clone, Copy, Debug)] +pub enum QueueSubmitInfo<'a> { + /// Submit using `vkQueueSubmit`. + /// + /// See [`vkQueueSubmit`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkQueueSubmit.html). + QueueSubmit { + /// Semaphores to wait on before execution begins. + waits: &'a [SemaphoreSubmitInfo], - let attachment = &mut attachments[*attachment_idx as usize]; - attachment.fmt = loaded_attachment.format; - attachment.sample_count = loaded_attachment.sample_count; - attachment.load_op = vk::AttachmentLoadOp::LOAD; - attachment.initial_layout = vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL; - *color_set = true; - } + /// Semaphores to signal after execution completes. + signals: &'a [SemaphoreSubmitInfo], + }, - // Cleared depth/stencil attachment - if !depth_stencil_set { - if let Some((cleared_attachment, _)) = exec.depth_stencil_clear { - let attachment = &mut attachments[color_attachment_count]; - attachment.fmt = cleared_attachment.format; - attachment.sample_count = cleared_attachment.sample_count; - attachment.initial_layout = if cleared_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) - { - attachment.load_op = vk::AttachmentLoadOp::CLEAR; - attachment.stencil_load_op = vk::AttachmentLoadOp::CLEAR; + /// Submit using `vkQueueSubmit2`. + /// + /// See [`vkQueueSubmit2`](https://registry.khronos.org/vulkan/specs/latest/man/html/vkQueueSubmit2.html). + QueueSubmit2 { + /// Semaphores to wait on before execution begins. + waits: &'a [SemaphoreSubmit2Info], - vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL - } else if cleared_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH) - { - attachment.load_op = vk::AttachmentLoadOp::CLEAR; + /// Semaphores to signal after execution completes. + signals: &'a [SemaphoreSubmit2Info], + }, +} - vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL - } else { - attachment.stencil_load_op = vk::AttachmentLoadOp::CLEAR; +impl QueueSubmitInfo<'static> { + /// A `vkQueueSubmit` payload with no waits or signals. + pub const QUEUE_SUBMIT: Self = Self::QueueSubmit { + waits: &[], + signals: &[], + }; + + /// A `vkQueueSubmit2` payload with no waits or signals. + pub const QUEUE_SUBMIT2: Self = Self::QueueSubmit2 { + waits: &[], + signals: &[], + }; +} - vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL - }; - depth_stencil_set = true; - } else if let Some(loaded_attachment) = exec.depth_stencil_load { - // Loaded depth/stencil attachment - let attachment = &mut attachments[color_attachment_count]; - attachment.fmt = loaded_attachment.format; - attachment.sample_count = loaded_attachment.sample_count; - attachment.initial_layout = if loaded_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) - { - attachment.load_op = vk::AttachmentLoadOp::LOAD; - attachment.stencil_load_op = vk::AttachmentLoadOp::LOAD; +impl<'a> QueueSubmitInfo<'a> { + /// Creates a `vkQueueSubmit` payload. + pub fn queue_submit( + waits: &'a [SemaphoreSubmitInfo], + signals: &'a [SemaphoreSubmitInfo], + ) -> Self { + Self::QueueSubmit { waits, signals } + } - vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL - } else if loaded_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH) - { - attachment.load_op = vk::AttachmentLoadOp::LOAD; + /// Creates a `vkQueueSubmit2` payload. + pub fn queue_submit2( + waits: &'a [SemaphoreSubmit2Info], + signals: &'a [SemaphoreSubmit2Info], + ) -> Self { + Self::QueueSubmit2 { waits, signals } + } +} - vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL - } else { - attachment.stencil_load_op = vk::AttachmentLoadOp::LOAD; +impl<'a> From<(&'a [SemaphoreSubmitInfo], &'a [SemaphoreSubmitInfo])> for QueueSubmitInfo<'a> { + fn from((waits, signals): (&'a [SemaphoreSubmitInfo], &'a [SemaphoreSubmitInfo])) -> Self { + Self::QueueSubmit { waits, signals } + } +} - vk::ImageLayout::STENCIL_READ_ONLY_OPTIMAL - }; - depth_stencil_set = true; - } else if exec.depth_stencil_clear.is_some() - || exec.depth_stencil_store.is_some() - { - depth_stencil_set = true; - } - } - } - } - - { - let mut color_set = vec![false; attachment_count]; - let mut depth_stencil_set = false; - let mut depth_stencil_resolve_set = false; +impl<'a> From<(&'a [SemaphoreSubmit2Info], &'a [SemaphoreSubmit2Info])> for QueueSubmitInfo<'a> { + fn from((waits, signals): (&'a [SemaphoreSubmit2Info], &'a [SemaphoreSubmit2Info])) -> Self { + Self::QueueSubmit2 { waits, signals } + } +} - // Add store op attachments using the last executions - for exec in pass.execs.iter().rev() { - // Resolved color attachments - for (attachment_idx, (resolved_attachment, _)) in &exec.color_resolves { - let color_set = &mut color_set[*attachment_idx as usize]; - if *color_set { - continue; - } +/// Selects which pending work from a [`Submission`] should be recorded. +#[derive(Clone, Copy, Debug)] +pub enum RecordSelection<'a> { + /// Record all remaining work. + All, - let attachment = &mut attachments[*attachment_idx as usize]; - attachment.fmt = resolved_attachment.format; - attachment.sample_count = resolved_attachment.sample_count; - attachment.final_layout = vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL; - *color_set = true; - } + /// Record prerequisite work, excluding commands that directly access the target node. + Dependencies(AnyNode), - // Stored color attachments - for (attachment_idx, stored_attachment) in &exec.color_stores { - let color_set = &mut color_set[*attachment_idx as usize]; - if *color_set { - continue; - } + /// Record work required by the target node. + Node(AnyNode), - let attachment = &mut attachments[*attachment_idx as usize]; - attachment.fmt = stored_attachment.format; - attachment.sample_count = stored_attachment.sample_count; - attachment.store_op = vk::AttachmentStoreOp::STORE; - attachment.final_layout = vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL; - *color_set = true; - } + /// Record work required by all of the target nodes. + /// + /// Nodes are processed sequentially in slice order against the same evolving submission state. + Nodes(&'a [AnyNode]), +} - // Stored depth/stencil attachment - if !depth_stencil_set && let Some(stored_attachment) = exec.depth_stencil_store { - let attachment = &mut attachments[color_attachment_count]; - attachment.fmt = stored_attachment.format; - attachment.sample_count = stored_attachment.sample_count; - attachment.final_layout = if stored_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) - { - attachment.store_op = vk::AttachmentStoreOp::STORE; - attachment.stencil_store_op = vk::AttachmentStoreOp::STORE; +impl<'a> RecordSelection<'a> { + /// Creates a selection that records prerequisite work for `node` without recording commands that + /// directly access it. + pub fn dependencies(node: impl Into) -> Self { + Self::Dependencies(node.into()) + } - vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL - } else if stored_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH) - { - attachment.store_op = vk::AttachmentStoreOp::STORE; + /// Creates a selection that records work required by `node`. + pub fn node(node: impl Into) -> Self { + Self::Node(node.into()) + } - vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL - } else { - attachment.stencil_store_op = vk::AttachmentStoreOp::STORE; + /// Creates a selection that records work required by all `nodes`. + /// + /// Nodes are processed in slice order. + pub fn nodes(nodes: &'a [AnyNode]) -> Self { + Self::Nodes(nodes) + } +} - vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL - }; - depth_stencil_set = true; - } +impl<'a> From for RecordSelection<'a> { + fn from(node: AnyNode) -> Self { + Self::Node(node) + } +} - // Resolved depth/stencil attachment - if !depth_stencil_resolve_set - && let Some((resolved_attachment, ..)) = exec.depth_stencil_resolve - { - let attachment = attachments - .last_mut() - .expect("missing depth stencil resolve attachment"); - attachment.fmt = resolved_attachment.format; - attachment.sample_count = resolved_attachment.sample_count; - attachment.final_layout = if resolved_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) - { - vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL - } else if resolved_attachment - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH) - { - vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL - } else { - vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL - }; - depth_stencil_resolve_set = true; - } +macro_rules! record_selection_from_node { + ($node:ty) => { + impl<'a> From<$node> for RecordSelection<'a> { + fn from(node: $node) -> Self { + Self::Node(node.into()) } } + }; +} - for attachment in &mut attachments { - if attachment.load_op == vk::AttachmentLoadOp::DONT_CARE { - attachment.initial_layout = attachment.final_layout; - } else if attachment.store_op == vk::AttachmentStoreOp::DONT_CARE - && attachment.stencil_store_op == vk::AttachmentStoreOp::DONT_CARE - { - attachment.final_layout = attachment.initial_layout; +record_selection_from_node!(crate::node::AnyAccelerationStructureNode); +record_selection_from_node!(crate::node::AnyBufferNode); +record_selection_from_node!(crate::node::AnyImageNode); +record_selection_from_node!(crate::node::AccelerationStructureNode); +record_selection_from_node!(crate::node::AccelerationStructureLeaseNode); +record_selection_from_node!(crate::node::BufferNode); +record_selection_from_node!(crate::node::BufferLeaseNode); +record_selection_from_node!(crate::node::ImageNode); +record_selection_from_node!(crate::node::ImageLeaseNode); +record_selection_from_node!(crate::node::SwapchainImageNode); + +/// Graph-side recorded payload for a command buffer that has already been recorded. +#[derive(Debug)] +#[read_only::cast] +pub struct RecordedSubmission { + cmd_buf: Cb, + queue_ownership_release_waits: Vec, + state: Arc>, +} + +impl RecordedSubmission +where + Cb: AsRef, +{ + fn attach(&self, fence: &mut Fence, queue_index: u32) { + let state = self + .state + .lock() + .expect("poisoned recorded submission state"); + + let queue_family_index = self.cmd_buf.as_ref().info.queue_family_index; + + for (node_idx, ranges) in &state.submission.exclusive_buffer_ranges { + if let Some(resource) = state.submission.graph.resources[*node_idx].as_buffer() { + resource.set_sharing_ranges( + SharingMode::Exclusive(Some((queue_family_index, queue_index))), + ranges.as_slice(), + ); } } - // Add subpasses - for (exec_idx, exec) in pass.execs.iter().enumerate() { - let pipeline = exec - .pipeline - .as_ref() - .expect("missing graphics pipeline") - .expect_graphic(); - let mut subpass_info = SubpassInfo::with_capacity(attachment_count); - - // Add input attachments - for attachment_idx in pipeline.inner.input_attachments.iter() { - debug_assert!( - !exec.color_clears.contains_key(attachment_idx), - "cannot clear color attachment {attachment_idx} because it uses subpass input", + for (node_idx, ranges) in &state.submission.exclusive_image_ranges { + if let Some(resource) = state.submission.graph.resources[*node_idx].as_image() { + resource.set_sharing_ranges( + SharingMode::Exclusive(Some((queue_family_index, queue_index))), + ranges.as_slice(), ); + } + } - let exec_attachment = exec - .color_attachments - .get(attachment_idx) - .or_else(|| exec.color_loads.get(attachment_idx)) - .or_else(|| exec.color_stores.get(attachment_idx)) - .expect("missing input attachment"); - let is_random_access = exec.color_stores.contains_key(attachment_idx); - subpass_info.input_attachments.push(AttachmentRef { - attachment: *attachment_idx, - aspect_mask: exec_attachment.aspect_mask, - layout: Self::attachment_layout( - exec_attachment.aspect_mask, - is_random_access, - true, - ), - }); + drop(state); - // We should preserve the attachment in the previous subpasses as needed - // (We're asserting that any input renderpasses are actually real subpasses - // here with prior passes..) - for prev_exec_idx in (0..exec_idx - 1).rev() { - let prev_exec = &pass.execs[prev_exec_idx]; - if prev_exec.color_stores.contains_key(attachment_idx) { - break; - } + fence.drop_when_signaled(self.state.clone()); + } - let prev_subpass = &mut subpasses[prev_exec_idx]; - prev_subpass.preserve_attachments.push(*attachment_idx); - } - } + /// Submits this recorded submission using either `vkQueueSubmit` or `vkQueueSubmit2`. + pub fn queue_submit<'a>( + &mut self, + fence: &mut Fence, + queue_index: u32, + submit_info: impl Into>, + ) -> Result<(), DriverError> { + #[cfg(feature = "checked")] + if fence.queued { + fence.wait_signaled()?; + fence.reset()?; + } - // Set color attachments to defaults - for attachment_idx in 0..color_attachment_count as u32 { - let is_input = subpass_info - .input_attachments - .iter() - .any(|input| input.attachment == attachment_idx); - subpass_info.color_attachments.push(AttachmentRef { - attachment: vk::ATTACHMENT_UNUSED, - aspect_mask: vk::ImageAspectFlags::COLOR, - layout: Self::attachment_layout(vk::ImageAspectFlags::COLOR, true, is_input), - }); - } + let command_buffer = self.cmd_buf.as_ref(); + let device = &command_buffer.device; + let queue_family_index = command_buffer.info.queue_family_index; + + match submit_info.into() { + QueueSubmitInfo::QueueSubmit { waits, signals } => { + check_queue_submit_args(waits, signals)?; + + let extra_waits = self.queue_ownership_release_waits.as_slice(); + let wait_count = waits.len() + extra_waits.len(); + + Device::with_queue(device, queue_family_index, queue_index, |queue| { + SUBMIT.with_borrow_mut(|tls| { + tls.wait_semaphores.clear(); + tls.wait_stage_masks.clear(); + tls.signal_semaphores.clear(); + tls.wait_semaphores.reserve(wait_count); + tls.wait_stage_masks.reserve(wait_count); + tls.signal_semaphores.reserve(signals.len()); + + tls.wait_semaphores + .extend(waits.iter().map(|wait| wait.semaphore)); + tls.wait_stage_masks.extend( + waits + .iter() + .map(|wait| submit_stage_mask_legacy(wait.stage_mask)), + ); + tls.wait_semaphores + .extend(extra_waits.iter().map(|wait| wait.semaphore)); + tls.wait_stage_masks.extend( + extra_waits + .iter() + .map(|wait| submit_stage_mask_legacy(wait.stage_mask)), + ); + tls.signal_semaphores + .extend(signals.iter().map(|signal| signal.semaphore)); - for attachment_idx in exec - .color_attachments - .keys() - .chain(exec.color_clears.keys()) - .chain(exec.color_loads.keys()) - .chain(exec.color_stores.keys()) - { - subpass_info.color_attachments[*attachment_idx as usize].attachment = - *attachment_idx; - } + let mut submit_info = vk::SubmitInfo::default() + .command_buffers(slice::from_ref(&command_buffer.handle)) + .signal_semaphores(tls.signal_semaphores.as_slice()); - // Set depth/stencil attachment - if let Some(depth_stencil) = exec - .depth_stencil_attachment - .or(exec.depth_stencil_load) - .or(exec.depth_stencil_store) - .or_else(|| exec.depth_stencil_clear.map(|(attachment, _)| attachment)) - { - let is_random_access = exec.depth_stencil_clear.is_some() - || exec.depth_stencil_load.is_some() - || exec.depth_stencil_store.is_some(); - subpass_info.depth_stencil_attachment = Some(AttachmentRef { - attachment: color_attachment_count as u32, - aspect_mask: depth_stencil.aspect_mask, - layout: Self::attachment_layout( - depth_stencil.aspect_mask, - is_random_access, - false, - ), - }); - } + if !tls.wait_semaphores.is_empty() { + submit_info = submit_info + .wait_semaphores(tls.wait_semaphores.as_slice()) + .wait_dst_stage_mask(tls.wait_stage_masks.as_slice()); + } - // Set color resolves to defaults - subpass_info.color_resolve_attachments.extend(repeat_n( - AttachmentRef { - attachment: vk::ATTACHMENT_UNUSED, - aspect_mask: vk::ImageAspectFlags::empty(), - layout: vk::ImageLayout::UNDEFINED, - }, - color_attachment_count, - )); + Device::queue_submit( + device, + queue, + slice::from_ref(&submit_info), + fence.handle, + )?; - // Set any used color resolve attachments now - for (dst_attachment_idx, (resolved_attachment, src_attachment_idx)) in - &exec.color_resolves - { - let is_input = subpass_info - .input_attachments - .iter() - .any(|input| input.attachment == *dst_attachment_idx); - subpass_info.color_resolve_attachments[*src_attachment_idx as usize] = - AttachmentRef { - attachment: *dst_attachment_idx, - aspect_mask: resolved_attachment.aspect_mask, - layout: Self::attachment_layout( - resolved_attachment.aspect_mask, - true, - is_input, - ), - }; + Ok::<(), DriverError>(()) + }) + })?; + fence.mark_queued(); } + QueueSubmitInfo::QueueSubmit2 { waits, signals } => { + check_queue_submit2_args(device, waits, signals)?; + + let extra_waits = self.queue_ownership_release_waits.as_slice(); + let wait_count = waits.len() + extra_waits.len(); + + Device::with_queue(device, queue_family_index, queue_index, |queue| { + SUBMIT.with_borrow_mut(|tls| { + tls.wait_infos.clear(); + tls.signal_infos.clear(); + tls.wait_infos.reserve(wait_count); + tls.signal_infos.reserve(signals.len()); + + tls.wait_infos.extend(waits.iter().map(|wait| { + vk::SemaphoreSubmitInfo::default() + .semaphore(wait.semaphore) + .stage_mask(wait.stage_mask) + .value(wait.value) + .device_index(wait.device_index) + })); + tls.wait_infos.extend(extra_waits.iter().map(|wait| { + vk::SemaphoreSubmitInfo::default() + .semaphore(wait.semaphore) + .stage_mask(wait.stage_mask) + .value(wait.value) + .device_index(wait.device_index) + })); + tls.signal_infos.extend(signals.iter().map(|signal| { + vk::SemaphoreSubmitInfo::default() + .semaphore(signal.semaphore) + .stage_mask(signal.stage_mask) + .value(signal.value) + .device_index(signal.device_index) + })); + + let command_buffer_info = vk::CommandBufferSubmitInfo::default() + .command_buffer(command_buffer.handle); + let mut submit_info = vk::SubmitInfo2::default() + .command_buffer_infos(slice::from_ref(&command_buffer_info)); + + if !tls.wait_infos.is_empty() { + submit_info = + submit_info.wait_semaphore_infos(tls.wait_infos.as_slice()); + } - if let Some(( - resolved_attachment, - dst_attachment_idx, - depth_resolve_mode, - stencil_resolve_mode, - )) = exec.depth_stencil_resolve - { - subpass_info.depth_stencil_resolve_attachment = Some(( - AttachmentRef { - attachment: dst_attachment_idx + 1, - aspect_mask: resolved_attachment.aspect_mask, - layout: Self::attachment_layout( - resolved_attachment.aspect_mask, - true, - false, - ), - }, - depth_resolve_mode, - stencil_resolve_mode, - )) - } + if !tls.signal_infos.is_empty() { + submit_info = + submit_info.signal_semaphore_infos(tls.signal_infos.as_slice()); + } - subpass_info.view_mask = exec.view_mask; - subpass_info.correlated_view_mask = exec.correlated_view_mask; + Device::queue_submit2( + device, + queue, + slice::from_ref(&submit_info), + fence.handle, + )?; - subpasses.push(subpass_info); + Ok::<(), DriverError>(()) + }) + })?; + fence.mark_queued(); + } } - // Add dependencies - let dependencies = - { - let mut dependencies = BTreeMap::new(); - for (exec_idx, exec) in pass.execs.iter().enumerate() { - // Check accesses - 'accesses: for (node_idx, accesses) in exec.accesses.iter() { - let (mut curr_stages, mut curr_access) = pipeline_stage_access_flags( - accesses.first().expect("missing resource access").access, - ); - if curr_stages.contains(vk::PipelineStageFlags::ALL_COMMANDS) { - curr_stages |= vk::PipelineStageFlags::ALL_GRAPHICS; - curr_stages &= !vk::PipelineStageFlags::ALL_COMMANDS; - } + self.attach(fence, queue_index); + self.queue_ownership_release_waits.clear(); - // First look for through earlier executions of this pass (in reverse order) - for (prev_exec_idx, prev_exec) in - pass.execs[0..exec_idx].iter().enumerate().rev() - { - if let Some(accesses) = prev_exec.accesses.get(node_idx) { - for &SubresourceAccess { access, .. } in accesses { - // Is this previous execution access dependent on anything the - // current execution access - // is dependent upon? - let (mut prev_stages, prev_access) = - pipeline_stage_access_flags(access); - if prev_stages.contains(vk::PipelineStageFlags::ALL_COMMANDS) { - prev_stages |= vk::PipelineStageFlags::ALL_GRAPHICS; - prev_stages &= !vk::PipelineStageFlags::ALL_COMMANDS; - } + Ok(()) + } +} - let common_stages = curr_stages & prev_stages; - if common_stages.is_empty() { - // No common dependencies - continue; - } +#[derive(Debug)] +struct RecordedSubmissionState { + _releases: Vec, + submission: Submission, +} - let dep = dependencies - .entry((prev_exec_idx, exec_idx)) - .or_insert_with(|| { - SubpassDependency::new( - prev_exec_idx as _, - exec_idx as _, - ) - }); - - // Wait for ... - dep.src_stage_mask |= common_stages; - dep.src_access_mask |= prev_access; - - // ... before we: - dep.dst_stage_mask |= curr_stages; - dep.dst_access_mask |= curr_access; - - // Do the source and destination stage masks both include - // framebuffer-space stages? - if (prev_stages | curr_stages).intersects( - vk::PipelineStageFlags::FRAGMENT_SHADER - | vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS - | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS - | vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, - ) { - dep.dependency_flags |= vk::DependencyFlags::BY_REGION; - } +/// A [`Submission`] bound to a specific command buffer for explicit recording and submission. +#[derive(Debug)] +#[read_only::cast] +pub struct Recording<'p, P, Cb> { + /// The command buffer bound to this recording. + /// + /// _Note:_ This field is read-only. + #[readonly] + pub cmd_buf: Cb, - curr_stages &= !common_stages; - curr_access &= !prev_access; + /// The pool used to allocate resources used during recording. + /// + /// _Note:_ This field may be mutated in between calls to `record`. The updated pool will be + /// used for future calls to record. + #[readonly] + pub resource_pool: &'p mut P, - // Have we found all dependencies for this stage? If so no need - // to check external passes - if curr_stages.is_empty() { - continue 'accesses; - } - } - } - } + submission: Submission, +} - // Second look in previous passes of the entire render graph - for prev_subpass in self.graph.cmds[0..pass_idx] - .iter() - .rev() - .flat_map(|pass| pass.execs.iter().rev()) - { - if let Some(accesses) = prev_subpass.accesses.get(node_idx) { - for &SubresourceAccess { access, .. } in accesses { - // Is this previous subpass access dependent on anything the - // current subpass access is - // dependent upon? - let (prev_stages, prev_access) = - pipeline_stage_access_flags(access); - let common_stages = curr_stages & prev_stages; - if common_stages.is_empty() { - // No common dependencies - continue; - } +impl<'p, P, Cb> Recording<'p, P, Cb> +where + Cb: AsRef, +{ + /// Returns `true` when this submission contains no more commands to record. + pub fn is_empty(&self) -> bool { + self.submission.is_empty() + } - let dep = dependencies - .entry((vk::SUBPASS_EXTERNAL as _, exec_idx)) - .or_insert_with(|| { - SubpassDependency::new( - vk::SUBPASS_EXTERNAL as _, - exec_idx as _, - ) - }); - - // Wait for ... - dep.src_stage_mask |= common_stages; - dep.src_access_mask |= prev_access; - - // ... before we: - dep.dst_stage_mask |= - curr_stages.min(vk::PipelineStageFlags::ALL_GRAPHICS); - dep.dst_access_mask |= curr_access; - - // If the source and destination stage masks both include - // framebuffer-space stages then we need the BY_REGION flag - if (prev_stages | curr_stages).intersects( - vk::PipelineStageFlags::FRAGMENT_SHADER - | vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS - | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS - | vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, - ) { - dep.dependency_flags |= vk::DependencyFlags::BY_REGION; - } + /// Returns a borrow of the original Vulkan resource (buffer, image or acceleration structure) + /// which the given node represents. + pub fn resource(&self, resource_node: N) -> &N::Resource + where + N: Node, + { + self.submission.resource(resource_node) + } - curr_stages &= !common_stages; - curr_access &= !prev_access; + /// Finalizes recording into a recorded submission for a caller-owned command buffer. + pub fn finish(self) -> Result, DriverError> + where + P: Pool, + { + let Self { + cmd_buf, + resource_pool, + submission, + } = self; + + let queue_family_index = cmd_buf.as_ref().info.queue_family_index; + let releases = submit_queue_ownership_releases( + resource_pool, + &submission.queue_ownership_release_groups, + queue_family_index, + |device, queue, cmd_handle, fence, semaphore| { + let submit_info = vk::SubmitInfo::default() + .command_buffers(slice::from_ref(&cmd_handle)) + .signal_semaphores(slice::from_ref(&semaphore)); + Device::queue_submit(device, queue, slice::from_ref(&submit_info), fence) + }, + )?; + let waits = releases + .iter() + .map(|release| QueueOwnershipReleaseWait { + semaphore: release.semaphore, + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + device_index: 0, + }) + .collect(); - // If we found all dependencies for this stage there is no need - // to check external passes - if curr_stages.is_empty() { - continue 'accesses; - } - } - } - } + Ok(submission.into_recorded_submission(cmd_buf, releases, waits)) + } +} - // Fall back to external dependencies - if !curr_stages.is_empty() { - let dep = dependencies - .entry((vk::SUBPASS_EXTERNAL as _, exec_idx)) - .or_insert_with(|| { - SubpassDependency::new(vk::SUBPASS_EXTERNAL as _, exec_idx as _) - }); +impl<'p, P, Cb> Recording<'p, P, Cb> +where + P: SubmissionPool, + Cb: AsRef, +{ + /// Records any remaining graph commands into this submission's command buffer. + /// + /// When `selection` is [`RecordSelection::Nodes`], nodes are processed sequentially in the + /// provided slice order and each step mutates the remaining submission state. + #[profiling::function] + pub fn record<'s>( + &mut self, + selection: impl Into>, + ) -> Result<(), DriverError> { + self.submission.record_selection_impl( + self.resource_pool, + self.cmd_buf.as_ref(), + selection.into(), + ) + } +} - // Wait for ... - dep.src_stage_mask |= curr_stages; - dep.src_access_mask |= curr_access; +#[derive(Default)] +struct Schedule { + access_index: CommandAccessIndex, + interdependent: Vec>, + cmds: Vec, +} - // ... before we: - dep.dst_stage_mask |= vk::PipelineStageFlags::TOP_OF_PIPE; - dep.dst_access_mask = - vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE; - } - } +impl Schedule { + #[profiling::function] + fn reorder_cmds(&mut self, end_cmd_idx: usize) { + if self.cmds.len() < 3 { + return; + } - // Look for attachments of this exec being read or written in other execs of the - // same pass - for (other_idx, other) in pass.execs[0..exec_idx].iter().enumerate() { - // Look for color attachments we're reading - for attachment_idx in exec.color_loads.keys() { - // Look for writes in the other exec - if other.color_clears.contains_key(attachment_idx) - || other.color_stores.contains_key(attachment_idx) - || other.color_resolves.contains_key(attachment_idx) - { - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + let cmd_count = self.cmds.len(); - // Wait for ... - dep.src_stage_mask |= - vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT; - dep.src_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_WRITE; + for dep_cmds in self.interdependent.iter_mut() { + dep_cmds.clear(); + } - // ... before we: - dep.dst_stage_mask |= vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS; - dep.dst_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_READ; - } + self.interdependent.resize_with(cmd_count, Vec::new); - // look for reads in the other exec - if other.color_loads.contains_key(attachment_idx) { - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + let mut local_of_global = vec![usize::MAX; end_cmd_idx]; - // Wait for ... - dep.src_stage_mask |= vk::PipelineStageFlags::LATE_FRAGMENT_TESTS; - dep.src_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_READ; + for (local_idx, &cmd_idx) in self.cmds.iter().enumerate() { + local_of_global[cmd_idx] = local_idx; + } - // ... before we: - dep.dst_stage_mask |= vk::PipelineStageFlags::FRAGMENT_SHADER; - dep.dst_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_READ; - } - } + let mut seen_deps = FixedBitSet::with_capacity(cmd_count); - // Look for a depth/stencil attachment read - if exec.depth_stencil_load.is_some() { - // Look for writes in the other exec - if other.depth_stencil_clear.is_some() - || other.depth_stencil_store.is_some() - || other.depth_stencil_resolve.is_some() - { - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + for (local_idx, &cmd_idx) in self.cmds.iter().enumerate() { + for dep_cmd_idx in self + .access_index + .prior_read_dependency_cmds(cmd_idx, end_cmd_idx) + { + let dep_local_idx = local_of_global[dep_cmd_idx]; + if dep_local_idx == usize::MAX || dep_local_idx == local_idx { + continue; + } - // Wait for ... - dep.src_stage_mask |= vk::PipelineStageFlags::LATE_FRAGMENT_TESTS; - dep.src_access_mask |= - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE; + if !seen_deps.put(dep_local_idx) { + self.interdependent[local_idx].push(dep_local_idx); + } + } - // ... before we: - dep.dst_stage_mask |= vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS; - dep.dst_access_mask |= - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ; - } - - // TODO: Do we need to depend on a READ..READ between subpasses? - // look for reads in the other exec - if other.depth_stencil_load.is_some() { - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + for dep_cmd_idx in self + .access_index + .prior_read_dependency_cmds(cmd_idx, end_cmd_idx) + { + let dep_local_idx = local_of_global[dep_cmd_idx]; + if dep_local_idx != usize::MAX && dep_local_idx != local_idx { + seen_deps.set(dep_local_idx, false); + } + } + } - // Wait for ... - dep.src_stage_mask |= vk::PipelineStageFlags::LATE_FRAGMENT_TESTS; - dep.src_access_mask |= - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ; + let mut scheduled = FixedBitSet::with_capacity(cmd_count); + let mut scheduled_count = 0; - // ... before we: - dep.dst_stage_mask |= vk::PipelineStageFlags::FRAGMENT_SHADER; - dep.dst_access_mask |= - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ; - } - } + while scheduled_count < cmd_count { + let mut best_idx = scheduled_count; + let mut best_overlap = self.interdependent[best_idx].len(); - // Look for color attachments we're writing - for (attachment_idx, aspect_mask) in - exec.color_clears - .iter() - .map(|(attachment_idx, (attachment, _))| { - (*attachment_idx, attachment.aspect_mask) - }) - .chain(exec.color_resolves.iter().map( - |(dst_attachment_idx, (resolved_attachment, _))| { - (*dst_attachment_idx, resolved_attachment.aspect_mask) - }, - )) - .chain(exec.color_stores.iter().map( - |(attachment_idx, attachment)| { - (*attachment_idx, attachment.aspect_mask) - }, - )) - { - let stage = match aspect_mask { - mask if mask.contains(vk::ImageAspectFlags::COLOR) => { - vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT - } - mask if mask.intersects( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) => - { - vk::PipelineStageFlags::LATE_FRAGMENT_TESTS - } - _ => vk::PipelineStageFlags::ALL_GRAPHICS, - }; + for idx in (scheduled_count + 1)..cmd_count { + let mut overlap = 0; - // Look for writes in the other exec - if other.color_clears.contains_key(&attachment_idx) - || other.color_stores.contains_key(&attachment_idx) - || other.color_resolves.contains_key(&attachment_idx) - { - let access = match aspect_mask { - mask if mask.contains(vk::ImageAspectFlags::COLOR) => { - vk::AccessFlags::COLOR_ATTACHMENT_WRITE - } - mask if mask.intersects( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) => - { - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE - } - _ => { - vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE - } - }; + for &dep_local in &self.interdependent[idx] { + if scheduled.contains(dep_local) { + overlap += 1; + } else { + break; + } + } - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + if overlap > best_overlap { + best_overlap = overlap; + best_idx = idx; + } + } - // Wait for ... - dep.src_stage_mask |= stage; - dep.src_access_mask |= access; + scheduled.insert(best_idx); + self.cmds.swap(scheduled_count, best_idx); + self.interdependent.swap(scheduled_count, best_idx); + scheduled_count += 1; + } + } +} - // ... before we: - dep.dst_stage_mask |= stage; - dep.dst_access_mask |= access; - } +/// Semaphore information used during submission. +/// +/// Used for both waits and signals. The legacy `vkQueueSubmit` path only supports binary +/// semaphores and coarse stage masks: [`value`](Self::value) must be `0`, and +/// [`stage_mask`](Self::stage_mask) must be [`vk::PipelineStageFlags2::ALL_COMMANDS`] or +/// [`vk::PipelineStageFlags2::NONE`]. Use [`SemaphoreSubmit2Info`] with +/// [`QueueSubmitInfo::QueueSubmit2`] when a more precise stage mask is required. +/// +/// See [`VkSubmitInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSubmitInfo.html). +#[derive(Clone, Copy, Debug, Default)] +pub struct SemaphoreSubmitInfo { + /// Semaphore to wait on or signal. + /// + /// Defaults to [`vk::Semaphore::null`]. + pub semaphore: vk::Semaphore, - // look for reads in the other exec - if other.color_loads.contains_key(&attachment_idx) { - let (src_access, dst_access) = match aspect_mask { - mask if mask.contains(vk::ImageAspectFlags::COLOR) => ( - vk::AccessFlags::COLOR_ATTACHMENT_READ, - vk::AccessFlags::COLOR_ATTACHMENT_WRITE, - ), - mask if mask.intersects( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) => - { - ( - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ, - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE, - ) - } - _ => ( - vk::AccessFlags::MEMORY_READ - | vk::AccessFlags::MEMORY_WRITE, - vk::AccessFlags::MEMORY_READ - | vk::AccessFlags::MEMORY_WRITE, - ), - }; + /// Stages blocked by this wait, or stages after which the semaphore is signaled. + /// + /// Defaults to [`vk::PipelineStageFlags2::empty`]. + pub stage_mask: vk::PipelineStageFlags2, - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + /// Timeline value to wait for or signal, or `0` for binary semaphores. + pub value: u64, +} - // Wait for ... - dep.src_stage_mask |= vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS; - dep.src_access_mask |= src_access; +impl SemaphoreSubmitInfo { + fn is_supported_legacy_submit(&self) -> bool { + self.value == 0 + && matches!( + self.stage_mask, + vk::PipelineStageFlags2::ALL_COMMANDS | vk::PipelineStageFlags2::NONE + ) + } +} - // ... before we: - dep.dst_stage_mask |= stage; - dep.dst_access_mask |= dst_access; - } - } +/// Semaphore information used during `queue_submit2` submission. +/// +/// Requires Vulkan 1.3 core or the `VK_KHR_synchronization2` extension. Using a non-zero +/// [`value`](Self::value) additionally requires the [`timeline_semaphore`] feature. +/// +/// See [`VkSemaphoreSubmitInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkSemaphoreSubmitInfo.html). +/// +/// [`timeline_semaphore`]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceTimelineSemaphoreFeatures.html +#[derive(Clone, Copy, Debug, Default)] +pub struct SemaphoreSubmit2Info { + /// Semaphore to wait on or signal. + /// + /// Defaults to [`vk::Semaphore::null`]. + pub semaphore: vk::Semaphore, - // Look for a depth/stencil attachment write - if let Some(aspect_mask) = exec - .depth_stencil_clear - .map(|(attachment, _)| attachment.aspect_mask) - .or_else(|| { - exec.depth_stencil_store - .map(|attachment| attachment.aspect_mask) - }) - .or_else(|| { - exec.depth_stencil_resolve - .map(|(attachment, ..)| attachment.aspect_mask) - }) - { - let stage = match aspect_mask { - mask if mask.contains(vk::ImageAspectFlags::COLOR) => { - vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT - } - mask if mask.intersects( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) => - { - vk::PipelineStageFlags::LATE_FRAGMENT_TESTS - } - _ => vk::PipelineStageFlags::ALL_GRAPHICS, - }; + /// Stages blocked by this wait, or stages after which the semaphore is signaled. + /// + /// Defaults to [`vk::PipelineStageFlags2::empty`]. + pub stage_mask: vk::PipelineStageFlags2, - // Look for writes in the other exec - if other.depth_stencil_clear.is_some() - || other.depth_stencil_store.is_some() - || other.depth_stencil_resolve.is_some() - { - let access = match aspect_mask { - mask if mask.contains(vk::ImageAspectFlags::COLOR) => { - vk::AccessFlags::COLOR_ATTACHMENT_WRITE - } - mask if mask.intersects( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) => - { - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE - } - _ => { - vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE - } - }; + /// Timeline value to wait for or signal, or `0` for binary semaphores. + pub value: u64, - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + /// Device index for device-group submissions. + pub device_index: u32, +} - // Wait for ... - dep.src_stage_mask |= stage; - dep.src_access_mask |= access; +/// A finalized graph execution plan. +/// +/// `Submission` owns the remaining commands of a [`Graph`] after [`Graph::finalize`] has ended the +/// graph-building phase. It supports two execution styles: +/// +/// - [`Submission::queue_submit`] for a one-shot submission path. +/// - [`Submission::record`] with a [`RecordSelection`] for explicit command-buffer recording, +/// returning a [`Recording`]. +#[derive(Debug)] +pub struct Submission { + exclusive_buffer_ranges: HashMap>, + exclusive_image_ranges: HashMap>, + graph: Graph, + pending_buffer_transfer_nodes: + Option>, + pending_image_transfer_nodes: + Option>, + queue_ownership_release_groups: Vec, + recorded_commands: Vec, + submit_retained: Vec>, +} - // ... before we: - dep.dst_stage_mask |= stage; - dep.dst_access_mask |= access; - } +impl Submission { + const GRAPHICS_STAGES: vk::PipelineStageFlags = vk::PipelineStageFlags::from_raw( + vk::PipelineStageFlags::DRAW_INDIRECT.as_raw() + | vk::PipelineStageFlags::VERTEX_INPUT.as_raw() + | vk::PipelineStageFlags::VERTEX_SHADER.as_raw() + | vk::PipelineStageFlags::TESSELLATION_CONTROL_SHADER.as_raw() + | vk::PipelineStageFlags::TESSELLATION_EVALUATION_SHADER.as_raw() + | vk::PipelineStageFlags::GEOMETRY_SHADER.as_raw() + | vk::PipelineStageFlags::FRAGMENT_SHADER.as_raw() + | vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS.as_raw() + | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS.as_raw() + | vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT.as_raw() + | vk::PipelineStageFlags::TASK_SHADER_EXT.as_raw() + | vk::PipelineStageFlags::MESH_SHADER_EXT.as_raw(), + ); - // look for reads in the other exec - if other.depth_stencil_load.is_some() { - let (src_access, dst_access) = match aspect_mask { - mask if mask.contains(vk::ImageAspectFlags::COLOR) => ( - vk::AccessFlags::COLOR_ATTACHMENT_READ, - vk::AccessFlags::COLOR_ATTACHMENT_WRITE, - ), - mask if mask.intersects( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) => - { - ( - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ, - vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE, - ) - } - _ => ( - vk::AccessFlags::MEMORY_READ - | vk::AccessFlags::MEMORY_WRITE, - vk::AccessFlags::MEMORY_READ - | vk::AccessFlags::MEMORY_WRITE, - ), - }; + pub(super) fn new(graph: Graph) -> Self { + let recorded_commands = Vec::with_capacity(graph.cmds.len()); + Self { + exclusive_buffer_ranges: HashMap::new(), + exclusive_image_ranges: HashMap::new(), + pending_buffer_transfer_nodes: None, + graph, + queue_ownership_release_groups: Vec::new(), + recorded_commands, + pending_image_transfer_nodes: None, + submit_retained: Vec::new(), + } + } - let dep = dependencies.entry((other_idx, exec_idx)).or_insert_with( - || SubpassDependency::new(other_idx as _, exec_idx as _), - ); + pub(crate) fn graph(&self) -> &Graph { + &self.graph + } - // Wait for ... - dep.src_stage_mask |= vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS; - dep.src_access_mask |= src_access; + pub(crate) fn assert_reusable_commands(&self) { + for cmd in &self.graph.cmds { + for exec in &cmd.execs { + assert!( + exec.func + .as_ref() + .is_some_and(crate::CommandFunction::is_reusable), + "command stream contains a one-shot callback" + ); + } + } + } - // ... before we: - dep.dst_stage_mask |= stage; - dep.dst_access_mask |= dst_access; - } - } - } - } + pub(crate) fn prepare_command_stream

(&mut self, pool: &mut P) -> Result<(), DriverError> + where + P: SubmissionPool, + { + if self.graph.cmds.is_empty() { + return Ok(()); + } - dependencies.into_values().collect::>() - }; + thread_local! { + static SCHEDULE: RefCell = Default::default(); + } - // let info = RenderPassInfo { - // attachments, - // dependencies, - // subpasses, - // }; + SCHEDULE.with_borrow_mut(|schedule| { + schedule + .access_index + .update(&self.graph, self.graph.cmds.len()); + schedule.cmds.clear(); + schedule.cmds.extend(0..self.graph.cmds.len()); - // trace!("{:#?}", info); + debug_assert!( + schedule.cmds.windows(2).all(|w| w[0] <= w[1]), + "Unsorted schedule" + ); - pool.resource(RenderPassInfo { - attachments, - dependencies, - subpasses, + schedule.reorder_cmds(self.graph.cmds.len()); + self.merge_scheduled_cmds(&mut schedule.cmds); + self.lease_scheduled_resources(pool, &schedule.cmds) }) } - #[profiling::function] - fn lease_scheduled_resources

( + pub(crate) fn record_prepared_command_stream( &mut self, - pool: &mut P, - schedule: &[usize], - ) -> Result<(), DriverError> - where - P: Pool + Pool, - { - for pass_idx in schedule.iter().copied() { - // At the time this function runs the pass will already have been optimized into a - // larger pass made out of anything that might have been merged into it - so we - // only care about one pass at a time here - let pass = &mut self.graph.cmds[pass_idx]; - - trace!("requesting [{pass_idx}: {}]", pass.name()); + cmd_buf: &CommandBuffer, + resources: crate::ResourceMap, + ) -> Result<(), DriverError> { + let original_resources = std::mem::replace(&mut self.graph.resources, resources); - let descriptor_pool = Self::lease_descriptor_pool(pool, pass)?; - let mut exec_descriptor_sets = HashMap::with_capacity( - descriptor_pool - .as_ref() - .map(|descriptor_pool| descriptor_pool.info.max_sets as usize) - .unwrap_or_default(), - ); - if let Some(descriptor_pool) = descriptor_pool.as_ref() { - for (exec_idx, pipeline) in - pass.execs - .iter() - .enumerate() - .filter_map(|(exec_idx, exec)| { - exec.pipeline.as_ref().map(|pipeline| (exec_idx, pipeline)) - }) - { - let layouts = pipeline.descriptor_info().layouts.values(); - let mut descriptor_sets = Vec::with_capacity(layouts.len()); - for descriptor_set_layout in layouts { - descriptor_sets.push(DescriptorPool::allocate_descriptor_set( - descriptor_pool, - descriptor_set_layout, - )?); - } - exec_descriptor_sets.insert(exec_idx, descriptor_sets); - } - } + let result = self.record_prepared_command_stream_inner(cmd_buf); - // Note that as a side effect of merging compatible passes all input passes should - // be globbed onto their preceeding passes by now. This allows subpasses to use - // input attachments without really doing anything, so we are provided a pass that - // starts with input we just blow up b/c we can't provide it, or at least shouldn't. - debug_assert!(!pass.execs.is_empty()); - debug_assert!( - pass.expect_first_exec().pipeline.is_none() - || !pass - .expect_first_exec() - .pipeline - .as_ref() - .is_some_and(|pipeline| pipeline.is_graphic()) - || pass - .expect_first_exec() - .pipeline - .as_ref() - .expect("missing graphics pipeline") - .expect_graphic() - .inner - .descriptor_info - .pool_sizes - .values() - .filter_map(|pool| pool.get(&vk::DescriptorType::INPUT_ATTACHMENT)) - .next() - .is_none() - ); + self.graph.resources = original_resources; - // Also the renderpass may just be None if the pass contained no graphic ops. - let render_pass = if pass - .expect_first_exec() - .pipeline - .as_ref() - .map(|pipeline| pipeline.is_graphic()) - .unwrap_or_default() - { - Some(self.lease_render_pass(pool, pass_idx)?) - } else { - None - }; + result + } - self.physical_passes.push(PhysicalPass { - descriptor_pool, - exec_descriptor_sets, - render_pass, - }); + fn record_prepared_command_stream_inner( + &mut self, + cmd_buf: &CommandBuffer, + ) -> Result<(), DriverError> { + thread_local! { + static SCHEDULE: RefCell = Default::default(); } + SCHEDULE.with_borrow_mut(|schedule| { + schedule + .access_index + .update(&self.graph, self.graph.cmds.len()); + schedule.cmds.clear(); + schedule.cmds.extend(0..self.graph.cmds.len()); + self.track_pending_transfers(schedule, cmd_buf.info.queue_family_index); + self.queue_ownership_release_groups + .extend(self.collect_queue_ownership_release_groups()); + }); + + self.record_cmd_indices(cmd_buf, 0..self.graph.cmds.len())?; + Ok(()) } - // Merges passes which are graphic with common-ish attachments - note that scheduled pass order - // is final during this function and so we must merge contiguous groups of passes - #[profiling::function] - fn merge_scheduled_passes(&mut self, schedule: &mut Vec) { - thread_local! { - static PASSES: RefCell>> = Default::default(); + fn into_recorded_submission( + self, + cmd_buf: Cb, + releases: Vec, + waits: Vec, + ) -> RecordedSubmission + where + Cb: AsRef, + { + RecordedSubmission { + cmd_buf, + queue_ownership_release_waits: waits, + state: Arc::new(Mutex::new(RecordedSubmissionState { + _releases: releases, + submission: self, + })), } + } - PASSES.with_borrow_mut(|passes| { - debug_assert!(passes.is_empty()); - - passes.extend(self.graph.cmds.drain(..).map(Some)); + fn is_framebuffer_space(stages: vk::PipelineStageFlags) -> bool { + stages.intersects( + vk::PipelineStageFlags::FRAGMENT_SHADER + | vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS + | vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, + ) + } - let mut idx = 0; + fn subpass_dependency_stage_masks( + previous: vk::PipelineStageFlags, + current: vk::PipelineStageFlags, + ) -> Option<(vk::PipelineStageFlags, vk::PipelineStageFlags)> { + let all_graphics = vk::PipelineStageFlags::ALL_GRAPHICS; + let previous_all_graphics = previous.contains(all_graphics); + let current_all_graphics = current.contains(all_graphics); + + let overlaps = if previous_all_graphics && current_all_graphics { + true + } else if previous_all_graphics { + current.intersects(Self::GRAPHICS_STAGES) + } else if current_all_graphics { + previous.intersects(Self::GRAPHICS_STAGES) + } else { + previous.intersects(current) + }; - // debug!("attempting to merge {} passes", schedule.len(),); + if !overlaps { + return None; + } - while idx < schedule.len() { - let mut pass = passes[schedule[idx]] - .take() - .expect("missing scheduled pass"); + if previous_all_graphics || current_all_graphics { + Some((previous, current)) + } else { + let stages = previous & current; - // Find candidates - let start = idx + 1; - let mut end = start; - while end < schedule.len() { - let other = passes[schedule[end]] - .as_ref() - .expect("missing scheduled pass"); + Some((stages, stages)) + } + } - debug!( - "attempting to merge [{idx}: {}] with [{end}: {}]", - pass.name(), - other.name() - ); + fn record_subpass_dependency( + dependencies: &mut BTreeMap<(usize, usize), SubpassDependency>, + src_subpass: usize, + dst_subpass: usize, + previous: PipelineStageAccessFlags, + dst_stage_mask: vk::PipelineStageFlags, + current: &mut PipelineStageAccessFlags, + ) -> bool { + let Some((src_stage_mask, matched_dst_stages)) = + Self::subpass_dependency_stage_masks(previous.stage_flags, current.stage_flags) + else { + return false; + }; - if Self::allow_merge_passes(&pass, other) { - end += 1; - } else { - break; - } - } + let dep = dependencies + .entry((src_subpass, dst_subpass)) + .or_insert_with(|| SubpassDependency::new(src_subpass as _, dst_subpass as _)); - if log_enabled!(Trace) && start != end { - trace!( - "merging {} passes into [{idx}: {}]", - end - start, - pass.name() - ); - } + dep.src_stage_mask |= src_stage_mask; + dep.src_access_mask |= previous.access_flags; + dep.dst_stage_mask |= dst_stage_mask; + dep.dst_access_mask |= current.access_flags; - let mut name = pass.name().to_owned(); + if Self::is_framebuffer_space(previous.stage_flags | current.stage_flags) { + dep.dependency_flags |= vk::DependencyFlags::BY_REGION; + } - // Grow the merged pass once, not per merge - { - let mut name_additional = 0; - let mut execs_additional = 0; - for idx in start..end { - let other = passes[schedule[idx]] - .as_ref() - .expect("missing scheduled pass"); - name_additional += other.name().len() + 3; - execs_additional += other.execs.len(); - } + current.stage_flags &= !matched_dst_stages; - name.reserve(name_additional); - pass.execs.reserve(execs_additional); - } + current.stage_flags.is_empty() + } - for idx in start..end { - let mut other = passes[schedule[idx]] - .take() - .expect("missing scheduled pass"); - name.push_str(" + "); - name.push_str(other.name()); - pass.execs.append(&mut other.execs); - } + #[profiling::function] + fn allow_merge_passes(lhs: &CommandData, rhs: &CommandData) -> bool { + fn first_graphic_pipeline(pass: &CommandData) -> Option<&GraphicsPipeline> { + pass.execs + .first() + .and_then(|exec| exec.pipeline.as_ref().map(ExecutionPipeline::as_graphics)) + .flatten() + } - #[cfg(debug_assertions)] - { - pass.name = Some(name); - } + fn is_multiview(view_mask: u32) -> bool { + view_mask != 0 + } - self.graph.cmds.push(pass); - idx += 1 + end - start; - } + let lhs_pipeline = first_graphic_pipeline(lhs); + if lhs_pipeline.is_none() { + trace!(" {} is not graphics", lhs.name()); - // Reschedule passes - schedule.truncate(self.graph.cmds.len()); + return false; + } - for (idx, pass_idx) in schedule.iter_mut().enumerate() { - *pass_idx = idx; - } + let rhs_pipeline = first_graphic_pipeline(rhs); + if rhs_pipeline.is_none() { + trace!(" {} is not graphics", rhs.name()); - // Add the remaining passes back into the graph for later - for pass in passes.drain(..).flatten() { - self.graph.cmds.push(pass); - } - }); - } + return false; + } - fn next_subpass(cmd: &CommandBuffer) { - trace!("next_subpass"); + let lhs_pipeline = unsafe { lhs_pipeline.unwrap_unchecked() }; + let rhs_pipeline = unsafe { rhs_pipeline.unwrap_unchecked() }; - unsafe { - cmd.device - .cmd_next_subpass(cmd.handle, vk::SubpassContents::INLINE); - } - } + // Must be same general rasterization modes + let lhs_info = lhs_pipeline.inner.info; + let rhs_info = rhs_pipeline.inner.info; + if lhs_info.blend != rhs_info.blend + || lhs_info.cull_mode != rhs_info.cull_mode + || lhs_info.front_face != rhs_info.front_face + || lhs_info.polygon_mode != rhs_info.polygon_mode + || lhs_info.samples != rhs_info.samples + { + trace!(" different rasterization modes",); - /// Returns the stages that process the given node. - /// - /// Note that this value must be retrieved before resolving a node as there will be no - /// data left to inspect afterwards! - #[profiling::function] - pub fn node_stages(&self, node: impl Node) -> vk::PipelineStageFlags { - self.graph.assert_node_owner(&node); + return false; + } - let node_idx = node.index(); - let mut res = Default::default(); + let rhs = rhs.execs.first(); - 'pass: for pass in self.graph.cmds.iter() { - for exec in pass.execs.iter() { - if exec.accesses.contains_key(&node_idx) { - res |= pass - .execs - .iter() - .filter_map(|exec| exec.pipeline.as_ref()) - .map(|pipeline| pipeline.stage()) - .reduce(|j, k| j | k) - .unwrap_or(vk::PipelineStageFlags::TRANSFER); + // Commands without executions are filtered before scheduling. + debug_assert!(rhs.is_some()); - // The execution pipelines of a pass are always the same type - continue 'pass; - } - } - } + let rhs = unsafe { rhs.unwrap_unchecked() }; - debug_assert_ne!( - res, - Default::default(), - "The given node was not accessed in this graph" - ); + let mut common_color_attachment = false; + let mut common_depth_attachment = false; - res - } + // Now we need to know what the subpasses (we may have prior merges) wrote + for lhs in lhs.execs.iter().rev() { + // Multiview subpasses cannot be combined with non-multiview subpasses + if is_multiview(lhs.view_mask) != is_multiview(rhs.view_mask) { + trace!(" incompatible multiview"); - #[profiling::function] - fn record_execution_barriers<'a>( - cmd_buf: &CommandBuffer, - resources: &mut [AnyResource], - accesses: impl Iterator)>, - ) { - // We store a Barriers in TLS to save an alloc; contents are POD - thread_local! { - static TLS: RefCell = Default::default(); - } + return false; + } - struct Barrier { - next_access: AccessType, - prev_access: AccessType, - resource: T, - } + // Compare individual color attachments for compatibility + for (attachment_idx, lhs_attachment) in lhs.attachments.color_attachments() { + let rhs_attachment = rhs + .attachments + .color_attachment(attachment_idx) + .map(|state| state.attachment); - struct BufferResource { - buffer: vk::Buffer, - offset: usize, - size: usize, - } + if !Attachment::are_compatible(Some(lhs_attachment.attachment), rhs_attachment) { + trace!(" incompatible color attachments"); - struct ImageResource { - image: vk::Image, - range: vk::ImageSubresourceRange, - } + return false; + } - #[derive(Default)] - struct Tls { - buffers: Vec>, - images: Vec>, - next_accesses: Vec, - prev_accesses: Vec, - } + common_color_attachment = true; + } - TLS.with_borrow_mut(|tls| { - // Initialize TLS from a previous call - tls.buffers.clear(); - tls.images.clear(); - tls.next_accesses.clear(); - tls.prev_accesses.clear(); + // Compare depth/stencil attachments for compatibility + let lhs_depth_stencil = lhs + .attachments + .depth_stencil_attachment() + .map(|state| state.attachment); - // Map remaining accesses into vk_sync barriers (some accesses may have been removed by - // the render pass request function) + let rhs_depth_stencil = rhs + .attachments + .depth_stencil_attachment() + .map(|state| state.attachment); - for (node_idx, accesses) in accesses { - let resource = &resources[*node_idx]; + if !Attachment::are_compatible(lhs_depth_stencil, rhs_depth_stencil) { + trace!(" incompatible depth/stencil attachments"); - match resource { - AnyResource::AccelerationStructure(..) - | AnyResource::AccelerationStructureLease(..) => { - let Some(accel_struct) = resource.as_accel_struct() else { - #[cfg(feature = "checked")] - unreachable!(); - - #[cfg(not(feature = "checked"))] - unsafe { - unreachable_unchecked() - } - }; + return false; + } - let prev_access = AccelerationStructure::access( - accel_struct, - accesses.last().expect("missing resource access").access, - ); + common_depth_attachment |= lhs_depth_stencil.is_some() && rhs_depth_stencil.is_some(); + } - tls.next_accesses.extend( - accesses - .iter() - .map(|&SubresourceAccess { access, .. }| access), - ); - tls.prev_accesses.push(prev_access); - } - AnyResource::Buffer(..) | AnyResource::BufferLease(..) => { - let Some(buffer) = resource.as_buffer() else { - #[cfg(feature = "checked")] - unreachable!(); + // Keep color and depth on tile + if common_color_attachment || common_depth_attachment { + trace!(" merging due to common image"); - #[cfg(not(feature = "checked"))] - unsafe { - unreachable_unchecked() - } - }; + return true; + } - for &SubresourceAccess { - access, - subresource, - } in accesses - { - let SubresourceRange::Buffer(range) = subresource else { - unreachable!() - }; + // Keep input on tile + if !rhs_pipeline.inner.input_attachments.is_empty() { + trace!(" merging due to subpass input"); - for (prev_access, range) in Buffer::access(buffer, access, range) { - tls.buffers.push(Barrier { - next_access: access, - prev_access, - resource: BufferResource { - buffer: buffer.handle, - offset: range.start as _, - size: (range.end - range.start) as _, - }, - }); - } - } - } - AnyResource::Image(..) - | AnyResource::ImageLease(..) - | AnyResource::SwapchainImage(..) => { - let Some(image) = resource.as_image() else { - #[cfg(feature = "checked")] - unreachable!(); + return true; + } - #[cfg(not(feature = "checked"))] - unsafe { - unreachable_unchecked() - } - }; + trace!(" not merging"); - for &SubresourceAccess { - access, - subresource, - } in accesses - { - let SubresourceRange::Image(range) = subresource else { - unreachable!() - }; + // No reason to merge, so don't + false + } - for (prev_access, range) in Image::access(image, access, range) { - tls.images.push(Barrier { - next_access: access, - prev_access, - resource: ImageResource { - image: image.handle, - range, - }, - }) - } - } - } + fn attachment_layout( + aspect_mask: vk::ImageAspectFlags, + is_random_access: bool, + is_input: bool, + ) -> vk::ImageLayout { + if aspect_mask.contains(vk::ImageAspectFlags::COLOR) { + if is_input { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL + } + } else if aspect_mask.contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) + { + if is_random_access { + if is_input { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL + } + } else { + vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL + } + } else if aspect_mask.contains(vk::ImageAspectFlags::DEPTH) { + if is_random_access { + if is_input { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL + } + } else { + vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL + } + } else if aspect_mask.contains(vk::ImageAspectFlags::STENCIL) { + if is_random_access { + if is_input { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL } + } else { + vk::ImageLayout::STENCIL_READ_ONLY_OPTIMAL } + } else { + vk::ImageLayout::UNDEFINED + } + } - let global_barrier = if !tls.next_accesses.is_empty() { - // No resource attached - we use a global barrier for these - trace!( - " global {:?}->{:?}", - tls.next_accesses, tls.prev_accesses - ); + fn attachment_stage(aspect_mask: vk::ImageAspectFlags) -> vk::PipelineStageFlags { + match aspect_mask { + mask if mask.contains(vk::ImageAspectFlags::COLOR) => { + vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT + } + mask if mask + .intersects(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) => + { + vk::PipelineStageFlags::LATE_FRAGMENT_TESTS + } + _ => vk::PipelineStageFlags::ALL_GRAPHICS, + } + } - Some(GlobalBarrier { - next_accesses: tls.next_accesses.as_slice(), - previous_accesses: tls.prev_accesses.as_slice(), - }) - } else { - None - }; - let buffer_barriers = tls.buffers.iter().map( - |Barrier { - next_access, - prev_access, - resource, - }| { - let BufferResource { - buffer, - offset, - size, - } = *resource; + fn attachment_read_stage(aspect_mask: vk::ImageAspectFlags) -> vk::PipelineStageFlags { + match aspect_mask { + mask if mask.contains(vk::ImageAspectFlags::COLOR) => { + vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT + } + mask if mask + .intersects(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) => + { + vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS + } + _ => vk::PipelineStageFlags::ALL_GRAPHICS, + } + } - trace!( - " buffer {:?} {:?} {:?}->{:?}", - buffer, - offset..offset + size, - prev_access, - next_access, - ); + fn subpass_stage_mask(stages: vk::PipelineStageFlags) -> vk::PipelineStageFlags { + if stages.is_empty() { + return stages; + } - BufferBarrier { - next_accesses: slice::from_ref(next_access), - previous_accesses: slice::from_ref(prev_access), - src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - buffer, - offset, - size, - } - }, - ); - let image_barriers = tls.images.iter().map( - |Barrier { - next_access, - prev_access, - resource, - }| { - let ImageResource { image, range } = *resource; + if stages.contains(vk::PipelineStageFlags::ALL_GRAPHICS) { + return vk::PipelineStageFlags::ALL_GRAPHICS; + } - struct ImageSubresourceRangeDebug(vk::ImageSubresourceRange); + let graphics_stages = stages & Self::GRAPHICS_STAGES; + if graphics_stages.is_empty() { + vk::PipelineStageFlags::ALL_GRAPHICS + } else { + graphics_stages + } + } - impl std::fmt::Debug for ImageSubresourceRangeDebug { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.aspect_mask.fmt(f)?; + fn attachment_write_access(aspect_mask: vk::ImageAspectFlags) -> vk::AccessFlags { + match aspect_mask { + mask if mask.contains(vk::ImageAspectFlags::COLOR) => { + vk::AccessFlags::COLOR_ATTACHMENT_WRITE + } + mask if mask + .intersects(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) => + { + vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE + } + _ => vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE, + } + } - f.write_str(" array: ")?; + fn accel_struct_canonical_accesses<'a>( + accesses: &'a [SubresourceAccess], + scratch: &'a mut Vec, + ) -> &'a [AccessType] { + scratch.clear(); - let array_layers = self.0.base_array_layer - ..self.0.base_array_layer + self.0.layer_count; - array_layers.fmt(f)?; + let [access] = accesses else { + for access in accesses { + if !scratch.contains(&access.access) { + scratch.push(access.access); + } + } - f.write_str(" mip: ")?; + return scratch.as_slice(); + }; - let mip_levels = - self.0.base_mip_level..self.0.base_mip_level + self.0.level_count; - mip_levels.fmt(f) - } - } + slice::from_ref(&access.access) + } - trace!( - " image {:?} {:?} {:?}->{:?}", - image, - ImageSubresourceRangeDebug(range), - prev_access, - next_access, - ); + fn attachment_read_write_access( + aspect_mask: vk::ImageAspectFlags, + ) -> (vk::AccessFlags, vk::AccessFlags) { + match aspect_mask { + mask if mask.contains(vk::ImageAspectFlags::COLOR) => ( + vk::AccessFlags::COLOR_ATTACHMENT_READ, + vk::AccessFlags::COLOR_ATTACHMENT_WRITE, + ), + mask if mask + .intersects(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) => + { + ( + vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ, + vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE, + ) + } + _ => ( + vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE, + vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE, + ), + } + } - ImageBarrier { - next_accesses: slice::from_ref(next_access), - next_layout: image_access_layout(*next_access), - previous_accesses: slice::from_ref(prev_access), - previous_layout: image_access_layout(*prev_access), - discard_contents: *prev_access == AccessType::Nothing - || is_write_access(*next_access), - src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - image, - range, - } - }, - ); + fn color_attachment_is_read(load: LoadOp<[f32; 4]>) -> bool { + matches!(load, LoadOp::Load) + } - pipeline_barrier( - &cmd_buf.device, - cmd_buf.handle, - global_barrier, - &buffer_barriers.collect::>(), - &image_barriers.collect::>(), - ); - }); + fn color_attachment_is_write( + load: LoadOp<[f32; 4]>, + store: StoreOp, + has_resolve: bool, + ) -> bool { + matches!(load, LoadOp::Clear(_)) || store == StoreOp::Store || has_resolve + } + + fn depth_stencil_attachment_is_read(load: LoadOp) -> bool { + matches!(load, LoadOp::Load) + } + + fn depth_stencil_attachment_is_write( + load: LoadOp, + store: StoreOp, + has_resolve: bool, + ) -> bool { + matches!(load, LoadOp::Clear(_)) || store == StoreOp::Store || has_resolve + } + + fn expect_attachment_image<'a>( + bindings: &'a [AnyResource], + attachment: &Attachment, + ) -> &'a Image { + bindings[attachment.target] + .as_image() + .expect("invalid attachment target image") } #[profiling::function] - fn record_image_layout_transitions( + fn begin_render_pass( cmd_buf: &CommandBuffer, - resources: &mut [AnyResource], - pass: &mut CommandData, - ) { - // We store a Barriers in TLS to save an alloc; contents are POD - thread_local! { - static TLS: RefCell = Default::default(); - } + bindings: &[AnyResource], + pass: &CommandData, + recorded_command: &mut CommandRecordingResources, + render_area: vk::Rect2D, + ) -> Result<(), DriverError> { + trace!(" begin render pass"); - struct ImageResourceBarrier { - image: vk::Image, - next_access: AccessType, - prev_access: AccessType, - range: vk::ImageSubresourceRange, - } + let render_pass = recorded_command.expect_render_pass_mut(); + let attachment_count = render_pass.info.attachments.len(); - #[derive(Default)] - struct Tls { - images: Vec, - initial_layouts: HashMap>, + let mut attachments = Vec::with_capacity(attachment_count); + attachments.resize( + attachment_count, + FramebufferAttachmentImageInfo { + flags: vk::ImageCreateFlags::empty(), + usage: vk::ImageUsageFlags::empty(), + width: 0, + height: 0, + layer_count: 0, + view_formats: vec![], + }, + ); + + thread_local! { + static CLEARS_VIEWS: RefCell<( + Vec, + Vec, + )> = Default::default(); } - TLS.with_borrow_mut(|tls| { - tls.images.clear(); - tls.initial_layouts.clear(); + CLEARS_VIEWS.with_borrow_mut(|(clear_values, image_views)| { + clear_values.resize_with(attachment_count, vk::ClearValue::default); + image_views.resize(attachment_count, vk::ImageView::null()); - for (node_idx, accesses) in pass - .execs - .iter_mut() - .flat_map(|exec| exec.accesses.iter()) - .map(|(node_idx, accesses)| (*node_idx, accesses)) - { - debug_assert!(resources.get(node_idx).is_some()); + for exec in &pass.execs { + for (attachment_idx, state) in exec.attachments.color_attachments() { + let attachment = state.attachment; + let attachment_image = &mut attachments[attachment_idx as usize]; + if let Err(idx) = attachment_image + .view_formats + .binary_search(&attachment.format) + { + if let LoadOp::Clear(clear_value) = state.load { + clear_values[attachment_idx as usize] = vk::ClearValue { + color: vk::ClearColorValue { + float32: clear_value, + }, + }; + } - let resource = unsafe { - // CommandRef enforces this during push_resource_access - resources.get_unchecked(node_idx) - }; + let image = Self::expect_attachment_image(bindings, &attachment); - match resource { - AnyResource::AccelerationStructure(..) - | AnyResource::AccelerationStructureLease(..) => { - let Some(accel_struct) = resource.as_accel_struct() else { - #[cfg(feature = "checked")] - unreachable!(); - - #[cfg(not(feature = "checked"))] - unsafe { - unreachable_unchecked() - } - }; + attachment_image.flags = image.info.flags; + attachment_image.usage = image.info.usage; + attachment_image.width = image.info.width >> attachment.base_mip_level; + attachment_image.height = image.info.height >> attachment.base_mip_level; + attachment_image.layer_count = attachment.array_layer_count; + attachment_image.view_formats.insert(idx, attachment.format); - AccelerationStructure::access(accel_struct, AccessType::Nothing); + image_views[attachment_idx as usize] = + Image::view(image, attachment.image_view_info(image.info))?; } - AnyResource::Buffer(..) | AnyResource::BufferLease(..) => { - let Some(buffer) = resource.as_buffer() else { - #[cfg(feature = "checked")] - unreachable!(); + } - #[cfg(not(feature = "checked"))] - unsafe { - unreachable_unchecked() - } - }; + if let Some(state) = exec.attachments.depth_stencil_attachment() + && state.is_attachment + { + let attachment = state.attachment; + let attachment_idx = attachments.len() - 1 - state.resolve.is_some() as usize; + let attachment_image = &mut attachments[attachment_idx]; + if let Err(idx) = attachment_image + .view_formats + .binary_search(&attachment.format) + { + if let LoadOp::Clear(depth_stencil) = state.load { + clear_values[attachment_idx] = vk::ClearValue { depth_stencil }; + } - for subresource_access in accesses { - let &SubresourceAccess { - subresource: SubresourceRange::Buffer(access_range), - .. - } = subresource_access - else { - #[cfg(feature = "checked")] - unreachable!(); + let image = Self::expect_attachment_image(bindings, &attachment); - #[cfg(not(feature = "checked"))] - unsafe { - // This cannot be reached because PassRef enforces the subrange - // is of type N::Subresource - // where N is the image node type - unreachable_unchecked() - } - }; + attachment_image.flags = image.info.flags; + attachment_image.usage = image.info.usage; + attachment_image.width = image.info.width >> attachment.base_mip_level; + attachment_image.height = image.info.height >> attachment.base_mip_level; + attachment_image.layer_count = attachment.array_layer_count; + attachment_image.view_formats.insert(idx, attachment.format); - for _ in Buffer::access(buffer, AccessType::Nothing, access_range) {} - } + image_views[attachment_idx] = + Image::view(image, attachment.image_view_info(image.info))?; } - AnyResource::Image(..) - | AnyResource::ImageLease(..) - | AnyResource::SwapchainImage(..) => { - let Some(image) = resource.as_image() else { - #[cfg(feature = "checked")] - unreachable!(); - - #[cfg(not(feature = "checked"))] - unsafe { - unreachable_unchecked() - } - }; - - // TODO: Optimize this path for single-aspect single-layer single-mip images - let initial_layout = tls - .initial_layouts - .entry(node_idx) - .or_insert_with(|| DenseAccess::new(image.info, true)); + } - for subresource_access in accesses { - let &SubresourceAccess { - access, - subresource: SubresourceRange::Image(access_range), - } = subresource_access - else { - #[cfg(feature = "checked")] - unreachable!(); + if let Some(state) = exec + .attachments + .depth_stencil_attachment() + .and_then(|state| state.resolve) + { + let attachment_idx = attachments.len() - 1; + let attachment_image = &mut attachments[attachment_idx]; + if let Err(idx) = attachment_image + .view_formats + .binary_search(&state.attachment.format) + { + let image = Self::expect_attachment_image(bindings, &state.attachment); - #[cfg(not(feature = "checked"))] - unsafe { - // This cannot be reached because PassRef enforces the subrange - // is of type N::Subresource - // where N is the image node type - unreachable_unchecked() - } - }; + attachment_image.flags = image.info.flags; + attachment_image.usage = image.info.usage; + attachment_image.width = + image.info.width >> state.attachment.base_mip_level; + attachment_image.height = + image.info.height >> state.attachment.base_mip_level; + attachment_image.layer_count = state.attachment.array_layer_count; + attachment_image + .view_formats + .insert(idx, state.attachment.format); - for (initial_layout, layout_range) in - initial_layout.swap(false, access_range) - { - for (prev_access, range) in - Image::access(image, access, layout_range) - { - if initial_layout { - tls.images.push(ImageResourceBarrier { - image: image.handle, - next_access: initial_image_layout_access(access), - prev_access, - range, - }); - } - } - } - } + image_views[attachment_idx] = + Image::view(image, state.attachment.image_view_info(image.info))?; } } } - let image_barriers = tls.images.iter().map( - |ImageResourceBarrier { - image, - next_access, - prev_access, - range, - }| { - trace!( - " image {:?} {:?} {:?}->{:?}", - image, - ImageSubresourceRangeDebug(*range), - prev_access, - next_access, - ); + let framebuffer = + RenderPass::framebuffer(render_pass, FramebufferInfo { attachments })?; - // Color Attachment Read/Write (blending) will prevent discarding contents. - // Note that we must check "not-read" because some reads write! - let discard_contents = - *prev_access == AccessType::Nothing || !is_read_access(*next_access); - - ImageBarrier { - next_accesses: slice::from_ref(next_access), - next_layout: image_access_layout(*next_access), - previous_accesses: slice::from_ref(prev_access), - previous_layout: image_access_layout(*prev_access), - discard_contents, - src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, - image: *image, - range: *range, - } - }, - ); + unsafe { + cmd_buf.device.cmd_begin_render_pass( + cmd_buf.handle, + &vk::RenderPassBeginInfo::default() + .render_pass(render_pass.handle) + .framebuffer(framebuffer) + .render_area(render_area) + .clear_values(clear_values) + .push_next( + &mut vk::RenderPassAttachmentBeginInfoKHR::default() + .attachments(image_views), + ), + vk::SubpassContents::INLINE, + ); + } - pipeline_barrier( - &cmd_buf.device, - cmd_buf.handle, - None, - &[], - &image_barriers.collect::>(), - ); - }); + Ok(()) + }) } #[profiling::function] - fn record_node_passes

( - &mut self, - pool: &mut P, - cmd_buf: &mut CommandBuffer, - node_idx: usize, - end_pass_idx: usize, - ) -> Result<(), DriverError> - where - P: Pool + Pool, - { - thread_local! { - static SCHEDULE: RefCell = Default::default(); - } + fn bind_descriptor_sets( + cmd_buf: &CommandBuffer, + pipeline: &ExecutionPipeline, + recorded_command: &CommandRecordingResources, + exec_idx: usize, + ) { + if let Some(exec_descriptor_sets) = recorded_command.descriptor_sets.get(exec_idx) { + thread_local! { + static DESCRIPTOR_SETS: RefCell> = Default::default(); + } - SCHEDULE.with_borrow_mut(|schedule| { - schedule.access_cache.update(&self.graph, end_pass_idx); - schedule.passes.clear(); + if exec_descriptor_sets.is_empty() { + return; + } - self.schedule_node_passes(node_idx, end_pass_idx, schedule); - self.record_scheduled_passes(pool, cmd_buf, schedule, end_pass_idx) - }) + DESCRIPTOR_SETS.with_borrow_mut(|descriptor_sets| { + descriptor_sets.clear(); + descriptor_sets.extend( + exec_descriptor_sets + .iter() + .map(|descriptor_set| **descriptor_set), + ); + + trace!(" bind descriptor sets {:?}", descriptor_sets); + + unsafe { + cmd_buf.device.cmd_bind_descriptor_sets( + cmd_buf.handle, + pipeline.bind_point(), + pipeline.layout(), + 0, + descriptor_sets, + &[], + ); + } + }); + } } #[profiling::function] - fn record_scheduled_passes

( - &mut self, - pool: &mut P, - cmd_buf: &mut CommandBuffer, - schedule: &mut Schedule, - end_pass_idx: usize, - ) -> Result<(), DriverError> - where - P: Pool + Pool, - { - if schedule.passes.is_empty() { - return Ok(()); + fn bind_pipeline( + cmd_buf: &CommandBuffer, + recorded_command: &mut CommandRecordingResources, + exec_idx: usize, + pipeline: &mut ExecutionPipeline, + depth_stencil: Option, + ) -> Result<(), DriverError> { + if log_enabled!(Trace) { + let (pipeline_kind, name, vk_pipeline) = match pipeline { + ExecutionPipeline::Compute(pipeline) => ( + "compute", + Device::private_data_object_name( + pipeline.device(), + vk::ObjectType::PIPELINE, + pipeline.handle(), + ), + pipeline.handle(), + ), + ExecutionPipeline::Graphics(pipeline) => ( + "graphics", + Device::private_data_object_name( + pipeline.device(), + vk::ObjectType::PIPELINE_LAYOUT, + pipeline.inner.layout, + ), + vk::Pipeline::null(), + ), + ExecutionPipeline::RayTracing(pipeline) => ( + "ray tracing", + Device::private_data_object_name( + pipeline.device(), + vk::ObjectType::PIPELINE, + pipeline.handle(), + ), + pipeline.handle(), + ), + }; + if let Some(name) = name { + trace!(" bind {pipeline_kind} pipeline {name} ({vk_pipeline:?})"); + } else { + trace!(" bind {pipeline_kind} pipeline {vk_pipeline:?}"); + } } - // // Print some handy details or hit a breakpoint if you set the flag - // if log_enabled!(Debug) && self.graph.debug { - // debug!("resolving the following graph:\n\n{:#?}\n\n", self.graph); - // } + // We store a shared reference to this pipeline inside the command buffer! + let bind_point = pipeline.bind_point(); + let pipeline = match pipeline { + ExecutionPipeline::Compute(pipeline) => pipeline.handle(), + ExecutionPipeline::Graphics(pipeline) => RenderPass::pipeline_handle( + recorded_command.expect_render_pass_mut(), + pipeline, + depth_stencil, + exec_idx as _, + )?, + ExecutionPipeline::RayTracing(pipeline) => pipeline.handle(), + }; - debug_assert!( - schedule.passes.windows(2).all(|w| w[0] <= w[1]), - "Unsorted schedule" - ); + unsafe { + cmd_buf + .device + .cmd_bind_pipeline(cmd_buf.handle, bind_point, pipeline); + } - // Optimize the schedule; requesting the required resources it needs - Self::reorder_scheduled_passes(schedule, end_pass_idx); - self.merge_scheduled_passes(&mut schedule.passes); - self.lease_scheduled_resources(pool, &schedule.passes)?; - - #[cfg(feature = "checked")] - let graph_id = self.graph.graph_id(); - - for pass_idx in schedule.passes.iter().copied() { - let pass = &mut self.graph.cmds[pass_idx]; - - profiling::scope!("Pass", pass.name()); - - let physical_pass = &mut self.physical_passes[pass_idx]; - let is_graphic = physical_pass.render_pass.is_some(); - - trace!("recording pass [{}: {}]", pass_idx, pass.name()); - - if !physical_pass.exec_descriptor_sets.is_empty() { - Self::write_descriptor_sets(cmd_buf, &self.graph.resources, pass, physical_pass)?; - } - - let render_area = if is_graphic { - Self::record_image_layout_transitions(cmd_buf, &mut self.graph.resources, pass); - - let render_area = vk::Rect2D { - offset: vk::Offset2D { x: 0, y: 0 }, - extent: Self::render_extent(&self.graph.resources, pass), - }; - - Self::begin_render_pass( - cmd_buf, - &self.graph.resources, - pass, - physical_pass, - render_area, - )?; + Ok(()) + } - Some(render_area) - } else { - None - }; + /// Collects release groups from pending ownership transfers. + fn collect_queue_ownership_release_groups(&self) -> Box<[QueueOwnershipReleaseGroup]> { + let mut release_groups = Vec::::new(); - for exec_idx in 0..pass.execs.len() { - let render_area = is_graphic.then(|| { - pass.execs[exec_idx] - .render_area - .unwrap_or(render_area.expect("missing render area")) - }); + thread_local! { + static TRANSFER_GROUP_INDICES: RefCell> = Default::default(); + } - let exec = &mut pass.execs[exec_idx]; + TRANSFER_GROUP_INDICES.with_borrow_mut(|tls| { + tls.clear(); - if is_graphic && exec_idx > 0 { - Self::next_subpass(cmd_buf); - } + if let Some(pending_buffer_transfer_nodes) = &self.pending_buffer_transfer_nodes { + for (_, buffer, transfers) in pending_buffer_transfer_nodes.iter() { + for transfer in transfers.iter().copied() { + let key = (transfer.src_queue_family_index, transfer.src_queue_index); + let buffer_transfer = (buffer, transfer.range); - if let Some(pipeline) = exec.pipeline.as_mut() { - Self::bind_pipeline( - cmd_buf, - physical_pass, - exec_idx, - pipeline, - exec.depth_stencil, - )?; + if let Some(&group_idx) = tls.get(&key) { + release_groups[group_idx].buffers.push(buffer_transfer); + } else { + let group_idx = release_groups.len(); - if is_graphic { - let render_area = render_area.expect("missing render area"); + release_groups.push(QueueOwnershipReleaseGroup { + src_queue_family_index: key.0, + src_queue_index: key.1, + buffers: vec![buffer_transfer], + images: Vec::new(), + }); - // In this case we set the viewport and scissor for the user - Self::set_viewport( - cmd_buf, - render_area.offset.x as _, - render_area.offset.y as _, - render_area.extent.width as _, - render_area.extent.height as _, - exec.depth_stencil - .map(|depth_stencil| { - let min = depth_stencil.min.0; - let max = depth_stencil.max.0; - min..max - }) - .unwrap_or(0.0..1.0), - ); - Self::set_scissor( - cmd_buf, - render_area.offset.x, - render_area.offset.y, - render_area.extent.width, - render_area.extent.height, - ); + tls.insert(key, group_idx); + } } - - Self::bind_descriptor_sets(cmd_buf, pipeline, physical_pass, exec_idx); } + } - if !is_graphic { - Self::record_execution_barriers( - cmd_buf, - &mut self.graph.resources, - exec.accesses.iter(), - ); - } + if let Some(pending_image_transfer_nodes) = &self.pending_image_transfer_nodes { + for (_, image, transfers) in pending_image_transfer_nodes.iter() { + for transfer in transfers.iter().copied() { + let key = (transfer.src_queue_family_index, transfer.src_queue_index); + let image_transfer = (image, transfer.layout, transfer.range); - trace!(" > exec[{exec_idx}]"); + if let Some(&group_idx) = tls.get(&key) { + release_groups[group_idx].images.push(image_transfer); + } else { + let group_idx = release_groups.len(); - { - profiling::scope!("Execute callback"); + release_groups.push(QueueOwnershipReleaseGroup { + src_queue_family_index: key.0, + src_queue_index: key.1, + buffers: Vec::new(), + images: vec![image_transfer], + }); - let exec_func = exec.func.take().expect("missing command function").0; - exec_func(crate::cmd::CommandRef::new( - cmd_buf, - &self.graph.resources, - #[cfg(feature = "checked")] - exec, - #[cfg(feature = "checked")] - graph_id, - )); + tls.insert(key, group_idx); + } + } } } + }); - if is_graphic { - self.end_render_pass(cmd_buf); - } - } - - thread_local! { - static PASSES: RefCell> = Default::default(); - } - - PASSES.with_borrow_mut(|passes| { - debug_assert!(passes.is_empty()); + release_groups.into_boxed_slice() + } - // We have to keep the bindings and pipelines alive until the gpu is done - schedule.passes.sort_unstable(); - while let Some(schedule_idx) = schedule.passes.pop() { - debug_assert!(!self.graph.cmds.is_empty()); + /// Returns `true` when this submission contains no more commands to record. + pub fn is_empty(&self) -> bool { + self.graph.cmds.is_empty() + } - while let Some(pass) = self.graph.cmds.pop() { - let pass_idx = self.graph.cmds.len(); + #[allow(clippy::type_complexity)] + #[profiling::function] + fn lease_descriptor_pool

( + pool: &mut P, + pass: &CommandData, + ) -> Result>, DriverError> + where + P: SubmissionPool, + { + let max_set_idx = pass + .execs + .iter() + .flat_map(|exec| exec.bindings.keys()) + .map(|descriptor| descriptor.set()) + .max() + .unwrap_or_default(); + let max_sets = pass.execs.len() as u32 * (max_set_idx + 1); + let mut info = DescriptorPoolInfo { + max_sets, + ..Default::default() + }; - if pass_idx == schedule_idx { - // This was a scheduled pass - store it! + // Find the total count of descriptors per type (there may be multiple pipelines!) + for pool_size in pass.descriptor_pools_sizes() { + for (&descriptor_ty, &descriptor_count) in pool_size { + debug_assert_ne!(descriptor_count, 0); - cmd_buf.drop_after_executed(( - pass, - self.physical_passes.pop().expect("missing physical pass"), - )); - break; - } else { - debug_assert!(pass_idx > schedule_idx); + match descriptor_ty { + vk::DescriptorType::ACCELERATION_STRUCTURE_KHR => { + info.acceleration_structure_count += descriptor_count; + } + vk::DescriptorType::COMBINED_IMAGE_SAMPLER => { + info.combined_image_sampler_count += descriptor_count; + } + vk::DescriptorType::INPUT_ATTACHMENT => { + info.input_attachment_count += descriptor_count; + } + vk::DescriptorType::SAMPLED_IMAGE => { + info.sampled_image_count += descriptor_count; + } + vk::DescriptorType::SAMPLER => { + info.sampler_count += descriptor_count; + } + vk::DescriptorType::STORAGE_BUFFER => { + info.storage_buffer_count += descriptor_count; + } + vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => { + info.storage_buffer_dynamic_count += descriptor_count; + } + vk::DescriptorType::STORAGE_IMAGE => { + info.storage_image_count += descriptor_count; + } + vk::DescriptorType::STORAGE_TEXEL_BUFFER => { + info.storage_texel_buffer_count += descriptor_count; + } + vk::DescriptorType::UNIFORM_BUFFER => { + info.uniform_buffer_count += descriptor_count; + } + vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC => { + info.uniform_buffer_dynamic_count += descriptor_count; + } + vk::DescriptorType::UNIFORM_TEXEL_BUFFER => { + info.uniform_texel_buffer_count += descriptor_count; + } + _ => { + warn!( + "unsupported descriptor type {:?} for command {}", + descriptor_ty, + pass.name(), + ); - passes.push(pass); + return Err(DriverError::Unsupported); } - } + }; } + } - debug_assert!(self.physical_passes.is_empty()); + // It's possible to execute a command-only pipeline + if info.is_empty() { + return Ok(None); + } - // Put the other passes back for future resolves - self.graph.cmds.extend(passes.drain(..).rev()); - }); + // Trivially round up the descriptor counts to increase cache coherence + const ATOM: u32 = 1 << 5; + info.acceleration_structure_count = + info.acceleration_structure_count.next_multiple_of(ATOM); + info.combined_image_sampler_count = + info.combined_image_sampler_count.next_multiple_of(ATOM); + info.input_attachment_count = info.input_attachment_count.next_multiple_of(ATOM); + info.sampled_image_count = info.sampled_image_count.next_multiple_of(ATOM); + info.sampler_count = info.sampler_count.next_multiple_of(ATOM); + info.storage_buffer_count = info.storage_buffer_count.next_multiple_of(ATOM); + info.storage_buffer_dynamic_count = + info.storage_buffer_dynamic_count.next_multiple_of(ATOM); + info.storage_image_count = info.storage_image_count.next_multiple_of(ATOM); + info.storage_texel_buffer_count = info.storage_texel_buffer_count.next_multiple_of(ATOM); + info.uniform_buffer_count = info.uniform_buffer_count.next_multiple_of(ATOM); + info.uniform_buffer_dynamic_count = + info.uniform_buffer_dynamic_count.next_multiple_of(ATOM); + info.uniform_texel_buffer_count = info.uniform_texel_buffer_count.next_multiple_of(ATOM); - log::trace!("Recorded passes"); + // Rounded descriptor counts make descriptor pools more reusable across similar pipelines - Ok(()) + // debug!("{:#?}", info); + + Ok(Some(pool.descriptor_pool(info)?)) } #[profiling::function] - fn render_extent(bindings: &[AnyResource], pass: &CommandData) -> vk::Extent2D { - // set_render_area was not specified so we're going to guess using the minimum common - // attachment extents - let first_exec = pass.expect_first_exec(); + fn lease_render_pass

( + &self, + pool: &mut P, + pass_idx: usize, + external_access_history: &ExternalRenderPassAccessHistory, + ) -> Result, DriverError> + where + P: SubmissionPool, + { + let pass = &self.graph.cmds[pass_idx]; + let (mut color_attachment_count, mut depth_stencil_attachment_count) = (0, 0); + for exec in &pass.execs { + color_attachment_count = color_attachment_count.max(exec.attachments.color.len()); - // We must be able to find the render area because render passes require at least one - // image to be attached - let (mut width, mut height) = (u32::MAX, u32::MAX); - for (attachment_width, attachment_height) in first_exec - .color_clears - .values() - .copied() - .map(|(attachment, _)| attachment) - .chain(first_exec.color_loads.values().copied()) - .chain(first_exec.color_stores.values().copied()) - .chain( - first_exec - .depth_stencil_clear - .map(|(attachment, _)| attachment), - ) - .chain(first_exec.depth_stencil_load) - .chain(first_exec.depth_stencil_store) - .map(|attachment| { - let info = Self::expect_attachment_image(bindings, &attachment).info; + let depth_stencil = exec.attachments.depth_stencil_attachment(); + let has_depth_stencil_attachment = + depth_stencil.is_some_and(|state| state.is_attachment); + let has_depth_stencil_resolve = depth_stencil.and_then(|state| state.resolve).is_some(); - ( - info.width >> attachment.base_mip_level, - info.height >> attachment.base_mip_level, - ) - }) - { - width = width.min(attachment_width); - height = height.min(attachment_height); + depth_stencil_attachment_count = depth_stencil_attachment_count + .max(has_depth_stencil_attachment as usize + has_depth_stencil_resolve as usize); } - vk::Extent2D { height, width } - } - - #[profiling::function] - fn reorder_scheduled_passes(schedule: &mut Schedule, end_pass_idx: usize) { - // It must be a party - if schedule.passes.len() < 3 { - return; - } + let attachment_count = color_attachment_count + depth_stencil_attachment_count; + let mut attachments = Vec::with_capacity(attachment_count); + attachments.resize_with(attachment_count, AttachmentInfo::default); - let mut scheduled = 0; + let mut subpasses = Vec::::with_capacity(pass.execs.len()); - thread_local! { - static UNSCHEDULED: RefCell> = Default::default(); - } + { + let mut color_set = FixedBitSet::with_capacity(attachment_count); + color_set.grow(attachment_count); + let mut depth_stencil_set = false; - UNSCHEDULED.with_borrow_mut(|unscheduled| { - unscheduled.truncate(end_pass_idx); - unscheduled.fill(true); - unscheduled.resize(end_pass_idx, true); + // Add load op attachments using the first executions + for exec in &pass.execs { + for (attachment_idx, state) in exec.attachments.color_attachments() { + let attachment_idx = attachment_idx as usize; + if color_set.put(attachment_idx) { + continue; + } - // Re-order passes by maximizing the distance between dependent nodes - while scheduled < schedule.passes.len() { - let mut best_idx = scheduled; - let pass_idx = schedule.passes[best_idx]; - let mut best_overlap_factor = schedule - .access_cache - .interdependent_passes(pass_idx, end_pass_idx) - .count(); + let attachment = &mut attachments[attachment_idx]; + attachment.format = state.attachment.format; + attachment.sample_count = state.attachment.sample_count; + attachment.initial_layout = vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL; + attachment.load_op = match state.load { + LoadOp::DontCare => vk::AttachmentLoadOp::DONT_CARE, + LoadOp::Load => vk::AttachmentLoadOp::LOAD, + LoadOp::Clear(_) => vk::AttachmentLoadOp::CLEAR, + }; + } - for (idx, pass_idx) in schedule.passes[best_idx + 1..schedule.passes.len()] - .iter() - .enumerate() - { - let mut overlap_factor = 0; - - for other_pass_idx in schedule - .access_cache - .interdependent_passes(*pass_idx, end_pass_idx) + if !depth_stencil_set { + if let Some(state) = exec + .attachments + .depth_stencil_attachment() + .filter(|state| state.is_attachment) { - if unscheduled[other_pass_idx] { - // This pass can't be the candidate because it depends on unfinished - // work - break; - } + let attachment = &mut attachments[color_attachment_count]; + attachment.format = state.attachment.format; + attachment.sample_count = state.attachment.sample_count; + let is_load = matches!(state.load, LoadOp::Load); + attachment.initial_layout = + if state.attachment.aspect_mask.contains( + vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, + ) { + attachment.load_op = match state.load { + LoadOp::DontCare => vk::AttachmentLoadOp::DONT_CARE, + LoadOp::Load => vk::AttachmentLoadOp::LOAD, + LoadOp::Clear(_) => vk::AttachmentLoadOp::CLEAR, + }; + attachment.stencil_load_op = match state.load { + LoadOp::DontCare => vk::AttachmentLoadOp::DONT_CARE, + LoadOp::Load => vk::AttachmentLoadOp::LOAD, + LoadOp::Clear(_) => vk::AttachmentLoadOp::CLEAR, + }; - overlap_factor += 1; - } + if is_load { + vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL + } else { + vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL + } + } else if state + .attachment + .aspect_mask + .contains(vk::ImageAspectFlags::DEPTH) + { + attachment.load_op = match state.load { + LoadOp::DontCare => vk::AttachmentLoadOp::DONT_CARE, + LoadOp::Load => vk::AttachmentLoadOp::LOAD, + LoadOp::Clear(_) => vk::AttachmentLoadOp::CLEAR, + }; - if overlap_factor > best_overlap_factor { - best_idx += idx + 1; - best_overlap_factor = overlap_factor; + if is_load { + vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL + } else { + vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL + } + } else { + attachment.stencil_load_op = match state.load { + LoadOp::DontCare => vk::AttachmentLoadOp::DONT_CARE, + LoadOp::Load => vk::AttachmentLoadOp::LOAD, + LoadOp::Clear(_) => vk::AttachmentLoadOp::CLEAR, + }; + + if is_load { + vk::ImageLayout::STENCIL_READ_ONLY_OPTIMAL + } else { + vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL + } + }; + depth_stencil_set = true; + } else if exec.attachments.depth_stencil_attachment().is_some() { + depth_stencil_set = true; } } - - unscheduled[schedule.passes[best_idx]] = false; - schedule.passes.swap(scheduled, best_idx); - scheduled += 1; } - }); - } + } - /// Returns a borrow of the original Vulkan resource (buffer, image or acceleration structure) - /// which the given node represents. - pub fn resource(&self, resource_node: N) -> &N::Resource - where - N: Node, - { - self.graph.resource(resource_node) - } + { + let mut color_set = FixedBitSet::with_capacity(attachment_count); + color_set.grow(attachment_count); + let mut depth_stencil_set = false; + let mut depth_stencil_resolve_set = false; - /// Returns a vec of pass indexes that are required to be executed, in order, for the given - /// node. - #[profiling::function] - fn schedule_node_passes(&self, node_idx: usize, end_pass_idx: usize, schedule: &mut Schedule) { - type UnscheduledUnresolvedUnchecked = (Vec, Vec, VecDeque<(usize, usize)>); + // Add store op attachments using the last executions + for exec in pass.execs.iter().rev() { + for (attachment_idx, state) in exec.attachments.color_attachments() { + let attachment_idx = attachment_idx as usize; + if color_set.put(attachment_idx) { + continue; + } - thread_local! { - static UNSCHEDULED_UNRESOLVED_UNCHECKED: RefCell< - UnscheduledUnresolvedUnchecked, - > = Default::default(); - } + let attachment = &mut attachments[attachment_idx]; + attachment.format = state.attachment.format; + attachment.sample_count = state.attachment.sample_count; + attachment.store_op = if state.store == StoreOp::Store { + vk::AttachmentStoreOp::STORE + } else { + vk::AttachmentStoreOp::DONT_CARE + }; + attachment.final_layout = vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL; + } - UNSCHEDULED_UNRESOLVED_UNCHECKED.with_borrow_mut(|(unscheduled, unresolved, unchecked)| { - unscheduled.truncate(end_pass_idx); - unscheduled.fill(true); - unscheduled.resize(end_pass_idx, true); + if !depth_stencil_set + && let Some(state) = exec + .attachments + .depth_stencil_attachment() + .filter(|state| state.is_attachment) + { + let attachment = &mut attachments[color_attachment_count]; + attachment.format = state.attachment.format; + attachment.sample_count = state.attachment.sample_count; + attachment.final_layout = if state + .attachment + .aspect_mask + .contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) + { + attachment.store_op = if state.store == StoreOp::Store { + vk::AttachmentStoreOp::STORE + } else { + vk::AttachmentStoreOp::DONT_CARE + }; + attachment.stencil_store_op = if state.store == StoreOp::Store { + vk::AttachmentStoreOp::STORE + } else { + vk::AttachmentStoreOp::DONT_CARE + }; - unresolved.truncate(self.graph.resources.len()); - unresolved.fill(true); - unresolved.resize(self.graph.resources.len(), true); + vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL + } else if state + .attachment + .aspect_mask + .contains(vk::ImageAspectFlags::DEPTH) + { + attachment.store_op = if state.store == StoreOp::Store { + vk::AttachmentStoreOp::STORE + } else { + vk::AttachmentStoreOp::DONT_CARE + }; - debug_assert!(unchecked.is_empty()); + vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL + } else { + attachment.stencil_store_op = if state.store == StoreOp::Store { + vk::AttachmentStoreOp::STORE + } else { + vk::AttachmentStoreOp::DONT_CARE + }; - trace!("scheduling node {node_idx}"); + vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL + }; + depth_stencil_set = true; + } - unresolved[node_idx] = false; + if !depth_stencil_resolve_set + && let Some(state) = exec + .attachments + .depth_stencil_attachment() + .and_then(|state| state.resolve) + { + let attachment = attachments + .last_mut() + .expect("missing depth stencil resolve attachment"); + attachment.format = state.attachment.format; + attachment.sample_count = state.attachment.sample_count; + attachment.final_layout = if state + .attachment + .aspect_mask + .contains(vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL) + { + vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL + } else if state + .attachment + .aspect_mask + .contains(vk::ImageAspectFlags::DEPTH) + { + vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL + } else { + vk::ImageLayout::STENCIL_ATTACHMENT_OPTIMAL + }; + depth_stencil_resolve_set = true; + } + } + } - // Schedule the first set of passes for the node we're trying to resolve - for pass_idx in schedule - .access_cache - .dependent_passes(node_idx, end_pass_idx) + for attachment in &mut attachments { + if attachment.load_op == vk::AttachmentLoadOp::DONT_CARE { + attachment.initial_layout = vk::ImageLayout::UNDEFINED; + } else if attachment.store_op == vk::AttachmentStoreOp::DONT_CARE + && attachment.stencil_store_op == vk::AttachmentStoreOp::DONT_CARE { - trace!( - " pass [{pass_idx}: {}] is dependent", - self.graph.cmds[pass_idx].name() - ); + attachment.final_layout = attachment.initial_layout; + } + } - debug_assert!(unscheduled[pass_idx]); + // Add subpasses + for (exec_idx, exec) in pass.execs.iter().enumerate() { + let pipeline = exec + .pipeline + .as_ref() + .expect("missing graphics pipeline") + .expect_graphics(); + let mut subpass_info = SubpassInfo::with_capacity(attachment_count); - unscheduled[pass_idx] = false; - schedule.passes.push(pass_idx); + // Add input attachments + for attachment_idx in pipeline.inner.input_attachments.iter() { + let exec_attachment = exec + .attachments + .color_attachment(*attachment_idx) + .expect("missing input attachment"); + debug_assert!( + !matches!(exec_attachment.load, LoadOp::Clear(_)), + "cannot clear color attachment {attachment_idx} because it uses subpass input", + ); - for node_idx in schedule.access_cache.dependent_nodes(pass_idx) { - trace!(" node {node_idx} is dependent"); + let is_random_access = exec_attachment.store == StoreOp::Store; + subpass_info.input_attachments.push(AttachmentRef { + attachment: *attachment_idx, + aspect_mask: exec_attachment.attachment.aspect_mask, + layout: Self::attachment_layout( + exec_attachment.attachment.aspect_mask, + is_random_access, + true, + ), + }); - let unresolved = &mut unresolved[node_idx]; - if *unresolved { - *unresolved = false; - unchecked.push_back((node_idx, pass_idx)); + // Preserve the attachment in previous subpasses as needed. Input render passes are + // expected to resolve to real prior subpasses here. + for prev_exec_idx in (0..exec_idx).rev() { + let prev_exec = &pass.execs[prev_exec_idx]; + if prev_exec + .attachments + .color_attachment(*attachment_idx) + .is_some_and(|state| state.store == StoreOp::Store) + { + break; } + + let prev_subpass = &mut subpasses[prev_exec_idx]; + prev_subpass.preserve_attachments.push(*attachment_idx); } } - trace!("secondary passes below"); + // Set color attachments to defaults + for attachment_idx in 0..color_attachment_count as u32 { + let is_input = subpass_info + .input_attachments + .iter() + .any(|input| input.attachment == attachment_idx); + subpass_info.color_attachments.push(AttachmentRef { + attachment: vk::ATTACHMENT_UNUSED, + aspect_mask: vk::ImageAspectFlags::COLOR, + layout: Self::attachment_layout(vk::ImageAspectFlags::COLOR, true, is_input), + }); + } - // Now schedule all nodes that are required, going through the tree to find them - while let Some((node_idx, pass_idx)) = unchecked.pop_front() { - trace!(" node {node_idx} is dependent"); + for (attachment_idx, state) in exec.attachments.color_attachments() { + if state.is_attachment { + subpass_info.color_attachments[attachment_idx as usize].attachment = + attachment_idx; + } + } - for pass_idx in schedule - .access_cache - .dependent_passes(node_idx, pass_idx + 1) - { - let unscheduled = &mut unscheduled[pass_idx]; - if *unscheduled { - *unscheduled = false; - schedule.passes.push(pass_idx); + // Set depth/stencil attachment + if let Some(state) = exec + .attachments + .depth_stencil_attachment() + .filter(|state| state.is_attachment) + { + let is_random_access = matches!(state.load, LoadOp::Clear(_)) + || matches!(state.load, LoadOp::Load) + || state.store == StoreOp::Store; + subpass_info.depth_stencil_attachment = Some(AttachmentRef { + attachment: color_attachment_count as u32, + aspect_mask: state.attachment.aspect_mask, + layout: Self::attachment_layout( + state.attachment.aspect_mask, + is_random_access, + false, + ), + }); + } - trace!( - " pass [{pass_idx}: {}] is dependent", - self.graph.cmds[pass_idx].name() - ); + // Set color resolves to defaults + subpass_info.color_resolve_attachments.extend(repeat_n( + AttachmentRef { + attachment: vk::ATTACHMENT_UNUSED, + aspect_mask: vk::ImageAspectFlags::empty(), + layout: vk::ImageLayout::UNDEFINED, + }, + color_attachment_count, + )); - for node_idx in schedule.access_cache.dependent_nodes(pass_idx) { - trace!(" node {node_idx} is dependent"); + // Set any used color resolve attachments now + for (dst_attachment_idx, state) in exec.attachments.color_attachments() { + let Some(state) = state.resolve else { + continue; + }; - let unresolved = &mut unresolved[node_idx]; - if *unresolved { - *unresolved = false; - unchecked.push_back((node_idx, pass_idx)); - } - } - } - } + let is_input = subpass_info + .input_attachments + .iter() + .any(|input| input.attachment == dst_attachment_idx); + subpass_info.color_resolve_attachments[state.src_attachment_idx as usize] = + AttachmentRef { + attachment: dst_attachment_idx, + aspect_mask: state.attachment.aspect_mask, + layout: Self::attachment_layout( + state.attachment.aspect_mask, + true, + is_input, + ), + }; } - schedule.passes.sort_unstable(); + if let Some(state) = exec + .attachments + .depth_stencil_attachment() + .and_then(|state| state.resolve) + { + subpass_info.depth_stencil_resolve_attachment = Some(( + AttachmentRef { + attachment: state.dst_attachment_idx + 1, + aspect_mask: state.attachment.aspect_mask, + layout: Self::attachment_layout(state.attachment.aspect_mask, true, false), + }, + state.depth_mode, + state.stencil_mode, + )) + } - if log_enabled!(Debug) { - if !schedule.passes.is_empty() { - // These are the indexes of the passes this thread is about to resolve - debug!( - "schedule: {}", - schedule - .passes - .iter() - .copied() - .map(|idx| format!("[{}: {}]", idx, self.graph.cmds[idx].name())) - .collect::>() - .join(", ") - ); - } + subpass_info.view_mask = exec.view_mask; + subpass_info.correlated_view_mask = exec.correlated_view_mask; - if log_enabled!(Trace) { - let unscheduled = (0..end_pass_idx) - .filter(|&pass_idx| unscheduled[pass_idx]) - .collect::>(); + subpasses.push(subpass_info); + } - if !unscheduled.is_empty() { - // These passes are within the range of passes we thought we had to do - // right now, but it turns out that nothing in "schedule" relies on them - trace!( - "delaying: {}", - unscheduled - .iter() - .copied() - .map(|idx| format!("[{}: {}]", idx, self.graph.cmds[idx].name())) - .collect::>() - .join(", ") - ); + let dependencies = Self::build_subpass_dependencies(pass, external_access_history); + + // let info = RenderPassInfo { + // attachments, + // dependencies, + // subpasses, + // }; + + // trace!("{:#?}", info); + + pool.render_pass(RenderPassInfo { + attachments, + dependencies, + subpasses, + }) + } + + fn build_subpass_dependencies( + pass: &CommandData, + external_access_history: &ExternalRenderPassAccessHistory, + ) -> Vec { + let mut dependencies = BTreeMap::new(); + let mut pass_access_history = + HashMap::>::new(); + + for (exec_idx, exec) in pass.execs.iter().enumerate() { + 'exec_accesses: for (node_idx, accesses) in exec.accesses.iter() { + for access in accesses { + let mut current = PipelineStageAccessFlags::new(access.access); + current.stage_flags = Self::subpass_stage_mask(current.stage_flags); + + if let Some(prev_accesses) = pass_access_history.get(&node_idx) { + for &(prev_exec_idx, previous) in prev_accesses.iter().rev() { + if Self::record_subpass_dependency( + &mut dependencies, + prev_exec_idx, + exec_idx, + previous, + current.stage_flags, + &mut current, + ) { + continue 'exec_accesses; + } + } } - if end_pass_idx < self.graph.cmds.len() { - // These passes existing on the graph but are not being considered right - // now because we've been told to stop work at the "end_pass_idx" point - trace!( - "ignoring: {}", - self.graph.cmds[end_pass_idx..] - .iter() - .enumerate() - .map(|(idx, pass)| format!( - "[{}: {}]", - idx + end_pass_idx, - pass.name() - )) - .collect::>() - .join(", ") - ); + for &previous in external_access_history.accesses(node_idx).iter().rev() { + if Self::record_subpass_dependency( + &mut dependencies, + vk::SUBPASS_EXTERNAL as usize, + exec_idx, + previous, + current.stage_flags, + &mut current, + ) { + continue 'exec_accesses; + } + } + + if !current.stage_flags.is_empty() { + let dep = dependencies + .entry((vk::SUBPASS_EXTERNAL as usize, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(vk::SUBPASS_EXTERNAL, exec_idx as _) + }); + + dep.src_stage_mask |= vk::PipelineStageFlags::ALL_COMMANDS; + dep.src_access_mask |= + vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE; + dep.dst_stage_mask |= current.stage_flags; + dep.dst_access_mask |= current.access_flags; } } } - }); - } - fn set_scissor(cmd_buf: &CommandBuffer, x: i32, y: i32, width: u32, height: u32) { - unsafe { - cmd_buf.device.cmd_set_scissor( - cmd_buf.handle, - 0, - slice::from_ref(&vk::Rect2D { - extent: vk::Extent2D { width, height }, - offset: vk::Offset2D { x, y }, - }), - ); - } - } + for (node_idx, accesses) in exec.accesses.iter() { + let prev_accesses = pass_access_history.entry(node_idx).or_default(); + prev_accesses.extend(accesses.iter().map(|access| { + let mut access_info = PipelineStageAccessFlags::new(access.access); + access_info.stage_flags = Self::subpass_stage_mask(access_info.stage_flags); - fn set_viewport( - cmd_buf: &CommandBuffer, - x: f32, - y: f32, - width: f32, - height: f32, - depth: Range, - ) { - unsafe { - cmd_buf.device.cmd_set_viewport( - cmd_buf.handle, - 0, - slice::from_ref(&vk::Viewport { - x, - y, - width, - height, - min_depth: depth.start, - max_depth: depth.end, - }), - ); - } - } + (exec_idx, access_info) + })); + } - /// Submits the remaining commands stored in this instance. - #[profiling::function] - pub fn queue_submit

( - mut self, - pool: &mut P, - queue_family_index: u32, - queue_index: u32, - ) -> Result, DriverError> - where - P: Pool - + Pool - + Pool, - { - trace!("submit"); + // Look for attachments of this exec being read or written in other execs of the + // same pass + for (other_idx, other) in pass.execs[0..exec_idx].iter().enumerate() { + // Look for color attachments we're reading + for (attachment_idx, state) in + exec.attachments.color_attachments().filter(|(_, state)| { + state.is_input || Self::color_attachment_is_read(state.load) + }) + { + // Look for writes in the other exec + if let Some(other_state) = other.attachments.color_attachment(attachment_idx) + && Self::color_attachment_is_write( + other_state.load, + other_state.store, + other_state.resolve.is_some(), + ) + { + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - let mut cmd = pool.resource(CommandBufferInfo::new(queue_family_index as _))?; + dep.src_stage_mask |= vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT; + dep.src_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_WRITE; - cmd.wait_until_executed()?; + if state.is_input { + dep.dst_stage_mask |= vk::PipelineStageFlags::FRAGMENT_SHADER; + dep.dst_access_mask |= vk::AccessFlags::INPUT_ATTACHMENT_READ; + } else { + dep.dst_stage_mask |= + Self::attachment_read_stage(state.attachment.aspect_mask); + dep.dst_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_READ; + } + } - Device::begin_command_buffer( - &cmd.device, - cmd.handle, - &vk::CommandBufferBeginInfo::default() - .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), - )?; + if let Some(other_state) = other.attachments.color_attachment(attachment_idx) + && (other_state.is_input + || Self::color_attachment_is_read(other_state.load)) + { + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - self.submit_cmd_buf(pool, &mut cmd)?; + if other_state.is_input { + dep.src_stage_mask |= vk::PipelineStageFlags::FRAGMENT_SHADER; + dep.src_access_mask |= vk::AccessFlags::INPUT_ATTACHMENT_READ; + } else { + dep.src_stage_mask |= + Self::attachment_read_stage(state.attachment.aspect_mask); + dep.src_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_READ; + } - Device::with_queue(&cmd.device, queue_family_index, queue_index, |queue| { - Device::end_command_buffer(&cmd.device, cmd.handle)?; - Device::reset_fences(&cmd.device, slice::from_ref(&cmd.fence))?; - Device::queue_submit( - &cmd.device, - queue, - slice::from_ref( - &vk::SubmitInfo::default().command_buffers(slice::from_ref(&cmd.handle)), - ), - cmd.fence, - )?; + if state.is_input { + dep.dst_stage_mask |= vk::PipelineStageFlags::FRAGMENT_SHADER; + dep.dst_access_mask |= vk::AccessFlags::INPUT_ATTACHMENT_READ; + } else { + dep.dst_stage_mask |= + Self::attachment_read_stage(state.attachment.aspect_mask); + dep.dst_access_mask |= vk::AccessFlags::COLOR_ATTACHMENT_READ; + } + } + } - Ok::<_, DriverError>(()) - })?; + if let Some(state) = exec.attachments.depth_stencil_attachment().filter(|state| { + state.is_attachment && Self::depth_stencil_attachment_is_read(state.load) + }) { + let aspect_mask = state.attachment.aspect_mask; + + if other + .attachments + .depth_stencil_attachment() + .is_some_and(|state| { + Self::depth_stencil_attachment_is_write( + state.load, + state.store, + state.resolve.is_some(), + ) + }) + { + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - // This graph contains references to buffers, images, and other resources which must be kept - // alive until this graph execution completes on the GPU. Once those references are dropped - // they will return to the pool for other things to use. The drop will happen the next time - // someone tries to lease a command buffer and we notice this one has returned and the fence - // has been signalled. - cmd.drop_after_executed(self); + dep.src_stage_mask |= vk::PipelineStageFlags::LATE_FRAGMENT_TESTS; + dep.src_access_mask |= vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE; + dep.dst_stage_mask |= vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS; + dep.dst_access_mask |= vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ; + } - Ok(cmd) - } + if other + .attachments + .depth_stencil_attachment() + .is_some_and(|state| Self::depth_stencil_attachment_is_read(state.load)) + { + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - /// Records any pending graph commands that have not been previously scheduled. - #[profiling::function] - pub fn submit_cmd_buf

( - &mut self, - pool: &mut P, - cmd_buf: &mut CommandBuffer, - ) -> Result<(), DriverError> - where - P: Pool + Pool, - { - if self.graph.cmds.is_empty() { - return Ok(()); - } + dep.src_stage_mask |= vk::PipelineStageFlags::LATE_FRAGMENT_TESTS; + dep.src_access_mask |= vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ; + dep.dst_stage_mask |= Self::attachment_read_stage(aspect_mask); + dep.dst_access_mask |= vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ; + } + } - thread_local! { - static SCHEDULE: RefCell = Default::default(); - } + for (attachment_idx, state) in + exec.attachments.color_attachments().filter(|(_, state)| { + Self::color_attachment_is_write( + state.load, + state.store, + state.resolve.is_some(), + ) + }) + { + let aspect_mask = state.attachment.aspect_mask; + let stage = Self::attachment_stage(aspect_mask); + + if other + .attachments + .color_attachment(attachment_idx) + .is_some_and(|state| { + Self::color_attachment_is_write( + state.load, + state.store, + state.resolve.is_some(), + ) + }) + { + let access = Self::attachment_write_access(aspect_mask); + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - SCHEDULE.with_borrow_mut(|schedule| { - schedule - .access_cache - .update(&self.graph, self.graph.cmds.len()); - schedule.passes.clear(); - schedule.passes.extend(0..self.graph.cmds.len()); + dep.src_stage_mask |= stage; + dep.src_access_mask |= access; + dep.dst_stage_mask |= stage; + dep.dst_access_mask |= access; + } - self.record_scheduled_passes(pool, cmd_buf, schedule, self.graph.cmds.len()) - }) - } + if let Some(other_state) = other.attachments.color_attachment(attachment_idx) + && (other_state.is_input + || Self::color_attachment_is_read(other_state.load)) + { + let (src_access, dst_access) = + Self::attachment_read_write_access(aspect_mask); + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - /// Records any pending graph commands that the given node requires into `cmd_buf`. - /// - /// This is a mutating execution step, not a pure query. It records work into the provided - /// command buffer and updates this submission's scheduling state so those commands are not - /// recorded again later. - #[profiling::function] - pub fn queue_cmds_for_resource

( - &mut self, - cmd_buf: &mut CommandBuffer, - resource_node: impl Node, - pool: &mut P, - ) -> Result<(), DriverError> - where - P: Pool + Pool, - { - self.graph.assert_node_owner(&resource_node); + if other_state.is_input { + dep.src_stage_mask |= vk::PipelineStageFlags::FRAGMENT_SHADER; + dep.src_access_mask |= vk::AccessFlags::INPUT_ATTACHMENT_READ; + } else { + dep.src_stage_mask |= Self::attachment_read_stage(aspect_mask); + dep.src_access_mask |= src_access; + } + dep.dst_stage_mask |= stage; + dep.dst_access_mask |= dst_access; + } + } - let node_idx = resource_node.index(); + if let Some(state) = exec.attachments.depth_stencil_attachment().filter(|state| { + Self::depth_stencil_attachment_is_write( + state.load, + state.store, + state.resolve.is_some(), + ) + }) { + let aspect_mask = state.attachment.aspect_mask; + let stage = Self::attachment_stage(aspect_mask); + + if other + .attachments + .depth_stencil_attachment() + .is_some_and(|state| { + Self::depth_stencil_attachment_is_write( + state.load, + state.store, + state.resolve.is_some(), + ) + }) + { + let access = Self::attachment_write_access(aspect_mask); + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); - debug_assert!(self.graph.resources.get(node_idx).is_some()); + dep.src_stage_mask |= stage; + dep.src_access_mask |= access; + dep.dst_stage_mask |= stage; + dep.dst_access_mask |= access; + } - if self.graph.cmds.is_empty() { - return Ok(()); + if other + .attachments + .depth_stencil_attachment() + .is_some_and(|state| Self::depth_stencil_attachment_is_read(state.load)) + { + let (src_access, dst_access) = + Self::attachment_read_write_access(aspect_mask); + let dep = dependencies + .entry((other_idx, exec_idx)) + .or_insert_with(|| { + SubpassDependency::new(other_idx as _, exec_idx as _) + }); + + dep.src_stage_mask |= Self::attachment_read_stage(aspect_mask); + dep.src_access_mask |= src_access; + dep.dst_stage_mask |= stage; + dep.dst_access_mask |= dst_access; + } + } + } } - let end_pass_idx = self.graph.cmds.len(); - self.record_node_passes(pool, cmd_buf, node_idx, end_pass_idx) + dependencies.into_values().collect() } - /// Records any pending graph commands required by the given node into `cmd`, but does not - /// record any command that actually accesses the given node. - /// - /// This is a mutating execution step, not a pure query. It records work into the provided - /// command buffer, updates this submission's scheduling state, and may reorder later recorded - /// work. - /// - /// The call order matters when extracting multiple outputs from the same submission. This - /// method optimizes the schedule for the requested node, and later calls can only optimize on - /// top of that existing state. If you are pulling multiple outputs and care about their final - /// ordering, record the most important output first. #[profiling::function] - pub fn queue_cmds_for_resource_dependencies

( + fn lease_scheduled_resources

( &mut self, - cmd_buf: &mut CommandBuffer, - resource_node: impl Node, pool: &mut P, + schedule: &[usize], ) -> Result<(), DriverError> where - P: Pool + Pool, + P: SubmissionPool, { - self.graph.assert_node_owner(&resource_node); + let mut render_pass_access_history = + ExternalRenderPassAccessHistory::new(self.graph.resources.len()); - let node_idx = resource_node.index(); + for pass_idx in schedule.iter().copied() { + // At the time this function runs the pass will already have been optimized into a + // larger pass made out of anything that might have been merged into it - so we + // only care about one pass at a time here + let pass = &self.graph.cmds[pass_idx]; - debug_assert!(self.graph.resources.get(node_idx).is_some()); + trace!("requesting [{pass_idx}: {}]", pass.name()); - // We record up to but not including the first pass which accesses the target node - if let Some(end_pass_idx) = self.graph.first_node_access_pass_index(resource_node) { - self.record_node_passes(pool, cmd_buf, node_idx, end_pass_idx)?; + let descriptor_pool = Self::lease_descriptor_pool(pool, pass)?; + let mut descriptor_sets = Vec::with_capacity(pass.execs.len()); + descriptor_sets.resize_with(pass.execs.len(), Vec::new); + if let Some(descriptor_pool) = descriptor_pool.as_ref() { + for (exec_idx, exec) in pass.execs.iter().enumerate() { + let Some(pipeline) = exec.pipeline.as_ref() else { + continue; + }; + + let layouts = pipeline.descriptor_info().layouts.values(); + descriptor_sets[exec_idx] = layouts + .into_iter() + .map(|descriptor_set_layout| { + DescriptorPool::allocate_descriptor_set( + descriptor_pool, + descriptor_set_layout, + ) + }) + .collect::>()?; + } + } + + /* + As a side effect of merging compatible passes, all input passes should be attached to + their preceding passes by now. This allows subpasses to use input attachments. If a pass + still starts with input-only work here, it cannot be represented correctly. + */ + debug_assert!(!pass.execs.is_empty()); + debug_assert!( + pass.expect_first_exec().pipeline.is_none() + || !pass + .expect_first_exec() + .pipeline + .as_ref() + .is_some_and(|pipeline| pipeline.is_graphics()) + || pass + .expect_first_exec() + .pipeline + .as_ref() + .expect("missing graphics pipeline") + .expect_graphics() + .inner + .descriptor_info + .pool_sizes + .values() + .filter_map(|pool| pool.get(&vk::DescriptorType::INPUT_ATTACHMENT)) + .next() + .is_none() + ); + + // Also, the render pass may be None if the pass contained no graphics operations. + let render_pass = if pass + .expect_first_exec() + .pipeline + .as_ref() + .map(|pipeline| pipeline.is_graphics()) + .unwrap_or_default() + { + Some(self.lease_render_pass(pool, pass_idx, &render_pass_access_history)?) + } else { + None + }; + + render_pass_access_history.record_cmd(pass); + + self.recorded_commands.push(CommandRecordingResources { + descriptor_pool, + descriptor_sets, + render_pass, + }); } Ok(()) } + // Merge contiguous scheduled graphics commands with compatible attachments. Scheduled command + // order is final during this function. #[profiling::function] - fn write_descriptor_sets( - cmd_buf: &CommandBuffer, - bindings: &[AnyResource], - pass: &CommandData, - physical_pass: &PhysicalPass, - ) -> Result<(), DriverError> { - struct IndexWrite<'a> { - idx: usize, - write: vk::WriteDescriptorSet<'a>, + fn merge_scheduled_cmds(&mut self, schedule: &mut Vec) { + thread_local! { + static CMD_SLOTS: RefCell>> = Default::default(); } - #[derive(Default)] - struct Tls<'a> { - accel_struct_infos: Vec>, - accel_struct_writes: Vec>, - buffer_infos: Vec, - buffer_writes: Vec>, - descriptors: Vec>, - image_infos: Vec, - image_writes: Vec>, - } + CMD_SLOTS.with_borrow_mut(|cmds| { + debug_assert!(cmds.is_empty()); - let mut tls = Tls::default(); + cmds.extend(self.graph.cmds.drain(..).map(Some)); - for (exec_idx, exec, pipeline) in pass - .execs - .iter() - .enumerate() - .filter_map(|(exec_idx, exec)| { - exec.pipeline - .as_ref() - .map(|pipeline| (exec_idx, exec, pipeline)) - }) - .filter(|(.., pipeline)| !pipeline.descriptor_info().layouts.is_empty()) - { - let descriptor_sets = &physical_pass.exec_descriptor_sets[&exec_idx]; + let mut schedule_idx = 0; - // Write the manually bound things (access, read, and write functions) - for (descriptor, (node_idx, view_info)) in exec.bindings.iter() { - let (descriptor_set_idx, dst_binding, binding_offset) = descriptor.into_tuple(); - let Some((descriptor_info, _)) = pipeline.descriptor_bindings().get(&Descriptor { - set: descriptor_set_idx, - binding: dst_binding, - }) else { - warn!( - "binding {}.{}[{}] not found in shader reflection for pass \"{}\"", - descriptor_set_idx, - dst_binding, - binding_offset, - pass.name(), + // debug!("attempting to merge {} passes", schedule.len(),); + + while schedule_idx < schedule.len() { + let mut cmd = cmds[schedule[schedule_idx]] + .take() + .expect("missing scheduled cmd"); + + // Find candidates + let merge_start = schedule_idx + 1; + let mut merge_end = merge_start; + while merge_end < schedule.len() { + let other = cmds[schedule[merge_end]] + .as_ref() + .expect("missing scheduled cmd"); + + debug!( + "attempting to merge [{schedule_idx}: {}] with [{merge_end}: {}]", + cmd.name(), + other.name() ); - return Err(DriverError::InvalidData); - }; - let descriptor_type = descriptor_info.descriptor_type(); - let bound_node = &bindings[*node_idx]; - if let Some(image) = bound_node.as_image() { - let mut image_view_info = *view_info.expect_image(); - // Handle default views which did not specify a particaular aspect - if image_view_info.aspect_mask.is_empty() { - image_view_info.aspect_mask = format_aspect_mask(image.info.fmt); + if Self::allow_merge_passes(&cmd, other) { + merge_end += 1; + } else { + break; } + } - let image_view = Image::view(image, image_view_info)?; - let image_layout = match descriptor_type { - vk::DescriptorType::COMBINED_IMAGE_SAMPLER - | vk::DescriptorType::SAMPLED_IMAGE => { - if image_view_info.aspect_mask.contains( - vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, - ) { - vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL - } else if image_view_info - .aspect_mask - .contains(vk::ImageAspectFlags::DEPTH) - { - vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL - } else if image_view_info - .aspect_mask - .contains(vk::ImageAspectFlags::STENCIL) + if log_enabled!(Trace) && merge_start != merge_end { + trace!( + "merging {} passes into [{schedule_idx}: {}]", + merge_end - merge_start, + cmd.name() + ); + } + + let mut name = cmd.name().to_owned(); + + // Grow the merged cmd once, not per merge + { + let mut additional_name_len = 0; + let mut additional_exec_count = 0; + for merge_idx in merge_start..merge_end { + let other = cmds[schedule[merge_idx]] + .as_ref() + .expect("missing scheduled cmd"); + additional_name_len += other.name().len() + 3; + additional_exec_count += other.execs.len(); + } + + name.reserve(additional_name_len); + cmd.execs.reserve(additional_exec_count); + } + + for merge_idx in merge_start..merge_end { + let mut other = cmds[schedule[merge_idx]] + .take() + .expect("missing scheduled cmd"); + name.push_str(" + "); + name.push_str(other.name()); + cmd.execs.append(&mut other.execs); + } + + #[cfg(debug_assertions)] + { + cmd.name = Some(name); + } + + self.graph.cmds.push(cmd); + schedule_idx += 1 + merge_end - merge_start; + } + + // Reschedule cmds + schedule.truncate(self.graph.cmds.len()); + + for (idx, cmd_idx) in schedule.iter_mut().enumerate() { + *cmd_idx = idx; + } + + // Add the remaining cmds back into the graph for later + for cmd in cmds.drain(..).flatten() { + self.graph.cmds.push(cmd); + } + }); + } + + fn next_subpass(cmd: &CommandBuffer) { + trace!("next_subpass"); + + unsafe { + cmd.device + .cmd_next_subpass(cmd.handle, vk::SubpassContents::INLINE); + } + } + + fn record_node

( + &mut self, + resource_pool: &mut P, + cmd_buf: &CommandBuffer, + node: AnyNode, + ) -> Result<(), DriverError> + where + P: SubmissionPool, + { + match node { + AnyNode::AccelerationStructure(node) => { + self.record_resource_impl(resource_pool, cmd_buf, node) + } + AnyNode::Buffer(node) => self.record_resource_impl(resource_pool, cmd_buf, node), + AnyNode::Image(node) => self.record_resource_impl(resource_pool, cmd_buf, node), + } + } + + #[profiling::function] + fn record_selection_impl<'a, P>( + &mut self, + resource_pool: &mut P, + cmd_buf: &CommandBuffer, + selection: RecordSelection<'a>, + ) -> Result<(), DriverError> + where + P: SubmissionPool, + { + let _ = CommandBufferDebugLabel::begin(cmd_buf, "graph submission"); + + match selection { + RecordSelection::All => self.record_impl(resource_pool, cmd_buf), + RecordSelection::Dependencies(node) => match node { + AnyNode::AccelerationStructure(node) => { + self.record_resource_dependencies_impl(resource_pool, cmd_buf, node) + } + AnyNode::Buffer(node) => { + self.record_resource_dependencies_impl(resource_pool, cmd_buf, node) + } + AnyNode::Image(node) => { + self.record_resource_dependencies_impl(resource_pool, cmd_buf, node) + } + }, + RecordSelection::Node(node) => self.record_node(resource_pool, cmd_buf, node), + RecordSelection::Nodes(nodes) => { + for &node in nodes { + self.record_node(resource_pool, cmd_buf, node)?; + } + + Ok(()) + } + } + } + + #[profiling::function] + fn record_execution_barriers<'a>( + cmd_buf: &CommandBuffer, + resources: &mut [AnyResource], + accesses: &'a ExecutionAccess, + pending_buffer_transfer_nodes: &mut Option< + PendingTransferNodes, + >, + pending_image_transfer_nodes: &mut Option< + PendingTransferNodes, + >, + ) { + // We store a Barriers in TLS to save an alloc; contents are POD + thread_local! { + static BARRIER: RefCell = Default::default(); + } + + struct AccessBarrier { + next_access: AccessType, + prev_access: AccessType, + resource: T, + } + + struct BufferBarrierTarget { + buffer: vk::Buffer, + range: BufferSubresourceRange, + } + + struct ImageBarrierTarget { + image: vk::Image, + range: vk::ImageSubresourceRange, + } + + #[derive(Default)] + struct BarrierScratch { + accel_struct_accesses: Vec, + buffers: Vec>, + images: Vec>, + next_accesses: Vec, + pending_buffers: NodeIndexedScratch>, + pending_images: NodeIndexedScratch>, + prev_accesses: Vec, + } + + BARRIER.with_borrow_mut(|tls| { + // Initialize TLS from a previous call + tls.accel_struct_accesses.clear(); + tls.buffers.clear(); + tls.images.clear(); + tls.next_accesses.clear(); + tls.pending_buffers.clear(); + tls.pending_images.clear(); + tls.prev_accesses.clear(); + + // Map remaining accesses into vk_sync barriers (some accesses may have been removed by + // the render pass request function) + + for (node_idx, node_accesses) in accesses.iter() { + enum ResourceRef<'a> { + AccelerationStructure(&'a AccelerationStructure), + Buffer(&'a Buffer), + Image(&'a Image), + } + + let resource = match &resources[node_idx] { + AnyResource::AccelerationStructure(resource) => { + ResourceRef::AccelerationStructure(resource) + } + AnyResource::AccelerationStructureArg(_) => { + panic!("unbound command stream acceleration structure argument") + } + AnyResource::AccelerationStructureLease(resource) => { + ResourceRef::AccelerationStructure(resource) + } + AnyResource::Buffer(resource) => ResourceRef::Buffer(resource), + AnyResource::BufferArg(_) => panic!("unbound command stream buffer argument"), + AnyResource::BufferLease(resource) => ResourceRef::Buffer(resource), + AnyResource::Image(resource) => ResourceRef::Image(resource), + AnyResource::ImageArg(_) => panic!("unbound command stream image argument"), + AnyResource::ImageLease(resource) => ResourceRef::Image(resource), + AnyResource::SwapchainImage(resource) => ResourceRef::Image(resource), + }; + + match resource { + ResourceRef::AccelerationStructure(accel_struct) => { + let canonical_accesses = Self::accel_struct_canonical_accesses( + node_accesses, + &mut tls.accel_struct_accesses, + ); + tls.next_accesses.extend(canonical_accesses.iter().copied()); + tls.prev_accesses + .extend(AccelerationStructure::swap_accesses( + accel_struct, + canonical_accesses, + )); + } + ResourceRef::Buffer(buffer) => { + for (next_access, prev_access, range) in Buffer::swap_accesses( + buffer, + node_accesses.iter().map( + |&SubresourceAccess { + access, + subresource, + }| { + let SubresourceRange::Buffer(range) = subresource else { + unreachable!() + }; + + (access, range) + }, + ), + ) { + let barrier = AccessBarrier { + next_access, + prev_access, + resource: BufferBarrierTarget { + buffer: buffer.handle, + range, + }, + }; + + if pending_buffer_transfer_nodes + .as_ref() + .is_some_and(|pending| pending.contains(node_idx)) { - vk::ImageLayout::STENCIL_READ_ONLY_OPTIMAL + tls.pending_buffers.push(node_idx, barrier); } else { - vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL + tls.buffers.push(barrier); } } - vk::DescriptorType::STORAGE_IMAGE => vk::ImageLayout::GENERAL, - _ => { - warn!( - "invalid image descriptor type at binding {}.{}[{}] in pass \"{}\"", - descriptor_set_idx, - dst_binding, - binding_offset, - pass.name() - ); + } + ResourceRef::Image(image) => { + for (next_access, prev_access, range) in Image::swap_accesses( + image, + node_accesses.iter().map( + |&SubresourceAccess { + access, + subresource, + }| { + let SubresourceRange::Image(range) = subresource else { + unreachable!() + }; + + (access, range) + }, + ), + ) { + let barrier = AccessBarrier { + next_access, + prev_access, + resource: ImageBarrierTarget { + image: image.handle, + range, + }, + }; + + if pending_image_transfer_nodes + .as_ref() + .is_some_and(|pending| pending.contains(node_idx)) + { + tls.pending_images.push(node_idx, barrier); + } else { + tls.images.push(barrier); + } + } + } + } + } + + let global_barrier = if !tls.next_accesses.is_empty() { + // No resource attached - we use a global barrier for these + trace!( + " global {:?}->{:?}", + tls.next_accesses, tls.prev_accesses + ); + + Some(GlobalBarrier { + next_accesses: tls.next_accesses.as_slice(), + previous_accesses: tls.prev_accesses.as_slice(), + }) + } else { + None + }; + let mut buffer_barriers = Vec::new(); + for AccessBarrier { + next_access, + prev_access, + resource, + } in tls.buffers.iter() + { + let BufferBarrierTarget { buffer, range, .. } = *resource; + + buffer_barriers.push(BufferBarrier { + next_accesses: slice::from_ref(next_access), + previous_accesses: slice::from_ref(prev_access), + src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, + dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, + buffer, + offset: range.start as _, + size: (range.end - range.start) as _, + }); + } + + if let Some(pending_buffer_transfer_nodes) = pending_buffer_transfer_nodes.as_ref() { + for (node_idx, _buffer, transfers) in pending_buffer_transfer_nodes.iter() { + for AccessBarrier { + next_access, + prev_access, + resource, + } in tls.pending_buffers.get(node_idx) + { + buffer_barriers.extend(buffer_barriers_from_transfers( + resource.buffer, + prev_access, + next_access, + resource.range, + transfers, + )); + } + } + } + + let mut image_barriers = Vec::new(); + for AccessBarrier { + next_access, + prev_access, + resource, + } in tls.images.iter() + { + let ImageBarrierTarget { image, range, .. } = *resource; + + image_barriers.push(ImageBarrier { + next_accesses: slice::from_ref(next_access), + previous_accesses: slice::from_ref(prev_access), + next_layout: image_access_layout(*next_access), + previous_layout: image_access_layout(*prev_access), + discard_contents: image_execution_discard_contents(*prev_access), + src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, + dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, + image, + range, + }); + } + + if let Some(pending_image_transfer_nodes) = pending_image_transfer_nodes.as_ref() { + for (node_idx, _image, transfers) in pending_image_transfer_nodes.iter() { + for AccessBarrier { + next_access, + prev_access, + resource, + } in tls.pending_images.get(node_idx) + { + image_barriers.extend(image_barriers_from_transfers( + resource.image, + prev_access, + next_access, + resource.range, + transfers, + image_execution_discard_contents(*prev_access), + )); + } + } + } + + pipeline_barrier_from_iters( + &cmd_buf.device, + cmd_buf.handle, + global_barrier, + buffer_barriers.into_iter(), + image_barriers.into_iter(), + ); + + if let Some(pending) = pending_buffer_transfer_nodes.as_mut() { + pending.remove_where(|node_idx, _buffer, transfers| { + for AccessBarrier { resource, .. } in tls.pending_buffers.get(node_idx) { + let range = resource.range; + + if consume_pending_buffer_transfers(transfers, range) { + return true; + } + } + + false + }); + + if pending.is_empty() { + *pending_buffer_transfer_nodes = None; + } + } + + if let Some(pending) = pending_image_transfer_nodes.as_mut() { + pending.remove_where(|node_idx, _image, transfers| { + for AccessBarrier { resource, .. } in tls.pending_images.get(node_idx) { + let range = resource.range; + + if consume_pending_image_transfers(transfers, range) { + return true; + } + } + + false + }); + + if pending.is_empty() { + *pending_image_transfer_nodes = None; + } + } + }); + } + + #[profiling::function] + fn record_image_layout_transitions( + cmd_buf: &CommandBuffer, + resources: &mut [AnyResource], + pass: &mut CommandData, + pending_buffer_transfer_nodes: &mut Option< + PendingTransferNodes, + >, + pending_image_transfer_nodes: &mut Option< + PendingTransferNodes, + >, + ) { + struct ImageResourceBarrier { + image: vk::Image, + node_idx: NodeIndex, + next_access: AccessType, + prev_access: AccessType, + range: vk::ImageSubresourceRange, + } + + struct BufferResourceBarrier { + buffer: vk::Buffer, + next_access: AccessType, + prev_access: AccessType, + range: BufferSubresourceRange, + } + + #[derive(Default)] + struct LayoutTransitionScratch { + buffers: Vec, + images: Vec, + first_layout_uses: HashMap>, + pending_buffers: NodeIndexedScratch, + pending_images: NodeIndexedScratch, + } + + // We store a LayoutTransitionScratch in TLS to save an alloc; contents are POD + thread_local! { + static LAYOUT_TRANSITION: RefCell = Default::default(); + } + + LAYOUT_TRANSITION.with_borrow_mut(|tls| { + tls.buffers.clear(); + tls.images.clear(); + tls.first_layout_uses.clear(); + tls.pending_buffers.clear(); + tls.pending_images.clear(); + + for (node_idx, accesses) in pass.execs.iter_mut().flat_map(|exec| exec.accesses.iter()) + { + debug_assert!(resources.get(node_idx).is_some()); + + let resource = unsafe { + // CommandRef enforces this during push_resource_access + resources.get_unchecked(node_idx) + }; + + enum ResourceRef<'a> { + AccelerationStructure(&'a AccelerationStructure), + Buffer(&'a Buffer), + Image(&'a Image), + } + + let resource = match resource { + AnyResource::AccelerationStructure(resource) => { + ResourceRef::AccelerationStructure(resource) + } + AnyResource::AccelerationStructureArg(_) => { + panic!("unbound command stream acceleration structure argument") + } + AnyResource::AccelerationStructureLease(resource) => { + ResourceRef::AccelerationStructure(resource) + } + AnyResource::Buffer(resource) => ResourceRef::Buffer(resource), + AnyResource::BufferArg(_) => panic!("unbound command stream buffer argument"), + AnyResource::BufferLease(resource) => ResourceRef::Buffer(resource), + AnyResource::Image(resource) => ResourceRef::Image(resource), + AnyResource::ImageArg(_) => panic!("unbound command stream image argument"), + AnyResource::ImageLease(resource) => ResourceRef::Image(resource), + AnyResource::SwapchainImage(resource) => ResourceRef::Image(resource), + }; + + match resource { + ResourceRef::AccelerationStructure(accel_struct) => { + AccelerationStructure::swap_access(accel_struct, AccessType::Nothing) + .for_each(drop); + } + ResourceRef::Buffer(buffer) => { + for subresource_access in accesses { + let &SubresourceAccess { + access, + subresource: SubresourceRange::Buffer(access_range), + } = subresource_access + else { + #[cfg(feature = "checked")] + unreachable!(); + + #[cfg(not(feature = "checked"))] + unsafe { + // This cannot be reached because command access recording + // preserves the buffer subresource type for this node. + unreachable_unchecked() + } + }; + + for (prev_access, range) in + Buffer::swap_access(buffer, AccessType::Nothing, access_range) + { + if !pending_buffer_transfer_nodes + .as_ref() + .is_some_and(|pending| pending.contains(node_idx)) + { + continue; + } + + tls.pending_buffers.push( + node_idx, + BufferResourceBarrier { + buffer: buffer.handle, + next_access: access, + prev_access, + range, + }, + ); + } + } + } + ResourceRef::Image(image) => { + let first_layout_uses = tls + .first_layout_uses + .entry(node_idx) + .or_insert_with(|| DenseMap::new(image.info, true)); + + for subresource_access in accesses { + let &SubresourceAccess { + access, + subresource: SubresourceRange::Image(access_range), + } = subresource_access + else { + #[cfg(feature = "checked")] + unreachable!(); + + #[cfg(not(feature = "checked"))] + unsafe { + // This cannot be reached because command access recording + // preserves the image subresource type for this node. + unreachable_unchecked() + } + }; + + let access_range = image.info.resolve_subresource_counts(access_range); + + for (is_initial_layout, layout_range) in + first_layout_uses.swap(false, access_range) + { + for (prev_access, range) in + Image::swap_access(image, access, layout_range) + { + if is_initial_layout { + let barrier = ImageResourceBarrier { + image: image.handle, + node_idx, + next_access: initial_image_layout_access(access), + prev_access, + range, + }; + + if pending_image_transfer_nodes + .as_ref() + .is_some_and(|pending| pending.contains(node_idx)) + { + tls.pending_images.push(node_idx, barrier); + } else { + tls.images.push(barrier); + } + } + } + } + } + } + } + } + + let mut buffer_barriers = Vec::new(); + if let Some(pending_buffer_transfer_nodes) = pending_buffer_transfer_nodes.as_ref() { + for (node_idx, _buffer, transfers) in pending_buffer_transfer_nodes.iter() { + for BufferResourceBarrier { + buffer, + next_access, + prev_access, + range, + .. + } in tls.pending_buffers.get(node_idx) + { + for transfer in transfers.iter().copied() { + let Some(range) = range.intersection(transfer.range) else { + continue; + }; + + trace!( + " buffer {:?} {:?} {:?}->{:?}", + buffer, + range.start..range.end, + prev_access, + next_access, + ); + + buffer_barriers.push(BufferBarrier { + next_accesses: slice::from_ref(next_access), + previous_accesses: slice::from_ref(prev_access), + src_queue_family_index: transfer.src_queue_family_index, + dst_queue_family_index: transfer.dst_queue_family_index, + buffer: *buffer, + offset: range.start as _, + size: (range.end - range.start) as _, + }); + } + } + } + } + + let mut image_barriers = Vec::new(); + for ImageResourceBarrier { + image, + node_idx, + next_access, + prev_access, + range, + } in tls.images.iter() + { + if pending_image_transfer_nodes + .as_ref() + .is_some_and(|pending| pending.contains(*node_idx)) + { + continue; + } + + image_barriers.extend(image_barriers_from_transfers( + *image, + prev_access, + next_access, + *range, + &[], + image_layout_transition_discard_contents(*prev_access, *next_access), + )); + } + + if let Some(pending_image_transfer_nodes) = pending_image_transfer_nodes.as_ref() { + for (node_idx, _image, transfers) in pending_image_transfer_nodes.iter() { + for ImageResourceBarrier { + image, + next_access, + prev_access, + range, + .. + } in tls.pending_images.get(node_idx) + { + image_barriers.extend(image_barriers_from_transfers( + *image, + prev_access, + next_access, + *range, + transfers, + image_layout_transition_discard_contents(*prev_access, *next_access), + )); + } + } + } + + pipeline_barrier_from_iters( + &cmd_buf.device, + cmd_buf.handle, + None, + buffer_barriers.into_iter(), + image_barriers.into_iter(), + ); + + if let Some(pending) = pending_buffer_transfer_nodes.as_mut() { + pending.remove_where(|node_idx, _buffer, transfers| { + for BufferResourceBarrier { range, .. } in tls.pending_buffers.get(node_idx) { + if consume_pending_buffer_transfers(transfers, *range) { + return true; + } + } + + false + }); + + if pending.is_empty() { + *pending_buffer_transfer_nodes = None; + } + } + + if let Some(pending) = pending_image_transfer_nodes.as_mut() { + pending.remove_where(|node_idx, _image, transfers| { + for ImageResourceBarrier { range, .. } in tls.pending_images.get(node_idx) { + if consume_pending_image_transfers(transfers, *range) { + return true; + } + } + + false + }); + + if pending.is_empty() { + *pending_image_transfer_nodes = None; + } + } + }); + } + + #[profiling::function] + fn record_node_cmds

( + &mut self, + pool: &mut P, + cmd_buf: &CommandBuffer, + node_idx: usize, + end_cmd_idx: usize, + ) -> Result<(), DriverError> + where + P: SubmissionPool, + { + thread_local! { + static SCHEDULE: RefCell = Default::default(); + } + + SCHEDULE.with_borrow_mut(|schedule| { + schedule.access_index.update(&self.graph, end_cmd_idx); + schedule.cmds.clear(); + + self.schedule_node_cmds(node_idx, end_cmd_idx, schedule); + self.record_scheduled_cmds(pool, cmd_buf, schedule, end_cmd_idx) + }) + } + + fn track_pending_transfers(&mut self, schedule: &Schedule, queue_family_index: u32) { + #[derive(Debug)] + struct BufferRangeSet { + buffer: vk::Buffer, + range_keys: HashSet, + } + + #[derive(Default)] + struct PendingTransferScratch { + buffers: HashMap, + images: HashMap, + } + + thread_local! { + static PENDING_TRANSFER: RefCell = Default::default(); + } + + PENDING_TRANSFER.with_borrow_mut(|tls| { + tls.buffers.clear(); + tls.images.clear(); + + for cmd_idx in schedule.cmds.iter().copied() { + let cmd = &self.graph.cmds[cmd_idx]; + + for (node_idx, accesses) in cmd.execs.iter().flat_map(|exec| exec.accesses.iter()) { + if let Some(buffer) = self.graph.resources[node_idx].as_buffer() { + if buffer.info.sharing_mode == vk::SharingMode::CONCURRENT { + continue; + } + + let transfer = + tls.buffers + .entry(node_idx) + .or_insert_with(|| BufferRangeSet { + buffer: buffer.handle, + range_keys: Default::default(), + }); + + for access in accesses.iter() { + let SubresourceRange::Buffer(access_range) = access.subresource else { + continue; + }; + + let access_range = BufferSubresourceRange { + start: access_range.start, + end: if access_range.end == vk::WHOLE_SIZE { + buffer.info.size + } else { + access_range.end + }, + }; + let access_key = BufferSubresourceRangeKey::from_range(access_range); + + if !transfer.range_keys.insert(access_key) { + continue; + } + + for (subresource, sharing) in + buffer.sync_info_with_sharing_range(access_range) + { + let Some(range) = subresource.range.intersection(access_range) + else { + continue; + }; + + let Some((src_queue_family_index, src_queue_index)) = + exclusive_transfer_source(sharing, queue_family_index) + else { + continue; + }; + + self.pending_buffer_transfer_nodes + .get_or_insert_with(|| { + PendingTransferNodes::new(self.graph.resources.len()) + }) + .push_transfer( + node_idx, + transfer.buffer, + BufferQueueOwnershipTransfer { + src_queue_family_index, + src_queue_index, + dst_queue_family_index: queue_family_index, + range, + }, + ); + } + } + + continue; + } + + let Some(image) = self.graph.resources[node_idx].as_image() else { + continue; + }; + + if image.info.sharing_mode == vk::SharingMode::CONCURRENT { + continue; + } + + let transfer = tls.images.entry(node_idx).or_insert_with(|| ImageRangeSet { + image: image.handle, + range_keys: Default::default(), + }); + + for access in accesses.iter() { + let SubresourceRange::Image(access_range) = access.subresource else { + continue; + }; + + let access_range = image.info.resolve_subresource_counts(access_range); + let access_key = ImageSubresourceRangeKey::from_range(access_range); + + if !transfer.range_keys.insert(access_key) { + continue; + } + + for (subresource, sharing) in + image.sync_info_with_sharing_range(access_range) + { + let Some(range) = image_subresource_range_intersection( + subresource.range, + access_range, + ) else { + continue; + }; + + let layout = subresource.layout.unwrap_or(vk::ImageLayout::UNDEFINED); + + let Some((src_queue_family_index, src_queue_index)) = + exclusive_transfer_source(sharing, queue_family_index) + else { + continue; + }; + + self.pending_image_transfer_nodes + .get_or_insert_with(|| { + PendingTransferNodes::new(self.graph.resources.len()) + }) + .push_transfer( + node_idx, + transfer.image, + ImageQueueOwnershipTransfer { + src_queue_family_index, + src_queue_index, + dst_queue_family_index: queue_family_index, + layout, + range, + }, + ); + } + } + } + } + + for (node_idx, transfer) in tls.buffers.iter() { + self.exclusive_buffer_ranges + .entry(*node_idx) + .or_default() + .extend( + transfer + .range_keys + .iter() + .copied() + .map(BufferSubresourceRangeKey::into_range), + ); + } + + for (node_idx, transfer) in tls.images.iter() { + self.exclusive_image_ranges + .entry(*node_idx) + .or_default() + .extend( + transfer + .range_keys + .iter() + .copied() + .map(ImageSubresourceRangeKey::into_range), + ); + } + }); + } + + fn record_cmd_indices( + &mut self, + cmd_buf: &CommandBuffer, + cmd_indices: impl IntoIterator, + ) -> Result<(), DriverError> { + #[cfg(feature = "checked")] + let graph_id = self.graph.graph_id(); + for cmd_idx in cmd_indices { + let cmd = &mut self.graph.cmds[cmd_idx]; + + profiling::scope!("Cmd", cmd.name()); + let stream_label = cmd + .stream_scope_id + .and_then(|_| CommandBufferDebugLabel::begin(cmd_buf, "command stream boundary")); + let _cmd_label = CommandBufferDebugLabel::begin(cmd_buf, cmd.name()); + + let recorded_command = &mut self.recorded_commands[cmd_idx]; + let is_graphics = recorded_command.render_pass.is_some(); + + trace!("recording cmd [{}: {}]", cmd_idx, cmd.name()); + + if !recorded_command.descriptor_sets.is_empty() { + Self::write_descriptor_sets(cmd_buf, &self.graph.resources, cmd, recorded_command)?; + } + + let (render_area, render_pass_label) = if is_graphics { + Self::record_image_layout_transitions( + cmd_buf, + &mut self.graph.resources, + cmd, + &mut self.pending_buffer_transfer_nodes, + &mut self.pending_image_transfer_nodes, + ); + + let render_area = vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent: Self::render_extent(&self.graph.resources, cmd), + }; + let render_pass_label = CommandBufferDebugLabel::begin( + cmd_buf, + format!("{} / render pass", cmd.name()), + ); + + Self::begin_render_pass( + cmd_buf, + &self.graph.resources, + cmd, + recorded_command, + render_area, + )?; + + (Some(render_area), render_pass_label) + } else { + (None, None) + }; + + for exec_idx in 0..cmd.execs.len() { + let render_area = if is_graphics { + Some( + cmd.execs[exec_idx] + .render_area + .unwrap_or(render_area.expect("missing render area")), + ) + } else { + None + }; + let exec_label_name = cmd_buf + .device + .physical + .instance + .info + .debug + .then(|| format!("{} / exec {exec_idx}", cmd.name())); + + let exec = &mut cmd.execs[exec_idx]; + + if is_graphics && exec_idx > 0 { + Self::next_subpass(cmd_buf); + } + + if let Some(pipeline) = exec.pipeline.as_mut() { + Self::bind_pipeline( + cmd_buf, + recorded_command, + exec_idx, + pipeline, + exec.depth_stencil, + )?; + + if is_graphics { + let render_area = render_area.expect("missing render area"); + + // In this case we set the viewport and scissor for the user + Self::set_viewport( + cmd_buf, + render_area.offset.x as _, + render_area.offset.y as _, + render_area.extent.width as _, + render_area.extent.height as _, + exec.depth_stencil + .map(|depth_stencil| { + let min = depth_stencil.min.0; + let max = depth_stencil.max.0; + min..max + }) + .unwrap_or(0.0..1.0), + ); + Self::set_scissor( + cmd_buf, + render_area.offset.x, + render_area.offset.y, + render_area.extent.width, + render_area.extent.height, + ); + } + + Self::bind_descriptor_sets(cmd_buf, pipeline, recorded_command, exec_idx); + } + + if !is_graphics { + Self::record_execution_barriers( + cmd_buf, + &mut self.graph.resources, + &exec.accesses, + &mut self.pending_buffer_transfer_nodes, + &mut self.pending_image_transfer_nodes, + ); + } + + trace!(" > exec[{exec_idx}]"); + + { + profiling::scope!("Execute callback"); + let _exec_label = exec_label_name.as_deref().and_then(|exec_label_name| { + CommandBufferDebugLabel::begin(cmd_buf, exec_label_name) + }); + + let exec_func = exec.func.take().expect("missing command function"); + exec.func = exec_func.record(CommandRef::new( + cmd_buf, + &self.graph.resources, + exec, + #[cfg(feature = "checked")] + graph_id, + )); + } + } + + if is_graphics { + trace!(" end render pass"); + + cmd_buf.end_render_pass(); + } + + drop(render_pass_label); + drop(stream_label); + } + Ok(()) + } + + #[profiling::function] + fn record_scheduled_cmds

( + &mut self, + pool: &mut P, + cmd_buf: &CommandBuffer, + schedule: &mut Schedule, + end_cmd_idx: usize, + ) -> Result<(), DriverError> + where + P: SubmissionPool, + { + if schedule.cmds.is_empty() { + return Ok(()); + } + + // // Print some handy details or hit a breakpoint if you set the flag + // if log_enabled!(Debug) && self.graph.debug { + // debug!("resolving the following graph:\n\n{:#?}\n\n", self.graph); + // } + + debug_assert!( + schedule.cmds.windows(2).all(|w| w[0] <= w[1]), + "Unsorted schedule" + ); + + // Optimize the schedule; requesting the required resources it needs + schedule.reorder_cmds(end_cmd_idx); + self.merge_scheduled_cmds(&mut schedule.cmds); + self.lease_scheduled_resources(pool, &schedule.cmds)?; + self.track_pending_transfers(schedule, cmd_buf.info.queue_family_index); + self.queue_ownership_release_groups + .extend(self.collect_queue_ownership_release_groups()); + + self.record_cmd_indices(cmd_buf, schedule.cmds.iter().copied())?; + + thread_local! { + static PASSES: RefCell> = Default::default(); + } + + PASSES.with_borrow_mut(|passes| { + debug_assert!(passes.is_empty()); + + // We have to keep the bindings and pipelines alive until the gpu is done + schedule.cmds.sort_unstable(); + while let Some(schedule_idx) = schedule.cmds.pop() { + debug_assert!(!self.graph.cmds.is_empty()); + + while let Some(cmd) = self.graph.cmds.pop() { + let cmd_idx = self.graph.cmds.len(); + + if cmd_idx == schedule_idx { + // This was a scheduled cmd - store it! + + self.submit_retained.push(Box::new(( + cmd, + self.recorded_commands + .pop() + .expect("missing recorded command"), + ))); + break; + } else { + debug_assert!(cmd_idx > schedule_idx); + + passes.push(cmd); + } + } + } + + debug_assert!(self.recorded_commands.is_empty()); + + // Put the other passes back for future resolves + self.graph.cmds.extend(passes.drain(..).rev()); + }); + + log::trace!("Recorded passes"); + + Ok(()) + } + + #[profiling::function] + fn render_extent(bindings: &[AnyResource], pass: &CommandData) -> vk::Extent2D { + // set_render_area was not specified so we're going to guess using the minimum common + // attachment extents + let first_exec = pass.expect_first_exec(); + + // We must be able to find the render area because render passes require at least one + // image to be attached + let (mut width, mut height) = (u32::MAX, u32::MAX); + for (attachment_width, attachment_height) in first_exec + .attachments + .color_attachments() + .map(|(_, state)| state.attachment) + .chain( + first_exec + .attachments + .depth_stencil_attachment() + .into_iter() + .filter(|state| state.is_attachment) + .map(|state| state.attachment), + ) + .map(|attachment| { + let info = Self::expect_attachment_image(bindings, &attachment).info; + + ( + info.width >> attachment.base_mip_level, + info.height >> attachment.base_mip_level, + ) + }) + { + width = width.min(attachment_width); + height = height.min(attachment_height); + } + + vk::Extent2D { height, width } + } + + /// Returns a borrow of the original Vulkan resource (buffer, image or acceleration structure) + /// which the given node represents. + pub fn resource(&self, resource_node: N) -> &N::Resource + where + N: Node, + { + self.graph.resource(resource_node) + } + + /// Mutates a schedule of command indices that are required to be executed, in order, for the + /// given node. + #[profiling::function] + fn schedule_node_cmds(&self, node_idx: usize, end_cmd_idx: usize, schedule: &mut Schedule) { + #[derive(Default)] + struct ScheduleSearchScratch { + pending_nodes: VecDeque<(usize, usize)>, + resolved_nodes: FixedBitSet, + scheduled_cmds: FixedBitSet, + } + + thread_local! { + static SCHEDULE_SEARCH: RefCell = Default::default(); + } + + SCHEDULE_SEARCH.with_borrow_mut(|tls| { + tls.scheduled_cmds.clear(); + tls.scheduled_cmds.grow(end_cmd_idx); + + tls.resolved_nodes.clear(); + tls.resolved_nodes.grow(self.graph.resources.len()); + + debug_assert!(tls.pending_nodes.is_empty()); + + trace!("scheduling node {node_idx}"); + + tls.resolved_nodes.insert(node_idx); + + // Schedule the first set of cmds for the node we're trying to resolve + for cmd_idx in schedule + .access_index + .prior_cmds_for_node(node_idx, end_cmd_idx) + { + trace!( + " cmd [{cmd_idx}: {}] is dependent", + self.graph.cmds[cmd_idx].name() + ); + + debug_assert!(!tls.scheduled_cmds.contains(cmd_idx)); + + tls.scheduled_cmds.insert(cmd_idx); + schedule.cmds.push(cmd_idx); + + for node_idx in schedule.access_index.read_nodes_for_cmd(cmd_idx) { + trace!(" node {node_idx} is dependent"); + + if !tls.resolved_nodes.put(node_idx) { + tls.pending_nodes.push_back((node_idx, cmd_idx)); + } + } + } + + trace!("secondary cmds below"); + + // Now schedule all nodes that are required, going through the tree to find them + while let Some((node_idx, cmd_idx)) = tls.pending_nodes.pop_front() { + trace!(" node {node_idx} is dependent"); + + for dep_cmd_idx in schedule + .access_index + .prior_cmds_for_node(node_idx, cmd_idx + 1) + { + if !tls.scheduled_cmds.put(dep_cmd_idx) { + schedule.cmds.push(dep_cmd_idx); + + trace!( + " cmd [{dep_cmd_idx}: {}] is dependent", + self.graph.cmds[dep_cmd_idx].name() + ); + + for node_idx in schedule.access_index.read_nodes_for_cmd(dep_cmd_idx) { + trace!(" node {node_idx} is dependent"); + + if !tls.resolved_nodes.put(node_idx) { + tls.pending_nodes.push_back((node_idx, dep_cmd_idx)); + } + } + } + } + } + + schedule.cmds.sort_unstable(); + + if log_enabled!(Debug) { + if !schedule.cmds.is_empty() { + // These are the indexes of the cmds this thread is about to resolve + debug!( + "schedule: {}", + schedule + .cmds + .iter() + .copied() + .map(|idx| format!("[{}: {}]", idx, self.graph.cmds[idx].name())) + .collect::>() + .join(", ") + ); + } + + if log_enabled!(Trace) { + let unscheduled = (0..end_cmd_idx) + .filter(|&cmd_idx| !tls.scheduled_cmds.contains(cmd_idx)) + .collect::>(); + + if !unscheduled.is_empty() { + // These cmds are within the range of cmds we thought we had to do + // right now, but it turns out that nothing in "schedule" relies on them + trace!( + "delaying: {}", + unscheduled + .iter() + .copied() + .map(|idx| format!("[{}: {}]", idx, self.graph.cmds[idx].name())) + .collect::>() + .join(", ") + ); + } + + if end_cmd_idx < self.graph.cmds.len() { + // These cmds existing on the graph but are not being considered right + // now because we've been told to stop work at the "end_cmd_idx" point + trace!( + "ignoring: {}", + self.graph.cmds[end_cmd_idx..] + .iter() + .enumerate() + .map(|(idx, cmd)| format!( + "[{}: {}]", + idx + end_cmd_idx, + cmd.name() + )) + .collect::>() + .join(", ") + ); + } + } + } + }); + } + + fn set_scissor(cmd_buf: &CommandBuffer, x: i32, y: i32, width: u32, height: u32) { + unsafe { + cmd_buf.device.cmd_set_scissor( + cmd_buf.handle, + 0, + slice::from_ref(&vk::Rect2D { + extent: vk::Extent2D { width, height }, + offset: vk::Offset2D { x, y }, + }), + ); + } + } + + fn set_viewport( + cmd_buf: &CommandBuffer, + x: f32, + y: f32, + width: f32, + height: f32, + depth: Range, + ) { + unsafe { + cmd_buf.device.cmd_set_viewport( + cmd_buf.handle, + 0, + slice::from_ref(&vk::Viewport { + x, + y, + width, + height, + min_depth: depth.start, + max_depth: depth.end, + }), + ); + } + } + + /// Records and submits all remaining commands using an internally allocated command buffer. + /// + /// This legacy submit path only supports binary semaphore behavior. All wait and signal + /// values must be `0`, and wait and signal stage masks must be `ALL_COMMANDS` or `NONE`. + pub fn queue_submit

( + self, + resource_pool: &mut P, + queue_family_index: u32, + queue_index: u32, + ) -> Result + where + P: Pool + SubmissionPool, + { + trace!("queue_submit"); + + /* + Phase 1: Get the main command buffer and record commands. This also discovers any ownership + transfers required by the scheduled work. + */ + let cmd_buf = resource_pool.resource(CommandBufferInfo::new(queue_family_index as _))?; + let mut fence = Fence::create(&cmd_buf.device, false)?; + cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + let recording = self.record(resource_pool, cmd_buf, RecordSelection::All)?; + recording.cmd_buf.end()?; + + let mut recorded = recording.finish()?; + recorded.queue_submit(&mut fence, queue_index, QueueSubmitInfo::QUEUE_SUBMIT)?; + + fence.drop_when_signaled(recorded); + + Ok(fence) + } + + /// Records any remaining graph commands into `cmd_buf` and returns a [`Recording`]. + /// + /// When `selection` is [`RecordSelection::Nodes`], nodes are processed sequentially in the + /// provided slice order and each step mutates the remaining submission state. + #[profiling::function] + pub fn record<'p, 's, P, Cb>( + mut self, + resource_pool: &'p mut P, + cmd_buf: Cb, + selection: impl Into>, + ) -> Result, DriverError> + where + P: SubmissionPool, + Cb: AsRef, + { + self.record_selection_impl(resource_pool, cmd_buf.as_ref(), selection.into())?; + + Ok(Recording { + cmd_buf, + resource_pool, + submission: self, + }) + } + + #[profiling::function] + fn record_impl

(&mut self, pool: &mut P, cmd_buf: &CommandBuffer) -> Result<(), DriverError> + where + P: SubmissionPool, + { + if self.graph.cmds.is_empty() { + return Ok(()); + } + + thread_local! { + static SCHEDULE: RefCell = Default::default(); + } + + SCHEDULE.with_borrow_mut(|schedule| { + schedule + .access_index + .update(&self.graph, self.graph.cmds.len()); + schedule.cmds.clear(); + schedule.cmds.extend(0..self.graph.cmds.len()); + + self.record_scheduled_cmds(pool, cmd_buf, schedule, self.graph.cmds.len()) + }) + } + + #[profiling::function] + fn record_resource_dependencies_impl

( + &mut self, + pool: &mut P, + cmd_buf: &CommandBuffer, + resource_node: impl Node, + ) -> Result<(), DriverError> + where + P: SubmissionPool, + { + self.graph.assert_node_owner(&resource_node); + + let node_idx = resource_node.index(); + + debug_assert!(self.graph.resources.get(node_idx).is_some()); + + // We record up to but not including the first command which accesses the target node. + if let Some(end_pass_idx) = self.graph.first_node_access_pass_index(resource_node) { + thread_local! { + static SCHEDULE: RefCell = Default::default(); + } + + SCHEDULE.with_borrow_mut(|tls| { + tls.access_index.update(&self.graph, end_pass_idx + 1); + tls.cmds.clear(); + schedule_dependency_cmds_before_target_access( + &tls.access_index, + node_idx, + end_pass_idx, + &mut tls.cmds, + ); + self.record_scheduled_cmds(pool, cmd_buf, tls, end_pass_idx) + })?; + } + + Ok(()) + } + + #[profiling::function] + fn record_resource_impl

( + &mut self, + pool: &mut P, + cmd_buf: &CommandBuffer, + resource_node: impl Node, + ) -> Result<(), DriverError> + where + P: SubmissionPool, + { + self.graph.assert_node_owner(&resource_node); + + let node_idx = resource_node.index(); + + debug_assert!(self.graph.resources.get(node_idx).is_some()); + + if self.graph.cmds.is_empty() { + return Ok(()); + } + + let end_pass_idx = self.graph.cmds.len(); + self.record_node_cmds(pool, cmd_buf, node_idx, end_pass_idx) + } + + #[profiling::function] + fn write_descriptor_sets( + cmd_buf: &CommandBuffer, + bindings: &[AnyResource], + pass: &CommandData, + recorded_command: &CommandRecordingResources, + ) -> Result<(), DriverError> { + #[derive(Clone, Copy)] + struct IndexedWrite<'a> { + info_idx: usize, + write: vk::WriteDescriptorSet<'a>, + } + + #[derive(Default)] + struct DescriptorScratch<'a> { + accel_struct_handles: Vec, + accel_struct_infos: Vec>, + accel_struct_writes: Vec>, + buffer_infos: Vec, + buffer_writes: Vec>, + descriptors: Vec>, + image_infos: Vec, + image_writes: Vec>, + } + + thread_local! { + static DESCRIPTOR: RefCell> = Default::default(); + } + + DESCRIPTOR.with_borrow_mut(|tls| { + tls.accel_struct_handles.clear(); + tls.accel_struct_infos.clear(); + tls.accel_struct_writes.clear(); + tls.buffer_infos.clear(); + tls.buffer_writes.clear(); + tls.descriptors.clear(); + tls.image_infos.clear(); + tls.image_writes.clear(); + + for (exec_idx, exec, pipeline) in pass + .execs + .iter() + .enumerate() + .filter_map(|(exec_idx, exec)| { + exec.pipeline + .as_ref() + .map(|pipeline| (exec_idx, exec, pipeline)) + }) + .filter(|(.., pipeline)| !pipeline.descriptor_info().layouts.is_empty()) + { + let descriptor_sets = &recorded_command.descriptor_sets[exec_idx]; + + // Write the manually bound things (access, read, and write functions) + for (descriptor, (node_idx, view_info)) in exec.bindings.iter() { + let (descriptor_set_idx, dst_binding, binding_offset) = descriptor.into_tuple(); + let Some((descriptor_info, _)) = pipeline.descriptor_bindings().get(&Descriptor { + set: descriptor_set_idx, + binding: dst_binding, + }) else { + warn!( + "binding {}.{}[{}] not found in shader reflection for command \"{}\"", + descriptor_set_idx, + dst_binding, + binding_offset, + pass.name(), + ); + return Err(DriverError::InvalidData); + }; + let descriptor_type = descriptor_info.descriptor_type(); + let bound_node = &bindings[*node_idx]; + if let Some(image) = bound_node.as_image() { + let mut image_view_info = *view_info.expect_image(); + + // Handle default views which did not specify a particular aspect + if image_view_info.aspect_mask.is_empty() { + image_view_info.aspect_mask = format_aspect_mask(image.info.format); + } + + let image_view = Image::view(image, image_view_info)?; + let image_layout = match descriptor_type { + vk::DescriptorType::COMBINED_IMAGE_SAMPLER + | vk::DescriptorType::SAMPLED_IMAGE => { + if image_view_info.aspect_mask.contains( + vk::ImageAspectFlags::DEPTH | vk::ImageAspectFlags::STENCIL, + ) { + vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL + } else if image_view_info + .aspect_mask + .contains(vk::ImageAspectFlags::DEPTH) + { + vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL + } else if image_view_info + .aspect_mask + .contains(vk::ImageAspectFlags::STENCIL) + { + vk::ImageLayout::STENCIL_READ_ONLY_OPTIMAL + } else { + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL + } + } + vk::DescriptorType::STORAGE_IMAGE => vk::ImageLayout::GENERAL, + _ => { + warn!( + "invalid image descriptor type at binding {}.{}[{}] in command \"{}\"", + descriptor_set_idx, + dst_binding, + binding_offset, + pass.name() + ); + + return Err(DriverError::InvalidData); + } + }; + + if binding_offset == 0 { + tls.image_writes.push(IndexedWrite { + info_idx: tls.image_infos.len(), + write: vk::WriteDescriptorSet { + dst_set: *descriptor_sets[descriptor_set_idx as usize], + dst_binding, + descriptor_type, + descriptor_count: 1, + ..Default::default() + }, + }); + } else { + tls.image_writes + .last_mut() + .expect("missing image descriptor write") + .write + .descriptor_count += 1; + } + + tls.image_infos.push( + vk::DescriptorImageInfo::default() + .image_layout(image_layout) + .image_view(image_view), + ); + } else if let Some(buffer) = bound_node.as_buffer() { + let buffer_view_info = view_info.expect_buffer(); + + if binding_offset == 0 { + tls.buffer_writes.push(IndexedWrite { + info_idx: tls.buffer_infos.len(), + write: vk::WriteDescriptorSet { + dst_set: *descriptor_sets[descriptor_set_idx as usize], + dst_binding, + descriptor_type, + descriptor_count: 1, + ..Default::default() + }, + }); + } else { + tls.buffer_writes + .last_mut() + .expect("missing buffer descriptor write") + .write + .descriptor_count += 1; + } + + tls.buffer_infos.push( + vk::DescriptorBufferInfo::default() + .buffer(buffer.handle) + .offset(buffer_view_info.start) + .range(buffer_view_info.end - buffer_view_info.start), + ); + } else if let Some(accel_struct) = bound_node.as_accel_struct() { + if binding_offset == 0 { + tls.accel_struct_writes.push(IndexedWrite { + info_idx: tls.accel_struct_handles.len(), + write: vk::WriteDescriptorSet::default() + .dst_set(*descriptor_sets[descriptor_set_idx as usize]) + .dst_binding(dst_binding) + .descriptor_type(descriptor_type) + .descriptor_count(1), + }); + } else { + tls + .accel_struct_writes + .last_mut() + .expect("missing acceleration structure descriptor write") + .write + .descriptor_count += 1; + } + + tls.accel_struct_handles.push(accel_struct.handle); + } else { + warn!( + "invalid bound resource kind at descriptor {}.{}[{}] in command \"{}\"", + descriptor_set_idx, + dst_binding, + binding_offset, + pass.name() + ); + + return Err(DriverError::InvalidData); + } + } + + if let ExecutionPipeline::Graphics(pipeline) = pipeline { + // Write graphics render pass input attachments (they're automatic) + if exec_idx > 0 { + for ( + &Descriptor { + set: descriptor_set_idx, + binding: dst_binding, + }, + (descriptor_info, _), + ) in &pipeline.inner.descriptor_bindings + { + if let DescriptorInfo::InputAttachment(_, attachment_idx) = *descriptor_info + { + let current_attachment = exec + .attachments + .color_attachment(attachment_idx) + .map(|state| state.attachment) + .expect("missing input attachment target"); + let attachment = pass.execs[0..exec_idx] + .iter() + .rev() + .find_map(|exec| { + exec.attachments + .color_attachment(attachment_idx) + .map(|state| state.attachment) + .filter(|attachment| { + Attachment::are_compatible( + Some(current_attachment), + Some(*attachment), + ) + }) + }) + .expect("input attachment not written"); + let image_binding = &bindings[attachment.target]; + let image = image_binding.expect_image(); + let image_view = + Image::view(image, attachment.image_view_info(image.info))?; + + tls.image_writes.push(IndexedWrite { + info_idx: tls.image_infos.len(), + write: vk::WriteDescriptorSet { + dst_set: *descriptor_sets[descriptor_set_idx as usize], + dst_binding, + descriptor_type: vk::DescriptorType::INPUT_ATTACHMENT, + descriptor_count: 1, + ..Default::default() + }, + }); + + tls.image_infos.push(vk::DescriptorImageInfo { + image_layout: Self::attachment_layout( + attachment.aspect_mask, + exec.attachments + .color_attachment(attachment_idx) + .map(|state| { + state.store == StoreOp::Store || state.resolve.is_some() + }) + .unwrap_or_default(), + true, + ), + image_view, + sampler: vk::Sampler::null(), + }); + } + } + } + } + } + + // NOTE: We assign the below pointers after the above insertions so they remain stable! + + let accel_struct_handles = tls.accel_struct_handles.as_ptr(); + for write_idx in 0..tls.accel_struct_writes.len() { + let IndexedWrite { + info_idx: handle_idx, + write, + } = tls.accel_struct_writes[write_idx]; + + unsafe { + tls.accel_struct_infos.push( + vk::WriteDescriptorSetAccelerationStructureKHR { + acceleration_structure_count: write.descriptor_count, + p_acceleration_structures: accel_struct_handles.add(handle_idx), + ..Default::default() + }, + ); + } + } + + let infos = tls.accel_struct_infos.as_ptr(); + for (write_idx, IndexedWrite { mut write, .. }) in + tls.accel_struct_writes.drain(..).enumerate() + { + unsafe { + write.p_next = infos.add(write_idx) as *const _; + } + + tls.descriptors.push(write); + } + + let buffer_infos_ptr = tls.buffer_infos.as_ptr(); + for write_idx in 0..tls.buffer_writes.len() { + let IndexedWrite { + info_idx, + mut write, + } = tls.buffer_writes[write_idx]; + unsafe { + write.p_buffer_info = buffer_infos_ptr.add(info_idx); + } + tls.descriptors.push(write); + } + + let image_infos_ptr = tls.image_infos.as_ptr(); + for write_idx in 0..tls.image_writes.len() { + let IndexedWrite { + info_idx, + mut write, + } = tls.image_writes[write_idx]; + unsafe { + write.p_image_info = image_infos_ptr.add(info_idx); + } + tls.descriptors.push(write); + } + + if !tls.descriptors.is_empty() { + trace!( + " writing {} descriptors ({} buffers, {} images)", + tls.descriptors.len(), + tls.buffer_infos.len(), + tls.image_infos.len() + ); + + unsafe { + cmd_buf + .device + .update_descriptor_sets(tls.descriptors.as_slice(), &[]); + } + } + + Ok(()) + }) + } +} + +#[derive(Default)] +struct SubmitScratch { + release_buffer_barriers: Vec>, + release_image_barriers: Vec>, + signal_infos: Vec>, + signal_semaphores: Vec, + wait_infos: Vec>, + wait_semaphores: Vec, + wait_stage_masks: Vec, +} + +#[doc(hidden)] +pub mod bench { + use super::{CommandAccessIndex, Schedule}; + + /// Synthetic workload description for scheduler benchmarks. + #[derive(Clone, Copy, Debug)] + pub struct ReorderBenchSpec { + /// Number of scheduled cmds. + pub cmd_count: usize, + + /// Number of resources participating in the schedule. + pub resource_count: usize, + + /// Typical cmd count for short-lived resources. + pub short_lived_uses: usize, + + /// Number of long-lived resources shared across many cmds. + pub long_lived_resource_count: usize, + + /// Typical cmd count for each long-lived resource. + pub long_lived_uses: usize, + } + + /// Reusable benchmark harness for `Schedule::reorder_cmds`. + pub struct ReorderBenchHarness { + schedule: Schedule, + original_cmds: Vec, + end_cmd_idx: usize, + } + + impl ReorderBenchHarness { + /// Builds a deterministic synthetic schedule for benchmarking. + pub fn new(spec: ReorderBenchSpec) -> Self { + assert!(spec.cmd_count > 0, "cmd_count must be greater than zero"); + assert!( + spec.resource_count > 0, + "resource_count must be greater than zero" + ); + assert!( + spec.short_lived_uses > 0, + "short_lived_uses must be greater than zero" + ); + + let mut cmds_by_node = vec![Vec::new(); spec.resource_count]; + let mut accessed_nodes_by_cmd = vec![Vec::new(); spec.cmd_count]; + + for (node_idx, cmds) in cmds_by_node.iter_mut().enumerate() { + let is_long_lived = node_idx < spec.long_lived_resource_count; + let uses = if is_long_lived { + spec.long_lived_uses.max(spec.short_lived_uses) + } else { + spec.short_lived_uses + } + .min(spec.cmd_count); + + let seed = splitmix64(node_idx as u64 ^ ((spec.cmd_count as u64) << 32)); + let stride = odd_stride(seed, spec.cmd_count); + let start = (seed as usize) % spec.cmd_count; + let cluster_len = uses.max(1).min(spec.cmd_count); + + cmds.reserve(uses); + + for use_idx in 0..uses { + let cmd_idx = if is_long_lived { + (start + use_idx * stride) % spec.cmd_count + } else { + (start + use_idx % cluster_len + (use_idx / cluster_len) * stride) + % spec.cmd_count + }; + + cmds.push(cmd_idx); + } + + cmds.sort_unstable(); + cmds.dedup(); + + while cmds.len() < uses { + let next_cmd = (start + cmds.len() * stride + cmds.len()) % spec.cmd_count; + if cmds.binary_search(&next_cmd).is_err() { + cmds.push(next_cmd); + } + } + + cmds.sort_unstable(); + + for &cmd_idx in cmds.iter() { + accessed_nodes_by_cmd[cmd_idx].push(node_idx); + } + } + + for nodes in &mut accessed_nodes_by_cmd { + nodes.sort_unstable(); + nodes.dedup(); + } + + let cmds = (0..spec.cmd_count).collect::>(); + + Self { + schedule: Schedule { + access_index: CommandAccessIndex { + cmds_by_node, + accessed_nodes_by_cmd, + }, + cmds: cmds.clone(), + ..Default::default() + }, + original_cmds: cmds, + end_cmd_idx: spec.cmd_count, + } + } + + /// Restores the original schedule, reorders it once, and returns a checksum. + pub fn reorder_once(&mut self) -> u64 { + self.schedule.cmds.clear(); + self.schedule + .cmds + .extend(self.original_cmds.iter().copied()); + + self.schedule.reorder_cmds(self.end_cmd_idx); + + self.schedule + .cmds + .iter() + .enumerate() + .fold(0u64, |checksum, (idx, &pass_idx)| { + checksum.wrapping_mul(1_099_511_628_211).wrapping_add( + ((idx as u64) << 32) ^ pass_idx as u64 ^ 0x9e37_79b9_7f4a_7c15, + ) + }) + } + } + + fn odd_stride(seed: u64, cmd_count: usize) -> usize { + let stride = ((seed >> 32) as usize % cmd_count.max(2)) | 1; + + stride.min(cmd_count.max(1) - 1).max(1) + } + + fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) + } +} + +#[doc(hidden)] +pub mod fuzz { + use { + super::{CommandAccessIndex, Schedule}, + fixedbitset::FixedBitSet, + }; + + #[derive(Clone, Copy, Debug)] + pub struct ResourceAccess { + pub cmd_idx: usize, + pub write: bool, + } + + pub fn check_schedule_reordering(cmd_count: usize, resource_accesses: &[Vec]) { + let cmd_count = cmd_count.min(256); + if cmd_count == 0 { + return; + } + + let (access_index, normalized_accesses) = build_access_index(cmd_count, resource_accesses); + + let mut schedule = Schedule { + access_index: access_index.clone(), + cmds: (0..cmd_count).collect(), + ..Default::default() + }; + + schedule.reorder_cmds(cmd_count); + + let reordered = schedule.cmds.clone(); + + assert_eq!(reordered.len(), cmd_count, "reordered cmd count changed"); + + let mut sorted = reordered.clone(); + sorted.sort_unstable(); + assert_eq!( + sorted, + (0..cmd_count).collect::>(), + "reordered cmds are not a permutation" + ); + + let mut repeat = Schedule { + access_index: access_index.clone(), + cmds: (0..cmd_count).collect(), + ..Default::default() + }; + repeat.reorder_cmds(cmd_count); + assert_eq!(reordered, repeat.cmds, "reordering is not deterministic"); + + assert_hazard_order_preserved(&reordered, &normalized_accesses); + + let expected = reference_reorder(access_index, cmd_count); + assert_eq!( + reordered, expected, + "reordering diverged from reference implementation" + ); + } + + fn build_access_index( + cmd_count: usize, + resource_accesses: &[Vec], + ) -> (CommandAccessIndex, Vec>) { + let mut cmds_by_node = Vec::with_capacity(resource_accesses.len()); + let mut accessed_nodes_by_cmd = vec![Vec::new(); cmd_count]; + let mut normalized_accesses = Vec::with_capacity(resource_accesses.len()); + + for (node_idx, accesses) in resource_accesses.iter().enumerate() { + let mut normalized = accesses + .iter() + .copied() + .filter(|access| access.cmd_idx < cmd_count) + .collect::>(); + normalized.sort_unstable_by_key(|access| access.cmd_idx); + + let mut deduped = Vec::::with_capacity(normalized.len()); + for access in normalized { + if let Some(prev) = deduped.last_mut() + && prev.cmd_idx == access.cmd_idx + { + prev.write |= access.write; + continue; + } + + deduped.push(access); + } + + for access in &deduped { + accessed_nodes_by_cmd[access.cmd_idx].push(node_idx); + } + + cmds_by_node.push(deduped.iter().map(|access| access.cmd_idx).collect()); + normalized_accesses.push(deduped); + } + + ( + CommandAccessIndex { + cmds_by_node, + accessed_nodes_by_cmd, + }, + normalized_accesses, + ) + } + + fn assert_hazard_order_preserved( + reordered: &[usize], + resource_accesses: &[Vec], + ) { + let mut positions = vec![usize::MAX; reordered.len()]; + for (position, &cmd_idx) in reordered.iter().enumerate() { + positions[cmd_idx] = position; + } + + for accesses in resource_accesses { + for (left_idx, left) in accesses.iter().enumerate() { + for right in &accesses[(left_idx + 1)..] { + if left.write || right.write { + assert!( + positions[left.cmd_idx] < positions[right.cmd_idx], + "hazard order changed for resource accesses {:?} -> {:?}: {:?}", + left, + right, + reordered + ); + } + } + } + } + } + + fn reference_reorder(access_index: CommandAccessIndex, cmd_count: usize) -> Vec { + let mut cmds = (0..cmd_count).collect::>(); + if cmds.len() < 3 { + return cmds; + } + + let mut interdependent = vec![Vec::new(); cmd_count]; + let mut local_of_global = vec![usize::MAX; cmd_count]; + let mut seen_deps = FixedBitSet::with_capacity(cmd_count); + let mut scheduled = FixedBitSet::with_capacity(cmd_count); + + for (local_idx, &cmd_idx) in cmds.iter().enumerate() { + local_of_global[cmd_idx] = local_idx; + } + + for (local_idx, &cmd_idx) in cmds.iter().enumerate() { + for dep_cmd_idx in access_index.prior_read_dependency_cmds(cmd_idx, cmd_count) { + let dep_local_idx = local_of_global[dep_cmd_idx]; + if dep_local_idx == usize::MAX || dep_local_idx == local_idx { + continue; + } + + if !seen_deps.put(dep_local_idx) { + interdependent[local_idx].push(dep_local_idx); + } + } + + for dep_cmd_idx in access_index.prior_read_dependency_cmds(cmd_idx, cmd_count) { + let dep_local_idx = local_of_global[dep_cmd_idx]; + if dep_local_idx != usize::MAX && dep_local_idx != local_idx { + seen_deps.set(dep_local_idx, false); + } + } + } + + let mut scheduled_count = 0; + while scheduled_count < cmd_count { + let mut best_idx = scheduled_count; + let mut best_overlap = interdependent[best_idx].len(); + + for (idx, dep_cmds) in interdependent[..cmd_count] + .iter() + .enumerate() + .skip(scheduled_count + 1) + { + let mut overlap = 0; + + for &dep_local in dep_cmds { + if scheduled.contains(dep_local) { + overlap += 1; + } else { + break; + } + } + + if overlap > best_overlap { + best_overlap = overlap; + best_idx = idx; + } + } + + scheduled.insert(best_idx); + cmds.swap(scheduled_count, best_idx); + interdependent.swap(scheduled_count, best_idx); + scheduled_count += 1; + } + + cmds + } +} + +#[cfg(test)] +mod tests { + use super::{ + BufferQueueOwnershipTransfer, CommandAccessIndex, CommandData, + ExternalRenderPassAccessHistory, ImageQueueOwnershipTransfer, NodeIndex, + PipelineStageAccessFlags, QueueSubmitInfo, RecordSelection, RecordedSubmission, + RecordedSubmissionState, Schedule, SemaphoreSubmitInfo, Submission, SubresourceAccess, + SubresourceRange, check_queue_submit_args, fuzz, + }; + use crate::{ + Attachment, DepthStencilAttachment, Execution, Graph, LoadOp, Node, StoreOp, + driver::{ + DriverError, SharingMode, + accel_struct::{AccelerationStructure, AccelerationStructureInfo}, + ash::vk, + buffer::{Buffer, BufferInfo, BufferSubresourceRange}, + cmd_buf::{CommandBuffer, CommandBufferInfo}, + device::{Device, DeviceInfo}, + fence::Fence, + graphics::{GraphicsPipeline, GraphicsPipelineInfo}, + image::{Image, ImageInfo, SampleCount}, + render_pass::SubpassDependency, + }, + node::{AnyNode, BufferNode}, + pool::{Pool, hash::HashPool}, + }; + use { + ash::vk::Handle, + std::{ + env::set_var, + mem::ManuallyDrop, + ops::Deref, + sync::{Arc, Mutex, MutexGuard, OnceLock}, + }, + vk_shader_macros::glsl, + vk_sync::AccessType, + }; + + fn color_subresource_range( + array_layers: std::ops::Range, + mip_levels: std::ops::Range, + ) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: array_layers.start, + layer_count: array_layers.end - array_layers.start, + base_mip_level: mip_levels.start, + level_count: mip_levels.end - mip_levels.start, + } + } + + #[cfg(test)] + fn sort_image_subresource_ranges(ranges: &mut [vk::ImageSubresourceRange]) { + ranges.sort_unstable_by_key(|range| { + ( + range.aspect_mask.as_raw(), + range.base_array_layer, + range.layer_count, + range.base_mip_level, + range.level_count, + ) + }); + } + + #[cfg(test)] + fn sort_pending_image_transfers(transfers: &mut [ImageQueueOwnershipTransfer]) { + transfers.sort_unstable_by_key(|transfer| { + ( + transfer.src_queue_family_index, + transfer.src_queue_index, + transfer.dst_queue_family_index, + transfer.layout.as_raw(), + transfer.range.aspect_mask.as_raw(), + transfer.range.base_array_layer, + transfer.range.layer_count, + transfer.range.base_mip_level, + transfer.range.level_count, + ) + }); + } + + #[cfg(test)] + fn sort_pending_buffer_transfers(transfers: &mut [BufferQueueOwnershipTransfer]) { + transfers.sort_unstable_by_key(|transfer| { + ( + transfer.src_queue_family_index, + transfer.src_queue_index, + transfer.dst_queue_family_index, + transfer.range.start, + transfer.range.end, + ) + }); + } + + fn pending_buffer_transfer_for_range( + transfers: &[BufferQueueOwnershipTransfer], + range: BufferSubresourceRange, + ) -> Option<&BufferQueueOwnershipTransfer> { + transfers.iter().find(|transfer| transfer.range == range) + } + + fn pending_transfer_for_node( + pending: &super::PendingTransferNodes, + node_idx: NodeIndex, + ) -> Option<(H, &[T])> { + pending + .iter() + .find_map(|(idx, handle, transfers)| (idx == node_idx).then_some((handle, transfers))) + } + + #[test] + fn pending_transfer_nodes_set_tracks_each_node_once() { + let mut pending = super::PendingTransferNodes::new(4); + + assert!(pending.push_transfer(2, 10, 20)); + assert!(!pending.push_transfer(2, 11, 21)); + + assert!(pending.contains(2)); + let (handle, transfers) = pending_transfer_for_node(&pending, 2).unwrap(); + assert_eq!(handle, 11); + assert_eq!(pending.indices, vec![2]); + assert_eq!(transfers, &[20, 21]); + assert_eq!(pending.iter().count(), 1); + } + + #[test] + fn pending_transfer_nodes_remove_where_uses_swap_remove() { + let mut pending = super::PendingTransferNodes::new(4); + + pending.push_transfer(0, 10, 20); + pending.push_transfer(1, 11, 21); + pending.push_transfer(2, 12, 22); + + pending.remove_where(|node_idx, _, _| node_idx == 1); + + assert!(pending_transfer_for_node(&pending, 1).is_none()); + assert_eq!(pending.indices.len(), 2); + assert!(pending.indices.contains(&0)); + assert!(pending.indices.contains(&2)); + assert_eq!(pending.iter().collect::>().len(), 2); + } + + #[test] + fn pending_transfer_nodes_remove_where_drops_stale_indices() { + let mut pending = super::PendingTransferNodes::new(3); + + pending.push_transfer(1, 11, 21); + pending.entries[1] = None; + pending.remove_where(|_, _, _| false); + + assert!(pending.indices.is_empty()); + assert_eq!(pending.iter().count(), 0); + } + + #[test] + fn node_indexed_scratch_tracks_each_node_once() { + let mut scratch = super::NodeIndexedScratch::default(); + + scratch.push(2, 20); + scratch.push(2, 21); + scratch.push(0, 10); + + assert_eq!(scratch.indices, vec![2, 0]); + assert_eq!(scratch.get(2), &[20, 21]); + assert_eq!(scratch.get(0), &[10]); + assert_eq!(scratch.get(1), &[] as &[i32]); + } + + #[test] + fn node_indexed_scratch_clear_resets_occupancy_and_reuses_entries() { + let mut scratch = super::NodeIndexedScratch::default(); + + scratch.push(1, 10); + scratch.clear(); + + assert!(scratch.indices.is_empty()); + assert_eq!(scratch.get(1), &[] as &[i32]); + + scratch.push(1, 11); + scratch.push(1, 12); + + assert_eq!(scratch.indices, vec![1]); + assert_eq!(scratch.get(1), &[11, 12]); + } + + #[test] + fn node_indexed_scratch_resizes_for_high_indices() { + let mut scratch = super::NodeIndexedScratch::default(); + + scratch.push(5, 50); + + assert_eq!(scratch.indices, vec![5]); + assert_eq!(scratch.get(5), &[50]); + assert_eq!(scratch.get(4), &[] as &[i32]); + } + + #[test] + fn pending_transfer_nodes_remove_where_keeps_partially_consumed_node() { + let mut pending = super::PendingTransferNodes::new(2); + + pending.push_transfer(1, 11, 20); + pending.push_transfer(1, 11, 21); + + pending.remove_where(|_, _, transfers| { + transfers.retain(|&transfer| transfer != 20); + transfers.is_empty() + }); + + assert!(pending.contains(1)); + assert_eq!(pending_transfer_for_node(&pending, 1).unwrap().1, &[21]); + assert!(!pending.is_empty()); + + pending.remove_where(|_, _, transfers| { + transfers.retain(|&transfer| transfer != 21); + transfers.is_empty() + }); + + assert!(!pending.contains(1)); + assert!(pending_transfer_for_node(&pending, 1).is_none()); + assert!(pending.is_empty()); + } + + #[test] + fn consume_pending_buffer_transfers_removes_intersecting_ranges() { + let consumed = BufferSubresourceRange { start: 4, end: 8 }; + let kept = BufferSubresourceRange { start: 8, end: 12 }; + let mut pending = vec![ + BufferQueueOwnershipTransfer { + dst_queue_family_index: 0, + range: consumed, + src_queue_family_index: 1, + src_queue_index: 0, + }, + BufferQueueOwnershipTransfer { + dst_queue_family_index: 0, + range: kept, + src_queue_family_index: 1, + src_queue_index: 0, + }, + ]; + + assert!(!super::consume_pending_buffer_transfers( + &mut pending, + consumed + )); + + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].range, kept); + } + + #[test] + fn consume_pending_image_transfers_removes_intersecting_ranges() { + let consumed = color_subresource_range(0..1, 0..1); + let kept = color_subresource_range(1..2, 0..1); + let mut pending = vec![ + ImageQueueOwnershipTransfer { + dst_queue_family_index: 0, + layout: vk::ImageLayout::GENERAL, + range: consumed, + src_queue_family_index: 1, + src_queue_index: 0, + }, + ImageQueueOwnershipTransfer { + dst_queue_family_index: 0, + layout: vk::ImageLayout::GENERAL, + range: kept, + src_queue_family_index: 1, + src_queue_index: 0, + }, + ]; + + assert!(!super::consume_pending_image_transfers( + &mut pending, + consumed + )); + + assert_eq!(pending.len(), 1); + assert!(super::image_subresource_range_eq(pending[0].range, kept)); + } + + #[test] + fn dependency_selection_schedules_inputs_to_first_target_access() { + let access_index = CommandAccessIndex { + /* + Node 0 is produced by cmd 0 and then read by cmd 1. Node 1 is the target written by + cmd 1, so dependencies(node 1) should include cmd 0 but not cmd 1. + */ + cmds_by_node: vec![vec![0, 1], vec![1]], + accessed_nodes_by_cmd: vec![vec![0], vec![0, 1]], + }; + let mut schedule = Vec::new(); + + super::schedule_dependency_cmds_before_target_access(&access_index, 1, 1, &mut schedule); + + assert_eq!(schedule, vec![0]); + } + + #[cfg(test)] + fn sort_queue_ownership_release_groups(groups: &mut [super::QueueOwnershipReleaseGroup]) { + for group in groups.iter_mut() { + group + .buffers + .sort_unstable_by_key(|(buffer, range)| (buffer.as_raw(), range.start, range.end)); + + group.images.sort_unstable_by_key(|(image, layout, range)| { + ( + image.as_raw(), + layout.as_raw(), + range.aspect_mask.as_raw(), + range.base_array_layer, + range.layer_count, + range.base_mip_level, + range.level_count, + ) + }); + } + + groups.sort_unstable_by_key(|group| (group.src_queue_family_index, group.src_queue_index)); + } + + #[cfg(test)] + fn sort_image_subresource_sync_infos( + subresources: &mut [crate::driver::image::ImageSubresourceSyncInfo], + ) { + subresources.sort_unstable_by_key(|subresource| { + ( + subresource.range.aspect_mask.as_raw(), + subresource.range.base_array_layer, + subresource.range.layer_count, + subresource.range.base_mip_level, + subresource.range.level_count, + ) + }); + } + + #[derive(Debug)] + struct TestDevice { + _guard: MutexGuard<'static, ()>, + device: ManuallyDrop, + } + + impl Drop for TestDevice { + fn drop(&mut self) { + // Drop the Vulkan device while the global test-device lock is still held + // `_guard` is a normal field, so it is released after this `Drop` returns + unsafe { + ManuallyDrop::drop(&mut self.device); + } + } + } + + impl Deref for TestDevice { + type Target = Device; + + fn deref(&self) -> &Self::Target { + &self.device + } + } + + fn test_device_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + + LOCK.get_or_init(|| Mutex::new(())) + } + + fn assert_no_invalid_attachment_stage_access_pairs(dep: &SubpassDependency) { + let dst_invalid_color_stages = dep.dst_stage_mask + & (vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS + | vk::PipelineStageFlags::FRAGMENT_SHADER); + assert!( + !dep.dst_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ) + || dst_invalid_color_stages.is_empty(), + "COLOR_ATTACHMENT_READ must not be paired with unsupported destination stages: {dep:?}" + ); + + let src_invalid_color_stages = dep.src_stage_mask + & (vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS + | vk::PipelineStageFlags::FRAGMENT_SHADER); + assert!( + !dep.src_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ) + || src_invalid_color_stages.is_empty(), + "COLOR_ATTACHMENT_READ must not be paired with unsupported source stages: {dep:?}" + ); + + assert!( + !(dep + .src_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ) + && dep + .src_stage_mask + .contains(vk::PipelineStageFlags::FRAGMENT_SHADER)), + "DEPTH_STENCIL_ATTACHMENT_READ must not be paired with FRAGMENT_SHADER in source stages: {dep:?}" + ); + assert!( + !(dep + .dst_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ) + && dep + .dst_stage_mask + .contains(vk::PipelineStageFlags::FRAGMENT_SHADER)), + "DEPTH_STENCIL_ATTACHMENT_READ must not be paired with FRAGMENT_SHADER in destination stages: {dep:?}" + ); + } + + fn assert_attachment_read_stage_mappings(dep: &SubpassDependency) { + if dep + .src_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ) + { + assert!( + dep.src_stage_mask + .contains(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT), + "COLOR_ATTACHMENT_READ source access should use COLOR_ATTACHMENT_OUTPUT: {dep:?}" + ); + } + + if dep + .dst_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ) + { + assert!( + dep.dst_stage_mask + .contains(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT), + "COLOR_ATTACHMENT_READ destination access should use COLOR_ATTACHMENT_OUTPUT: {dep:?}" + ); + } + + let fragment_tests = vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags::LATE_FRAGMENT_TESTS; + + if dep + .src_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ) + { + assert!( + dep.src_stage_mask.intersects(fragment_tests), + "DEPTH_STENCIL_ATTACHMENT_READ source access should use fragment-test stages: {dep:?}" + ); + } + + if dep + .dst_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ) + { + assert!( + dep.dst_stage_mask.intersects(fragment_tests), + "DEPTH_STENCIL_ATTACHMENT_READ destination access should use fragment-test stages: {dep:?}" + ); + } + } + + fn exec_with_buffer_access(access: AccessType) -> Execution { + let mut exec = Execution::default(); + exec.accesses.push( + 0, + SubresourceAccess { + access, + subresource: SubresourceRange::Buffer((0..16).into()), + }, + ); + + exec + } + + fn subpass_dependencies_for_accesses( + previous: AccessType, + current: AccessType, + ) -> Vec { + let pass = CommandData { + execs: vec![ + exec_with_buffer_access(previous), + exec_with_buffer_access(current), + ], + + #[cfg(debug_assertions)] + name: None, + + stream_scope_id: None, + }; + + Submission::build_subpass_dependencies(&pass, &ExternalRenderPassAccessHistory::new(1)) + } + + fn depth_attachment_exec( + load: LoadOp, + store: StoreOp, + ) -> Execution { + let mut exec = Execution::default(); + exec.attachments.depth_stencil = Some(DepthStencilAttachment { + attachment: Attachment { + array_layer_count: 1, + aspect_mask: vk::ImageAspectFlags::DEPTH, + base_array_layer: 0, + base_mip_level: 0, + format: vk::Format::D32_SFLOAT, + mip_level_count: 1, + sample_count: SampleCount::Type1, + target: 0, + }, + load, + store, + resolve: None, + is_attachment: true, + }); + + exec + } + + fn depth_attachment_dependencies( + previous_load: LoadOp, + previous_store: StoreOp, + current_load: LoadOp, + current_store: StoreOp, + ) -> Vec { + let pass = CommandData { + execs: vec![ + depth_attachment_exec(previous_load, previous_store), + depth_attachment_exec(current_load, current_store), + ], + + #[cfg(debug_assertions)] + name: None, + + stream_scope_id: None, + }; + + Submission::build_subpass_dependencies(&pass, &ExternalRenderPassAccessHistory::new(1)) + } + + fn schedule_with_access_index( + cmds: &[usize], + cmds_by_node: &[&[usize]], + accessed_nodes_by_cmd: &[&[usize]], + ) -> Schedule { + Schedule { + access_index: CommandAccessIndex { + cmds_by_node: cmds_by_node.iter().map(|cmds| cmds.to_vec()).collect(), + accessed_nodes_by_cmd: accessed_nodes_by_cmd + .iter() + .map(|nodes| nodes.to_vec()) + .collect(), + }, + cmds: cmds.to_vec(), + ..Default::default() + } + } + + #[test] + fn image_execution_discard_only_when_previous_access_is_nothing() { + assert!(super::image_execution_discard_contents(AccessType::Nothing)); + assert!(!super::image_execution_discard_contents( + AccessType::TransferRead + )); + assert!(!super::image_execution_discard_contents( + AccessType::TransferWrite + )); + assert!(!super::image_execution_discard_contents( + AccessType::ColorAttachmentReadWrite + )); + } + + #[test] + fn image_layout_transition_discard_keeps_attachment_write_policy() { + assert!(super::image_layout_transition_discard_contents( + AccessType::Nothing, + AccessType::TransferWrite, + )); + assert!(super::image_layout_transition_discard_contents( + AccessType::TransferRead, + AccessType::TransferWrite, + )); + assert!(!super::image_layout_transition_discard_contents( + AccessType::TransferWrite, + AccessType::ColorAttachmentReadWrite, + )); + } + + fn command_with_accesses(accesses: &[(usize, AccessType)]) -> CommandData { + let mut exec = Execution::default(); + + for &(node_idx, access) in accesses { + exec.accesses.push( + node_idx, + SubresourceAccess { + access, + subresource: SubresourceRange::Buffer(BufferSubresourceRange { + start: 0, + end: 1, + }), + }, + ); + } + + CommandData { + execs: vec![exec], + + #[cfg(debug_assertions)] + name: None, + + stream_scope_id: None, + } + } + + #[test] + fn command_access_index_includes_read_and_write_accesses() { + let cmds = vec![ + command_with_accesses(&[(0, AccessType::TransferRead)]), + command_with_accesses(&[(1, AccessType::TransferWrite)]), + command_with_accesses(&[(1, AccessType::TransferRead)]), + command_with_accesses(&[(1, AccessType::TransferWrite)]), + ]; + let mut access_index = CommandAccessIndex::default(); + + access_index.update_from_cmds(&cmds, 2); + + assert_eq!(access_index.cmds_by_node[0], vec![0]); + assert_eq!(access_index.cmds_by_node[1], vec![1, 2, 3]); + assert_eq!(access_index.accessed_nodes_by_cmd[0], vec![0]); + assert_eq!(access_index.accessed_nodes_by_cmd[1], vec![1]); + assert_eq!(access_index.accessed_nodes_by_cmd[2], vec![1]); + assert_eq!(access_index.accessed_nodes_by_cmd[3], vec![1]); + } + + #[test] + fn command_access_index_dedupes_accesses_per_command_and_resets_between_commands() { + let cmds = vec![ + command_with_accesses(&[ + (0, AccessType::TransferRead), + (0, AccessType::TransferWrite), + (1, AccessType::TransferRead), + (1, AccessType::TransferWrite), + ]), + command_with_accesses(&[(0, AccessType::TransferRead), (1, AccessType::TransferRead)]), + ]; + let mut access_index = CommandAccessIndex::default(); + + access_index.update_from_cmds(&cmds, 2); + + assert_eq!(access_index.cmds_by_node[0], vec![0, 1]); + assert_eq!(access_index.cmds_by_node[1], vec![0, 1]); + assert_eq!(access_index.accessed_nodes_by_cmd[0], vec![0, 1]); + assert_eq!(access_index.accessed_nodes_by_cmd[1], vec![0, 1]); + } + + #[test] + fn dependency_selection_dedupes_repeated_read_dependencies() { + let access_index = CommandAccessIndex { + /* + Cmd 1 reads node 0 twice and writes node 1. Dependency selection for node 1 must + schedule cmd 0 once, not once per repeated read of node 0. + */ + cmds_by_node: vec![vec![0, 1], vec![1]], + accessed_nodes_by_cmd: vec![vec![0], vec![0, 0, 1]], + }; + let mut schedule = Vec::new(); + + super::schedule_dependency_cmds_before_target_access(&access_index, 1, 1, &mut schedule); + + assert_eq!(schedule, vec![0]); + } + + #[test] + fn reorder_scheduled_cmds_preserves_hazards_from_command_access_index_update() { + let cmds = vec![ + command_with_accesses(&[(0, AccessType::TransferRead)]), + command_with_accesses(&[(1, AccessType::TransferWrite)]), + command_with_accesses(&[(1, AccessType::TransferRead)]), + command_with_accesses(&[(1, AccessType::TransferWrite)]), + command_with_accesses(&[(0, AccessType::TransferRead)]), + ]; + let mut access_index = CommandAccessIndex::default(); + access_index.update_from_cmds(&cmds, 2); + let mut schedule = Schedule { + access_index, + cmds: (0..cmds.len()).collect(), + ..Default::default() + }; + + schedule.reorder_cmds(cmds.len()); + + let position = |cmd_idx| { + schedule + .cmds + .iter() + .position(|&scheduled_cmd_idx| scheduled_cmd_idx == cmd_idx) + .expect("command was not scheduled") + }; + assert!(position(1) < position(2), "write-read hazard crossed"); + assert!(position(2) < position(3), "read-write hazard crossed"); + } + + #[test] + fn reorder_scheduled_cmds_matches_original_seed_choice() { + let mut schedule = schedule_with_access_index( + &[0, 1, 2, 3], + &[&[0, 1], &[1, 2], &[1, 3]], + &[&[0], &[0, 1, 2], &[1], &[2]], + ); + + schedule.reorder_cmds(4); + + assert_eq!(schedule.cmds, vec![0, 1, 2, 3]); + } + + #[test] + fn queue_ownership_release_groups_group_by_source_queue() { + use super::{ImageQueueOwnershipTransfer, image_subresource_range_eq}; + + let mut submission = Submission::new(Graph::new()); + let image = vk::Image::null(); + + let range_a = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: 0, + layer_count: 1, + base_mip_level: 0, + level_count: 1, + }; + let range_b = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_array_layer: 1, + layer_count: 1, + base_mip_level: 0, + level_count: 1, + }; + + let pending_image_transfer_nodes = submission + .pending_image_transfer_nodes + .get_or_insert_with(|| super::PendingTransferNodes::new(1)); + pending_image_transfer_nodes.push_transfer( + 0, + image, + ImageQueueOwnershipTransfer { + src_queue_family_index: 1, + src_queue_index: 2, + dst_queue_family_index: 3, + layout: vk::ImageLayout::GENERAL, + range: range_a, + }, + ); + pending_image_transfer_nodes.push_transfer( + 0, + image, + ImageQueueOwnershipTransfer { + src_queue_family_index: 1, + src_queue_index: 2, + dst_queue_family_index: 3, + layout: vk::ImageLayout::GENERAL, + range: range_b, + }, + ); + pending_image_transfer_nodes.push_transfer( + 0, + image, + ImageQueueOwnershipTransfer { + src_queue_family_index: 4, + src_queue_index: 5, + dst_queue_family_index: 3, + layout: vk::ImageLayout::GENERAL, + range: range_a, + }, + ); + + let groups = submission.collect_queue_ownership_release_groups(); + let mut groups = groups.into_vec(); + sort_queue_ownership_release_groups(&mut groups); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].images.len(), 2); + assert_eq!(groups[1].images.len(), 1); + assert_eq!(groups[0].images[0].0, image); + assert!(image_subresource_range_eq(groups[0].images[0].2, range_a)); + } + + #[test] + fn barrier_transfer_ranges_only_marks_overlapping_ranges() { + use super::{image_barrier_transfer_ranges, image_subresource_range_eq}; + + let range_a = color_subresource_range(0..1, 0..1); + let range_b = color_subresource_range(1..2, 0..1); + let transfers = [ImageQueueOwnershipTransfer { + src_queue_family_index: 1, + src_queue_index: 2, + dst_queue_family_index: 3, + layout: vk::ImageLayout::GENERAL, + range: range_a, + }]; + + let ranges = image_barrier_transfer_ranges(&transfers, color_subresource_range(0..2, 0..1)) + .collect::>(); + + assert_eq!(ranges.len(), 2); + assert!(image_subresource_range_eq(ranges[0].0, range_a)); + assert_eq!( + ranges[0].1.map(|transfer| ( + transfer.src_queue_family_index, + transfer.src_queue_index, + transfer.dst_queue_family_index, + )), + Some((1, 2, 3)) + ); + assert!(image_subresource_range_eq(ranges[1].0, range_b)); + assert!(ranges[1].1.is_none()); + } + + #[test] + #[ignore = "requires Vulkan device"] + fn track_pending_transfers_only_collects_touched_subresources() -> Result<(), DriverError> { + let device = test_device()?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d_array(1, 1, 2, vk::Format::R8_UINT, vk::ImageUsageFlags::SAMPLED), + )?); + let range_a = color_subresource_range(0..1, 0..1); + let range_b = color_subresource_range(1..2, 0..1); + let image_handle = graph.resource(image).handle; + + { + let image_resource = graph.resource(image); + image_resource.set_sharing_ranges(SharingMode::Exclusive(Some((1, 0))), &[range_a]); + image_resource.set_sharing_ranges(SharingMode::Exclusive(Some((2, 0))), &[range_b]); + + image_resource + .swap_access(AccessType::TransferRead, range_a) + .for_each(drop); + image_resource + .swap_access(AccessType::TransferRead, range_b) + .for_each(drop); + } + + graph + .begin_cmd() + .debug_name("touch first layer only") + .subresource_access(image, range_a, AccessType::TransferWrite) + .record_cmd(|_| {}) + .end_cmd(); + + let mut submission = graph.finalize(); + submission.track_pending_transfers( + &Schedule { + cmds: vec![0], + ..Default::default() + }, + 3, + ); + + let (handle, transfers) = pending_transfer_for_node( + submission + .pending_image_transfer_nodes + .as_ref() + .expect("missing pending transfer nodes"), + image.index(), + ) + .expect("missing pending transfer for touched subresource"); + assert_eq!(handle, image_handle); + assert_eq!( + submission + .pending_image_transfer_nodes + .as_ref() + .expect("missing pending transfer nodes") + .indices, + vec![image.index()] + ); + let mut transfers = transfers.to_vec(); + sort_pending_image_transfers(&mut transfers); + + assert_eq!(transfers.len(), 1); + assert!(super::image_subresource_range_eq( + transfers[0].range, + range_a + )); + let ranges = &submission.exclusive_image_ranges[&image.index()]; + let mut ranges = ranges.clone(); + sort_image_subresource_ranges(&mut ranges); + assert_eq!(ranges.len(), 1); + assert!(super::image_subresource_range_eq(ranges[0], range_a)); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn track_pending_transfers_only_collects_touched_buffer_ranges() -> Result<(), DriverError> { + let device = test_device()?; + let mut graph = Graph::new(); + let buffer = graph.bind_resource(Buffer::create( + &device, + BufferInfo::device_mem(16, vk::BufferUsageFlags::TRANSFER_DST), + )?); + let range_a = BufferSubresourceRange { start: 0, end: 8 }; + let range_b = BufferSubresourceRange { start: 8, end: 16 }; + let buffer_handle = graph.resource(buffer).handle; + + { + let buffer_resource = graph.resource(buffer); + buffer_resource.set_sharing_ranges(SharingMode::Exclusive(Some((1, 0))), &[range_a]); + buffer_resource.set_sharing_ranges(SharingMode::Exclusive(Some((2, 0))), &[range_b]); + + buffer_resource + .swap_access(AccessType::TransferRead, range_a) + .for_each(drop); + buffer_resource + .swap_access(AccessType::TransferRead, range_b) + .for_each(drop); + } + + graph + .begin_cmd() + .debug_name("touch first buffer range only") + .subresource_access(buffer, range_a, AccessType::TransferWrite) + .record_cmd(|_| {}) + .end_cmd(); + + let mut submission = graph.finalize(); + submission.track_pending_transfers( + &Schedule { + cmds: vec![0], + ..Default::default() + }, + 3, + ); + + let (handle, transfers) = pending_transfer_for_node( + submission + .pending_buffer_transfer_nodes + .as_ref() + .expect("missing pending transfer nodes"), + buffer.index(), + ) + .expect("missing pending transfer for touched buffer range"); + assert_eq!(handle, buffer_handle); + assert_eq!( + submission + .pending_buffer_transfer_nodes + .as_ref() + .expect("missing pending transfer nodes") + .indices, + vec![buffer.index()] + ); + let mut transfers = transfers.to_vec(); + sort_pending_buffer_transfers(&mut transfers); + + assert_eq!(transfers.len(), 1); + assert!(pending_buffer_transfer_for_range(&transfers, range_a).is_some()); + assert!(pending_buffer_transfer_for_range(&transfers, range_b).is_none()); + + let ranges = &submission.exclusive_buffer_ranges[&buffer.index()]; + let mut ranges = ranges.clone(); + ranges.sort_unstable_by_key(|range| (range.start, range.end)); + assert_eq!(ranges, vec![range_a]); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn track_pending_transfers_keeps_exclusive_owner_without_known_layout() + -> Result<(), DriverError> { + let device = test_device()?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d_array(1, 1, 2, vk::Format::R8_UINT, vk::ImageUsageFlags::SAMPLED), + )?); + let range_a = color_subresource_range(0..1, 0..1); + let range_b = color_subresource_range(1..2, 0..1); + let image_handle = graph.resource(image).handle; + + { + let image_resource = graph.resource(image); + image_resource.set_sharing_ranges(SharingMode::Exclusive(Some((1, 0))), &[range_a]); + image_resource.set_sharing_ranges(SharingMode::Exclusive(Some((2, 0))), &[range_b]); + } + + graph + .begin_cmd() + .debug_name("touch first layer only") + .subresource_access(image, range_a, AccessType::TransferWrite) + .record_cmd(|_| {}) + .end_cmd(); + + let mut submission = graph.finalize(); + submission.track_pending_transfers( + &Schedule { + cmds: vec![0], + ..Default::default() + }, + 3, + ); + + let (handle, transfers) = pending_transfer_for_node( + submission + .pending_image_transfer_nodes + .as_ref() + .expect("missing pending transfer nodes"), + image.index(), + ) + .expect("missing pending transfer for touched subresource"); + assert_eq!(handle, image_handle); + assert_eq!( + submission + .pending_image_transfer_nodes + .as_ref() + .expect("missing pending transfer nodes") + .indices, + vec![image.index()] + ); + let mut transfers = transfers.to_vec(); + sort_pending_image_transfers(&mut transfers); + + assert_eq!(transfers.len(), 1); + assert!(super::image_subresource_range_eq( + transfers[0].range, + range_a + )); + assert_eq!(transfers[0].layout, vk::ImageLayout::UNDEFINED); + + let ranges = &submission.exclusive_image_ranges[&image.index()]; + let mut ranges = ranges.clone(); + sort_image_subresource_ranges(&mut ranges); + assert_eq!(ranges.len(), 1); + assert!(super::image_subresource_range_eq(ranges[0], range_a)); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn recorded_submission_attach_updates_only_touched_subresources() -> Result<(), DriverError> { + let device = test_device()?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d_array(1, 1, 2, vk::Format::R8_UINT, vk::ImageUsageFlags::SAMPLED), + )?); + let range_a = color_subresource_range(0..1, 0..1); + let range_b = color_subresource_range(1..2, 0..1); + + { + let image_resource = graph.resource(image); + image_resource.set_sharing_ranges(SharingMode::Exclusive(Some((1, 0))), &[range_a]); + image_resource.set_sharing_ranges(SharingMode::Exclusive(Some((2, 0))), &[range_b]); + + image_resource + .swap_access(AccessType::TransferRead, range_a) + .for_each(drop); + image_resource + .swap_access(AccessType::TransferRead, range_b) + .for_each(drop); + } + + let mut submission = graph.finalize(); + submission + .exclusive_image_ranges + .insert(image.index(), vec![range_a]); + + let mut fence = Fence::create(&device, false)?; + let cmd_buf = CommandBuffer::create(&device, CommandBufferInfo::new(3))?; + let recorded = RecordedSubmission { + cmd_buf, + queue_ownership_release_waits: Vec::new(), + state: Arc::new(Mutex::new(RecordedSubmissionState { + submission, + _releases: Vec::new(), + })), + }; + + recorded.attach(&mut fence, 7); + + let state = recorded.state.lock().expect("poisoned recorded state"); + let sync_info = state.submission.graph.resource(image).sync_info(); + let mut subresources = sync_info.subresources.into_vec(); + sort_image_subresource_sync_infos(&mut subresources); + + assert_eq!(subresources.len(), 2); + assert_eq!(subresources[0].queue_family_index, Some(3)); + assert_eq!(subresources[1].queue_family_index, Some(2)); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn recorded_submission_attach_updates_only_touched_buffer_ranges() -> Result<(), DriverError> { + let device = test_device()?; + let mut graph = Graph::new(); + let buffer = graph.bind_resource(Buffer::create( + &device, + BufferInfo::device_mem(16, vk::BufferUsageFlags::TRANSFER_DST), + )?); + let range_a = BufferSubresourceRange { start: 0, end: 8 }; + let range_b = BufferSubresourceRange { start: 8, end: 16 }; + + { + let buffer_resource = graph.resource(buffer); + buffer_resource.set_sharing_ranges(SharingMode::Exclusive(Some((1, 0))), &[range_a]); + buffer_resource.set_sharing_ranges(SharingMode::Exclusive(Some((2, 0))), &[range_b]); + + buffer_resource + .swap_access(AccessType::TransferRead, range_a) + .for_each(drop); + buffer_resource + .swap_access(AccessType::TransferRead, range_b) + .for_each(drop); + } + + let mut submission = graph.finalize(); + submission + .exclusive_buffer_ranges + .insert(buffer.index(), vec![range_a]); + + let mut fence = Fence::create(&device, false)?; + let cmd_buf = CommandBuffer::create(&device, CommandBufferInfo::new(3))?; + let recorded = RecordedSubmission { + cmd_buf, + queue_ownership_release_waits: Vec::new(), + state: Arc::new(Mutex::new(RecordedSubmissionState { + submission, + _releases: Vec::new(), + })), + }; + + recorded.attach(&mut fence, 7); + + let state = recorded.state.lock().expect("poisoned recorded state"); + let sync_info = state.submission.graph.resource(buffer).sync_info(); + let mut ranges = sync_info.ranges.into_vec(); + ranges.sort_unstable_by_key(|range| (range.range.start, range.range.end)); + + assert_eq!(ranges.len(), 2); + assert_eq!(ranges[0].queue_family_index, Some(3)); + assert_eq!(ranges[1].queue_family_index, Some(2)); + + Ok(()) + } + + #[test] + fn reorder_scheduled_cmds_keeps_disconnected_groups_deterministic() { + let mut schedule = schedule_with_access_index( + &[0, 1, 2, 3, 4], + &[&[0, 1, 2], &[3, 4]], + &[&[0], &[0], &[0], &[1], &[1]], + ); + + schedule.reorder_cmds(5); + + assert_eq!(schedule.cmds, vec![0, 1, 2, 3, 4]); + } + + #[test] + fn reorder_scheduled_cmds_preserves_write_only_order() { + let mut schedule = schedule_with_access_index( + &[0, 1, 2, 3], + /* + Resource 0 is written by cmd 0 and read by cmd 3. Resource 1 is written by cmds 1 and + 2, so their relative order must be preserved even though neither cmd reads it. + */ + &[&[0, 3], &[1, 2]], + &[&[0], &[1], &[1], &[0]], + ); + + schedule.reorder_cmds(4); + + let cmd_1_position = schedule + .cmds + .iter() + .position(|&cmd_idx| cmd_idx == 1) + .expect("cmd 1 was not scheduled"); + let cmd_2_position = schedule + .cmds + .iter() + .position(|&cmd_idx| cmd_idx == 2) + .expect("cmd 2 was not scheduled"); + + assert!( + cmd_1_position < cmd_2_position, + "write-only commands were reordered: {:?}", + schedule.cmds + ); + } + + #[test] + fn reorder_scheduled_cmds_preserves_write_after_write_hazard() { + fuzz::check_schedule_reordering( + 4, + &[ + vec![ + fuzz::ResourceAccess { + cmd_idx: 1, + write: true, + }, + fuzz::ResourceAccess { + cmd_idx: 2, + write: true, + }, + ], + vec![fuzz::ResourceAccess { + cmd_idx: 0, + write: false, + }], + vec![fuzz::ResourceAccess { + cmd_idx: 3, + write: false, + }], + ], + ); + } + + #[test] + fn reorder_scheduled_cmds_preserves_write_then_read_hazard() { + fuzz::check_schedule_reordering( + 4, + &[ + vec![ + fuzz::ResourceAccess { + cmd_idx: 1, + write: true, + }, + fuzz::ResourceAccess { + cmd_idx: 2, + write: false, + }, + ], + vec![fuzz::ResourceAccess { + cmd_idx: 0, + write: false, + }], + vec![fuzz::ResourceAccess { + cmd_idx: 3, + write: false, + }], + ], + ); + } + + #[test] + fn reorder_scheduled_cmds_preserves_read_then_write_hazard() { + fuzz::check_schedule_reordering( + 4, + &[ + vec![ + fuzz::ResourceAccess { + cmd_idx: 1, + write: false, + }, + fuzz::ResourceAccess { + cmd_idx: 2, + write: true, + }, + ], + vec![fuzz::ResourceAccess { + cmd_idx: 0, + write: false, + }], + vec![fuzz::ResourceAccess { + cmd_idx: 3, + write: false, + }], + ], + ); + } + + #[test] + fn reorder_scheduled_cmds_allows_unrelated_moves_without_crossing_hazard() { + fuzz::check_schedule_reordering( + 6, + &[ + vec![ + fuzz::ResourceAccess { + cmd_idx: 1, + write: true, + }, + fuzz::ResourceAccess { + cmd_idx: 4, + write: true, + }, + ], + vec![ + fuzz::ResourceAccess { + cmd_idx: 0, + write: false, + }, + fuzz::ResourceAccess { + cmd_idx: 2, + write: false, + }, + fuzz::ResourceAccess { + cmd_idx: 5, + write: false, + }, + ], + vec![fuzz::ResourceAccess { + cmd_idx: 3, + write: false, + }], + ], + ); + } + + #[test] + fn record_selection_from_node_creates_node_variant() { + let node = BufferNode::new( + 7, + #[cfg(feature = "checked")] + crate::GraphId(1), + ); + + let selection = RecordSelection::from(node); + + match selection { + RecordSelection::Node(AnyNode::Buffer(actual)) => assert_eq!(actual.index(), 7), + _ => panic!("expected RecordSelection::Node(Buffer)"), + } + } + + #[test] + fn record_selection_nodes_preserves_slice() { + let lhs = AnyNode::from(BufferNode::new( + 1, + #[cfg(feature = "checked")] + crate::GraphId(1), + )); + let rhs = AnyNode::from(BufferNode::new( + 2, + #[cfg(feature = "checked")] + crate::GraphId(1), + )); + let nodes = [lhs, rhs]; + + match RecordSelection::nodes(&nodes) { + RecordSelection::Nodes(actual) => assert_eq!(actual.len(), 2), + _ => panic!("expected RecordSelection::Nodes"), + } + } + + #[test] + fn legacy_submit_accepts_all_commands_and_none_wait_masks() { + let waits = [ + SemaphoreSubmitInfo { + semaphore: vk::Semaphore::null(), + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + }, + SemaphoreSubmitInfo { + semaphore: vk::Semaphore::null(), + stage_mask: vk::PipelineStageFlags2::NONE, + value: 0, + }, + ]; + let signals = [SemaphoreSubmitInfo { + semaphore: vk::Semaphore::null(), + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 0, + }]; + + assert!(check_queue_submit_args(&waits, &signals).is_ok()); + } + + #[test] + fn legacy_submit_rejects_precise_wait_stage_masks() { + let waits = [SemaphoreSubmitInfo { + semaphore: vk::Semaphore::null(), + stage_mask: vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, + value: 0, + }]; + + assert!(matches!( + check_queue_submit_args(&waits, &[]), + Err(DriverError::Unsupported) + )); + } + + #[test] + fn legacy_submit_rejects_timeline_values() { + let waits = [SemaphoreSubmitInfo { + semaphore: vk::Semaphore::null(), + stage_mask: vk::PipelineStageFlags2::ALL_COMMANDS, + value: 1, + }]; + + assert!(matches!( + check_queue_submit_args(&waits, &[]), + Err(DriverError::Unsupported) + )); + } + + fn test_device() -> Result { + let guard = test_device_lock() + .lock() + .expect("poisoned test device lock"); + let device = Device::create(DeviceInfo::default())?; + + Ok(TestDevice { + _guard: guard, + device: ManuallyDrop::new(device), + }) + } + + fn test_debug_device() -> Result { + let guard = test_device_lock() + .lock() + .expect("poisoned test device lock"); + let device = Device::create(DeviceInfo::builder().debug(true).build())?; + + Ok(TestDevice { + _guard: guard, + device: ManuallyDrop::new(device), + }) + } + + fn init_validation_test_logging() { + static INIT: OnceLock<()> = OnceLock::new(); + + INIT.get_or_init(|| { + unsafe { + set_var("RUST_LOG", "trace"); + set_var("VK_GRAPH_SKIP_VALIDATION_PARK", "1"); + } + + let _ = pretty_env_logger::try_init(); + }); + } + + fn test_triangle_pipeline(device: &Device) -> Result { + GraphicsPipeline::create( + device, + GraphicsPipelineInfo::default(), + [ + glsl!( + r#" + #version 460 core + #pragma shader_stage(vertex) + + vec2 POSITIONS[3] = vec2[]( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0) + ); + + void main() { + gl_Position = vec4(POSITIONS[gl_VertexIndex], 0.0, 1.0); + } + "# + ) + .as_slice(), + glsl!( + r#" + #version 460 core + #pragma shader_stage(fragment) + + layout(location = 0) out vec4 vk_Color; + + void main() { + vk_Color = vec4(1.0, 0.0, 0.0, 1.0); + } + "# + ) + .as_slice(), + ], + ) + } + + fn test_input_attachment_pipelines( + device: &Device, + ) -> Result<(GraphicsPipeline, GraphicsPipeline), DriverError> { + let vertex = glsl!( + r#" + #version 460 core + #pragma shader_stage(vertex) + + vec2 POSITIONS[3] = vec2[]( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0) + ); + + void main() { + gl_Position = vec4(POSITIONS[gl_VertexIndex], 0.0, 1.0); + } + "# + ); + let pipeline_a = GraphicsPipeline::create( + device, + GraphicsPipelineInfo::default(), + [ + vertex.as_slice(), + glsl!( + kind: frag, + r#" + #version 460 core + #pragma shader_stage(fragment) + + layout(location = 0) out vec4 color_out; + + void main() { + color_out = vec4(0.25, 0.5, 0.75, 1.0); + } + "# + ) + .as_slice(), + ], + )?; + let pipeline_b = GraphicsPipeline::create( + device, + GraphicsPipelineInfo::default(), + [ + vertex.as_slice(), + glsl!( + kind: frag, + r#" + #version 460 core + #pragma shader_stage(fragment) + + layout(input_attachment_index = 0, binding = 0) uniform subpassInput color_in; + layout(location = 0) out vec4 color_out; + + void main() { + color_out = subpassLoad(color_in); + } + "# + ) + .as_slice(), + ], + )?; + + Ok((pipeline_a, pipeline_b)) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn submission_record_all_consumes_single_pass_graph() -> Result<(), DriverError> { + let device = test_device()?; + let mut pool = HashPool::new(&device); + let mut graph = Graph::new(); + let buffer = graph.bind_resource(Buffer::create( + &device, + BufferInfo::device_mem(16, vk::BufferUsageFlags::TRANSFER_DST), + )?); + + graph.fill_buffer(buffer, 0..16, 0xdead_beef); + + let submission = graph.finalize(); + let mut cmd_buf = pool.resource(CommandBufferInfo::new(0))?; + + cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + + let recorded = submission.record(&mut pool, &mut cmd_buf, RecordSelection::All)?; + + assert!(recorded.is_empty()); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn submission_record_nodes_consumes_requested_outputs() -> Result<(), DriverError> { + let device = test_device()?; + let mut pool = HashPool::new(&device); + let mut graph = Graph::new(); + let lhs = graph.bind_resource(Buffer::create( + &device, + BufferInfo::device_mem(16, vk::BufferUsageFlags::TRANSFER_DST), + )?); + let rhs = graph.bind_resource(Buffer::create( + &device, + BufferInfo::device_mem(16, vk::BufferUsageFlags::TRANSFER_DST), + )?); + + graph.fill_buffer(lhs, 0..16, 1); + graph.fill_buffer(rhs, 0..16, 2); + + let nodes = [AnyNode::from(lhs), AnyNode::from(rhs)]; + let submission = graph.finalize(); + let mut cmd_buf = pool.resource(CommandBufferInfo::new(0))?; + + cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + + let recorded = + submission.record(&mut pool, &mut cmd_buf, RecordSelection::nodes(&nodes))?; + + assert!(recorded.is_empty()); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn submission_record_can_be_reused() -> Result<(), DriverError> { + let device = test_device()?; + let mut pool = HashPool::new(&device); + let mut graph = Graph::new(); + let buffer = graph.bind_resource(Buffer::create( + &device, + BufferInfo::device_mem(16, vk::BufferUsageFlags::TRANSFER_DST), + )?); + + graph.fill_buffer(buffer, 0..16, 0xdead_beef); + + let submission = graph.finalize(); + let mut cmd_buf = pool.resource(CommandBufferInfo::new(0))?; + let mut fence = Fence::create(&device, false)?; + + cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::SIMULTANEOUS_USE), + )?; + + let recorded = submission.record(&mut pool, &mut cmd_buf, RecordSelection::All)?; + recorded.cmd_buf.end()?; + let replay = recorded.finish()?; + replay.attach(&mut fence, 0); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn accel_struct_mixed_accesses_preserve_all_stage_bits() -> Result<(), DriverError> { + let device = test_device()?; + let mut pool = HashPool::new(&device); + let mut graph = Graph::new(); + let accel_struct = graph.bind_resource(AccelerationStructure::create( + &device, + AccelerationStructureInfo::blas(1024), + )?); + + graph + .begin_cmd() + .debug_name("mixed accel struct accesses") + .resource_access(accel_struct, AccessType::AccelerationStructureBuildRead) + .resource_access( + accel_struct, + AccessType::RayTracingShaderReadAccelerationStructure, + ) + .record_cmd(|_| {}); + + let submission = graph.finalize(); + let mut cmd_buf = pool.resource(CommandBufferInfo::new(0))?; + + cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + + let recording = submission.record(&mut pool, &mut cmd_buf, RecordSelection::All)?; + let sync_info = recording.resource(accel_struct).sync_info(); + + assert!( + sync_info + .stage_mask + .contains(vk::PipelineStageFlags::ACCELERATION_STRUCTURE_BUILD_KHR), + "sync info should preserve build-read stage bits" + ); + assert!( + sync_info + .stage_mask + .contains(vk::PipelineStageFlags::RAY_TRACING_SHADER_KHR), + "sync info should preserve ray-tracing-read stage bits" + ); + assert_eq!( + sync_info.access_mask, + vk::AccessFlags::ACCELERATION_STRUCTURE_READ_KHR, + "mixed read-only accesses should stay read-only" + ); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan validation layers; inspect validation output"] + fn submission_external_subpass_dependency_validation_repro() -> Result<(), DriverError> { + init_validation_test_logging(); + + let device = test_debug_device()?; + let mut pool = HashPool::new(&device); + let pipeline = test_triangle_pipeline(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::TRANSFER_DST | vk::ImageUsageFlags::COLOR_ATTACHMENT, + ), + )?); + + // Seed external_access_history with a transfer write so the later render pass relies on + // the synthesized EXTERNAL -> first subpass dependency + graph.clear_color_image(image, [0.0, 0.0, 0.0, 1.0]); + graph + .begin_cmd() + .debug_name("validation repro render pass") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::Load, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + + let submission = graph.finalize(); + let mut cmd_buf = pool.resource(CommandBufferInfo::new(0))?; + + cmd_buf.begin( + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + + let recorded = submission.record(&mut pool, &mut cmd_buf, RecordSelection::All)?; + recorded.cmd_buf.end()?; + + let mut fence = Fence::create(&device, false)?; + let mut recorded = recorded.finish()?; + + recorded.queue_submit(&mut fence, 0, QueueSubmitInfo::QUEUE_SUBMIT)?; + fence.wait_signaled()?; + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn external_subpass_dependency_targets_first_subpass_consumer() -> Result<(), DriverError> { + let device = test_device()?; + let pipeline = test_triangle_pipeline(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::TRANSFER_DST | vk::ImageUsageFlags::COLOR_ATTACHMENT, + ), + )?); + + graph.clear_color_image(image, [0.0, 0.0, 0.0, 1.0]); + graph + .begin_cmd() + .debug_name("dependency inspection render pass") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::Load, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + + let submission = graph.finalize(); + let mut external_access_history = + ExternalRenderPassAccessHistory::new(submission.graph.resources.len()); + external_access_history.record_cmd(&submission.graph.cmds[0]); + + let dependencies = Submission::build_subpass_dependencies( + &submission.graph.cmds[1], + &external_access_history, + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == vk::SUBPASS_EXTERNAL && dep.dst_subpass == 0) + .expect("missing external -> first subpass dependency"); + + assert_eq!( + dep.dst_stage_mask, + vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, + "destination stage should describe the first subpass consumer" + ); + assert_eq!( + dep.dst_access_mask, + vk::AccessFlags::COLOR_ATTACHMENT_READ | vk::AccessFlags::COLOR_ATTACHMENT_WRITE, + "destination access should describe the first subpass attachment access" + ); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn color_input_attachment_dependencies_use_fragment_shader_input_reads() + -> Result<(), DriverError> { + let device = test_device()?; + let (pipeline_a, pipeline_b) = test_input_attachment_pipelines(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::COLOR_ATTACHMENT + | vk::ImageUsageFlags::INPUT_ATTACHMENT + | vk::ImageUsageFlags::TRANSFER_DST, + ), + )?); + + graph + .begin_cmd() + .debug_name("input attachment writer") + .bind_pipeline(&pipeline_a) + .color_attachment_image(0, image, LoadOp::CLEAR_BLACK_ALPHA_ZERO, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + graph + .begin_cmd() + .debug_name("input attachment reader") + .bind_pipeline(&pipeline_b) + .color_attachment_image(0, image, LoadOp::DontCare, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + + let mut submission = graph.finalize(); + let mut schedule = vec![0, 1]; + submission.merge_scheduled_cmds(&mut schedule); + + let dependencies = Submission::build_subpass_dependencies( + &submission.graph.cmds[0], + &ExternalRenderPassAccessHistory::new(submission.graph.resources.len()), + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for input attachment read"); + + assert!( + dep.src_stage_mask + .contains(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT), + "source stage should include color attachment output" + ); + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_WRITE), + "source access should include color attachment write" + ); + assert!( + dep.dst_stage_mask + .contains(vk::PipelineStageFlags::FRAGMENT_SHADER), + "destination stage should include fragment shader input attachment reads" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::INPUT_ATTACHMENT_READ), + "destination access should include input attachment reads" + ); + + Ok(()) + } - return Err(DriverError::InvalidData); - } - }; + #[test] + #[ignore = "requires Vulkan device"] + fn color_attachment_load_dependencies_avoid_invalid_stage_access_pairs() + -> Result<(), DriverError> { + let device = test_device()?; + let pipeline = test_triangle_pipeline(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::COLOR_ATTACHMENT, + ), + )?); + + graph + .begin_cmd() + .debug_name("color attachment writer") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::CLEAR_BLACK_ALPHA_ZERO, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + graph + .begin_cmd() + .debug_name("color attachment reader") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::Load, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); - if binding_offset == 0 { - tls.image_writes.push(IndexWrite { - idx: tls.image_infos.len(), - write: vk::WriteDescriptorSet { - dst_set: *descriptor_sets[descriptor_set_idx as usize], - dst_binding, - descriptor_type, - descriptor_count: 1, - ..Default::default() - }, - }); - } else { - tls.image_writes - .last_mut() - .expect("missing image descriptor write") - .write - .descriptor_count += 1; - } + let mut submission = graph.finalize(); + let mut schedule = vec![0, 1]; + submission.merge_scheduled_cmds(&mut schedule); - tls.image_infos.push( - vk::DescriptorImageInfo::default() - .image_layout(image_layout) - .image_view(image_view), - ); - } else if let Some(buffer) = bound_node.as_buffer() { - let buffer_view_info = view_info.expect_buffer(); + let dependencies = Submission::build_subpass_dependencies( + &submission.graph.cmds[0], + &ExternalRenderPassAccessHistory::new(submission.graph.resources.len()), + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for color attachment load"); - if binding_offset == 0 { - tls.buffer_writes.push(IndexWrite { - idx: tls.buffer_infos.len(), - write: vk::WriteDescriptorSet { - dst_set: *descriptor_sets[descriptor_set_idx as usize], - dst_binding, - descriptor_type, - descriptor_count: 1, - ..Default::default() - }, - }); - } else { - tls.buffer_writes - .last_mut() - .expect("missing buffer descriptor write") - .write - .descriptor_count += 1; - } + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_WRITE), + "source access should include color attachment writes" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ), + "destination access should include color attachment reads" + ); + assert_no_invalid_attachment_stage_access_pairs(dep); + assert_attachment_read_stage_mappings(dep); - tls.buffer_infos.push( - vk::DescriptorBufferInfo::default() - .buffer(buffer.handle) - .offset(buffer_view_info.start) - .range(buffer_view_info.end - buffer_view_info.start), - ); - } else if let Some(accel_struct) = bound_node.as_accel_struct() { - if binding_offset == 0 { - tls.accel_struct_writes.push(IndexWrite { - idx: tls.accel_struct_infos.len(), - write: vk::WriteDescriptorSet::default() - .dst_set(*descriptor_sets[descriptor_set_idx as usize]) - .dst_binding(dst_binding) - .descriptor_type(descriptor_type) - .descriptor_count(1), - }); - } else { - tls.accel_struct_writes - .last_mut() - .expect("missing acceleration structure descriptor write") - .write - .descriptor_count += 1; - } + Ok(()) + } - tls.accel_struct_infos.push( - vk::WriteDescriptorSetAccelerationStructureKHR::default() - .acceleration_structures(std::slice::from_ref(&accel_struct.handle)), - ); - } else { - warn!( - "invalid bound resource kind at descriptor {}.{}[{}] in pass \"{}\"", - descriptor_set_idx, - dst_binding, - binding_offset, - pass.name() - ); + #[test] + #[ignore = "requires Vulkan device"] + fn color_attachment_read_dependencies_avoid_invalid_stage_access_pairs() + -> Result<(), DriverError> { + let device = test_device()?; + let pipeline = test_triangle_pipeline(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::COLOR_ATTACHMENT, + ), + )?); + + graph + .begin_cmd() + .debug_name("color attachment first reader") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::Load, StoreOp::DontCare) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + graph + .begin_cmd() + .debug_name("color attachment second reader") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::Load, StoreOp::DontCare) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); - return Err(DriverError::InvalidData); - } - } + let mut submission = graph.finalize(); + let mut schedule = vec![0, 1]; + submission.merge_scheduled_cmds(&mut schedule); - if let ExecutionPipeline::Graphic(pipeline) = pipeline { - // Write graphic render pass input attachments (they're automatic) - if exec_idx > 0 { - for ( - &Descriptor { - set: descriptor_set_idx, - binding: dst_binding, - }, - (descriptor_info, _), - ) in &pipeline.inner.descriptor_bindings - { - if let DescriptorInfo::InputAttachment(_, attachment_idx) = *descriptor_info - { - let is_random_access = exec.color_stores.contains_key(&attachment_idx) - || exec.color_resolves.contains_key(&attachment_idx); - let current_attachment = exec - .color_attachments - .get(&attachment_idx) - .copied() - .or_else(|| { - exec.color_clears - .get(&attachment_idx) - .map(|(attachment, _)| *attachment) - }) - .or_else(|| exec.color_loads.get(&attachment_idx).copied()) - .or_else(|| exec.color_stores.get(&attachment_idx).copied()) - .or_else(|| { - exec.color_resolves - .get(&attachment_idx) - .map(|(attachment, _)| *attachment) - }) - .expect("missing input attachment target"); - let (attachment, write_exec) = pass.execs[0..exec_idx] - .iter() - .rev() - .find_map(|exec| { - exec.color_attachments - .get(&attachment_idx) - .copied() - .or_else(|| { - exec.color_clears - .get(&attachment_idx) - .map(|(attachment, _)| *attachment) - }) - .or_else(|| exec.color_loads.get(&attachment_idx).copied()) - .or_else(|| exec.color_stores.get(&attachment_idx).copied()) - .or_else(|| { - exec.color_resolves.get(&attachment_idx).map( - |(resolved_attachment, _)| *resolved_attachment, - ) - }) - .filter(|attachment| { - Attachment::are_compatible( - Some(current_attachment), - Some(*attachment), - ) - }) - .map(|attachment| (attachment, exec)) - }) - .expect("input attachment not written"); - let late = &write_exec.accesses[&attachment.target] - .last() - .expect("missing input attachment access"); - let image_range = late.subresource.expect_image(); - let image_binding = &bindings[attachment.target]; - let image = image_binding.expect_image(); - let image_view_info = attachment - .image_view_info(image.info) - .into_builder() - .array_layer_count(image_range.layer_count) - .base_array_layer(image_range.base_array_layer) - .base_mip_level(image_range.base_mip_level) - .mip_level_count(image_range.level_count) - .build(); - let image_view = Image::view(image, image_view_info)?; - - tls.image_writes.push(IndexWrite { - idx: tls.image_infos.len(), - write: vk::WriteDescriptorSet { - dst_set: *descriptor_sets[descriptor_set_idx as usize], - dst_binding, - descriptor_type: vk::DescriptorType::INPUT_ATTACHMENT, - descriptor_count: 1, - ..Default::default() - }, - }); + let dependencies = Submission::build_subpass_dependencies( + &submission.graph.cmds[0], + &ExternalRenderPassAccessHistory::new(submission.graph.resources.len()), + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for color attachment read"); - tls.image_infos.push(vk::DescriptorImageInfo { - image_layout: Self::attachment_layout( - attachment.aspect_mask, - is_random_access, - true, - ), - image_view, - sampler: vk::Sampler::null(), - }); - } - } - } - } - } + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ), + "source access should include color attachment reads" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ), + "destination access should include color attachment reads" + ); + assert_no_invalid_attachment_stage_access_pairs(dep); + assert_attachment_read_stage_mappings(dep); - // NOTE: We assign the below pointers after the above insertions so they remain stable! + Ok(()) + } - tls.descriptors - .extend(tls.accel_struct_writes.drain(..).map( - |IndexWrite { idx, mut write }| unsafe { - write.p_next = tls.accel_struct_infos.as_ptr().add(idx) as *const _; - write - }, - )); - tls.descriptors.extend(tls.buffer_writes.drain(..).map( - |IndexWrite { idx, mut write }| unsafe { - write.p_buffer_info = tls.buffer_infos.as_ptr().add(idx); - write + #[test] + #[ignore = "requires Vulkan device"] + fn color_attachment_read_to_write_dependencies_avoid_invalid_stage_access_pairs() + -> Result<(), DriverError> { + let device = test_device()?; + let pipeline = test_triangle_pipeline(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::COLOR_ATTACHMENT, + ), + )?); + + graph + .begin_cmd() + .debug_name("color attachment reader") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::Load, StoreOp::DontCare) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + graph + .begin_cmd() + .debug_name("color attachment writer") + .bind_pipeline(&pipeline) + .color_attachment_image(0, image, LoadOp::CLEAR_BLACK_ALPHA_ZERO, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + + let mut submission = graph.finalize(); + let mut schedule = vec![0, 1]; + submission.merge_scheduled_cmds(&mut schedule); + + let dependencies = Submission::build_subpass_dependencies( + &submission.graph.cmds[0], + &ExternalRenderPassAccessHistory::new(submission.graph.resources.len()), + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for color attachment read to write"); + + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_READ), + "source access should include color attachment reads" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::COLOR_ATTACHMENT_WRITE), + "destination access should include color attachment writes" + ); + assert_no_invalid_attachment_stage_access_pairs(dep); + assert_attachment_read_stage_mappings(dep); + + Ok(()) + } + + #[test] + #[ignore = "requires Vulkan device"] + fn depth_attachment_load_dependencies_avoid_invalid_stage_access_pairs() + -> Result<(), DriverError> { + let device = test_device()?; + let pipeline = test_triangle_pipeline(&device)?; + let mut graph = Graph::new(); + let image = graph.bind_resource(Image::create( + &device, + ImageInfo::image_2d( + 4, + 4, + vk::Format::D32_SFLOAT, + vk::ImageUsageFlags::DEPTH_STENCIL_ATTACHMENT, + ), + )?); + + graph + .begin_cmd() + .debug_name("depth attachment first reader") + .bind_pipeline(&pipeline) + .depth_stencil_attachment_image(image, LoadOp::Load, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + graph + .begin_cmd() + .debug_name("depth attachment second reader") + .bind_pipeline(&pipeline) + .depth_stencil_attachment_image(image, LoadOp::Load, StoreOp::Store) + .record_cmd(|cmd| { + cmd.draw(3, 1, 0, 0); + }); + + let mut submission = graph.finalize(); + let mut schedule = vec![0, 1]; + submission.merge_scheduled_cmds(&mut schedule); + + let dependencies = Submission::build_subpass_dependencies( + &submission.graph.cmds[0], + &ExternalRenderPassAccessHistory::new(submission.graph.resources.len()), + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for depth attachment load"); + + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ), + "source access should include depth/stencil attachment reads" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ), + "destination access should include depth/stencil attachment reads" + ); + assert_no_invalid_attachment_stage_access_pairs(dep); + assert_attachment_read_stage_mappings(dep); + + Ok(()) + } + + #[test] + fn depth_attachment_read_to_write_dependency_includes_late_read_stage() { + let dependencies = depth_attachment_dependencies( + LoadOp::Load, + StoreOp::DontCare, + LoadOp::CLEAR_ONE_STENCIL_ZERO, + StoreOp::Store, + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for depth attachment read to write"); + + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ), + "source access should include depth/stencil attachment reads" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE), + "destination access should include depth/stencil attachment writes" + ); + assert!( + dep.src_stage_mask + .contains(vk::PipelineStageFlags::EARLY_FRAGMENT_TESTS), + "source stage should include early fragment tests" + ); + assert!( + dep.src_stage_mask + .contains(vk::PipelineStageFlags::LATE_FRAGMENT_TESTS), + "source stage should include late fragment tests" + ); + } + + #[test] + fn depth_attachment_write_to_write_dependency_uses_write_access() { + let dependencies = depth_attachment_dependencies( + LoadOp::CLEAR_ONE_STENCIL_ZERO, + StoreOp::Store, + LoadOp::CLEAR_ONE_STENCIL_ZERO, + StoreOp::Store, + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for depth attachment write to write"); + + assert!( + dep.src_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE), + "source access should include depth/stencil attachment writes" + ); + assert!( + !dep.src_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_READ), + "source access should not include depth/stencil attachment reads" + ); + assert!( + dep.dst_access_mask + .contains(vk::AccessFlags::DEPTH_STENCIL_ATTACHMENT_WRITE), + "destination access should include depth/stencil attachment writes" + ); + } + + #[test] + fn subpass_stage_mask_clamps_non_graphics_stages() { + assert_eq!( + Submission::subpass_stage_mask(vk::PipelineStageFlags::RAY_TRACING_SHADER_KHR), + vk::PipelineStageFlags::ALL_GRAPHICS, + ); + assert_eq!( + Submission::subpass_stage_mask( + vk::PipelineStageFlags::FRAGMENT_SHADER + | vk::PipelineStageFlags::RAY_TRACING_SHADER_KHR, + ), + vk::PipelineStageFlags::FRAGMENT_SHADER, + ); + } + + #[test] + fn subpass_dependency_matches_all_graphics_source_stage() { + let dependencies = subpass_dependencies_for_accesses( + AccessType::AnyShaderWrite, + AccessType::FragmentShaderReadOther, + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for ALL_GRAPHICS source stage"); + + assert!( + dep.src_stage_mask + .contains(vk::PipelineStageFlags::ALL_GRAPHICS), + "source stage should include ALL_GRAPHICS" + ); + assert!( + dep.src_access_mask.contains(vk::AccessFlags::SHADER_WRITE), + "source access should include shader writes" + ); + assert!( + dep.dst_stage_mask + .contains(vk::PipelineStageFlags::FRAGMENT_SHADER), + "destination stage should include fragment shader" + ); + assert!( + dep.dst_access_mask.contains(vk::AccessFlags::SHADER_READ), + "destination access should include shader reads" + ); + } + + #[test] + fn subpass_dependency_matches_all_graphics_destination_stage() { + let dependencies = subpass_dependencies_for_accesses( + AccessType::FragmentShaderWrite, + AccessType::AnyShaderReadOther, + ); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == 0 && dep.dst_subpass == 1) + .expect("missing subpass dependency for ALL_GRAPHICS destination stage"); + + assert!( + dep.src_stage_mask + .contains(vk::PipelineStageFlags::FRAGMENT_SHADER), + "source stage should include fragment shader" + ); + assert!( + dep.src_access_mask.contains(vk::AccessFlags::SHADER_WRITE), + "source access should include shader writes" + ); + assert!( + dep.dst_stage_mask + .contains(vk::PipelineStageFlags::ALL_GRAPHICS), + "destination stage should include ALL_GRAPHICS" + ); + assert!( + dep.dst_access_mask.contains(vk::AccessFlags::SHADER_READ), + "destination access should include shader reads" + ); + } + + #[test] + fn record_subpass_dependency_preserves_dst_access_for_unmatched_stages() { + let mut dependencies = std::collections::BTreeMap::new(); + let mut current = PipelineStageAccessFlags { + stage_flags: vk::PipelineStageFlags::VERTEX_SHADER + | vk::PipelineStageFlags::FRAGMENT_SHADER, + access_flags: vk::AccessFlags::SHADER_READ, + }; + + assert!(!Submission::record_subpass_dependency( + &mut dependencies, + 0, + 2, + PipelineStageAccessFlags { + stage_flags: vk::PipelineStageFlags::VERTEX_SHADER, + access_flags: vk::AccessFlags::SHADER_READ, }, + current.stage_flags, + &mut current, )); - tls.descriptors.extend(tls.image_writes.drain(..).map( - |IndexWrite { idx, mut write }| unsafe { - write.p_image_info = tls.image_infos.as_ptr().add(idx); - write + assert!(Submission::record_subpass_dependency( + &mut dependencies, + 1, + 2, + PipelineStageAccessFlags { + stage_flags: vk::PipelineStageFlags::FRAGMENT_SHADER, + access_flags: vk::AccessFlags::SHADER_READ, }, + current.stage_flags, + &mut current, )); - if !tls.descriptors.is_empty() { - trace!( - " writing {} descriptors ({} buffers, {} images)", - tls.descriptors.len(), - tls.buffer_infos.len(), - tls.image_infos.len() - ); + let dep = dependencies + .get(&(1, 2)) + .expect("missing dependency for later matched stage"); + assert!( + dep.dst_access_mask.contains(vk::AccessFlags::SHADER_READ), + "later matched stage should retain destination access mask" + ); + } - unsafe { - cmd_buf - .device - .update_descriptor_sets(tls.descriptors.as_slice(), &[]); - } - } + #[test] + fn record_subpass_dependency_ignores_non_overlapping_stage() { + let mut dependencies = std::collections::BTreeMap::new(); + let mut current = PipelineStageAccessFlags { + stage_flags: vk::PipelineStageFlags::FRAGMENT_SHADER, + access_flags: vk::AccessFlags::SHADER_READ, + }; - Ok(()) + assert!(!Submission::record_subpass_dependency( + &mut dependencies, + 0, + 1, + PipelineStageAccessFlags { + stage_flags: vk::PipelineStageFlags::VERTEX_SHADER, + access_flags: vk::AccessFlags::SHADER_WRITE, + }, + current.stage_flags, + &mut current, + )); + + assert!(dependencies.is_empty()); + assert_eq!(current.stage_flags, vk::PipelineStageFlags::FRAGMENT_SHADER); + assert_eq!(current.access_flags, vk::AccessFlags::SHADER_READ); } -} -impl From for Submission { - fn from(val: Graph) -> Self { - val.into_submission() + #[test] + fn build_subpass_dependencies_includes_later_access_stage_bits() { + let mut exec = Execution::default(); + + exec.accesses.push( + 0, + SubresourceAccess { + access: AccessType::IndexBuffer, + subresource: SubresourceRange::Buffer((0..16).into()), + }, + ); + exec.accesses.push( + 0, + SubresourceAccess { + access: AccessType::FragmentShaderReadOther, + subresource: SubresourceRange::Buffer((0..16).into()), + }, + ); + + let pass = CommandData { + execs: vec![exec], + + #[cfg(debug_assertions)] + name: None, + + stream_scope_id: None, + }; + let dependencies = + Submission::build_subpass_dependencies(&pass, &ExternalRenderPassAccessHistory::new(1)); + let dep = dependencies + .iter() + .find(|dep| dep.src_subpass == vk::SUBPASS_EXTERNAL && dep.dst_subpass == 0) + .expect("missing external dependency for mixed access slice"); + + assert!( + dep.dst_stage_mask + .contains(vk::PipelineStageFlags::VERTEX_INPUT), + "first access stage should be preserved" + ); + assert!( + dep.dst_stage_mask + .contains(vk::PipelineStageFlags::FRAGMENT_SHADER), + "later access stages should also contribute" + ); } -} -#[derive(Default)] -struct Schedule { - access_cache: AccessCache, - passes: Vec, + #[test] + fn accel_struct_canonical_accesses_preserves_mixed_slice_accesses() { + let accesses = [ + SubresourceAccess { + access: AccessType::AccelerationStructureBuildRead, + subresource: SubresourceRange::AccelerationStructure, + }, + SubresourceAccess { + access: AccessType::RayTracingShaderReadAccelerationStructure, + subresource: SubresourceRange::AccelerationStructure, + }, + ]; + + let mut scratch = Vec::new(); + assert_eq!( + Submission::accel_struct_canonical_accesses(&accesses, &mut scratch), + &[ + AccessType::AccelerationStructureBuildRead, + AccessType::RayTracingShaderReadAccelerationStructure, + ], + "mixed acceleration-structure slices should preserve all accesses for next-state tracking" + ); + } } diff --git a/tests/integration.rs b/tests/vulkan.rs similarity index 52% rename from tests/integration.rs rename to tests/vulkan.rs index b5df691a..09a125e1 100644 --- a/tests/integration.rs +++ b/tests/vulkan.rs @@ -1,17 +1,17 @@ use std::{env, ffi::OsString, process::Command}; -fn run_cpu_readback_example(extra_args: &[&str]) { +fn run_example(target_name: &str, extra_args: &[&str]) { let cargo = env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")); let output = Command::new(cargo) - .args(["run", "--quiet", "--example", "cpu_readback", "--"]) + .args(["run", "--quiet", "--example", target_name, "--"]) .args(extra_args) .current_dir(env!("CARGO_MANIFEST_DIR")) .output() - .expect("unable to run cpu_readback example"); + .unwrap_or_else(|err| panic!("unable to run {target_name} example: {err}")); assert!( output.status.success(), - "cpu_readback example failed with status {:?}\nstdout:\n{}\nstderr:\n{}", + "run error: example `{target_name}` exited with status {:?}\nstdout:\n{}\nstderr:\n{}", output.status.code(), String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), @@ -19,7 +19,7 @@ fn run_cpu_readback_example(extra_args: &[&str]) { } #[test] -#[ignore = "requires a working Vulkan device"] -fn live_device_cpu_readback() { - run_cpu_readback_example(&[]); +#[ignore = "requires Vulkan device"] +fn vulkan_cpu_readback() { + run_example("cpu_readback", &[]); }