Skip to main content

wgpu_hal/vulkan/
mod.rs

1/*!
2# Vulkan API internals.
3
4## Stack memory
5
6Ash expects slices, which we don't generally have available.
7We cope with this requirement by the combination of the following ways:
8  - temporarily allocating `Vec` on heap, where overhead is permitted
9  - growing temporary local storage
10
11## Framebuffers and Render passes
12
13Render passes are cached on the device and kept forever.
14
15Framebuffers are also cached on the device, but they are removed when
16any of the image views (they have) gets removed.
17If Vulkan supports image-less framebuffers,
18then the actual views are excluded from the framebuffer key.
19
20## Fences
21
22If timeline semaphores are available, they are used 1:1 with wgpu-hal fences.
23Otherwise, we manage a pool of `VkFence` objects behind each `hal::Fence`.
24
25!*/
26
27mod adapter;
28mod command;
29pub mod conv;
30mod descriptor;
31mod device;
32mod drm;
33mod instance;
34mod memory;
35mod pnext_chain;
36mod sampler;
37mod semaphore_list;
38mod swapchain;
39
40pub use adapter::PhysicalDeviceFeatures;
41pub(crate) use pnext_chain::PnextChain;
42
43use alloc::{boxed::Box, ffi::CString, sync::Arc, vec::Vec};
44use core::{
45    borrow::Borrow,
46    ffi::{c_void, CStr},
47    fmt,
48    marker::PhantomData,
49    mem,
50    num::NonZeroU32,
51};
52
53use arrayvec::ArrayVec;
54use ash::{ext, khr, vk};
55use bytemuck::{Pod, Zeroable};
56use hashbrown::HashSet;
57use wgpu_sync::{Mutex, RwLock};
58
59use naga::FastHashMap;
60use wgt::InternalCounter;
61
62use semaphore_list::SemaphoreList;
63
64pub use memory::MemoryAllocation;
65use memory::{MemoryAllocators, MemoryPool};
66
67use crate::vulkan::semaphore_list::{SemaphoreListMode, SemaphoreType};
68
69const MAX_TOTAL_ATTACHMENTS: usize = crate::MAX_COLOR_ATTACHMENTS * 2 + 1;
70
71#[derive(Clone, Debug)]
72pub struct Api;
73
74impl crate::Api for Api {
75    const VARIANT: wgt::Backend = wgt::Backend::Vulkan;
76
77    type Instance = Instance;
78    type Surface = Surface;
79    type Adapter = Adapter;
80    type Device = Device;
81
82    type Queue = Queue;
83    type CommandEncoder = CommandEncoder;
84    type CommandBuffer = CommandBuffer;
85
86    type Buffer = Buffer;
87    type Texture = Texture;
88    type SurfaceTexture = SurfaceTexture;
89    type TextureView = TextureView;
90    type Sampler = Sampler;
91    type QuerySet = QuerySet;
92    type Fence = Fence;
93    type AccelerationStructure = AccelerationStructure;
94    type PipelineCache = PipelineCache;
95
96    type BindGroupLayout = BindGroupLayout;
97    type BindGroup = BindGroup;
98    type PipelineLayout = PipelineLayout;
99    type ShaderModule = ShaderModule;
100    type RenderPipeline = RenderPipeline;
101    type ComputePipeline = ComputePipeline;
102    type RayTracingPipeline = RayTracingPipeline;
103}
104
105crate::impl_dyn_resource!(
106    Adapter,
107    AccelerationStructure,
108    BindGroup,
109    BindGroupLayout,
110    Buffer,
111    CommandBuffer,
112    CommandEncoder,
113    ComputePipeline,
114    Device,
115    Fence,
116    Instance,
117    PipelineCache,
118    PipelineLayout,
119    QuerySet,
120    Queue,
121    RenderPipeline,
122    RayTracingPipeline,
123    Sampler,
124    ShaderModule,
125    Surface,
126    SurfaceTexture,
127    Texture,
128    TextureView
129);
130
131struct DebugUtils {
132    extension: ext::debug_utils::Instance,
133    messenger: vk::DebugUtilsMessengerEXT,
134
135    /// Owning pointer to the debug messenger callback user data.
136    ///
137    /// `InstanceShared::drop` destroys the debug messenger before
138    /// dropping this, so the callback should never receive a dangling
139    /// user data pointer.
140    #[allow(dead_code)]
141    callback_data: Box<DebugUtilsMessengerUserData>,
142}
143
144#[derive(Debug)]
145pub struct DebugUtilsCreateInfo {
146    severity: vk::DebugUtilsMessageSeverityFlagsEXT,
147    message_type: vk::DebugUtilsMessageTypeFlagsEXT,
148    callback_data: Box<DebugUtilsMessengerUserData>,
149}
150
151#[derive(Debug)]
152/// The properties related to the validation layer needed for the
153/// DebugUtilsMessenger for their workarounds
154struct ValidationLayerProperties {
155    /// Validation layer description, from `vk::LayerProperties`.
156    layer_description: CString,
157
158    /// Validation layer specification version, from `vk::LayerProperties`.
159    layer_spec_version: u32,
160}
161
162/// User data needed by `instance::debug_utils_messenger_callback`.
163///
164/// When we create the [`vk::DebugUtilsMessengerEXT`], the `pUserData`
165/// pointer refers to one of these values.
166#[derive(Debug)]
167pub struct DebugUtilsMessengerUserData {
168    /// The properties related to the validation layer, if present
169    validation_layer_properties: Option<ValidationLayerProperties>,
170
171    /// If the OBS layer is present. OBS never increments the version of their layer,
172    /// so there's no reason to have the version.
173    has_obs_layer: bool,
174}
175
176pub struct InstanceShared {
177    raw: ash::Instance,
178    extensions: Vec<&'static CStr>,
179    flags: wgt::InstanceFlags,
180    memory_budget_thresholds: wgt::MemoryBudgetThresholds,
181    debug_utils: Option<DebugUtils>,
182    get_physical_device_properties: Option<khr::get_physical_device_properties2::Instance>,
183    entry: ash::Entry,
184    has_nv_optimus: bool,
185    android_sdk_version: u32,
186    /// The instance API version.
187    ///
188    /// Which is the version of Vulkan supported for instance-level functionality.
189    ///
190    /// It is associated with a `VkInstance` and its children,
191    /// except for a `VkPhysicalDevice` and its children.
192    instance_api_version: u32,
193
194    // The `drop_guard` field must be the last field of this struct so it is dropped last.
195    // Do not add new fields after it.
196    drop_guard: Option<crate::DropGuard>,
197}
198
199impl fmt::Debug for InstanceShared {
200    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
201        let Self {
202            raw: _,
203            extensions,
204            flags,
205            memory_budget_thresholds,
206            debug_utils: _,
207            get_physical_device_properties: _,
208            entry: _,
209            has_nv_optimus,
210            android_sdk_version,
211            instance_api_version,
212            drop_guard: _,
213        } = self;
214        f.debug_struct("InstanceShared")
215            .field("extensions", extensions)
216            .field("flags", flags)
217            .field("memory_budget_thresholds", memory_budget_thresholds)
218            .field("has_nv_optimus", has_nv_optimus)
219            .field("android_sdk_version", android_sdk_version)
220            .field("instance_api_version", instance_api_version)
221            .finish_non_exhaustive()
222    }
223}
224
225#[derive(Debug)]
226pub struct Instance {
227    shared: Arc<InstanceShared>,
228}
229
230#[expect(missing_debug_implementations, reason = "TODO?")]
231pub struct Surface {
232    swapchain: RwLock<Option<Box<dyn swapchain::Swapchain>>>,
233    inner: Box<dyn swapchain::Surface>,
234}
235
236impl Surface {
237    /// Returns the raw Vulkan surface handle.
238    ///
239    /// Returns `None` if the surface is a DXGI surface.
240    pub unsafe fn raw_native_handle(&self) -> Option<vk::SurfaceKHR> {
241        Some(
242            self.inner
243                .as_any()
244                .downcast_ref::<swapchain::NativeSurface>()?
245                .as_raw(),
246        )
247    }
248
249    /// Get the raw Vulkan swapchain associated with this surface.
250    ///
251    /// Returns [`None`] if the surface is not configured or if the swapchain
252    /// is a DXGI swapchain.
253    pub fn raw_native_swapchain(&self) -> Option<vk::SwapchainKHR> {
254        let read = self.swapchain.read();
255        Some(
256            read.as_ref()?
257                .as_any()
258                .downcast_ref::<swapchain::NativeSwapchain>()?
259                .as_raw(),
260        )
261    }
262
263    /// Set the present timing information which will be used for the next [presentation](crate::Queue::present()) of this surface,
264    /// using [VK_GOOGLE_display_timing].
265    ///
266    /// This can be used to give an id to presentations, for future use of [`vk::PastPresentationTimingGOOGLE`].
267    /// Note that `wgpu-hal` does *not* provide a way to use that API - you should manually access this through [`ash`].
268    ///
269    /// This can also be used to add a "not before" timestamp to the presentation.
270    ///
271    /// The exact semantics of the fields are also documented in the [specification](https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPresentTimeGOOGLE.html) for the extension.
272    ///
273    /// # Panics
274    ///
275    /// - If the surface hasn't been configured.
276    /// - If the surface has been configured for a DXGI swapchain.
277    /// - If the device doesn't have the [VK_GOOGLE_display_timing] extension enabled.
278    ///
279    /// [VK_GOOGLE_display_timing]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_GOOGLE_display_timing.html
280    #[track_caller]
281    pub fn set_next_present_time(&self, present_timing: vk::PresentTimeGOOGLE) {
282        let mut swapchain = self.swapchain.write();
283        swapchain
284            .as_mut()
285            .expect("Surface should have been configured")
286            .as_any_mut()
287            .downcast_mut::<swapchain::NativeSwapchain>()
288            .expect("Surface should have a native Vulkan swapchain")
289            .set_next_present_time(present_timing);
290    }
291
292    /// Set a `pNext` chain of extension structs to be attached to the [`vk::PresentInfoKHR`]
293    /// of the next [presentation](crate::Queue::present()) of this surface.
294    ///
295    /// This supports presentation extensions that `wgpu-hal` has no dedicated support
296    /// for, such as [VK_NV_present_metering]. The corresponding device extension must be
297    /// enabled at device creation, e.g. with
298    /// [`Adapter::open_with_callback()`](super::vulkan::Adapter::open_with_callback).
299    ///
300    /// The chain is consumed by the next presentation: it replaces any chain set by a
301    /// previous call that hasn't been presented yet, and is discarded unread if the
302    /// surface is reconfigured first.
303    ///
304    /// # Safety
305    ///
306    /// - `chain` must point to a valid `pNext` chain of structs that can extend
307    ///   [`vk::PresentInfoKHR`], whose extensions are enabled on the device.
308    /// - The chain must stay valid, and must not be read or written, until the next
309    ///   presentation of this surface completes or the surface is reconfigured. Fields
310    ///   the driver wrote during presentation may be read afterwards.
311    ///
312    /// # Panics
313    ///
314    /// - If the surface hasn't been configured.
315    /// - If the surface has been configured for a DXGI swapchain.
316    ///
317    /// [VK_NV_present_metering]: https://registry.khronos.org/vulkan/specs/latest/man/html/VK_NV_present_metering.html
318    #[track_caller]
319    pub unsafe fn set_next_present_chain(&self, chain: *mut c_void) {
320        let mut swapchain = self.swapchain.write();
321        let swapchain = swapchain
322            .as_mut()
323            .expect("Surface should have been configured")
324            .as_any_mut()
325            .downcast_mut::<swapchain::NativeSwapchain>()
326            .expect("Surface should have a native Vulkan swapchain");
327        unsafe { swapchain.set_next_present_chain(chain) };
328    }
329
330    /// Set a `pNext` chain of extension structs to attach to the
331    /// [`vk::SwapchainCreateInfoKHR`] used by the next configuration of this surface.
332    ///
333    /// This supports swapchain extensions that `wgpu-hal` has no dedicated support
334    /// for. One example is `VkSwapchainLatencyCreateInfoNV` from [VK_NV_low_latency2].
335    /// The device extension itself must be enabled at device creation, for example
336    /// with [`Adapter::open_with_callback()`](super::vulkan::Adapter::open_with_callback).
337    ///
338    /// The next configuration consumes the chain. A later call replaces a chain that
339    /// hasn't been consumed yet. If the surface is dropped first, the chain is
340    /// discarded unread. Every configuration creates a new swapchain, so you must set
341    /// the chain again before each configuration, including on resize.
342    ///
343    /// # Safety
344    ///
345    /// - `chain` must point to a valid `pNext` chain of structs that can extend
346    ///   [`vk::SwapchainCreateInfoKHR`]. The extensions in the chain must be enabled
347    ///   on the device the surface is configured with.
348    /// - The chain must stay valid until the configuration that consumes it returns
349    ///   or the surface is dropped. Don't read or write the chain during that time.
350    ///   Fields the driver wrote during swapchain creation may be read afterwards.
351    ///
352    /// # Panics
353    ///
354    /// - If the surface isn't a native Vulkan surface, such as a DXGI surface.
355    ///
356    /// [VK_NV_low_latency2]: https://registry.khronos.org/vulkan/specs/latest/man/html/VK_NV_low_latency2.html
357    #[track_caller]
358    pub unsafe fn set_next_swapchain_create_chain(&self, chain: *mut c_void) {
359        let surface = self
360            .inner
361            .as_any()
362            .downcast_ref::<swapchain::NativeSurface>()
363            .expect("Surface should be a native Vulkan surface");
364        unsafe { surface.set_next_swapchain_create_chain(chain) };
365    }
366}
367
368#[derive(Debug)]
369pub struct SurfaceTexture {
370    index: u32,
371    texture: Texture,
372    metadata: Box<dyn swapchain::SurfaceTextureMetadata>,
373}
374
375impl crate::DynSurfaceTexture for SurfaceTexture {}
376
377impl Borrow<Texture> for SurfaceTexture {
378    fn borrow(&self) -> &Texture {
379        &self.texture
380    }
381}
382
383impl Borrow<dyn crate::DynTexture> for SurfaceTexture {
384    fn borrow(&self) -> &dyn crate::DynTexture {
385        &self.texture
386    }
387}
388
389#[derive(Debug)]
390pub struct Adapter {
391    raw: vk::PhysicalDevice,
392    instance: Arc<InstanceShared>,
393    //queue_families: Vec<vk::QueueFamilyProperties>,
394    known_memory_flags: vk::MemoryPropertyFlags,
395    phd_capabilities: adapter::PhysicalDeviceProperties,
396    phd_features: PhysicalDeviceFeatures,
397    downlevel_flags: wgt::DownlevelFlags,
398    private_caps: PrivateCapabilities,
399    workarounds: Workarounds,
400}
401
402// TODO there's no reason why this can't be unified--the function pointers should all be the same--it's not clear how to do this with `ash`.
403enum ExtensionFn<T> {
404    /// The loaded function pointer struct for an extension.
405    Extension(T),
406    /// The extension was promoted to a core version of Vulkan and the functions on `ash`'s `DeviceV1_x` traits should be used.
407    Promoted,
408}
409
410struct DeviceExtensionFunctions {
411    debug_utils: Option<ext::debug_utils::Device>,
412    draw_indirect_count: Option<khr::draw_indirect_count::Device>,
413    timeline_semaphore: Option<ExtensionFn<khr::timeline_semaphore::Device>>,
414    ray_tracing: Option<RayTracingDeviceExtensionFunctions>,
415    ray_tracing_pipelines: Option<khr::ray_tracing_pipeline::Device>,
416    mesh_shading: Option<ext::mesh_shader::Device>,
417    #[cfg_attr(not(unix), allow(dead_code))]
418    external_memory_fd: Option<khr::external_memory_fd::Device>,
419}
420
421struct RayTracingDeviceExtensionFunctions {
422    acceleration_structure: khr::acceleration_structure::Device,
423    buffer_device_address: khr::buffer_device_address::Device,
424}
425
426/// Set of internal capabilities, which don't show up in the exposed
427/// device geometry, but affect the code paths taken internally.
428#[derive(Clone, Debug)]
429struct PrivateCapabilities {
430    image_view_usage: bool,
431    timeline_semaphores: bool,
432    texture_d24: bool,
433    texture_d24_s8: bool,
434    texture_s8: bool,
435    /// Ability to present contents to any screen. Only needed to work around broken platform configurations.
436    can_present: bool,
437    non_coherent_map_mask: wgt::BufferAddress,
438    multi_draw_indirect: bool,
439    max_draw_indirect_count: u32,
440
441    /// True if this adapter advertises the [`robustBufferAccess`][vrba] feature.
442    ///
443    /// Note that Vulkan's `robustBufferAccess` is not sufficient to implement
444    /// `wgpu_hal`'s guarantee that shaders will not access buffer contents via
445    /// a given bindgroup binding outside that binding's [accessible
446    /// region][ar]. Enabling `robustBufferAccess` does ensure that
447    /// out-of-bounds reads and writes are not undefined behavior (that's good),
448    /// but still permits out-of-bounds reads to return data from anywhere
449    /// within the buffer, not just the accessible region.
450    ///
451    /// [ar]: ../struct.BufferBinding.html#accessible-region
452    /// [vrba]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#features-robustBufferAccess
453    robust_buffer_access: bool,
454
455    robust_image_access: bool,
456
457    /// True if this adapter supports the [`VK_EXT_robustness2`] extension's
458    /// [`robustBufferAccess2`] feature.
459    ///
460    /// This is sufficient to implement `wgpu_hal`'s [required bounds-checking][ar] of
461    /// shader accesses to buffer contents. If this feature is not available,
462    /// this backend must have Naga inject bounds checks in the generated
463    /// SPIR-V.
464    ///
465    /// [`VK_EXT_robustness2`]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_robustness2.html
466    /// [`robustBufferAccess2`]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPhysicalDeviceRobustness2FeaturesEXT.html#features-robustBufferAccess2
467    /// [ar]: ../struct.BufferBinding.html#accessible-region
468    robust_buffer_access2: bool,
469
470    robust_image_access2: bool,
471    zero_initialize_workgroup_memory: bool,
472    image_format_list: bool,
473    maximum_samplers: u32,
474
475    /// True if this adapter supports the [`VK_KHR_shader_integer_dot_product`] extension
476    /// (promoted to Vulkan 1.3).
477    ///
478    /// This is used to generate optimized code for WGSL's `dot4{I, U}8Packed`.
479    ///
480    /// [`VK_KHR_shader_integer_dot_product`]: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_shader_integer_dot_product.html
481    shader_integer_dot_product: bool,
482
483    /// True if this adapter supports 8-bit integers provided by the
484    /// [`VK_KHR_shader_float16_int8`] extension (promoted to Vulkan 1.2).
485    ///
486    /// Allows shaders to declare the "Int8" capability. Note, however, that this
487    /// feature alone allows the use of 8-bit integers "only in the `Private`,
488    /// `Workgroup` (for non-Block variables), and `Function` storage classes"
489    /// ([see spec]). To use 8-bit integers in the interface storage classes (e.g.,
490    /// `StorageBuffer`), you also need to enable the corresponding feature in
491    /// `VkPhysicalDevice8BitStorageFeatures` and declare the corresponding SPIR-V
492    /// capability (e.g., `StorageBuffer8BitAccess`).
493    ///
494    /// [`VK_KHR_shader_float16_int8`]: https://registry.khronos.org/vulkan/specs/latest/man/html/VK_KHR_shader_float16_int8.html
495    /// [see spec]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkPhysicalDeviceShaderFloat16Int8Features.html#extension-features-shaderInt8
496    shader_int8: bool,
497
498    /// This is done to panic before undefined behavior, and is imperfect.
499    /// Basically, to allow implementations to emulate mv using instancing, if you
500    /// want to draw `n` instances to VR, you must draw `2n` instances, but you
501    /// can never draw more than `u32::MAX` instances. Therefore, when drawing
502    /// multiview on some vulkan implementations, it might restrict the instance
503    /// count, which isn't usually a thing in webgpu. We don't expose this limit
504    /// because its strange, i.e. only occurs on certain vulkan implementations
505    /// if you are drawing more than 128 million instances. We still want to avoid
506    /// undefined behavior in this situation, so we panic if the limit is violated.
507    multiview_instance_index_limit: u32,
508
509    /// BufferUsages::ACCELERATION_STRUCTURE_SCRATCH allows usage as a scratch buffer.
510    /// Vulkan has no way to specify this as a usage, and it maps to other usages, but
511    /// these usages do not have as high of an alignment requirement using the buffer as
512    ///  a scratch buffer when building acceleration structures.
513    scratch_buffer_alignment: u32,
514
515    /// Indicating that depth/stencil texturing operations with `VK_COMPONENT_SWIZZLE_ONE` have defined behavior.
516    depth_stencil_swizzle_one_support: bool,
517
518    /// `get_raytracing_pipeline_group_data` requires both a group count and a data size.
519    /// The data size parameter is just this * the group count, so we store this to not
520    /// require an unnecessary parameter.
521    ray_tracing_pipeline_group_data_size: u32,
522
523    /// `shaderGroupBaseAlignment`. Shader binding table regions must start at a device
524    /// address that is a multiple of this, so shader binding table buffers use it as their alignment.
525    shader_binding_table_alignment: u32,
526
527    /// Whether `VK_ATTACHMENT_STORE_OP_NONE` is supported. This includes:
528    /// - `VK_ATTACHMENT_STORE_OP_NONE` provided by `VK_VERSION_1_3`.
529    /// - `VK_ATTACHMENT_STORE_OP_NONE_KHR` provided by `VK_KHR_dynamic_rendering`, or `VK_KHR_load_store_op_none` (promoted to Vulkan 1.4).
530    /// - `VK_ATTACHMENT_STORE_OP_NONE_QCOM` provided by `VK_QCOM_render_pass_store_ops`.
531    /// - `VK_ATTACHMENT_STORE_OP_NONE_EXT` provided by `VK_EXT_load_store_op_none`.
532    store_op_none: bool,
533
534    /// True if `VK_KHR_incremental_present` is enabled on the device.
535    incremental_present: bool,
536}
537
538bitflags::bitflags!(
539    /// Workaround flags.
540    #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
541    pub struct Workarounds: u32 {
542        /// Only generate SPIR-V for one entry point at a time.
543        const SEPARATE_ENTRY_POINTS = 1 << 0;
544        /// Qualcomm OOMs when there are zero color attachments but a non-null pointer
545        /// to a subpass resolve attachment array. This nulls out that pointer in that case.
546        const EMPTY_RESOLVE_ATTACHMENT_LISTS = 1 << 1;
547        /// If the following code returns false, then nvidia will end up filling the wrong range.
548        ///
549        /// ```skip
550        /// fn nvidia_succeeds() -> bool {
551        ///   # let (copy_length, start_offset) = (0, 0);
552        ///     if copy_length >= 4096 {
553        ///         if start_offset % 16 != 0 {
554        ///             if copy_length == 4096 {
555        ///                 return true;
556        ///             }
557        ///             if copy_length % 16 == 0 {
558        ///                 return false;
559        ///             }
560        ///         }
561        ///     }
562        ///     true
563        /// }
564        /// ```
565        ///
566        /// As such, we need to make sure all calls to vkCmdFillBuffer are aligned to 16 bytes
567        /// if they cover a range of 4096 bytes or more.
568        const FORCE_FILL_BUFFER_WITH_SIZE_GREATER_4096_ALIGNED_OFFSET_16 = 1 << 2;
569        /// AMD's proprietary driver fails with VK_ERROR_INITIALIZATION_FAILED on mesh
570        /// and task shaders that zero-initialize workgroup memory with `OpVariable`
571        /// initializers, so polyfill it in those stages.
572        const POLYFILL_MESH_TASK_WORKGROUP_ZERO_INIT = 1 << 3;
573        /// Don't use timeline semaphores, even if the device supports them, and fall back
574        /// to binary fences ([`FencePool`]) for [`Fence`]s instead.
575        ///
576        /// Arm's proprietary Mali driver keeps memory for every value a timeline semaphore
577        /// has ever been signaled with, and only releases it when the semaphore is
578        /// destroyed. wgpu signals the device's timeline semaphore once per submission, so
579        /// the memory grows without bound until the kernel driver's limit is reached and the
580        /// device is lost. Observed with driver `v1.g18p0-01eac0` on a Mali-G610 (RK3588,
581        /// Android 12): 16 bytes per signal, device lost after ~4.2 million submissions,
582        /// i.e. ~35 minutes for a 960 FPS `Mailbox` loop or ~10 hours at 60 FPS.
583        ///
584        /// Set for Arm drivers older than `r46` and Rockchip drivers older than `g24`, and
585        /// for drivers whose version is not known.
586        ///
587        /// [`Fence`]: super::Fence
588        /// [`FencePool`]: super::FencePool
589        const DISABLE_TIMELINE_SEMAPHORES = 1 << 4;
590    }
591);
592
593#[derive(Clone, Debug, Eq, Hash, PartialEq)]
594struct AttachmentKey {
595    format: vk::Format,
596    layout: vk::ImageLayout,
597    ops: crate::AttachmentOps,
598}
599
600impl AttachmentKey {
601    /// Returns an attachment key for a compatible attachment.
602    fn compatible(format: vk::Format, layout: vk::ImageLayout) -> Self {
603        Self {
604            format,
605            layout,
606            ops: crate::AttachmentOps::all(),
607        }
608    }
609}
610
611#[derive(Clone, Eq, Hash, PartialEq)]
612struct ColorAttachmentKey {
613    base: AttachmentKey,
614    resolve: Option<AttachmentKey>,
615}
616
617#[derive(Clone, Eq, Hash, PartialEq)]
618struct DepthStencilAttachmentKey {
619    base: AttachmentKey,
620    stencil_ops: crate::AttachmentOps,
621}
622
623#[derive(Clone, Eq, Default, Hash, PartialEq)]
624struct RenderPassKey {
625    colors: ArrayVec<Option<ColorAttachmentKey>, { crate::MAX_COLOR_ATTACHMENTS }>,
626    depth_stencil: Option<DepthStencilAttachmentKey>,
627    sample_count: u32,
628    multiview_mask: Option<NonZeroU32>,
629    depth_read_only: bool,
630    stencil_read_only: bool,
631}
632
633struct DeviceShared {
634    raw: ash::Device,
635    family_index: u32,
636    queue_flags: vk::QueueFlags,
637    queue_index: u32,
638    raw_queue: vk::Queue,
639    instance: Arc<InstanceShared>,
640    physical_device: vk::PhysicalDevice,
641    enabled_extensions: Vec<&'static CStr>,
642    extension_fns: DeviceExtensionFunctions,
643    vendor_id: u32,
644    pipeline_cache_validation_key: [u8; 16],
645    timestamp_period: f32,
646    private_caps: PrivateCapabilities,
647    workarounds: Workarounds,
648    features: wgt::Features,
649    render_passes: Mutex<FastHashMap<RenderPassKey, vk::RenderPass>>,
650    sampler_cache: Mutex<sampler::SamplerCache>,
651    memory_allocations_counter: InternalCounter,
652
653    /// Because we have cached framebuffers which are not deleted from until
654    /// the device is destroyed, if the implementation of vulkan re-uses handles
655    /// we need some way to differentiate between the old handle and the new handle.
656    /// This factory allows us to have a dedicated identity value for each texture.
657    texture_identity_factory: ResourceIdentityFactory<vk::Image>,
658    /// As above, for texture views.
659    texture_view_identity_factory: ResourceIdentityFactory<vk::ImageView>,
660
661    empty_descriptor_set_layout: vk::DescriptorSetLayout,
662
663    // The `drop_guard` field must be the last field of this struct so it is dropped last.
664    // Do not add new fields after it.
665    drop_guard: Option<crate::DropGuard>,
666}
667
668impl Drop for DeviceShared {
669    fn drop(&mut self) {
670        for &raw in self.render_passes.lock().values() {
671            unsafe { self.raw.destroy_render_pass(raw, None) };
672        }
673        unsafe {
674            self.raw
675                .destroy_descriptor_set_layout(self.empty_descriptor_set_layout, None)
676        };
677        if self.drop_guard.is_none() {
678            unsafe { self.raw.destroy_device(None) };
679        }
680    }
681}
682
683#[expect(
684    missing_debug_implementations,
685    reason = "needs work to not be disastrously verbose"
686)]
687pub struct Device {
688    mem_allocator: MemoryAllocators,
689    desc_allocator: Mutex<descriptor::DescriptorAllocator>,
690    valid_ash_memory_types: u32,
691    naga_options: naga::back::spv::Options<'static>,
692    #[cfg(feature = "renderdoc")]
693    render_doc: crate::auxil::renderdoc::RenderDoc,
694    counters: Arc<wgt::HalCounters>,
695    // Struct members are dropped from first to last, put the Device last to ensure that
696    // all resources that depends on it are destroyed before it like the mem_allocator
697    shared: Arc<DeviceShared>,
698}
699
700impl Drop for Device {
701    fn drop(&mut self) {}
702}
703
704/// Semaphores for forcing queue submissions to run in order.
705///
706/// The [`wgpu_hal::Queue`] trait promises that if two calls to [`submit`] are
707/// ordered, then the first submission will finish on the GPU before the second
708/// submission begins. To get this behavior on Vulkan we need to pass semaphores
709/// to [`vkQueueSubmit`] for the commands to wait on before beginning execution,
710/// and to signal when their execution is done.
711///
712/// Normally this can be done with a single semaphore, waited on and then
713/// signalled for each submission. At any given time there's exactly one
714/// submission that would signal the semaphore, and exactly one waiting on it,
715/// as Vulkan requires.
716///
717/// However, as of Oct 2021, bug [#5508] in the Mesa ANV drivers caused them to
718/// hang if we use a single semaphore. The workaround is to alternate between
719/// two semaphores. The bug has been fixed in Mesa, but we should probably keep
720/// the workaround until, say, Oct 2026.
721///
722/// [`wgpu_hal::Queue`]: crate::Queue
723/// [`submit`]: crate::Queue::submit
724/// [`vkQueueSubmit`]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#vkQueueSubmit
725/// [#5508]: https://gitlab.freedesktop.org/mesa/mesa/-/issues/5508
726#[derive(Clone)]
727struct RelaySemaphores {
728    /// The semaphore the next submission should wait on before beginning
729    /// execution on the GPU. This is `None` for the first submission, which
730    /// should not wait on anything at all.
731    wait: Option<vk::Semaphore>,
732
733    /// The semaphore the next submission should signal when it has finished
734    /// execution on the GPU.
735    signal: vk::Semaphore,
736}
737
738impl RelaySemaphores {
739    fn new(device: &DeviceShared) -> Result<Self, crate::DeviceError> {
740        Ok(Self {
741            wait: None,
742            signal: device.new_binary_semaphore("RelaySemaphores: 1")?,
743        })
744    }
745
746    /// Advances the semaphores, returning the semaphores that should be used for a submission.
747    fn advance(&mut self, device: &DeviceShared) -> Result<Self, crate::DeviceError> {
748        let old = self.clone();
749
750        // Build the state for the next submission.
751        match self.wait {
752            None => {
753                // The `old` values describe the first submission to this queue.
754                // The second submission should wait on `old.signal`, and then
755                // signal a new semaphore which we'll create now.
756                self.wait = Some(old.signal);
757                self.signal = device.new_binary_semaphore("RelaySemaphores: 2")?;
758            }
759            Some(ref mut wait) => {
760                // What this submission signals, the next should wait.
761                mem::swap(wait, &mut self.signal);
762            }
763        };
764
765        Ok(old)
766    }
767
768    /// Destroys the semaphores.
769    unsafe fn destroy(&self, device: &ash::Device) {
770        unsafe {
771            if let Some(wait) = self.wait {
772                device.destroy_semaphore(wait, None);
773            }
774            device.destroy_semaphore(self.signal, None);
775        }
776    }
777}
778
779pub struct Queue {
780    raw: vk::Queue,
781    device: Arc<DeviceShared>,
782    family_index: u32,
783    relay_semaphores: Mutex<RelaySemaphores>,
784    signal_semaphores: Mutex<SemaphoreList>,
785    wait_semaphores: Mutex<SemaphoreList>,
786    /// A caller-provided `pNext` chain to attach to the [`vk::SubmitInfo`] of the
787    /// next call to [`submit()`](crate::Queue::submit).
788    ///
789    /// Set only through [`Queue::set_next_submit_chain()`].
790    next_submit_chain: Mutex<Option<PnextChain>>,
791}
792
793impl fmt::Debug for Queue {
794    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
795        let Self {
796            raw: _,
797            device: _,
798            family_index,
799            relay_semaphores: _,
800            signal_semaphores: _,
801            wait_semaphores: _,
802            next_submit_chain: _,
803        } = self;
804        f.debug_struct("Queue")
805            .field("family_index", family_index)
806            .finish_non_exhaustive()
807    }
808}
809
810impl Queue {
811    pub fn as_raw(&self) -> vk::Queue {
812        self.raw
813    }
814}
815
816impl Drop for Queue {
817    fn drop(&mut self) {
818        unsafe { self.relay_semaphores.lock().destroy(&self.device.raw) };
819    }
820}
821#[derive(Debug)]
822enum BufferMemoryBacking {
823    Managed(MemoryAllocation),
824    VulkanMemory {
825        memory: vk::DeviceMemory,
826        offset: u64,
827        size: u64,
828    },
829}
830impl BufferMemoryBacking {
831    fn memory(&self) -> vk::DeviceMemory {
832        match self {
833            Self::Managed(m) => unsafe { m.memory() },
834            Self::VulkanMemory { memory, .. } => *memory,
835        }
836    }
837    fn offset(&self) -> u64 {
838        match self {
839            Self::Managed(m) => m.offset(),
840            Self::VulkanMemory { offset, .. } => *offset,
841        }
842    }
843    fn size(&self) -> u64 {
844        match self {
845            Self::Managed(m) => m.size(),
846            Self::VulkanMemory { size, .. } => *size,
847        }
848    }
849}
850/// Describes who owns a [`Buffer`]'s `vk::Buffer` handle and its backing memory,
851/// and therefore what cleanup is required when the buffer is destroyed.
852#[derive(Debug)]
853enum BufferOwnership {
854    /// wgpu-hal owns the `vk::Buffer` and its backing memory. On cleanup the buffer
855    /// handle is destroyed and the memory is released.
856    Managed(Mutex<BufferMemoryBacking>),
857    /// wgpu-hal owns the `vk::Buffer` handle but the backing memory is kept alive
858    /// by the caller. On cleanup only the buffer handle is destroyed.
859    RawHandle,
860    /// Caller owns the `vk::Buffer` and its backing memory. On cleanup the
861    /// [`crate::DropGuard`] runs the caller's cleanup callback and wgpu-hal touches
862    /// neither the handle nor the memory.
863    External(crate::DropGuard),
864}
865
866#[derive(Debug)]
867pub struct Buffer {
868    raw: vk::Buffer,
869
870    // This field must be last, because it may contain a `DropGuard` which needs to be dropped after all other fields.
871    ownership: BufferOwnership,
872}
873impl Buffer {
874    /// # Safety
875    ///
876    /// - `vk_buffer`'s memory must be managed by the caller
877    /// - Externally imported buffers can't be mapped by `wgpu`
878    pub unsafe fn from_raw(vk_buffer: vk::Buffer) -> Self {
879        Self {
880            raw: vk_buffer,
881            ownership: BufferOwnership::RawHandle,
882        }
883    }
884
885    /// # Safety
886    /// - `vk_buffer` must outlive the returned `Buffer`.
887    /// - wgpu-hal will NOT call `vkDestroyBuffer`; the caller remains responsible for the buffer handle's destruction.
888    ///   The `drop_callback` runs when the `Buffer` drops and may be used to release caller-side bookkeeping.
889    /// - Externally imported buffers can't be mapped by `wgpu`.
890    pub unsafe fn from_raw_externally_owned(
891        vk_buffer: vk::Buffer,
892        drop_callback: crate::DropCallback,
893    ) -> Self {
894        Self {
895            raw: vk_buffer,
896            ownership: BufferOwnership::External(crate::DropGuard::new(drop_callback)),
897        }
898    }
899
900    /// # Safety
901    /// - We will use this buffer and the buffer's backing memory range as if we have exclusive ownership over it, until the wgpu resource is dropped and the wgpu-hal object is cleaned up
902    /// - Externally imported buffers can't be mapped by `wgpu`
903    /// - `offset` and `size` must be valid with the allocation of `memory`
904    pub unsafe fn from_raw_managed(
905        vk_buffer: vk::Buffer,
906        memory: vk::DeviceMemory,
907        offset: u64,
908        size: u64,
909    ) -> Self {
910        Self {
911            raw: vk_buffer,
912            ownership: BufferOwnership::Managed(Mutex::new(BufferMemoryBacking::VulkanMemory {
913                memory,
914                offset,
915                size,
916            })),
917        }
918    }
919
920    /// # Safety
921    /// - The buffer handle must not be manually destroyed
922    pub unsafe fn raw_handle(&self) -> vk::Buffer {
923        self.raw
924    }
925}
926
927impl crate::DynBuffer for Buffer {}
928
929#[derive(Debug)]
930pub struct AccelerationStructure {
931    raw: vk::AccelerationStructureKHR,
932    buffer: vk::Buffer,
933    allocation: MemoryAllocation,
934    compacted_size_query: Option<vk::QueryPool>,
935}
936
937impl crate::DynAccelerationStructure for AccelerationStructure {}
938
939impl AccelerationStructure {
940    /// Returns the raw Vulkan acceleration structure handle.
941    ///
942    /// This allows recording acceleration structure commands that `wgpu-hal` doesn't
943    /// support. If those commands come from a device extension, enable it with
944    /// [`Adapter::open_with_callback`].
945    ///
946    /// For a device address, use
947    /// [`crate::Device::get_acceleration_structure_device_address`] instead.
948    ///
949    /// `wgpu` doesn't observe an external build, and rejects an acceleration structure
950    /// that it never built. Call `wgpu::CommandEncoder::mark_acceleration_structures_built`
951    /// on the encoder that recorded the build.
952    ///
953    /// # Safety
954    ///
955    /// - The acceleration structure handle must not be manually destroyed, or used after
956    ///   the acceleration structure is dropped
957    /// - External work using the handle must be synchronized against `wgpu`'s own reads
958    ///   and writes, such as compaction
959    /// - An external build must not write more data than the size the acceleration
960    ///   structure was created with
961    pub unsafe fn raw_handle(&self) -> vk::AccelerationStructureKHR {
962        self.raw
963    }
964}
965
966#[derive(Debug)]
967pub enum TextureMemory {
968    // shared memory in GPU allocator (owned by wgpu-hal)
969    Allocation(MemoryAllocation),
970
971    // dedicated memory (owned by wgpu-hal)
972    Dedicated(vk::DeviceMemory),
973
974    // memory not owned by wgpu
975    External,
976}
977
978#[derive(Debug)]
979pub struct Texture {
980    raw: vk::Image,
981    memory: TextureMemory,
982    format: wgt::TextureFormat,
983    copy_size: crate::CopyExtent,
984    identity: ResourceIdentity<vk::Image>,
985
986    // The `drop_guard` field must be the last field of this struct so it is dropped last.
987    // Do not add new fields after it.
988    drop_guard: Option<crate::DropGuard>,
989}
990
991impl crate::DynTexture for Texture {}
992
993impl Texture {
994    /// # Safety
995    ///
996    /// - The image handle must not be manually destroyed
997    pub unsafe fn raw_handle(&self) -> vk::Image {
998        self.raw
999    }
1000
1001    /// # Safety
1002    ///
1003    /// - The caller must not free the `vk::DeviceMemory` or
1004    ///   `gpu_alloc::MemoryBlock` in the returned `TextureMemory`.
1005    pub unsafe fn memory(&self) -> &TextureMemory {
1006        &self.memory
1007    }
1008}
1009
1010#[derive(Debug)]
1011pub struct TextureView {
1012    raw_texture: vk::Image,
1013    raw: vk::ImageView,
1014    _layers: NonZeroU32,
1015    format: wgt::TextureFormat,
1016    raw_format: vk::Format,
1017    base_mip_level: u32,
1018    dimension: wgt::TextureViewDimension,
1019    texture_identity: ResourceIdentity<vk::Image>,
1020    view_identity: ResourceIdentity<vk::ImageView>,
1021}
1022
1023impl crate::DynTextureView for TextureView {}
1024
1025impl TextureView {
1026    /// # Safety
1027    ///
1028    /// - The image view handle must not be manually destroyed
1029    pub unsafe fn raw_handle(&self) -> vk::ImageView {
1030        self.raw
1031    }
1032
1033    /// Returns the raw texture view, along with its identity.
1034    fn identified_raw_view(&self) -> IdentifiedTextureView {
1035        IdentifiedTextureView {
1036            raw: self.raw,
1037            identity: self.view_identity,
1038        }
1039    }
1040}
1041
1042#[derive(Debug)]
1043pub struct Sampler {
1044    raw: vk::Sampler,
1045    create_info: vk::SamplerCreateInfo<'static>,
1046}
1047
1048impl crate::DynSampler for Sampler {}
1049
1050/// Information about a binding within a specific BindGroupLayout / BindGroup.
1051/// This will be used to construct a [`naga::back::spv::BindingInfo`], where
1052/// the descriptor set value will be taken from the index of the group.
1053#[derive(Copy, Clone, Debug)]
1054struct BindingInfo {
1055    binding: u32,
1056    binding_array_size: Option<NonZeroU32>,
1057    external_texture: bool,
1058}
1059
1060#[derive(Debug)]
1061pub struct BindGroupLayout {
1062    raw: vk::DescriptorSetLayout,
1063    desc_count: descriptor::DescriptorCounts,
1064    /// Sorted list of entries.
1065    entries: Box<[wgt::BindGroupLayoutEntry]>,
1066    /// Map of original binding index to remapped binding index and optional
1067    /// array size.
1068    binding_map: Vec<(u32, BindingInfo)>,
1069    contains_binding_arrays: bool,
1070}
1071
1072impl crate::DynBindGroupLayout for BindGroupLayout {}
1073
1074#[derive(Debug)]
1075pub struct PipelineLayout {
1076    raw: vk::PipelineLayout,
1077    binding_map: naga::back::spv::BindingMap,
1078}
1079
1080impl crate::DynPipelineLayout for PipelineLayout {}
1081
1082#[derive(Debug)]
1083pub struct BindGroup {
1084    set: descriptor::DescriptorSet,
1085}
1086
1087impl crate::DynBindGroup for BindGroup {}
1088
1089/// Miscellaneous allocation recycling pool for `CommandAllocator`.
1090#[derive(Default)]
1091struct Temp {
1092    marker: Vec<u8>,
1093    buffer_barriers: Vec<vk::BufferMemoryBarrier<'static>>,
1094    image_barriers: Vec<vk::ImageMemoryBarrier<'static>>,
1095}
1096
1097impl Temp {
1098    fn clear(&mut self) {
1099        self.marker.clear();
1100        self.buffer_barriers.clear();
1101        self.image_barriers.clear();
1102    }
1103
1104    fn make_c_str(&mut self, name: &str) -> &CStr {
1105        self.marker.clear();
1106        self.marker.extend_from_slice(name.as_bytes());
1107        self.marker.push(0);
1108        unsafe { CStr::from_bytes_with_nul_unchecked(&self.marker) }
1109    }
1110}
1111
1112/// Generates unique IDs for each resource of type `T`.
1113///
1114/// Because vk handles are not permanently unique, this
1115/// provides a way to generate unique IDs for each resource.
1116struct ResourceIdentityFactory<T> {
1117    #[cfg(not(target_has_atomic = "64"))]
1118    next_id: Mutex<u64>,
1119    #[cfg(target_has_atomic = "64")]
1120    next_id: wgpu_sync::atomic::AtomicU64,
1121    _phantom: PhantomData<T>,
1122}
1123
1124impl<T> ResourceIdentityFactory<T> {
1125    fn new() -> Self {
1126        Self {
1127            #[cfg(not(target_has_atomic = "64"))]
1128            next_id: Mutex::new(0),
1129            #[cfg(target_has_atomic = "64")]
1130            next_id: wgpu_sync::atomic::AtomicU64::new(0),
1131            _phantom: PhantomData,
1132        }
1133    }
1134
1135    /// Returns a new unique ID for a resource of type `T`.
1136    fn next(&self) -> ResourceIdentity<T> {
1137        #[cfg(not(target_has_atomic = "64"))]
1138        {
1139            let mut next_id = self.next_id.lock();
1140            let id = *next_id;
1141            *next_id += 1;
1142            ResourceIdentity {
1143                id,
1144                _phantom: PhantomData,
1145            }
1146        }
1147
1148        #[cfg(target_has_atomic = "64")]
1149        ResourceIdentity {
1150            id: self
1151                .next_id
1152                .fetch_add(1, core::sync::atomic::Ordering::Relaxed),
1153            _phantom: PhantomData,
1154        }
1155    }
1156}
1157
1158/// A unique identifier for a resource of type `T`.
1159///
1160/// This is used as a hashable key for resources, which
1161/// is permanently unique through the lifetime of the program.
1162#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq)]
1163struct ResourceIdentity<T> {
1164    id: u64,
1165    _phantom: PhantomData<T>,
1166}
1167
1168#[derive(Clone, Eq, Hash, PartialEq)]
1169struct FramebufferKey {
1170    raw_pass: vk::RenderPass,
1171    /// Because this is used as a key in a hash map, we need to include the identity
1172    /// so that this hashes differently, even if the ImageView handles are the same
1173    /// between different views.
1174    attachment_identities: ArrayVec<ResourceIdentity<vk::ImageView>, { MAX_TOTAL_ATTACHMENTS }>,
1175    /// While this is redundant for calculating the hash, we need access to an array
1176    /// of all the raw ImageViews when we are creating the actual framebuffer,
1177    /// so we store this here.
1178    attachment_views: ArrayVec<vk::ImageView, { MAX_TOTAL_ATTACHMENTS }>,
1179    extent: wgt::Extent3d,
1180}
1181
1182impl FramebufferKey {
1183    fn push_view(&mut self, view: IdentifiedTextureView) {
1184        self.attachment_identities.push(view.identity);
1185        self.attachment_views.push(view.raw);
1186    }
1187}
1188
1189/// A texture view paired with its identity.
1190#[derive(Copy, Clone)]
1191struct IdentifiedTextureView {
1192    raw: vk::ImageView,
1193    identity: ResourceIdentity<vk::ImageView>,
1194}
1195
1196#[derive(Clone, Eq, Hash, PartialEq)]
1197struct TempTextureViewKey {
1198    texture: vk::Image,
1199    /// As this is used in a hashmap, we need to
1200    /// include the identity so that this hashes differently,
1201    /// even if the Image handles are the same between different images.
1202    texture_identity: ResourceIdentity<vk::Image>,
1203    format: vk::Format,
1204    mip_level: u32,
1205    depth_slice: u32,
1206}
1207
1208// Any state in this struct that may be dirty after an abandoned encoding must
1209// be reset for reused encoders in `begin_encoding`.
1210pub struct CommandEncoder {
1211    raw: vk::CommandPool,
1212    device: Arc<DeviceShared>,
1213
1214    /// The current command buffer, if `self` is in the ["recording"]
1215    /// state.
1216    ///
1217    /// ["recording"]: crate::CommandEncoder
1218    ///
1219    /// If non-`null`, the buffer is in the Vulkan "recording" state.
1220    active: vk::CommandBuffer,
1221
1222    /// What kind of pass we are currently within: compute or render.
1223    bind_point: vk::PipelineBindPoint,
1224
1225    /// Allocation recycling pool for this encoder.
1226    temp: Temp,
1227
1228    /// A pool of available command buffers.
1229    ///
1230    /// These are all in the Vulkan "initial" state.
1231    free: Vec<vk::CommandBuffer>,
1232
1233    /// A pool of discarded command buffers.
1234    ///
1235    /// These could be in any Vulkan state except "pending".
1236    discarded: Vec<vk::CommandBuffer>,
1237
1238    /// If this is true, the active renderpass enabled a debug span,
1239    /// and needs to be disabled on renderpass close.
1240    rpass_debug_marker_active: bool,
1241
1242    /// If set, the end of the next render/compute pass will write a timestamp at
1243    /// the given pool & location.
1244    end_of_pass_timer_query: Option<(vk::QueryPool, u32)>,
1245
1246    framebuffers: FastHashMap<FramebufferKey, vk::Framebuffer>,
1247    temp_texture_views: FastHashMap<TempTextureViewKey, IdentifiedTextureView>,
1248
1249    counters: Arc<wgt::HalCounters>,
1250
1251    current_pipeline_is_multiview: bool,
1252}
1253
1254impl Drop for CommandEncoder {
1255    fn drop(&mut self) {
1256        // SAFETY:
1257        //
1258        // VUID-vkDestroyCommandPool-commandPool-00041: wgpu_hal requires that a
1259        // `CommandBuffer` must live until its execution is complete, and that a
1260        // `CommandBuffer` must not outlive the `CommandEncoder` that built it.
1261        // Thus, we know that none of our `CommandBuffers` are in the "pending"
1262        // state.
1263        //
1264        // The other VUIDs are pretty obvious.
1265        unsafe {
1266            // `vkDestroyCommandPool` also frees any command buffers allocated
1267            // from that pool, so there's no need to explicitly call
1268            // `vkFreeCommandBuffers` on `cmd_encoder`'s `free` and `discarded`
1269            // fields.
1270            self.device.raw.destroy_command_pool(self.raw, None);
1271        }
1272
1273        for (_, fb) in self.framebuffers.drain() {
1274            unsafe { self.device.raw.destroy_framebuffer(fb, None) };
1275        }
1276
1277        for (_, view) in self.temp_texture_views.drain() {
1278            unsafe { self.device.raw.destroy_image_view(view.raw, None) };
1279        }
1280
1281        self.counters.command_encoders.sub(1);
1282    }
1283}
1284
1285impl CommandEncoder {
1286    /// # Safety
1287    ///
1288    /// - The command buffer handle must not be manually destroyed
1289    pub unsafe fn raw_handle(&self) -> vk::CommandBuffer {
1290        self.active
1291    }
1292}
1293
1294impl fmt::Debug for CommandEncoder {
1295    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1296        f.debug_struct("CommandEncoder")
1297            .field("raw", &self.raw)
1298            .finish()
1299    }
1300}
1301
1302#[derive(Debug)]
1303pub struct CommandBuffer {
1304    raw: vk::CommandBuffer,
1305}
1306
1307impl crate::DynCommandBuffer for CommandBuffer {}
1308
1309#[derive(Debug)]
1310pub enum ShaderModule {
1311    Raw(vk::ShaderModule),
1312    Intermediate {
1313        naga_shader: crate::NagaShader,
1314        runtime_checks: wgt::ShaderRuntimeChecks,
1315    },
1316}
1317
1318impl crate::DynShaderModule for ShaderModule {}
1319
1320#[derive(Debug)]
1321pub struct RenderPipeline {
1322    raw: vk::Pipeline,
1323    is_multiview: bool,
1324}
1325
1326impl crate::DynRenderPipeline for RenderPipeline {}
1327
1328#[derive(Debug)]
1329pub struct ComputePipeline {
1330    raw: vk::Pipeline,
1331}
1332
1333impl crate::DynComputePipeline for ComputePipeline {}
1334
1335#[derive(Debug)]
1336pub struct RayTracingPipeline {
1337    raw: vk::Pipeline,
1338}
1339
1340impl crate::DynRayTracingPipeline for RayTracingPipeline {}
1341
1342#[derive(Debug)]
1343pub struct PipelineCache {
1344    raw: vk::PipelineCache,
1345}
1346
1347impl crate::DynPipelineCache for PipelineCache {}
1348
1349#[derive(Debug)]
1350pub struct QuerySet {
1351    raw: vk::QueryPool,
1352}
1353
1354impl crate::DynQuerySet for QuerySet {}
1355
1356/// The [`Api::Fence`] type for [`vulkan::Api`].
1357///
1358/// This is an `enum` because there are two possible implementations of
1359/// `wgpu-hal` fences on Vulkan: Vulkan fences, which work on any version of
1360/// Vulkan, and Vulkan timeline semaphores, which are easier and cheaper but
1361/// require non-1.0 features.
1362///
1363/// [`Device::create_fence`] returns a [`TimelineSemaphore`] if
1364/// [`VK_KHR_timeline_semaphore`] is available and enabled, and a [`FencePool`]
1365/// otherwise.
1366///
1367/// [`Api::Fence`]: crate::Api::Fence
1368/// [`vulkan::Api`]: Api
1369/// [`Device::create_fence`]: crate::Device::create_fence
1370/// [`TimelineSemaphore`]: Fence::TimelineSemaphore
1371/// [`VK_KHR_timeline_semaphore`]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#VK_KHR_timeline_semaphore
1372/// [`FencePool`]: Fence::FencePool
1373#[derive(Debug)]
1374pub enum Fence {
1375    /// A Vulkan [timeline semaphore].
1376    ///
1377    /// These are simpler to use than Vulkan fences, since timeline semaphores
1378    /// work exactly the way [`wpgu_hal::Api::Fence`] is specified to work.
1379    ///
1380    /// [timeline semaphore]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#synchronization-semaphores
1381    /// [`wpgu_hal::Api::Fence`]: crate::Api::Fence
1382    TimelineSemaphore(vk::Semaphore),
1383
1384    /// A collection of Vulkan [fence]s, each associated with a [`FenceValue`].
1385    ///
1386    /// The effective [`FenceValue`] of this variant is the greater of
1387    /// `last_completed` and the maximum value associated with a signalled fence
1388    /// in `active`.
1389    ///
1390    /// Fences are available in all versions of Vulkan, but since they only have
1391    /// two states, "signaled" and "unsignaled", we need to use a separate fence
1392    /// for each queue submission we might want to wait for, and remember which
1393    /// [`FenceValue`] each one represents.
1394    ///
1395    /// One should keep the fence pool read while there are any references to the
1396    /// fences inside of them. This ensures there are no race conditions when
1397    /// resetting the fences
1398    ///
1399    /// [fence]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#synchronization-fences
1400    /// [`FenceValue`]: crate::FenceValue
1401    FencePool(RwLock<FencePool>),
1402}
1403
1404/// A shared fence type. The arc is expect to have a ref-count of one once a function has finished being called
1405///
1406/// A fence should have access synchronised as fence resetting might happen at any point. Resetting checks the ref-count
1407/// of the fence, so instead of copying the fence, it should have its `Arc` container cloned which shows not to reset
1408/// this fence as it is being used.
1409pub(super) type SynchronizedFence = Arc<vk::Fence>;
1410
1411#[derive(Debug)]
1412pub struct FencePool {
1413    last_completed: crate::FenceValue,
1414    /// The pending fence values have to be ascending.
1415    active: Vec<(crate::FenceValue, SynchronizedFence)>,
1416    // Don't need extra synchronisation around the fences here, if they are used they should be put into active.
1417    free: Vec<vk::Fence>,
1418}
1419
1420impl crate::DynFence for Fence {}
1421
1422impl Fence {
1423    /// Return the highest [`FenceValue`] among the signalled fences in `active`.
1424    ///
1425    /// As an optimization, assume that we already know that the fence has
1426    /// reached `last_completed`, and don't bother checking fences whose values
1427    /// are less than that: those fences remain in the `active` array only
1428    /// because we haven't called `maintain` yet to clean them up.
1429    ///
1430    /// [`FenceValue`]: crate::FenceValue
1431    fn check_active(
1432        device: &ash::Device,
1433        mut last_completed: crate::FenceValue,
1434        active: &[(crate::FenceValue, SynchronizedFence)],
1435    ) -> Result<crate::FenceValue, crate::DeviceError> {
1436        for &(value, ref raw) in active.iter() {
1437            unsafe {
1438                if value > last_completed
1439                    && device
1440                        // Don't need to clone as active should be from a read or
1441                        // write lock which means this is already synchronised.
1442                        .get_fence_status(**raw)
1443                        .map_err(map_host_device_oom_and_lost_err)?
1444                {
1445                    last_completed = value;
1446                }
1447            }
1448        }
1449        Ok(last_completed)
1450    }
1451
1452    /// Return the highest signalled [`FenceValue`] for `self`.
1453    ///
1454    /// [`FenceValue`]: crate::FenceValue
1455    fn get_latest(
1456        &self,
1457        device: &ash::Device,
1458        extension: Option<&ExtensionFn<khr::timeline_semaphore::Device>>,
1459    ) -> Result<crate::FenceValue, crate::DeviceError> {
1460        match *self {
1461            Self::TimelineSemaphore(raw) => unsafe {
1462                Ok(match *extension.unwrap() {
1463                    ExtensionFn::Extension(ref ext) => ext
1464                        .get_semaphore_counter_value(raw)
1465                        .map_err(map_host_device_oom_and_lost_err)?,
1466                    ExtensionFn::Promoted => device
1467                        .get_semaphore_counter_value(raw)
1468                        .map_err(map_host_device_oom_and_lost_err)?,
1469                })
1470            },
1471            Self::FencePool(ref pool) => {
1472                let FencePool {
1473                    last_completed,
1474                    ref active,
1475                    free: _,
1476                } = *pool.read();
1477                Self::check_active(device, last_completed, active)
1478            }
1479        }
1480    }
1481
1482    /// Trim the internal state of this [`Fence`].
1483    ///
1484    /// This function has no externally visible effect, but you should call it
1485    /// periodically to keep this fence's resource consumption under control.
1486    ///
1487    /// For fences using the [`FencePool`] implementation, this function
1488    /// recycles fences that have been signaled. If you don't call this,
1489    /// [`Queue::submit`] will just keep allocating a new Vulkan fence every
1490    /// time it's called.
1491    ///
1492    /// [`FencePool`]: Fence::FencePool
1493    /// [`Queue::submit`]: crate::Queue::submit
1494    fn maintain(&self, device: &ash::Device) -> Result<(), crate::DeviceError> {
1495        match *self {
1496            Self::TimelineSemaphore(_) => {}
1497            Self::FencePool(ref pool) => {
1498                let FencePool {
1499                    ref mut last_completed,
1500                    ref mut active,
1501                    ref mut free,
1502                } = *pool.write();
1503
1504                let base_free = free.len();
1505                let latest = Self::check_active(device, *last_completed, active)?;
1506
1507                active.retain_mut(|&mut (value, ref mut fence)| {
1508                    if value > latest {
1509                        true
1510                    } else if let Some(fence) = Arc::get_mut(fence) {
1511                        // No other references to these, so we have exclusive access. Add them to free and reset them later,
1512                        // but drop them from active immediately
1513                        free.push(*fence);
1514                        false
1515                    } else {
1516                        // some other function is using it. Although this shouldn't be to long,
1517                        // maintain shouldn't block, and it should be cleared up by the next time it happens
1518                        true
1519                    }
1520                });
1521
1522                if free.len() != base_free {
1523                    unsafe { device.reset_fences(&free[base_free..]) }
1524                        .map_err(map_device_oom_err)?
1525                }
1526                *last_completed = latest;
1527            }
1528        }
1529        Ok(())
1530    }
1531}
1532
1533impl crate::Queue for Queue {
1534    type A = Api;
1535
1536    unsafe fn submit(
1537        &self,
1538        command_buffers: &[&CommandBuffer],
1539        surface_textures: &[&SurfaceTexture],
1540        (signal_fence, signal_value): (&Fence, crate::FenceValue),
1541    ) -> Result<(), crate::DeviceError> {
1542        let mut fence_raw = vk::Fence::null();
1543
1544        let mut wait_semaphores = SemaphoreList::new(SemaphoreListMode::Wait);
1545        let mut signal_semaphores = SemaphoreList::new(SemaphoreListMode::Signal);
1546
1547        // Assert that we will not deadlock as we try to lock each `SurfaceTexture`'s
1548        // metadata.
1549        //
1550        // The documentation for [`wgpu_hal::Queue::submit`] requires that
1551        // `surface_textures` must not contain duplicates, so simply iterating over
1552        // the slice and locking each one should be fine.
1553        debug_assert!(
1554            {
1555                let mut check = HashSet::with_capacity(surface_textures.len());
1556                // We compare the Box by pointer, as Eq isn't well defined for SurfaceSemaphores.
1557                for st in surface_textures {
1558                    let ptr: *const () = <*const _>::cast(&*st.metadata);
1559                    check.insert(ptr as usize);
1560                }
1561                check.len() == surface_textures.len()
1562            },
1563            "More than one surface texture is being used from the same swapchain. This will cause a deadlock in release."
1564        );
1565
1566        let locked_swapchain_semaphores = surface_textures
1567            .iter()
1568            .map(|st| st.metadata.get_semaphore_guard())
1569            .collect::<Vec<_>>();
1570
1571        for mut semaphores in locked_swapchain_semaphores {
1572            semaphores.set_used_fence_value(signal_value);
1573
1574            // If we're the first submission to operate on this image, wait on
1575            // its acquire semaphore, to make sure the presentation engine is
1576            // done with it.
1577            if let Some(sem) = semaphores.get_acquire_wait_semaphore() {
1578                wait_semaphores.push_wait(sem, vk::PipelineStageFlags::TOP_OF_PIPE);
1579            }
1580
1581            // Get a semaphore to signal when we're done writing to this surface
1582            // image. Presentation of this image will wait for this.
1583            let signal_semaphore = semaphores.get_submit_signal_semaphore(&self.device)?;
1584            signal_semaphores.push_signal(signal_semaphore);
1585        }
1586
1587        let mut guard = self.signal_semaphores.lock();
1588        if !guard.is_empty() {
1589            signal_semaphores.append(&mut guard);
1590        }
1591
1592        let mut wait_guard = self.wait_semaphores.lock();
1593        if !wait_guard.is_empty() {
1594            wait_semaphores.append(&mut wait_guard);
1595        }
1596
1597        // In order for submissions to be strictly ordered, we encode a dependency between each submission
1598        // using a pair of semaphores. This adds a wait if it is needed, and signals the next semaphore.
1599        let semaphore_state = self.relay_semaphores.lock().advance(&self.device)?;
1600
1601        if let Some(sem) = semaphore_state.wait {
1602            wait_semaphores.push_wait(
1603                SemaphoreType::Binary(sem),
1604                vk::PipelineStageFlags::TOP_OF_PIPE,
1605            );
1606        }
1607
1608        signal_semaphores.push_signal(SemaphoreType::Binary(semaphore_state.signal));
1609
1610        // We need to signal our wgpu::Fence if we have one, this adds it to the signal list.
1611        signal_fence.maintain(&self.device.raw)?;
1612        // `vkQueueSubmit` requires its fence to be externally synchronized, and other threads
1613        // access the fences in `active` (for example with `vkGetFenceStatus`) while holding the
1614        // pool's lock. Holding the lock across the submit keeps them off the new fence.
1615        let mut pool_guard = None;
1616        match *signal_fence {
1617            Fence::TimelineSemaphore(raw) => {
1618                signal_semaphores.push_signal(SemaphoreType::Timeline(raw, signal_value));
1619            }
1620            Fence::FencePool(ref pool) => {
1621                let locked_pool = pool_guard.insert(pool.write());
1622                let FencePool {
1623                    ref mut active,
1624                    ref mut free,
1625                    ..
1626                } = **locked_pool;
1627                let shared_fence = match free.pop() {
1628                    Some(raw) => Arc::new(raw),
1629                    None => unsafe {
1630                        let fence = self
1631                            .device
1632                            .raw
1633                            .create_fence(&vk::FenceCreateInfo::default(), None)
1634                            .map_err(map_host_device_oom_err)?;
1635                        Arc::new(fence)
1636                    },
1637                };
1638                fence_raw = *shared_fence;
1639                active.push((signal_value, shared_fence));
1640            }
1641        }
1642
1643        let vk_cmd_buffers = command_buffers
1644            .iter()
1645            .map(|cmd| cmd.raw)
1646            .collect::<Vec<_>>();
1647
1648        let mut vk_info = vk::SubmitInfo::default().command_buffers(&vk_cmd_buffers);
1649        let mut vk_timeline_info = mem::MaybeUninit::uninit();
1650        vk_info = SemaphoreList::add_to_submit(
1651            &mut wait_semaphores,
1652            &mut signal_semaphores,
1653            vk_info,
1654            &mut vk_timeline_info,
1655        );
1656
1657        let submit_chain = self.next_submit_chain.lock().take();
1658        if let Some(chain) = submit_chain {
1659            // SAFETY: The contract on `Queue::set_next_submit_chain()` keeps the chain
1660            // valid and unaliased until this submission is made.
1661            vk_info.p_next = unsafe { chain.splice_into(vk_info.p_next) };
1662        }
1663
1664        profiling::scope!("vkQueueSubmit");
1665        unsafe {
1666            self.device
1667                .raw
1668                .queue_submit(self.raw, &[vk_info], fence_raw)
1669                .map_err(map_host_device_oom_and_lost_err)?
1670        };
1671        Ok(())
1672    }
1673
1674    unsafe fn present(
1675        &self,
1676        surface: &Surface,
1677        texture: SurfaceTexture,
1678        damage_rects: &[wgt::Rect<u32>],
1679    ) -> Result<(), crate::SurfaceError> {
1680        let mut swapchain = surface.swapchain.write();
1681
1682        unsafe {
1683            swapchain
1684                .as_mut()
1685                .unwrap()
1686                .present(self, texture, damage_rects)
1687        }
1688    }
1689
1690    unsafe fn get_timestamp_period(&self) -> f32 {
1691        self.device.timestamp_period
1692    }
1693
1694    unsafe fn wait_for_idle(&self) -> Result<(), crate::DeviceError> {
1695        unsafe { self.device.raw.queue_wait_idle(self.raw) }
1696            .map_err(map_host_device_oom_and_lost_err)
1697    }
1698}
1699
1700impl Queue {
1701    pub fn raw_device(&self) -> &ash::Device {
1702        &self.device.raw
1703    }
1704
1705    pub fn add_signal_semaphore(&self, semaphore: vk::Semaphore, semaphore_value: Option<u64>) {
1706        let mut guard = self.signal_semaphores.lock();
1707        if let Some(value) = semaphore_value {
1708            guard.push_signal(SemaphoreType::Timeline(semaphore, value));
1709        } else {
1710            guard.push_signal(SemaphoreType::Binary(semaphore));
1711        }
1712    }
1713
1714    /// Remove `semaphore` from the pending signal list if it is still present.
1715    ///
1716    /// Returns `true` if the semaphore was found and removed. If the submit
1717    /// already consumed it, this is a harmless no-op that returns `false`.
1718    pub fn remove_signal_semaphore(&self, semaphore: vk::Semaphore) -> bool {
1719        self.signal_semaphores.lock().remove(semaphore)
1720    }
1721
1722    /// Stage a semaphore wait on the next [`crate::Queue::submit`] call.
1723    ///
1724    /// `semaphore_value` selects the kind of payload the wait targets:
1725    ///
1726    /// - `Some(value)` - wait until `semaphore` (a timeline semaphore) has been signalled to at least `value`.
1727    /// - `None` - wait on a binary semaphore signal.
1728    ///
1729    /// `stage` is the pipeline stage at which the wait blocks downstream
1730    /// work (e.g. `vk::PipelineStageFlags::TOP_OF_PIPE` to gate the
1731    /// entire submission, or a more specific stage when only that stage
1732    /// reads the synchronised resource).
1733    pub fn add_wait_semaphore(
1734        &self,
1735        semaphore: vk::Semaphore,
1736        semaphore_value: Option<u64>,
1737        stage: vk::PipelineStageFlags,
1738    ) {
1739        let mut guard = self.wait_semaphores.lock();
1740        if let Some(value) = semaphore_value {
1741            guard.push_wait(SemaphoreType::Timeline(semaphore, value), stage);
1742        } else {
1743            guard.push_wait(SemaphoreType::Binary(semaphore), stage);
1744        }
1745    }
1746
1747    /// Remove `semaphore` from the pending wait list if it is still present.
1748    ///
1749    /// Returns `true` if the semaphore was found and removed. If the submit
1750    /// already consumed it, this is a no-op that returns `false`.
1751    pub fn remove_wait_semaphore(&self, semaphore: vk::Semaphore) -> bool {
1752        self.wait_semaphores.lock().remove(semaphore)
1753    }
1754
1755    /// Set a `pNext` chain of extension structs to attach to the [`vk::SubmitInfo`]
1756    /// of the next [`submit()`](crate::Queue::submit) on this queue.
1757    ///
1758    /// This supports submission extensions that `wgpu-hal` has no dedicated support
1759    /// for. One example is `VkLatencySubmissionPresentIdNV` from `VK_NV_low_latency2`.
1760    /// The device extension itself must be enabled at device creation, for example
1761    /// with [`Adapter::open_with_callback()`](super::vulkan::Adapter::open_with_callback).
1762    ///
1763    /// The next submission on this queue consumes the chain, and a later call
1764    /// replaces a chain that hasn't been submitted yet. One `wgpu` or `wgpu-core`
1765    /// submit maps to one submission here, but layers above `wgpu-hal` also make
1766    /// housekeeping submissions of their own, for example when a buffer is mapped,
1767    /// and those consume the chain too. Set the chain right before the submission
1768    /// it applies to, with no other queue or buffer-mapping work in between.
1769    ///
1770    /// # Safety
1771    ///
1772    /// - `chain` must point to a valid `pNext` chain of structs that can extend
1773    ///   [`vk::SubmitInfo`]. The extensions in the chain must be enabled on this
1774    ///   queue's device.
1775    /// - The chain must stay valid until the next submission on this queue is made
1776    ///   or the queue is dropped. Don't read or write the chain during that time.
1777    ///   Fields the driver wrote during submission may be read afterwards.
1778    pub unsafe fn set_next_submit_chain(&self, chain: *mut c_void) {
1779        *self.next_submit_chain.lock() = Some(PnextChain::new(chain));
1780    }
1781}
1782
1783/// Maps
1784///
1785/// - VK_ERROR_OUT_OF_HOST_MEMORY
1786/// - VK_ERROR_OUT_OF_DEVICE_MEMORY
1787fn map_host_device_oom_err(err: vk::Result) -> crate::DeviceError {
1788    match err {
1789        vk::Result::ERROR_OUT_OF_HOST_MEMORY | vk::Result::ERROR_OUT_OF_DEVICE_MEMORY => {
1790            get_oom_err(err)
1791        }
1792        e => get_unexpected_err(e),
1793    }
1794}
1795
1796/// Maps
1797///
1798/// - VK_ERROR_OUT_OF_HOST_MEMORY
1799/// - VK_ERROR_OUT_OF_DEVICE_MEMORY
1800/// - VK_ERROR_DEVICE_LOST
1801fn map_host_device_oom_and_lost_err(err: vk::Result) -> crate::DeviceError {
1802    match err {
1803        vk::Result::ERROR_DEVICE_LOST => get_lost_err(),
1804        other => map_host_device_oom_err(other),
1805    }
1806}
1807
1808/// Maps
1809///
1810/// - VK_ERROR_OUT_OF_HOST_MEMORY
1811/// - VK_ERROR_OUT_OF_DEVICE_MEMORY
1812/// - VK_ERROR_FRAGMENTATION
1813fn map_host_device_oom_and_fragmentation_err(err: vk::Result) -> crate::DeviceError {
1814    match err {
1815        vk::Result::ERROR_FRAGMENTATION => get_oom_err(err),
1816        other => map_host_device_oom_err(other),
1817    }
1818}
1819
1820/// Maps
1821///
1822/// - VK_ERROR_OUT_OF_HOST_MEMORY
1823/// - VK_ERROR_OUT_OF_DEVICE_MEMORY
1824/// - VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
1825fn map_host_device_oom_and_ioca_err(err: vk::Result) -> crate::DeviceError {
1826    // We don't use VK_KHR_buffer_device_address
1827    // VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
1828    map_host_device_oom_err(err)
1829}
1830
1831/// Maps
1832///
1833/// - VK_ERROR_OUT_OF_HOST_MEMORY
1834fn map_host_oom_err(err: vk::Result) -> crate::DeviceError {
1835    match err {
1836        vk::Result::ERROR_OUT_OF_HOST_MEMORY => get_oom_err(err),
1837        e => get_unexpected_err(e),
1838    }
1839}
1840
1841/// Maps
1842///
1843/// - VK_ERROR_OUT_OF_DEVICE_MEMORY
1844fn map_device_oom_err(err: vk::Result) -> crate::DeviceError {
1845    match err {
1846        vk::Result::ERROR_OUT_OF_DEVICE_MEMORY => get_oom_err(err),
1847        e => get_unexpected_err(e),
1848    }
1849}
1850
1851/// Maps
1852///
1853/// - VK_ERROR_OUT_OF_HOST_MEMORY
1854/// - VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
1855fn map_host_oom_and_ioca_err(err: vk::Result) -> crate::DeviceError {
1856    // We don't use VK_KHR_buffer_device_address
1857    // VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
1858    map_host_oom_err(err)
1859}
1860
1861/// Maps
1862///
1863/// - VK_ERROR_OUT_OF_HOST_MEMORY
1864/// - VK_ERROR_OUT_OF_DEVICE_MEMORY
1865/// - VK_PIPELINE_COMPILE_REQUIRED_EXT
1866/// - VK_ERROR_INVALID_SHADER_NV
1867fn map_pipeline_err(err: vk::Result) -> crate::DeviceError {
1868    // We don't use VK_EXT_pipeline_creation_cache_control
1869    // VK_PIPELINE_COMPILE_REQUIRED_EXT
1870    // We don't use VK_NV_glsl_shader
1871    // VK_ERROR_INVALID_SHADER_NV
1872    map_host_device_oom_err(err)
1873}
1874
1875/// Returns [`crate::DeviceError::Unexpected`] or panics if the `internal_error_panic`
1876/// feature flag is enabled.
1877fn get_unexpected_err(_err: vk::Result) -> crate::DeviceError {
1878    #[cfg(feature = "internal_error_panic")]
1879    panic!("Unexpected Vulkan error: {_err:?}");
1880
1881    #[allow(unreachable_code)]
1882    crate::DeviceError::Unexpected
1883}
1884
1885/// Returns [`crate::DeviceError::OutOfMemory`].
1886fn get_oom_err(_err: vk::Result) -> crate::DeviceError {
1887    crate::DeviceError::OutOfMemory
1888}
1889
1890/// Returns [`crate::DeviceError::Lost`] or panics if the `device_lost_panic`
1891/// feature flag is enabled.
1892fn get_lost_err() -> crate::DeviceError {
1893    #[cfg(feature = "device_lost_panic")]
1894    panic!("Device lost");
1895
1896    #[allow(unreachable_code)]
1897    crate::DeviceError::Lost
1898}
1899
1900#[derive(Clone, Copy, Pod, Zeroable)]
1901#[repr(C)]
1902struct RawTlasInstance {
1903    transform: [f32; 12],
1904    custom_data_and_mask: u32,
1905    shader_binding_table_record_offset_and_flags: u32,
1906    acceleration_structure_reference: u64,
1907}
1908
1909/// Arguments to the [`CreateDeviceCallback`].
1910#[derive(Debug)]
1911pub struct CreateDeviceCallbackArgs<'arg, 'pnext, 'this>
1912where
1913    'this: 'pnext,
1914{
1915    /// The extensions to enable for the device. You must not remove anything from this list,
1916    /// but you may add to it.
1917    pub extensions: &'arg mut Vec<&'static CStr>,
1918    /// The physical device features to enable. You may enable features, but must not disable any.
1919    pub device_features: &'arg mut PhysicalDeviceFeatures,
1920    /// The queue create infos for the device. You may substitute a different queue, but:
1921    /// 1. Any queue you provide must be compatible with `wgpu`'s usage,
1922    /// 2. You must not leave the vector empty,
1923    /// 3. `wgpu` currently only uses the first entry.
1924    pub queue_create_infos: &'arg mut Vec<vk::DeviceQueueCreateInfo<'pnext>>,
1925    /// The create info for the device. You may add or modify things in the pnext chain, but
1926    /// do not turn features off. Additionally, do not add things to the list of extensions,
1927    /// or to the feature set, as all changes to that member will be overwritten.
1928    pub create_info: &'arg mut vk::DeviceCreateInfo<'pnext>,
1929    /// We need to have `'this` in the struct, so we can declare that all lifetimes coming from
1930    /// captures in the closure will live longer (and hence satisfy) `'pnext`. However, we
1931    /// don't actually directly use `'this`
1932    _phantom: PhantomData<&'this ()>,
1933}
1934
1935/// Callback to allow changing the vulkan device creation parameters.
1936///
1937/// # Safety:
1938/// - If you want to add extensions, add the to the `Vec<'static CStr>` not the create info,
1939///   as the create info value will be overwritten.
1940/// - Callback must not remove features.
1941/// - Callback must not change anything to what the instance does not support.
1942pub type CreateDeviceCallback<'this> =
1943    dyn for<'arg, 'pnext> FnOnce(CreateDeviceCallbackArgs<'arg, 'pnext, 'this>) + 'this;
1944
1945/// Arguments to the [`CreateInstanceCallback`].
1946#[expect(missing_debug_implementations, reason = "TODO?")]
1947pub struct CreateInstanceCallbackArgs<'arg, 'pnext, 'this>
1948where
1949    'this: 'pnext,
1950{
1951    /// The extensions to enable for the instance. You must not remove anything from this list,
1952    /// but you may add to it.
1953    pub extensions: &'arg mut Vec<&'static CStr>,
1954    /// The create info for the instance. You may add or modify things in the pnext chain, but
1955    /// do not turn features off. Additionally, do not add things to the list of extensions,
1956    /// all changes to that member will be overwritten.
1957    pub create_info: &'arg mut vk::InstanceCreateInfo<'pnext>,
1958    /// Vulkan entry point.
1959    pub entry: &'arg ash::Entry,
1960    /// We need to have `'this` in the struct, so we can declare that all lifetimes coming from
1961    /// captures in the closure will live longer (and hence satisfy) `'pnext`. However, we
1962    /// don't actually directly use `'this`
1963    _phantom: PhantomData<&'this ()>,
1964}
1965
1966/// Callback to allow changing the vulkan instance creation parameters.
1967///
1968/// # Safety:
1969/// - If you want to add extensions, add the to the `Vec<'static CStr>` not the create info,
1970///   as the create info value will be overwritten.
1971/// - Callback must not remove features.
1972/// - Callback must not change anything to what the instance does not support.
1973pub type CreateInstanceCallback<'this> =
1974    dyn for<'arg, 'pnext> FnOnce(CreateInstanceCallbackArgs<'arg, 'pnext, 'this>) + 'this;