Skip to main content

wgpu_hal/vulkan/
device.rs

1use alloc::{borrow::ToOwned as _, collections::BTreeMap, ffi::CString, sync::Arc, vec::Vec};
2use core::{
3    ffi::CStr,
4    mem::{self, MaybeUninit},
5    num::NonZeroU32,
6    ptr,
7    time::Duration,
8};
9
10use arrayvec::ArrayVec;
11use ash::{ext, vk};
12use hashbrown::hash_map::Entry;
13use wgpu_sync::{Mutex, RwLock};
14
15use super::{conv, descriptor::DescriptorCounts, RawTlasInstance};
16use crate::TlasInstance;
17
18impl super::DeviceShared {
19    /// Set the name of `object` to `name`.
20    ///
21    /// If `name` contains an interior null byte, then the name set will be truncated to that byte.
22    ///
23    /// # Safety
24    ///
25    /// This method inherits the safety contract from [`vkSetDebugUtilsObjectName`]. In particular:
26    ///
27    /// - `object` must be a valid handle for one of the following:
28    ///   - An instance-level object from the same instance as this device.
29    ///   - A physical-device-level object that descends from the same physical device as this
30    ///     device.
31    ///   - A device-level object that descends from this device.
32    /// - `object` must be externally synchronized—only the calling thread should access it during
33    ///   this call.
34    ///
35    /// [`vkSetDebugUtilsObjectName`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkSetDebugUtilsObjectNameEXT.html
36    pub(super) unsafe fn set_object_name(&self, object: impl vk::Handle, name: &str) {
37        let Some(extension) = self.extension_fns.debug_utils.as_ref() else {
38            return;
39        };
40
41        // Keep variables outside the if-else block to ensure they do not
42        // go out of scope while we hold a pointer to them
43        let mut buffer: [u8; 64] = [0u8; 64];
44        let buffer_vec: Vec<u8>;
45
46        // Append a null terminator to the string
47        let name_bytes = if name.len() < buffer.len() {
48            // Common case, string is very small. Allocate a copy on the stack.
49            buffer[..name.len()].copy_from_slice(name.as_bytes());
50            // Add null terminator
51            buffer[name.len()] = 0;
52            &buffer[..name.len() + 1]
53        } else {
54            // Less common case, the string is large.
55            // This requires a heap allocation.
56            buffer_vec = name
57                .as_bytes()
58                .iter()
59                .cloned()
60                .chain(core::iter::once(0))
61                .collect();
62            &buffer_vec
63        };
64
65        let name = CStr::from_bytes_until_nul(name_bytes).expect("We have added a null byte");
66
67        let _result = unsafe {
68            extension.set_debug_utils_object_name(
69                &vk::DebugUtilsObjectNameInfoEXT::default()
70                    .object_handle(object)
71                    .object_name(name),
72            )
73        };
74    }
75
76    pub fn make_render_pass(
77        &self,
78        key: super::RenderPassKey,
79    ) -> Result<vk::RenderPass, crate::DeviceError> {
80        Ok(match self.render_passes.lock().entry(key) {
81            Entry::Occupied(e) => *e.get(),
82            Entry::Vacant(e) => {
83                let super::RenderPassKey {
84                    ref colors,
85                    ref depth_stencil,
86                    sample_count,
87                    multiview_mask,
88                    depth_read_only,
89                    stencil_read_only,
90                } = *e.key();
91
92                let mut vk_attachments = Vec::new();
93                let mut color_refs = Vec::with_capacity(colors.len());
94                let mut resolve_refs = Vec::with_capacity(color_refs.capacity());
95                let mut ds_ref = None;
96                let samples = vk::SampleCountFlags::from_raw(sample_count);
97                let unused = vk::AttachmentReference {
98                    attachment: vk::ATTACHMENT_UNUSED,
99                    layout: vk::ImageLayout::UNDEFINED,
100                };
101                for cat in colors.iter() {
102                    let (color_ref, resolve_ref) =
103                        if let Some(super::ColorAttachmentKey { base, resolve }) = cat {
104                            let super::AttachmentKey {
105                                format,
106                                layout,
107                                ops,
108                            } = *base;
109
110                            let color_ref = vk::AttachmentReference {
111                                attachment: vk_attachments.len() as u32,
112                                layout,
113                            };
114                            vk_attachments.push({
115                                let (load_op, store_op) = conv::map_attachment_ops(ops);
116                                vk::AttachmentDescription::default()
117                                    .format(format)
118                                    .samples(samples)
119                                    .load_op(load_op)
120                                    .store_op(store_op)
121                                    .initial_layout(layout)
122                                    .final_layout(layout)
123                            });
124                            let resolve_ref = if let Some(rat) = resolve {
125                                let super::AttachmentKey {
126                                    format,
127                                    layout,
128                                    ops,
129                                } = *rat;
130
131                                let (load_op, store_op) = conv::map_attachment_ops(ops);
132                                let vk_attachment = vk::AttachmentDescription::default()
133                                    .format(format)
134                                    .samples(vk::SampleCountFlags::TYPE_1)
135                                    .load_op(load_op)
136                                    .store_op(store_op)
137                                    .initial_layout(layout)
138                                    .final_layout(layout);
139                                vk_attachments.push(vk_attachment);
140
141                                vk::AttachmentReference {
142                                    attachment: vk_attachments.len() as u32 - 1,
143                                    layout,
144                                }
145                            } else {
146                                unused
147                            };
148
149                            (color_ref, resolve_ref)
150                        } else {
151                            (unused, unused)
152                        };
153
154                    color_refs.push(color_ref);
155                    resolve_refs.push(resolve_ref);
156                }
157
158                if let Some(ds) = depth_stencil {
159                    let super::DepthStencilAttachmentKey {
160                        ref base,
161                        stencil_ops,
162                    } = *ds;
163
164                    let super::AttachmentKey {
165                        format,
166                        layout,
167                        ops,
168                    } = *base;
169
170                    ds_ref = Some(vk::AttachmentReference {
171                        attachment: vk_attachments.len() as u32,
172                        layout,
173                    });
174                    let (load_op, mut store_op) = conv::map_attachment_ops(ops);
175                    let (stencil_load_op, mut stencil_store_op) =
176                        conv::map_attachment_ops(stencil_ops);
177
178                    let store_op_for_read_only = if self.private_caps.store_op_none {
179                        vk::AttachmentStoreOp::NONE
180                    } else {
181                        vk::AttachmentStoreOp::STORE
182                    };
183                    if depth_read_only {
184                        store_op = store_op_for_read_only;
185                    }
186                    if stencil_read_only {
187                        stencil_store_op = store_op_for_read_only;
188                    }
189
190                    let vk_attachment = vk::AttachmentDescription::default()
191                        .format(format)
192                        .samples(samples)
193                        .load_op(load_op)
194                        .store_op(store_op)
195                        .stencil_load_op(stencil_load_op)
196                        .stencil_store_op(stencil_store_op)
197                        .initial_layout(layout)
198                        .final_layout(layout);
199                    vk_attachments.push(vk_attachment);
200                }
201
202                let vk_subpasses = [{
203                    let mut vk_subpass = vk::SubpassDescription::default()
204                        .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
205                        .color_attachments(&color_refs)
206                        .resolve_attachments(&resolve_refs);
207
208                    if self
209                        .workarounds
210                        .contains(super::Workarounds::EMPTY_RESOLVE_ATTACHMENT_LISTS)
211                        && resolve_refs.is_empty()
212                    {
213                        vk_subpass.p_resolve_attachments = ptr::null();
214                    }
215
216                    if let Some(ref reference) = ds_ref {
217                        vk_subpass = vk_subpass.depth_stencil_attachment(reference)
218                    }
219                    vk_subpass
220                }];
221
222                let mut vk_info = vk::RenderPassCreateInfo::default()
223                    .attachments(&vk_attachments)
224                    .subpasses(&vk_subpasses);
225
226                let mut multiview_info;
227                let mask;
228                if let Some(multiview_mask) = multiview_mask {
229                    mask = [multiview_mask.get()];
230
231                    // On Vulkan 1.1 or later, this is an alias for core functionality
232                    multiview_info = vk::RenderPassMultiviewCreateInfoKHR::default()
233                        .view_masks(&mask)
234                        .correlation_masks(&mask);
235                    vk_info = vk_info.push_next(&mut multiview_info);
236                }
237
238                let raw = unsafe {
239                    self.raw
240                        .create_render_pass(&vk_info, None)
241                        .map_err(super::map_host_device_oom_err)?
242                };
243
244                *e.insert(raw)
245            }
246        })
247    }
248
249    fn make_memory_ranges<'a, I: 'a + Iterator<Item = crate::MemoryRange>>(
250        &self,
251        buffer: &'a super::Buffer,
252        ranges: I,
253    ) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
254        let super::BufferOwnership::Managed(ref allocation) = buffer.ownership else {
255            return None;
256        };
257        let allocation = allocation.lock();
258        let mask = self.private_caps.non_coherent_map_mask;
259        Some(ranges.map(move |range| {
260            vk::MappedMemoryRange::default()
261                .memory(allocation.memory())
262                .offset((allocation.offset() + range.start) & !mask)
263                .size((range.end - range.start + mask) & !mask)
264        }))
265    }
266}
267
268struct CompiledStage {
269    create_info: vk::PipelineShaderStageCreateInfo<'static>,
270    _entry_point: CString,
271    temp_raw_module: Option<vk::ShaderModule>,
272}
273
274struct MemoryProperties {
275    base: vk::PhysicalDeviceMemoryProperties,
276    heap_budget: ArrayVec<vk::DeviceSize, { vk::MAX_MEMORY_HEAPS }>,
277    heap_usage: ArrayVec<vk::DeviceSize, { vk::MAX_MEMORY_HEAPS }>,
278}
279
280impl MemoryProperties {
281    fn types(&self) -> &[vk::MemoryType] {
282        let count = self.base.memory_type_count as usize;
283        &self.base.memory_types[0..count]
284    }
285
286    fn heaps(&self) -> &[vk::MemoryHeap] {
287        let count = self.base.memory_heap_count as usize;
288        &self.base.memory_heaps[0..count]
289    }
290
291    fn heap_budget(&self) -> &[vk::DeviceSize] {
292        let count = self.base.memory_heap_count as usize;
293        &self.heap_budget[0..count]
294    }
295
296    fn heap_usage(&self) -> &[vk::DeviceSize] {
297        let count = self.base.memory_heap_count as usize;
298        &self.heap_usage[0..count]
299    }
300}
301
302impl super::Device {
303    /// # Safety
304    ///
305    /// - `vk_image` must be created respecting `desc`
306    /// - If `drop_callback` is [`None`], wgpu-hal will take ownership of `vk_image`. If
307    ///   `drop_callback` is [`Some`], `vk_image` must be valid until the callback is called.
308    /// - If the `ImageCreateFlags` does not contain `MUTABLE_FORMAT`, the `view_formats` of `desc` must be empty.
309    /// - If `memory` is not [`super::TextureMemory::External`], wgpu-hal will take ownership of the
310    ///   memory (which is presumed to back `vk_image`). Otherwise, the memory must remain valid until
311    ///   `drop_callback` is called.
312    pub unsafe fn texture_from_raw(
313        &self,
314        vk_image: vk::Image,
315        desc: &crate::TextureDescriptor,
316        drop_callback: Option<crate::DropCallback>,
317        memory: super::TextureMemory,
318    ) -> super::Texture {
319        let identity = self.shared.texture_identity_factory.next();
320        let drop_guard = crate::DropGuard::from_option(drop_callback);
321
322        if let Some(label) = desc.label {
323            unsafe { self.shared.set_object_name(vk_image, label) };
324        }
325
326        super::Texture {
327            raw: vk_image,
328            drop_guard,
329            memory,
330            format: desc.format,
331            copy_size: desc.copy_extent(),
332            identity,
333        }
334    }
335
336    fn find_memory_type_index(
337        &self,
338        type_bits_req: u32,
339        flags_req: vk::MemoryPropertyFlags,
340    ) -> Option<usize> {
341        let mem_properties = unsafe {
342            self.shared
343                .instance
344                .raw
345                .get_physical_device_memory_properties(self.shared.physical_device)
346        };
347
348        // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPhysicalDeviceMemoryProperties.html
349        for (i, mem_ty) in mem_properties.memory_types_as_slice().iter().enumerate() {
350            let types_bits = 1 << i;
351            let is_required_memory_type = type_bits_req & types_bits != 0;
352            let has_required_properties = mem_ty.property_flags & flags_req == flags_req;
353            if is_required_memory_type && has_required_properties {
354                return Some(i);
355            }
356        }
357
358        None
359    }
360
361    fn create_image_without_memory(
362        &self,
363        desc: &crate::TextureDescriptor,
364        external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
365    ) -> Result<ImageWithoutMemory, crate::DeviceError> {
366        self.create_image_without_memory_with_tiling(
367            desc,
368            vk::ImageTiling::OPTIMAL,
369            external_memory_image_create_info,
370            None,
371        )
372    }
373
374    fn create_image_without_memory_with_tiling(
375        &self,
376        desc: &crate::TextureDescriptor,
377        tiling: vk::ImageTiling,
378        external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
379        drm_modifier_info: Option<&mut vk::ImageDrmFormatModifierExplicitCreateInfoEXT>,
380    ) -> Result<ImageWithoutMemory, crate::DeviceError> {
381        let copy_size = desc.copy_extent();
382
383        let mut raw_flags = vk::ImageCreateFlags::empty();
384        if desc.dimension == wgt::TextureDimension::D3
385            && desc.usage.contains(wgt::TextureUses::COLOR_TARGET)
386        {
387            raw_flags |= vk::ImageCreateFlags::TYPE_2D_ARRAY_COMPATIBLE;
388        }
389        if desc.is_cube_compatible() {
390            raw_flags |= vk::ImageCreateFlags::CUBE_COMPATIBLE;
391        }
392
393        let original_format = self.shared.private_caps.map_texture_format(desc.format);
394        let mut vk_view_formats = vec![];
395        if !desc.view_formats.is_empty() {
396            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
397
398            if self.shared.private_caps.image_format_list {
399                vk_view_formats = desc
400                    .view_formats
401                    .iter()
402                    .map(|f| self.shared.private_caps.map_texture_format(*f))
403                    .collect();
404                vk_view_formats.push(original_format)
405            }
406        }
407        if desc.format.is_multi_planar_format() {
408            raw_flags |=
409                vk::ImageCreateFlags::MUTABLE_FORMAT | vk::ImageCreateFlags::EXTENDED_USAGE;
410        }
411
412        let mut usage = conv::map_texture_usage(desc.usage);
413        // wgpu allows views of textures that are only ever copied to or from, but Vulkan
414        // requires a viewable image to carry at least one of `VIEWABLE_IMAGE_USAGES`. Add a
415        // usage the format supports so that such views stay valid. Images backed by external
416        // memory are left alone: their usage has to match the resource being imported.
417        if !usage.intersects(conv::VIEWABLE_IMAGE_USAGES)
418            && external_memory_image_create_info.is_none()
419            && super::adapter::supports_format(
420                &self.shared.instance.raw,
421                self.shared.physical_device,
422                original_format,
423                tiling,
424                vk::FormatFeatureFlags::SAMPLED_IMAGE,
425            )
426        {
427            usage |= vk::ImageUsageFlags::SAMPLED;
428        }
429
430        let mut vk_info = vk::ImageCreateInfo::default()
431            .flags(raw_flags)
432            .image_type(conv::map_texture_dimension(desc.dimension))
433            .format(original_format)
434            .extent(conv::map_copy_extent(&copy_size))
435            .mip_levels(desc.mip_level_count)
436            .array_layers(desc.array_layer_count())
437            .samples(vk::SampleCountFlags::from_raw(desc.sample_count))
438            .tiling(tiling)
439            .usage(usage)
440            .sharing_mode(vk::SharingMode::EXCLUSIVE)
441            .initial_layout(vk::ImageLayout::UNDEFINED);
442
443        let mut format_list_info = vk::ImageFormatListCreateInfo::default();
444        if !vk_view_formats.is_empty() {
445            format_list_info = format_list_info.view_formats(&vk_view_formats);
446            vk_info = vk_info.push_next(&mut format_list_info);
447        }
448
449        if let Some(ext_info) = external_memory_image_create_info {
450            vk_info = vk_info.push_next(ext_info);
451        }
452
453        if let Some(drm_info) = drm_modifier_info {
454            vk_info = vk_info.push_next(drm_info);
455        }
456
457        let raw = unsafe { self.shared.raw.create_image(&vk_info, None) }.map_err(map_err)?;
458        fn map_err(err: vk::Result) -> crate::DeviceError {
459            // We don't use VK_EXT_image_compression_control
460            // VK_ERROR_COMPRESSION_EXHAUSTED_EXT
461            super::map_host_device_oom_and_ioca_err(err)
462        }
463        let mut req = unsafe { self.shared.raw.get_image_memory_requirements(raw) };
464
465        if desc.usage.contains(wgt::TextureUses::TRANSIENT) {
466            let mem_type_index = self.find_memory_type_index(
467                req.memory_type_bits,
468                vk::MemoryPropertyFlags::LAZILY_ALLOCATED,
469            );
470            if let Some(mem_type_index) = mem_type_index {
471                req.memory_type_bits = 1 << mem_type_index;
472            }
473        }
474
475        Ok(ImageWithoutMemory {
476            raw,
477            requirements: req,
478        })
479    }
480
481    /// # Safety
482    ///
483    /// - Vulkan (with VK_KHR_external_memory_win32)
484    /// - The `d3d11_shared_handle` must be valid and respecting `desc`
485    /// - `VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_BIT` flag is used because we need to hold a reference to the handle
486    #[cfg(windows)]
487    pub unsafe fn texture_from_d3d11_shared_handle(
488        &self,
489        d3d11_shared_handle: windows::Win32::Foundation::HANDLE,
490        desc: &crate::TextureDescriptor,
491    ) -> Result<super::Texture, crate::DeviceError> {
492        if !self
493            .shared
494            .enabled_extensions
495            .contains(&ash::khr::external_memory_win32::NAME)
496        {
497            log::error!("Vulkan driver does not support VK_KHR_external_memory_win32");
498            return Err(crate::DeviceError::Unexpected);
499        }
500
501        let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
502            .handle_types(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE);
503
504        let image =
505            self.create_image_without_memory(desc, Some(&mut external_memory_image_info))?;
506
507        // Some external memory types require dedicated allocation
508        // https://docs.vulkan.org/guide/latest/extensions/external.html#_importing_memory
509        let mut dedicated_allocate_info =
510            vk::MemoryDedicatedAllocateInfo::default().image(image.raw);
511
512        let mut import_memory_info = vk::ImportMemoryWin32HandleInfoKHR::default()
513            .handle_type(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE)
514            .handle(d3d11_shared_handle.0 as _);
515        // TODO: We should use `push_next` instead, but currently ash does not provide this method for the `ImportMemoryWin32HandleInfoKHR` type.
516        #[allow(clippy::unnecessary_mut_passed)]
517        {
518            import_memory_info.p_next = <*const _>::cast(&mut dedicated_allocate_info);
519        }
520
521        let mem_type_index = self
522            .find_memory_type_index(
523                image.requirements.memory_type_bits,
524                vk::MemoryPropertyFlags::DEVICE_LOCAL,
525            )
526            .ok_or(crate::DeviceError::Unexpected)?;
527
528        let memory_allocate_info = vk::MemoryAllocateInfo::default()
529            .allocation_size(image.requirements.size)
530            .memory_type_index(mem_type_index as _)
531            .push_next(&mut import_memory_info);
532        let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
533            .map_err(super::map_host_device_oom_err)?;
534
535        unsafe { self.shared.raw.bind_image_memory(image.raw, memory, 0) }
536            .map_err(super::map_host_device_oom_err)?;
537
538        Ok(unsafe {
539            self.texture_from_raw(
540                image.raw,
541                desc,
542                None,
543                super::TextureMemory::Dedicated(memory),
544            )
545        })
546    }
547
548    /// Import a DMA-buf as a texture. Currently only supports single-plane DMA-bufs.
549    ///
550    /// # Safety
551    ///
552    /// - Requires the VK_KHR_external_memory_fd, VK_EXT_external_memory_dma_buf, and
553    ///   VK_EXT_image_drm_format_modifier extensions
554    /// - The `fd` must be a valid DMA-buf file descriptor matching `desc`
555    /// - On success, Vulkan takes ownership of the file descriptor. On failure,
556    ///   the file descriptor is closed.
557    /// - The `drm_modifier`, `stride`, and `offset` must match the DMA-buf layout
558    #[cfg(unix)]
559    pub unsafe fn texture_from_dmabuf_fd(
560        &self,
561        fd: std::os::unix::io::OwnedFd,
562        desc: &crate::TextureDescriptor,
563        drm_modifier: u64,
564        stride: u64,
565        offset: u64,
566    ) -> Result<super::Texture, crate::DeviceError> {
567        use std::os::unix::io::IntoRawFd;
568
569        let enabled_extensions = &self.shared.enabled_extensions;
570        if !enabled_extensions.contains(&ext::external_memory_dma_buf::NAME)
571            || !enabled_extensions.contains(&ext::image_drm_format_modifier::NAME)
572        {
573            log::error!(
574                "Vulkan driver does not support VK_EXT_external_memory_dma_buf \
575                 or VK_EXT_image_drm_format_modifier"
576            );
577            return Err(crate::DeviceError::Unexpected);
578        }
579
580        let external_memory_fd_fn = self
581            .shared
582            .extension_fns
583            .external_memory_fd
584            .as_ref()
585            .ok_or_else(|| {
586                log::error!("VK_KHR_external_memory_fd extension not loaded");
587                crate::DeviceError::Unexpected
588            })?;
589
590        let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
591            .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
592
593        let plane_layout = vk::SubresourceLayout {
594            offset,
595            row_pitch: stride,
596            size: 0,
597            array_pitch: 0,
598            depth_pitch: 0,
599        };
600        let mut drm_modifier_info = vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default()
601            .drm_format_modifier(drm_modifier)
602            .plane_layouts(core::slice::from_ref(&plane_layout));
603
604        let image = self.create_image_without_memory_with_tiling(
605            desc,
606            vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT,
607            Some(&mut external_memory_image_info),
608            Some(&mut drm_modifier_info),
609        )?;
610
611        // Convert to raw fd. We must close it ourselves if any operation below
612        // fails, since Vulkan only takes ownership on successful vkAllocateMemory.
613        let fd_raw = fd.into_raw_fd();
614
615        let result = self.import_dmabuf_memory(
616            external_memory_fd_fn,
617            fd_raw,
618            image.raw,
619            &image.requirements,
620        );
621
622        match result {
623            Ok(memory) => Ok(unsafe {
624                self.texture_from_raw(
625                    image.raw,
626                    desc,
627                    None,
628                    super::TextureMemory::Dedicated(memory),
629                )
630            }),
631            Err(e) => {
632                // Clean up the VkImage on failure.
633                unsafe { self.shared.raw.destroy_image(image.raw, None) };
634                Err(e)
635            }
636        }
637    }
638
639    /// Import DMA-buf memory and bind it to the image.
640    ///
641    /// On failure, the raw fd is closed (if not yet consumed by Vulkan) and the
642    /// caller is responsible for destroying the VkImage.
643    #[cfg(unix)]
644    fn import_dmabuf_memory(
645        &self,
646        external_memory_fd_fn: &ash::khr::external_memory_fd::Device,
647        fd_raw: i32,
648        image: vk::Image,
649        requirements: &vk::MemoryRequirements,
650    ) -> Result<vk::DeviceMemory, crate::DeviceError> {
651        let mut fd_props = vk::MemoryFdPropertiesKHR::default();
652        unsafe {
653            external_memory_fd_fn.get_memory_fd_properties(
654                vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
655                fd_raw,
656                &mut fd_props,
657            )
658        }
659        .map_err(|e| {
660            unsafe { libc::close(fd_raw) };
661            super::map_host_device_oom_err(e)
662        })?;
663
664        let mem_type_index = self
665            .find_memory_type_index(
666                requirements.memory_type_bits & fd_props.memory_type_bits,
667                vk::MemoryPropertyFlags::empty(),
668            )
669            .ok_or_else(|| {
670                unsafe { libc::close(fd_raw) };
671                crate::DeviceError::Unexpected
672            })?;
673
674        let mut dedicated_allocate_info = vk::MemoryDedicatedAllocateInfo::default().image(image);
675
676        let mut import_memory_info = vk::ImportMemoryFdInfoKHR::default()
677            .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT)
678            .fd(fd_raw);
679
680        let memory_allocate_info = vk::MemoryAllocateInfo::default()
681            .allocation_size(requirements.size)
682            .memory_type_index(mem_type_index as _)
683            .push_next(&mut import_memory_info)
684            .push_next(&mut dedicated_allocate_info);
685
686        // vkAllocateMemory takes ownership of the fd on success.
687        // On failure, the fd is NOT consumed and we must close it.
688        let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
689            .map_err(|e| {
690                unsafe { libc::close(fd_raw) };
691                super::map_host_device_oom_err(e)
692            })?;
693
694        // From this point, the fd is consumed. Only VkDeviceMemory needs cleanup on error.
695        unsafe { self.shared.raw.bind_image_memory(image, memory, 0) }.map_err(|e| {
696            unsafe { self.shared.raw.free_memory(memory, None) };
697            super::map_host_device_oom_err(e)
698        })?;
699
700        Ok(memory)
701    }
702
703    fn create_shader_module_impl(
704        &self,
705        spv: &[u32],
706        label: &crate::Label<'_>,
707    ) -> Result<vk::ShaderModule, crate::DeviceError> {
708        let vk_info = vk::ShaderModuleCreateInfo::default()
709            .flags(vk::ShaderModuleCreateFlags::empty())
710            .code(spv);
711
712        let raw = unsafe {
713            profiling::scope!("vkCreateShaderModule");
714            self.shared
715                .raw
716                .create_shader_module(&vk_info, None)
717                .map_err(map_err)?
718        };
719        fn map_err(err: vk::Result) -> crate::DeviceError {
720            // We don't use VK_NV_glsl_shader
721            // VK_ERROR_INVALID_SHADER_NV
722            super::map_host_device_oom_err(err)
723        }
724
725        if let Some(label) = label {
726            unsafe { self.shared.set_object_name(raw, label) };
727        }
728
729        Ok(raw)
730    }
731
732    fn compile_stage(
733        &self,
734        stage: &crate::ProgrammableStage<super::ShaderModule>,
735        naga_stage: naga::ShaderStage,
736        binding_map: &naga::back::spv::BindingMap,
737    ) -> Result<CompiledStage, crate::PipelineError> {
738        let stage_flags = crate::auxil::map_naga_stage(naga_stage);
739        let vk_module = match *stage.module {
740            super::ShaderModule::Raw(raw) => raw,
741            super::ShaderModule::Intermediate {
742                ref naga_shader,
743                runtime_checks,
744            } => {
745                let pipeline_options = naga::back::spv::PipelineOptions {
746                    entry_point: stage.entry_point.to_owned(),
747                    shader_stage: naga_stage,
748                };
749                let polyfill_zero_init = matches!(
750                    naga_stage,
751                    naga::ShaderStage::Task | naga::ShaderStage::Mesh
752                ) && self
753                    .shared
754                    .workarounds
755                    .contains(super::Workarounds::POLYFILL_MESH_TASK_WORKGROUP_ZERO_INIT)
756                    && self.naga_options.zero_initialize_workgroup_memory
757                        == naga::back::spv::ZeroInitializeWorkgroupMemoryMode::Native;
758                let needs_temp_options = polyfill_zero_init
759                    || !runtime_checks.bounds_checks
760                    || !runtime_checks.force_loop_bounding
761                    || !runtime_checks.ray_query_initialization_tracking
762                    || !binding_map.is_empty()
763                    || naga_shader.debug_source.is_some()
764                    || !stage.zero_initialize_workgroup_memory
765                    || !runtime_checks.task_shader_dispatch_tracking
766                    || !runtime_checks.mesh_shader_primitive_indices_clamp
767                    || !runtime_checks.int_div_checks;
768
769                let mut temp_options;
770                let options = if needs_temp_options {
771                    temp_options = self.naga_options.clone();
772                    if !runtime_checks.bounds_checks {
773                        temp_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
774                            index: naga::proc::BoundsCheckPolicy::Unchecked,
775                            buffer: naga::proc::BoundsCheckPolicy::Unchecked,
776                            image_load: naga::proc::BoundsCheckPolicy::Unchecked,
777                            binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
778                        };
779                    }
780                    if !runtime_checks.force_loop_bounding {
781                        temp_options.force_loop_bounding = false;
782                    }
783                    if !runtime_checks.ray_query_initialization_tracking {
784                        temp_options.ray_query_initialization_tracking = false;
785                    }
786                    if !binding_map.is_empty() {
787                        temp_options.binding_map = binding_map.clone();
788                    }
789
790                    if let Some(ref debug) = naga_shader.debug_source {
791                        temp_options.debug_info = Some(naga::back::spv::DebugInfo {
792                            source_code: &debug.source_code,
793                            file_name: debug.file_name.as_ref(),
794                            language: naga::back::spv::SourceLanguage::WGSL,
795                        })
796                    }
797                    if !stage.zero_initialize_workgroup_memory {
798                        temp_options.zero_initialize_workgroup_memory =
799                            naga::back::spv::ZeroInitializeWorkgroupMemoryMode::None;
800                    } else if polyfill_zero_init {
801                        temp_options.zero_initialize_workgroup_memory =
802                            naga::back::spv::ZeroInitializeWorkgroupMemoryMode::Polyfill;
803                    }
804                    if !runtime_checks.task_shader_dispatch_tracking {
805                        temp_options.task_dispatch_limits = None;
806                    }
807                    temp_options.mesh_shader_primitive_indices_clamp =
808                        runtime_checks.mesh_shader_primitive_indices_clamp;
809                    temp_options.emit_int_div_checks = runtime_checks.int_div_checks;
810
811                    &temp_options
812                } else {
813                    &self.naga_options
814                };
815
816                let (module, info) = naga::back::pipeline_constants::process_overrides(
817                    &naga_shader.module,
818                    &naga_shader.info,
819                    Some((naga_stage, stage.entry_point)),
820                    stage.constants,
821                )
822                .map_err(|e| {
823                    crate::PipelineError::PipelineConstants(stage_flags, format!("{e}"))
824                })?;
825
826                let spv = {
827                    profiling::scope!("naga::spv::write_vec");
828                    naga::back::spv::write_vec(&module, &info, options, Some(&pipeline_options))
829                }
830                .map_err(|e| crate::PipelineError::Linkage(stage_flags, format!("{e}")))?;
831                self.create_shader_module_impl(&spv, &None)?
832            }
833        };
834
835        let mut flags = vk::PipelineShaderStageCreateFlags::empty();
836        if self.shared.features.contains(wgt::Features::SUBGROUP) {
837            flags |= vk::PipelineShaderStageCreateFlags::ALLOW_VARYING_SUBGROUP_SIZE
838        }
839
840        let entry_point = CString::new(stage.entry_point).unwrap();
841        let mut create_info = vk::PipelineShaderStageCreateInfo::default()
842            .flags(flags)
843            .stage(conv::map_shader_stage(stage_flags))
844            .module(vk_module);
845
846        // Circumvent struct lifetime check because of a self-reference inside CompiledStage
847        create_info.p_name = entry_point.as_ptr();
848
849        Ok(CompiledStage {
850            create_info,
851            _entry_point: entry_point,
852            temp_raw_module: match *stage.module {
853                super::ShaderModule::Raw(_) => None,
854                super::ShaderModule::Intermediate { .. } => Some(vk_module),
855            },
856        })
857    }
858
859    /// Returns the queue family index of the device's internal queue.
860    ///
861    /// This is useful for constructing memory barriers needed for queue family ownership transfer when
862    /// external memory is involved (from/to `VK_QUEUE_FAMILY_EXTERNAL_KHR` and `VK_QUEUE_FAMILY_FOREIGN_EXT`
863    /// for example).
864    pub fn queue_family_index(&self) -> u32 {
865        self.shared.family_index
866    }
867
868    pub fn queue_index(&self) -> u32 {
869        self.shared.queue_index
870    }
871
872    pub fn raw_device(&self) -> &ash::Device {
873        &self.shared.raw
874    }
875
876    pub fn raw_physical_device(&self) -> vk::PhysicalDevice {
877        self.shared.physical_device
878    }
879
880    pub fn raw_queue(&self) -> vk::Queue {
881        self.shared.raw_queue
882    }
883
884    pub fn enabled_device_extensions(&self) -> &[&'static CStr] {
885        &self.shared.enabled_extensions
886    }
887
888    pub fn shared_instance(&self) -> &super::InstanceShared {
889        &self.shared.instance
890    }
891
892    fn get_memory_properties(&self) -> Option<MemoryProperties> {
893        if !self
894            .shared
895            .enabled_extensions
896            .contains(&ext::memory_budget::NAME)
897        {
898            return None;
899        }
900
901        let get_physical_device_properties = self
902            .shared
903            .instance
904            .get_physical_device_properties
905            .as_ref()
906            .unwrap();
907
908        let mut budget = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
909        let mut props = vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut budget);
910
911        unsafe {
912            get_physical_device_properties
913                .get_physical_device_memory_properties2(self.shared.physical_device, &mut props);
914        }
915
916        let vk::PhysicalDeviceMemoryProperties2 {
917            memory_properties, ..
918        } = props;
919
920        let mut heap_budget = ArrayVec::from(budget.heap_budget);
921        let mut heap_usage = ArrayVec::from(budget.heap_usage);
922        heap_budget.truncate(memory_properties.memory_heap_count as usize);
923        heap_usage.truncate(memory_properties.memory_heap_count as usize);
924
925        Some(MemoryProperties {
926            base: memory_properties,
927            heap_budget,
928            heap_usage,
929        })
930    }
931
932    /// Predict whether a proposed allocation will result in an OOM condition.
933    ///
934    /// If so, returns `Err(crate::DeviceError::OutOfMemory)`. If not, returns
935    /// `Ok(())`.
936    ///
937    /// The prediction quality depends on accurately selecting the heap that
938    /// [`gpu_allocator`] will use for the allocation, and is subject to
939    /// deteriorate if the logic in [`gpu_allocator`] changes.
940    fn error_if_would_oom_on_resource_allocation(
941        &self,
942        location: gpu_allocator::MemoryLocation,
943        requirements: &vk::MemoryRequirements,
944    ) -> Result<(), crate::DeviceError> {
945        use gpu_allocator::MemoryLocation;
946
947        let Some(threshold) = self
948            .shared
949            .instance
950            .memory_budget_thresholds
951            .for_resource_creation
952        else {
953            return Ok(());
954        };
955
956        let Some(memory_properties) = self.get_memory_properties() else {
957            return Ok(());
958        };
959
960        let preferred_flags = match location {
961            MemoryLocation::GpuOnly => vk::MemoryPropertyFlags::DEVICE_LOCAL,
962            MemoryLocation::CpuToGpu => {
963                vk::MemoryPropertyFlags::HOST_VISIBLE
964                    | vk::MemoryPropertyFlags::HOST_COHERENT
965                    | vk::MemoryPropertyFlags::DEVICE_LOCAL
966            }
967            MemoryLocation::GpuToCpu => {
968                vk::MemoryPropertyFlags::HOST_VISIBLE
969                    | vk::MemoryPropertyFlags::HOST_COHERENT
970                    | vk::MemoryPropertyFlags::HOST_CACHED
971            }
972            MemoryLocation::Unknown => vk::MemoryPropertyFlags::empty(),
973        };
974
975        let mut selected_heap = memory_properties
976            .types()
977            .iter()
978            .enumerate()
979            .find(|(i, ty)| {
980                (1 << i) & requirements.memory_type_bits != 0
981                    && ty.property_flags.contains(preferred_flags)
982            });
983
984        if selected_heap.is_none() {
985            let required_flags = match location {
986                MemoryLocation::GpuOnly => vk::MemoryPropertyFlags::DEVICE_LOCAL,
987                MemoryLocation::CpuToGpu | MemoryLocation::GpuToCpu => {
988                    vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT
989                }
990                MemoryLocation::Unknown => vk::MemoryPropertyFlags::empty(),
991            };
992            selected_heap = memory_properties
993                .types()
994                .iter()
995                .enumerate()
996                .find(|(i, ty)| {
997                    (1 << i) & requirements.memory_type_bits != 0
998                        && ty.property_flags.contains(required_flags)
999                });
1000        }
1001
1002        if let Some((_, ty)) = selected_heap {
1003            let i = ty.heap_index as usize;
1004            let heap_usage = memory_properties.heap_usage()[i];
1005            let heap_budget = memory_properties.heap_budget()[i];
1006            if heap_usage + requirements.size < heap_budget / 100 * threshold as u64 {
1007                Ok(())
1008            } else {
1009                log::warn!(
1010                    "Allocation would result in an OOM condition\n\
1011                    Request: {requirements:?}\n\
1012                    Heap {index} had {heap_usage}B used of {heap_budget}B total before this request.",
1013                    index = ty.heap_index,
1014                );
1015                Err(crate::DeviceError::OutOfMemory)
1016            }
1017        } else {
1018            log::warn!("Failed to find a suitable heap for {requirements:?}");
1019            Err(crate::DeviceError::OutOfMemory)
1020        }
1021    }
1022}
1023
1024impl crate::Device for super::Device {
1025    type A = super::Api;
1026
1027    unsafe fn create_buffer(
1028        &self,
1029        desc: &crate::BufferDescriptor,
1030    ) -> Result<(super::Buffer, wgt::BufferAddress), crate::DeviceError> {
1031        let vk_info = vk::BufferCreateInfo::default()
1032            .size(desc.size)
1033            .usage(conv::map_buffer_usage(desc.usage))
1034            .sharing_mode(vk::SharingMode::EXCLUSIVE);
1035
1036        let raw = unsafe {
1037            self.shared
1038                .raw
1039                .create_buffer(&vk_info, None)
1040                .map_err(super::map_host_device_oom_and_ioca_err)?
1041        };
1042
1043        let mut requirements = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
1044
1045        let is_cpu_read = desc.usage.contains(wgt::BufferUses::MAP_READ);
1046        let is_cpu_write = desc.usage.contains(wgt::BufferUses::MAP_WRITE);
1047
1048        let location = match (is_cpu_read, is_cpu_write) {
1049            (true, true) => gpu_allocator::MemoryLocation::CpuToGpu,
1050            (true, false) => gpu_allocator::MemoryLocation::GpuToCpu,
1051            (false, true) => gpu_allocator::MemoryLocation::CpuToGpu,
1052            (false, false) => gpu_allocator::MemoryLocation::GpuOnly,
1053        };
1054
1055        self.error_if_would_oom_on_resource_allocation(location, &requirements)
1056            .inspect_err(|_| {
1057                unsafe { self.shared.raw.destroy_buffer(raw, None) };
1058            })?;
1059
1060        let name = desc.label.unwrap_or("Unlabeled buffer");
1061
1062        if desc
1063            .usage
1064            .contains(wgt::BufferUses::ACCELERATION_STRUCTURE_SCRATCH)
1065        {
1066            // There is no way to specify this usage to Vulkan so we must make sure the alignment requirement is large enough.
1067            requirements.alignment = requirements
1068                .alignment
1069                .max(self.shared.private_caps.scratch_buffer_alignment as u64);
1070        }
1071
1072        if desc
1073            .usage
1074            .contains(wgt::BufferUses::RAY_TRACING_PIPELINE_SHADER_DATA)
1075        {
1076            requirements.alignment = requirements
1077                .alignment
1078                .max(self.shared.private_caps.shader_binding_table_alignment as u64);
1079        }
1080
1081        let allocation = self
1082            .mem_allocator
1083            .allocate(
1084                super::MemoryPool::from_memory_flags(desc.memory_flags),
1085                &gpu_allocator::vulkan::AllocationCreateDesc {
1086                    name,
1087                    requirements: vk::MemoryRequirements {
1088                        memory_type_bits: requirements.memory_type_bits
1089                            & self.valid_ash_memory_types,
1090                        ..requirements
1091                    },
1092                    location,
1093                    linear: true, // Buffers are always linear
1094                    allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
1095                },
1096            )
1097            .inspect_err(|_| {
1098                unsafe { self.shared.raw.destroy_buffer(raw, None) };
1099            })?;
1100
1101        unsafe {
1102            self.shared
1103                .raw
1104                .bind_buffer_memory(raw, allocation.memory(), allocation.offset())
1105        }
1106        .map_err(super::map_host_device_oom_and_ioca_err)
1107        .inspect_err(|_| {
1108            unsafe { self.shared.raw.destroy_buffer(raw, None) };
1109        })?;
1110
1111        if let Some(label) = desc.label {
1112            unsafe { self.shared.set_object_name(raw, label) };
1113        }
1114
1115        self.counters.buffer_memory.add(allocation.size() as isize);
1116        self.counters.buffers.add(1);
1117
1118        Ok((
1119            super::Buffer {
1120                raw,
1121                ownership: super::BufferOwnership::Managed(Mutex::new(
1122                    super::BufferMemoryBacking::Managed(allocation),
1123                )),
1124            },
1125            desc.size,
1126        ))
1127    }
1128    unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
1129        match buffer.ownership {
1130            super::BufferOwnership::Managed(allocation) => {
1131                unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1132                let allocation = allocation.into_inner();
1133                self.counters.buffer_memory.sub(allocation.size() as isize);
1134                match allocation {
1135                    super::BufferMemoryBacking::Managed(allocation) => {
1136                        let result = self.mem_allocator.free(allocation);
1137                        if let Err(err) = result {
1138                            log::warn!("Failed to free buffer allocation: {err}");
1139                        }
1140                    }
1141                    super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
1142                        self.shared.raw.free_memory(memory, None);
1143                    },
1144                }
1145            }
1146            super::BufferOwnership::RawHandle => {
1147                unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1148            }
1149            super::BufferOwnership::External(_drop_guard) => {
1150                // The caller owns the `vk::Buffer` and its memory. Dropping
1151                // `_drop_guard` at the end of this arm runs the cleanup callback.
1152            }
1153        }
1154
1155        self.counters.buffers.sub(1);
1156    }
1157
1158    unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) {
1159        self.counters.buffers.add(1);
1160    }
1161
1162    unsafe fn map_buffer(
1163        &self,
1164        buffer: &super::Buffer,
1165        range: crate::MemoryRange,
1166    ) -> Result<crate::BufferMapping, crate::DeviceError> {
1167        let super::BufferOwnership::Managed(ref allocation) = buffer.ownership else {
1168            crate::hal_usage_error("tried to map external buffer")
1169        };
1170        let mut allocation = allocation.lock();
1171        let super::BufferMemoryBacking::Managed(ref mut allocation) = *allocation else {
1172            crate::hal_usage_error("tried to map externally created buffer")
1173        };
1174        let is_coherent = allocation
1175            .memory_properties()
1176            .contains(vk::MemoryPropertyFlags::HOST_COHERENT);
1177        Ok(crate::BufferMapping {
1178            ptr: unsafe {
1179                allocation
1180                    .mapped_ptr()
1181                    .unwrap()
1182                    .cast()
1183                    .offset(range.start as isize)
1184            },
1185            is_coherent,
1186        })
1187    }
1188
1189    unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
1190        match buffer.ownership {
1191            super::BufferOwnership::Managed(_) => {
1192                // gpu-allocator maps the buffer when allocated and unmaps it when free'd
1193            }
1194            super::BufferOwnership::RawHandle | super::BufferOwnership::External(_) => {
1195                crate::hal_usage_error("tried to unmap external buffer")
1196            }
1197        }
1198    }
1199
1200    unsafe fn flush_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1201    where
1202        I: Iterator<Item = crate::MemoryRange>,
1203    {
1204        if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1205            unsafe {
1206                self.shared
1207                    .raw
1208                    .flush_mapped_memory_ranges(
1209                        &smallvec::SmallVec::<[vk::MappedMemoryRange; 32]>::from_iter(vk_ranges),
1210                    )
1211            }
1212            .unwrap();
1213        }
1214    }
1215    unsafe fn invalidate_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1216    where
1217        I: Iterator<Item = crate::MemoryRange>,
1218    {
1219        if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1220            unsafe {
1221                self.shared
1222                    .raw
1223                    .invalidate_mapped_memory_ranges(&smallvec::SmallVec::<
1224                        [vk::MappedMemoryRange; 32],
1225                    >::from_iter(vk_ranges))
1226            }
1227            .unwrap();
1228        }
1229    }
1230
1231    unsafe fn create_texture(
1232        &self,
1233        desc: &crate::TextureDescriptor,
1234    ) -> Result<super::Texture, crate::DeviceError> {
1235        let image = self.create_image_without_memory(desc, None)?;
1236
1237        self.error_if_would_oom_on_resource_allocation(
1238            gpu_allocator::MemoryLocation::GpuOnly,
1239            &image.requirements,
1240        )
1241        .inspect_err(|_| {
1242            unsafe { self.shared.raw.destroy_image(image.raw, None) };
1243        })?;
1244
1245        let name = desc.label.unwrap_or("Unlabeled texture");
1246
1247        let allocation = self
1248            .mem_allocator
1249            .allocate(
1250                super::MemoryPool::from_memory_flags(desc.memory_flags),
1251                &gpu_allocator::vulkan::AllocationCreateDesc {
1252                    name,
1253                    requirements: vk::MemoryRequirements {
1254                        memory_type_bits: image.requirements.memory_type_bits
1255                            & self.valid_ash_memory_types,
1256                        ..image.requirements
1257                    },
1258                    location: gpu_allocator::MemoryLocation::GpuOnly,
1259                    linear: false,
1260                    allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
1261                },
1262            )
1263            .inspect_err(|_| {
1264                unsafe { self.shared.raw.destroy_image(image.raw, None) };
1265            })?;
1266
1267        unsafe {
1268            self.shared
1269                .raw
1270                .bind_image_memory(image.raw, allocation.memory(), allocation.offset())
1271        }
1272        .map_err(super::map_host_device_oom_err)
1273        .inspect_err(|_| {
1274            unsafe { self.shared.raw.destroy_image(image.raw, None) };
1275        })?;
1276
1277        self.counters.texture_memory.add(allocation.size() as isize);
1278        self.counters.textures.add(1);
1279
1280        Ok(unsafe {
1281            self.texture_from_raw(
1282                image.raw,
1283                desc,
1284                None,
1285                super::TextureMemory::Allocation(allocation),
1286            )
1287        })
1288    }
1289
1290    unsafe fn destroy_texture(&self, texture: super::Texture) {
1291        if texture.drop_guard.is_none() {
1292            unsafe { self.shared.raw.destroy_image(texture.raw, None) };
1293        }
1294
1295        match texture.memory {
1296            super::TextureMemory::Allocation(allocation) => {
1297                self.counters.texture_memory.sub(allocation.size() as isize);
1298                let result = self.mem_allocator.free(allocation);
1299                if let Err(err) = result {
1300                    log::warn!("Failed to free texture allocation: {err}");
1301                }
1302            }
1303            super::TextureMemory::Dedicated(memory) => unsafe {
1304                self.shared.raw.free_memory(memory, None);
1305            },
1306            super::TextureMemory::External => {}
1307        }
1308
1309        self.counters.textures.sub(1);
1310    }
1311
1312    unsafe fn add_raw_texture(&self, _texture: &super::Texture) {
1313        self.counters.textures.add(1);
1314    }
1315
1316    unsafe fn create_texture_view(
1317        &self,
1318        texture: &super::Texture,
1319        desc: &crate::TextureViewDescriptor,
1320    ) -> Result<super::TextureView, crate::DeviceError> {
1321        let mut swizzle = desc.swizzle;
1322
1323        // https://registry.khronos.org/vulkan/specs/latest/html/vkspec.html#textures-component-swizzle
1324        // If the image view has a depth/stencil format and the VkComponentSwizzle is VK_COMPONENT_SWIZZLE_ONE,
1325        // and VkPhysicalDeviceMaintenance5Properties::depthStencilSwizzleOneSupport is not VK_TRUE,
1326        // the value of the texel after swizzle is undefined.
1327        //
1328        // We convert `One` to `A` which should sample as 1.0,
1329        // according to https://registry.khronos.org/vulkan/specs/latest/html/vkspec.html#images-component-substitution
1330        if texture.format.is_depth_stencil_format()
1331            && !self.shared.private_caps.depth_stencil_swizzle_one_support
1332        {
1333            for component in [
1334                &mut swizzle.r,
1335                &mut swizzle.g,
1336                &mut swizzle.b,
1337                &mut swizzle.a,
1338            ] {
1339                if *component == wgt::ComponentSwizzle::One {
1340                    *component = wgt::ComponentSwizzle::A;
1341                }
1342            }
1343        }
1344
1345        let subresource_range = conv::map_subresource_range(&desc.range, texture.format);
1346        let raw_format = self.shared.private_caps.map_texture_format(desc.format);
1347        let mut vk_info = vk::ImageViewCreateInfo::default()
1348            .flags(vk::ImageViewCreateFlags::empty())
1349            .image(texture.raw)
1350            .view_type(conv::map_view_dimension(desc.dimension))
1351            .format(raw_format)
1352            .subresource_range(subresource_range)
1353            .components(conv::map_texture_component_swizzle(swizzle));
1354        let layers =
1355            NonZeroU32::new(subresource_range.layer_count).expect("Unexpected zero layer count");
1356
1357        let mut image_view_info;
1358        if self.shared.private_caps.image_view_usage && !desc.usage.is_empty() {
1359            image_view_info =
1360                vk::ImageViewUsageCreateInfo::default().usage(conv::map_texture_usage(desc.usage));
1361            vk_info = vk_info.push_next(&mut image_view_info);
1362        }
1363
1364        let raw = unsafe { self.shared.raw.create_image_view(&vk_info, None) }
1365            .map_err(super::map_host_device_oom_and_ioca_err)?;
1366
1367        if let Some(label) = desc.label {
1368            unsafe { self.shared.set_object_name(raw, label) };
1369        }
1370
1371        let identity = self.shared.texture_view_identity_factory.next();
1372
1373        self.counters.texture_views.add(1);
1374
1375        Ok(super::TextureView {
1376            raw_texture: texture.raw,
1377            raw,
1378            _layers: layers,
1379            format: desc.format,
1380            raw_format,
1381            base_mip_level: desc.range.base_mip_level,
1382            dimension: desc.dimension,
1383            texture_identity: texture.identity,
1384            view_identity: identity,
1385        })
1386    }
1387    unsafe fn destroy_texture_view(&self, view: super::TextureView) {
1388        unsafe { self.shared.raw.destroy_image_view(view.raw, None) };
1389
1390        self.counters.texture_views.sub(1);
1391    }
1392
1393    unsafe fn create_sampler(
1394        &self,
1395        desc: &crate::SamplerDescriptor,
1396    ) -> Result<super::Sampler, crate::DeviceError> {
1397        let mut create_info = vk::SamplerCreateInfo::default()
1398            .flags(vk::SamplerCreateFlags::empty())
1399            .mag_filter(conv::map_filter_mode(desc.mag_filter))
1400            .min_filter(conv::map_filter_mode(desc.min_filter))
1401            .mipmap_mode(conv::map_mip_filter_mode(desc.mipmap_filter))
1402            .address_mode_u(conv::map_address_mode(desc.address_modes[0]))
1403            .address_mode_v(conv::map_address_mode(desc.address_modes[1]))
1404            .address_mode_w(conv::map_address_mode(desc.address_modes[2]))
1405            .min_lod(desc.lod_clamp.start)
1406            .max_lod(desc.lod_clamp.end);
1407
1408        if let Some(fun) = desc.compare {
1409            create_info = create_info
1410                .compare_enable(true)
1411                .compare_op(conv::map_comparison(fun));
1412        }
1413
1414        if desc.anisotropy_clamp != 1 {
1415            // We only enable anisotropy if it is supported, and wgpu-hal interface guarantees
1416            // the clamp is in the range [1, 16] which is always supported if anisotropy is.
1417            create_info = create_info
1418                .anisotropy_enable(true)
1419                .max_anisotropy(desc.anisotropy_clamp as f32);
1420        }
1421
1422        if let Some(color) = desc.border_color {
1423            create_info = create_info.border_color(conv::map_border_color(color));
1424        }
1425
1426        let mut sampler_cache_guard = self.shared.sampler_cache.lock();
1427
1428        let raw = sampler_cache_guard.create_sampler(&self.shared.raw, create_info)?;
1429
1430        // Note: Cached samplers will just continually overwrite the label
1431        //
1432        // https://github.com/gfx-rs/wgpu/issues/6867
1433        if let Some(label) = desc.label {
1434            // SAFETY: we are holding a lock on the sampler cache,
1435            // so we can only be setting the name from one thread.
1436            unsafe { self.shared.set_object_name(raw, label) };
1437        }
1438
1439        drop(sampler_cache_guard);
1440
1441        self.counters.samplers.add(1);
1442
1443        Ok(super::Sampler { raw, create_info })
1444    }
1445    unsafe fn destroy_sampler(&self, sampler: super::Sampler) {
1446        self.shared.sampler_cache.lock().destroy_sampler(
1447            &self.shared.raw,
1448            sampler.create_info,
1449            sampler.raw,
1450        );
1451
1452        self.counters.samplers.sub(1);
1453    }
1454
1455    unsafe fn create_command_encoder(
1456        &self,
1457        desc: &crate::CommandEncoderDescriptor<super::Queue>,
1458    ) -> Result<super::CommandEncoder, crate::DeviceError> {
1459        let vk_info = vk::CommandPoolCreateInfo::default()
1460            .queue_family_index(desc.queue.family_index)
1461            .flags(vk::CommandPoolCreateFlags::TRANSIENT);
1462
1463        let raw = unsafe {
1464            self.shared
1465                .raw
1466                .create_command_pool(&vk_info, None)
1467                .map_err(super::map_host_device_oom_err)?
1468        };
1469
1470        self.counters.command_encoders.add(1);
1471
1472        Ok(super::CommandEncoder {
1473            raw,
1474            device: Arc::clone(&self.shared),
1475            active: vk::CommandBuffer::null(),
1476            bind_point: vk::PipelineBindPoint::default(),
1477            temp: super::Temp::default(),
1478            free: Vec::new(),
1479            discarded: Vec::new(),
1480            rpass_debug_marker_active: false,
1481            end_of_pass_timer_query: None,
1482            framebuffers: Default::default(),
1483            temp_texture_views: Default::default(),
1484            counters: Arc::clone(&self.counters),
1485            current_pipeline_is_multiview: false,
1486        })
1487    }
1488
1489    unsafe fn create_bind_group_layout(
1490        &self,
1491        desc: &crate::BindGroupLayoutDescriptor,
1492    ) -> Result<super::BindGroupLayout, crate::DeviceError> {
1493        // Iterate through the entries and accumulate our Vulkan
1494        // DescriptorSetLayoutBindings and DescriptorBindingFlags, as well as
1495        // our binding map and our descriptor counts.
1496        // Note: not bothering with on stack arrays here as it's low frequency
1497        let mut vk_bindings = Vec::new();
1498        let mut binding_flags = Vec::new();
1499        let mut binding_map = Vec::new();
1500        let mut next_binding = 0;
1501        let mut contains_binding_arrays = false;
1502        let mut desc_count = DescriptorCounts::default();
1503        for entry in desc.entries {
1504            if entry.count.is_some() {
1505                contains_binding_arrays = true;
1506            }
1507
1508            let partially_bound = desc
1509                .flags
1510                .contains(crate::BindGroupLayoutFlags::PARTIALLY_BOUND);
1511            let mut flags = vk::DescriptorBindingFlags::empty();
1512            if partially_bound && entry.count.is_some() {
1513                flags |= vk::DescriptorBindingFlags::PARTIALLY_BOUND;
1514            }
1515            if entry.count.is_some() {
1516                flags |= vk::DescriptorBindingFlags::UPDATE_AFTER_BIND;
1517            }
1518
1519            let count = entry.count.map_or(1, |c| c.get());
1520            match entry.ty {
1521                wgt::BindingType::ExternalTexture => {
1522                    assert!(
1523                        count == 1 && entry.count.is_none() && flags.is_empty(),
1524                        "ExternalTexture binding arrays are not supported"
1525                    );
1526                    vk_bindings.extend((next_binding..next_binding + 3).map(|binding| {
1527                        vk::DescriptorSetLayoutBinding {
1528                            binding,
1529                            descriptor_type: vk::DescriptorType::SAMPLED_IMAGE,
1530                            descriptor_count: 1,
1531                            stage_flags: conv::map_shader_stage(entry.visibility),
1532                            p_immutable_samplers: ptr::null(),
1533                            _marker: Default::default(),
1534                        }
1535                    }));
1536                    vk_bindings.push(vk::DescriptorSetLayoutBinding {
1537                        binding: next_binding + 3,
1538                        descriptor_type: vk::DescriptorType::UNIFORM_BUFFER,
1539                        descriptor_count: 1,
1540                        stage_flags: conv::map_shader_stage(entry.visibility),
1541                        p_immutable_samplers: ptr::null(),
1542                        _marker: Default::default(),
1543                    });
1544                    binding_flags.extend(core::iter::repeat_n(flags, 4));
1545                    binding_map.push((
1546                        entry.binding,
1547                        super::BindingInfo {
1548                            binding: next_binding,
1549                            binding_array_size: None,
1550                            external_texture: true,
1551                        },
1552                    ));
1553                    next_binding += 4;
1554                }
1555                _ => {
1556                    vk_bindings.push(vk::DescriptorSetLayoutBinding {
1557                        binding: next_binding,
1558                        descriptor_type: conv::map_binding_type(entry.ty),
1559                        descriptor_count: count,
1560                        stage_flags: conv::map_shader_stage(entry.visibility),
1561                        p_immutable_samplers: ptr::null(),
1562                        _marker: Default::default(),
1563                    });
1564                    binding_flags.push(flags);
1565                    binding_map.push((
1566                        entry.binding,
1567                        super::BindingInfo {
1568                            binding: next_binding,
1569                            binding_array_size: entry.count,
1570                            external_texture: false,
1571                        },
1572                    ));
1573                    next_binding += 1;
1574                }
1575            }
1576
1577            match entry.ty {
1578                wgt::BindingType::Buffer {
1579                    ty,
1580                    has_dynamic_offset,
1581                    ..
1582                } => match ty {
1583                    wgt::BufferBindingType::Uniform => {
1584                        if has_dynamic_offset {
1585                            desc_count.uniform_buffer_dynamic += count;
1586                        } else {
1587                            desc_count.uniform_buffer += count;
1588                        }
1589                    }
1590                    wgt::BufferBindingType::Storage { .. } => {
1591                        if has_dynamic_offset {
1592                            desc_count.storage_buffer_dynamic += count;
1593                        } else {
1594                            desc_count.storage_buffer += count;
1595                        }
1596                    }
1597                },
1598                wgt::BindingType::Sampler { .. } => {
1599                    desc_count.sampler += count;
1600                }
1601                wgt::BindingType::Texture { .. } => {
1602                    desc_count.sampled_image += count;
1603                }
1604                wgt::BindingType::StorageTexture { .. } => {
1605                    desc_count.storage_image += count;
1606                }
1607                wgt::BindingType::AccelerationStructure { .. } => {
1608                    desc_count.acceleration_structure += count;
1609                }
1610                wgt::BindingType::ExternalTexture => {
1611                    desc_count.sampled_image += 3;
1612                    desc_count.uniform_buffer += 1;
1613                }
1614            }
1615        }
1616
1617        let vk_info = vk::DescriptorSetLayoutCreateInfo::default()
1618            .bindings(&vk_bindings)
1619            .flags(if contains_binding_arrays {
1620                vk::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND_POOL
1621            } else {
1622                vk::DescriptorSetLayoutCreateFlags::empty()
1623            });
1624
1625        let mut binding_flag_info =
1626            vk::DescriptorSetLayoutBindingFlagsCreateInfo::default().binding_flags(&binding_flags);
1627
1628        let vk_info = vk_info.push_next(&mut binding_flag_info);
1629
1630        let raw = unsafe {
1631            self.shared
1632                .raw
1633                .create_descriptor_set_layout(&vk_info, None)
1634                .map_err(super::map_host_device_oom_err)?
1635        };
1636
1637        if let Some(label) = desc.label {
1638            unsafe { self.shared.set_object_name(raw, label) };
1639        }
1640
1641        let layout = super::BindGroupLayout {
1642            raw,
1643            desc_count,
1644            entries: desc.entries.into(),
1645            binding_map,
1646            contains_binding_arrays,
1647        };
1648
1649        let result = self
1650            .desc_allocator
1651            .lock()
1652            .register_layout(&self.shared.raw, &layout);
1653        if let Err(err) = result {
1654            unsafe {
1655                self.shared
1656                    .raw
1657                    .destroy_descriptor_set_layout(layout.raw, None)
1658            };
1659            return Err(err);
1660        }
1661
1662        self.counters.bind_group_layouts.add(1);
1663
1664        Ok(layout)
1665    }
1666    unsafe fn destroy_bind_group_layout(&self, bg_layout: super::BindGroupLayout) {
1667        self.desc_allocator
1668            .lock()
1669            .unregister_layout(&self.shared.raw, &bg_layout);
1670
1671        unsafe {
1672            self.shared
1673                .raw
1674                .destroy_descriptor_set_layout(bg_layout.raw, None)
1675        };
1676
1677        self.counters.bind_group_layouts.sub(1);
1678    }
1679
1680    unsafe fn create_pipeline_layout(
1681        &self,
1682        desc: &crate::PipelineLayoutDescriptor<super::BindGroupLayout>,
1683    ) -> Result<super::PipelineLayout, crate::DeviceError> {
1684        //Note: not bothering with on stack array here as it's low frequency
1685        let vk_set_layouts = desc
1686            .bind_group_layouts
1687            .iter()
1688            .map(|bgl| match bgl {
1689                Some(bgl) => bgl.raw,
1690                None => {
1691                    // `VUID-VkPipelineLayoutCreateInfo-pSetLayouts-parameter`
1692                    // says `VK_NULL_HANDLE` is allowed but
1693                    // `VUID-VkPipelineLayoutCreateInfo-graphicsPipelineLibrary-06753`
1694                    // says it's not, unless the `graphicsPipelineLibrary`
1695                    // feature is enabled.
1696                    //
1697                    // We use an empty descriptor set layout to work around this.
1698                    self.shared.empty_descriptor_set_layout
1699                }
1700            })
1701            .collect::<Vec<_>>();
1702        let vk_immediates_ranges: Option<vk::PushConstantRange> = if desc.immediate_size != 0 {
1703            Some(vk::PushConstantRange {
1704                stage_flags: vk::ShaderStageFlags::ALL,
1705                offset: 0,
1706                size: desc.immediate_size,
1707            })
1708        } else {
1709            None
1710        };
1711
1712        let vk_info = vk::PipelineLayoutCreateInfo::default()
1713            .flags(vk::PipelineLayoutCreateFlags::empty())
1714            .set_layouts(&vk_set_layouts)
1715            .push_constant_ranges(vk_immediates_ranges.as_slice());
1716
1717        let raw = {
1718            profiling::scope!("vkCreatePipelineLayout");
1719            unsafe {
1720                self.shared
1721                    .raw
1722                    .create_pipeline_layout(&vk_info, None)
1723                    .map_err(super::map_host_device_oom_err)?
1724            }
1725        };
1726
1727        if let Some(label) = desc.label {
1728            unsafe { self.shared.set_object_name(raw, label) };
1729        }
1730
1731        let mut binding_map = BTreeMap::new();
1732        for (group, layout) in desc.bind_group_layouts.iter().enumerate() {
1733            let Some(layout) = layout else {
1734                continue;
1735            };
1736
1737            for &(binding, binding_info) in &layout.binding_map {
1738                binding_map.insert(
1739                    naga::ResourceBinding {
1740                        group: group as u32,
1741                        binding,
1742                    },
1743                    naga::back::spv::BindingInfo {
1744                        descriptor_set: group as u32,
1745                        binding: binding_info.binding,
1746                        binding_array_size: binding_info.binding_array_size.map(NonZeroU32::get),
1747                        external_texture: binding_info.external_texture,
1748                    },
1749                );
1750            }
1751        }
1752
1753        self.counters.pipeline_layouts.add(1);
1754        Ok(super::PipelineLayout { raw, binding_map })
1755    }
1756    unsafe fn destroy_pipeline_layout(&self, pipeline_layout: super::PipelineLayout) {
1757        unsafe {
1758            self.shared
1759                .raw
1760                .destroy_pipeline_layout(pipeline_layout.raw, None)
1761        };
1762
1763        self.counters.pipeline_layouts.sub(1);
1764    }
1765
1766    unsafe fn create_bind_group(
1767        &self,
1768        desc: &crate::BindGroupDescriptor<
1769            super::BindGroupLayout,
1770            super::Buffer,
1771            super::Sampler,
1772            super::TextureView,
1773            super::AccelerationStructure,
1774        >,
1775    ) -> Result<super::BindGroup, crate::DeviceError> {
1776        let set = unsafe {
1777            self.desc_allocator
1778                .lock()
1779                .alloc(&self.shared.raw, desc.layout)?
1780        };
1781
1782        if let Some(label) = desc.label {
1783            unsafe { self.shared.set_object_name(set.raw(), label) };
1784        }
1785
1786        /// Helper for splitting off and initializing a given number of elements on a pre-allocated
1787        /// stack, based on items returned from an [`ExactSizeIterator`].  Typically created from a
1788        /// [`MaybeUninit`] slice (see [`Vec::spare_capacity_mut()`]).
1789        /// The updated [`ExtensionStack`] of remaining uninitialized elements is returned, safely
1790        /// representing that the initialized and remaining elements are two independent mutable
1791        /// borrows.
1792        struct ExtendStack<'a, T> {
1793            remainder: &'a mut [MaybeUninit<T>],
1794        }
1795
1796        impl<'a, T> ExtendStack<'a, T> {
1797            fn from_vec_capacity(vec: &'a mut Vec<T>) -> Self {
1798                Self {
1799                    remainder: vec.spare_capacity_mut(),
1800                }
1801            }
1802
1803            fn extend_one(self, value: T) -> (Self, &'a mut T) {
1804                let (to_init, remainder) = self.remainder.split_first_mut().unwrap();
1805                let init = to_init.write(value);
1806                (Self { remainder }, init)
1807            }
1808
1809            fn extend(
1810                self,
1811                iter: impl IntoIterator<Item = T> + ExactSizeIterator,
1812            ) -> (Self, &'a mut [T]) {
1813                let (to_init, remainder) = self.remainder.split_at_mut(iter.len());
1814
1815                for (value, to_init) in iter.into_iter().zip(to_init.iter_mut()) {
1816                    to_init.write(value);
1817                }
1818
1819                // we can't use the safe (yet unstable) MaybeUninit::write_slice() here because of having an iterator to write
1820
1821                let init = {
1822                    // SAFETY: The loop above has initialized exactly as many items as to_init is
1823                    // long, so it is safe to cast away the MaybeUninit<T> wrapper into T.
1824
1825                    // Additional safety docs from unstable slice_assume_init_mut
1826                    // SAFETY: similar to safety notes for `slice_get_ref`, but we have a
1827                    // mutable reference which is also guaranteed to be valid for writes.
1828                    unsafe { mem::transmute::<&mut [MaybeUninit<T>], &mut [T]>(to_init) }
1829                };
1830                (Self { remainder }, init)
1831            }
1832        }
1833
1834        let mut writes = Vec::with_capacity(desc.entries.len());
1835        let mut buffer_infos =
1836            Vec::with_capacity(desc.buffers.len() + desc.external_textures.len());
1837        let mut buffer_infos = ExtendStack::from_vec_capacity(&mut buffer_infos);
1838        let mut image_infos = Vec::with_capacity(
1839            desc.samplers.len() + desc.textures.len() + 3 * desc.external_textures.len(),
1840        );
1841        let mut image_infos = ExtendStack::from_vec_capacity(&mut image_infos);
1842        // TODO: This length could be reduced to just the number of top-level acceleration
1843        // structure bindings, where multiple consecutive TLAS bindings that are set via
1844        // one `WriteDescriptorSet` count towards one "info" struct, not the total number of
1845        // acceleration structure bindings to write:
1846        let mut acceleration_structure_infos =
1847            Vec::with_capacity(desc.acceleration_structures.len());
1848        let mut acceleration_structure_infos =
1849            ExtendStack::from_vec_capacity(&mut acceleration_structure_infos);
1850        let mut raw_acceleration_structures =
1851            Vec::with_capacity(desc.acceleration_structures.len());
1852        let mut raw_acceleration_structures =
1853            ExtendStack::from_vec_capacity(&mut raw_acceleration_structures);
1854
1855        let layout_and_entry_iter = desc.entries.iter().map(|entry| {
1856            let layout = desc
1857                .layout
1858                .entries
1859                .iter()
1860                .find(|layout_entry| layout_entry.binding == entry.binding)
1861                .expect("internal error: no layout entry found with binding slot");
1862            (layout, entry)
1863        });
1864        let mut next_binding = 0;
1865        for (layout, entry) in layout_and_entry_iter {
1866            let dst_set = set.raw();
1867
1868            fn add_image_descriptor_writes<'a>(
1869                mut infos: ExtendStack<'a, vk::DescriptorImageInfo>,
1870                writes: &mut Vec<vk::WriteDescriptorSet<'a>>,
1871                dst_set: vk::DescriptorSet,
1872                next_binding: &mut u32,
1873                add_infos: &'a [crate::TextureBinding<super::TextureView>],
1874                add_ty: vk::DescriptorType,
1875            ) -> ExtendStack<'a, vk::DescriptorImageInfo> {
1876                let local_image_infos;
1877                (infos, local_image_infos) = infos.extend(add_infos.iter().map(|binding| {
1878                    let layout = conv::derive_image_layout(binding.usage, binding.view.format);
1879                    vk::DescriptorImageInfo::default()
1880                        .image_view(binding.view.raw)
1881                        .image_layout(layout)
1882                }));
1883                writes.push(
1884                    vk::WriteDescriptorSet::default()
1885                        .dst_set(dst_set)
1886                        .dst_binding(*next_binding)
1887                        .descriptor_type(add_ty)
1888                        .image_info(local_image_infos),
1889                );
1890                *next_binding += 1;
1891                infos
1892            }
1893
1894            fn add_buffer_descriptor_writes<'a>(
1895                mut infos: ExtendStack<'a, vk::DescriptorBufferInfo>,
1896                writes: &mut Vec<vk::WriteDescriptorSet<'a>>,
1897                dst_set: vk::DescriptorSet,
1898                next_binding: &mut u32,
1899                add_infos: &'a [crate::BufferBinding<super::Buffer, wgt::BufferSize>],
1900                add_ty: vk::DescriptorType,
1901            ) -> ExtendStack<'a, vk::DescriptorBufferInfo> {
1902                let local_buffer_infos;
1903                (infos, local_buffer_infos) = infos.extend(add_infos.iter().map(|binding| {
1904                    vk::DescriptorBufferInfo::default()
1905                        .buffer(binding.buffer.raw)
1906                        .offset(binding.offset)
1907                        .range(binding.size.get())
1908                }));
1909                writes.push(
1910                    vk::WriteDescriptorSet::default()
1911                        .dst_set(dst_set)
1912                        .dst_binding(*next_binding)
1913                        .descriptor_type(add_ty)
1914                        .buffer_info(local_buffer_infos),
1915                );
1916                *next_binding += 1;
1917                infos
1918            }
1919
1920            match layout.ty {
1921                wgt::BindingType::Sampler(_) => {
1922                    let start = entry.resource_index;
1923                    let end = start + entry.count;
1924                    let local_image_infos;
1925                    (image_infos, local_image_infos) =
1926                        image_infos.extend(desc.samplers[start as usize..end as usize].iter().map(
1927                            |sampler| vk::DescriptorImageInfo::default().sampler(sampler.raw),
1928                        ));
1929                    writes.push(
1930                        vk::WriteDescriptorSet::default()
1931                            .dst_set(dst_set)
1932                            .dst_binding(next_binding)
1933                            .descriptor_type(conv::map_binding_type(layout.ty))
1934                            .image_info(local_image_infos),
1935                    );
1936                    next_binding += 1;
1937                }
1938                wgt::BindingType::Texture { .. } | wgt::BindingType::StorageTexture { .. } => {
1939                    let start = entry.resource_index;
1940                    let end = start + entry.count;
1941                    image_infos = add_image_descriptor_writes(
1942                        image_infos,
1943                        &mut writes,
1944                        dst_set,
1945                        &mut next_binding,
1946                        &desc.textures[start as usize..end as usize],
1947                        conv::map_binding_type(layout.ty),
1948                    );
1949                }
1950                wgt::BindingType::Buffer { .. } => {
1951                    let start = entry.resource_index;
1952                    let end = start + entry.count;
1953                    buffer_infos = add_buffer_descriptor_writes(
1954                        buffer_infos,
1955                        &mut writes,
1956                        dst_set,
1957                        &mut next_binding,
1958                        &desc.buffers[start as usize..end as usize],
1959                        conv::map_binding_type(layout.ty),
1960                    );
1961                }
1962                wgt::BindingType::AccelerationStructure { .. } => {
1963                    let start = entry.resource_index;
1964                    let end = start + entry.count;
1965
1966                    let local_raw_acceleration_structures;
1967                    (
1968                        raw_acceleration_structures,
1969                        local_raw_acceleration_structures,
1970                    ) = raw_acceleration_structures.extend(
1971                        desc.acceleration_structures[start as usize..end as usize]
1972                            .iter()
1973                            .map(|acceleration_structure| acceleration_structure.raw),
1974                    );
1975
1976                    let local_acceleration_structure_infos;
1977                    (
1978                        acceleration_structure_infos,
1979                        local_acceleration_structure_infos,
1980                    ) = acceleration_structure_infos.extend_one(
1981                        vk::WriteDescriptorSetAccelerationStructureKHR::default()
1982                            .acceleration_structures(local_raw_acceleration_structures),
1983                    );
1984
1985                    writes.push(
1986                        vk::WriteDescriptorSet::default()
1987                            .dst_set(dst_set)
1988                            .dst_binding(next_binding)
1989                            .descriptor_type(conv::map_binding_type(layout.ty))
1990                            .descriptor_count(entry.count)
1991                            .push_next(local_acceleration_structure_infos),
1992                    );
1993                    next_binding += 1;
1994                }
1995                wgt::BindingType::ExternalTexture => {
1996                    assert!(
1997                        entry.count == 1,
1998                        "ExternalTexture binding arrays are not supported"
1999                    );
2000                    let index = entry.resource_index as usize;
2001                    for plane in &desc.external_textures[index].planes {
2002                        image_infos = add_image_descriptor_writes(
2003                            image_infos,
2004                            &mut writes,
2005                            dst_set,
2006                            &mut next_binding,
2007                            core::slice::from_ref(plane),
2008                            vk::DescriptorType::SAMPLED_IMAGE,
2009                        );
2010                    }
2011                    buffer_infos = add_buffer_descriptor_writes(
2012                        buffer_infos,
2013                        &mut writes,
2014                        dst_set,
2015                        &mut next_binding,
2016                        core::slice::from_ref(&desc.external_textures[index].params),
2017                        vk::DescriptorType::UNIFORM_BUFFER,
2018                    )
2019                }
2020            }
2021        }
2022
2023        unsafe { self.shared.raw.update_descriptor_sets(&writes, &[]) };
2024
2025        self.counters.bind_groups.add(1);
2026
2027        Ok(super::BindGroup { set })
2028    }
2029
2030    unsafe fn destroy_bind_group(&self, group: super::BindGroup) {
2031        unsafe { self.desc_allocator.lock().free(&self.shared.raw, group.set) };
2032
2033        self.counters.bind_groups.sub(1);
2034    }
2035
2036    unsafe fn create_shader_module(
2037        &self,
2038        desc: &crate::ShaderModuleDescriptor,
2039        shader: crate::ShaderInput,
2040    ) -> Result<super::ShaderModule, crate::ShaderError> {
2041        let shader_module = match shader {
2042            crate::ShaderInput::Naga(naga_shader)
2043                if self
2044                    .shared
2045                    .workarounds
2046                    .contains(super::Workarounds::SEPARATE_ENTRY_POINTS)
2047                    || !naga_shader.module.overrides.is_empty() =>
2048            {
2049                super::ShaderModule::Intermediate {
2050                    naga_shader,
2051                    runtime_checks: desc.runtime_checks,
2052                }
2053            }
2054            crate::ShaderInput::Naga(naga_shader) => {
2055                let mut naga_options = self.naga_options.clone();
2056                naga_options.debug_info =
2057                    naga_shader
2058                        .debug_source
2059                        .as_ref()
2060                        .map(|d| naga::back::spv::DebugInfo {
2061                            source_code: d.source_code.as_ref(),
2062                            file_name: d.file_name.as_ref(),
2063                            language: naga::back::spv::SourceLanguage::WGSL,
2064                        });
2065                if !desc.runtime_checks.bounds_checks {
2066                    naga_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
2067                        index: naga::proc::BoundsCheckPolicy::Unchecked,
2068                        buffer: naga::proc::BoundsCheckPolicy::Unchecked,
2069                        image_load: naga::proc::BoundsCheckPolicy::Unchecked,
2070                        binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
2071                    };
2072                }
2073                let spv = naga::back::spv::write_vec(
2074                    &naga_shader.module,
2075                    &naga_shader.info,
2076                    &naga_options,
2077                    None,
2078                )
2079                .map_err(|e| crate::ShaderError::Compilation(format!("{e}")))?;
2080                super::ShaderModule::Raw(self.create_shader_module_impl(&spv, &desc.label)?)
2081            }
2082            crate::ShaderInput::SpirV(data) => {
2083                super::ShaderModule::Raw(self.create_shader_module_impl(data, &desc.label)?)
2084            }
2085            crate::ShaderInput::MetalLib { .. }
2086            | crate::ShaderInput::Msl { .. }
2087            | crate::ShaderInput::Dxil { .. }
2088            | crate::ShaderInput::Hlsl { .. }
2089            | crate::ShaderInput::Glsl { .. } => unreachable!(),
2090        };
2091
2092        self.counters.shader_modules.add(1);
2093
2094        Ok(shader_module)
2095    }
2096
2097    unsafe fn destroy_shader_module(&self, module: super::ShaderModule) {
2098        match module {
2099            super::ShaderModule::Raw(raw) => {
2100                unsafe { self.shared.raw.destroy_shader_module(raw, None) };
2101            }
2102            super::ShaderModule::Intermediate { .. } => {}
2103        }
2104
2105        self.counters.shader_modules.sub(1);
2106    }
2107
2108    unsafe fn create_render_pipeline(
2109        &self,
2110        desc: &crate::RenderPipelineDescriptor<
2111            super::PipelineLayout,
2112            super::ShaderModule,
2113            super::PipelineCache,
2114        >,
2115    ) -> Result<super::RenderPipeline, crate::PipelineError> {
2116        let dynamic_states = [
2117            vk::DynamicState::VIEWPORT,
2118            vk::DynamicState::SCISSOR,
2119            vk::DynamicState::BLEND_CONSTANTS,
2120            vk::DynamicState::STENCIL_REFERENCE,
2121        ];
2122        let mut compatible_rp_key = super::RenderPassKey {
2123            sample_count: desc.multisample.count,
2124            multiview_mask: desc.multiview_mask,
2125            ..Default::default()
2126        };
2127        let mut stages = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new();
2128        let mut vertex_buffers = Vec::new();
2129        let mut vertex_attributes = Vec::new();
2130
2131        if let crate::VertexProcessor::Standard {
2132            vertex_buffers: desc_vertex_buffers,
2133            vertex_stage: _,
2134        } = &desc.vertex_processor
2135        {
2136            vertex_buffers = Vec::with_capacity(desc_vertex_buffers.len());
2137            for (i, vb) in desc_vertex_buffers.iter().enumerate() {
2138                let Some(vb) = vb else {
2139                    continue;
2140                };
2141                vertex_buffers.push(vk::VertexInputBindingDescription {
2142                    binding: i as u32,
2143                    stride: vb.array_stride as u32,
2144                    input_rate: match vb.step_mode {
2145                        wgt::VertexStepMode::Vertex => vk::VertexInputRate::VERTEX,
2146                        wgt::VertexStepMode::Instance => vk::VertexInputRate::INSTANCE,
2147                    },
2148                });
2149                for at in vb.attributes {
2150                    vertex_attributes.push(vk::VertexInputAttributeDescription {
2151                        location: at.shader_location,
2152                        binding: i as u32,
2153                        format: conv::map_vertex_format(at.format),
2154                        offset: at.offset as u32,
2155                    });
2156                }
2157            }
2158        }
2159
2160        let vk_vertex_input = vk::PipelineVertexInputStateCreateInfo::default()
2161            .vertex_binding_descriptions(&vertex_buffers)
2162            .vertex_attribute_descriptions(&vertex_attributes);
2163
2164        let vk_input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
2165            .topology(conv::map_topology(desc.primitive.topology))
2166            .primitive_restart_enable(desc.primitive.strip_index_format.is_some());
2167
2168        let mut compiled_vs = None;
2169        let mut compiled_ms = None;
2170        let mut compiled_ts = None;
2171        match &desc.vertex_processor {
2172            crate::VertexProcessor::Standard {
2173                vertex_buffers: _,
2174                vertex_stage,
2175            } => {
2176                compiled_vs = Some(self.compile_stage(
2177                    vertex_stage,
2178                    naga::ShaderStage::Vertex,
2179                    &desc.layout.binding_map,
2180                )?);
2181                stages.push(compiled_vs.as_ref().unwrap().create_info);
2182            }
2183            crate::VertexProcessor::Mesh {
2184                task_stage,
2185                mesh_stage,
2186            } => {
2187                if let Some(t) = task_stage.as_ref() {
2188                    compiled_ts = Some(self.compile_stage(
2189                        t,
2190                        naga::ShaderStage::Task,
2191                        &desc.layout.binding_map,
2192                    )?);
2193                    stages.push(compiled_ts.as_ref().unwrap().create_info);
2194                }
2195                compiled_ms = Some(self.compile_stage(
2196                    mesh_stage,
2197                    naga::ShaderStage::Mesh,
2198                    &desc.layout.binding_map,
2199                )?);
2200                stages.push(compiled_ms.as_ref().unwrap().create_info);
2201            }
2202        }
2203        let compiled_fs = match desc.fragment_stage {
2204            Some(ref stage) => {
2205                let compiled = self.compile_stage(
2206                    stage,
2207                    naga::ShaderStage::Fragment,
2208                    &desc.layout.binding_map,
2209                )?;
2210                stages.push(compiled.create_info);
2211                Some(compiled)
2212            }
2213            None => None,
2214        };
2215
2216        let mut vk_rasterization = vk::PipelineRasterizationStateCreateInfo::default()
2217            .polygon_mode(conv::map_polygon_mode(desc.primitive.polygon_mode))
2218            .front_face(conv::map_front_face(desc.primitive.front_face))
2219            .line_width(1.0)
2220            .depth_clamp_enable(desc.primitive.unclipped_depth);
2221        if let Some(face) = desc.primitive.cull_mode {
2222            vk_rasterization = vk_rasterization.cull_mode(conv::map_cull_face(face))
2223        }
2224        let mut vk_rasterization_conservative_state =
2225            vk::PipelineRasterizationConservativeStateCreateInfoEXT::default()
2226                .conservative_rasterization_mode(
2227                    vk::ConservativeRasterizationModeEXT::OVERESTIMATE,
2228                );
2229        if desc.primitive.conservative {
2230            vk_rasterization = vk_rasterization.push_next(&mut vk_rasterization_conservative_state);
2231        }
2232
2233        let mut vk_depth_stencil = vk::PipelineDepthStencilStateCreateInfo::default();
2234        if let Some(ref ds) = desc.depth_stencil {
2235            (
2236                compatible_rp_key.depth_read_only,
2237                compatible_rp_key.stencil_read_only,
2238            ) = (
2239                ds.is_depth_read_only(),
2240                ds.is_stencil_read_only(desc.primitive.cull_mode),
2241            );
2242            let vk_format = self.shared.private_caps.map_texture_format(ds.format);
2243            let vk_layout = match (
2244                compatible_rp_key.depth_read_only,
2245                compatible_rp_key.stencil_read_only,
2246            ) {
2247                (true, true) => vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL,
2248                (true, false) => vk::ImageLayout::DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL,
2249                (false, true) => vk::ImageLayout::DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL,
2250                (false, false) => vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL,
2251            };
2252            compatible_rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
2253                base: super::AttachmentKey::compatible(vk_format, vk_layout),
2254                stencil_ops: crate::AttachmentOps::all(),
2255            });
2256
2257            if ds.is_depth_enabled() {
2258                vk_depth_stencil = vk_depth_stencil
2259                    .depth_test_enable(true)
2260                    .depth_write_enable(ds.depth_write_enabled.unwrap_or_default())
2261                    .depth_compare_op(conv::map_comparison(ds.depth_compare.unwrap_or_default()));
2262            }
2263            if ds.stencil.is_enabled() {
2264                let s = &ds.stencil;
2265                let front = conv::map_stencil_face(&s.front, s.read_mask, s.write_mask);
2266                let back = conv::map_stencil_face(&s.back, s.read_mask, s.write_mask);
2267                vk_depth_stencil = vk_depth_stencil
2268                    .stencil_test_enable(true)
2269                    .front(front)
2270                    .back(back);
2271            }
2272
2273            if ds.bias.is_enabled() {
2274                vk_rasterization = vk_rasterization
2275                    .depth_bias_enable(true)
2276                    .depth_bias_constant_factor(ds.bias.constant as f32)
2277                    .depth_bias_clamp(ds.bias.clamp)
2278                    .depth_bias_slope_factor(ds.bias.slope_scale);
2279            }
2280        }
2281
2282        let vk_viewport = vk::PipelineViewportStateCreateInfo::default()
2283            .flags(vk::PipelineViewportStateCreateFlags::empty())
2284            .scissor_count(1)
2285            .viewport_count(1);
2286
2287        let vk_sample_mask = [
2288            desc.multisample.mask as u32,
2289            (desc.multisample.mask >> 32) as u32,
2290        ];
2291        let vk_multisample = vk::PipelineMultisampleStateCreateInfo::default()
2292            .rasterization_samples(vk::SampleCountFlags::from_raw(desc.multisample.count))
2293            .alpha_to_coverage_enable(desc.multisample.alpha_to_coverage_enabled)
2294            .sample_mask(&vk_sample_mask);
2295
2296        let mut vk_attachments = Vec::with_capacity(desc.color_targets.len());
2297        for cat in desc.color_targets {
2298            let (key, attarchment) = if let Some(cat) = cat.as_ref() {
2299                let mut vk_attachment = vk::PipelineColorBlendAttachmentState::default()
2300                    .color_write_mask(vk::ColorComponentFlags::from_raw(cat.write_mask.bits()));
2301                if let Some(ref blend) = cat.blend {
2302                    let (color_op, color_src, color_dst) = conv::map_blend_component(&blend.color);
2303                    let (alpha_op, alpha_src, alpha_dst) = conv::map_blend_component(&blend.alpha);
2304                    vk_attachment = vk_attachment
2305                        .blend_enable(true)
2306                        .color_blend_op(color_op)
2307                        .src_color_blend_factor(color_src)
2308                        .dst_color_blend_factor(color_dst)
2309                        .alpha_blend_op(alpha_op)
2310                        .src_alpha_blend_factor(alpha_src)
2311                        .dst_alpha_blend_factor(alpha_dst);
2312                }
2313
2314                let vk_format = self.shared.private_caps.map_texture_format(cat.format);
2315                (
2316                    Some(super::ColorAttachmentKey {
2317                        base: super::AttachmentKey::compatible(
2318                            vk_format,
2319                            vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
2320                        ),
2321                        resolve: None,
2322                    }),
2323                    vk_attachment,
2324                )
2325            } else {
2326                (None, vk::PipelineColorBlendAttachmentState::default())
2327            };
2328
2329            compatible_rp_key.colors.push(key);
2330            vk_attachments.push(attarchment);
2331        }
2332
2333        let vk_color_blend =
2334            vk::PipelineColorBlendStateCreateInfo::default().attachments(&vk_attachments);
2335
2336        let vk_dynamic_state =
2337            vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic_states);
2338
2339        let raw_pass = self.shared.make_render_pass(compatible_rp_key)?;
2340
2341        let vk_infos = [{
2342            vk::GraphicsPipelineCreateInfo::default()
2343                .layout(desc.layout.raw)
2344                .stages(&stages)
2345                .vertex_input_state(&vk_vertex_input)
2346                .input_assembly_state(&vk_input_assembly)
2347                .rasterization_state(&vk_rasterization)
2348                .viewport_state(&vk_viewport)
2349                .multisample_state(&vk_multisample)
2350                .depth_stencil_state(&vk_depth_stencil)
2351                .color_blend_state(&vk_color_blend)
2352                .dynamic_state(&vk_dynamic_state)
2353                .render_pass(raw_pass)
2354        }];
2355
2356        let pipeline_cache = desc
2357            .cache
2358            .map(|it| it.raw)
2359            .unwrap_or(vk::PipelineCache::null());
2360
2361        let mut raw_vec = {
2362            profiling::scope!("vkCreateGraphicsPipelines");
2363            unsafe {
2364                self.shared
2365                    .raw
2366                    .create_graphics_pipelines(pipeline_cache, &vk_infos, None)
2367                    .map_err(|(_, e)| super::map_pipeline_err(e))
2368            }?
2369        };
2370
2371        let raw = raw_vec.pop().unwrap();
2372        if let Some(label) = desc.label {
2373            unsafe { self.shared.set_object_name(raw, label) };
2374        }
2375
2376        if let Some(CompiledStage {
2377            temp_raw_module: Some(raw_module),
2378            ..
2379        }) = compiled_vs
2380        {
2381            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2382        }
2383        if let Some(CompiledStage {
2384            temp_raw_module: Some(raw_module),
2385            ..
2386        }) = compiled_ts
2387        {
2388            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2389        }
2390        if let Some(CompiledStage {
2391            temp_raw_module: Some(raw_module),
2392            ..
2393        }) = compiled_ms
2394        {
2395            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2396        }
2397        if let Some(CompiledStage {
2398            temp_raw_module: Some(raw_module),
2399            ..
2400        }) = compiled_fs
2401        {
2402            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2403        }
2404
2405        self.counters.render_pipelines.add(1);
2406
2407        Ok(super::RenderPipeline {
2408            raw,
2409            is_multiview: desc.multiview_mask.is_some(),
2410        })
2411    }
2412
2413    unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) {
2414        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2415
2416        self.counters.render_pipelines.sub(1);
2417    }
2418
2419    unsafe fn create_compute_pipeline(
2420        &self,
2421        desc: &crate::ComputePipelineDescriptor<
2422            super::PipelineLayout,
2423            super::ShaderModule,
2424            super::PipelineCache,
2425        >,
2426    ) -> Result<super::ComputePipeline, crate::PipelineError> {
2427        let compiled = self.compile_stage(
2428            &desc.stage,
2429            naga::ShaderStage::Compute,
2430            &desc.layout.binding_map,
2431        )?;
2432
2433        let vk_infos = [{
2434            vk::ComputePipelineCreateInfo::default()
2435                .layout(desc.layout.raw)
2436                .stage(compiled.create_info)
2437        }];
2438
2439        let pipeline_cache = desc
2440            .cache
2441            .map(|it| it.raw)
2442            .unwrap_or(vk::PipelineCache::null());
2443
2444        let mut raw_vec = {
2445            profiling::scope!("vkCreateComputePipelines");
2446            unsafe {
2447                self.shared
2448                    .raw
2449                    .create_compute_pipelines(pipeline_cache, &vk_infos, None)
2450                    .map_err(|(_, e)| super::map_pipeline_err(e))
2451            }?
2452        };
2453
2454        let raw = raw_vec.pop().unwrap();
2455        if let Some(label) = desc.label {
2456            unsafe { self.shared.set_object_name(raw, label) };
2457        }
2458
2459        if let Some(raw_module) = compiled.temp_raw_module {
2460            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2461        }
2462
2463        self.counters.compute_pipelines.add(1);
2464
2465        Ok(super::ComputePipeline { raw })
2466    }
2467
2468    unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) {
2469        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2470
2471        self.counters.compute_pipelines.sub(1);
2472    }
2473
2474    unsafe fn create_ray_tracing_pipeline(
2475        &self,
2476        desc: &crate::RayTracingPipelineDescriptor<
2477            super::PipelineLayout,
2478            super::ShaderModule,
2479            super::PipelineCache,
2480        >,
2481    ) -> Result<super::RayTracingPipeline, crate::PipelineError> {
2482        let mut stages = Vec::new();
2483        let mut groups = Vec::new();
2484
2485        let compiled_ray_gen = self.compile_stage(
2486            &desc.ray_generation,
2487            naga::ShaderStage::RayGeneration,
2488            &desc.layout.binding_map,
2489        )?;
2490
2491        groups.push(
2492            vk::RayTracingShaderGroupCreateInfoKHR::default()
2493                .closest_hit_shader(vk::SHADER_UNUSED_KHR)
2494                .any_hit_shader(vk::SHADER_UNUSED_KHR)
2495                .intersection_shader(vk::SHADER_UNUSED_KHR)
2496                .general_shader(0) // stages is empty so next index is 0.
2497                .ty(vk::RayTracingShaderGroupTypeKHR::GENERAL),
2498        );
2499
2500        stages.push(compiled_ray_gen.create_info);
2501
2502        let compiled_miss = self.compile_stage(
2503            &desc.miss,
2504            naga::ShaderStage::Miss,
2505            &desc.layout.binding_map,
2506        )?;
2507
2508        groups.push(
2509            vk::RayTracingShaderGroupCreateInfoKHR::default()
2510                .closest_hit_shader(vk::SHADER_UNUSED_KHR)
2511                .any_hit_shader(vk::SHADER_UNUSED_KHR)
2512                .intersection_shader(vk::SHADER_UNUSED_KHR)
2513                .general_shader(1) // stages always has one element so next index is 1.
2514                .ty(vk::RayTracingShaderGroupTypeKHR::GENERAL),
2515        );
2516
2517        stages.push(compiled_miss.create_info);
2518
2519        // This is to keep alive the CStrings, as the ones in the loop would be deallocated
2520        // causing UB otherwise.
2521        let mut compiled_stages = Vec::new();
2522
2523        for group in desc.intersection {
2524            let compiled_closest_hits = self.compile_stage(
2525                &group.closest_hit,
2526                naga::ShaderStage::ClosestHit,
2527                &desc.layout.binding_map,
2528            )?;
2529
2530            let closest_idx = stages.len();
2531
2532            stages.push(compiled_closest_hits.create_info);
2533
2534            compiled_stages.push(compiled_closest_hits);
2535
2536            let mut raw_hit: vk::RayTracingShaderGroupCreateInfoKHR<'_> =
2537                vk::RayTracingShaderGroupCreateInfoKHR::default()
2538                    .closest_hit_shader(closest_idx as _)
2539                    .any_hit_shader(vk::SHADER_UNUSED_KHR)
2540                    .intersection_shader(vk::SHADER_UNUSED_KHR)
2541                    .general_shader(vk::SHADER_UNUSED_KHR)
2542                    .ty(vk::RayTracingShaderGroupTypeKHR::TRIANGLES_HIT_GROUP);
2543
2544            if let Some(any_hit) = &group.any_hit {
2545                let compiled_any_hit = self.compile_stage(
2546                    any_hit,
2547                    naga::ShaderStage::AnyHit,
2548                    &desc.layout.binding_map,
2549                )?;
2550
2551                let any_idx = stages.len();
2552
2553                stages.push(compiled_any_hit.create_info);
2554
2555                compiled_stages.push(compiled_any_hit);
2556
2557                raw_hit = raw_hit.any_hit_shader(any_idx as _);
2558            }
2559
2560            groups.push(raw_hit);
2561        }
2562
2563        let create_infos = [{
2564            vk::RayTracingPipelineCreateInfoKHR::default()
2565                .layout(desc.layout.raw)
2566                .max_pipeline_ray_recursion_depth(desc.max_recursion_depth)
2567                .stages(&stages)
2568                .groups(&groups)
2569        }];
2570
2571        let pipeline_cache = desc
2572            .cache
2573            .map(|it| it.raw)
2574            .unwrap_or(vk::PipelineCache::null());
2575
2576        let fns = self
2577            .shared
2578            .extension_fns
2579            .ray_tracing_pipelines
2580            .as_ref()
2581            .unwrap();
2582        let pipelines = unsafe {
2583            fns.create_ray_tracing_pipelines(
2584                vk::DeferredOperationKHR::null(),
2585                pipeline_cache,
2586                &create_infos,
2587                None,
2588            )
2589            .map_err(|(_, e)| super::map_pipeline_err(e))
2590        }?;
2591
2592        if let Some(raw_module) = compiled_ray_gen.temp_raw_module {
2593            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2594        }
2595
2596        if let Some(raw_module) = compiled_miss.temp_raw_module {
2597            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2598        }
2599
2600        for raw_module in compiled_stages
2601            .into_iter()
2602            .flat_map(|stage| stage.temp_raw_module)
2603        {
2604            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2605        }
2606
2607        self.counters.ray_tracing_pipelines.add(1);
2608
2609        Ok(super::RayTracingPipeline { raw: pipelines[0] })
2610    }
2611
2612    unsafe fn destroy_ray_tracing_pipeline(&self, pipeline: super::RayTracingPipeline) {
2613        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2614
2615        self.counters.ray_tracing_pipelines.sub(1);
2616    }
2617
2618    unsafe fn get_raytracing_pipeline_group_data(
2619        &self,
2620        pipeline: &super::RayTracingPipeline,
2621        groups: core::ops::Range<u32>,
2622    ) -> Result<Vec<u8>, crate::DeviceError> {
2623        let fns = self
2624            .shared
2625            .extension_fns
2626            .ray_tracing_pipelines
2627            .as_ref()
2628            .unwrap();
2629
2630        let num = groups.end - groups.start;
2631
2632        unsafe {
2633            fns.get_ray_tracing_shader_group_handles(
2634                pipeline.raw,
2635                groups.start,
2636                num,
2637                (num * self
2638                    .shared
2639                    .private_caps
2640                    .ray_tracing_pipeline_group_data_size) as usize,
2641            )
2642        }
2643        .map_err(super::map_host_device_oom_err)
2644    }
2645
2646    unsafe fn create_pipeline_cache(
2647        &self,
2648        desc: &crate::PipelineCacheDescriptor<'_>,
2649    ) -> Result<super::PipelineCache, crate::PipelineCacheError> {
2650        let mut info = vk::PipelineCacheCreateInfo::default();
2651        if let Some(data) = desc.data {
2652            info = info.initial_data(data)
2653        }
2654        profiling::scope!("vkCreatePipelineCache");
2655        let raw = unsafe { self.shared.raw.create_pipeline_cache(&info, None) }
2656            .map_err(super::map_host_device_oom_err)?;
2657
2658        Ok(super::PipelineCache { raw })
2659    }
2660    fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> {
2661        Some(self.shared.pipeline_cache_validation_key)
2662    }
2663    unsafe fn destroy_pipeline_cache(&self, cache: super::PipelineCache) {
2664        unsafe { self.shared.raw.destroy_pipeline_cache(cache.raw, None) }
2665    }
2666    unsafe fn create_query_set(
2667        &self,
2668        desc: &wgt::QuerySetDescriptor<crate::Label>,
2669    ) -> Result<super::QuerySet, crate::DeviceError> {
2670        // Assume each query is 256 bytes. This is the case for occlusion
2671        // queries on an AMD W6800 with driver version 32.0.12030.9. The
2672        // size and allocation policy may vary; this is an approximate
2673        // check only.
2674        self.error_if_would_oom_on_resource_allocation(
2675            gpu_allocator::MemoryLocation::GpuToCpu,
2676            &vk::MemoryRequirements {
2677                size: desc.count as u64 * 256,
2678                alignment: 256,
2679                memory_type_bits: self.valid_ash_memory_types,
2680            },
2681        )?;
2682
2683        let (vk_type, pipeline_statistics) = match desc.ty {
2684            wgt::QueryType::Occlusion => (
2685                vk::QueryType::OCCLUSION,
2686                vk::QueryPipelineStatisticFlags::empty(),
2687            ),
2688            wgt::QueryType::PipelineStatistics(statistics) => (
2689                vk::QueryType::PIPELINE_STATISTICS,
2690                conv::map_pipeline_statistics(statistics),
2691            ),
2692            wgt::QueryType::Timestamp => (
2693                vk::QueryType::TIMESTAMP,
2694                vk::QueryPipelineStatisticFlags::empty(),
2695            ),
2696        };
2697
2698        let vk_info = vk::QueryPoolCreateInfo::default()
2699            .query_type(vk_type)
2700            .query_count(desc.count)
2701            .pipeline_statistics(pipeline_statistics);
2702
2703        let raw = unsafe { self.shared.raw.create_query_pool(&vk_info, None) }
2704            .map_err(super::map_host_device_oom_err)?;
2705        if let Some(label) = desc.label {
2706            unsafe { self.shared.set_object_name(raw, label) };
2707        }
2708
2709        self.counters.query_sets.add(1);
2710
2711        Ok(super::QuerySet { raw })
2712    }
2713
2714    unsafe fn destroy_query_set(&self, set: super::QuerySet) {
2715        unsafe { self.shared.raw.destroy_query_pool(set.raw, None) };
2716
2717        self.counters.query_sets.sub(1);
2718    }
2719
2720    unsafe fn create_fence(&self) -> Result<super::Fence, crate::DeviceError> {
2721        self.counters.fences.add(1);
2722
2723        Ok(if self.shared.private_caps.timeline_semaphores {
2724            let mut sem_type_info =
2725                vk::SemaphoreTypeCreateInfo::default().semaphore_type(vk::SemaphoreType::TIMELINE);
2726            let vk_info = vk::SemaphoreCreateInfo::default().push_next(&mut sem_type_info);
2727            let raw = unsafe { self.shared.raw.create_semaphore(&vk_info, None) }
2728                .map_err(super::map_host_device_oom_err)?;
2729
2730            super::Fence::TimelineSemaphore(raw)
2731        } else {
2732            super::Fence::FencePool(RwLock::new(super::FencePool {
2733                last_completed: 0,
2734                active: Vec::new(),
2735                free: Vec::new(),
2736            }))
2737        })
2738    }
2739    unsafe fn destroy_fence(&self, fence: super::Fence) {
2740        match fence {
2741            super::Fence::TimelineSemaphore(raw) => {
2742                unsafe { self.shared.raw.destroy_semaphore(raw, None) };
2743            }
2744            super::Fence::FencePool(pool) => {
2745                let super::FencePool {
2746                    active,
2747                    free,
2748                    last_completed: _,
2749                } = pool.into_inner();
2750
2751                for (_, raw) in active {
2752                    unsafe {
2753                        self.shared.raw.destroy_fence(Arc::into_inner(raw).expect("Fence should have its reference count be one by the end of each function"), None)
2754                    };
2755                }
2756                for raw in free {
2757                    unsafe { self.shared.raw.destroy_fence(raw, None) };
2758                }
2759            }
2760        }
2761
2762        self.counters.fences.sub(1);
2763    }
2764    unsafe fn get_fence_value(
2765        &self,
2766        fence: &super::Fence,
2767    ) -> Result<crate::FenceValue, crate::DeviceError> {
2768        fence.get_latest(
2769            &self.shared.raw,
2770            self.shared.extension_fns.timeline_semaphore.as_ref(),
2771        )
2772    }
2773    unsafe fn wait(
2774        &self,
2775        fence: &super::Fence,
2776        wait_value: crate::FenceValue,
2777        timeout: Option<Duration>,
2778    ) -> Result<bool, crate::DeviceError> {
2779        let timeout_ns = timeout
2780            .unwrap_or(Duration::MAX)
2781            .as_nanos()
2782            .min(u64::MAX as _) as u64;
2783        self.shared.wait_for_fence(fence, wait_value, timeout_ns)
2784    }
2785
2786    unsafe fn start_graphics_debugger_capture(&self) -> bool {
2787        #[cfg(feature = "renderdoc")]
2788        {
2789            // Renderdoc requires us to give us the pointer that vkInstance _points to_.
2790            let raw_vk_instance =
2791                vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2792            let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2793            unsafe {
2794                self.render_doc
2795                    .start_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2796            }
2797        }
2798        #[cfg(not(feature = "renderdoc"))]
2799        false
2800    }
2801    unsafe fn stop_graphics_debugger_capture(&self) {
2802        #[cfg(feature = "renderdoc")]
2803        {
2804            // Renderdoc requires us to give us the pointer that vkInstance _points to_.
2805            let raw_vk_instance =
2806                vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2807            let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2808
2809            unsafe {
2810                self.render_doc
2811                    .end_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2812            }
2813        }
2814    }
2815
2816    unsafe fn pipeline_cache_get_data(&self, cache: &super::PipelineCache) -> Option<Vec<u8>> {
2817        let data = unsafe { self.raw_device().get_pipeline_cache_data(cache.raw) };
2818        data.ok()
2819    }
2820
2821    unsafe fn get_acceleration_structure_build_sizes<'a>(
2822        &self,
2823        desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>,
2824    ) -> crate::AccelerationStructureBuildSizes {
2825        const CAPACITY: usize = 8;
2826
2827        let ray_tracing_functions = self
2828            .shared
2829            .extension_fns
2830            .ray_tracing
2831            .as_ref()
2832            .expect("Feature `RAY_TRACING` not enabled");
2833
2834        let (geometries, primitive_counts) = match *desc.entries {
2835            crate::AccelerationStructureEntries::Instances(ref instances) => {
2836                let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default();
2837
2838                let geometry = vk::AccelerationStructureGeometryKHR::default()
2839                    .geometry_type(vk::GeometryTypeKHR::INSTANCES)
2840                    .geometry(vk::AccelerationStructureGeometryDataKHR {
2841                        instances: instance_data,
2842                    });
2843
2844                (
2845                    smallvec::smallvec![geometry],
2846                    smallvec::smallvec![instances.count],
2847                )
2848            }
2849            crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
2850                let mut primitive_counts =
2851                    smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2852                let mut geometries = smallvec::SmallVec::<
2853                    [vk::AccelerationStructureGeometryKHR; CAPACITY],
2854                >::with_capacity(in_geometries.len());
2855
2856                for triangles in in_geometries {
2857                    let mut triangle_data =
2858                        vk::AccelerationStructureGeometryTrianglesDataKHR::default()
2859                            .index_type(vk::IndexType::NONE_KHR)
2860                            .vertex_format(conv::map_vertex_format(triangles.vertex_format))
2861                            .max_vertex(triangles.vertex_count)
2862                            .vertex_stride(triangles.vertex_stride)
2863                            // The vulkan spec suggests we could pass a non-zero invalid address here if fetching
2864                            // the real address has significant overhead, but we pass the real one to be on the
2865                            // safe side for now.
2866                            // from https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetAccelerationStructureBuildSizesKHR.html
2867                            // > The srcAccelerationStructure, dstAccelerationStructure, and mode members
2868                            // > of pBuildInfo are ignored. Any VkDeviceOrHostAddressKHR or VkDeviceOrHostAddressConstKHR
2869                            // > members of pBuildInfo are ignored by this command, except that the hostAddress
2870                            // > member of VkAccelerationStructureGeometryTrianglesDataKHR::transformData will
2871                            // > be examined to check if it is NULL.
2872                            .transform_data(vk::DeviceOrHostAddressConstKHR {
2873                                device_address: if desc
2874                                    .flags
2875                                    .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
2876                                {
2877                                    unsafe {
2878                                        ray_tracing_functions
2879                                            .buffer_device_address
2880                                            .get_buffer_device_address(
2881                                                &vk::BufferDeviceAddressInfo::default().buffer(
2882                                                    triangles
2883                                                        .transform
2884                                                        .as_ref()
2885                                                        .unwrap()
2886                                                        .buffer
2887                                                        .raw,
2888                                                ),
2889                                            )
2890                                    }
2891                                } else {
2892                                    0
2893                                },
2894                            });
2895
2896                    let pritive_count = if let Some(ref indices) = triangles.indices {
2897                        triangle_data =
2898                            triangle_data.index_type(conv::map_index_format(indices.format));
2899                        indices.count / 3
2900                    } else {
2901                        triangles.vertex_count / 3
2902                    };
2903
2904                    let geometry = vk::AccelerationStructureGeometryKHR::default()
2905                        .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
2906                        .geometry(vk::AccelerationStructureGeometryDataKHR {
2907                            triangles: triangle_data,
2908                        })
2909                        .flags(conv::map_acceleration_structure_geometry_flags(
2910                            triangles.flags,
2911                        ));
2912
2913                    geometries.push(geometry);
2914                    primitive_counts.push(pritive_count);
2915                }
2916                (geometries, primitive_counts)
2917            }
2918            crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
2919                let mut primitive_counts =
2920                    smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2921                let mut geometries = smallvec::SmallVec::<
2922                    [vk::AccelerationStructureGeometryKHR; CAPACITY],
2923                >::with_capacity(in_geometries.len());
2924                for aabb in in_geometries {
2925                    let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
2926                        .stride(aabb.stride);
2927
2928                    let geometry = vk::AccelerationStructureGeometryKHR::default()
2929                        .geometry_type(vk::GeometryTypeKHR::AABBS)
2930                        .geometry(vk::AccelerationStructureGeometryDataKHR { aabbs: aabbs_data })
2931                        .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
2932
2933                    geometries.push(geometry);
2934                    primitive_counts.push(aabb.count);
2935                }
2936                (geometries, primitive_counts)
2937            }
2938        };
2939
2940        let ty = match *desc.entries {
2941            crate::AccelerationStructureEntries::Instances(_) => {
2942                vk::AccelerationStructureTypeKHR::TOP_LEVEL
2943            }
2944            _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
2945        };
2946
2947        let geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
2948            .ty(ty)
2949            .flags(conv::map_acceleration_structure_flags(desc.flags))
2950            .geometries(&geometries);
2951
2952        let mut raw = Default::default();
2953        unsafe {
2954            ray_tracing_functions
2955                .acceleration_structure
2956                .get_acceleration_structure_build_sizes(
2957                    vk::AccelerationStructureBuildTypeKHR::DEVICE,
2958                    &geometry_info,
2959                    &primitive_counts,
2960                    &mut raw,
2961                )
2962        }
2963
2964        crate::AccelerationStructureBuildSizes {
2965            acceleration_structure_size: raw.acceleration_structure_size,
2966            update_scratch_size: raw.update_scratch_size,
2967            build_scratch_size: raw.build_scratch_size,
2968        }
2969    }
2970
2971    unsafe fn get_acceleration_structure_device_address(
2972        &self,
2973        acceleration_structure: &super::AccelerationStructure,
2974    ) -> wgt::BufferAddress {
2975        let ray_tracing_functions = self
2976            .shared
2977            .extension_fns
2978            .ray_tracing
2979            .as_ref()
2980            .expect("Feature `RAY_TRACING` not enabled");
2981
2982        unsafe {
2983            ray_tracing_functions
2984                .acceleration_structure
2985                .get_acceleration_structure_device_address(
2986                    &vk::AccelerationStructureDeviceAddressInfoKHR::default()
2987                        .acceleration_structure(acceleration_structure.raw),
2988                )
2989        }
2990    }
2991
2992    unsafe fn create_acceleration_structure(
2993        &self,
2994        desc: &crate::AccelerationStructureDescriptor,
2995    ) -> Result<super::AccelerationStructure, crate::DeviceError> {
2996        let ray_tracing_functions = self
2997            .shared
2998            .extension_fns
2999            .ray_tracing
3000            .as_ref()
3001            .expect("Feature `RAY_TRACING` not enabled");
3002
3003        let vk_buffer_info = vk::BufferCreateInfo::default()
3004            .size(desc.size)
3005            .usage(
3006                vk::BufferUsageFlags::ACCELERATION_STRUCTURE_STORAGE_KHR
3007                    | vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS,
3008            )
3009            .sharing_mode(vk::SharingMode::EXCLUSIVE);
3010
3011        unsafe {
3012            let raw_buffer = self
3013                .shared
3014                .raw
3015                .create_buffer(&vk_buffer_info, None)
3016                .map_err(super::map_host_device_oom_and_ioca_err)?;
3017
3018            let requirements = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
3019
3020            self.error_if_would_oom_on_resource_allocation(
3021                gpu_allocator::MemoryLocation::GpuOnly,
3022                &requirements,
3023            )
3024            .inspect_err(|_| {
3025                self.shared.raw.destroy_buffer(raw_buffer, None);
3026            })?;
3027
3028            let name = desc
3029                .label
3030                .unwrap_or("Unlabeled acceleration structure buffer");
3031
3032            let pool = if desc.allow_compaction {
3033                super::MemoryPool::Transient
3034            } else {
3035                super::MemoryPool::General
3036            };
3037            let allocation = self
3038                .mem_allocator
3039                .allocate(
3040                    pool,
3041                    &gpu_allocator::vulkan::AllocationCreateDesc {
3042                        name,
3043                        requirements,
3044                        location: gpu_allocator::MemoryLocation::GpuOnly,
3045                        linear: true, // Buffers are always linear
3046                        allocation_scheme:
3047                            gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
3048                    },
3049                )
3050                .inspect_err(|_| {
3051                    self.shared.raw.destroy_buffer(raw_buffer, None);
3052                })?;
3053
3054            self.shared
3055                .raw
3056                .bind_buffer_memory(raw_buffer, allocation.memory(), allocation.offset())
3057                .map_err(super::map_host_device_oom_and_ioca_err)
3058                .inspect_err(|_| {
3059                    self.shared.raw.destroy_buffer(raw_buffer, None);
3060                })?;
3061
3062            if let Some(label) = desc.label {
3063                self.shared.set_object_name(raw_buffer, label);
3064            }
3065
3066            let vk_info = vk::AccelerationStructureCreateInfoKHR::default()
3067                .buffer(raw_buffer)
3068                .offset(0)
3069                .size(desc.size)
3070                .ty(conv::map_acceleration_structure_format(desc.format));
3071
3072            let raw_acceleration_structure = ray_tracing_functions
3073                .acceleration_structure
3074                .create_acceleration_structure(&vk_info, None)
3075                .map_err(super::map_host_oom_and_ioca_err)
3076                .inspect_err(|_| {
3077                    self.shared.raw.destroy_buffer(raw_buffer, None);
3078                })?;
3079
3080            if let Some(label) = desc.label {
3081                self.shared
3082                    .set_object_name(raw_acceleration_structure, label);
3083            }
3084
3085            let pool = if desc.allow_compaction {
3086                let vk_info = vk::QueryPoolCreateInfo::default()
3087                    .query_type(vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR)
3088                    .query_count(1);
3089
3090                let raw = self
3091                    .shared
3092                    .raw
3093                    .create_query_pool(&vk_info, None)
3094                    .map_err(super::map_host_device_oom_err)
3095                    .inspect_err(|_| {
3096                        ray_tracing_functions
3097                            .acceleration_structure
3098                            .destroy_acceleration_structure(raw_acceleration_structure, None);
3099                        self.shared.raw.destroy_buffer(raw_buffer, None);
3100                    })?;
3101                Some(raw)
3102            } else {
3103                None
3104            };
3105
3106            Ok(super::AccelerationStructure {
3107                raw: raw_acceleration_structure,
3108                buffer: raw_buffer,
3109                allocation,
3110                compacted_size_query: pool,
3111            })
3112        }
3113    }
3114
3115    unsafe fn destroy_acceleration_structure(
3116        &self,
3117        acceleration_structure: super::AccelerationStructure,
3118    ) {
3119        let ray_tracing_functions = self
3120            .shared
3121            .extension_fns
3122            .ray_tracing
3123            .as_ref()
3124            .expect("Feature `RAY_TRACING` not enabled");
3125
3126        unsafe {
3127            ray_tracing_functions
3128                .acceleration_structure
3129                .destroy_acceleration_structure(acceleration_structure.raw, None);
3130            self.shared
3131                .raw
3132                .destroy_buffer(acceleration_structure.buffer, None);
3133            let result = self.mem_allocator.free(acceleration_structure.allocation);
3134            if let Err(err) = result {
3135                log::warn!("Failed to free buffer acceleration structure: {err}");
3136            }
3137            if let Some(query) = acceleration_structure.compacted_size_query {
3138                self.shared.raw.destroy_query_pool(query, None)
3139            }
3140        }
3141    }
3142
3143    fn get_internal_counters(&self) -> wgt::HalCounters {
3144        self.counters
3145            .memory_allocations
3146            .set(self.shared.memory_allocations_counter.read());
3147
3148        self.counters.as_ref().clone()
3149    }
3150
3151    fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
3152        Some(self.mem_allocator.generate_report())
3153    }
3154
3155    fn tlas_instance_to_bytes(&self, instance: TlasInstance, to_extend: &mut Vec<u8>) {
3156        const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
3157        let temp = RawTlasInstance {
3158            transform: instance.transform,
3159            custom_data_and_mask: (instance.custom_data & MAX_U24)
3160                | (u32::from(instance.mask) << 24),
3161            shader_binding_table_record_offset_and_flags: (instance
3162                .pipeline_intersection_data_offset
3163                & MAX_U24),
3164            acceleration_structure_reference: instance.blas_address,
3165        };
3166        to_extend.extend_from_slice(bytemuck::bytes_of(&temp))
3167    }
3168
3169    fn check_if_oom(&self) -> Result<(), crate::DeviceError> {
3170        let Some(threshold) = self
3171            .shared
3172            .instance
3173            .memory_budget_thresholds
3174            .for_device_loss
3175        else {
3176            return Ok(());
3177        };
3178
3179        let Some(memory_properties) = self.get_memory_properties() else {
3180            return Ok(());
3181        };
3182
3183        for i in 0..memory_properties.heaps().len() {
3184            let heap_usage = memory_properties.heap_usage()[i];
3185            let heap_budget = memory_properties.heap_budget()[i];
3186
3187            if heap_usage >= heap_budget / 100 * threshold as u64 {
3188                return Err(crate::DeviceError::OutOfMemory);
3189            }
3190        }
3191
3192        Ok(())
3193    }
3194}
3195
3196impl super::DeviceShared {
3197    pub(super) fn new_binary_semaphore(
3198        &self,
3199        name: &str,
3200    ) -> Result<vk::Semaphore, crate::DeviceError> {
3201        unsafe {
3202            let semaphore = self
3203                .raw
3204                .create_semaphore(&vk::SemaphoreCreateInfo::default(), None)
3205                .map_err(super::map_host_device_oom_err)?;
3206
3207            if !self
3208                .instance
3209                .flags
3210                .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS)
3211            {
3212                self.set_object_name(semaphore, name);
3213            }
3214
3215            Ok(semaphore)
3216        }
3217    }
3218
3219    pub(super) fn wait_for_fence(
3220        &self,
3221        fence: &super::Fence,
3222        wait_value: crate::FenceValue,
3223        timeout_ns: u64,
3224    ) -> Result<bool, crate::DeviceError> {
3225        profiling::scope!("Device::wait");
3226        match *fence {
3227            super::Fence::TimelineSemaphore(raw) => {
3228                let semaphores = [raw];
3229                let values = [wait_value];
3230                let vk_info = vk::SemaphoreWaitInfo::default()
3231                    .semaphores(&semaphores)
3232                    .values(&values);
3233                let result = match self.extension_fns.timeline_semaphore {
3234                    Some(super::ExtensionFn::Extension(ref ext)) => unsafe {
3235                        ext.wait_semaphores(&vk_info, timeout_ns)
3236                    },
3237                    Some(super::ExtensionFn::Promoted) => unsafe {
3238                        self.raw.wait_semaphores(&vk_info, timeout_ns)
3239                    },
3240                    None => unreachable!(),
3241                };
3242                match result {
3243                    Ok(()) => Ok(true),
3244                    Err(vk::Result::TIMEOUT) => Ok(false),
3245                    Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
3246                }
3247            }
3248            super::Fence::FencePool(ref pool) => {
3249                let pool = pool.read();
3250                let super::FencePool {
3251                    last_completed,
3252                    ref active,
3253                    free: _,
3254                } = *pool;
3255                if wait_value <= last_completed {
3256                    Ok(true)
3257                } else {
3258                    match active.iter().find(|&&(value, _)| value >= wait_value) {
3259                        Some((_, fence)) => {
3260                            // clone to show we are using this fence while the pool is unlocked.
3261                            let fence = fence.clone();
3262                            drop(pool);
3263                            match unsafe {
3264                                self.raw.wait_for_fences(
3265                                    core::slice::from_ref(&fence),
3266                                    true,
3267                                    timeout_ns,
3268                                )
3269                            } {
3270                                Ok(()) => Ok(true),
3271                                Err(vk::Result::TIMEOUT) => Ok(false),
3272                                Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
3273                            }
3274                        }
3275                        None => {
3276                            crate::hal_usage_error(format!(
3277                                "no signals reached value {wait_value}"
3278                            ));
3279                        }
3280                    }
3281                }
3282            }
3283        }
3284    }
3285}
3286
3287struct ImageWithoutMemory {
3288    raw: vk::Image,
3289    requirements: vk::MemoryRequirements,
3290}