wgpu_hal/vulkan/
device.rs

1use alloc::{
2    borrow::{Cow, ToOwned as _},
3    collections::BTreeMap,
4    ffi::CString,
5    sync::Arc,
6    vec::Vec,
7};
8use core::{
9    ffi::CStr,
10    mem::{self, MaybeUninit},
11    num::NonZeroU32,
12    ptr,
13};
14
15use arrayvec::ArrayVec;
16use ash::{ext, khr, vk};
17use hashbrown::hash_map::Entry;
18use parking_lot::Mutex;
19
20use super::{conv, RawTlasInstance};
21use crate::TlasInstance;
22
23impl super::DeviceShared {
24    /// Set the name of `object` to `name`.
25    ///
26    /// If `name` contains an interior null byte, then the name set will be truncated to that byte.
27    ///
28    /// # Safety
29    ///
30    /// It must be valid to set `object`'s debug name
31    pub(super) unsafe fn set_object_name(&self, object: impl vk::Handle, name: &str) {
32        let Some(extension) = self.extension_fns.debug_utils.as_ref() else {
33            return;
34        };
35
36        // Keep variables outside the if-else block to ensure they do not
37        // go out of scope while we hold a pointer to them
38        let mut buffer: [u8; 64] = [0u8; 64];
39        let buffer_vec: Vec<u8>;
40
41        // Append a null terminator to the string
42        let name_bytes = if name.len() < buffer.len() {
43            // Common case, string is very small. Allocate a copy on the stack.
44            buffer[..name.len()].copy_from_slice(name.as_bytes());
45            // Add null terminator
46            buffer[name.len()] = 0;
47            &buffer[..name.len() + 1]
48        } else {
49            // Less common case, the string is large.
50            // This requires a heap allocation.
51            buffer_vec = name
52                .as_bytes()
53                .iter()
54                .cloned()
55                .chain(core::iter::once(0))
56                .collect();
57            &buffer_vec
58        };
59
60        let name = CStr::from_bytes_until_nul(name_bytes).expect("We have added a null byte");
61
62        let _result = unsafe {
63            extension.set_debug_utils_object_name(
64                &vk::DebugUtilsObjectNameInfoEXT::default()
65                    .object_handle(object)
66                    .object_name(name),
67            )
68        };
69    }
70
71    pub fn make_render_pass(
72        &self,
73        key: super::RenderPassKey,
74    ) -> Result<vk::RenderPass, crate::DeviceError> {
75        Ok(match self.render_passes.lock().entry(key) {
76            Entry::Occupied(e) => *e.get(),
77            Entry::Vacant(e) => {
78                let super::RenderPassKey {
79                    ref colors,
80                    ref depth_stencil,
81                    sample_count,
82                    multiview,
83                } = *e.key();
84
85                let mut vk_attachments = Vec::new();
86                let mut color_refs = Vec::with_capacity(colors.len());
87                let mut resolve_refs = Vec::with_capacity(color_refs.capacity());
88                let mut ds_ref = None;
89                let samples = vk::SampleCountFlags::from_raw(sample_count);
90                let unused = vk::AttachmentReference {
91                    attachment: vk::ATTACHMENT_UNUSED,
92                    layout: vk::ImageLayout::UNDEFINED,
93                };
94                for cat in colors.iter() {
95                    let (color_ref, resolve_ref) =
96                        if let Some(super::ColorAttachmentKey { base, resolve }) = cat {
97                            let super::AttachmentKey {
98                                format,
99                                layout,
100                                ops,
101                            } = *base;
102
103                            let color_ref = vk::AttachmentReference {
104                                attachment: vk_attachments.len() as u32,
105                                layout,
106                            };
107                            vk_attachments.push({
108                                let (load_op, store_op) = conv::map_attachment_ops(ops);
109                                vk::AttachmentDescription::default()
110                                    .format(format)
111                                    .samples(samples)
112                                    .load_op(load_op)
113                                    .store_op(store_op)
114                                    .initial_layout(layout)
115                                    .final_layout(layout)
116                            });
117                            let resolve_ref = if let Some(rat) = resolve {
118                                let super::AttachmentKey {
119                                    format,
120                                    layout,
121                                    ops,
122                                } = *rat;
123
124                                let (load_op, store_op) = conv::map_attachment_ops(ops);
125                                let vk_attachment = vk::AttachmentDescription::default()
126                                    .format(format)
127                                    .samples(vk::SampleCountFlags::TYPE_1)
128                                    .load_op(load_op)
129                                    .store_op(store_op)
130                                    .initial_layout(layout)
131                                    .final_layout(layout);
132                                vk_attachments.push(vk_attachment);
133
134                                vk::AttachmentReference {
135                                    attachment: vk_attachments.len() as u32 - 1,
136                                    layout,
137                                }
138                            } else {
139                                unused
140                            };
141
142                            (color_ref, resolve_ref)
143                        } else {
144                            (unused, unused)
145                        };
146
147                    color_refs.push(color_ref);
148                    resolve_refs.push(resolve_ref);
149                }
150
151                if let Some(ds) = depth_stencil {
152                    let super::DepthStencilAttachmentKey {
153                        ref base,
154                        stencil_ops,
155                    } = *ds;
156
157                    let super::AttachmentKey {
158                        format,
159                        layout,
160                        ops,
161                    } = *base;
162
163                    ds_ref = Some(vk::AttachmentReference {
164                        attachment: vk_attachments.len() as u32,
165                        layout,
166                    });
167                    let (load_op, store_op) = conv::map_attachment_ops(ops);
168                    let (stencil_load_op, stencil_store_op) = conv::map_attachment_ops(stencil_ops);
169                    let vk_attachment = vk::AttachmentDescription::default()
170                        .format(format)
171                        .samples(samples)
172                        .load_op(load_op)
173                        .store_op(store_op)
174                        .stencil_load_op(stencil_load_op)
175                        .stencil_store_op(stencil_store_op)
176                        .initial_layout(layout)
177                        .final_layout(layout);
178                    vk_attachments.push(vk_attachment);
179                }
180
181                let vk_subpasses = [{
182                    let mut vk_subpass = vk::SubpassDescription::default()
183                        .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
184                        .color_attachments(&color_refs)
185                        .resolve_attachments(&resolve_refs);
186
187                    if self
188                        .workarounds
189                        .contains(super::Workarounds::EMPTY_RESOLVE_ATTACHMENT_LISTS)
190                        && resolve_refs.is_empty()
191                    {
192                        vk_subpass.p_resolve_attachments = ptr::null();
193                    }
194
195                    if let Some(ref reference) = ds_ref {
196                        vk_subpass = vk_subpass.depth_stencil_attachment(reference)
197                    }
198                    vk_subpass
199                }];
200
201                let mut vk_info = vk::RenderPassCreateInfo::default()
202                    .attachments(&vk_attachments)
203                    .subpasses(&vk_subpasses);
204
205                let mut multiview_info;
206                let mask;
207                if let Some(multiview) = multiview {
208                    // Sanity checks, better to panic here than cause a driver crash
209                    assert!(multiview.get() <= 8);
210                    assert!(multiview.get() > 1);
211
212                    // Right now we enable all bits on the view masks and correlation masks.
213                    // This means we're rendering to all views in the subpass, and that all views
214                    // can be rendered concurrently.
215                    mask = [(1 << multiview.get()) - 1];
216
217                    // On Vulkan 1.1 or later, this is an alias for core functionality
218                    multiview_info = vk::RenderPassMultiviewCreateInfoKHR::default()
219                        .view_masks(&mask)
220                        .correlation_masks(&mask);
221                    vk_info = vk_info.push_next(&mut multiview_info);
222                }
223
224                let raw = unsafe {
225                    self.raw
226                        .create_render_pass(&vk_info, None)
227                        .map_err(super::map_host_device_oom_err)?
228                };
229
230                *e.insert(raw)
231            }
232        })
233    }
234
235    fn make_memory_ranges<'a, I: 'a + Iterator<Item = crate::MemoryRange>>(
236        &self,
237        buffer: &'a super::Buffer,
238        ranges: I,
239    ) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
240        let block = buffer.block.as_ref()?.lock();
241        let mask = self.private_caps.non_coherent_map_mask;
242        Some(ranges.map(move |range| {
243            vk::MappedMemoryRange::default()
244                .memory(*block.memory())
245                .offset((block.offset() + range.start) & !mask)
246                .size((range.end - range.start + mask) & !mask)
247        }))
248    }
249}
250
251impl gpu_alloc::MemoryDevice<vk::DeviceMemory> for super::DeviceShared {
252    unsafe fn allocate_memory(
253        &self,
254        size: u64,
255        memory_type: u32,
256        flags: gpu_alloc::AllocationFlags,
257    ) -> Result<vk::DeviceMemory, gpu_alloc::OutOfMemory> {
258        let mut info = vk::MemoryAllocateInfo::default()
259            .allocation_size(size)
260            .memory_type_index(memory_type);
261
262        let mut info_flags;
263
264        if flags.contains(gpu_alloc::AllocationFlags::DEVICE_ADDRESS) {
265            info_flags = vk::MemoryAllocateFlagsInfo::default()
266                .flags(vk::MemoryAllocateFlags::DEVICE_ADDRESS);
267            info = info.push_next(&mut info_flags);
268        }
269
270        match unsafe { self.raw.allocate_memory(&info, None) } {
271            Ok(memory) => {
272                self.memory_allocations_counter.add(1);
273                Ok(memory)
274            }
275            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
276                Err(gpu_alloc::OutOfMemory::OutOfDeviceMemory)
277            }
278            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
279                Err(gpu_alloc::OutOfMemory::OutOfHostMemory)
280            }
281            // We don't use VK_KHR_external_memory
282            // VK_ERROR_INVALID_EXTERNAL_HANDLE
283            // We don't use VK_KHR_buffer_device_address
284            // VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
285            Err(err) => handle_unexpected(err),
286        }
287    }
288
289    unsafe fn deallocate_memory(&self, memory: vk::DeviceMemory) {
290        self.memory_allocations_counter.sub(1);
291
292        unsafe { self.raw.free_memory(memory, None) };
293    }
294
295    unsafe fn map_memory(
296        &self,
297        memory: &mut vk::DeviceMemory,
298        offset: u64,
299        size: u64,
300    ) -> Result<ptr::NonNull<u8>, gpu_alloc::DeviceMapError> {
301        match unsafe {
302            self.raw
303                .map_memory(*memory, offset, size, vk::MemoryMapFlags::empty())
304        } {
305            Ok(ptr) => Ok(ptr::NonNull::new(ptr.cast::<u8>())
306                .expect("Pointer to memory mapping must not be null")),
307            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
308                Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory)
309            }
310            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
311                Err(gpu_alloc::DeviceMapError::OutOfHostMemory)
312            }
313            Err(vk::Result::ERROR_MEMORY_MAP_FAILED) => Err(gpu_alloc::DeviceMapError::MapFailed),
314            Err(err) => handle_unexpected(err),
315        }
316    }
317
318    unsafe fn unmap_memory(&self, memory: &mut vk::DeviceMemory) {
319        unsafe { self.raw.unmap_memory(*memory) };
320    }
321
322    unsafe fn invalidate_memory_ranges(
323        &self,
324        _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
325    ) -> Result<(), gpu_alloc::OutOfMemory> {
326        // should never be called
327        unimplemented!()
328    }
329
330    unsafe fn flush_memory_ranges(
331        &self,
332        _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
333    ) -> Result<(), gpu_alloc::OutOfMemory> {
334        // should never be called
335        unimplemented!()
336    }
337}
338
339impl
340    gpu_descriptor::DescriptorDevice<vk::DescriptorSetLayout, vk::DescriptorPool, vk::DescriptorSet>
341    for super::DeviceShared
342{
343    unsafe fn create_descriptor_pool(
344        &self,
345        descriptor_count: &gpu_descriptor::DescriptorTotalCount,
346        max_sets: u32,
347        flags: gpu_descriptor::DescriptorPoolCreateFlags,
348    ) -> Result<vk::DescriptorPool, gpu_descriptor::CreatePoolError> {
349        //Note: ignoring other types, since they can't appear here
350        let unfiltered_counts = [
351            (vk::DescriptorType::SAMPLER, descriptor_count.sampler),
352            (
353                vk::DescriptorType::SAMPLED_IMAGE,
354                descriptor_count.sampled_image,
355            ),
356            (
357                vk::DescriptorType::STORAGE_IMAGE,
358                descriptor_count.storage_image,
359            ),
360            (
361                vk::DescriptorType::UNIFORM_BUFFER,
362                descriptor_count.uniform_buffer,
363            ),
364            (
365                vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC,
366                descriptor_count.uniform_buffer_dynamic,
367            ),
368            (
369                vk::DescriptorType::STORAGE_BUFFER,
370                descriptor_count.storage_buffer,
371            ),
372            (
373                vk::DescriptorType::STORAGE_BUFFER_DYNAMIC,
374                descriptor_count.storage_buffer_dynamic,
375            ),
376            (
377                vk::DescriptorType::ACCELERATION_STRUCTURE_KHR,
378                descriptor_count.acceleration_structure,
379            ),
380        ];
381
382        let filtered_counts = unfiltered_counts
383            .iter()
384            .cloned()
385            .filter(|&(_, count)| count != 0)
386            .map(|(ty, count)| vk::DescriptorPoolSize {
387                ty,
388                descriptor_count: count,
389            })
390            .collect::<ArrayVec<_, 8>>();
391
392        let mut vk_flags =
393            if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND) {
394                vk::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND
395            } else {
396                vk::DescriptorPoolCreateFlags::empty()
397            };
398        if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET) {
399            vk_flags |= vk::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET;
400        }
401        let vk_info = vk::DescriptorPoolCreateInfo::default()
402            .max_sets(max_sets)
403            .flags(vk_flags)
404            .pool_sizes(&filtered_counts);
405
406        match unsafe { self.raw.create_descriptor_pool(&vk_info, None) } {
407            Ok(pool) => Ok(pool),
408            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
409                Err(gpu_descriptor::CreatePoolError::OutOfHostMemory)
410            }
411            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
412                Err(gpu_descriptor::CreatePoolError::OutOfDeviceMemory)
413            }
414            Err(vk::Result::ERROR_FRAGMENTATION) => {
415                Err(gpu_descriptor::CreatePoolError::Fragmentation)
416            }
417            Err(err) => handle_unexpected(err),
418        }
419    }
420
421    unsafe fn destroy_descriptor_pool(&self, pool: vk::DescriptorPool) {
422        unsafe { self.raw.destroy_descriptor_pool(pool, None) }
423    }
424
425    unsafe fn alloc_descriptor_sets<'a>(
426        &self,
427        pool: &mut vk::DescriptorPool,
428        layouts: impl ExactSizeIterator<Item = &'a vk::DescriptorSetLayout>,
429        sets: &mut impl Extend<vk::DescriptorSet>,
430    ) -> Result<(), gpu_descriptor::DeviceAllocationError> {
431        let result = unsafe {
432            self.raw.allocate_descriptor_sets(
433                &vk::DescriptorSetAllocateInfo::default()
434                    .descriptor_pool(*pool)
435                    .set_layouts(
436                        &smallvec::SmallVec::<[vk::DescriptorSetLayout; 32]>::from_iter(
437                            layouts.cloned(),
438                        ),
439                    ),
440            )
441        };
442
443        match result {
444            Ok(vk_sets) => {
445                sets.extend(vk_sets);
446                Ok(())
447            }
448            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY)
449            | Err(vk::Result::ERROR_OUT_OF_POOL_MEMORY) => {
450                Err(gpu_descriptor::DeviceAllocationError::OutOfHostMemory)
451            }
452            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
453                Err(gpu_descriptor::DeviceAllocationError::OutOfDeviceMemory)
454            }
455            Err(vk::Result::ERROR_FRAGMENTED_POOL) => {
456                Err(gpu_descriptor::DeviceAllocationError::FragmentedPool)
457            }
458            Err(err) => handle_unexpected(err),
459        }
460    }
461
462    unsafe fn dealloc_descriptor_sets<'a>(
463        &self,
464        pool: &mut vk::DescriptorPool,
465        sets: impl Iterator<Item = vk::DescriptorSet>,
466    ) {
467        let result = unsafe {
468            self.raw.free_descriptor_sets(
469                *pool,
470                &smallvec::SmallVec::<[vk::DescriptorSet; 32]>::from_iter(sets),
471            )
472        };
473        match result {
474            Ok(()) => {}
475            Err(err) => handle_unexpected(err),
476        }
477    }
478}
479
480struct CompiledStage {
481    create_info: vk::PipelineShaderStageCreateInfo<'static>,
482    _entry_point: CString,
483    temp_raw_module: Option<vk::ShaderModule>,
484}
485
486impl super::Device {
487    pub(super) unsafe fn create_swapchain(
488        &self,
489        surface: &super::Surface,
490        config: &crate::SurfaceConfiguration,
491        provided_old_swapchain: Option<super::Swapchain>,
492    ) -> Result<super::Swapchain, crate::SurfaceError> {
493        profiling::scope!("Device::create_swapchain");
494        let functor = khr::swapchain::Device::new(&surface.instance.raw, &self.shared.raw);
495
496        let old_swapchain = match provided_old_swapchain {
497            Some(osc) => osc.raw,
498            None => vk::SwapchainKHR::null(),
499        };
500
501        let color_space = if config.format == wgt::TextureFormat::Rgba16Float {
502            // Enable wide color gamut mode
503            // Vulkan swapchain for Android only supports DISPLAY_P3_NONLINEAR_EXT and EXTENDED_SRGB_LINEAR_EXT
504            vk::ColorSpaceKHR::EXTENDED_SRGB_LINEAR_EXT
505        } else {
506            vk::ColorSpaceKHR::SRGB_NONLINEAR
507        };
508
509        let original_format = self.shared.private_caps.map_texture_format(config.format);
510        let mut raw_flags = vk::SwapchainCreateFlagsKHR::empty();
511        let mut raw_view_formats: Vec<vk::Format> = vec![];
512        if !config.view_formats.is_empty() {
513            raw_flags |= vk::SwapchainCreateFlagsKHR::MUTABLE_FORMAT;
514            raw_view_formats = config
515                .view_formats
516                .iter()
517                .map(|f| self.shared.private_caps.map_texture_format(*f))
518                .collect();
519            raw_view_formats.push(original_format);
520        }
521
522        let mut info = vk::SwapchainCreateInfoKHR::default()
523            .flags(raw_flags)
524            .surface(surface.raw)
525            .min_image_count(config.maximum_frame_latency + 1) // TODO: https://github.com/gfx-rs/wgpu/issues/2869
526            .image_format(original_format)
527            .image_color_space(color_space)
528            .image_extent(vk::Extent2D {
529                width: config.extent.width,
530                height: config.extent.height,
531            })
532            .image_array_layers(config.extent.depth_or_array_layers)
533            .image_usage(conv::map_texture_usage(config.usage))
534            .image_sharing_mode(vk::SharingMode::EXCLUSIVE)
535            .pre_transform(vk::SurfaceTransformFlagsKHR::IDENTITY)
536            .composite_alpha(conv::map_composite_alpha_mode(config.composite_alpha_mode))
537            .present_mode(conv::map_present_mode(config.present_mode))
538            .clipped(true)
539            .old_swapchain(old_swapchain);
540
541        let mut format_list_info = vk::ImageFormatListCreateInfo::default();
542        if !raw_view_formats.is_empty() {
543            format_list_info = format_list_info.view_formats(&raw_view_formats);
544            info = info.push_next(&mut format_list_info);
545        }
546
547        let result = {
548            profiling::scope!("vkCreateSwapchainKHR");
549            unsafe { functor.create_swapchain(&info, None) }
550        };
551
552        // doing this before bailing out with error
553        if old_swapchain != vk::SwapchainKHR::null() {
554            unsafe { functor.destroy_swapchain(old_swapchain, None) }
555        }
556
557        let raw = match result {
558            Ok(swapchain) => swapchain,
559            Err(error) => {
560                return Err(match error {
561                    vk::Result::ERROR_SURFACE_LOST_KHR
562                    | vk::Result::ERROR_INITIALIZATION_FAILED => crate::SurfaceError::Lost,
563                    vk::Result::ERROR_NATIVE_WINDOW_IN_USE_KHR => {
564                        crate::SurfaceError::Other("Native window is in use")
565                    }
566                    // We don't use VK_EXT_image_compression_control
567                    // VK_ERROR_COMPRESSION_EXHAUSTED_EXT
568                    other => super::map_host_device_oom_and_lost_err(other).into(),
569                });
570            }
571        };
572
573        let images =
574            unsafe { functor.get_swapchain_images(raw) }.map_err(super::map_host_device_oom_err)?;
575
576        // NOTE: It's important that we define the same number of acquire/present semaphores
577        // as we will need to index into them with the image index.
578        let acquire_semaphores = (0..=images.len())
579            .map(|i| {
580                super::SwapchainAcquireSemaphore::new(&self.shared, i)
581                    .map(Mutex::new)
582                    .map(Arc::new)
583            })
584            .collect::<Result<Vec<_>, _>>()?;
585
586        let present_semaphores = (0..=images.len())
587            .map(|i| Arc::new(Mutex::new(super::SwapchainPresentSemaphores::new(i))))
588            .collect::<Vec<_>>();
589
590        Ok(super::Swapchain {
591            raw,
592            functor,
593            device: Arc::clone(&self.shared),
594            images,
595            config: config.clone(),
596            acquire_semaphores,
597            next_acquire_index: 0,
598            present_semaphores,
599            next_present_time: None,
600        })
601    }
602
603    /// # Safety
604    ///
605    /// - `vk_image` must be created respecting `desc`
606    /// - If `drop_callback` is [`None`], wgpu-hal will take ownership of `vk_image`. If
607    ///   `drop_callback` is [`Some`], `vk_image` must be valid until the callback is called.
608    /// - If the `ImageCreateFlags` does not contain `MUTABLE_FORMAT`, the `view_formats` of `desc` must be empty.
609    pub unsafe fn texture_from_raw(
610        &self,
611        vk_image: vk::Image,
612        desc: &crate::TextureDescriptor,
613        drop_callback: Option<crate::DropCallback>,
614    ) -> super::Texture {
615        let mut raw_flags = vk::ImageCreateFlags::empty();
616        let mut view_formats = vec![];
617        for tf in desc.view_formats.iter() {
618            if *tf == desc.format {
619                continue;
620            }
621            view_formats.push(*tf);
622        }
623        if !view_formats.is_empty() {
624            raw_flags |=
625                vk::ImageCreateFlags::MUTABLE_FORMAT | vk::ImageCreateFlags::EXTENDED_USAGE;
626            view_formats.push(desc.format)
627        }
628        if desc.format.is_multi_planar_format() {
629            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
630        }
631
632        let identity = self.shared.texture_identity_factory.next();
633
634        let drop_guard = crate::DropGuard::from_option(drop_callback);
635
636        super::Texture {
637            raw: vk_image,
638            drop_guard,
639            external_memory: None,
640            block: None,
641            format: desc.format,
642            copy_size: desc.copy_extent(),
643            identity,
644        }
645    }
646
647    #[cfg(windows)]
648    fn find_memory_type_index(
649        &self,
650        type_bits_req: u32,
651        flags_req: vk::MemoryPropertyFlags,
652    ) -> Option<usize> {
653        let mem_properties = unsafe {
654            self.shared
655                .instance
656                .raw
657                .get_physical_device_memory_properties(self.shared.physical_device)
658        };
659
660        // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPhysicalDeviceMemoryProperties.html
661        for (i, mem_ty) in mem_properties.memory_types_as_slice().iter().enumerate() {
662            let types_bits = 1 << i;
663            let is_required_memory_type = type_bits_req & types_bits != 0;
664            let has_required_properties = mem_ty.property_flags & flags_req == flags_req;
665            if is_required_memory_type && has_required_properties {
666                return Some(i);
667            }
668        }
669
670        None
671    }
672
673    fn create_image_without_memory(
674        &self,
675        desc: &crate::TextureDescriptor,
676        external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
677    ) -> Result<ImageWithoutMemory, crate::DeviceError> {
678        let copy_size = desc.copy_extent();
679
680        let mut raw_flags = vk::ImageCreateFlags::empty();
681        if desc.dimension == wgt::TextureDimension::D3
682            && desc.usage.contains(wgt::TextureUses::COLOR_TARGET)
683        {
684            raw_flags |= vk::ImageCreateFlags::TYPE_2D_ARRAY_COMPATIBLE;
685        }
686        if desc.is_cube_compatible() {
687            raw_flags |= vk::ImageCreateFlags::CUBE_COMPATIBLE;
688        }
689
690        let original_format = self.shared.private_caps.map_texture_format(desc.format);
691        let mut vk_view_formats = vec![];
692        if !desc.view_formats.is_empty() {
693            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
694
695            if self.shared.private_caps.image_format_list {
696                vk_view_formats = desc
697                    .view_formats
698                    .iter()
699                    .map(|f| self.shared.private_caps.map_texture_format(*f))
700                    .collect();
701                vk_view_formats.push(original_format)
702            }
703        }
704        if desc.format.is_multi_planar_format() {
705            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
706        }
707
708        let mut vk_info = vk::ImageCreateInfo::default()
709            .flags(raw_flags)
710            .image_type(conv::map_texture_dimension(desc.dimension))
711            .format(original_format)
712            .extent(conv::map_copy_extent(&copy_size))
713            .mip_levels(desc.mip_level_count)
714            .array_layers(desc.array_layer_count())
715            .samples(vk::SampleCountFlags::from_raw(desc.sample_count))
716            .tiling(vk::ImageTiling::OPTIMAL)
717            .usage(conv::map_texture_usage(desc.usage))
718            .sharing_mode(vk::SharingMode::EXCLUSIVE)
719            .initial_layout(vk::ImageLayout::UNDEFINED);
720
721        let mut format_list_info = vk::ImageFormatListCreateInfo::default();
722        if !vk_view_formats.is_empty() {
723            format_list_info = format_list_info.view_formats(&vk_view_formats);
724            vk_info = vk_info.push_next(&mut format_list_info);
725        }
726
727        if let Some(ext_info) = external_memory_image_create_info {
728            vk_info = vk_info.push_next(ext_info);
729        }
730
731        let raw = unsafe { self.shared.raw.create_image(&vk_info, None) }.map_err(map_err)?;
732        fn map_err(err: vk::Result) -> crate::DeviceError {
733            // We don't use VK_EXT_image_compression_control
734            // VK_ERROR_COMPRESSION_EXHAUSTED_EXT
735            super::map_host_device_oom_and_ioca_err(err)
736        }
737        let req = unsafe { self.shared.raw.get_image_memory_requirements(raw) };
738
739        Ok(ImageWithoutMemory {
740            raw,
741            requirements: req,
742            copy_size,
743        })
744    }
745
746    /// # Safety
747    ///
748    /// - Vulkan (with VK_KHR_external_memory_win32)
749    /// - The `d3d11_shared_handle` must be valid and respecting `desc`
750    /// - `VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_BIT` flag is used because we need to hold a reference to the handle
751    #[cfg(windows)]
752    pub unsafe fn texture_from_d3d11_shared_handle(
753        &self,
754        d3d11_shared_handle: windows::Win32::Foundation::HANDLE,
755        desc: &crate::TextureDescriptor,
756    ) -> Result<super::Texture, crate::DeviceError> {
757        if !self
758            .shared
759            .features
760            .contains(wgt::Features::VULKAN_EXTERNAL_MEMORY_WIN32)
761        {
762            log::error!("Vulkan driver does not support VK_KHR_external_memory_win32");
763            return Err(crate::DeviceError::Unexpected);
764        }
765
766        let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
767            .handle_types(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE);
768
769        let image =
770            self.create_image_without_memory(desc, Some(&mut external_memory_image_info))?;
771
772        // Some external memory types require dedicated allocation
773        // https://docs.vulkan.org/guide/latest/extensions/external.html#_importing_memory
774        let mut dedicated_allocate_info =
775            vk::MemoryDedicatedAllocateInfo::default().image(image.raw);
776
777        let mut import_memory_info = vk::ImportMemoryWin32HandleInfoKHR::default()
778            .handle_type(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE)
779            .handle(d3d11_shared_handle.0 as _);
780        // TODO: We should use `push_next` instead, but currently ash does not provide this method for the `ImportMemoryWin32HandleInfoKHR` type.
781        #[allow(clippy::unnecessary_mut_passed)]
782        {
783            import_memory_info.p_next = <*const _>::cast(&mut dedicated_allocate_info);
784        }
785
786        let mem_type_index = self
787            .find_memory_type_index(
788                image.requirements.memory_type_bits,
789                vk::MemoryPropertyFlags::DEVICE_LOCAL,
790            )
791            .ok_or(crate::DeviceError::Unexpected)?;
792
793        let memory_allocate_info = vk::MemoryAllocateInfo::default()
794            .allocation_size(image.requirements.size)
795            .memory_type_index(mem_type_index as _)
796            .push_next(&mut import_memory_info);
797        let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
798            .map_err(super::map_host_device_oom_err)?;
799
800        unsafe { self.shared.raw.bind_image_memory(image.raw, memory, 0) }
801            .map_err(super::map_host_device_oom_err)?;
802
803        if let Some(label) = desc.label {
804            unsafe { self.shared.set_object_name(image.raw, label) };
805        }
806
807        let identity = self.shared.texture_identity_factory.next();
808
809        self.counters.textures.add(1);
810
811        Ok(super::Texture {
812            raw: image.raw,
813            drop_guard: None,
814            external_memory: Some(memory),
815            block: None,
816            format: desc.format,
817            copy_size: image.copy_size,
818            identity,
819        })
820    }
821
822    fn create_shader_module_impl(
823        &self,
824        spv: &[u32],
825    ) -> Result<vk::ShaderModule, crate::DeviceError> {
826        let vk_info = vk::ShaderModuleCreateInfo::default()
827            .flags(vk::ShaderModuleCreateFlags::empty())
828            .code(spv);
829
830        let raw = unsafe {
831            profiling::scope!("vkCreateShaderModule");
832            self.shared
833                .raw
834                .create_shader_module(&vk_info, None)
835                .map_err(map_err)?
836        };
837        fn map_err(err: vk::Result) -> crate::DeviceError {
838            // We don't use VK_NV_glsl_shader
839            // VK_ERROR_INVALID_SHADER_NV
840            super::map_host_device_oom_err(err)
841        }
842        Ok(raw)
843    }
844
845    fn compile_stage(
846        &self,
847        stage: &crate::ProgrammableStage<super::ShaderModule>,
848        naga_stage: naga::ShaderStage,
849        binding_map: &naga::back::spv::BindingMap,
850    ) -> Result<CompiledStage, crate::PipelineError> {
851        let stage_flags = crate::auxil::map_naga_stage(naga_stage);
852        let vk_module = match *stage.module {
853            super::ShaderModule::Raw(raw) => raw,
854            super::ShaderModule::Intermediate {
855                ref naga_shader,
856                runtime_checks,
857            } => {
858                let pipeline_options = naga::back::spv::PipelineOptions {
859                    entry_point: stage.entry_point.to_owned(),
860                    shader_stage: naga_stage,
861                };
862                let needs_temp_options = !runtime_checks.bounds_checks
863                    || !runtime_checks.force_loop_bounding
864                    || !binding_map.is_empty()
865                    || naga_shader.debug_source.is_some()
866                    || !stage.zero_initialize_workgroup_memory;
867                let mut temp_options;
868                let options = if needs_temp_options {
869                    temp_options = self.naga_options.clone();
870                    if !runtime_checks.bounds_checks {
871                        temp_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
872                            index: naga::proc::BoundsCheckPolicy::Unchecked,
873                            buffer: naga::proc::BoundsCheckPolicy::Unchecked,
874                            image_load: naga::proc::BoundsCheckPolicy::Unchecked,
875                            binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
876                        };
877                    }
878                    if !runtime_checks.force_loop_bounding {
879                        temp_options.force_loop_bounding = false;
880                    }
881                    if !binding_map.is_empty() {
882                        temp_options.binding_map = binding_map.clone();
883                    }
884
885                    if let Some(ref debug) = naga_shader.debug_source {
886                        temp_options.debug_info = Some(naga::back::spv::DebugInfo {
887                            source_code: &debug.source_code,
888                            file_name: debug.file_name.as_ref().into(),
889                            language: naga::back::spv::SourceLanguage::WGSL,
890                        })
891                    }
892                    if !stage.zero_initialize_workgroup_memory {
893                        temp_options.zero_initialize_workgroup_memory =
894                            naga::back::spv::ZeroInitializeWorkgroupMemoryMode::None;
895                    }
896
897                    &temp_options
898                } else {
899                    &self.naga_options
900                };
901
902                let (module, info) = naga::back::pipeline_constants::process_overrides(
903                    &naga_shader.module,
904                    &naga_shader.info,
905                    Some((naga_stage, stage.entry_point)),
906                    stage.constants,
907                )
908                .map_err(|e| {
909                    crate::PipelineError::PipelineConstants(stage_flags, format!("{e}"))
910                })?;
911
912                let spv = {
913                    profiling::scope!("naga::spv::write_vec");
914                    naga::back::spv::write_vec(&module, &info, options, Some(&pipeline_options))
915                }
916                .map_err(|e| crate::PipelineError::Linkage(stage_flags, format!("{e}")))?;
917                self.create_shader_module_impl(&spv)?
918            }
919        };
920
921        let mut flags = vk::PipelineShaderStageCreateFlags::empty();
922        if self.shared.features.contains(wgt::Features::SUBGROUP) {
923            flags |= vk::PipelineShaderStageCreateFlags::ALLOW_VARYING_SUBGROUP_SIZE
924        }
925
926        let entry_point = CString::new(stage.entry_point).unwrap();
927        let mut create_info = vk::PipelineShaderStageCreateInfo::default()
928            .flags(flags)
929            .stage(conv::map_shader_stage(stage_flags))
930            .module(vk_module);
931
932        // Circumvent struct lifetime check because of a self-reference inside CompiledStage
933        create_info.p_name = entry_point.as_ptr();
934
935        Ok(CompiledStage {
936            create_info,
937            _entry_point: entry_point,
938            temp_raw_module: match *stage.module {
939                super::ShaderModule::Raw(_) => None,
940                super::ShaderModule::Intermediate { .. } => Some(vk_module),
941            },
942        })
943    }
944
945    /// Returns the queue family index of the device's internal queue.
946    ///
947    /// This is useful for constructing memory barriers needed for queue family ownership transfer when
948    /// external memory is involved (from/to `VK_QUEUE_FAMILY_EXTERNAL_KHR` and `VK_QUEUE_FAMILY_FOREIGN_EXT`
949    /// for example).
950    pub fn queue_family_index(&self) -> u32 {
951        self.shared.family_index
952    }
953
954    pub fn queue_index(&self) -> u32 {
955        self.shared.queue_index
956    }
957
958    pub fn raw_device(&self) -> &ash::Device {
959        &self.shared.raw
960    }
961
962    pub fn raw_physical_device(&self) -> vk::PhysicalDevice {
963        self.shared.physical_device
964    }
965
966    pub fn raw_queue(&self) -> vk::Queue {
967        self.shared.raw_queue
968    }
969
970    pub fn enabled_device_extensions(&self) -> &[&'static CStr] {
971        &self.shared.enabled_extensions
972    }
973
974    pub fn shared_instance(&self) -> &super::InstanceShared {
975        &self.shared.instance
976    }
977
978    fn error_if_would_oom_on_resource_allocation(
979        &self,
980        needs_host_access: bool,
981        size: u64,
982    ) -> Result<(), crate::DeviceError> {
983        let Some(threshold) = self
984            .shared
985            .instance
986            .memory_budget_thresholds
987            .for_resource_creation
988        else {
989            return Ok(());
990        };
991
992        if !self
993            .shared
994            .enabled_extensions
995            .contains(&ext::memory_budget::NAME)
996        {
997            return Ok(());
998        }
999
1000        let get_physical_device_properties = self
1001            .shared
1002            .instance
1003            .get_physical_device_properties
1004            .as_ref()
1005            .unwrap();
1006
1007        let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
1008
1009        let mut memory_properties =
1010            vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
1011
1012        unsafe {
1013            get_physical_device_properties.get_physical_device_memory_properties2(
1014                self.shared.physical_device,
1015                &mut memory_properties,
1016            );
1017        }
1018
1019        let mut host_visible_heaps = [false; vk::MAX_MEMORY_HEAPS];
1020        let mut device_local_heaps = [false; vk::MAX_MEMORY_HEAPS];
1021
1022        let memory_properties = memory_properties.memory_properties;
1023
1024        for i in 0..memory_properties.memory_type_count {
1025            let memory_type = memory_properties.memory_types[i as usize];
1026            let flags = memory_type.property_flags;
1027
1028            if flags.intersects(
1029                vk::MemoryPropertyFlags::LAZILY_ALLOCATED | vk::MemoryPropertyFlags::PROTECTED,
1030            ) {
1031                continue; // not used by gpu-alloc
1032            }
1033
1034            if flags.contains(vk::MemoryPropertyFlags::HOST_VISIBLE) {
1035                host_visible_heaps[memory_type.heap_index as usize] = true;
1036            }
1037
1038            if flags.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL) {
1039                device_local_heaps[memory_type.heap_index as usize] = true;
1040            }
1041        }
1042
1043        let heaps = if needs_host_access {
1044            host_visible_heaps
1045        } else {
1046            device_local_heaps
1047        };
1048
1049        // NOTE: We might end up checking multiple heaps since gpu-alloc doesn't have a way
1050        // for us to query the heap the resource will end up on. But this is unlikely,
1051        // there is usually only one heap on integrated GPUs and two on dedicated GPUs.
1052
1053        for (i, check) in heaps.iter().enumerate() {
1054            if !check {
1055                continue;
1056            }
1057
1058            let heap_usage = memory_budget_properties.heap_usage[i];
1059            let heap_budget = memory_budget_properties.heap_budget[i];
1060
1061            if heap_usage + size >= heap_budget / 100 * threshold as u64 {
1062                return Err(crate::DeviceError::OutOfMemory);
1063            }
1064        }
1065
1066        Ok(())
1067    }
1068}
1069
1070impl crate::Device for super::Device {
1071    type A = super::Api;
1072
1073    unsafe fn create_buffer(
1074        &self,
1075        desc: &crate::BufferDescriptor,
1076    ) -> Result<super::Buffer, crate::DeviceError> {
1077        let vk_info = vk::BufferCreateInfo::default()
1078            .size(desc.size)
1079            .usage(conv::map_buffer_usage(desc.usage))
1080            .sharing_mode(vk::SharingMode::EXCLUSIVE);
1081
1082        let raw = unsafe {
1083            self.shared
1084                .raw
1085                .create_buffer(&vk_info, None)
1086                .map_err(super::map_host_device_oom_and_ioca_err)?
1087        };
1088        let req = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
1089
1090        let mut alloc_usage = if desc
1091            .usage
1092            .intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE)
1093        {
1094            let mut flags = gpu_alloc::UsageFlags::HOST_ACCESS;
1095            //TODO: find a way to use `crate::MemoryFlags::PREFER_COHERENT`
1096            flags.set(
1097                gpu_alloc::UsageFlags::DOWNLOAD,
1098                desc.usage.contains(wgt::BufferUses::MAP_READ),
1099            );
1100            flags.set(
1101                gpu_alloc::UsageFlags::UPLOAD,
1102                desc.usage.contains(wgt::BufferUses::MAP_WRITE),
1103            );
1104            flags
1105        } else {
1106            gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS
1107        };
1108        alloc_usage.set(
1109            gpu_alloc::UsageFlags::TRANSIENT,
1110            desc.memory_flags.contains(crate::MemoryFlags::TRANSIENT),
1111        );
1112
1113        let needs_host_access = alloc_usage.contains(gpu_alloc::UsageFlags::HOST_ACCESS);
1114
1115        self.error_if_would_oom_on_resource_allocation(needs_host_access, req.size)
1116            .inspect_err(|_| {
1117                unsafe { self.shared.raw.destroy_buffer(raw, None) };
1118            })?;
1119
1120        let alignment_mask = req.alignment - 1;
1121
1122        let block = unsafe {
1123            self.mem_allocator.lock().alloc(
1124                &*self.shared,
1125                gpu_alloc::Request {
1126                    size: req.size,
1127                    align_mask: alignment_mask,
1128                    usage: alloc_usage,
1129                    memory_types: req.memory_type_bits & self.valid_ash_memory_types,
1130                },
1131            )
1132        }
1133        .inspect_err(|_| {
1134            unsafe { self.shared.raw.destroy_buffer(raw, None) };
1135        })?;
1136
1137        unsafe {
1138            self.shared
1139                .raw
1140                .bind_buffer_memory(raw, *block.memory(), block.offset())
1141        }
1142        .map_err(super::map_host_device_oom_and_ioca_err)
1143        .inspect_err(|_| {
1144            unsafe { self.shared.raw.destroy_buffer(raw, None) };
1145        })?;
1146
1147        if let Some(label) = desc.label {
1148            unsafe { self.shared.set_object_name(raw, label) };
1149        }
1150
1151        self.counters.buffer_memory.add(block.size() as isize);
1152        self.counters.buffers.add(1);
1153
1154        Ok(super::Buffer {
1155            raw,
1156            block: Some(Mutex::new(super::BufferMemoryBacking::Managed(block))),
1157        })
1158    }
1159    unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
1160        unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1161        if let Some(block) = buffer.block {
1162            let block = block.into_inner();
1163            self.counters.buffer_memory.sub(block.size() as isize);
1164            match block {
1165                super::BufferMemoryBacking::Managed(block) => unsafe {
1166                    self.mem_allocator.lock().dealloc(&*self.shared, block)
1167                },
1168                super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
1169                    self.shared.raw.free_memory(memory, None);
1170                },
1171            }
1172        }
1173
1174        self.counters.buffers.sub(1);
1175    }
1176
1177    unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) {
1178        self.counters.buffers.add(1);
1179    }
1180
1181    unsafe fn map_buffer(
1182        &self,
1183        buffer: &super::Buffer,
1184        range: crate::MemoryRange,
1185    ) -> Result<crate::BufferMapping, crate::DeviceError> {
1186        if let Some(ref block) = buffer.block {
1187            let size = range.end - range.start;
1188            let mut block = block.lock();
1189            if let super::BufferMemoryBacking::Managed(ref mut block) = *block {
1190                let ptr = unsafe { block.map(&*self.shared, range.start, size as usize)? };
1191                let is_coherent = block
1192                    .props()
1193                    .contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT);
1194                Ok(crate::BufferMapping { ptr, is_coherent })
1195            } else {
1196                crate::hal_usage_error("tried to map externally created buffer")
1197            }
1198        } else {
1199            crate::hal_usage_error("tried to map external buffer")
1200        }
1201    }
1202    unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
1203        if let Some(ref block) = buffer.block {
1204            match &mut *block.lock() {
1205                super::BufferMemoryBacking::Managed(block) => unsafe { block.unmap(&*self.shared) },
1206                super::BufferMemoryBacking::VulkanMemory { .. } => {
1207                    crate::hal_usage_error("tried to unmap externally created buffer")
1208                }
1209            };
1210        } else {
1211            crate::hal_usage_error("tried to unmap external buffer")
1212        }
1213    }
1214
1215    unsafe fn flush_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1216    where
1217        I: Iterator<Item = crate::MemoryRange>,
1218    {
1219        if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1220            unsafe {
1221                self.shared
1222                    .raw
1223                    .flush_mapped_memory_ranges(
1224                        &smallvec::SmallVec::<[vk::MappedMemoryRange; 32]>::from_iter(vk_ranges),
1225                    )
1226            }
1227            .unwrap();
1228        }
1229    }
1230    unsafe fn invalidate_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1231    where
1232        I: Iterator<Item = crate::MemoryRange>,
1233    {
1234        if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1235            unsafe {
1236                self.shared
1237                    .raw
1238                    .invalidate_mapped_memory_ranges(&smallvec::SmallVec::<
1239                        [vk::MappedMemoryRange; 32],
1240                    >::from_iter(vk_ranges))
1241            }
1242            .unwrap();
1243        }
1244    }
1245
1246    unsafe fn create_texture(
1247        &self,
1248        desc: &crate::TextureDescriptor,
1249    ) -> Result<super::Texture, crate::DeviceError> {
1250        let image = self.create_image_without_memory(desc, None)?;
1251
1252        self.error_if_would_oom_on_resource_allocation(false, image.requirements.size)
1253            .inspect_err(|_| {
1254                unsafe { self.shared.raw.destroy_image(image.raw, None) };
1255            })?;
1256
1257        let block = unsafe {
1258            self.mem_allocator.lock().alloc(
1259                &*self.shared,
1260                gpu_alloc::Request {
1261                    size: image.requirements.size,
1262                    align_mask: image.requirements.alignment - 1,
1263                    usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
1264                    memory_types: image.requirements.memory_type_bits & self.valid_ash_memory_types,
1265                },
1266            )
1267        }
1268        .inspect_err(|_| {
1269            unsafe { self.shared.raw.destroy_image(image.raw, None) };
1270        })?;
1271
1272        self.counters.texture_memory.add(block.size() as isize);
1273
1274        unsafe {
1275            self.shared
1276                .raw
1277                .bind_image_memory(image.raw, *block.memory(), block.offset())
1278        }
1279        .map_err(super::map_host_device_oom_err)
1280        .inspect_err(|_| {
1281            unsafe { self.shared.raw.destroy_image(image.raw, None) };
1282        })?;
1283
1284        if let Some(label) = desc.label {
1285            unsafe { self.shared.set_object_name(image.raw, label) };
1286        }
1287
1288        let identity = self.shared.texture_identity_factory.next();
1289
1290        self.counters.textures.add(1);
1291
1292        Ok(super::Texture {
1293            raw: image.raw,
1294            drop_guard: None,
1295            external_memory: None,
1296            block: Some(block),
1297            format: desc.format,
1298            copy_size: image.copy_size,
1299            identity,
1300        })
1301    }
1302    unsafe fn destroy_texture(&self, texture: super::Texture) {
1303        if texture.drop_guard.is_none() {
1304            unsafe { self.shared.raw.destroy_image(texture.raw, None) };
1305        }
1306        if let Some(memory) = texture.external_memory {
1307            unsafe { self.shared.raw.free_memory(memory, None) };
1308        }
1309        if let Some(block) = texture.block {
1310            self.counters.texture_memory.sub(block.size() as isize);
1311
1312            unsafe { self.mem_allocator.lock().dealloc(&*self.shared, block) };
1313        }
1314
1315        self.counters.textures.sub(1);
1316    }
1317
1318    unsafe fn add_raw_texture(&self, _texture: &super::Texture) {
1319        self.counters.textures.add(1);
1320    }
1321
1322    unsafe fn create_texture_view(
1323        &self,
1324        texture: &super::Texture,
1325        desc: &crate::TextureViewDescriptor,
1326    ) -> Result<super::TextureView, crate::DeviceError> {
1327        let subresource_range = conv::map_subresource_range(&desc.range, texture.format);
1328        let raw_format = self.shared.private_caps.map_texture_format(desc.format);
1329        let mut vk_info = vk::ImageViewCreateInfo::default()
1330            .flags(vk::ImageViewCreateFlags::empty())
1331            .image(texture.raw)
1332            .view_type(conv::map_view_dimension(desc.dimension))
1333            .format(raw_format)
1334            .subresource_range(subresource_range);
1335        let layers =
1336            NonZeroU32::new(subresource_range.layer_count).expect("Unexpected zero layer count");
1337
1338        let mut image_view_info;
1339        if self.shared.private_caps.image_view_usage && !desc.usage.is_empty() {
1340            image_view_info =
1341                vk::ImageViewUsageCreateInfo::default().usage(conv::map_texture_usage(desc.usage));
1342            vk_info = vk_info.push_next(&mut image_view_info);
1343        }
1344
1345        let raw = unsafe { self.shared.raw.create_image_view(&vk_info, None) }
1346            .map_err(super::map_host_device_oom_and_ioca_err)?;
1347
1348        if let Some(label) = desc.label {
1349            unsafe { self.shared.set_object_name(raw, label) };
1350        }
1351
1352        let identity = self.shared.texture_view_identity_factory.next();
1353
1354        self.counters.texture_views.add(1);
1355
1356        Ok(super::TextureView {
1357            raw_texture: texture.raw,
1358            raw,
1359            layers,
1360            format: desc.format,
1361            raw_format,
1362            base_mip_level: desc.range.base_mip_level,
1363            dimension: desc.dimension,
1364            texture_identity: texture.identity,
1365            view_identity: identity,
1366        })
1367    }
1368    unsafe fn destroy_texture_view(&self, view: super::TextureView) {
1369        unsafe { self.shared.raw.destroy_image_view(view.raw, None) };
1370
1371        self.counters.texture_views.sub(1);
1372    }
1373
1374    unsafe fn create_sampler(
1375        &self,
1376        desc: &crate::SamplerDescriptor,
1377    ) -> Result<super::Sampler, crate::DeviceError> {
1378        let mut create_info = vk::SamplerCreateInfo::default()
1379            .flags(vk::SamplerCreateFlags::empty())
1380            .mag_filter(conv::map_filter_mode(desc.mag_filter))
1381            .min_filter(conv::map_filter_mode(desc.min_filter))
1382            .mipmap_mode(conv::map_mip_filter_mode(desc.mipmap_filter))
1383            .address_mode_u(conv::map_address_mode(desc.address_modes[0]))
1384            .address_mode_v(conv::map_address_mode(desc.address_modes[1]))
1385            .address_mode_w(conv::map_address_mode(desc.address_modes[2]))
1386            .min_lod(desc.lod_clamp.start)
1387            .max_lod(desc.lod_clamp.end);
1388
1389        if let Some(fun) = desc.compare {
1390            create_info = create_info
1391                .compare_enable(true)
1392                .compare_op(conv::map_comparison(fun));
1393        }
1394
1395        if desc.anisotropy_clamp != 1 {
1396            // We only enable anisotropy if it is supported, and wgpu-hal interface guarantees
1397            // the clamp is in the range [1, 16] which is always supported if anisotropy is.
1398            create_info = create_info
1399                .anisotropy_enable(true)
1400                .max_anisotropy(desc.anisotropy_clamp as f32);
1401        }
1402
1403        if let Some(color) = desc.border_color {
1404            create_info = create_info.border_color(conv::map_border_color(color));
1405        }
1406
1407        let mut sampler_cache_guard = self.shared.sampler_cache.lock();
1408
1409        let raw = sampler_cache_guard.create_sampler(&self.shared.raw, create_info)?;
1410
1411        // Note: Cached samplers will just continually overwrite the label
1412        //
1413        // https://github.com/gfx-rs/wgpu/issues/6867
1414        if let Some(label) = desc.label {
1415            // SAFETY: we are holding a lock on the sampler cache,
1416            // so we can only be setting the name from one thread.
1417            unsafe { self.shared.set_object_name(raw, label) };
1418        }
1419
1420        drop(sampler_cache_guard);
1421
1422        self.counters.samplers.add(1);
1423
1424        Ok(super::Sampler { raw, create_info })
1425    }
1426    unsafe fn destroy_sampler(&self, sampler: super::Sampler) {
1427        self.shared.sampler_cache.lock().destroy_sampler(
1428            &self.shared.raw,
1429            sampler.create_info,
1430            sampler.raw,
1431        );
1432
1433        self.counters.samplers.sub(1);
1434    }
1435
1436    unsafe fn create_command_encoder(
1437        &self,
1438        desc: &crate::CommandEncoderDescriptor<super::Queue>,
1439    ) -> Result<super::CommandEncoder, crate::DeviceError> {
1440        let vk_info = vk::CommandPoolCreateInfo::default()
1441            .queue_family_index(desc.queue.family_index)
1442            .flags(vk::CommandPoolCreateFlags::TRANSIENT);
1443
1444        let raw = unsafe {
1445            self.shared
1446                .raw
1447                .create_command_pool(&vk_info, None)
1448                .map_err(super::map_host_device_oom_err)?
1449        };
1450
1451        self.counters.command_encoders.add(1);
1452
1453        Ok(super::CommandEncoder {
1454            raw,
1455            device: Arc::clone(&self.shared),
1456            active: vk::CommandBuffer::null(),
1457            bind_point: vk::PipelineBindPoint::default(),
1458            temp: super::Temp::default(),
1459            free: Vec::new(),
1460            discarded: Vec::new(),
1461            rpass_debug_marker_active: false,
1462            end_of_pass_timer_query: None,
1463            framebuffers: Default::default(),
1464            temp_texture_views: Default::default(),
1465            counters: Arc::clone(&self.counters),
1466        })
1467    }
1468
1469    unsafe fn create_bind_group_layout(
1470        &self,
1471        desc: &crate::BindGroupLayoutDescriptor,
1472    ) -> Result<super::BindGroupLayout, crate::DeviceError> {
1473        let mut desc_count = gpu_descriptor::DescriptorTotalCount::default();
1474        let mut types = Vec::new();
1475        for entry in desc.entries {
1476            let count = entry.count.map_or(1, |c| c.get());
1477            if entry.binding as usize >= types.len() {
1478                types.resize(
1479                    entry.binding as usize + 1,
1480                    (vk::DescriptorType::INPUT_ATTACHMENT, 0),
1481                );
1482            }
1483            types[entry.binding as usize] = (
1484                conv::map_binding_type(entry.ty),
1485                entry.count.map_or(1, |c| c.get()),
1486            );
1487
1488            match entry.ty {
1489                wgt::BindingType::Buffer {
1490                    ty,
1491                    has_dynamic_offset,
1492                    ..
1493                } => match ty {
1494                    wgt::BufferBindingType::Uniform => {
1495                        if has_dynamic_offset {
1496                            desc_count.uniform_buffer_dynamic += count;
1497                        } else {
1498                            desc_count.uniform_buffer += count;
1499                        }
1500                    }
1501                    wgt::BufferBindingType::Storage { .. } => {
1502                        if has_dynamic_offset {
1503                            desc_count.storage_buffer_dynamic += count;
1504                        } else {
1505                            desc_count.storage_buffer += count;
1506                        }
1507                    }
1508                },
1509                wgt::BindingType::Sampler { .. } => {
1510                    desc_count.sampler += count;
1511                }
1512                wgt::BindingType::Texture { .. } => {
1513                    desc_count.sampled_image += count;
1514                }
1515                wgt::BindingType::StorageTexture { .. } => {
1516                    desc_count.storage_image += count;
1517                }
1518                wgt::BindingType::AccelerationStructure { .. } => {
1519                    desc_count.acceleration_structure += count;
1520                }
1521                wgt::BindingType::ExternalTexture => unimplemented!(),
1522            }
1523        }
1524
1525        //Note: not bothering with on stack array here as it's low frequency
1526        let vk_bindings = desc
1527            .entries
1528            .iter()
1529            .map(|entry| vk::DescriptorSetLayoutBinding {
1530                binding: entry.binding,
1531                descriptor_type: types[entry.binding as usize].0,
1532                descriptor_count: types[entry.binding as usize].1,
1533                stage_flags: conv::map_shader_stage(entry.visibility),
1534                p_immutable_samplers: ptr::null(),
1535                _marker: Default::default(),
1536            })
1537            .collect::<Vec<_>>();
1538
1539        let binding_arrays: Vec<_> = desc
1540            .entries
1541            .iter()
1542            .enumerate()
1543            .filter_map(|(idx, entry)| entry.count.map(|count| (idx as u32, count)))
1544            .collect();
1545
1546        let vk_info = vk::DescriptorSetLayoutCreateInfo::default()
1547            .bindings(&vk_bindings)
1548            .flags(if !binding_arrays.is_empty() {
1549                vk::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND_POOL
1550            } else {
1551                vk::DescriptorSetLayoutCreateFlags::empty()
1552            });
1553
1554        let partially_bound = desc
1555            .flags
1556            .contains(crate::BindGroupLayoutFlags::PARTIALLY_BOUND);
1557
1558        let binding_flag_vec = desc
1559            .entries
1560            .iter()
1561            .map(|entry| {
1562                let mut flags = vk::DescriptorBindingFlags::empty();
1563
1564                if partially_bound && entry.count.is_some() {
1565                    flags |= vk::DescriptorBindingFlags::PARTIALLY_BOUND;
1566                }
1567
1568                if entry.count.is_some() {
1569                    flags |= vk::DescriptorBindingFlags::UPDATE_AFTER_BIND;
1570                }
1571
1572                flags
1573            })
1574            .collect::<Vec<_>>();
1575
1576        let mut binding_flag_info = vk::DescriptorSetLayoutBindingFlagsCreateInfo::default()
1577            .binding_flags(&binding_flag_vec);
1578
1579        let vk_info = vk_info.push_next(&mut binding_flag_info);
1580
1581        let raw = unsafe {
1582            self.shared
1583                .raw
1584                .create_descriptor_set_layout(&vk_info, None)
1585                .map_err(super::map_host_device_oom_err)?
1586        };
1587
1588        if let Some(label) = desc.label {
1589            unsafe { self.shared.set_object_name(raw, label) };
1590        }
1591
1592        self.counters.bind_group_layouts.add(1);
1593
1594        Ok(super::BindGroupLayout {
1595            raw,
1596            desc_count,
1597            types: types.into_boxed_slice(),
1598            binding_arrays,
1599        })
1600    }
1601    unsafe fn destroy_bind_group_layout(&self, bg_layout: super::BindGroupLayout) {
1602        unsafe {
1603            self.shared
1604                .raw
1605                .destroy_descriptor_set_layout(bg_layout.raw, None)
1606        };
1607
1608        self.counters.bind_group_layouts.sub(1);
1609    }
1610
1611    unsafe fn create_pipeline_layout(
1612        &self,
1613        desc: &crate::PipelineLayoutDescriptor<super::BindGroupLayout>,
1614    ) -> Result<super::PipelineLayout, crate::DeviceError> {
1615        //Note: not bothering with on stack array here as it's low frequency
1616        let vk_set_layouts = desc
1617            .bind_group_layouts
1618            .iter()
1619            .map(|bgl| bgl.raw)
1620            .collect::<Vec<_>>();
1621        let vk_push_constant_ranges = desc
1622            .push_constant_ranges
1623            .iter()
1624            .map(|pcr| vk::PushConstantRange {
1625                stage_flags: conv::map_shader_stage(pcr.stages),
1626                offset: pcr.range.start,
1627                size: pcr.range.end - pcr.range.start,
1628            })
1629            .collect::<Vec<_>>();
1630
1631        let vk_info = vk::PipelineLayoutCreateInfo::default()
1632            .flags(vk::PipelineLayoutCreateFlags::empty())
1633            .set_layouts(&vk_set_layouts)
1634            .push_constant_ranges(&vk_push_constant_ranges);
1635
1636        let raw = {
1637            profiling::scope!("vkCreatePipelineLayout");
1638            unsafe {
1639                self.shared
1640                    .raw
1641                    .create_pipeline_layout(&vk_info, None)
1642                    .map_err(super::map_host_device_oom_err)?
1643            }
1644        };
1645
1646        if let Some(label) = desc.label {
1647            unsafe { self.shared.set_object_name(raw, label) };
1648        }
1649
1650        let mut binding_arrays = BTreeMap::new();
1651        for (group, &layout) in desc.bind_group_layouts.iter().enumerate() {
1652            for &(binding, binding_array_size) in &layout.binding_arrays {
1653                binding_arrays.insert(
1654                    naga::ResourceBinding {
1655                        group: group as u32,
1656                        binding,
1657                    },
1658                    naga::back::spv::BindingInfo {
1659                        binding_array_size: Some(binding_array_size.get()),
1660                    },
1661                );
1662            }
1663        }
1664
1665        self.counters.pipeline_layouts.add(1);
1666
1667        Ok(super::PipelineLayout {
1668            raw,
1669            binding_arrays,
1670        })
1671    }
1672    unsafe fn destroy_pipeline_layout(&self, pipeline_layout: super::PipelineLayout) {
1673        unsafe {
1674            self.shared
1675                .raw
1676                .destroy_pipeline_layout(pipeline_layout.raw, None)
1677        };
1678
1679        self.counters.pipeline_layouts.sub(1);
1680    }
1681
1682    unsafe fn create_bind_group(
1683        &self,
1684        desc: &crate::BindGroupDescriptor<
1685            super::BindGroupLayout,
1686            super::Buffer,
1687            super::Sampler,
1688            super::TextureView,
1689            super::AccelerationStructure,
1690        >,
1691    ) -> Result<super::BindGroup, crate::DeviceError> {
1692        let contains_binding_arrays = !desc.layout.binding_arrays.is_empty();
1693
1694        let desc_set_layout_flags = if contains_binding_arrays {
1695            gpu_descriptor::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND
1696        } else {
1697            gpu_descriptor::DescriptorSetLayoutCreateFlags::empty()
1698        };
1699
1700        let mut vk_sets = unsafe {
1701            self.desc_allocator.lock().allocate(
1702                &*self.shared,
1703                &desc.layout.raw,
1704                desc_set_layout_flags,
1705                &desc.layout.desc_count,
1706                1,
1707            )?
1708        };
1709
1710        let set = vk_sets.pop().unwrap();
1711        if let Some(label) = desc.label {
1712            unsafe { self.shared.set_object_name(*set.raw(), label) };
1713        }
1714
1715        /// Helper for splitting off and initializing a given number of elements on a pre-allocated
1716        /// stack, based on items returned from an [`ExactSizeIterator`].  Typically created from a
1717        /// [`MaybeUninit`] slice (see [`Vec::spare_capacity_mut()`]).
1718        /// The updated [`ExtensionStack`] of remaining uninitialized elements is returned, safely
1719        /// representing that the initialized and remaining elements are two independent mutable
1720        /// borrows.
1721        struct ExtendStack<'a, T> {
1722            remainder: &'a mut [MaybeUninit<T>],
1723        }
1724
1725        impl<'a, T> ExtendStack<'a, T> {
1726            fn from_vec_capacity(vec: &'a mut Vec<T>) -> Self {
1727                Self {
1728                    remainder: vec.spare_capacity_mut(),
1729                }
1730            }
1731
1732            fn extend_one(self, value: T) -> (Self, &'a mut T) {
1733                let (to_init, remainder) = self.remainder.split_first_mut().unwrap();
1734                let init = to_init.write(value);
1735                (Self { remainder }, init)
1736            }
1737
1738            fn extend(
1739                self,
1740                iter: impl IntoIterator<Item = T> + ExactSizeIterator,
1741            ) -> (Self, &'a mut [T]) {
1742                let (to_init, remainder) = self.remainder.split_at_mut(iter.len());
1743
1744                for (value, to_init) in iter.into_iter().zip(to_init.iter_mut()) {
1745                    to_init.write(value);
1746                }
1747
1748                // we can't use the safe (yet unstable) MaybeUninit::write_slice() here because of having an iterator to write
1749
1750                let init = {
1751                    // SAFETY: The loop above has initialized exactly as many items as to_init is
1752                    // long, so it is safe to cast away the MaybeUninit<T> wrapper into T.
1753
1754                    // Additional safety docs from unstable slice_assume_init_mut
1755                    // SAFETY: similar to safety notes for `slice_get_ref`, but we have a
1756                    // mutable reference which is also guaranteed to be valid for writes.
1757                    unsafe { mem::transmute::<&mut [MaybeUninit<T>], &mut [T]>(to_init) }
1758                };
1759                (Self { remainder }, init)
1760            }
1761        }
1762
1763        let mut writes = Vec::with_capacity(desc.entries.len());
1764        let mut buffer_infos = Vec::with_capacity(desc.buffers.len());
1765        let mut buffer_infos = ExtendStack::from_vec_capacity(&mut buffer_infos);
1766        let mut image_infos = Vec::with_capacity(desc.samplers.len() + desc.textures.len());
1767        let mut image_infos = ExtendStack::from_vec_capacity(&mut image_infos);
1768        // TODO: This length could be reduced to just the number of top-level acceleration
1769        // structure bindings, where multiple consecutive TLAS bindings that are set via
1770        // one `WriteDescriptorSet` count towards one "info" struct, not the total number of
1771        // acceleration structure bindings to write:
1772        let mut acceleration_structure_infos =
1773            Vec::with_capacity(desc.acceleration_structures.len());
1774        let mut acceleration_structure_infos =
1775            ExtendStack::from_vec_capacity(&mut acceleration_structure_infos);
1776        let mut raw_acceleration_structures =
1777            Vec::with_capacity(desc.acceleration_structures.len());
1778        let mut raw_acceleration_structures =
1779            ExtendStack::from_vec_capacity(&mut raw_acceleration_structures);
1780        for entry in desc.entries {
1781            let (ty, size) = desc.layout.types[entry.binding as usize];
1782            if size == 0 {
1783                continue; // empty slot
1784            }
1785            let mut write = vk::WriteDescriptorSet::default()
1786                .dst_set(*set.raw())
1787                .dst_binding(entry.binding)
1788                .descriptor_type(ty);
1789
1790            write = match ty {
1791                vk::DescriptorType::SAMPLER => {
1792                    let start = entry.resource_index;
1793                    let end = start + entry.count;
1794                    let local_image_infos;
1795                    (image_infos, local_image_infos) =
1796                        image_infos.extend(desc.samplers[start as usize..end as usize].iter().map(
1797                            |sampler| vk::DescriptorImageInfo::default().sampler(sampler.raw),
1798                        ));
1799                    write.image_info(local_image_infos)
1800                }
1801                vk::DescriptorType::SAMPLED_IMAGE | vk::DescriptorType::STORAGE_IMAGE => {
1802                    let start = entry.resource_index;
1803                    let end = start + entry.count;
1804                    let local_image_infos;
1805                    (image_infos, local_image_infos) =
1806                        image_infos.extend(desc.textures[start as usize..end as usize].iter().map(
1807                            |binding| {
1808                                let layout =
1809                                    conv::derive_image_layout(binding.usage, binding.view.format);
1810                                vk::DescriptorImageInfo::default()
1811                                    .image_view(binding.view.raw)
1812                                    .image_layout(layout)
1813                            },
1814                        ));
1815                    write.image_info(local_image_infos)
1816                }
1817                vk::DescriptorType::UNIFORM_BUFFER
1818                | vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC
1819                | vk::DescriptorType::STORAGE_BUFFER
1820                | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => {
1821                    let start = entry.resource_index;
1822                    let end = start + entry.count;
1823                    let local_buffer_infos;
1824                    (buffer_infos, local_buffer_infos) =
1825                        buffer_infos.extend(desc.buffers[start as usize..end as usize].iter().map(
1826                            |binding| {
1827                                vk::DescriptorBufferInfo::default()
1828                                    .buffer(binding.buffer.raw)
1829                                    .offset(binding.offset)
1830                                    .range(
1831                                        binding.size.map_or(vk::WHOLE_SIZE, wgt::BufferSize::get),
1832                                    )
1833                            },
1834                        ));
1835                    write.buffer_info(local_buffer_infos)
1836                }
1837                vk::DescriptorType::ACCELERATION_STRUCTURE_KHR => {
1838                    let start = entry.resource_index;
1839                    let end = start + entry.count;
1840
1841                    let local_raw_acceleration_structures;
1842                    (
1843                        raw_acceleration_structures,
1844                        local_raw_acceleration_structures,
1845                    ) = raw_acceleration_structures.extend(
1846                        desc.acceleration_structures[start as usize..end as usize]
1847                            .iter()
1848                            .map(|acceleration_structure| acceleration_structure.raw),
1849                    );
1850
1851                    let local_acceleration_structure_infos;
1852                    (
1853                        acceleration_structure_infos,
1854                        local_acceleration_structure_infos,
1855                    ) = acceleration_structure_infos.extend_one(
1856                        vk::WriteDescriptorSetAccelerationStructureKHR::default()
1857                            .acceleration_structures(local_raw_acceleration_structures),
1858                    );
1859
1860                    write
1861                        .descriptor_count(entry.count)
1862                        .push_next(local_acceleration_structure_infos)
1863                }
1864                _ => unreachable!(),
1865            };
1866
1867            writes.push(write);
1868        }
1869
1870        unsafe { self.shared.raw.update_descriptor_sets(&writes, &[]) };
1871
1872        self.counters.bind_groups.add(1);
1873
1874        Ok(super::BindGroup { set })
1875    }
1876
1877    unsafe fn destroy_bind_group(&self, group: super::BindGroup) {
1878        unsafe {
1879            self.desc_allocator
1880                .lock()
1881                .free(&*self.shared, Some(group.set))
1882        };
1883
1884        self.counters.bind_groups.sub(1);
1885    }
1886
1887    unsafe fn create_shader_module(
1888        &self,
1889        desc: &crate::ShaderModuleDescriptor,
1890        shader: crate::ShaderInput,
1891    ) -> Result<super::ShaderModule, crate::ShaderError> {
1892        let spv = match shader {
1893            crate::ShaderInput::Naga(naga_shader) => {
1894                if self
1895                    .shared
1896                    .workarounds
1897                    .contains(super::Workarounds::SEPARATE_ENTRY_POINTS)
1898                    || !naga_shader.module.overrides.is_empty()
1899                {
1900                    return Ok(super::ShaderModule::Intermediate {
1901                        naga_shader,
1902                        runtime_checks: desc.runtime_checks,
1903                    });
1904                }
1905                let mut naga_options = self.naga_options.clone();
1906                naga_options.debug_info =
1907                    naga_shader
1908                        .debug_source
1909                        .as_ref()
1910                        .map(|d| naga::back::spv::DebugInfo {
1911                            source_code: d.source_code.as_ref(),
1912                            file_name: d.file_name.as_ref().into(),
1913                            language: naga::back::spv::SourceLanguage::WGSL,
1914                        });
1915                if !desc.runtime_checks.bounds_checks {
1916                    naga_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
1917                        index: naga::proc::BoundsCheckPolicy::Unchecked,
1918                        buffer: naga::proc::BoundsCheckPolicy::Unchecked,
1919                        image_load: naga::proc::BoundsCheckPolicy::Unchecked,
1920                        binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
1921                    };
1922                }
1923                Cow::Owned(
1924                    naga::back::spv::write_vec(
1925                        &naga_shader.module,
1926                        &naga_shader.info,
1927                        &naga_options,
1928                        None,
1929                    )
1930                    .map_err(|e| crate::ShaderError::Compilation(format!("{e}")))?,
1931                )
1932            }
1933            crate::ShaderInput::Msl { .. } => {
1934                panic!("MSL_SHADER_PASSTHROUGH is not enabled for this backend")
1935            }
1936            crate::ShaderInput::Dxil { .. } | crate::ShaderInput::Hlsl { .. } => {
1937                panic!("`Features::HLSL_DXIL_SHADER_PASSTHROUGH` is not enabled")
1938            }
1939            crate::ShaderInput::SpirV(spv) => Cow::Borrowed(spv),
1940        };
1941
1942        let raw = self.create_shader_module_impl(&spv)?;
1943
1944        if let Some(label) = desc.label {
1945            unsafe { self.shared.set_object_name(raw, label) };
1946        }
1947
1948        self.counters.shader_modules.add(1);
1949
1950        Ok(super::ShaderModule::Raw(raw))
1951    }
1952
1953    unsafe fn destroy_shader_module(&self, module: super::ShaderModule) {
1954        match module {
1955            super::ShaderModule::Raw(raw) => {
1956                unsafe { self.shared.raw.destroy_shader_module(raw, None) };
1957            }
1958            super::ShaderModule::Intermediate { .. } => {}
1959        }
1960
1961        self.counters.shader_modules.sub(1);
1962    }
1963
1964    unsafe fn create_render_pipeline(
1965        &self,
1966        desc: &crate::RenderPipelineDescriptor<
1967            super::PipelineLayout,
1968            super::ShaderModule,
1969            super::PipelineCache,
1970        >,
1971    ) -> Result<super::RenderPipeline, crate::PipelineError> {
1972        let dynamic_states = [
1973            vk::DynamicState::VIEWPORT,
1974            vk::DynamicState::SCISSOR,
1975            vk::DynamicState::BLEND_CONSTANTS,
1976            vk::DynamicState::STENCIL_REFERENCE,
1977        ];
1978        let mut compatible_rp_key = super::RenderPassKey {
1979            sample_count: desc.multisample.count,
1980            multiview: desc.multiview,
1981            ..Default::default()
1982        };
1983        let mut stages = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new();
1984        let mut vertex_buffers = Vec::new();
1985        let mut vertex_attributes = Vec::new();
1986
1987        if let crate::VertexProcessor::Standard {
1988            vertex_buffers: desc_vertex_buffers,
1989            vertex_stage: _,
1990        } = &desc.vertex_processor
1991        {
1992            vertex_buffers = Vec::with_capacity(desc_vertex_buffers.len());
1993            for (i, vb) in desc_vertex_buffers.iter().enumerate() {
1994                vertex_buffers.push(vk::VertexInputBindingDescription {
1995                    binding: i as u32,
1996                    stride: vb.array_stride as u32,
1997                    input_rate: match vb.step_mode {
1998                        wgt::VertexStepMode::Vertex => vk::VertexInputRate::VERTEX,
1999                        wgt::VertexStepMode::Instance => vk::VertexInputRate::INSTANCE,
2000                    },
2001                });
2002                for at in vb.attributes {
2003                    vertex_attributes.push(vk::VertexInputAttributeDescription {
2004                        location: at.shader_location,
2005                        binding: i as u32,
2006                        format: conv::map_vertex_format(at.format),
2007                        offset: at.offset as u32,
2008                    });
2009                }
2010            }
2011        }
2012
2013        let vk_vertex_input = vk::PipelineVertexInputStateCreateInfo::default()
2014            .vertex_binding_descriptions(&vertex_buffers)
2015            .vertex_attribute_descriptions(&vertex_attributes);
2016
2017        let vk_input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
2018            .topology(conv::map_topology(desc.primitive.topology))
2019            .primitive_restart_enable(desc.primitive.strip_index_format.is_some());
2020
2021        let mut compiled_vs = None;
2022        let mut compiled_ms = None;
2023        let mut compiled_ts = None;
2024        match &desc.vertex_processor {
2025            crate::VertexProcessor::Standard {
2026                vertex_buffers: _,
2027                vertex_stage,
2028            } => {
2029                compiled_vs = Some(self.compile_stage(
2030                    vertex_stage,
2031                    naga::ShaderStage::Vertex,
2032                    &desc.layout.binding_arrays,
2033                )?);
2034                stages.push(compiled_vs.as_ref().unwrap().create_info);
2035            }
2036            crate::VertexProcessor::Mesh {
2037                task_stage,
2038                mesh_stage,
2039            } => {
2040                if let Some(t) = task_stage.as_ref() {
2041                    compiled_ts = Some(self.compile_stage(
2042                        t,
2043                        naga::ShaderStage::Task,
2044                        &desc.layout.binding_arrays,
2045                    )?);
2046                    stages.push(compiled_ts.as_ref().unwrap().create_info);
2047                }
2048                compiled_ms = Some(self.compile_stage(
2049                    mesh_stage,
2050                    naga::ShaderStage::Mesh,
2051                    &desc.layout.binding_arrays,
2052                )?);
2053                stages.push(compiled_ms.as_ref().unwrap().create_info);
2054            }
2055        }
2056        let compiled_fs = match desc.fragment_stage {
2057            Some(ref stage) => {
2058                let compiled = self.compile_stage(
2059                    stage,
2060                    naga::ShaderStage::Fragment,
2061                    &desc.layout.binding_arrays,
2062                )?;
2063                stages.push(compiled.create_info);
2064                Some(compiled)
2065            }
2066            None => None,
2067        };
2068
2069        let mut vk_rasterization = vk::PipelineRasterizationStateCreateInfo::default()
2070            .polygon_mode(conv::map_polygon_mode(desc.primitive.polygon_mode))
2071            .front_face(conv::map_front_face(desc.primitive.front_face))
2072            .line_width(1.0)
2073            .depth_clamp_enable(desc.primitive.unclipped_depth);
2074        if let Some(face) = desc.primitive.cull_mode {
2075            vk_rasterization = vk_rasterization.cull_mode(conv::map_cull_face(face))
2076        }
2077        let mut vk_rasterization_conservative_state =
2078            vk::PipelineRasterizationConservativeStateCreateInfoEXT::default()
2079                .conservative_rasterization_mode(
2080                    vk::ConservativeRasterizationModeEXT::OVERESTIMATE,
2081                );
2082        if desc.primitive.conservative {
2083            vk_rasterization = vk_rasterization.push_next(&mut vk_rasterization_conservative_state);
2084        }
2085
2086        let mut vk_depth_stencil = vk::PipelineDepthStencilStateCreateInfo::default();
2087        if let Some(ref ds) = desc.depth_stencil {
2088            let vk_format = self.shared.private_caps.map_texture_format(ds.format);
2089            let vk_layout = if ds.is_read_only(desc.primitive.cull_mode) {
2090                vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL
2091            } else {
2092                vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL
2093            };
2094            compatible_rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
2095                base: super::AttachmentKey::compatible(vk_format, vk_layout),
2096                stencil_ops: crate::AttachmentOps::all(),
2097            });
2098
2099            if ds.is_depth_enabled() {
2100                vk_depth_stencil = vk_depth_stencil
2101                    .depth_test_enable(true)
2102                    .depth_write_enable(ds.depth_write_enabled)
2103                    .depth_compare_op(conv::map_comparison(ds.depth_compare));
2104            }
2105            if ds.stencil.is_enabled() {
2106                let s = &ds.stencil;
2107                let front = conv::map_stencil_face(&s.front, s.read_mask, s.write_mask);
2108                let back = conv::map_stencil_face(&s.back, s.read_mask, s.write_mask);
2109                vk_depth_stencil = vk_depth_stencil
2110                    .stencil_test_enable(true)
2111                    .front(front)
2112                    .back(back);
2113            }
2114
2115            if ds.bias.is_enabled() {
2116                vk_rasterization = vk_rasterization
2117                    .depth_bias_enable(true)
2118                    .depth_bias_constant_factor(ds.bias.constant as f32)
2119                    .depth_bias_clamp(ds.bias.clamp)
2120                    .depth_bias_slope_factor(ds.bias.slope_scale);
2121            }
2122        }
2123
2124        let vk_viewport = vk::PipelineViewportStateCreateInfo::default()
2125            .flags(vk::PipelineViewportStateCreateFlags::empty())
2126            .scissor_count(1)
2127            .viewport_count(1);
2128
2129        let vk_sample_mask = [
2130            desc.multisample.mask as u32,
2131            (desc.multisample.mask >> 32) as u32,
2132        ];
2133        let vk_multisample = vk::PipelineMultisampleStateCreateInfo::default()
2134            .rasterization_samples(vk::SampleCountFlags::from_raw(desc.multisample.count))
2135            .alpha_to_coverage_enable(desc.multisample.alpha_to_coverage_enabled)
2136            .sample_mask(&vk_sample_mask);
2137
2138        let mut vk_attachments = Vec::with_capacity(desc.color_targets.len());
2139        for cat in desc.color_targets {
2140            let (key, attarchment) = if let Some(cat) = cat.as_ref() {
2141                let mut vk_attachment = vk::PipelineColorBlendAttachmentState::default()
2142                    .color_write_mask(vk::ColorComponentFlags::from_raw(cat.write_mask.bits()));
2143                if let Some(ref blend) = cat.blend {
2144                    let (color_op, color_src, color_dst) = conv::map_blend_component(&blend.color);
2145                    let (alpha_op, alpha_src, alpha_dst) = conv::map_blend_component(&blend.alpha);
2146                    vk_attachment = vk_attachment
2147                        .blend_enable(true)
2148                        .color_blend_op(color_op)
2149                        .src_color_blend_factor(color_src)
2150                        .dst_color_blend_factor(color_dst)
2151                        .alpha_blend_op(alpha_op)
2152                        .src_alpha_blend_factor(alpha_src)
2153                        .dst_alpha_blend_factor(alpha_dst);
2154                }
2155
2156                let vk_format = self.shared.private_caps.map_texture_format(cat.format);
2157                (
2158                    Some(super::ColorAttachmentKey {
2159                        base: super::AttachmentKey::compatible(
2160                            vk_format,
2161                            vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
2162                        ),
2163                        resolve: None,
2164                    }),
2165                    vk_attachment,
2166                )
2167            } else {
2168                (None, vk::PipelineColorBlendAttachmentState::default())
2169            };
2170
2171            compatible_rp_key.colors.push(key);
2172            vk_attachments.push(attarchment);
2173        }
2174
2175        let vk_color_blend =
2176            vk::PipelineColorBlendStateCreateInfo::default().attachments(&vk_attachments);
2177
2178        let vk_dynamic_state =
2179            vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic_states);
2180
2181        let raw_pass = self.shared.make_render_pass(compatible_rp_key)?;
2182
2183        let vk_infos = [{
2184            vk::GraphicsPipelineCreateInfo::default()
2185                .layout(desc.layout.raw)
2186                .stages(&stages)
2187                .vertex_input_state(&vk_vertex_input)
2188                .input_assembly_state(&vk_input_assembly)
2189                .rasterization_state(&vk_rasterization)
2190                .viewport_state(&vk_viewport)
2191                .multisample_state(&vk_multisample)
2192                .depth_stencil_state(&vk_depth_stencil)
2193                .color_blend_state(&vk_color_blend)
2194                .dynamic_state(&vk_dynamic_state)
2195                .render_pass(raw_pass)
2196        }];
2197
2198        let pipeline_cache = desc
2199            .cache
2200            .map(|it| it.raw)
2201            .unwrap_or(vk::PipelineCache::null());
2202
2203        let mut raw_vec = {
2204            profiling::scope!("vkCreateGraphicsPipelines");
2205            unsafe {
2206                self.shared
2207                    .raw
2208                    .create_graphics_pipelines(pipeline_cache, &vk_infos, None)
2209                    .map_err(|(_, e)| super::map_pipeline_err(e))
2210            }?
2211        };
2212
2213        let raw = raw_vec.pop().unwrap();
2214        if let Some(label) = desc.label {
2215            unsafe { self.shared.set_object_name(raw, label) };
2216        }
2217
2218        if let Some(CompiledStage {
2219            temp_raw_module: Some(raw_module),
2220            ..
2221        }) = compiled_vs
2222        {
2223            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2224        }
2225        if let Some(CompiledStage {
2226            temp_raw_module: Some(raw_module),
2227            ..
2228        }) = compiled_ts
2229        {
2230            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2231        }
2232        if let Some(CompiledStage {
2233            temp_raw_module: Some(raw_module),
2234            ..
2235        }) = compiled_ms
2236        {
2237            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2238        }
2239        if let Some(CompiledStage {
2240            temp_raw_module: Some(raw_module),
2241            ..
2242        }) = compiled_fs
2243        {
2244            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2245        }
2246
2247        self.counters.render_pipelines.add(1);
2248
2249        Ok(super::RenderPipeline { raw })
2250    }
2251
2252    unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) {
2253        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2254
2255        self.counters.render_pipelines.sub(1);
2256    }
2257
2258    unsafe fn create_compute_pipeline(
2259        &self,
2260        desc: &crate::ComputePipelineDescriptor<
2261            super::PipelineLayout,
2262            super::ShaderModule,
2263            super::PipelineCache,
2264        >,
2265    ) -> Result<super::ComputePipeline, crate::PipelineError> {
2266        let compiled = self.compile_stage(
2267            &desc.stage,
2268            naga::ShaderStage::Compute,
2269            &desc.layout.binding_arrays,
2270        )?;
2271
2272        let vk_infos = [{
2273            vk::ComputePipelineCreateInfo::default()
2274                .layout(desc.layout.raw)
2275                .stage(compiled.create_info)
2276        }];
2277
2278        let pipeline_cache = desc
2279            .cache
2280            .map(|it| it.raw)
2281            .unwrap_or(vk::PipelineCache::null());
2282
2283        let mut raw_vec = {
2284            profiling::scope!("vkCreateComputePipelines");
2285            unsafe {
2286                self.shared
2287                    .raw
2288                    .create_compute_pipelines(pipeline_cache, &vk_infos, None)
2289                    .map_err(|(_, e)| super::map_pipeline_err(e))
2290            }?
2291        };
2292
2293        let raw = raw_vec.pop().unwrap();
2294        if let Some(label) = desc.label {
2295            unsafe { self.shared.set_object_name(raw, label) };
2296        }
2297
2298        if let Some(raw_module) = compiled.temp_raw_module {
2299            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2300        }
2301
2302        self.counters.compute_pipelines.add(1);
2303
2304        Ok(super::ComputePipeline { raw })
2305    }
2306
2307    unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) {
2308        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2309
2310        self.counters.compute_pipelines.sub(1);
2311    }
2312
2313    unsafe fn create_pipeline_cache(
2314        &self,
2315        desc: &crate::PipelineCacheDescriptor<'_>,
2316    ) -> Result<super::PipelineCache, crate::PipelineCacheError> {
2317        let mut info = vk::PipelineCacheCreateInfo::default();
2318        if let Some(data) = desc.data {
2319            info = info.initial_data(data)
2320        }
2321        profiling::scope!("vkCreatePipelineCache");
2322        let raw = unsafe { self.shared.raw.create_pipeline_cache(&info, None) }
2323            .map_err(super::map_host_device_oom_err)?;
2324
2325        Ok(super::PipelineCache { raw })
2326    }
2327    fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> {
2328        Some(self.shared.pipeline_cache_validation_key)
2329    }
2330    unsafe fn destroy_pipeline_cache(&self, cache: super::PipelineCache) {
2331        unsafe { self.shared.raw.destroy_pipeline_cache(cache.raw, None) }
2332    }
2333    unsafe fn create_query_set(
2334        &self,
2335        desc: &wgt::QuerySetDescriptor<crate::Label>,
2336    ) -> Result<super::QuerySet, crate::DeviceError> {
2337        // Assume each query is 256 bytes.
2338        // On an AMD W6800 with driver version 32.0.12030.9, occlusion queries are 256.
2339        self.error_if_would_oom_on_resource_allocation(true, desc.count as u64 * 256)?;
2340
2341        let (vk_type, pipeline_statistics) = match desc.ty {
2342            wgt::QueryType::Occlusion => (
2343                vk::QueryType::OCCLUSION,
2344                vk::QueryPipelineStatisticFlags::empty(),
2345            ),
2346            wgt::QueryType::PipelineStatistics(statistics) => (
2347                vk::QueryType::PIPELINE_STATISTICS,
2348                conv::map_pipeline_statistics(statistics),
2349            ),
2350            wgt::QueryType::Timestamp => (
2351                vk::QueryType::TIMESTAMP,
2352                vk::QueryPipelineStatisticFlags::empty(),
2353            ),
2354        };
2355
2356        let vk_info = vk::QueryPoolCreateInfo::default()
2357            .query_type(vk_type)
2358            .query_count(desc.count)
2359            .pipeline_statistics(pipeline_statistics);
2360
2361        let raw = unsafe { self.shared.raw.create_query_pool(&vk_info, None) }
2362            .map_err(super::map_host_device_oom_err)?;
2363        if let Some(label) = desc.label {
2364            unsafe { self.shared.set_object_name(raw, label) };
2365        }
2366
2367        self.counters.query_sets.add(1);
2368
2369        Ok(super::QuerySet { raw })
2370    }
2371
2372    unsafe fn destroy_query_set(&self, set: super::QuerySet) {
2373        unsafe { self.shared.raw.destroy_query_pool(set.raw, None) };
2374
2375        self.counters.query_sets.sub(1);
2376    }
2377
2378    unsafe fn create_fence(&self) -> Result<super::Fence, crate::DeviceError> {
2379        self.counters.fences.add(1);
2380
2381        Ok(if self.shared.private_caps.timeline_semaphores {
2382            let mut sem_type_info =
2383                vk::SemaphoreTypeCreateInfo::default().semaphore_type(vk::SemaphoreType::TIMELINE);
2384            let vk_info = vk::SemaphoreCreateInfo::default().push_next(&mut sem_type_info);
2385            let raw = unsafe { self.shared.raw.create_semaphore(&vk_info, None) }
2386                .map_err(super::map_host_device_oom_err)?;
2387
2388            super::Fence::TimelineSemaphore(raw)
2389        } else {
2390            super::Fence::FencePool {
2391                last_completed: 0,
2392                active: Vec::new(),
2393                free: Vec::new(),
2394            }
2395        })
2396    }
2397    unsafe fn destroy_fence(&self, fence: super::Fence) {
2398        match fence {
2399            super::Fence::TimelineSemaphore(raw) => {
2400                unsafe { self.shared.raw.destroy_semaphore(raw, None) };
2401            }
2402            super::Fence::FencePool {
2403                active,
2404                free,
2405                last_completed: _,
2406            } => {
2407                for (_, raw) in active {
2408                    unsafe { self.shared.raw.destroy_fence(raw, None) };
2409                }
2410                for raw in free {
2411                    unsafe { self.shared.raw.destroy_fence(raw, None) };
2412                }
2413            }
2414        }
2415
2416        self.counters.fences.sub(1);
2417    }
2418    unsafe fn get_fence_value(
2419        &self,
2420        fence: &super::Fence,
2421    ) -> Result<crate::FenceValue, crate::DeviceError> {
2422        fence.get_latest(
2423            &self.shared.raw,
2424            self.shared.extension_fns.timeline_semaphore.as_ref(),
2425        )
2426    }
2427    unsafe fn wait(
2428        &self,
2429        fence: &super::Fence,
2430        wait_value: crate::FenceValue,
2431        timeout_ms: u32,
2432    ) -> Result<bool, crate::DeviceError> {
2433        let timeout_ns = timeout_ms as u64 * super::MILLIS_TO_NANOS;
2434        self.shared.wait_for_fence(fence, wait_value, timeout_ns)
2435    }
2436
2437    unsafe fn start_graphics_debugger_capture(&self) -> bool {
2438        #[cfg(feature = "renderdoc")]
2439        {
2440            // Renderdoc requires us to give us the pointer that vkInstance _points to_.
2441            let raw_vk_instance =
2442                vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2443            let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2444            unsafe {
2445                self.render_doc
2446                    .start_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2447            }
2448        }
2449        #[cfg(not(feature = "renderdoc"))]
2450        false
2451    }
2452    unsafe fn stop_graphics_debugger_capture(&self) {
2453        #[cfg(feature = "renderdoc")]
2454        {
2455            // Renderdoc requires us to give us the pointer that vkInstance _points to_.
2456            let raw_vk_instance =
2457                vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2458            let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2459
2460            unsafe {
2461                self.render_doc
2462                    .end_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2463            }
2464        }
2465    }
2466
2467    unsafe fn pipeline_cache_get_data(&self, cache: &super::PipelineCache) -> Option<Vec<u8>> {
2468        let data = unsafe { self.raw_device().get_pipeline_cache_data(cache.raw) };
2469        data.ok()
2470    }
2471
2472    unsafe fn get_acceleration_structure_build_sizes<'a>(
2473        &self,
2474        desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>,
2475    ) -> crate::AccelerationStructureBuildSizes {
2476        const CAPACITY: usize = 8;
2477
2478        let ray_tracing_functions = self
2479            .shared
2480            .extension_fns
2481            .ray_tracing
2482            .as_ref()
2483            .expect("Feature `RAY_TRACING` not enabled");
2484
2485        let (geometries, primitive_counts) = match *desc.entries {
2486            crate::AccelerationStructureEntries::Instances(ref instances) => {
2487                let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default();
2488
2489                let geometry = vk::AccelerationStructureGeometryKHR::default()
2490                    .geometry_type(vk::GeometryTypeKHR::INSTANCES)
2491                    .geometry(vk::AccelerationStructureGeometryDataKHR {
2492                        instances: instance_data,
2493                    });
2494
2495                (
2496                    smallvec::smallvec![geometry],
2497                    smallvec::smallvec![instances.count],
2498                )
2499            }
2500            crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
2501                let mut primitive_counts =
2502                    smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2503                let mut geometries = smallvec::SmallVec::<
2504                    [vk::AccelerationStructureGeometryKHR; CAPACITY],
2505                >::with_capacity(in_geometries.len());
2506
2507                for triangles in in_geometries {
2508                    let mut triangle_data =
2509                        vk::AccelerationStructureGeometryTrianglesDataKHR::default()
2510                            .index_type(vk::IndexType::NONE_KHR)
2511                            .vertex_format(conv::map_vertex_format(triangles.vertex_format))
2512                            .max_vertex(triangles.vertex_count)
2513                            .vertex_stride(triangles.vertex_stride)
2514                            // The vulkan spec suggests we could pass a non-zero invalid address here if fetching
2515                            // the real address has significant overhead, but we pass the real one to be on the
2516                            // safe side for now.
2517                            // from https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetAccelerationStructureBuildSizesKHR.html
2518                            // > The srcAccelerationStructure, dstAccelerationStructure, and mode members
2519                            // > of pBuildInfo are ignored. Any VkDeviceOrHostAddressKHR or VkDeviceOrHostAddressConstKHR
2520                            // > members of pBuildInfo are ignored by this command, except that the hostAddress
2521                            // > member of VkAccelerationStructureGeometryTrianglesDataKHR::transformData will
2522                            // > be examined to check if it is NULL.
2523                            .transform_data(vk::DeviceOrHostAddressConstKHR {
2524                                device_address: if desc
2525                                    .flags
2526                                    .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
2527                                {
2528                                    unsafe {
2529                                        ray_tracing_functions
2530                                            .buffer_device_address
2531                                            .get_buffer_device_address(
2532                                                &vk::BufferDeviceAddressInfo::default().buffer(
2533                                                    triangles
2534                                                        .transform
2535                                                        .as_ref()
2536                                                        .unwrap()
2537                                                        .buffer
2538                                                        .raw,
2539                                                ),
2540                                            )
2541                                    }
2542                                } else {
2543                                    0
2544                                },
2545                            });
2546
2547                    let pritive_count = if let Some(ref indices) = triangles.indices {
2548                        triangle_data =
2549                            triangle_data.index_type(conv::map_index_format(indices.format));
2550                        indices.count / 3
2551                    } else {
2552                        triangles.vertex_count
2553                    };
2554
2555                    let geometry = vk::AccelerationStructureGeometryKHR::default()
2556                        .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
2557                        .geometry(vk::AccelerationStructureGeometryDataKHR {
2558                            triangles: triangle_data,
2559                        })
2560                        .flags(conv::map_acceleration_structure_geometry_flags(
2561                            triangles.flags,
2562                        ));
2563
2564                    geometries.push(geometry);
2565                    primitive_counts.push(pritive_count);
2566                }
2567                (geometries, primitive_counts)
2568            }
2569            crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
2570                let mut primitive_counts =
2571                    smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2572                let mut geometries = smallvec::SmallVec::<
2573                    [vk::AccelerationStructureGeometryKHR; CAPACITY],
2574                >::with_capacity(in_geometries.len());
2575                for aabb in in_geometries {
2576                    let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
2577                        .stride(aabb.stride);
2578
2579                    let geometry = vk::AccelerationStructureGeometryKHR::default()
2580                        .geometry_type(vk::GeometryTypeKHR::AABBS)
2581                        .geometry(vk::AccelerationStructureGeometryDataKHR { aabbs: aabbs_data })
2582                        .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
2583
2584                    geometries.push(geometry);
2585                    primitive_counts.push(aabb.count);
2586                }
2587                (geometries, primitive_counts)
2588            }
2589        };
2590
2591        let ty = match *desc.entries {
2592            crate::AccelerationStructureEntries::Instances(_) => {
2593                vk::AccelerationStructureTypeKHR::TOP_LEVEL
2594            }
2595            _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
2596        };
2597
2598        let geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
2599            .ty(ty)
2600            .flags(conv::map_acceleration_structure_flags(desc.flags))
2601            .geometries(&geometries);
2602
2603        let mut raw = Default::default();
2604        unsafe {
2605            ray_tracing_functions
2606                .acceleration_structure
2607                .get_acceleration_structure_build_sizes(
2608                    vk::AccelerationStructureBuildTypeKHR::DEVICE,
2609                    &geometry_info,
2610                    &primitive_counts,
2611                    &mut raw,
2612                )
2613        }
2614
2615        crate::AccelerationStructureBuildSizes {
2616            acceleration_structure_size: raw.acceleration_structure_size,
2617            update_scratch_size: raw.update_scratch_size,
2618            build_scratch_size: raw.build_scratch_size,
2619        }
2620    }
2621
2622    unsafe fn get_acceleration_structure_device_address(
2623        &self,
2624        acceleration_structure: &super::AccelerationStructure,
2625    ) -> wgt::BufferAddress {
2626        let ray_tracing_functions = self
2627            .shared
2628            .extension_fns
2629            .ray_tracing
2630            .as_ref()
2631            .expect("Feature `RAY_TRACING` not enabled");
2632
2633        unsafe {
2634            ray_tracing_functions
2635                .acceleration_structure
2636                .get_acceleration_structure_device_address(
2637                    &vk::AccelerationStructureDeviceAddressInfoKHR::default()
2638                        .acceleration_structure(acceleration_structure.raw),
2639                )
2640        }
2641    }
2642
2643    unsafe fn create_acceleration_structure(
2644        &self,
2645        desc: &crate::AccelerationStructureDescriptor,
2646    ) -> Result<super::AccelerationStructure, crate::DeviceError> {
2647        let ray_tracing_functions = self
2648            .shared
2649            .extension_fns
2650            .ray_tracing
2651            .as_ref()
2652            .expect("Feature `RAY_TRACING` not enabled");
2653
2654        let vk_buffer_info = vk::BufferCreateInfo::default()
2655            .size(desc.size)
2656            .usage(
2657                vk::BufferUsageFlags::ACCELERATION_STRUCTURE_STORAGE_KHR
2658                    | vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS,
2659            )
2660            .sharing_mode(vk::SharingMode::EXCLUSIVE);
2661
2662        unsafe {
2663            let raw_buffer = self
2664                .shared
2665                .raw
2666                .create_buffer(&vk_buffer_info, None)
2667                .map_err(super::map_host_device_oom_and_ioca_err)?;
2668            let req = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
2669
2670            self.error_if_would_oom_on_resource_allocation(false, req.size)
2671                .inspect_err(|_| {
2672                    self.shared.raw.destroy_buffer(raw_buffer, None);
2673                })?;
2674
2675            let block = self
2676                .mem_allocator
2677                .lock()
2678                .alloc(
2679                    &*self.shared,
2680                    gpu_alloc::Request {
2681                        size: req.size,
2682                        align_mask: req.alignment - 1,
2683                        usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
2684                        memory_types: req.memory_type_bits & self.valid_ash_memory_types,
2685                    },
2686                )
2687                .inspect_err(|_| {
2688                    self.shared.raw.destroy_buffer(raw_buffer, None);
2689                })?;
2690
2691            self.shared
2692                .raw
2693                .bind_buffer_memory(raw_buffer, *block.memory(), block.offset())
2694                .map_err(super::map_host_device_oom_and_ioca_err)
2695                .inspect_err(|_| {
2696                    self.shared.raw.destroy_buffer(raw_buffer, None);
2697                })?;
2698
2699            if let Some(label) = desc.label {
2700                self.shared.set_object_name(raw_buffer, label);
2701            }
2702
2703            let vk_info = vk::AccelerationStructureCreateInfoKHR::default()
2704                .buffer(raw_buffer)
2705                .offset(0)
2706                .size(desc.size)
2707                .ty(conv::map_acceleration_structure_format(desc.format));
2708
2709            let raw_acceleration_structure = ray_tracing_functions
2710                .acceleration_structure
2711                .create_acceleration_structure(&vk_info, None)
2712                .map_err(super::map_host_oom_and_ioca_err)
2713                .inspect_err(|_| {
2714                    self.shared.raw.destroy_buffer(raw_buffer, None);
2715                })?;
2716
2717            if let Some(label) = desc.label {
2718                self.shared
2719                    .set_object_name(raw_acceleration_structure, label);
2720            }
2721
2722            let pool = if desc.allow_compaction {
2723                let vk_info = vk::QueryPoolCreateInfo::default()
2724                    .query_type(vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR)
2725                    .query_count(1);
2726
2727                let raw = self
2728                    .shared
2729                    .raw
2730                    .create_query_pool(&vk_info, None)
2731                    .map_err(super::map_host_device_oom_err)
2732                    .inspect_err(|_| {
2733                        ray_tracing_functions
2734                            .acceleration_structure
2735                            .destroy_acceleration_structure(raw_acceleration_structure, None);
2736                        self.shared.raw.destroy_buffer(raw_buffer, None);
2737                    })?;
2738                Some(raw)
2739            } else {
2740                None
2741            };
2742
2743            Ok(super::AccelerationStructure {
2744                raw: raw_acceleration_structure,
2745                buffer: raw_buffer,
2746                block: Mutex::new(block),
2747                compacted_size_query: pool,
2748            })
2749        }
2750    }
2751
2752    unsafe fn destroy_acceleration_structure(
2753        &self,
2754        acceleration_structure: super::AccelerationStructure,
2755    ) {
2756        let ray_tracing_functions = self
2757            .shared
2758            .extension_fns
2759            .ray_tracing
2760            .as_ref()
2761            .expect("Feature `RAY_TRACING` not enabled");
2762
2763        unsafe {
2764            ray_tracing_functions
2765                .acceleration_structure
2766                .destroy_acceleration_structure(acceleration_structure.raw, None);
2767            self.shared
2768                .raw
2769                .destroy_buffer(acceleration_structure.buffer, None);
2770            self.mem_allocator
2771                .lock()
2772                .dealloc(&*self.shared, acceleration_structure.block.into_inner());
2773            if let Some(query) = acceleration_structure.compacted_size_query {
2774                self.shared.raw.destroy_query_pool(query, None)
2775            }
2776        }
2777    }
2778
2779    fn get_internal_counters(&self) -> wgt::HalCounters {
2780        self.counters
2781            .memory_allocations
2782            .set(self.shared.memory_allocations_counter.read());
2783
2784        self.counters.as_ref().clone()
2785    }
2786
2787    fn tlas_instance_to_bytes(&self, instance: TlasInstance) -> Vec<u8> {
2788        const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
2789        let temp = RawTlasInstance {
2790            transform: instance.transform,
2791            custom_data_and_mask: (instance.custom_data & MAX_U24)
2792                | (u32::from(instance.mask) << 24),
2793            shader_binding_table_record_offset_and_flags: 0,
2794            acceleration_structure_reference: instance.blas_address,
2795        };
2796        bytemuck::bytes_of(&temp).to_vec()
2797    }
2798
2799    fn check_if_oom(&self) -> Result<(), crate::DeviceError> {
2800        let Some(threshold) = self
2801            .shared
2802            .instance
2803            .memory_budget_thresholds
2804            .for_device_loss
2805        else {
2806            return Ok(());
2807        };
2808
2809        if !self
2810            .shared
2811            .enabled_extensions
2812            .contains(&ext::memory_budget::NAME)
2813        {
2814            return Ok(());
2815        }
2816
2817        let get_physical_device_properties = self
2818            .shared
2819            .instance
2820            .get_physical_device_properties
2821            .as_ref()
2822            .unwrap();
2823
2824        let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
2825
2826        let mut memory_properties =
2827            vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
2828
2829        unsafe {
2830            get_physical_device_properties.get_physical_device_memory_properties2(
2831                self.shared.physical_device,
2832                &mut memory_properties,
2833            );
2834        }
2835
2836        let memory_properties = memory_properties.memory_properties;
2837
2838        for i in 0..memory_properties.memory_heap_count {
2839            let heap_usage = memory_budget_properties.heap_usage[i as usize];
2840            let heap_budget = memory_budget_properties.heap_budget[i as usize];
2841
2842            if heap_usage >= heap_budget / 100 * threshold as u64 {
2843                return Err(crate::DeviceError::OutOfMemory);
2844            }
2845        }
2846
2847        Ok(())
2848    }
2849}
2850
2851impl super::DeviceShared {
2852    pub(super) fn new_binary_semaphore(
2853        &self,
2854        name: &str,
2855    ) -> Result<vk::Semaphore, crate::DeviceError> {
2856        unsafe {
2857            let semaphore = self
2858                .raw
2859                .create_semaphore(&vk::SemaphoreCreateInfo::default(), None)
2860                .map_err(super::map_host_device_oom_err)?;
2861
2862            self.set_object_name(semaphore, name);
2863
2864            Ok(semaphore)
2865        }
2866    }
2867
2868    pub(super) fn wait_for_fence(
2869        &self,
2870        fence: &super::Fence,
2871        wait_value: crate::FenceValue,
2872        timeout_ns: u64,
2873    ) -> Result<bool, crate::DeviceError> {
2874        profiling::scope!("Device::wait");
2875        match *fence {
2876            super::Fence::TimelineSemaphore(raw) => {
2877                let semaphores = [raw];
2878                let values = [wait_value];
2879                let vk_info = vk::SemaphoreWaitInfo::default()
2880                    .semaphores(&semaphores)
2881                    .values(&values);
2882                let result = match self.extension_fns.timeline_semaphore {
2883                    Some(super::ExtensionFn::Extension(ref ext)) => unsafe {
2884                        ext.wait_semaphores(&vk_info, timeout_ns)
2885                    },
2886                    Some(super::ExtensionFn::Promoted) => unsafe {
2887                        self.raw.wait_semaphores(&vk_info, timeout_ns)
2888                    },
2889                    None => unreachable!(),
2890                };
2891                match result {
2892                    Ok(()) => Ok(true),
2893                    Err(vk::Result::TIMEOUT) => Ok(false),
2894                    Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2895                }
2896            }
2897            super::Fence::FencePool {
2898                last_completed,
2899                ref active,
2900                free: _,
2901            } => {
2902                if wait_value <= last_completed {
2903                    Ok(true)
2904                } else {
2905                    match active.iter().find(|&&(value, _)| value >= wait_value) {
2906                        Some(&(_, raw)) => {
2907                            match unsafe { self.raw.wait_for_fences(&[raw], true, timeout_ns) } {
2908                                Ok(()) => Ok(true),
2909                                Err(vk::Result::TIMEOUT) => Ok(false),
2910                                Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2911                            }
2912                        }
2913                        None => {
2914                            crate::hal_usage_error(format!(
2915                                "no signals reached value {wait_value}"
2916                            ));
2917                        }
2918                    }
2919                }
2920            }
2921        }
2922    }
2923}
2924
2925impl From<gpu_alloc::AllocationError> for crate::DeviceError {
2926    fn from(error: gpu_alloc::AllocationError) -> Self {
2927        use gpu_alloc::AllocationError as Ae;
2928        match error {
2929            Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::TooManyObjects => Self::OutOfMemory,
2930            Ae::NoCompatibleMemoryTypes => crate::hal_usage_error(error),
2931        }
2932    }
2933}
2934impl From<gpu_alloc::MapError> for crate::DeviceError {
2935    fn from(error: gpu_alloc::MapError) -> Self {
2936        use gpu_alloc::MapError as Me;
2937        match error {
2938            Me::OutOfDeviceMemory | Me::OutOfHostMemory | Me::MapFailed => Self::OutOfMemory,
2939            Me::NonHostVisible | Me::AlreadyMapped => crate::hal_usage_error(error),
2940        }
2941    }
2942}
2943impl From<gpu_descriptor::AllocationError> for crate::DeviceError {
2944    fn from(error: gpu_descriptor::AllocationError) -> Self {
2945        use gpu_descriptor::AllocationError as Ae;
2946        match error {
2947            Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::Fragmentation => Self::OutOfMemory,
2948        }
2949    }
2950}
2951
2952/// We usually map unexpected vulkan errors to the [`crate::DeviceError::Unexpected`]
2953/// variant to be more robust even in cases where the driver is not
2954/// complying with the spec.
2955///
2956/// However, we implement a few Trait methods that don't have an equivalent
2957/// error variant. In those cases we use this function.
2958fn handle_unexpected(err: vk::Result) -> ! {
2959    panic!("Unexpected Vulkan error: `{err}`")
2960}
2961
2962struct ImageWithoutMemory {
2963    raw: vk::Image,
2964    requirements: vk::MemoryRequirements,
2965    copy_size: crate::CopyExtent,
2966}