wgpu_hal/vulkan/
device.rs

1use alloc::{borrow::ToOwned as _, collections::BTreeMap, ffi::CString, sync::Arc, vec::Vec};
2use core::{
3    ffi::CStr,
4    mem::{self, MaybeUninit},
5    num::NonZeroU32,
6    ptr,
7};
8
9use arrayvec::ArrayVec;
10use ash::{ext, khr, vk};
11use hashbrown::hash_map::Entry;
12use parking_lot::Mutex;
13
14use super::{conv, RawTlasInstance};
15use crate::TlasInstance;
16
17impl super::DeviceShared {
18    /// Set the name of `object` to `name`.
19    ///
20    /// If `name` contains an interior null byte, then the name set will be truncated to that byte.
21    ///
22    /// # Safety
23    ///
24    /// This method inherits the safety contract from [`vkSetDebugUtilsObjectName`]. In particular:
25    ///
26    /// - `object` must be a valid handle for one of the following:
27    ///   - An instance-level object from the same instance as this device.
28    ///   - A physical-device-level object that descends from the same physical device as this
29    ///     device.
30    ///   - A device-level object that descends from this device.
31    /// - `object` must be externally synchronized—only the calling thread should access it during
32    ///   this call.
33    ///
34    /// [`vkSetDebugUtilsObjectName`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkSetDebugUtilsObjectNameEXT.html
35    pub(super) unsafe fn set_object_name(&self, object: impl vk::Handle, name: &str) {
36        let Some(extension) = self.extension_fns.debug_utils.as_ref() else {
37            return;
38        };
39
40        // Keep variables outside the if-else block to ensure they do not
41        // go out of scope while we hold a pointer to them
42        let mut buffer: [u8; 64] = [0u8; 64];
43        let buffer_vec: Vec<u8>;
44
45        // Append a null terminator to the string
46        let name_bytes = if name.len() < buffer.len() {
47            // Common case, string is very small. Allocate a copy on the stack.
48            buffer[..name.len()].copy_from_slice(name.as_bytes());
49            // Add null terminator
50            buffer[name.len()] = 0;
51            &buffer[..name.len() + 1]
52        } else {
53            // Less common case, the string is large.
54            // This requires a heap allocation.
55            buffer_vec = name
56                .as_bytes()
57                .iter()
58                .cloned()
59                .chain(core::iter::once(0))
60                .collect();
61            &buffer_vec
62        };
63
64        let name = CStr::from_bytes_until_nul(name_bytes).expect("We have added a null byte");
65
66        let _result = unsafe {
67            extension.set_debug_utils_object_name(
68                &vk::DebugUtilsObjectNameInfoEXT::default()
69                    .object_handle(object)
70                    .object_name(name),
71            )
72        };
73    }
74
75    pub fn make_render_pass(
76        &self,
77        key: super::RenderPassKey,
78    ) -> Result<vk::RenderPass, crate::DeviceError> {
79        Ok(match self.render_passes.lock().entry(key) {
80            Entry::Occupied(e) => *e.get(),
81            Entry::Vacant(e) => {
82                let super::RenderPassKey {
83                    ref colors,
84                    ref depth_stencil,
85                    sample_count,
86                    multiview,
87                } = *e.key();
88
89                let mut vk_attachments = Vec::new();
90                let mut color_refs = Vec::with_capacity(colors.len());
91                let mut resolve_refs = Vec::with_capacity(color_refs.capacity());
92                let mut ds_ref = None;
93                let samples = vk::SampleCountFlags::from_raw(sample_count);
94                let unused = vk::AttachmentReference {
95                    attachment: vk::ATTACHMENT_UNUSED,
96                    layout: vk::ImageLayout::UNDEFINED,
97                };
98                for cat in colors.iter() {
99                    let (color_ref, resolve_ref) =
100                        if let Some(super::ColorAttachmentKey { base, resolve }) = cat {
101                            let super::AttachmentKey {
102                                format,
103                                layout,
104                                ops,
105                            } = *base;
106
107                            let color_ref = vk::AttachmentReference {
108                                attachment: vk_attachments.len() as u32,
109                                layout,
110                            };
111                            vk_attachments.push({
112                                let (load_op, store_op) = conv::map_attachment_ops(ops);
113                                vk::AttachmentDescription::default()
114                                    .format(format)
115                                    .samples(samples)
116                                    .load_op(load_op)
117                                    .store_op(store_op)
118                                    .initial_layout(layout)
119                                    .final_layout(layout)
120                            });
121                            let resolve_ref = if let Some(rat) = resolve {
122                                let super::AttachmentKey {
123                                    format,
124                                    layout,
125                                    ops,
126                                } = *rat;
127
128                                let (load_op, store_op) = conv::map_attachment_ops(ops);
129                                let vk_attachment = vk::AttachmentDescription::default()
130                                    .format(format)
131                                    .samples(vk::SampleCountFlags::TYPE_1)
132                                    .load_op(load_op)
133                                    .store_op(store_op)
134                                    .initial_layout(layout)
135                                    .final_layout(layout);
136                                vk_attachments.push(vk_attachment);
137
138                                vk::AttachmentReference {
139                                    attachment: vk_attachments.len() as u32 - 1,
140                                    layout,
141                                }
142                            } else {
143                                unused
144                            };
145
146                            (color_ref, resolve_ref)
147                        } else {
148                            (unused, unused)
149                        };
150
151                    color_refs.push(color_ref);
152                    resolve_refs.push(resolve_ref);
153                }
154
155                if let Some(ds) = depth_stencil {
156                    let super::DepthStencilAttachmentKey {
157                        ref base,
158                        stencil_ops,
159                    } = *ds;
160
161                    let super::AttachmentKey {
162                        format,
163                        layout,
164                        ops,
165                    } = *base;
166
167                    ds_ref = Some(vk::AttachmentReference {
168                        attachment: vk_attachments.len() as u32,
169                        layout,
170                    });
171                    let (load_op, store_op) = conv::map_attachment_ops(ops);
172                    let (stencil_load_op, stencil_store_op) = conv::map_attachment_ops(stencil_ops);
173                    let vk_attachment = vk::AttachmentDescription::default()
174                        .format(format)
175                        .samples(samples)
176                        .load_op(load_op)
177                        .store_op(store_op)
178                        .stencil_load_op(stencil_load_op)
179                        .stencil_store_op(stencil_store_op)
180                        .initial_layout(layout)
181                        .final_layout(layout);
182                    vk_attachments.push(vk_attachment);
183                }
184
185                let vk_subpasses = [{
186                    let mut vk_subpass = vk::SubpassDescription::default()
187                        .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
188                        .color_attachments(&color_refs)
189                        .resolve_attachments(&resolve_refs);
190
191                    if self
192                        .workarounds
193                        .contains(super::Workarounds::EMPTY_RESOLVE_ATTACHMENT_LISTS)
194                        && resolve_refs.is_empty()
195                    {
196                        vk_subpass.p_resolve_attachments = ptr::null();
197                    }
198
199                    if let Some(ref reference) = ds_ref {
200                        vk_subpass = vk_subpass.depth_stencil_attachment(reference)
201                    }
202                    vk_subpass
203                }];
204
205                let mut vk_info = vk::RenderPassCreateInfo::default()
206                    .attachments(&vk_attachments)
207                    .subpasses(&vk_subpasses);
208
209                let mut multiview_info;
210                let mask;
211                if let Some(multiview) = multiview {
212                    // Sanity checks, better to panic here than cause a driver crash
213                    assert!(multiview.get() <= 8);
214                    assert!(multiview.get() > 1);
215
216                    // Right now we enable all bits on the view masks and correlation masks.
217                    // This means we're rendering to all views in the subpass, and that all views
218                    // can be rendered concurrently.
219                    mask = [(1 << multiview.get()) - 1];
220
221                    // On Vulkan 1.1 or later, this is an alias for core functionality
222                    multiview_info = vk::RenderPassMultiviewCreateInfoKHR::default()
223                        .view_masks(&mask)
224                        .correlation_masks(&mask);
225                    vk_info = vk_info.push_next(&mut multiview_info);
226                }
227
228                let raw = unsafe {
229                    self.raw
230                        .create_render_pass(&vk_info, None)
231                        .map_err(super::map_host_device_oom_err)?
232                };
233
234                *e.insert(raw)
235            }
236        })
237    }
238
239    fn make_memory_ranges<'a, I: 'a + Iterator<Item = crate::MemoryRange>>(
240        &self,
241        buffer: &'a super::Buffer,
242        ranges: I,
243    ) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
244        let block = buffer.block.as_ref()?.lock();
245        let mask = self.private_caps.non_coherent_map_mask;
246        Some(ranges.map(move |range| {
247            vk::MappedMemoryRange::default()
248                .memory(*block.memory())
249                .offset((block.offset() + range.start) & !mask)
250                .size((range.end - range.start + mask) & !mask)
251        }))
252    }
253}
254
255impl gpu_alloc::MemoryDevice<vk::DeviceMemory> for super::DeviceShared {
256    unsafe fn allocate_memory(
257        &self,
258        size: u64,
259        memory_type: u32,
260        flags: gpu_alloc::AllocationFlags,
261    ) -> Result<vk::DeviceMemory, gpu_alloc::OutOfMemory> {
262        let mut info = vk::MemoryAllocateInfo::default()
263            .allocation_size(size)
264            .memory_type_index(memory_type);
265
266        let mut info_flags;
267
268        if flags.contains(gpu_alloc::AllocationFlags::DEVICE_ADDRESS) {
269            info_flags = vk::MemoryAllocateFlagsInfo::default()
270                .flags(vk::MemoryAllocateFlags::DEVICE_ADDRESS);
271            info = info.push_next(&mut info_flags);
272        }
273
274        match unsafe { self.raw.allocate_memory(&info, None) } {
275            Ok(memory) => {
276                self.memory_allocations_counter.add(1);
277                Ok(memory)
278            }
279            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
280                Err(gpu_alloc::OutOfMemory::OutOfDeviceMemory)
281            }
282            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
283                Err(gpu_alloc::OutOfMemory::OutOfHostMemory)
284            }
285            // We don't use VK_KHR_external_memory
286            // VK_ERROR_INVALID_EXTERNAL_HANDLE
287            // We don't use VK_KHR_buffer_device_address
288            // VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
289            Err(err) => handle_unexpected(err),
290        }
291    }
292
293    unsafe fn deallocate_memory(&self, memory: vk::DeviceMemory) {
294        self.memory_allocations_counter.sub(1);
295
296        unsafe { self.raw.free_memory(memory, None) };
297    }
298
299    unsafe fn map_memory(
300        &self,
301        memory: &mut vk::DeviceMemory,
302        offset: u64,
303        size: u64,
304    ) -> Result<ptr::NonNull<u8>, gpu_alloc::DeviceMapError> {
305        match unsafe {
306            self.raw
307                .map_memory(*memory, offset, size, vk::MemoryMapFlags::empty())
308        } {
309            Ok(ptr) => Ok(ptr::NonNull::new(ptr.cast::<u8>())
310                .expect("Pointer to memory mapping must not be null")),
311            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
312                Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory)
313            }
314            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
315                Err(gpu_alloc::DeviceMapError::OutOfHostMemory)
316            }
317            Err(vk::Result::ERROR_MEMORY_MAP_FAILED) => Err(gpu_alloc::DeviceMapError::MapFailed),
318            Err(err) => handle_unexpected(err),
319        }
320    }
321
322    unsafe fn unmap_memory(&self, memory: &mut vk::DeviceMemory) {
323        unsafe { self.raw.unmap_memory(*memory) };
324    }
325
326    unsafe fn invalidate_memory_ranges(
327        &self,
328        _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
329    ) -> Result<(), gpu_alloc::OutOfMemory> {
330        // should never be called
331        unimplemented!()
332    }
333
334    unsafe fn flush_memory_ranges(
335        &self,
336        _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
337    ) -> Result<(), gpu_alloc::OutOfMemory> {
338        // should never be called
339        unimplemented!()
340    }
341}
342
343impl
344    gpu_descriptor::DescriptorDevice<vk::DescriptorSetLayout, vk::DescriptorPool, vk::DescriptorSet>
345    for super::DeviceShared
346{
347    unsafe fn create_descriptor_pool(
348        &self,
349        descriptor_count: &gpu_descriptor::DescriptorTotalCount,
350        max_sets: u32,
351        flags: gpu_descriptor::DescriptorPoolCreateFlags,
352    ) -> Result<vk::DescriptorPool, gpu_descriptor::CreatePoolError> {
353        //Note: ignoring other types, since they can't appear here
354        let unfiltered_counts = [
355            (vk::DescriptorType::SAMPLER, descriptor_count.sampler),
356            (
357                vk::DescriptorType::SAMPLED_IMAGE,
358                descriptor_count.sampled_image,
359            ),
360            (
361                vk::DescriptorType::STORAGE_IMAGE,
362                descriptor_count.storage_image,
363            ),
364            (
365                vk::DescriptorType::UNIFORM_BUFFER,
366                descriptor_count.uniform_buffer,
367            ),
368            (
369                vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC,
370                descriptor_count.uniform_buffer_dynamic,
371            ),
372            (
373                vk::DescriptorType::STORAGE_BUFFER,
374                descriptor_count.storage_buffer,
375            ),
376            (
377                vk::DescriptorType::STORAGE_BUFFER_DYNAMIC,
378                descriptor_count.storage_buffer_dynamic,
379            ),
380            (
381                vk::DescriptorType::ACCELERATION_STRUCTURE_KHR,
382                descriptor_count.acceleration_structure,
383            ),
384        ];
385
386        let filtered_counts = unfiltered_counts
387            .iter()
388            .cloned()
389            .filter(|&(_, count)| count != 0)
390            .map(|(ty, count)| vk::DescriptorPoolSize {
391                ty,
392                descriptor_count: count,
393            })
394            .collect::<ArrayVec<_, 8>>();
395
396        let mut vk_flags =
397            if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND) {
398                vk::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND
399            } else {
400                vk::DescriptorPoolCreateFlags::empty()
401            };
402        if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET) {
403            vk_flags |= vk::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET;
404        }
405        let vk_info = vk::DescriptorPoolCreateInfo::default()
406            .max_sets(max_sets)
407            .flags(vk_flags)
408            .pool_sizes(&filtered_counts);
409
410        match unsafe { self.raw.create_descriptor_pool(&vk_info, None) } {
411            Ok(pool) => Ok(pool),
412            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
413                Err(gpu_descriptor::CreatePoolError::OutOfHostMemory)
414            }
415            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
416                Err(gpu_descriptor::CreatePoolError::OutOfDeviceMemory)
417            }
418            Err(vk::Result::ERROR_FRAGMENTATION) => {
419                Err(gpu_descriptor::CreatePoolError::Fragmentation)
420            }
421            Err(err) => handle_unexpected(err),
422        }
423    }
424
425    unsafe fn destroy_descriptor_pool(&self, pool: vk::DescriptorPool) {
426        unsafe { self.raw.destroy_descriptor_pool(pool, None) }
427    }
428
429    unsafe fn alloc_descriptor_sets<'a>(
430        &self,
431        pool: &mut vk::DescriptorPool,
432        layouts: impl ExactSizeIterator<Item = &'a vk::DescriptorSetLayout>,
433        sets: &mut impl Extend<vk::DescriptorSet>,
434    ) -> Result<(), gpu_descriptor::DeviceAllocationError> {
435        let result = unsafe {
436            self.raw.allocate_descriptor_sets(
437                &vk::DescriptorSetAllocateInfo::default()
438                    .descriptor_pool(*pool)
439                    .set_layouts(
440                        &smallvec::SmallVec::<[vk::DescriptorSetLayout; 32]>::from_iter(
441                            layouts.cloned(),
442                        ),
443                    ),
444            )
445        };
446
447        match result {
448            Ok(vk_sets) => {
449                sets.extend(vk_sets);
450                Ok(())
451            }
452            Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY)
453            | Err(vk::Result::ERROR_OUT_OF_POOL_MEMORY) => {
454                Err(gpu_descriptor::DeviceAllocationError::OutOfHostMemory)
455            }
456            Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
457                Err(gpu_descriptor::DeviceAllocationError::OutOfDeviceMemory)
458            }
459            Err(vk::Result::ERROR_FRAGMENTED_POOL) => {
460                Err(gpu_descriptor::DeviceAllocationError::FragmentedPool)
461            }
462            Err(err) => handle_unexpected(err),
463        }
464    }
465
466    unsafe fn dealloc_descriptor_sets<'a>(
467        &self,
468        pool: &mut vk::DescriptorPool,
469        sets: impl Iterator<Item = vk::DescriptorSet>,
470    ) {
471        let result = unsafe {
472            self.raw.free_descriptor_sets(
473                *pool,
474                &smallvec::SmallVec::<[vk::DescriptorSet; 32]>::from_iter(sets),
475            )
476        };
477        match result {
478            Ok(()) => {}
479            Err(err) => handle_unexpected(err),
480        }
481    }
482}
483
484struct CompiledStage {
485    create_info: vk::PipelineShaderStageCreateInfo<'static>,
486    _entry_point: CString,
487    temp_raw_module: Option<vk::ShaderModule>,
488}
489
490impl super::Device {
491    pub(super) unsafe fn create_swapchain(
492        &self,
493        surface: &super::Surface,
494        config: &crate::SurfaceConfiguration,
495        provided_old_swapchain: Option<super::Swapchain>,
496    ) -> Result<super::Swapchain, crate::SurfaceError> {
497        profiling::scope!("Device::create_swapchain");
498        let functor = khr::swapchain::Device::new(&surface.instance.raw, &self.shared.raw);
499
500        let old_swapchain = match provided_old_swapchain {
501            Some(osc) => osc.raw,
502            None => vk::SwapchainKHR::null(),
503        };
504
505        let color_space = if config.format == wgt::TextureFormat::Rgba16Float {
506            // Enable wide color gamut mode
507            // Vulkan swapchain for Android only supports DISPLAY_P3_NONLINEAR_EXT and EXTENDED_SRGB_LINEAR_EXT
508            vk::ColorSpaceKHR::EXTENDED_SRGB_LINEAR_EXT
509        } else {
510            vk::ColorSpaceKHR::SRGB_NONLINEAR
511        };
512
513        let original_format = self.shared.private_caps.map_texture_format(config.format);
514        let mut raw_flags = vk::SwapchainCreateFlagsKHR::empty();
515        let mut raw_view_formats: Vec<vk::Format> = vec![];
516        if !config.view_formats.is_empty() {
517            raw_flags |= vk::SwapchainCreateFlagsKHR::MUTABLE_FORMAT;
518            raw_view_formats = config
519                .view_formats
520                .iter()
521                .map(|f| self.shared.private_caps.map_texture_format(*f))
522                .collect();
523            raw_view_formats.push(original_format);
524        }
525
526        let mut info = vk::SwapchainCreateInfoKHR::default()
527            .flags(raw_flags)
528            .surface(surface.raw)
529            .min_image_count(config.maximum_frame_latency + 1) // TODO: https://github.com/gfx-rs/wgpu/issues/2869
530            .image_format(original_format)
531            .image_color_space(color_space)
532            .image_extent(vk::Extent2D {
533                width: config.extent.width,
534                height: config.extent.height,
535            })
536            .image_array_layers(config.extent.depth_or_array_layers)
537            .image_usage(conv::map_texture_usage(config.usage))
538            .image_sharing_mode(vk::SharingMode::EXCLUSIVE)
539            .pre_transform(vk::SurfaceTransformFlagsKHR::IDENTITY)
540            .composite_alpha(conv::map_composite_alpha_mode(config.composite_alpha_mode))
541            .present_mode(conv::map_present_mode(config.present_mode))
542            .clipped(true)
543            .old_swapchain(old_swapchain);
544
545        let mut format_list_info = vk::ImageFormatListCreateInfo::default();
546        if !raw_view_formats.is_empty() {
547            format_list_info = format_list_info.view_formats(&raw_view_formats);
548            info = info.push_next(&mut format_list_info);
549        }
550
551        let result = {
552            profiling::scope!("vkCreateSwapchainKHR");
553            unsafe { functor.create_swapchain(&info, None) }
554        };
555
556        // doing this before bailing out with error
557        if old_swapchain != vk::SwapchainKHR::null() {
558            unsafe { functor.destroy_swapchain(old_swapchain, None) }
559        }
560
561        let raw = match result {
562            Ok(swapchain) => swapchain,
563            Err(error) => {
564                return Err(match error {
565                    vk::Result::ERROR_SURFACE_LOST_KHR
566                    | vk::Result::ERROR_INITIALIZATION_FAILED => crate::SurfaceError::Lost,
567                    vk::Result::ERROR_NATIVE_WINDOW_IN_USE_KHR => {
568                        crate::SurfaceError::Other("Native window is in use")
569                    }
570                    // We don't use VK_EXT_image_compression_control
571                    // VK_ERROR_COMPRESSION_EXHAUSTED_EXT
572                    other => super::map_host_device_oom_and_lost_err(other).into(),
573                });
574            }
575        };
576
577        let images =
578            unsafe { functor.get_swapchain_images(raw) }.map_err(super::map_host_device_oom_err)?;
579
580        // NOTE: It's important that we define the same number of acquire/present semaphores
581        // as we will need to index into them with the image index.
582        let acquire_semaphores = (0..=images.len())
583            .map(|i| {
584                super::SwapchainAcquireSemaphore::new(&self.shared, i)
585                    .map(Mutex::new)
586                    .map(Arc::new)
587            })
588            .collect::<Result<Vec<_>, _>>()?;
589
590        let present_semaphores = (0..=images.len())
591            .map(|i| Arc::new(Mutex::new(super::SwapchainPresentSemaphores::new(i))))
592            .collect::<Vec<_>>();
593
594        Ok(super::Swapchain {
595            raw,
596            functor,
597            device: Arc::clone(&self.shared),
598            images,
599            config: config.clone(),
600            acquire_semaphores,
601            next_acquire_index: 0,
602            present_semaphores,
603            next_present_time: None,
604        })
605    }
606
607    /// # Safety
608    ///
609    /// - `vk_image` must be created respecting `desc`
610    /// - If `drop_callback` is [`None`], wgpu-hal will take ownership of `vk_image`. If
611    ///   `drop_callback` is [`Some`], `vk_image` must be valid until the callback is called.
612    /// - If the `ImageCreateFlags` does not contain `MUTABLE_FORMAT`, the `view_formats` of `desc` must be empty.
613    pub unsafe fn texture_from_raw(
614        &self,
615        vk_image: vk::Image,
616        desc: &crate::TextureDescriptor,
617        drop_callback: Option<crate::DropCallback>,
618    ) -> super::Texture {
619        let mut raw_flags = vk::ImageCreateFlags::empty();
620        let mut view_formats = vec![];
621        for tf in desc.view_formats.iter() {
622            if *tf == desc.format {
623                continue;
624            }
625            view_formats.push(*tf);
626        }
627        if !view_formats.is_empty() {
628            raw_flags |=
629                vk::ImageCreateFlags::MUTABLE_FORMAT | vk::ImageCreateFlags::EXTENDED_USAGE;
630            view_formats.push(desc.format)
631        }
632        if desc.format.is_multi_planar_format() {
633            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
634        }
635
636        let identity = self.shared.texture_identity_factory.next();
637
638        let drop_guard = crate::DropGuard::from_option(drop_callback);
639
640        super::Texture {
641            raw: vk_image,
642            drop_guard,
643            external_memory: None,
644            block: None,
645            format: desc.format,
646            copy_size: desc.copy_extent(),
647            identity,
648        }
649    }
650
651    #[cfg(windows)]
652    fn find_memory_type_index(
653        &self,
654        type_bits_req: u32,
655        flags_req: vk::MemoryPropertyFlags,
656    ) -> Option<usize> {
657        let mem_properties = unsafe {
658            self.shared
659                .instance
660                .raw
661                .get_physical_device_memory_properties(self.shared.physical_device)
662        };
663
664        // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPhysicalDeviceMemoryProperties.html
665        for (i, mem_ty) in mem_properties.memory_types_as_slice().iter().enumerate() {
666            let types_bits = 1 << i;
667            let is_required_memory_type = type_bits_req & types_bits != 0;
668            let has_required_properties = mem_ty.property_flags & flags_req == flags_req;
669            if is_required_memory_type && has_required_properties {
670                return Some(i);
671            }
672        }
673
674        None
675    }
676
677    fn create_image_without_memory(
678        &self,
679        desc: &crate::TextureDescriptor,
680        external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
681    ) -> Result<ImageWithoutMemory, crate::DeviceError> {
682        let copy_size = desc.copy_extent();
683
684        let mut raw_flags = vk::ImageCreateFlags::empty();
685        if desc.dimension == wgt::TextureDimension::D3
686            && desc.usage.contains(wgt::TextureUses::COLOR_TARGET)
687        {
688            raw_flags |= vk::ImageCreateFlags::TYPE_2D_ARRAY_COMPATIBLE;
689        }
690        if desc.is_cube_compatible() {
691            raw_flags |= vk::ImageCreateFlags::CUBE_COMPATIBLE;
692        }
693
694        let original_format = self.shared.private_caps.map_texture_format(desc.format);
695        let mut vk_view_formats = vec![];
696        if !desc.view_formats.is_empty() {
697            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
698
699            if self.shared.private_caps.image_format_list {
700                vk_view_formats = desc
701                    .view_formats
702                    .iter()
703                    .map(|f| self.shared.private_caps.map_texture_format(*f))
704                    .collect();
705                vk_view_formats.push(original_format)
706            }
707        }
708        if desc.format.is_multi_planar_format() {
709            raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
710        }
711
712        let mut vk_info = vk::ImageCreateInfo::default()
713            .flags(raw_flags)
714            .image_type(conv::map_texture_dimension(desc.dimension))
715            .format(original_format)
716            .extent(conv::map_copy_extent(&copy_size))
717            .mip_levels(desc.mip_level_count)
718            .array_layers(desc.array_layer_count())
719            .samples(vk::SampleCountFlags::from_raw(desc.sample_count))
720            .tiling(vk::ImageTiling::OPTIMAL)
721            .usage(conv::map_texture_usage(desc.usage))
722            .sharing_mode(vk::SharingMode::EXCLUSIVE)
723            .initial_layout(vk::ImageLayout::UNDEFINED);
724
725        let mut format_list_info = vk::ImageFormatListCreateInfo::default();
726        if !vk_view_formats.is_empty() {
727            format_list_info = format_list_info.view_formats(&vk_view_formats);
728            vk_info = vk_info.push_next(&mut format_list_info);
729        }
730
731        if let Some(ext_info) = external_memory_image_create_info {
732            vk_info = vk_info.push_next(ext_info);
733        }
734
735        let raw = unsafe { self.shared.raw.create_image(&vk_info, None) }.map_err(map_err)?;
736        fn map_err(err: vk::Result) -> crate::DeviceError {
737            // We don't use VK_EXT_image_compression_control
738            // VK_ERROR_COMPRESSION_EXHAUSTED_EXT
739            super::map_host_device_oom_and_ioca_err(err)
740        }
741        let req = unsafe { self.shared.raw.get_image_memory_requirements(raw) };
742
743        Ok(ImageWithoutMemory {
744            raw,
745            requirements: req,
746            copy_size,
747        })
748    }
749
750    /// # Safety
751    ///
752    /// - Vulkan (with VK_KHR_external_memory_win32)
753    /// - The `d3d11_shared_handle` must be valid and respecting `desc`
754    /// - `VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_BIT` flag is used because we need to hold a reference to the handle
755    #[cfg(windows)]
756    pub unsafe fn texture_from_d3d11_shared_handle(
757        &self,
758        d3d11_shared_handle: windows::Win32::Foundation::HANDLE,
759        desc: &crate::TextureDescriptor,
760    ) -> Result<super::Texture, crate::DeviceError> {
761        if !self
762            .shared
763            .features
764            .contains(wgt::Features::VULKAN_EXTERNAL_MEMORY_WIN32)
765        {
766            log::error!("Vulkan driver does not support VK_KHR_external_memory_win32");
767            return Err(crate::DeviceError::Unexpected);
768        }
769
770        let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
771            .handle_types(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE);
772
773        let image =
774            self.create_image_without_memory(desc, Some(&mut external_memory_image_info))?;
775
776        // Some external memory types require dedicated allocation
777        // https://docs.vulkan.org/guide/latest/extensions/external.html#_importing_memory
778        let mut dedicated_allocate_info =
779            vk::MemoryDedicatedAllocateInfo::default().image(image.raw);
780
781        let mut import_memory_info = vk::ImportMemoryWin32HandleInfoKHR::default()
782            .handle_type(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE)
783            .handle(d3d11_shared_handle.0 as _);
784        // TODO: We should use `push_next` instead, but currently ash does not provide this method for the `ImportMemoryWin32HandleInfoKHR` type.
785        #[allow(clippy::unnecessary_mut_passed)]
786        {
787            import_memory_info.p_next = <*const _>::cast(&mut dedicated_allocate_info);
788        }
789
790        let mem_type_index = self
791            .find_memory_type_index(
792                image.requirements.memory_type_bits,
793                vk::MemoryPropertyFlags::DEVICE_LOCAL,
794            )
795            .ok_or(crate::DeviceError::Unexpected)?;
796
797        let memory_allocate_info = vk::MemoryAllocateInfo::default()
798            .allocation_size(image.requirements.size)
799            .memory_type_index(mem_type_index as _)
800            .push_next(&mut import_memory_info);
801        let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
802            .map_err(super::map_host_device_oom_err)?;
803
804        unsafe { self.shared.raw.bind_image_memory(image.raw, memory, 0) }
805            .map_err(super::map_host_device_oom_err)?;
806
807        if let Some(label) = desc.label {
808            unsafe { self.shared.set_object_name(image.raw, label) };
809        }
810
811        let identity = self.shared.texture_identity_factory.next();
812
813        self.counters.textures.add(1);
814
815        Ok(super::Texture {
816            raw: image.raw,
817            drop_guard: None,
818            external_memory: Some(memory),
819            block: None,
820            format: desc.format,
821            copy_size: image.copy_size,
822            identity,
823        })
824    }
825
826    fn create_shader_module_impl(
827        &self,
828        spv: &[u32],
829        label: &crate::Label<'_>,
830    ) -> Result<vk::ShaderModule, crate::DeviceError> {
831        let vk_info = vk::ShaderModuleCreateInfo::default()
832            .flags(vk::ShaderModuleCreateFlags::empty())
833            .code(spv);
834
835        let raw = unsafe {
836            profiling::scope!("vkCreateShaderModule");
837            self.shared
838                .raw
839                .create_shader_module(&vk_info, None)
840                .map_err(map_err)?
841        };
842        fn map_err(err: vk::Result) -> crate::DeviceError {
843            // We don't use VK_NV_glsl_shader
844            // VK_ERROR_INVALID_SHADER_NV
845            super::map_host_device_oom_err(err)
846        }
847
848        if let Some(label) = label {
849            unsafe { self.shared.set_object_name(raw, label) };
850        }
851
852        Ok(raw)
853    }
854
855    fn compile_stage(
856        &self,
857        stage: &crate::ProgrammableStage<super::ShaderModule>,
858        naga_stage: naga::ShaderStage,
859        binding_map: &naga::back::spv::BindingMap,
860    ) -> Result<CompiledStage, crate::PipelineError> {
861        let stage_flags = crate::auxil::map_naga_stage(naga_stage);
862        let vk_module = match *stage.module {
863            super::ShaderModule::Raw(raw) => raw,
864            super::ShaderModule::Intermediate {
865                ref naga_shader,
866                runtime_checks,
867            } => {
868                let pipeline_options = naga::back::spv::PipelineOptions {
869                    entry_point: stage.entry_point.to_owned(),
870                    shader_stage: naga_stage,
871                };
872                let needs_temp_options = !runtime_checks.bounds_checks
873                    || !runtime_checks.force_loop_bounding
874                    || !binding_map.is_empty()
875                    || naga_shader.debug_source.is_some()
876                    || !stage.zero_initialize_workgroup_memory;
877                let mut temp_options;
878                let options = if needs_temp_options {
879                    temp_options = self.naga_options.clone();
880                    if !runtime_checks.bounds_checks {
881                        temp_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
882                            index: naga::proc::BoundsCheckPolicy::Unchecked,
883                            buffer: naga::proc::BoundsCheckPolicy::Unchecked,
884                            image_load: naga::proc::BoundsCheckPolicy::Unchecked,
885                            binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
886                        };
887                    }
888                    if !runtime_checks.force_loop_bounding {
889                        temp_options.force_loop_bounding = false;
890                    }
891                    if !binding_map.is_empty() {
892                        temp_options.binding_map = binding_map.clone();
893                    }
894
895                    if let Some(ref debug) = naga_shader.debug_source {
896                        temp_options.debug_info = Some(naga::back::spv::DebugInfo {
897                            source_code: &debug.source_code,
898                            file_name: debug.file_name.as_ref().into(),
899                            language: naga::back::spv::SourceLanguage::WGSL,
900                        })
901                    }
902                    if !stage.zero_initialize_workgroup_memory {
903                        temp_options.zero_initialize_workgroup_memory =
904                            naga::back::spv::ZeroInitializeWorkgroupMemoryMode::None;
905                    }
906
907                    &temp_options
908                } else {
909                    &self.naga_options
910                };
911
912                let (module, info) = naga::back::pipeline_constants::process_overrides(
913                    &naga_shader.module,
914                    &naga_shader.info,
915                    Some((naga_stage, stage.entry_point)),
916                    stage.constants,
917                )
918                .map_err(|e| {
919                    crate::PipelineError::PipelineConstants(stage_flags, format!("{e}"))
920                })?;
921
922                let spv = {
923                    profiling::scope!("naga::spv::write_vec");
924                    naga::back::spv::write_vec(&module, &info, options, Some(&pipeline_options))
925                }
926                .map_err(|e| crate::PipelineError::Linkage(stage_flags, format!("{e}")))?;
927                self.create_shader_module_impl(&spv, &None)?
928            }
929        };
930
931        let mut flags = vk::PipelineShaderStageCreateFlags::empty();
932        if self.shared.features.contains(wgt::Features::SUBGROUP) {
933            flags |= vk::PipelineShaderStageCreateFlags::ALLOW_VARYING_SUBGROUP_SIZE
934        }
935
936        let entry_point = CString::new(stage.entry_point).unwrap();
937        let mut create_info = vk::PipelineShaderStageCreateInfo::default()
938            .flags(flags)
939            .stage(conv::map_shader_stage(stage_flags))
940            .module(vk_module);
941
942        // Circumvent struct lifetime check because of a self-reference inside CompiledStage
943        create_info.p_name = entry_point.as_ptr();
944
945        Ok(CompiledStage {
946            create_info,
947            _entry_point: entry_point,
948            temp_raw_module: match *stage.module {
949                super::ShaderModule::Raw(_) => None,
950                super::ShaderModule::Intermediate { .. } => Some(vk_module),
951            },
952        })
953    }
954
955    /// Returns the queue family index of the device's internal queue.
956    ///
957    /// This is useful for constructing memory barriers needed for queue family ownership transfer when
958    /// external memory is involved (from/to `VK_QUEUE_FAMILY_EXTERNAL_KHR` and `VK_QUEUE_FAMILY_FOREIGN_EXT`
959    /// for example).
960    pub fn queue_family_index(&self) -> u32 {
961        self.shared.family_index
962    }
963
964    pub fn queue_index(&self) -> u32 {
965        self.shared.queue_index
966    }
967
968    pub fn raw_device(&self) -> &ash::Device {
969        &self.shared.raw
970    }
971
972    pub fn raw_physical_device(&self) -> vk::PhysicalDevice {
973        self.shared.physical_device
974    }
975
976    pub fn raw_queue(&self) -> vk::Queue {
977        self.shared.raw_queue
978    }
979
980    pub fn enabled_device_extensions(&self) -> &[&'static CStr] {
981        &self.shared.enabled_extensions
982    }
983
984    pub fn shared_instance(&self) -> &super::InstanceShared {
985        &self.shared.instance
986    }
987
988    fn error_if_would_oom_on_resource_allocation(
989        &self,
990        needs_host_access: bool,
991        size: u64,
992    ) -> Result<(), crate::DeviceError> {
993        let Some(threshold) = self
994            .shared
995            .instance
996            .memory_budget_thresholds
997            .for_resource_creation
998        else {
999            return Ok(());
1000        };
1001
1002        if !self
1003            .shared
1004            .enabled_extensions
1005            .contains(&ext::memory_budget::NAME)
1006        {
1007            return Ok(());
1008        }
1009
1010        let get_physical_device_properties = self
1011            .shared
1012            .instance
1013            .get_physical_device_properties
1014            .as_ref()
1015            .unwrap();
1016
1017        let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
1018
1019        let mut memory_properties =
1020            vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
1021
1022        unsafe {
1023            get_physical_device_properties.get_physical_device_memory_properties2(
1024                self.shared.physical_device,
1025                &mut memory_properties,
1026            );
1027        }
1028
1029        let mut host_visible_heaps = [false; vk::MAX_MEMORY_HEAPS];
1030        let mut device_local_heaps = [false; vk::MAX_MEMORY_HEAPS];
1031
1032        let memory_properties = memory_properties.memory_properties;
1033
1034        for i in 0..memory_properties.memory_type_count {
1035            let memory_type = memory_properties.memory_types[i as usize];
1036            let flags = memory_type.property_flags;
1037
1038            if flags.intersects(
1039                vk::MemoryPropertyFlags::LAZILY_ALLOCATED | vk::MemoryPropertyFlags::PROTECTED,
1040            ) {
1041                continue; // not used by gpu-alloc
1042            }
1043
1044            if flags.contains(vk::MemoryPropertyFlags::HOST_VISIBLE) {
1045                host_visible_heaps[memory_type.heap_index as usize] = true;
1046            }
1047
1048            if flags.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL) {
1049                device_local_heaps[memory_type.heap_index as usize] = true;
1050            }
1051        }
1052
1053        let heaps = if needs_host_access {
1054            host_visible_heaps
1055        } else {
1056            device_local_heaps
1057        };
1058
1059        // NOTE: We might end up checking multiple heaps since gpu-alloc doesn't have a way
1060        // for us to query the heap the resource will end up on. But this is unlikely,
1061        // there is usually only one heap on integrated GPUs and two on dedicated GPUs.
1062
1063        for (i, check) in heaps.iter().enumerate() {
1064            if !check {
1065                continue;
1066            }
1067
1068            let heap_usage = memory_budget_properties.heap_usage[i];
1069            let heap_budget = memory_budget_properties.heap_budget[i];
1070
1071            if heap_usage + size >= heap_budget / 100 * threshold as u64 {
1072                return Err(crate::DeviceError::OutOfMemory);
1073            }
1074        }
1075
1076        Ok(())
1077    }
1078}
1079
1080impl crate::Device for super::Device {
1081    type A = super::Api;
1082
1083    unsafe fn create_buffer(
1084        &self,
1085        desc: &crate::BufferDescriptor,
1086    ) -> Result<super::Buffer, crate::DeviceError> {
1087        let vk_info = vk::BufferCreateInfo::default()
1088            .size(desc.size)
1089            .usage(conv::map_buffer_usage(desc.usage))
1090            .sharing_mode(vk::SharingMode::EXCLUSIVE);
1091
1092        let raw = unsafe {
1093            self.shared
1094                .raw
1095                .create_buffer(&vk_info, None)
1096                .map_err(super::map_host_device_oom_and_ioca_err)?
1097        };
1098        let req = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
1099
1100        let mut alloc_usage = if desc
1101            .usage
1102            .intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE)
1103        {
1104            let mut flags = gpu_alloc::UsageFlags::HOST_ACCESS;
1105            //TODO: find a way to use `crate::MemoryFlags::PREFER_COHERENT`
1106            flags.set(
1107                gpu_alloc::UsageFlags::DOWNLOAD,
1108                desc.usage.contains(wgt::BufferUses::MAP_READ),
1109            );
1110            flags.set(
1111                gpu_alloc::UsageFlags::UPLOAD,
1112                desc.usage.contains(wgt::BufferUses::MAP_WRITE),
1113            );
1114            flags
1115        } else {
1116            gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS
1117        };
1118        alloc_usage.set(
1119            gpu_alloc::UsageFlags::TRANSIENT,
1120            desc.memory_flags.contains(crate::MemoryFlags::TRANSIENT),
1121        );
1122
1123        let needs_host_access = alloc_usage.contains(gpu_alloc::UsageFlags::HOST_ACCESS);
1124
1125        self.error_if_would_oom_on_resource_allocation(needs_host_access, req.size)
1126            .inspect_err(|_| {
1127                unsafe { self.shared.raw.destroy_buffer(raw, None) };
1128            })?;
1129
1130        let alignment_mask = req.alignment - 1;
1131
1132        let block = unsafe {
1133            self.mem_allocator.lock().alloc(
1134                &*self.shared,
1135                gpu_alloc::Request {
1136                    size: req.size,
1137                    align_mask: alignment_mask,
1138                    usage: alloc_usage,
1139                    memory_types: req.memory_type_bits & self.valid_ash_memory_types,
1140                },
1141            )
1142        }
1143        .inspect_err(|_| {
1144            unsafe { self.shared.raw.destroy_buffer(raw, None) };
1145        })?;
1146
1147        unsafe {
1148            self.shared
1149                .raw
1150                .bind_buffer_memory(raw, *block.memory(), block.offset())
1151        }
1152        .map_err(super::map_host_device_oom_and_ioca_err)
1153        .inspect_err(|_| {
1154            unsafe { self.shared.raw.destroy_buffer(raw, None) };
1155        })?;
1156
1157        if let Some(label) = desc.label {
1158            unsafe { self.shared.set_object_name(raw, label) };
1159        }
1160
1161        self.counters.buffer_memory.add(block.size() as isize);
1162        self.counters.buffers.add(1);
1163
1164        Ok(super::Buffer {
1165            raw,
1166            block: Some(Mutex::new(super::BufferMemoryBacking::Managed(block))),
1167        })
1168    }
1169    unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
1170        unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1171        if let Some(block) = buffer.block {
1172            let block = block.into_inner();
1173            self.counters.buffer_memory.sub(block.size() as isize);
1174            match block {
1175                super::BufferMemoryBacking::Managed(block) => unsafe {
1176                    self.mem_allocator.lock().dealloc(&*self.shared, block)
1177                },
1178                super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
1179                    self.shared.raw.free_memory(memory, None);
1180                },
1181            }
1182        }
1183
1184        self.counters.buffers.sub(1);
1185    }
1186
1187    unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) {
1188        self.counters.buffers.add(1);
1189    }
1190
1191    unsafe fn map_buffer(
1192        &self,
1193        buffer: &super::Buffer,
1194        range: crate::MemoryRange,
1195    ) -> Result<crate::BufferMapping, crate::DeviceError> {
1196        if let Some(ref block) = buffer.block {
1197            let size = range.end - range.start;
1198            let mut block = block.lock();
1199            if let super::BufferMemoryBacking::Managed(ref mut block) = *block {
1200                let ptr = unsafe { block.map(&*self.shared, range.start, size as usize)? };
1201                let is_coherent = block
1202                    .props()
1203                    .contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT);
1204                Ok(crate::BufferMapping { ptr, is_coherent })
1205            } else {
1206                crate::hal_usage_error("tried to map externally created buffer")
1207            }
1208        } else {
1209            crate::hal_usage_error("tried to map external buffer")
1210        }
1211    }
1212    unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
1213        if let Some(ref block) = buffer.block {
1214            match &mut *block.lock() {
1215                super::BufferMemoryBacking::Managed(block) => unsafe { block.unmap(&*self.shared) },
1216                super::BufferMemoryBacking::VulkanMemory { .. } => {
1217                    crate::hal_usage_error("tried to unmap externally created buffer")
1218                }
1219            };
1220        } else {
1221            crate::hal_usage_error("tried to unmap external buffer")
1222        }
1223    }
1224
1225    unsafe fn flush_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1226    where
1227        I: Iterator<Item = crate::MemoryRange>,
1228    {
1229        if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1230            unsafe {
1231                self.shared
1232                    .raw
1233                    .flush_mapped_memory_ranges(
1234                        &smallvec::SmallVec::<[vk::MappedMemoryRange; 32]>::from_iter(vk_ranges),
1235                    )
1236            }
1237            .unwrap();
1238        }
1239    }
1240    unsafe fn invalidate_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1241    where
1242        I: Iterator<Item = crate::MemoryRange>,
1243    {
1244        if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1245            unsafe {
1246                self.shared
1247                    .raw
1248                    .invalidate_mapped_memory_ranges(&smallvec::SmallVec::<
1249                        [vk::MappedMemoryRange; 32],
1250                    >::from_iter(vk_ranges))
1251            }
1252            .unwrap();
1253        }
1254    }
1255
1256    unsafe fn create_texture(
1257        &self,
1258        desc: &crate::TextureDescriptor,
1259    ) -> Result<super::Texture, crate::DeviceError> {
1260        let image = self.create_image_without_memory(desc, None)?;
1261
1262        self.error_if_would_oom_on_resource_allocation(false, image.requirements.size)
1263            .inspect_err(|_| {
1264                unsafe { self.shared.raw.destroy_image(image.raw, None) };
1265            })?;
1266
1267        let block = unsafe {
1268            self.mem_allocator.lock().alloc(
1269                &*self.shared,
1270                gpu_alloc::Request {
1271                    size: image.requirements.size,
1272                    align_mask: image.requirements.alignment - 1,
1273                    usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
1274                    memory_types: image.requirements.memory_type_bits & self.valid_ash_memory_types,
1275                },
1276            )
1277        }
1278        .inspect_err(|_| {
1279            unsafe { self.shared.raw.destroy_image(image.raw, None) };
1280        })?;
1281
1282        self.counters.texture_memory.add(block.size() as isize);
1283
1284        unsafe {
1285            self.shared
1286                .raw
1287                .bind_image_memory(image.raw, *block.memory(), block.offset())
1288        }
1289        .map_err(super::map_host_device_oom_err)
1290        .inspect_err(|_| {
1291            unsafe { self.shared.raw.destroy_image(image.raw, None) };
1292        })?;
1293
1294        if let Some(label) = desc.label {
1295            unsafe { self.shared.set_object_name(image.raw, label) };
1296        }
1297
1298        let identity = self.shared.texture_identity_factory.next();
1299
1300        self.counters.textures.add(1);
1301
1302        Ok(super::Texture {
1303            raw: image.raw,
1304            drop_guard: None,
1305            external_memory: None,
1306            block: Some(block),
1307            format: desc.format,
1308            copy_size: image.copy_size,
1309            identity,
1310        })
1311    }
1312    unsafe fn destroy_texture(&self, texture: super::Texture) {
1313        if texture.drop_guard.is_none() {
1314            unsafe { self.shared.raw.destroy_image(texture.raw, None) };
1315        }
1316        if let Some(memory) = texture.external_memory {
1317            unsafe { self.shared.raw.free_memory(memory, None) };
1318        }
1319        if let Some(block) = texture.block {
1320            self.counters.texture_memory.sub(block.size() as isize);
1321
1322            unsafe { self.mem_allocator.lock().dealloc(&*self.shared, block) };
1323        }
1324
1325        self.counters.textures.sub(1);
1326    }
1327
1328    unsafe fn add_raw_texture(&self, _texture: &super::Texture) {
1329        self.counters.textures.add(1);
1330    }
1331
1332    unsafe fn create_texture_view(
1333        &self,
1334        texture: &super::Texture,
1335        desc: &crate::TextureViewDescriptor,
1336    ) -> Result<super::TextureView, crate::DeviceError> {
1337        let subresource_range = conv::map_subresource_range(&desc.range, texture.format);
1338        let raw_format = self.shared.private_caps.map_texture_format(desc.format);
1339        let mut vk_info = vk::ImageViewCreateInfo::default()
1340            .flags(vk::ImageViewCreateFlags::empty())
1341            .image(texture.raw)
1342            .view_type(conv::map_view_dimension(desc.dimension))
1343            .format(raw_format)
1344            .subresource_range(subresource_range);
1345        let layers =
1346            NonZeroU32::new(subresource_range.layer_count).expect("Unexpected zero layer count");
1347
1348        let mut image_view_info;
1349        if self.shared.private_caps.image_view_usage && !desc.usage.is_empty() {
1350            image_view_info =
1351                vk::ImageViewUsageCreateInfo::default().usage(conv::map_texture_usage(desc.usage));
1352            vk_info = vk_info.push_next(&mut image_view_info);
1353        }
1354
1355        let raw = unsafe { self.shared.raw.create_image_view(&vk_info, None) }
1356            .map_err(super::map_host_device_oom_and_ioca_err)?;
1357
1358        if let Some(label) = desc.label {
1359            unsafe { self.shared.set_object_name(raw, label) };
1360        }
1361
1362        let identity = self.shared.texture_view_identity_factory.next();
1363
1364        self.counters.texture_views.add(1);
1365
1366        Ok(super::TextureView {
1367            raw_texture: texture.raw,
1368            raw,
1369            layers,
1370            format: desc.format,
1371            raw_format,
1372            base_mip_level: desc.range.base_mip_level,
1373            dimension: desc.dimension,
1374            texture_identity: texture.identity,
1375            view_identity: identity,
1376        })
1377    }
1378    unsafe fn destroy_texture_view(&self, view: super::TextureView) {
1379        unsafe { self.shared.raw.destroy_image_view(view.raw, None) };
1380
1381        self.counters.texture_views.sub(1);
1382    }
1383
1384    unsafe fn create_sampler(
1385        &self,
1386        desc: &crate::SamplerDescriptor,
1387    ) -> Result<super::Sampler, crate::DeviceError> {
1388        let mut create_info = vk::SamplerCreateInfo::default()
1389            .flags(vk::SamplerCreateFlags::empty())
1390            .mag_filter(conv::map_filter_mode(desc.mag_filter))
1391            .min_filter(conv::map_filter_mode(desc.min_filter))
1392            .mipmap_mode(conv::map_mip_filter_mode(desc.mipmap_filter))
1393            .address_mode_u(conv::map_address_mode(desc.address_modes[0]))
1394            .address_mode_v(conv::map_address_mode(desc.address_modes[1]))
1395            .address_mode_w(conv::map_address_mode(desc.address_modes[2]))
1396            .min_lod(desc.lod_clamp.start)
1397            .max_lod(desc.lod_clamp.end);
1398
1399        if let Some(fun) = desc.compare {
1400            create_info = create_info
1401                .compare_enable(true)
1402                .compare_op(conv::map_comparison(fun));
1403        }
1404
1405        if desc.anisotropy_clamp != 1 {
1406            // We only enable anisotropy if it is supported, and wgpu-hal interface guarantees
1407            // the clamp is in the range [1, 16] which is always supported if anisotropy is.
1408            create_info = create_info
1409                .anisotropy_enable(true)
1410                .max_anisotropy(desc.anisotropy_clamp as f32);
1411        }
1412
1413        if let Some(color) = desc.border_color {
1414            create_info = create_info.border_color(conv::map_border_color(color));
1415        }
1416
1417        let mut sampler_cache_guard = self.shared.sampler_cache.lock();
1418
1419        let raw = sampler_cache_guard.create_sampler(&self.shared.raw, create_info)?;
1420
1421        // Note: Cached samplers will just continually overwrite the label
1422        //
1423        // https://github.com/gfx-rs/wgpu/issues/6867
1424        if let Some(label) = desc.label {
1425            // SAFETY: we are holding a lock on the sampler cache,
1426            // so we can only be setting the name from one thread.
1427            unsafe { self.shared.set_object_name(raw, label) };
1428        }
1429
1430        drop(sampler_cache_guard);
1431
1432        self.counters.samplers.add(1);
1433
1434        Ok(super::Sampler { raw, create_info })
1435    }
1436    unsafe fn destroy_sampler(&self, sampler: super::Sampler) {
1437        self.shared.sampler_cache.lock().destroy_sampler(
1438            &self.shared.raw,
1439            sampler.create_info,
1440            sampler.raw,
1441        );
1442
1443        self.counters.samplers.sub(1);
1444    }
1445
1446    unsafe fn create_command_encoder(
1447        &self,
1448        desc: &crate::CommandEncoderDescriptor<super::Queue>,
1449    ) -> Result<super::CommandEncoder, crate::DeviceError> {
1450        let vk_info = vk::CommandPoolCreateInfo::default()
1451            .queue_family_index(desc.queue.family_index)
1452            .flags(vk::CommandPoolCreateFlags::TRANSIENT);
1453
1454        let raw = unsafe {
1455            self.shared
1456                .raw
1457                .create_command_pool(&vk_info, None)
1458                .map_err(super::map_host_device_oom_err)?
1459        };
1460
1461        self.counters.command_encoders.add(1);
1462
1463        Ok(super::CommandEncoder {
1464            raw,
1465            device: Arc::clone(&self.shared),
1466            active: vk::CommandBuffer::null(),
1467            bind_point: vk::PipelineBindPoint::default(),
1468            temp: super::Temp::default(),
1469            free: Vec::new(),
1470            discarded: Vec::new(),
1471            rpass_debug_marker_active: false,
1472            end_of_pass_timer_query: None,
1473            framebuffers: Default::default(),
1474            temp_texture_views: Default::default(),
1475            counters: Arc::clone(&self.counters),
1476        })
1477    }
1478
1479    unsafe fn create_bind_group_layout(
1480        &self,
1481        desc: &crate::BindGroupLayoutDescriptor,
1482    ) -> Result<super::BindGroupLayout, crate::DeviceError> {
1483        let mut desc_count = gpu_descriptor::DescriptorTotalCount::default();
1484        let mut types = Vec::new();
1485        for entry in desc.entries {
1486            let count = entry.count.map_or(1, |c| c.get());
1487            if entry.binding as usize >= types.len() {
1488                types.resize(
1489                    entry.binding as usize + 1,
1490                    (vk::DescriptorType::INPUT_ATTACHMENT, 0),
1491                );
1492            }
1493            types[entry.binding as usize] = (
1494                conv::map_binding_type(entry.ty),
1495                entry.count.map_or(1, |c| c.get()),
1496            );
1497
1498            match entry.ty {
1499                wgt::BindingType::Buffer {
1500                    ty,
1501                    has_dynamic_offset,
1502                    ..
1503                } => match ty {
1504                    wgt::BufferBindingType::Uniform => {
1505                        if has_dynamic_offset {
1506                            desc_count.uniform_buffer_dynamic += count;
1507                        } else {
1508                            desc_count.uniform_buffer += count;
1509                        }
1510                    }
1511                    wgt::BufferBindingType::Storage { .. } => {
1512                        if has_dynamic_offset {
1513                            desc_count.storage_buffer_dynamic += count;
1514                        } else {
1515                            desc_count.storage_buffer += count;
1516                        }
1517                    }
1518                },
1519                wgt::BindingType::Sampler { .. } => {
1520                    desc_count.sampler += count;
1521                }
1522                wgt::BindingType::Texture { .. } => {
1523                    desc_count.sampled_image += count;
1524                }
1525                wgt::BindingType::StorageTexture { .. } => {
1526                    desc_count.storage_image += count;
1527                }
1528                wgt::BindingType::AccelerationStructure { .. } => {
1529                    desc_count.acceleration_structure += count;
1530                }
1531                wgt::BindingType::ExternalTexture => unimplemented!(),
1532            }
1533        }
1534
1535        //Note: not bothering with on stack array here as it's low frequency
1536        let vk_bindings = desc
1537            .entries
1538            .iter()
1539            .map(|entry| vk::DescriptorSetLayoutBinding {
1540                binding: entry.binding,
1541                descriptor_type: types[entry.binding as usize].0,
1542                descriptor_count: types[entry.binding as usize].1,
1543                stage_flags: conv::map_shader_stage(entry.visibility),
1544                p_immutable_samplers: ptr::null(),
1545                _marker: Default::default(),
1546            })
1547            .collect::<Vec<_>>();
1548
1549        let binding_arrays: Vec<_> = desc
1550            .entries
1551            .iter()
1552            .enumerate()
1553            .filter_map(|(idx, entry)| entry.count.map(|count| (idx as u32, count)))
1554            .collect();
1555
1556        let vk_info = vk::DescriptorSetLayoutCreateInfo::default()
1557            .bindings(&vk_bindings)
1558            .flags(if !binding_arrays.is_empty() {
1559                vk::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND_POOL
1560            } else {
1561                vk::DescriptorSetLayoutCreateFlags::empty()
1562            });
1563
1564        let partially_bound = desc
1565            .flags
1566            .contains(crate::BindGroupLayoutFlags::PARTIALLY_BOUND);
1567
1568        let binding_flag_vec = desc
1569            .entries
1570            .iter()
1571            .map(|entry| {
1572                let mut flags = vk::DescriptorBindingFlags::empty();
1573
1574                if partially_bound && entry.count.is_some() {
1575                    flags |= vk::DescriptorBindingFlags::PARTIALLY_BOUND;
1576                }
1577
1578                if entry.count.is_some() {
1579                    flags |= vk::DescriptorBindingFlags::UPDATE_AFTER_BIND;
1580                }
1581
1582                flags
1583            })
1584            .collect::<Vec<_>>();
1585
1586        let mut binding_flag_info = vk::DescriptorSetLayoutBindingFlagsCreateInfo::default()
1587            .binding_flags(&binding_flag_vec);
1588
1589        let vk_info = vk_info.push_next(&mut binding_flag_info);
1590
1591        let raw = unsafe {
1592            self.shared
1593                .raw
1594                .create_descriptor_set_layout(&vk_info, None)
1595                .map_err(super::map_host_device_oom_err)?
1596        };
1597
1598        if let Some(label) = desc.label {
1599            unsafe { self.shared.set_object_name(raw, label) };
1600        }
1601
1602        self.counters.bind_group_layouts.add(1);
1603
1604        Ok(super::BindGroupLayout {
1605            raw,
1606            desc_count,
1607            types: types.into_boxed_slice(),
1608            binding_arrays,
1609        })
1610    }
1611    unsafe fn destroy_bind_group_layout(&self, bg_layout: super::BindGroupLayout) {
1612        unsafe {
1613            self.shared
1614                .raw
1615                .destroy_descriptor_set_layout(bg_layout.raw, None)
1616        };
1617
1618        self.counters.bind_group_layouts.sub(1);
1619    }
1620
1621    unsafe fn create_pipeline_layout(
1622        &self,
1623        desc: &crate::PipelineLayoutDescriptor<super::BindGroupLayout>,
1624    ) -> Result<super::PipelineLayout, crate::DeviceError> {
1625        //Note: not bothering with on stack array here as it's low frequency
1626        let vk_set_layouts = desc
1627            .bind_group_layouts
1628            .iter()
1629            .map(|bgl| bgl.raw)
1630            .collect::<Vec<_>>();
1631        let vk_push_constant_ranges = desc
1632            .push_constant_ranges
1633            .iter()
1634            .map(|pcr| vk::PushConstantRange {
1635                stage_flags: conv::map_shader_stage(pcr.stages),
1636                offset: pcr.range.start,
1637                size: pcr.range.end - pcr.range.start,
1638            })
1639            .collect::<Vec<_>>();
1640
1641        let vk_info = vk::PipelineLayoutCreateInfo::default()
1642            .flags(vk::PipelineLayoutCreateFlags::empty())
1643            .set_layouts(&vk_set_layouts)
1644            .push_constant_ranges(&vk_push_constant_ranges);
1645
1646        let raw = {
1647            profiling::scope!("vkCreatePipelineLayout");
1648            unsafe {
1649                self.shared
1650                    .raw
1651                    .create_pipeline_layout(&vk_info, None)
1652                    .map_err(super::map_host_device_oom_err)?
1653            }
1654        };
1655
1656        if let Some(label) = desc.label {
1657            unsafe { self.shared.set_object_name(raw, label) };
1658        }
1659
1660        let mut binding_arrays = BTreeMap::new();
1661        for (group, &layout) in desc.bind_group_layouts.iter().enumerate() {
1662            for &(binding, binding_array_size) in &layout.binding_arrays {
1663                binding_arrays.insert(
1664                    naga::ResourceBinding {
1665                        group: group as u32,
1666                        binding,
1667                    },
1668                    naga::back::spv::BindingInfo {
1669                        binding_array_size: Some(binding_array_size.get()),
1670                    },
1671                );
1672            }
1673        }
1674
1675        self.counters.pipeline_layouts.add(1);
1676
1677        Ok(super::PipelineLayout {
1678            raw,
1679            binding_arrays,
1680        })
1681    }
1682    unsafe fn destroy_pipeline_layout(&self, pipeline_layout: super::PipelineLayout) {
1683        unsafe {
1684            self.shared
1685                .raw
1686                .destroy_pipeline_layout(pipeline_layout.raw, None)
1687        };
1688
1689        self.counters.pipeline_layouts.sub(1);
1690    }
1691
1692    unsafe fn create_bind_group(
1693        &self,
1694        desc: &crate::BindGroupDescriptor<
1695            super::BindGroupLayout,
1696            super::Buffer,
1697            super::Sampler,
1698            super::TextureView,
1699            super::AccelerationStructure,
1700        >,
1701    ) -> Result<super::BindGroup, crate::DeviceError> {
1702        let contains_binding_arrays = !desc.layout.binding_arrays.is_empty();
1703
1704        let desc_set_layout_flags = if contains_binding_arrays {
1705            gpu_descriptor::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND
1706        } else {
1707            gpu_descriptor::DescriptorSetLayoutCreateFlags::empty()
1708        };
1709
1710        let mut vk_sets = unsafe {
1711            self.desc_allocator.lock().allocate(
1712                &*self.shared,
1713                &desc.layout.raw,
1714                desc_set_layout_flags,
1715                &desc.layout.desc_count,
1716                1,
1717            )?
1718        };
1719
1720        let set = vk_sets.pop().unwrap();
1721        if let Some(label) = desc.label {
1722            unsafe { self.shared.set_object_name(*set.raw(), label) };
1723        }
1724
1725        /// Helper for splitting off and initializing a given number of elements on a pre-allocated
1726        /// stack, based on items returned from an [`ExactSizeIterator`].  Typically created from a
1727        /// [`MaybeUninit`] slice (see [`Vec::spare_capacity_mut()`]).
1728        /// The updated [`ExtensionStack`] of remaining uninitialized elements is returned, safely
1729        /// representing that the initialized and remaining elements are two independent mutable
1730        /// borrows.
1731        struct ExtendStack<'a, T> {
1732            remainder: &'a mut [MaybeUninit<T>],
1733        }
1734
1735        impl<'a, T> ExtendStack<'a, T> {
1736            fn from_vec_capacity(vec: &'a mut Vec<T>) -> Self {
1737                Self {
1738                    remainder: vec.spare_capacity_mut(),
1739                }
1740            }
1741
1742            fn extend_one(self, value: T) -> (Self, &'a mut T) {
1743                let (to_init, remainder) = self.remainder.split_first_mut().unwrap();
1744                let init = to_init.write(value);
1745                (Self { remainder }, init)
1746            }
1747
1748            fn extend(
1749                self,
1750                iter: impl IntoIterator<Item = T> + ExactSizeIterator,
1751            ) -> (Self, &'a mut [T]) {
1752                let (to_init, remainder) = self.remainder.split_at_mut(iter.len());
1753
1754                for (value, to_init) in iter.into_iter().zip(to_init.iter_mut()) {
1755                    to_init.write(value);
1756                }
1757
1758                // we can't use the safe (yet unstable) MaybeUninit::write_slice() here because of having an iterator to write
1759
1760                let init = {
1761                    // SAFETY: The loop above has initialized exactly as many items as to_init is
1762                    // long, so it is safe to cast away the MaybeUninit<T> wrapper into T.
1763
1764                    // Additional safety docs from unstable slice_assume_init_mut
1765                    // SAFETY: similar to safety notes for `slice_get_ref`, but we have a
1766                    // mutable reference which is also guaranteed to be valid for writes.
1767                    unsafe { mem::transmute::<&mut [MaybeUninit<T>], &mut [T]>(to_init) }
1768                };
1769                (Self { remainder }, init)
1770            }
1771        }
1772
1773        let mut writes = Vec::with_capacity(desc.entries.len());
1774        let mut buffer_infos = Vec::with_capacity(desc.buffers.len());
1775        let mut buffer_infos = ExtendStack::from_vec_capacity(&mut buffer_infos);
1776        let mut image_infos = Vec::with_capacity(desc.samplers.len() + desc.textures.len());
1777        let mut image_infos = ExtendStack::from_vec_capacity(&mut image_infos);
1778        // TODO: This length could be reduced to just the number of top-level acceleration
1779        // structure bindings, where multiple consecutive TLAS bindings that are set via
1780        // one `WriteDescriptorSet` count towards one "info" struct, not the total number of
1781        // acceleration structure bindings to write:
1782        let mut acceleration_structure_infos =
1783            Vec::with_capacity(desc.acceleration_structures.len());
1784        let mut acceleration_structure_infos =
1785            ExtendStack::from_vec_capacity(&mut acceleration_structure_infos);
1786        let mut raw_acceleration_structures =
1787            Vec::with_capacity(desc.acceleration_structures.len());
1788        let mut raw_acceleration_structures =
1789            ExtendStack::from_vec_capacity(&mut raw_acceleration_structures);
1790        for entry in desc.entries {
1791            let (ty, size) = desc.layout.types[entry.binding as usize];
1792            if size == 0 {
1793                continue; // empty slot
1794            }
1795            let mut write = vk::WriteDescriptorSet::default()
1796                .dst_set(*set.raw())
1797                .dst_binding(entry.binding)
1798                .descriptor_type(ty);
1799
1800            write = match ty {
1801                vk::DescriptorType::SAMPLER => {
1802                    let start = entry.resource_index;
1803                    let end = start + entry.count;
1804                    let local_image_infos;
1805                    (image_infos, local_image_infos) =
1806                        image_infos.extend(desc.samplers[start as usize..end as usize].iter().map(
1807                            |sampler| vk::DescriptorImageInfo::default().sampler(sampler.raw),
1808                        ));
1809                    write.image_info(local_image_infos)
1810                }
1811                vk::DescriptorType::SAMPLED_IMAGE | vk::DescriptorType::STORAGE_IMAGE => {
1812                    let start = entry.resource_index;
1813                    let end = start + entry.count;
1814                    let local_image_infos;
1815                    (image_infos, local_image_infos) =
1816                        image_infos.extend(desc.textures[start as usize..end as usize].iter().map(
1817                            |binding| {
1818                                let layout =
1819                                    conv::derive_image_layout(binding.usage, binding.view.format);
1820                                vk::DescriptorImageInfo::default()
1821                                    .image_view(binding.view.raw)
1822                                    .image_layout(layout)
1823                            },
1824                        ));
1825                    write.image_info(local_image_infos)
1826                }
1827                vk::DescriptorType::UNIFORM_BUFFER
1828                | vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC
1829                | vk::DescriptorType::STORAGE_BUFFER
1830                | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => {
1831                    let start = entry.resource_index;
1832                    let end = start + entry.count;
1833                    let local_buffer_infos;
1834                    (buffer_infos, local_buffer_infos) =
1835                        buffer_infos.extend(desc.buffers[start as usize..end as usize].iter().map(
1836                            |binding| {
1837                                vk::DescriptorBufferInfo::default()
1838                                    .buffer(binding.buffer.raw)
1839                                    .offset(binding.offset)
1840                                    .range(
1841                                        binding.size.map_or(vk::WHOLE_SIZE, wgt::BufferSize::get),
1842                                    )
1843                            },
1844                        ));
1845                    write.buffer_info(local_buffer_infos)
1846                }
1847                vk::DescriptorType::ACCELERATION_STRUCTURE_KHR => {
1848                    let start = entry.resource_index;
1849                    let end = start + entry.count;
1850
1851                    let local_raw_acceleration_structures;
1852                    (
1853                        raw_acceleration_structures,
1854                        local_raw_acceleration_structures,
1855                    ) = raw_acceleration_structures.extend(
1856                        desc.acceleration_structures[start as usize..end as usize]
1857                            .iter()
1858                            .map(|acceleration_structure| acceleration_structure.raw),
1859                    );
1860
1861                    let local_acceleration_structure_infos;
1862                    (
1863                        acceleration_structure_infos,
1864                        local_acceleration_structure_infos,
1865                    ) = acceleration_structure_infos.extend_one(
1866                        vk::WriteDescriptorSetAccelerationStructureKHR::default()
1867                            .acceleration_structures(local_raw_acceleration_structures),
1868                    );
1869
1870                    write
1871                        .descriptor_count(entry.count)
1872                        .push_next(local_acceleration_structure_infos)
1873                }
1874                _ => unreachable!(),
1875            };
1876
1877            writes.push(write);
1878        }
1879
1880        unsafe { self.shared.raw.update_descriptor_sets(&writes, &[]) };
1881
1882        self.counters.bind_groups.add(1);
1883
1884        Ok(super::BindGroup { set })
1885    }
1886
1887    unsafe fn destroy_bind_group(&self, group: super::BindGroup) {
1888        unsafe {
1889            self.desc_allocator
1890                .lock()
1891                .free(&*self.shared, Some(group.set))
1892        };
1893
1894        self.counters.bind_groups.sub(1);
1895    }
1896
1897    unsafe fn create_shader_module(
1898        &self,
1899        desc: &crate::ShaderModuleDescriptor,
1900        shader: crate::ShaderInput,
1901    ) -> Result<super::ShaderModule, crate::ShaderError> {
1902        let shader_module = match shader {
1903            crate::ShaderInput::Naga(naga_shader)
1904                if self
1905                    .shared
1906                    .workarounds
1907                    .contains(super::Workarounds::SEPARATE_ENTRY_POINTS)
1908                    || !naga_shader.module.overrides.is_empty() =>
1909            {
1910                super::ShaderModule::Intermediate {
1911                    naga_shader,
1912                    runtime_checks: desc.runtime_checks,
1913                }
1914            }
1915            crate::ShaderInput::Naga(naga_shader) => {
1916                let mut naga_options = self.naga_options.clone();
1917                naga_options.debug_info =
1918                    naga_shader
1919                        .debug_source
1920                        .as_ref()
1921                        .map(|d| naga::back::spv::DebugInfo {
1922                            source_code: d.source_code.as_ref(),
1923                            file_name: d.file_name.as_ref().into(),
1924                            language: naga::back::spv::SourceLanguage::WGSL,
1925                        });
1926                if !desc.runtime_checks.bounds_checks {
1927                    naga_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
1928                        index: naga::proc::BoundsCheckPolicy::Unchecked,
1929                        buffer: naga::proc::BoundsCheckPolicy::Unchecked,
1930                        image_load: naga::proc::BoundsCheckPolicy::Unchecked,
1931                        binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
1932                    };
1933                }
1934                let spv = naga::back::spv::write_vec(
1935                    &naga_shader.module,
1936                    &naga_shader.info,
1937                    &naga_options,
1938                    None,
1939                )
1940                .map_err(|e| crate::ShaderError::Compilation(format!("{e}")))?;
1941                super::ShaderModule::Raw(self.create_shader_module_impl(&spv, &desc.label)?)
1942            }
1943            crate::ShaderInput::SpirV(data) => {
1944                super::ShaderModule::Raw(self.create_shader_module_impl(data, &desc.label)?)
1945            }
1946            crate::ShaderInput::Msl { .. }
1947            | crate::ShaderInput::Dxil { .. }
1948            | crate::ShaderInput::Hlsl { .. }
1949            | crate::ShaderInput::Glsl { .. } => unreachable!(),
1950        };
1951
1952        self.counters.shader_modules.add(1);
1953
1954        Ok(shader_module)
1955    }
1956
1957    unsafe fn destroy_shader_module(&self, module: super::ShaderModule) {
1958        match module {
1959            super::ShaderModule::Raw(raw) => {
1960                unsafe { self.shared.raw.destroy_shader_module(raw, None) };
1961            }
1962            super::ShaderModule::Intermediate { .. } => {}
1963        }
1964
1965        self.counters.shader_modules.sub(1);
1966    }
1967
1968    unsafe fn create_render_pipeline(
1969        &self,
1970        desc: &crate::RenderPipelineDescriptor<
1971            super::PipelineLayout,
1972            super::ShaderModule,
1973            super::PipelineCache,
1974        >,
1975    ) -> Result<super::RenderPipeline, crate::PipelineError> {
1976        let dynamic_states = [
1977            vk::DynamicState::VIEWPORT,
1978            vk::DynamicState::SCISSOR,
1979            vk::DynamicState::BLEND_CONSTANTS,
1980            vk::DynamicState::STENCIL_REFERENCE,
1981        ];
1982        let mut compatible_rp_key = super::RenderPassKey {
1983            sample_count: desc.multisample.count,
1984            multiview: desc.multiview,
1985            ..Default::default()
1986        };
1987        let mut stages = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new();
1988        let mut vertex_buffers = Vec::new();
1989        let mut vertex_attributes = Vec::new();
1990
1991        if let crate::VertexProcessor::Standard {
1992            vertex_buffers: desc_vertex_buffers,
1993            vertex_stage: _,
1994        } = &desc.vertex_processor
1995        {
1996            vertex_buffers = Vec::with_capacity(desc_vertex_buffers.len());
1997            for (i, vb) in desc_vertex_buffers.iter().enumerate() {
1998                vertex_buffers.push(vk::VertexInputBindingDescription {
1999                    binding: i as u32,
2000                    stride: vb.array_stride as u32,
2001                    input_rate: match vb.step_mode {
2002                        wgt::VertexStepMode::Vertex => vk::VertexInputRate::VERTEX,
2003                        wgt::VertexStepMode::Instance => vk::VertexInputRate::INSTANCE,
2004                    },
2005                });
2006                for at in vb.attributes {
2007                    vertex_attributes.push(vk::VertexInputAttributeDescription {
2008                        location: at.shader_location,
2009                        binding: i as u32,
2010                        format: conv::map_vertex_format(at.format),
2011                        offset: at.offset as u32,
2012                    });
2013                }
2014            }
2015        }
2016
2017        let vk_vertex_input = vk::PipelineVertexInputStateCreateInfo::default()
2018            .vertex_binding_descriptions(&vertex_buffers)
2019            .vertex_attribute_descriptions(&vertex_attributes);
2020
2021        let vk_input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
2022            .topology(conv::map_topology(desc.primitive.topology))
2023            .primitive_restart_enable(desc.primitive.strip_index_format.is_some());
2024
2025        let mut compiled_vs = None;
2026        let mut compiled_ms = None;
2027        let mut compiled_ts = None;
2028        match &desc.vertex_processor {
2029            crate::VertexProcessor::Standard {
2030                vertex_buffers: _,
2031                vertex_stage,
2032            } => {
2033                compiled_vs = Some(self.compile_stage(
2034                    vertex_stage,
2035                    naga::ShaderStage::Vertex,
2036                    &desc.layout.binding_arrays,
2037                )?);
2038                stages.push(compiled_vs.as_ref().unwrap().create_info);
2039            }
2040            crate::VertexProcessor::Mesh {
2041                task_stage,
2042                mesh_stage,
2043            } => {
2044                if let Some(t) = task_stage.as_ref() {
2045                    compiled_ts = Some(self.compile_stage(
2046                        t,
2047                        naga::ShaderStage::Task,
2048                        &desc.layout.binding_arrays,
2049                    )?);
2050                    stages.push(compiled_ts.as_ref().unwrap().create_info);
2051                }
2052                compiled_ms = Some(self.compile_stage(
2053                    mesh_stage,
2054                    naga::ShaderStage::Mesh,
2055                    &desc.layout.binding_arrays,
2056                )?);
2057                stages.push(compiled_ms.as_ref().unwrap().create_info);
2058            }
2059        }
2060        let compiled_fs = match desc.fragment_stage {
2061            Some(ref stage) => {
2062                let compiled = self.compile_stage(
2063                    stage,
2064                    naga::ShaderStage::Fragment,
2065                    &desc.layout.binding_arrays,
2066                )?;
2067                stages.push(compiled.create_info);
2068                Some(compiled)
2069            }
2070            None => None,
2071        };
2072
2073        let mut vk_rasterization = vk::PipelineRasterizationStateCreateInfo::default()
2074            .polygon_mode(conv::map_polygon_mode(desc.primitive.polygon_mode))
2075            .front_face(conv::map_front_face(desc.primitive.front_face))
2076            .line_width(1.0)
2077            .depth_clamp_enable(desc.primitive.unclipped_depth);
2078        if let Some(face) = desc.primitive.cull_mode {
2079            vk_rasterization = vk_rasterization.cull_mode(conv::map_cull_face(face))
2080        }
2081        let mut vk_rasterization_conservative_state =
2082            vk::PipelineRasterizationConservativeStateCreateInfoEXT::default()
2083                .conservative_rasterization_mode(
2084                    vk::ConservativeRasterizationModeEXT::OVERESTIMATE,
2085                );
2086        if desc.primitive.conservative {
2087            vk_rasterization = vk_rasterization.push_next(&mut vk_rasterization_conservative_state);
2088        }
2089
2090        let mut vk_depth_stencil = vk::PipelineDepthStencilStateCreateInfo::default();
2091        if let Some(ref ds) = desc.depth_stencil {
2092            let vk_format = self.shared.private_caps.map_texture_format(ds.format);
2093            let vk_layout = if ds.is_read_only(desc.primitive.cull_mode) {
2094                vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL
2095            } else {
2096                vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL
2097            };
2098            compatible_rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
2099                base: super::AttachmentKey::compatible(vk_format, vk_layout),
2100                stencil_ops: crate::AttachmentOps::all(),
2101            });
2102
2103            if ds.is_depth_enabled() {
2104                vk_depth_stencil = vk_depth_stencil
2105                    .depth_test_enable(true)
2106                    .depth_write_enable(ds.depth_write_enabled)
2107                    .depth_compare_op(conv::map_comparison(ds.depth_compare));
2108            }
2109            if ds.stencil.is_enabled() {
2110                let s = &ds.stencil;
2111                let front = conv::map_stencil_face(&s.front, s.read_mask, s.write_mask);
2112                let back = conv::map_stencil_face(&s.back, s.read_mask, s.write_mask);
2113                vk_depth_stencil = vk_depth_stencil
2114                    .stencil_test_enable(true)
2115                    .front(front)
2116                    .back(back);
2117            }
2118
2119            if ds.bias.is_enabled() {
2120                vk_rasterization = vk_rasterization
2121                    .depth_bias_enable(true)
2122                    .depth_bias_constant_factor(ds.bias.constant as f32)
2123                    .depth_bias_clamp(ds.bias.clamp)
2124                    .depth_bias_slope_factor(ds.bias.slope_scale);
2125            }
2126        }
2127
2128        let vk_viewport = vk::PipelineViewportStateCreateInfo::default()
2129            .flags(vk::PipelineViewportStateCreateFlags::empty())
2130            .scissor_count(1)
2131            .viewport_count(1);
2132
2133        let vk_sample_mask = [
2134            desc.multisample.mask as u32,
2135            (desc.multisample.mask >> 32) as u32,
2136        ];
2137        let vk_multisample = vk::PipelineMultisampleStateCreateInfo::default()
2138            .rasterization_samples(vk::SampleCountFlags::from_raw(desc.multisample.count))
2139            .alpha_to_coverage_enable(desc.multisample.alpha_to_coverage_enabled)
2140            .sample_mask(&vk_sample_mask);
2141
2142        let mut vk_attachments = Vec::with_capacity(desc.color_targets.len());
2143        for cat in desc.color_targets {
2144            let (key, attarchment) = if let Some(cat) = cat.as_ref() {
2145                let mut vk_attachment = vk::PipelineColorBlendAttachmentState::default()
2146                    .color_write_mask(vk::ColorComponentFlags::from_raw(cat.write_mask.bits()));
2147                if let Some(ref blend) = cat.blend {
2148                    let (color_op, color_src, color_dst) = conv::map_blend_component(&blend.color);
2149                    let (alpha_op, alpha_src, alpha_dst) = conv::map_blend_component(&blend.alpha);
2150                    vk_attachment = vk_attachment
2151                        .blend_enable(true)
2152                        .color_blend_op(color_op)
2153                        .src_color_blend_factor(color_src)
2154                        .dst_color_blend_factor(color_dst)
2155                        .alpha_blend_op(alpha_op)
2156                        .src_alpha_blend_factor(alpha_src)
2157                        .dst_alpha_blend_factor(alpha_dst);
2158                }
2159
2160                let vk_format = self.shared.private_caps.map_texture_format(cat.format);
2161                (
2162                    Some(super::ColorAttachmentKey {
2163                        base: super::AttachmentKey::compatible(
2164                            vk_format,
2165                            vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
2166                        ),
2167                        resolve: None,
2168                    }),
2169                    vk_attachment,
2170                )
2171            } else {
2172                (None, vk::PipelineColorBlendAttachmentState::default())
2173            };
2174
2175            compatible_rp_key.colors.push(key);
2176            vk_attachments.push(attarchment);
2177        }
2178
2179        let vk_color_blend =
2180            vk::PipelineColorBlendStateCreateInfo::default().attachments(&vk_attachments);
2181
2182        let vk_dynamic_state =
2183            vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic_states);
2184
2185        let raw_pass = self.shared.make_render_pass(compatible_rp_key)?;
2186
2187        let vk_infos = [{
2188            vk::GraphicsPipelineCreateInfo::default()
2189                .layout(desc.layout.raw)
2190                .stages(&stages)
2191                .vertex_input_state(&vk_vertex_input)
2192                .input_assembly_state(&vk_input_assembly)
2193                .rasterization_state(&vk_rasterization)
2194                .viewport_state(&vk_viewport)
2195                .multisample_state(&vk_multisample)
2196                .depth_stencil_state(&vk_depth_stencil)
2197                .color_blend_state(&vk_color_blend)
2198                .dynamic_state(&vk_dynamic_state)
2199                .render_pass(raw_pass)
2200        }];
2201
2202        let pipeline_cache = desc
2203            .cache
2204            .map(|it| it.raw)
2205            .unwrap_or(vk::PipelineCache::null());
2206
2207        let mut raw_vec = {
2208            profiling::scope!("vkCreateGraphicsPipelines");
2209            unsafe {
2210                self.shared
2211                    .raw
2212                    .create_graphics_pipelines(pipeline_cache, &vk_infos, None)
2213                    .map_err(|(_, e)| super::map_pipeline_err(e))
2214            }?
2215        };
2216
2217        let raw = raw_vec.pop().unwrap();
2218        if let Some(label) = desc.label {
2219            unsafe { self.shared.set_object_name(raw, label) };
2220        }
2221
2222        if let Some(CompiledStage {
2223            temp_raw_module: Some(raw_module),
2224            ..
2225        }) = compiled_vs
2226        {
2227            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2228        }
2229        if let Some(CompiledStage {
2230            temp_raw_module: Some(raw_module),
2231            ..
2232        }) = compiled_ts
2233        {
2234            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2235        }
2236        if let Some(CompiledStage {
2237            temp_raw_module: Some(raw_module),
2238            ..
2239        }) = compiled_ms
2240        {
2241            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2242        }
2243        if let Some(CompiledStage {
2244            temp_raw_module: Some(raw_module),
2245            ..
2246        }) = compiled_fs
2247        {
2248            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2249        }
2250
2251        self.counters.render_pipelines.add(1);
2252
2253        Ok(super::RenderPipeline { raw })
2254    }
2255
2256    unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) {
2257        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2258
2259        self.counters.render_pipelines.sub(1);
2260    }
2261
2262    unsafe fn create_compute_pipeline(
2263        &self,
2264        desc: &crate::ComputePipelineDescriptor<
2265            super::PipelineLayout,
2266            super::ShaderModule,
2267            super::PipelineCache,
2268        >,
2269    ) -> Result<super::ComputePipeline, crate::PipelineError> {
2270        let compiled = self.compile_stage(
2271            &desc.stage,
2272            naga::ShaderStage::Compute,
2273            &desc.layout.binding_arrays,
2274        )?;
2275
2276        let vk_infos = [{
2277            vk::ComputePipelineCreateInfo::default()
2278                .layout(desc.layout.raw)
2279                .stage(compiled.create_info)
2280        }];
2281
2282        let pipeline_cache = desc
2283            .cache
2284            .map(|it| it.raw)
2285            .unwrap_or(vk::PipelineCache::null());
2286
2287        let mut raw_vec = {
2288            profiling::scope!("vkCreateComputePipelines");
2289            unsafe {
2290                self.shared
2291                    .raw
2292                    .create_compute_pipelines(pipeline_cache, &vk_infos, None)
2293                    .map_err(|(_, e)| super::map_pipeline_err(e))
2294            }?
2295        };
2296
2297        let raw = raw_vec.pop().unwrap();
2298        if let Some(label) = desc.label {
2299            unsafe { self.shared.set_object_name(raw, label) };
2300        }
2301
2302        if let Some(raw_module) = compiled.temp_raw_module {
2303            unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2304        }
2305
2306        self.counters.compute_pipelines.add(1);
2307
2308        Ok(super::ComputePipeline { raw })
2309    }
2310
2311    unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) {
2312        unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2313
2314        self.counters.compute_pipelines.sub(1);
2315    }
2316
2317    unsafe fn create_pipeline_cache(
2318        &self,
2319        desc: &crate::PipelineCacheDescriptor<'_>,
2320    ) -> Result<super::PipelineCache, crate::PipelineCacheError> {
2321        let mut info = vk::PipelineCacheCreateInfo::default();
2322        if let Some(data) = desc.data {
2323            info = info.initial_data(data)
2324        }
2325        profiling::scope!("vkCreatePipelineCache");
2326        let raw = unsafe { self.shared.raw.create_pipeline_cache(&info, None) }
2327            .map_err(super::map_host_device_oom_err)?;
2328
2329        Ok(super::PipelineCache { raw })
2330    }
2331    fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> {
2332        Some(self.shared.pipeline_cache_validation_key)
2333    }
2334    unsafe fn destroy_pipeline_cache(&self, cache: super::PipelineCache) {
2335        unsafe { self.shared.raw.destroy_pipeline_cache(cache.raw, None) }
2336    }
2337    unsafe fn create_query_set(
2338        &self,
2339        desc: &wgt::QuerySetDescriptor<crate::Label>,
2340    ) -> Result<super::QuerySet, crate::DeviceError> {
2341        // Assume each query is 256 bytes.
2342        // On an AMD W6800 with driver version 32.0.12030.9, occlusion queries are 256.
2343        self.error_if_would_oom_on_resource_allocation(true, desc.count as u64 * 256)?;
2344
2345        let (vk_type, pipeline_statistics) = match desc.ty {
2346            wgt::QueryType::Occlusion => (
2347                vk::QueryType::OCCLUSION,
2348                vk::QueryPipelineStatisticFlags::empty(),
2349            ),
2350            wgt::QueryType::PipelineStatistics(statistics) => (
2351                vk::QueryType::PIPELINE_STATISTICS,
2352                conv::map_pipeline_statistics(statistics),
2353            ),
2354            wgt::QueryType::Timestamp => (
2355                vk::QueryType::TIMESTAMP,
2356                vk::QueryPipelineStatisticFlags::empty(),
2357            ),
2358        };
2359
2360        let vk_info = vk::QueryPoolCreateInfo::default()
2361            .query_type(vk_type)
2362            .query_count(desc.count)
2363            .pipeline_statistics(pipeline_statistics);
2364
2365        let raw = unsafe { self.shared.raw.create_query_pool(&vk_info, None) }
2366            .map_err(super::map_host_device_oom_err)?;
2367        if let Some(label) = desc.label {
2368            unsafe { self.shared.set_object_name(raw, label) };
2369        }
2370
2371        self.counters.query_sets.add(1);
2372
2373        Ok(super::QuerySet { raw })
2374    }
2375
2376    unsafe fn destroy_query_set(&self, set: super::QuerySet) {
2377        unsafe { self.shared.raw.destroy_query_pool(set.raw, None) };
2378
2379        self.counters.query_sets.sub(1);
2380    }
2381
2382    unsafe fn create_fence(&self) -> Result<super::Fence, crate::DeviceError> {
2383        self.counters.fences.add(1);
2384
2385        Ok(if self.shared.private_caps.timeline_semaphores {
2386            let mut sem_type_info =
2387                vk::SemaphoreTypeCreateInfo::default().semaphore_type(vk::SemaphoreType::TIMELINE);
2388            let vk_info = vk::SemaphoreCreateInfo::default().push_next(&mut sem_type_info);
2389            let raw = unsafe { self.shared.raw.create_semaphore(&vk_info, None) }
2390                .map_err(super::map_host_device_oom_err)?;
2391
2392            super::Fence::TimelineSemaphore(raw)
2393        } else {
2394            super::Fence::FencePool {
2395                last_completed: 0,
2396                active: Vec::new(),
2397                free: Vec::new(),
2398            }
2399        })
2400    }
2401    unsafe fn destroy_fence(&self, fence: super::Fence) {
2402        match fence {
2403            super::Fence::TimelineSemaphore(raw) => {
2404                unsafe { self.shared.raw.destroy_semaphore(raw, None) };
2405            }
2406            super::Fence::FencePool {
2407                active,
2408                free,
2409                last_completed: _,
2410            } => {
2411                for (_, raw) in active {
2412                    unsafe { self.shared.raw.destroy_fence(raw, None) };
2413                }
2414                for raw in free {
2415                    unsafe { self.shared.raw.destroy_fence(raw, None) };
2416                }
2417            }
2418        }
2419
2420        self.counters.fences.sub(1);
2421    }
2422    unsafe fn get_fence_value(
2423        &self,
2424        fence: &super::Fence,
2425    ) -> Result<crate::FenceValue, crate::DeviceError> {
2426        fence.get_latest(
2427            &self.shared.raw,
2428            self.shared.extension_fns.timeline_semaphore.as_ref(),
2429        )
2430    }
2431    unsafe fn wait(
2432        &self,
2433        fence: &super::Fence,
2434        wait_value: crate::FenceValue,
2435        timeout_ms: u32,
2436    ) -> Result<bool, crate::DeviceError> {
2437        let timeout_ns = timeout_ms as u64 * super::MILLIS_TO_NANOS;
2438        self.shared.wait_for_fence(fence, wait_value, timeout_ns)
2439    }
2440
2441    unsafe fn start_graphics_debugger_capture(&self) -> bool {
2442        #[cfg(feature = "renderdoc")]
2443        {
2444            // Renderdoc requires us to give us the pointer that vkInstance _points to_.
2445            let raw_vk_instance =
2446                vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2447            let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2448            unsafe {
2449                self.render_doc
2450                    .start_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2451            }
2452        }
2453        #[cfg(not(feature = "renderdoc"))]
2454        false
2455    }
2456    unsafe fn stop_graphics_debugger_capture(&self) {
2457        #[cfg(feature = "renderdoc")]
2458        {
2459            // Renderdoc requires us to give us the pointer that vkInstance _points to_.
2460            let raw_vk_instance =
2461                vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2462            let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2463
2464            unsafe {
2465                self.render_doc
2466                    .end_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2467            }
2468        }
2469    }
2470
2471    unsafe fn pipeline_cache_get_data(&self, cache: &super::PipelineCache) -> Option<Vec<u8>> {
2472        let data = unsafe { self.raw_device().get_pipeline_cache_data(cache.raw) };
2473        data.ok()
2474    }
2475
2476    unsafe fn get_acceleration_structure_build_sizes<'a>(
2477        &self,
2478        desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>,
2479    ) -> crate::AccelerationStructureBuildSizes {
2480        const CAPACITY: usize = 8;
2481
2482        let ray_tracing_functions = self
2483            .shared
2484            .extension_fns
2485            .ray_tracing
2486            .as_ref()
2487            .expect("Feature `RAY_TRACING` not enabled");
2488
2489        let (geometries, primitive_counts) = match *desc.entries {
2490            crate::AccelerationStructureEntries::Instances(ref instances) => {
2491                let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default();
2492
2493                let geometry = vk::AccelerationStructureGeometryKHR::default()
2494                    .geometry_type(vk::GeometryTypeKHR::INSTANCES)
2495                    .geometry(vk::AccelerationStructureGeometryDataKHR {
2496                        instances: instance_data,
2497                    });
2498
2499                (
2500                    smallvec::smallvec![geometry],
2501                    smallvec::smallvec![instances.count],
2502                )
2503            }
2504            crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
2505                let mut primitive_counts =
2506                    smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2507                let mut geometries = smallvec::SmallVec::<
2508                    [vk::AccelerationStructureGeometryKHR; CAPACITY],
2509                >::with_capacity(in_geometries.len());
2510
2511                for triangles in in_geometries {
2512                    let mut triangle_data =
2513                        vk::AccelerationStructureGeometryTrianglesDataKHR::default()
2514                            .index_type(vk::IndexType::NONE_KHR)
2515                            .vertex_format(conv::map_vertex_format(triangles.vertex_format))
2516                            .max_vertex(triangles.vertex_count)
2517                            .vertex_stride(triangles.vertex_stride)
2518                            // The vulkan spec suggests we could pass a non-zero invalid address here if fetching
2519                            // the real address has significant overhead, but we pass the real one to be on the
2520                            // safe side for now.
2521                            // from https://registry.khronos.org/vulkan/specs/latest/man/html/vkGetAccelerationStructureBuildSizesKHR.html
2522                            // > The srcAccelerationStructure, dstAccelerationStructure, and mode members
2523                            // > of pBuildInfo are ignored. Any VkDeviceOrHostAddressKHR or VkDeviceOrHostAddressConstKHR
2524                            // > members of pBuildInfo are ignored by this command, except that the hostAddress
2525                            // > member of VkAccelerationStructureGeometryTrianglesDataKHR::transformData will
2526                            // > be examined to check if it is NULL.
2527                            .transform_data(vk::DeviceOrHostAddressConstKHR {
2528                                device_address: if desc
2529                                    .flags
2530                                    .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
2531                                {
2532                                    unsafe {
2533                                        ray_tracing_functions
2534                                            .buffer_device_address
2535                                            .get_buffer_device_address(
2536                                                &vk::BufferDeviceAddressInfo::default().buffer(
2537                                                    triangles
2538                                                        .transform
2539                                                        .as_ref()
2540                                                        .unwrap()
2541                                                        .buffer
2542                                                        .raw,
2543                                                ),
2544                                            )
2545                                    }
2546                                } else {
2547                                    0
2548                                },
2549                            });
2550
2551                    let pritive_count = if let Some(ref indices) = triangles.indices {
2552                        triangle_data =
2553                            triangle_data.index_type(conv::map_index_format(indices.format));
2554                        indices.count / 3
2555                    } else {
2556                        triangles.vertex_count / 3
2557                    };
2558
2559                    let geometry = vk::AccelerationStructureGeometryKHR::default()
2560                        .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
2561                        .geometry(vk::AccelerationStructureGeometryDataKHR {
2562                            triangles: triangle_data,
2563                        })
2564                        .flags(conv::map_acceleration_structure_geometry_flags(
2565                            triangles.flags,
2566                        ));
2567
2568                    geometries.push(geometry);
2569                    primitive_counts.push(pritive_count);
2570                }
2571                (geometries, primitive_counts)
2572            }
2573            crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
2574                let mut primitive_counts =
2575                    smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2576                let mut geometries = smallvec::SmallVec::<
2577                    [vk::AccelerationStructureGeometryKHR; CAPACITY],
2578                >::with_capacity(in_geometries.len());
2579                for aabb in in_geometries {
2580                    let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
2581                        .stride(aabb.stride);
2582
2583                    let geometry = vk::AccelerationStructureGeometryKHR::default()
2584                        .geometry_type(vk::GeometryTypeKHR::AABBS)
2585                        .geometry(vk::AccelerationStructureGeometryDataKHR { aabbs: aabbs_data })
2586                        .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
2587
2588                    geometries.push(geometry);
2589                    primitive_counts.push(aabb.count);
2590                }
2591                (geometries, primitive_counts)
2592            }
2593        };
2594
2595        let ty = match *desc.entries {
2596            crate::AccelerationStructureEntries::Instances(_) => {
2597                vk::AccelerationStructureTypeKHR::TOP_LEVEL
2598            }
2599            _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
2600        };
2601
2602        let geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
2603            .ty(ty)
2604            .flags(conv::map_acceleration_structure_flags(desc.flags))
2605            .geometries(&geometries);
2606
2607        let mut raw = Default::default();
2608        unsafe {
2609            ray_tracing_functions
2610                .acceleration_structure
2611                .get_acceleration_structure_build_sizes(
2612                    vk::AccelerationStructureBuildTypeKHR::DEVICE,
2613                    &geometry_info,
2614                    &primitive_counts,
2615                    &mut raw,
2616                )
2617        }
2618
2619        crate::AccelerationStructureBuildSizes {
2620            acceleration_structure_size: raw.acceleration_structure_size,
2621            update_scratch_size: raw.update_scratch_size,
2622            build_scratch_size: raw.build_scratch_size,
2623        }
2624    }
2625
2626    unsafe fn get_acceleration_structure_device_address(
2627        &self,
2628        acceleration_structure: &super::AccelerationStructure,
2629    ) -> wgt::BufferAddress {
2630        let ray_tracing_functions = self
2631            .shared
2632            .extension_fns
2633            .ray_tracing
2634            .as_ref()
2635            .expect("Feature `RAY_TRACING` not enabled");
2636
2637        unsafe {
2638            ray_tracing_functions
2639                .acceleration_structure
2640                .get_acceleration_structure_device_address(
2641                    &vk::AccelerationStructureDeviceAddressInfoKHR::default()
2642                        .acceleration_structure(acceleration_structure.raw),
2643                )
2644        }
2645    }
2646
2647    unsafe fn create_acceleration_structure(
2648        &self,
2649        desc: &crate::AccelerationStructureDescriptor,
2650    ) -> Result<super::AccelerationStructure, crate::DeviceError> {
2651        let ray_tracing_functions = self
2652            .shared
2653            .extension_fns
2654            .ray_tracing
2655            .as_ref()
2656            .expect("Feature `RAY_TRACING` not enabled");
2657
2658        let vk_buffer_info = vk::BufferCreateInfo::default()
2659            .size(desc.size)
2660            .usage(
2661                vk::BufferUsageFlags::ACCELERATION_STRUCTURE_STORAGE_KHR
2662                    | vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS,
2663            )
2664            .sharing_mode(vk::SharingMode::EXCLUSIVE);
2665
2666        unsafe {
2667            let raw_buffer = self
2668                .shared
2669                .raw
2670                .create_buffer(&vk_buffer_info, None)
2671                .map_err(super::map_host_device_oom_and_ioca_err)?;
2672            let req = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
2673
2674            self.error_if_would_oom_on_resource_allocation(false, req.size)
2675                .inspect_err(|_| {
2676                    self.shared.raw.destroy_buffer(raw_buffer, None);
2677                })?;
2678
2679            let block = self
2680                .mem_allocator
2681                .lock()
2682                .alloc(
2683                    &*self.shared,
2684                    gpu_alloc::Request {
2685                        size: req.size,
2686                        align_mask: req.alignment - 1,
2687                        usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
2688                        memory_types: req.memory_type_bits & self.valid_ash_memory_types,
2689                    },
2690                )
2691                .inspect_err(|_| {
2692                    self.shared.raw.destroy_buffer(raw_buffer, None);
2693                })?;
2694
2695            self.shared
2696                .raw
2697                .bind_buffer_memory(raw_buffer, *block.memory(), block.offset())
2698                .map_err(super::map_host_device_oom_and_ioca_err)
2699                .inspect_err(|_| {
2700                    self.shared.raw.destroy_buffer(raw_buffer, None);
2701                })?;
2702
2703            if let Some(label) = desc.label {
2704                self.shared.set_object_name(raw_buffer, label);
2705            }
2706
2707            let vk_info = vk::AccelerationStructureCreateInfoKHR::default()
2708                .buffer(raw_buffer)
2709                .offset(0)
2710                .size(desc.size)
2711                .ty(conv::map_acceleration_structure_format(desc.format));
2712
2713            let raw_acceleration_structure = ray_tracing_functions
2714                .acceleration_structure
2715                .create_acceleration_structure(&vk_info, None)
2716                .map_err(super::map_host_oom_and_ioca_err)
2717                .inspect_err(|_| {
2718                    self.shared.raw.destroy_buffer(raw_buffer, None);
2719                })?;
2720
2721            if let Some(label) = desc.label {
2722                self.shared
2723                    .set_object_name(raw_acceleration_structure, label);
2724            }
2725
2726            let pool = if desc.allow_compaction {
2727                let vk_info = vk::QueryPoolCreateInfo::default()
2728                    .query_type(vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR)
2729                    .query_count(1);
2730
2731                let raw = self
2732                    .shared
2733                    .raw
2734                    .create_query_pool(&vk_info, None)
2735                    .map_err(super::map_host_device_oom_err)
2736                    .inspect_err(|_| {
2737                        ray_tracing_functions
2738                            .acceleration_structure
2739                            .destroy_acceleration_structure(raw_acceleration_structure, None);
2740                        self.shared.raw.destroy_buffer(raw_buffer, None);
2741                    })?;
2742                Some(raw)
2743            } else {
2744                None
2745            };
2746
2747            Ok(super::AccelerationStructure {
2748                raw: raw_acceleration_structure,
2749                buffer: raw_buffer,
2750                block: Mutex::new(block),
2751                compacted_size_query: pool,
2752            })
2753        }
2754    }
2755
2756    unsafe fn destroy_acceleration_structure(
2757        &self,
2758        acceleration_structure: super::AccelerationStructure,
2759    ) {
2760        let ray_tracing_functions = self
2761            .shared
2762            .extension_fns
2763            .ray_tracing
2764            .as_ref()
2765            .expect("Feature `RAY_TRACING` not enabled");
2766
2767        unsafe {
2768            ray_tracing_functions
2769                .acceleration_structure
2770                .destroy_acceleration_structure(acceleration_structure.raw, None);
2771            self.shared
2772                .raw
2773                .destroy_buffer(acceleration_structure.buffer, None);
2774            self.mem_allocator
2775                .lock()
2776                .dealloc(&*self.shared, acceleration_structure.block.into_inner());
2777            if let Some(query) = acceleration_structure.compacted_size_query {
2778                self.shared.raw.destroy_query_pool(query, None)
2779            }
2780        }
2781    }
2782
2783    fn get_internal_counters(&self) -> wgt::HalCounters {
2784        self.counters
2785            .memory_allocations
2786            .set(self.shared.memory_allocations_counter.read());
2787
2788        self.counters.as_ref().clone()
2789    }
2790
2791    fn tlas_instance_to_bytes(&self, instance: TlasInstance) -> Vec<u8> {
2792        const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
2793        let temp = RawTlasInstance {
2794            transform: instance.transform,
2795            custom_data_and_mask: (instance.custom_data & MAX_U24)
2796                | (u32::from(instance.mask) << 24),
2797            shader_binding_table_record_offset_and_flags: 0,
2798            acceleration_structure_reference: instance.blas_address,
2799        };
2800        bytemuck::bytes_of(&temp).to_vec()
2801    }
2802
2803    fn check_if_oom(&self) -> Result<(), crate::DeviceError> {
2804        let Some(threshold) = self
2805            .shared
2806            .instance
2807            .memory_budget_thresholds
2808            .for_device_loss
2809        else {
2810            return Ok(());
2811        };
2812
2813        if !self
2814            .shared
2815            .enabled_extensions
2816            .contains(&ext::memory_budget::NAME)
2817        {
2818            return Ok(());
2819        }
2820
2821        let get_physical_device_properties = self
2822            .shared
2823            .instance
2824            .get_physical_device_properties
2825            .as_ref()
2826            .unwrap();
2827
2828        let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
2829
2830        let mut memory_properties =
2831            vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
2832
2833        unsafe {
2834            get_physical_device_properties.get_physical_device_memory_properties2(
2835                self.shared.physical_device,
2836                &mut memory_properties,
2837            );
2838        }
2839
2840        let memory_properties = memory_properties.memory_properties;
2841
2842        for i in 0..memory_properties.memory_heap_count {
2843            let heap_usage = memory_budget_properties.heap_usage[i as usize];
2844            let heap_budget = memory_budget_properties.heap_budget[i as usize];
2845
2846            if heap_usage >= heap_budget / 100 * threshold as u64 {
2847                return Err(crate::DeviceError::OutOfMemory);
2848            }
2849        }
2850
2851        Ok(())
2852    }
2853}
2854
2855impl super::DeviceShared {
2856    pub(super) fn new_binary_semaphore(
2857        &self,
2858        name: &str,
2859    ) -> Result<vk::Semaphore, crate::DeviceError> {
2860        unsafe {
2861            let semaphore = self
2862                .raw
2863                .create_semaphore(&vk::SemaphoreCreateInfo::default(), None)
2864                .map_err(super::map_host_device_oom_err)?;
2865
2866            self.set_object_name(semaphore, name);
2867
2868            Ok(semaphore)
2869        }
2870    }
2871
2872    pub(super) fn wait_for_fence(
2873        &self,
2874        fence: &super::Fence,
2875        wait_value: crate::FenceValue,
2876        timeout_ns: u64,
2877    ) -> Result<bool, crate::DeviceError> {
2878        profiling::scope!("Device::wait");
2879        match *fence {
2880            super::Fence::TimelineSemaphore(raw) => {
2881                let semaphores = [raw];
2882                let values = [wait_value];
2883                let vk_info = vk::SemaphoreWaitInfo::default()
2884                    .semaphores(&semaphores)
2885                    .values(&values);
2886                let result = match self.extension_fns.timeline_semaphore {
2887                    Some(super::ExtensionFn::Extension(ref ext)) => unsafe {
2888                        ext.wait_semaphores(&vk_info, timeout_ns)
2889                    },
2890                    Some(super::ExtensionFn::Promoted) => unsafe {
2891                        self.raw.wait_semaphores(&vk_info, timeout_ns)
2892                    },
2893                    None => unreachable!(),
2894                };
2895                match result {
2896                    Ok(()) => Ok(true),
2897                    Err(vk::Result::TIMEOUT) => Ok(false),
2898                    Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2899                }
2900            }
2901            super::Fence::FencePool {
2902                last_completed,
2903                ref active,
2904                free: _,
2905            } => {
2906                if wait_value <= last_completed {
2907                    Ok(true)
2908                } else {
2909                    match active.iter().find(|&&(value, _)| value >= wait_value) {
2910                        Some(&(_, raw)) => {
2911                            match unsafe { self.raw.wait_for_fences(&[raw], true, timeout_ns) } {
2912                                Ok(()) => Ok(true),
2913                                Err(vk::Result::TIMEOUT) => Ok(false),
2914                                Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2915                            }
2916                        }
2917                        None => {
2918                            crate::hal_usage_error(format!(
2919                                "no signals reached value {wait_value}"
2920                            ));
2921                        }
2922                    }
2923                }
2924            }
2925        }
2926    }
2927}
2928
2929impl From<gpu_alloc::AllocationError> for crate::DeviceError {
2930    fn from(error: gpu_alloc::AllocationError) -> Self {
2931        use gpu_alloc::AllocationError as Ae;
2932        match error {
2933            Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::TooManyObjects => Self::OutOfMemory,
2934            Ae::NoCompatibleMemoryTypes => crate::hal_usage_error(error),
2935        }
2936    }
2937}
2938impl From<gpu_alloc::MapError> for crate::DeviceError {
2939    fn from(error: gpu_alloc::MapError) -> Self {
2940        use gpu_alloc::MapError as Me;
2941        match error {
2942            Me::OutOfDeviceMemory | Me::OutOfHostMemory | Me::MapFailed => Self::OutOfMemory,
2943            Me::NonHostVisible | Me::AlreadyMapped => crate::hal_usage_error(error),
2944        }
2945    }
2946}
2947impl From<gpu_descriptor::AllocationError> for crate::DeviceError {
2948    fn from(error: gpu_descriptor::AllocationError) -> Self {
2949        use gpu_descriptor::AllocationError as Ae;
2950        match error {
2951            Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::Fragmentation => Self::OutOfMemory,
2952        }
2953    }
2954}
2955
2956/// We usually map unexpected vulkan errors to the [`crate::DeviceError::Unexpected`]
2957/// variant to be more robust even in cases where the driver is not
2958/// complying with the spec.
2959///
2960/// However, we implement a few Trait methods that don't have an equivalent
2961/// error variant. In those cases we use this function.
2962fn handle_unexpected(err: vk::Result) -> ! {
2963    panic!("Unexpected Vulkan error: `{err}`")
2964}
2965
2966struct ImageWithoutMemory {
2967    raw: vk::Image,
2968    requirements: vk::MemoryRequirements,
2969    copy_size: crate::CopyExtent,
2970}