1use alloc::{
2 borrow::{Cow, ToOwned as _},
3 collections::BTreeMap,
4 ffi::CString,
5 sync::Arc,
6 vec::Vec,
7};
8use core::{
9 ffi::CStr,
10 mem::{self, MaybeUninit},
11 num::NonZeroU32,
12 ptr,
13};
14
15use arrayvec::ArrayVec;
16use ash::{ext, khr, vk};
17use hashbrown::hash_map::Entry;
18use parking_lot::Mutex;
19
20use super::{conv, RawTlasInstance};
21use crate::TlasInstance;
22
23impl super::DeviceShared {
24 pub(super) unsafe fn set_object_name(&self, object: impl vk::Handle, name: &str) {
32 let Some(extension) = self.extension_fns.debug_utils.as_ref() else {
33 return;
34 };
35
36 let mut buffer: [u8; 64] = [0u8; 64];
39 let buffer_vec: Vec<u8>;
40
41 let name_bytes = if name.len() < buffer.len() {
43 buffer[..name.len()].copy_from_slice(name.as_bytes());
45 buffer[name.len()] = 0;
47 &buffer[..name.len() + 1]
48 } else {
49 buffer_vec = name
52 .as_bytes()
53 .iter()
54 .cloned()
55 .chain(core::iter::once(0))
56 .collect();
57 &buffer_vec
58 };
59
60 let name = CStr::from_bytes_until_nul(name_bytes).expect("We have added a null byte");
61
62 let _result = unsafe {
63 extension.set_debug_utils_object_name(
64 &vk::DebugUtilsObjectNameInfoEXT::default()
65 .object_handle(object)
66 .object_name(name),
67 )
68 };
69 }
70
71 pub fn make_render_pass(
72 &self,
73 key: super::RenderPassKey,
74 ) -> Result<vk::RenderPass, crate::DeviceError> {
75 Ok(match self.render_passes.lock().entry(key) {
76 Entry::Occupied(e) => *e.get(),
77 Entry::Vacant(e) => {
78 let super::RenderPassKey {
79 ref colors,
80 ref depth_stencil,
81 sample_count,
82 multiview,
83 } = *e.key();
84
85 let mut vk_attachments = Vec::new();
86 let mut color_refs = Vec::with_capacity(colors.len());
87 let mut resolve_refs = Vec::with_capacity(color_refs.capacity());
88 let mut ds_ref = None;
89 let samples = vk::SampleCountFlags::from_raw(sample_count);
90 let unused = vk::AttachmentReference {
91 attachment: vk::ATTACHMENT_UNUSED,
92 layout: vk::ImageLayout::UNDEFINED,
93 };
94 for cat in colors.iter() {
95 let (color_ref, resolve_ref) =
96 if let Some(super::ColorAttachmentKey { base, resolve }) = cat {
97 let super::AttachmentKey {
98 format,
99 layout,
100 ops,
101 } = *base;
102
103 let color_ref = vk::AttachmentReference {
104 attachment: vk_attachments.len() as u32,
105 layout,
106 };
107 vk_attachments.push({
108 let (load_op, store_op) = conv::map_attachment_ops(ops);
109 vk::AttachmentDescription::default()
110 .format(format)
111 .samples(samples)
112 .load_op(load_op)
113 .store_op(store_op)
114 .initial_layout(layout)
115 .final_layout(layout)
116 });
117 let resolve_ref = if let Some(rat) = resolve {
118 let super::AttachmentKey {
119 format,
120 layout,
121 ops,
122 } = *rat;
123
124 let (load_op, store_op) = conv::map_attachment_ops(ops);
125 let vk_attachment = vk::AttachmentDescription::default()
126 .format(format)
127 .samples(vk::SampleCountFlags::TYPE_1)
128 .load_op(load_op)
129 .store_op(store_op)
130 .initial_layout(layout)
131 .final_layout(layout);
132 vk_attachments.push(vk_attachment);
133
134 vk::AttachmentReference {
135 attachment: vk_attachments.len() as u32 - 1,
136 layout,
137 }
138 } else {
139 unused
140 };
141
142 (color_ref, resolve_ref)
143 } else {
144 (unused, unused)
145 };
146
147 color_refs.push(color_ref);
148 resolve_refs.push(resolve_ref);
149 }
150
151 if let Some(ds) = depth_stencil {
152 let super::DepthStencilAttachmentKey {
153 ref base,
154 stencil_ops,
155 } = *ds;
156
157 let super::AttachmentKey {
158 format,
159 layout,
160 ops,
161 } = *base;
162
163 ds_ref = Some(vk::AttachmentReference {
164 attachment: vk_attachments.len() as u32,
165 layout,
166 });
167 let (load_op, store_op) = conv::map_attachment_ops(ops);
168 let (stencil_load_op, stencil_store_op) = conv::map_attachment_ops(stencil_ops);
169 let vk_attachment = vk::AttachmentDescription::default()
170 .format(format)
171 .samples(samples)
172 .load_op(load_op)
173 .store_op(store_op)
174 .stencil_load_op(stencil_load_op)
175 .stencil_store_op(stencil_store_op)
176 .initial_layout(layout)
177 .final_layout(layout);
178 vk_attachments.push(vk_attachment);
179 }
180
181 let vk_subpasses = [{
182 let mut vk_subpass = vk::SubpassDescription::default()
183 .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
184 .color_attachments(&color_refs)
185 .resolve_attachments(&resolve_refs);
186
187 if self
188 .workarounds
189 .contains(super::Workarounds::EMPTY_RESOLVE_ATTACHMENT_LISTS)
190 && resolve_refs.is_empty()
191 {
192 vk_subpass.p_resolve_attachments = ptr::null();
193 }
194
195 if let Some(ref reference) = ds_ref {
196 vk_subpass = vk_subpass.depth_stencil_attachment(reference)
197 }
198 vk_subpass
199 }];
200
201 let mut vk_info = vk::RenderPassCreateInfo::default()
202 .attachments(&vk_attachments)
203 .subpasses(&vk_subpasses);
204
205 let mut multiview_info;
206 let mask;
207 if let Some(multiview) = multiview {
208 assert!(multiview.get() <= 8);
210 assert!(multiview.get() > 1);
211
212 mask = [(1 << multiview.get()) - 1];
216
217 multiview_info = vk::RenderPassMultiviewCreateInfoKHR::default()
219 .view_masks(&mask)
220 .correlation_masks(&mask);
221 vk_info = vk_info.push_next(&mut multiview_info);
222 }
223
224 let raw = unsafe {
225 self.raw
226 .create_render_pass(&vk_info, None)
227 .map_err(super::map_host_device_oom_err)?
228 };
229
230 *e.insert(raw)
231 }
232 })
233 }
234
235 fn make_memory_ranges<'a, I: 'a + Iterator<Item = crate::MemoryRange>>(
236 &self,
237 buffer: &'a super::Buffer,
238 ranges: I,
239 ) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
240 let block = buffer.block.as_ref()?.lock();
241 let mask = self.private_caps.non_coherent_map_mask;
242 Some(ranges.map(move |range| {
243 vk::MappedMemoryRange::default()
244 .memory(*block.memory())
245 .offset((block.offset() + range.start) & !mask)
246 .size((range.end - range.start + mask) & !mask)
247 }))
248 }
249}
250
251impl gpu_alloc::MemoryDevice<vk::DeviceMemory> for super::DeviceShared {
252 unsafe fn allocate_memory(
253 &self,
254 size: u64,
255 memory_type: u32,
256 flags: gpu_alloc::AllocationFlags,
257 ) -> Result<vk::DeviceMemory, gpu_alloc::OutOfMemory> {
258 let mut info = vk::MemoryAllocateInfo::default()
259 .allocation_size(size)
260 .memory_type_index(memory_type);
261
262 let mut info_flags;
263
264 if flags.contains(gpu_alloc::AllocationFlags::DEVICE_ADDRESS) {
265 info_flags = vk::MemoryAllocateFlagsInfo::default()
266 .flags(vk::MemoryAllocateFlags::DEVICE_ADDRESS);
267 info = info.push_next(&mut info_flags);
268 }
269
270 match unsafe { self.raw.allocate_memory(&info, None) } {
271 Ok(memory) => {
272 self.memory_allocations_counter.add(1);
273 Ok(memory)
274 }
275 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
276 Err(gpu_alloc::OutOfMemory::OutOfDeviceMemory)
277 }
278 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
279 Err(gpu_alloc::OutOfMemory::OutOfHostMemory)
280 }
281 Err(err) => handle_unexpected(err),
286 }
287 }
288
289 unsafe fn deallocate_memory(&self, memory: vk::DeviceMemory) {
290 self.memory_allocations_counter.sub(1);
291
292 unsafe { self.raw.free_memory(memory, None) };
293 }
294
295 unsafe fn map_memory(
296 &self,
297 memory: &mut vk::DeviceMemory,
298 offset: u64,
299 size: u64,
300 ) -> Result<ptr::NonNull<u8>, gpu_alloc::DeviceMapError> {
301 match unsafe {
302 self.raw
303 .map_memory(*memory, offset, size, vk::MemoryMapFlags::empty())
304 } {
305 Ok(ptr) => Ok(ptr::NonNull::new(ptr.cast::<u8>())
306 .expect("Pointer to memory mapping must not be null")),
307 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
308 Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory)
309 }
310 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
311 Err(gpu_alloc::DeviceMapError::OutOfHostMemory)
312 }
313 Err(vk::Result::ERROR_MEMORY_MAP_FAILED) => Err(gpu_alloc::DeviceMapError::MapFailed),
314 Err(err) => handle_unexpected(err),
315 }
316 }
317
318 unsafe fn unmap_memory(&self, memory: &mut vk::DeviceMemory) {
319 unsafe { self.raw.unmap_memory(*memory) };
320 }
321
322 unsafe fn invalidate_memory_ranges(
323 &self,
324 _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
325 ) -> Result<(), gpu_alloc::OutOfMemory> {
326 unimplemented!()
328 }
329
330 unsafe fn flush_memory_ranges(
331 &self,
332 _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
333 ) -> Result<(), gpu_alloc::OutOfMemory> {
334 unimplemented!()
336 }
337}
338
339impl
340 gpu_descriptor::DescriptorDevice<vk::DescriptorSetLayout, vk::DescriptorPool, vk::DescriptorSet>
341 for super::DeviceShared
342{
343 unsafe fn create_descriptor_pool(
344 &self,
345 descriptor_count: &gpu_descriptor::DescriptorTotalCount,
346 max_sets: u32,
347 flags: gpu_descriptor::DescriptorPoolCreateFlags,
348 ) -> Result<vk::DescriptorPool, gpu_descriptor::CreatePoolError> {
349 let unfiltered_counts = [
351 (vk::DescriptorType::SAMPLER, descriptor_count.sampler),
352 (
353 vk::DescriptorType::SAMPLED_IMAGE,
354 descriptor_count.sampled_image,
355 ),
356 (
357 vk::DescriptorType::STORAGE_IMAGE,
358 descriptor_count.storage_image,
359 ),
360 (
361 vk::DescriptorType::UNIFORM_BUFFER,
362 descriptor_count.uniform_buffer,
363 ),
364 (
365 vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC,
366 descriptor_count.uniform_buffer_dynamic,
367 ),
368 (
369 vk::DescriptorType::STORAGE_BUFFER,
370 descriptor_count.storage_buffer,
371 ),
372 (
373 vk::DescriptorType::STORAGE_BUFFER_DYNAMIC,
374 descriptor_count.storage_buffer_dynamic,
375 ),
376 (
377 vk::DescriptorType::ACCELERATION_STRUCTURE_KHR,
378 descriptor_count.acceleration_structure,
379 ),
380 ];
381
382 let filtered_counts = unfiltered_counts
383 .iter()
384 .cloned()
385 .filter(|&(_, count)| count != 0)
386 .map(|(ty, count)| vk::DescriptorPoolSize {
387 ty,
388 descriptor_count: count,
389 })
390 .collect::<ArrayVec<_, 8>>();
391
392 let mut vk_flags =
393 if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND) {
394 vk::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND
395 } else {
396 vk::DescriptorPoolCreateFlags::empty()
397 };
398 if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET) {
399 vk_flags |= vk::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET;
400 }
401 let vk_info = vk::DescriptorPoolCreateInfo::default()
402 .max_sets(max_sets)
403 .flags(vk_flags)
404 .pool_sizes(&filtered_counts);
405
406 match unsafe { self.raw.create_descriptor_pool(&vk_info, None) } {
407 Ok(pool) => Ok(pool),
408 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
409 Err(gpu_descriptor::CreatePoolError::OutOfHostMemory)
410 }
411 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
412 Err(gpu_descriptor::CreatePoolError::OutOfDeviceMemory)
413 }
414 Err(vk::Result::ERROR_FRAGMENTATION) => {
415 Err(gpu_descriptor::CreatePoolError::Fragmentation)
416 }
417 Err(err) => handle_unexpected(err),
418 }
419 }
420
421 unsafe fn destroy_descriptor_pool(&self, pool: vk::DescriptorPool) {
422 unsafe { self.raw.destroy_descriptor_pool(pool, None) }
423 }
424
425 unsafe fn alloc_descriptor_sets<'a>(
426 &self,
427 pool: &mut vk::DescriptorPool,
428 layouts: impl ExactSizeIterator<Item = &'a vk::DescriptorSetLayout>,
429 sets: &mut impl Extend<vk::DescriptorSet>,
430 ) -> Result<(), gpu_descriptor::DeviceAllocationError> {
431 let result = unsafe {
432 self.raw.allocate_descriptor_sets(
433 &vk::DescriptorSetAllocateInfo::default()
434 .descriptor_pool(*pool)
435 .set_layouts(
436 &smallvec::SmallVec::<[vk::DescriptorSetLayout; 32]>::from_iter(
437 layouts.cloned(),
438 ),
439 ),
440 )
441 };
442
443 match result {
444 Ok(vk_sets) => {
445 sets.extend(vk_sets);
446 Ok(())
447 }
448 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY)
449 | Err(vk::Result::ERROR_OUT_OF_POOL_MEMORY) => {
450 Err(gpu_descriptor::DeviceAllocationError::OutOfHostMemory)
451 }
452 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
453 Err(gpu_descriptor::DeviceAllocationError::OutOfDeviceMemory)
454 }
455 Err(vk::Result::ERROR_FRAGMENTED_POOL) => {
456 Err(gpu_descriptor::DeviceAllocationError::FragmentedPool)
457 }
458 Err(err) => handle_unexpected(err),
459 }
460 }
461
462 unsafe fn dealloc_descriptor_sets<'a>(
463 &self,
464 pool: &mut vk::DescriptorPool,
465 sets: impl Iterator<Item = vk::DescriptorSet>,
466 ) {
467 let result = unsafe {
468 self.raw.free_descriptor_sets(
469 *pool,
470 &smallvec::SmallVec::<[vk::DescriptorSet; 32]>::from_iter(sets),
471 )
472 };
473 match result {
474 Ok(()) => {}
475 Err(err) => handle_unexpected(err),
476 }
477 }
478}
479
480struct CompiledStage {
481 create_info: vk::PipelineShaderStageCreateInfo<'static>,
482 _entry_point: CString,
483 temp_raw_module: Option<vk::ShaderModule>,
484}
485
486impl super::Device {
487 pub(super) unsafe fn create_swapchain(
488 &self,
489 surface: &super::Surface,
490 config: &crate::SurfaceConfiguration,
491 provided_old_swapchain: Option<super::Swapchain>,
492 ) -> Result<super::Swapchain, crate::SurfaceError> {
493 profiling::scope!("Device::create_swapchain");
494 let functor = khr::swapchain::Device::new(&surface.instance.raw, &self.shared.raw);
495
496 let old_swapchain = match provided_old_swapchain {
497 Some(osc) => osc.raw,
498 None => vk::SwapchainKHR::null(),
499 };
500
501 let color_space = if config.format == wgt::TextureFormat::Rgba16Float {
502 vk::ColorSpaceKHR::EXTENDED_SRGB_LINEAR_EXT
505 } else {
506 vk::ColorSpaceKHR::SRGB_NONLINEAR
507 };
508
509 let original_format = self.shared.private_caps.map_texture_format(config.format);
510 let mut raw_flags = vk::SwapchainCreateFlagsKHR::empty();
511 let mut raw_view_formats: Vec<vk::Format> = vec![];
512 if !config.view_formats.is_empty() {
513 raw_flags |= vk::SwapchainCreateFlagsKHR::MUTABLE_FORMAT;
514 raw_view_formats = config
515 .view_formats
516 .iter()
517 .map(|f| self.shared.private_caps.map_texture_format(*f))
518 .collect();
519 raw_view_formats.push(original_format);
520 }
521
522 let mut info = vk::SwapchainCreateInfoKHR::default()
523 .flags(raw_flags)
524 .surface(surface.raw)
525 .min_image_count(config.maximum_frame_latency + 1) .image_format(original_format)
527 .image_color_space(color_space)
528 .image_extent(vk::Extent2D {
529 width: config.extent.width,
530 height: config.extent.height,
531 })
532 .image_array_layers(config.extent.depth_or_array_layers)
533 .image_usage(conv::map_texture_usage(config.usage))
534 .image_sharing_mode(vk::SharingMode::EXCLUSIVE)
535 .pre_transform(vk::SurfaceTransformFlagsKHR::IDENTITY)
536 .composite_alpha(conv::map_composite_alpha_mode(config.composite_alpha_mode))
537 .present_mode(conv::map_present_mode(config.present_mode))
538 .clipped(true)
539 .old_swapchain(old_swapchain);
540
541 let mut format_list_info = vk::ImageFormatListCreateInfo::default();
542 if !raw_view_formats.is_empty() {
543 format_list_info = format_list_info.view_formats(&raw_view_formats);
544 info = info.push_next(&mut format_list_info);
545 }
546
547 let result = {
548 profiling::scope!("vkCreateSwapchainKHR");
549 unsafe { functor.create_swapchain(&info, None) }
550 };
551
552 if old_swapchain != vk::SwapchainKHR::null() {
554 unsafe { functor.destroy_swapchain(old_swapchain, None) }
555 }
556
557 let raw = match result {
558 Ok(swapchain) => swapchain,
559 Err(error) => {
560 return Err(match error {
561 vk::Result::ERROR_SURFACE_LOST_KHR
562 | vk::Result::ERROR_INITIALIZATION_FAILED => crate::SurfaceError::Lost,
563 vk::Result::ERROR_NATIVE_WINDOW_IN_USE_KHR => {
564 crate::SurfaceError::Other("Native window is in use")
565 }
566 other => super::map_host_device_oom_and_lost_err(other).into(),
569 });
570 }
571 };
572
573 let images =
574 unsafe { functor.get_swapchain_images(raw) }.map_err(super::map_host_device_oom_err)?;
575
576 let acquire_semaphores = (0..=images.len())
579 .map(|i| {
580 super::SwapchainAcquireSemaphore::new(&self.shared, i)
581 .map(Mutex::new)
582 .map(Arc::new)
583 })
584 .collect::<Result<Vec<_>, _>>()?;
585
586 let present_semaphores = (0..=images.len())
587 .map(|i| Arc::new(Mutex::new(super::SwapchainPresentSemaphores::new(i))))
588 .collect::<Vec<_>>();
589
590 Ok(super::Swapchain {
591 raw,
592 functor,
593 device: Arc::clone(&self.shared),
594 images,
595 config: config.clone(),
596 acquire_semaphores,
597 next_acquire_index: 0,
598 present_semaphores,
599 next_present_time: None,
600 })
601 }
602
603 pub unsafe fn texture_from_raw(
610 &self,
611 vk_image: vk::Image,
612 desc: &crate::TextureDescriptor,
613 drop_callback: Option<crate::DropCallback>,
614 ) -> super::Texture {
615 let mut raw_flags = vk::ImageCreateFlags::empty();
616 let mut view_formats = vec![];
617 for tf in desc.view_formats.iter() {
618 if *tf == desc.format {
619 continue;
620 }
621 view_formats.push(*tf);
622 }
623 if !view_formats.is_empty() {
624 raw_flags |=
625 vk::ImageCreateFlags::MUTABLE_FORMAT | vk::ImageCreateFlags::EXTENDED_USAGE;
626 view_formats.push(desc.format)
627 }
628 if desc.format.is_multi_planar_format() {
629 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
630 }
631
632 let identity = self.shared.texture_identity_factory.next();
633
634 let drop_guard = crate::DropGuard::from_option(drop_callback);
635
636 super::Texture {
637 raw: vk_image,
638 drop_guard,
639 external_memory: None,
640 block: None,
641 format: desc.format,
642 copy_size: desc.copy_extent(),
643 identity,
644 }
645 }
646
647 #[cfg(windows)]
648 fn find_memory_type_index(
649 &self,
650 type_bits_req: u32,
651 flags_req: vk::MemoryPropertyFlags,
652 ) -> Option<usize> {
653 let mem_properties = unsafe {
654 self.shared
655 .instance
656 .raw
657 .get_physical_device_memory_properties(self.shared.physical_device)
658 };
659
660 for (i, mem_ty) in mem_properties.memory_types_as_slice().iter().enumerate() {
662 let types_bits = 1 << i;
663 let is_required_memory_type = type_bits_req & types_bits != 0;
664 let has_required_properties = mem_ty.property_flags & flags_req == flags_req;
665 if is_required_memory_type && has_required_properties {
666 return Some(i);
667 }
668 }
669
670 None
671 }
672
673 fn create_image_without_memory(
674 &self,
675 desc: &crate::TextureDescriptor,
676 external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
677 ) -> Result<ImageWithoutMemory, crate::DeviceError> {
678 let copy_size = desc.copy_extent();
679
680 let mut raw_flags = vk::ImageCreateFlags::empty();
681 if desc.dimension == wgt::TextureDimension::D3
682 && desc.usage.contains(wgt::TextureUses::COLOR_TARGET)
683 {
684 raw_flags |= vk::ImageCreateFlags::TYPE_2D_ARRAY_COMPATIBLE;
685 }
686 if desc.is_cube_compatible() {
687 raw_flags |= vk::ImageCreateFlags::CUBE_COMPATIBLE;
688 }
689
690 let original_format = self.shared.private_caps.map_texture_format(desc.format);
691 let mut vk_view_formats = vec![];
692 if !desc.view_formats.is_empty() {
693 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
694
695 if self.shared.private_caps.image_format_list {
696 vk_view_formats = desc
697 .view_formats
698 .iter()
699 .map(|f| self.shared.private_caps.map_texture_format(*f))
700 .collect();
701 vk_view_formats.push(original_format)
702 }
703 }
704 if desc.format.is_multi_planar_format() {
705 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
706 }
707
708 let mut vk_info = vk::ImageCreateInfo::default()
709 .flags(raw_flags)
710 .image_type(conv::map_texture_dimension(desc.dimension))
711 .format(original_format)
712 .extent(conv::map_copy_extent(©_size))
713 .mip_levels(desc.mip_level_count)
714 .array_layers(desc.array_layer_count())
715 .samples(vk::SampleCountFlags::from_raw(desc.sample_count))
716 .tiling(vk::ImageTiling::OPTIMAL)
717 .usage(conv::map_texture_usage(desc.usage))
718 .sharing_mode(vk::SharingMode::EXCLUSIVE)
719 .initial_layout(vk::ImageLayout::UNDEFINED);
720
721 let mut format_list_info = vk::ImageFormatListCreateInfo::default();
722 if !vk_view_formats.is_empty() {
723 format_list_info = format_list_info.view_formats(&vk_view_formats);
724 vk_info = vk_info.push_next(&mut format_list_info);
725 }
726
727 if let Some(ext_info) = external_memory_image_create_info {
728 vk_info = vk_info.push_next(ext_info);
729 }
730
731 let raw = unsafe { self.shared.raw.create_image(&vk_info, None) }.map_err(map_err)?;
732 fn map_err(err: vk::Result) -> crate::DeviceError {
733 super::map_host_device_oom_and_ioca_err(err)
736 }
737 let req = unsafe { self.shared.raw.get_image_memory_requirements(raw) };
738
739 Ok(ImageWithoutMemory {
740 raw,
741 requirements: req,
742 copy_size,
743 })
744 }
745
746 #[cfg(windows)]
752 pub unsafe fn texture_from_d3d11_shared_handle(
753 &self,
754 d3d11_shared_handle: windows::Win32::Foundation::HANDLE,
755 desc: &crate::TextureDescriptor,
756 ) -> Result<super::Texture, crate::DeviceError> {
757 if !self
758 .shared
759 .features
760 .contains(wgt::Features::VULKAN_EXTERNAL_MEMORY_WIN32)
761 {
762 log::error!("Vulkan driver does not support VK_KHR_external_memory_win32");
763 return Err(crate::DeviceError::Unexpected);
764 }
765
766 let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
767 .handle_types(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE);
768
769 let image =
770 self.create_image_without_memory(desc, Some(&mut external_memory_image_info))?;
771
772 let mut dedicated_allocate_info =
775 vk::MemoryDedicatedAllocateInfo::default().image(image.raw);
776
777 let mut import_memory_info = vk::ImportMemoryWin32HandleInfoKHR::default()
778 .handle_type(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE)
779 .handle(d3d11_shared_handle.0 as _);
780 #[allow(clippy::unnecessary_mut_passed)]
782 {
783 import_memory_info.p_next = <*const _>::cast(&mut dedicated_allocate_info);
784 }
785
786 let mem_type_index = self
787 .find_memory_type_index(
788 image.requirements.memory_type_bits,
789 vk::MemoryPropertyFlags::DEVICE_LOCAL,
790 )
791 .ok_or(crate::DeviceError::Unexpected)?;
792
793 let memory_allocate_info = vk::MemoryAllocateInfo::default()
794 .allocation_size(image.requirements.size)
795 .memory_type_index(mem_type_index as _)
796 .push_next(&mut import_memory_info);
797 let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
798 .map_err(super::map_host_device_oom_err)?;
799
800 unsafe { self.shared.raw.bind_image_memory(image.raw, memory, 0) }
801 .map_err(super::map_host_device_oom_err)?;
802
803 if let Some(label) = desc.label {
804 unsafe { self.shared.set_object_name(image.raw, label) };
805 }
806
807 let identity = self.shared.texture_identity_factory.next();
808
809 self.counters.textures.add(1);
810
811 Ok(super::Texture {
812 raw: image.raw,
813 drop_guard: None,
814 external_memory: Some(memory),
815 block: None,
816 format: desc.format,
817 copy_size: image.copy_size,
818 identity,
819 })
820 }
821
822 fn create_shader_module_impl(
823 &self,
824 spv: &[u32],
825 ) -> Result<vk::ShaderModule, crate::DeviceError> {
826 let vk_info = vk::ShaderModuleCreateInfo::default()
827 .flags(vk::ShaderModuleCreateFlags::empty())
828 .code(spv);
829
830 let raw = unsafe {
831 profiling::scope!("vkCreateShaderModule");
832 self.shared
833 .raw
834 .create_shader_module(&vk_info, None)
835 .map_err(map_err)?
836 };
837 fn map_err(err: vk::Result) -> crate::DeviceError {
838 super::map_host_device_oom_err(err)
841 }
842 Ok(raw)
843 }
844
845 fn compile_stage(
846 &self,
847 stage: &crate::ProgrammableStage<super::ShaderModule>,
848 naga_stage: naga::ShaderStage,
849 binding_map: &naga::back::spv::BindingMap,
850 ) -> Result<CompiledStage, crate::PipelineError> {
851 let stage_flags = crate::auxil::map_naga_stage(naga_stage);
852 let vk_module = match *stage.module {
853 super::ShaderModule::Raw(raw) => raw,
854 super::ShaderModule::Intermediate {
855 ref naga_shader,
856 runtime_checks,
857 } => {
858 let pipeline_options = naga::back::spv::PipelineOptions {
859 entry_point: stage.entry_point.to_owned(),
860 shader_stage: naga_stage,
861 };
862 let needs_temp_options = !runtime_checks.bounds_checks
863 || !runtime_checks.force_loop_bounding
864 || !binding_map.is_empty()
865 || naga_shader.debug_source.is_some()
866 || !stage.zero_initialize_workgroup_memory;
867 let mut temp_options;
868 let options = if needs_temp_options {
869 temp_options = self.naga_options.clone();
870 if !runtime_checks.bounds_checks {
871 temp_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
872 index: naga::proc::BoundsCheckPolicy::Unchecked,
873 buffer: naga::proc::BoundsCheckPolicy::Unchecked,
874 image_load: naga::proc::BoundsCheckPolicy::Unchecked,
875 binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
876 };
877 }
878 if !runtime_checks.force_loop_bounding {
879 temp_options.force_loop_bounding = false;
880 }
881 if !binding_map.is_empty() {
882 temp_options.binding_map = binding_map.clone();
883 }
884
885 if let Some(ref debug) = naga_shader.debug_source {
886 temp_options.debug_info = Some(naga::back::spv::DebugInfo {
887 source_code: &debug.source_code,
888 file_name: debug.file_name.as_ref().into(),
889 language: naga::back::spv::SourceLanguage::WGSL,
890 })
891 }
892 if !stage.zero_initialize_workgroup_memory {
893 temp_options.zero_initialize_workgroup_memory =
894 naga::back::spv::ZeroInitializeWorkgroupMemoryMode::None;
895 }
896
897 &temp_options
898 } else {
899 &self.naga_options
900 };
901
902 let (module, info) = naga::back::pipeline_constants::process_overrides(
903 &naga_shader.module,
904 &naga_shader.info,
905 Some((naga_stage, stage.entry_point)),
906 stage.constants,
907 )
908 .map_err(|e| {
909 crate::PipelineError::PipelineConstants(stage_flags, format!("{e}"))
910 })?;
911
912 let spv = {
913 profiling::scope!("naga::spv::write_vec");
914 naga::back::spv::write_vec(&module, &info, options, Some(&pipeline_options))
915 }
916 .map_err(|e| crate::PipelineError::Linkage(stage_flags, format!("{e}")))?;
917 self.create_shader_module_impl(&spv)?
918 }
919 };
920
921 let mut flags = vk::PipelineShaderStageCreateFlags::empty();
922 if self.shared.features.contains(wgt::Features::SUBGROUP) {
923 flags |= vk::PipelineShaderStageCreateFlags::ALLOW_VARYING_SUBGROUP_SIZE
924 }
925
926 let entry_point = CString::new(stage.entry_point).unwrap();
927 let mut create_info = vk::PipelineShaderStageCreateInfo::default()
928 .flags(flags)
929 .stage(conv::map_shader_stage(stage_flags))
930 .module(vk_module);
931
932 create_info.p_name = entry_point.as_ptr();
934
935 Ok(CompiledStage {
936 create_info,
937 _entry_point: entry_point,
938 temp_raw_module: match *stage.module {
939 super::ShaderModule::Raw(_) => None,
940 super::ShaderModule::Intermediate { .. } => Some(vk_module),
941 },
942 })
943 }
944
945 pub fn queue_family_index(&self) -> u32 {
951 self.shared.family_index
952 }
953
954 pub fn queue_index(&self) -> u32 {
955 self.shared.queue_index
956 }
957
958 pub fn raw_device(&self) -> &ash::Device {
959 &self.shared.raw
960 }
961
962 pub fn raw_physical_device(&self) -> vk::PhysicalDevice {
963 self.shared.physical_device
964 }
965
966 pub fn raw_queue(&self) -> vk::Queue {
967 self.shared.raw_queue
968 }
969
970 pub fn enabled_device_extensions(&self) -> &[&'static CStr] {
971 &self.shared.enabled_extensions
972 }
973
974 pub fn shared_instance(&self) -> &super::InstanceShared {
975 &self.shared.instance
976 }
977
978 fn error_if_would_oom_on_resource_allocation(
979 &self,
980 needs_host_access: bool,
981 size: u64,
982 ) -> Result<(), crate::DeviceError> {
983 let Some(threshold) = self
984 .shared
985 .instance
986 .memory_budget_thresholds
987 .for_resource_creation
988 else {
989 return Ok(());
990 };
991
992 if !self
993 .shared
994 .enabled_extensions
995 .contains(&ext::memory_budget::NAME)
996 {
997 return Ok(());
998 }
999
1000 let get_physical_device_properties = self
1001 .shared
1002 .instance
1003 .get_physical_device_properties
1004 .as_ref()
1005 .unwrap();
1006
1007 let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
1008
1009 let mut memory_properties =
1010 vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
1011
1012 unsafe {
1013 get_physical_device_properties.get_physical_device_memory_properties2(
1014 self.shared.physical_device,
1015 &mut memory_properties,
1016 );
1017 }
1018
1019 let mut host_visible_heaps = [false; vk::MAX_MEMORY_HEAPS];
1020 let mut device_local_heaps = [false; vk::MAX_MEMORY_HEAPS];
1021
1022 let memory_properties = memory_properties.memory_properties;
1023
1024 for i in 0..memory_properties.memory_type_count {
1025 let memory_type = memory_properties.memory_types[i as usize];
1026 let flags = memory_type.property_flags;
1027
1028 if flags.intersects(
1029 vk::MemoryPropertyFlags::LAZILY_ALLOCATED | vk::MemoryPropertyFlags::PROTECTED,
1030 ) {
1031 continue; }
1033
1034 if flags.contains(vk::MemoryPropertyFlags::HOST_VISIBLE) {
1035 host_visible_heaps[memory_type.heap_index as usize] = true;
1036 }
1037
1038 if flags.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL) {
1039 device_local_heaps[memory_type.heap_index as usize] = true;
1040 }
1041 }
1042
1043 let heaps = if needs_host_access {
1044 host_visible_heaps
1045 } else {
1046 device_local_heaps
1047 };
1048
1049 for (i, check) in heaps.iter().enumerate() {
1054 if !check {
1055 continue;
1056 }
1057
1058 let heap_usage = memory_budget_properties.heap_usage[i];
1059 let heap_budget = memory_budget_properties.heap_budget[i];
1060
1061 if heap_usage + size >= heap_budget / 100 * threshold as u64 {
1062 return Err(crate::DeviceError::OutOfMemory);
1063 }
1064 }
1065
1066 Ok(())
1067 }
1068}
1069
1070impl crate::Device for super::Device {
1071 type A = super::Api;
1072
1073 unsafe fn create_buffer(
1074 &self,
1075 desc: &crate::BufferDescriptor,
1076 ) -> Result<super::Buffer, crate::DeviceError> {
1077 let vk_info = vk::BufferCreateInfo::default()
1078 .size(desc.size)
1079 .usage(conv::map_buffer_usage(desc.usage))
1080 .sharing_mode(vk::SharingMode::EXCLUSIVE);
1081
1082 let raw = unsafe {
1083 self.shared
1084 .raw
1085 .create_buffer(&vk_info, None)
1086 .map_err(super::map_host_device_oom_and_ioca_err)?
1087 };
1088 let req = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
1089
1090 let mut alloc_usage = if desc
1091 .usage
1092 .intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE)
1093 {
1094 let mut flags = gpu_alloc::UsageFlags::HOST_ACCESS;
1095 flags.set(
1097 gpu_alloc::UsageFlags::DOWNLOAD,
1098 desc.usage.contains(wgt::BufferUses::MAP_READ),
1099 );
1100 flags.set(
1101 gpu_alloc::UsageFlags::UPLOAD,
1102 desc.usage.contains(wgt::BufferUses::MAP_WRITE),
1103 );
1104 flags
1105 } else {
1106 gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS
1107 };
1108 alloc_usage.set(
1109 gpu_alloc::UsageFlags::TRANSIENT,
1110 desc.memory_flags.contains(crate::MemoryFlags::TRANSIENT),
1111 );
1112
1113 let needs_host_access = alloc_usage.contains(gpu_alloc::UsageFlags::HOST_ACCESS);
1114
1115 self.error_if_would_oom_on_resource_allocation(needs_host_access, req.size)
1116 .inspect_err(|_| {
1117 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1118 })?;
1119
1120 let alignment_mask = req.alignment - 1;
1121
1122 let block = unsafe {
1123 self.mem_allocator.lock().alloc(
1124 &*self.shared,
1125 gpu_alloc::Request {
1126 size: req.size,
1127 align_mask: alignment_mask,
1128 usage: alloc_usage,
1129 memory_types: req.memory_type_bits & self.valid_ash_memory_types,
1130 },
1131 )
1132 }
1133 .inspect_err(|_| {
1134 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1135 })?;
1136
1137 unsafe {
1138 self.shared
1139 .raw
1140 .bind_buffer_memory(raw, *block.memory(), block.offset())
1141 }
1142 .map_err(super::map_host_device_oom_and_ioca_err)
1143 .inspect_err(|_| {
1144 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1145 })?;
1146
1147 if let Some(label) = desc.label {
1148 unsafe { self.shared.set_object_name(raw, label) };
1149 }
1150
1151 self.counters.buffer_memory.add(block.size() as isize);
1152 self.counters.buffers.add(1);
1153
1154 Ok(super::Buffer {
1155 raw,
1156 block: Some(Mutex::new(super::BufferMemoryBacking::Managed(block))),
1157 })
1158 }
1159 unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
1160 unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1161 if let Some(block) = buffer.block {
1162 let block = block.into_inner();
1163 self.counters.buffer_memory.sub(block.size() as isize);
1164 match block {
1165 super::BufferMemoryBacking::Managed(block) => unsafe {
1166 self.mem_allocator.lock().dealloc(&*self.shared, block)
1167 },
1168 super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
1169 self.shared.raw.free_memory(memory, None);
1170 },
1171 }
1172 }
1173
1174 self.counters.buffers.sub(1);
1175 }
1176
1177 unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) {
1178 self.counters.buffers.add(1);
1179 }
1180
1181 unsafe fn map_buffer(
1182 &self,
1183 buffer: &super::Buffer,
1184 range: crate::MemoryRange,
1185 ) -> Result<crate::BufferMapping, crate::DeviceError> {
1186 if let Some(ref block) = buffer.block {
1187 let size = range.end - range.start;
1188 let mut block = block.lock();
1189 if let super::BufferMemoryBacking::Managed(ref mut block) = *block {
1190 let ptr = unsafe { block.map(&*self.shared, range.start, size as usize)? };
1191 let is_coherent = block
1192 .props()
1193 .contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT);
1194 Ok(crate::BufferMapping { ptr, is_coherent })
1195 } else {
1196 crate::hal_usage_error("tried to map externally created buffer")
1197 }
1198 } else {
1199 crate::hal_usage_error("tried to map external buffer")
1200 }
1201 }
1202 unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
1203 if let Some(ref block) = buffer.block {
1204 match &mut *block.lock() {
1205 super::BufferMemoryBacking::Managed(block) => unsafe { block.unmap(&*self.shared) },
1206 super::BufferMemoryBacking::VulkanMemory { .. } => {
1207 crate::hal_usage_error("tried to unmap externally created buffer")
1208 }
1209 };
1210 } else {
1211 crate::hal_usage_error("tried to unmap external buffer")
1212 }
1213 }
1214
1215 unsafe fn flush_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1216 where
1217 I: Iterator<Item = crate::MemoryRange>,
1218 {
1219 if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1220 unsafe {
1221 self.shared
1222 .raw
1223 .flush_mapped_memory_ranges(
1224 &smallvec::SmallVec::<[vk::MappedMemoryRange; 32]>::from_iter(vk_ranges),
1225 )
1226 }
1227 .unwrap();
1228 }
1229 }
1230 unsafe fn invalidate_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1231 where
1232 I: Iterator<Item = crate::MemoryRange>,
1233 {
1234 if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1235 unsafe {
1236 self.shared
1237 .raw
1238 .invalidate_mapped_memory_ranges(&smallvec::SmallVec::<
1239 [vk::MappedMemoryRange; 32],
1240 >::from_iter(vk_ranges))
1241 }
1242 .unwrap();
1243 }
1244 }
1245
1246 unsafe fn create_texture(
1247 &self,
1248 desc: &crate::TextureDescriptor,
1249 ) -> Result<super::Texture, crate::DeviceError> {
1250 let image = self.create_image_without_memory(desc, None)?;
1251
1252 self.error_if_would_oom_on_resource_allocation(false, image.requirements.size)
1253 .inspect_err(|_| {
1254 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1255 })?;
1256
1257 let block = unsafe {
1258 self.mem_allocator.lock().alloc(
1259 &*self.shared,
1260 gpu_alloc::Request {
1261 size: image.requirements.size,
1262 align_mask: image.requirements.alignment - 1,
1263 usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
1264 memory_types: image.requirements.memory_type_bits & self.valid_ash_memory_types,
1265 },
1266 )
1267 }
1268 .inspect_err(|_| {
1269 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1270 })?;
1271
1272 self.counters.texture_memory.add(block.size() as isize);
1273
1274 unsafe {
1275 self.shared
1276 .raw
1277 .bind_image_memory(image.raw, *block.memory(), block.offset())
1278 }
1279 .map_err(super::map_host_device_oom_err)
1280 .inspect_err(|_| {
1281 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1282 })?;
1283
1284 if let Some(label) = desc.label {
1285 unsafe { self.shared.set_object_name(image.raw, label) };
1286 }
1287
1288 let identity = self.shared.texture_identity_factory.next();
1289
1290 self.counters.textures.add(1);
1291
1292 Ok(super::Texture {
1293 raw: image.raw,
1294 drop_guard: None,
1295 external_memory: None,
1296 block: Some(block),
1297 format: desc.format,
1298 copy_size: image.copy_size,
1299 identity,
1300 })
1301 }
1302 unsafe fn destroy_texture(&self, texture: super::Texture) {
1303 if texture.drop_guard.is_none() {
1304 unsafe { self.shared.raw.destroy_image(texture.raw, None) };
1305 }
1306 if let Some(memory) = texture.external_memory {
1307 unsafe { self.shared.raw.free_memory(memory, None) };
1308 }
1309 if let Some(block) = texture.block {
1310 self.counters.texture_memory.sub(block.size() as isize);
1311
1312 unsafe { self.mem_allocator.lock().dealloc(&*self.shared, block) };
1313 }
1314
1315 self.counters.textures.sub(1);
1316 }
1317
1318 unsafe fn add_raw_texture(&self, _texture: &super::Texture) {
1319 self.counters.textures.add(1);
1320 }
1321
1322 unsafe fn create_texture_view(
1323 &self,
1324 texture: &super::Texture,
1325 desc: &crate::TextureViewDescriptor,
1326 ) -> Result<super::TextureView, crate::DeviceError> {
1327 let subresource_range = conv::map_subresource_range(&desc.range, texture.format);
1328 let raw_format = self.shared.private_caps.map_texture_format(desc.format);
1329 let mut vk_info = vk::ImageViewCreateInfo::default()
1330 .flags(vk::ImageViewCreateFlags::empty())
1331 .image(texture.raw)
1332 .view_type(conv::map_view_dimension(desc.dimension))
1333 .format(raw_format)
1334 .subresource_range(subresource_range);
1335 let layers =
1336 NonZeroU32::new(subresource_range.layer_count).expect("Unexpected zero layer count");
1337
1338 let mut image_view_info;
1339 if self.shared.private_caps.image_view_usage && !desc.usage.is_empty() {
1340 image_view_info =
1341 vk::ImageViewUsageCreateInfo::default().usage(conv::map_texture_usage(desc.usage));
1342 vk_info = vk_info.push_next(&mut image_view_info);
1343 }
1344
1345 let raw = unsafe { self.shared.raw.create_image_view(&vk_info, None) }
1346 .map_err(super::map_host_device_oom_and_ioca_err)?;
1347
1348 if let Some(label) = desc.label {
1349 unsafe { self.shared.set_object_name(raw, label) };
1350 }
1351
1352 let identity = self.shared.texture_view_identity_factory.next();
1353
1354 self.counters.texture_views.add(1);
1355
1356 Ok(super::TextureView {
1357 raw_texture: texture.raw,
1358 raw,
1359 layers,
1360 format: desc.format,
1361 raw_format,
1362 base_mip_level: desc.range.base_mip_level,
1363 dimension: desc.dimension,
1364 texture_identity: texture.identity,
1365 view_identity: identity,
1366 })
1367 }
1368 unsafe fn destroy_texture_view(&self, view: super::TextureView) {
1369 unsafe { self.shared.raw.destroy_image_view(view.raw, None) };
1370
1371 self.counters.texture_views.sub(1);
1372 }
1373
1374 unsafe fn create_sampler(
1375 &self,
1376 desc: &crate::SamplerDescriptor,
1377 ) -> Result<super::Sampler, crate::DeviceError> {
1378 let mut create_info = vk::SamplerCreateInfo::default()
1379 .flags(vk::SamplerCreateFlags::empty())
1380 .mag_filter(conv::map_filter_mode(desc.mag_filter))
1381 .min_filter(conv::map_filter_mode(desc.min_filter))
1382 .mipmap_mode(conv::map_mip_filter_mode(desc.mipmap_filter))
1383 .address_mode_u(conv::map_address_mode(desc.address_modes[0]))
1384 .address_mode_v(conv::map_address_mode(desc.address_modes[1]))
1385 .address_mode_w(conv::map_address_mode(desc.address_modes[2]))
1386 .min_lod(desc.lod_clamp.start)
1387 .max_lod(desc.lod_clamp.end);
1388
1389 if let Some(fun) = desc.compare {
1390 create_info = create_info
1391 .compare_enable(true)
1392 .compare_op(conv::map_comparison(fun));
1393 }
1394
1395 if desc.anisotropy_clamp != 1 {
1396 create_info = create_info
1399 .anisotropy_enable(true)
1400 .max_anisotropy(desc.anisotropy_clamp as f32);
1401 }
1402
1403 if let Some(color) = desc.border_color {
1404 create_info = create_info.border_color(conv::map_border_color(color));
1405 }
1406
1407 let mut sampler_cache_guard = self.shared.sampler_cache.lock();
1408
1409 let raw = sampler_cache_guard.create_sampler(&self.shared.raw, create_info)?;
1410
1411 if let Some(label) = desc.label {
1415 unsafe { self.shared.set_object_name(raw, label) };
1418 }
1419
1420 drop(sampler_cache_guard);
1421
1422 self.counters.samplers.add(1);
1423
1424 Ok(super::Sampler { raw, create_info })
1425 }
1426 unsafe fn destroy_sampler(&self, sampler: super::Sampler) {
1427 self.shared.sampler_cache.lock().destroy_sampler(
1428 &self.shared.raw,
1429 sampler.create_info,
1430 sampler.raw,
1431 );
1432
1433 self.counters.samplers.sub(1);
1434 }
1435
1436 unsafe fn create_command_encoder(
1437 &self,
1438 desc: &crate::CommandEncoderDescriptor<super::Queue>,
1439 ) -> Result<super::CommandEncoder, crate::DeviceError> {
1440 let vk_info = vk::CommandPoolCreateInfo::default()
1441 .queue_family_index(desc.queue.family_index)
1442 .flags(vk::CommandPoolCreateFlags::TRANSIENT);
1443
1444 let raw = unsafe {
1445 self.shared
1446 .raw
1447 .create_command_pool(&vk_info, None)
1448 .map_err(super::map_host_device_oom_err)?
1449 };
1450
1451 self.counters.command_encoders.add(1);
1452
1453 Ok(super::CommandEncoder {
1454 raw,
1455 device: Arc::clone(&self.shared),
1456 active: vk::CommandBuffer::null(),
1457 bind_point: vk::PipelineBindPoint::default(),
1458 temp: super::Temp::default(),
1459 free: Vec::new(),
1460 discarded: Vec::new(),
1461 rpass_debug_marker_active: false,
1462 end_of_pass_timer_query: None,
1463 framebuffers: Default::default(),
1464 temp_texture_views: Default::default(),
1465 counters: Arc::clone(&self.counters),
1466 })
1467 }
1468
1469 unsafe fn create_bind_group_layout(
1470 &self,
1471 desc: &crate::BindGroupLayoutDescriptor,
1472 ) -> Result<super::BindGroupLayout, crate::DeviceError> {
1473 let mut desc_count = gpu_descriptor::DescriptorTotalCount::default();
1474 let mut types = Vec::new();
1475 for entry in desc.entries {
1476 let count = entry.count.map_or(1, |c| c.get());
1477 if entry.binding as usize >= types.len() {
1478 types.resize(
1479 entry.binding as usize + 1,
1480 (vk::DescriptorType::INPUT_ATTACHMENT, 0),
1481 );
1482 }
1483 types[entry.binding as usize] = (
1484 conv::map_binding_type(entry.ty),
1485 entry.count.map_or(1, |c| c.get()),
1486 );
1487
1488 match entry.ty {
1489 wgt::BindingType::Buffer {
1490 ty,
1491 has_dynamic_offset,
1492 ..
1493 } => match ty {
1494 wgt::BufferBindingType::Uniform => {
1495 if has_dynamic_offset {
1496 desc_count.uniform_buffer_dynamic += count;
1497 } else {
1498 desc_count.uniform_buffer += count;
1499 }
1500 }
1501 wgt::BufferBindingType::Storage { .. } => {
1502 if has_dynamic_offset {
1503 desc_count.storage_buffer_dynamic += count;
1504 } else {
1505 desc_count.storage_buffer += count;
1506 }
1507 }
1508 },
1509 wgt::BindingType::Sampler { .. } => {
1510 desc_count.sampler += count;
1511 }
1512 wgt::BindingType::Texture { .. } => {
1513 desc_count.sampled_image += count;
1514 }
1515 wgt::BindingType::StorageTexture { .. } => {
1516 desc_count.storage_image += count;
1517 }
1518 wgt::BindingType::AccelerationStructure { .. } => {
1519 desc_count.acceleration_structure += count;
1520 }
1521 wgt::BindingType::ExternalTexture => unimplemented!(),
1522 }
1523 }
1524
1525 let vk_bindings = desc
1527 .entries
1528 .iter()
1529 .map(|entry| vk::DescriptorSetLayoutBinding {
1530 binding: entry.binding,
1531 descriptor_type: types[entry.binding as usize].0,
1532 descriptor_count: types[entry.binding as usize].1,
1533 stage_flags: conv::map_shader_stage(entry.visibility),
1534 p_immutable_samplers: ptr::null(),
1535 _marker: Default::default(),
1536 })
1537 .collect::<Vec<_>>();
1538
1539 let binding_arrays: Vec<_> = desc
1540 .entries
1541 .iter()
1542 .enumerate()
1543 .filter_map(|(idx, entry)| entry.count.map(|count| (idx as u32, count)))
1544 .collect();
1545
1546 let vk_info = vk::DescriptorSetLayoutCreateInfo::default()
1547 .bindings(&vk_bindings)
1548 .flags(if !binding_arrays.is_empty() {
1549 vk::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND_POOL
1550 } else {
1551 vk::DescriptorSetLayoutCreateFlags::empty()
1552 });
1553
1554 let partially_bound = desc
1555 .flags
1556 .contains(crate::BindGroupLayoutFlags::PARTIALLY_BOUND);
1557
1558 let binding_flag_vec = desc
1559 .entries
1560 .iter()
1561 .map(|entry| {
1562 let mut flags = vk::DescriptorBindingFlags::empty();
1563
1564 if partially_bound && entry.count.is_some() {
1565 flags |= vk::DescriptorBindingFlags::PARTIALLY_BOUND;
1566 }
1567
1568 if entry.count.is_some() {
1569 flags |= vk::DescriptorBindingFlags::UPDATE_AFTER_BIND;
1570 }
1571
1572 flags
1573 })
1574 .collect::<Vec<_>>();
1575
1576 let mut binding_flag_info = vk::DescriptorSetLayoutBindingFlagsCreateInfo::default()
1577 .binding_flags(&binding_flag_vec);
1578
1579 let vk_info = vk_info.push_next(&mut binding_flag_info);
1580
1581 let raw = unsafe {
1582 self.shared
1583 .raw
1584 .create_descriptor_set_layout(&vk_info, None)
1585 .map_err(super::map_host_device_oom_err)?
1586 };
1587
1588 if let Some(label) = desc.label {
1589 unsafe { self.shared.set_object_name(raw, label) };
1590 }
1591
1592 self.counters.bind_group_layouts.add(1);
1593
1594 Ok(super::BindGroupLayout {
1595 raw,
1596 desc_count,
1597 types: types.into_boxed_slice(),
1598 binding_arrays,
1599 })
1600 }
1601 unsafe fn destroy_bind_group_layout(&self, bg_layout: super::BindGroupLayout) {
1602 unsafe {
1603 self.shared
1604 .raw
1605 .destroy_descriptor_set_layout(bg_layout.raw, None)
1606 };
1607
1608 self.counters.bind_group_layouts.sub(1);
1609 }
1610
1611 unsafe fn create_pipeline_layout(
1612 &self,
1613 desc: &crate::PipelineLayoutDescriptor<super::BindGroupLayout>,
1614 ) -> Result<super::PipelineLayout, crate::DeviceError> {
1615 let vk_set_layouts = desc
1617 .bind_group_layouts
1618 .iter()
1619 .map(|bgl| bgl.raw)
1620 .collect::<Vec<_>>();
1621 let vk_push_constant_ranges = desc
1622 .push_constant_ranges
1623 .iter()
1624 .map(|pcr| vk::PushConstantRange {
1625 stage_flags: conv::map_shader_stage(pcr.stages),
1626 offset: pcr.range.start,
1627 size: pcr.range.end - pcr.range.start,
1628 })
1629 .collect::<Vec<_>>();
1630
1631 let vk_info = vk::PipelineLayoutCreateInfo::default()
1632 .flags(vk::PipelineLayoutCreateFlags::empty())
1633 .set_layouts(&vk_set_layouts)
1634 .push_constant_ranges(&vk_push_constant_ranges);
1635
1636 let raw = {
1637 profiling::scope!("vkCreatePipelineLayout");
1638 unsafe {
1639 self.shared
1640 .raw
1641 .create_pipeline_layout(&vk_info, None)
1642 .map_err(super::map_host_device_oom_err)?
1643 }
1644 };
1645
1646 if let Some(label) = desc.label {
1647 unsafe { self.shared.set_object_name(raw, label) };
1648 }
1649
1650 let mut binding_arrays = BTreeMap::new();
1651 for (group, &layout) in desc.bind_group_layouts.iter().enumerate() {
1652 for &(binding, binding_array_size) in &layout.binding_arrays {
1653 binding_arrays.insert(
1654 naga::ResourceBinding {
1655 group: group as u32,
1656 binding,
1657 },
1658 naga::back::spv::BindingInfo {
1659 binding_array_size: Some(binding_array_size.get()),
1660 },
1661 );
1662 }
1663 }
1664
1665 self.counters.pipeline_layouts.add(1);
1666
1667 Ok(super::PipelineLayout {
1668 raw,
1669 binding_arrays,
1670 })
1671 }
1672 unsafe fn destroy_pipeline_layout(&self, pipeline_layout: super::PipelineLayout) {
1673 unsafe {
1674 self.shared
1675 .raw
1676 .destroy_pipeline_layout(pipeline_layout.raw, None)
1677 };
1678
1679 self.counters.pipeline_layouts.sub(1);
1680 }
1681
1682 unsafe fn create_bind_group(
1683 &self,
1684 desc: &crate::BindGroupDescriptor<
1685 super::BindGroupLayout,
1686 super::Buffer,
1687 super::Sampler,
1688 super::TextureView,
1689 super::AccelerationStructure,
1690 >,
1691 ) -> Result<super::BindGroup, crate::DeviceError> {
1692 let contains_binding_arrays = !desc.layout.binding_arrays.is_empty();
1693
1694 let desc_set_layout_flags = if contains_binding_arrays {
1695 gpu_descriptor::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND
1696 } else {
1697 gpu_descriptor::DescriptorSetLayoutCreateFlags::empty()
1698 };
1699
1700 let mut vk_sets = unsafe {
1701 self.desc_allocator.lock().allocate(
1702 &*self.shared,
1703 &desc.layout.raw,
1704 desc_set_layout_flags,
1705 &desc.layout.desc_count,
1706 1,
1707 )?
1708 };
1709
1710 let set = vk_sets.pop().unwrap();
1711 if let Some(label) = desc.label {
1712 unsafe { self.shared.set_object_name(*set.raw(), label) };
1713 }
1714
1715 struct ExtendStack<'a, T> {
1722 remainder: &'a mut [MaybeUninit<T>],
1723 }
1724
1725 impl<'a, T> ExtendStack<'a, T> {
1726 fn from_vec_capacity(vec: &'a mut Vec<T>) -> Self {
1727 Self {
1728 remainder: vec.spare_capacity_mut(),
1729 }
1730 }
1731
1732 fn extend_one(self, value: T) -> (Self, &'a mut T) {
1733 let (to_init, remainder) = self.remainder.split_first_mut().unwrap();
1734 let init = to_init.write(value);
1735 (Self { remainder }, init)
1736 }
1737
1738 fn extend(
1739 self,
1740 iter: impl IntoIterator<Item = T> + ExactSizeIterator,
1741 ) -> (Self, &'a mut [T]) {
1742 let (to_init, remainder) = self.remainder.split_at_mut(iter.len());
1743
1744 for (value, to_init) in iter.into_iter().zip(to_init.iter_mut()) {
1745 to_init.write(value);
1746 }
1747
1748 let init = {
1751 unsafe { mem::transmute::<&mut [MaybeUninit<T>], &mut [T]>(to_init) }
1758 };
1759 (Self { remainder }, init)
1760 }
1761 }
1762
1763 let mut writes = Vec::with_capacity(desc.entries.len());
1764 let mut buffer_infos = Vec::with_capacity(desc.buffers.len());
1765 let mut buffer_infos = ExtendStack::from_vec_capacity(&mut buffer_infos);
1766 let mut image_infos = Vec::with_capacity(desc.samplers.len() + desc.textures.len());
1767 let mut image_infos = ExtendStack::from_vec_capacity(&mut image_infos);
1768 let mut acceleration_structure_infos =
1773 Vec::with_capacity(desc.acceleration_structures.len());
1774 let mut acceleration_structure_infos =
1775 ExtendStack::from_vec_capacity(&mut acceleration_structure_infos);
1776 let mut raw_acceleration_structures =
1777 Vec::with_capacity(desc.acceleration_structures.len());
1778 let mut raw_acceleration_structures =
1779 ExtendStack::from_vec_capacity(&mut raw_acceleration_structures);
1780 for entry in desc.entries {
1781 let (ty, size) = desc.layout.types[entry.binding as usize];
1782 if size == 0 {
1783 continue; }
1785 let mut write = vk::WriteDescriptorSet::default()
1786 .dst_set(*set.raw())
1787 .dst_binding(entry.binding)
1788 .descriptor_type(ty);
1789
1790 write = match ty {
1791 vk::DescriptorType::SAMPLER => {
1792 let start = entry.resource_index;
1793 let end = start + entry.count;
1794 let local_image_infos;
1795 (image_infos, local_image_infos) =
1796 image_infos.extend(desc.samplers[start as usize..end as usize].iter().map(
1797 |sampler| vk::DescriptorImageInfo::default().sampler(sampler.raw),
1798 ));
1799 write.image_info(local_image_infos)
1800 }
1801 vk::DescriptorType::SAMPLED_IMAGE | vk::DescriptorType::STORAGE_IMAGE => {
1802 let start = entry.resource_index;
1803 let end = start + entry.count;
1804 let local_image_infos;
1805 (image_infos, local_image_infos) =
1806 image_infos.extend(desc.textures[start as usize..end as usize].iter().map(
1807 |binding| {
1808 let layout =
1809 conv::derive_image_layout(binding.usage, binding.view.format);
1810 vk::DescriptorImageInfo::default()
1811 .image_view(binding.view.raw)
1812 .image_layout(layout)
1813 },
1814 ));
1815 write.image_info(local_image_infos)
1816 }
1817 vk::DescriptorType::UNIFORM_BUFFER
1818 | vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC
1819 | vk::DescriptorType::STORAGE_BUFFER
1820 | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => {
1821 let start = entry.resource_index;
1822 let end = start + entry.count;
1823 let local_buffer_infos;
1824 (buffer_infos, local_buffer_infos) =
1825 buffer_infos.extend(desc.buffers[start as usize..end as usize].iter().map(
1826 |binding| {
1827 vk::DescriptorBufferInfo::default()
1828 .buffer(binding.buffer.raw)
1829 .offset(binding.offset)
1830 .range(
1831 binding.size.map_or(vk::WHOLE_SIZE, wgt::BufferSize::get),
1832 )
1833 },
1834 ));
1835 write.buffer_info(local_buffer_infos)
1836 }
1837 vk::DescriptorType::ACCELERATION_STRUCTURE_KHR => {
1838 let start = entry.resource_index;
1839 let end = start + entry.count;
1840
1841 let local_raw_acceleration_structures;
1842 (
1843 raw_acceleration_structures,
1844 local_raw_acceleration_structures,
1845 ) = raw_acceleration_structures.extend(
1846 desc.acceleration_structures[start as usize..end as usize]
1847 .iter()
1848 .map(|acceleration_structure| acceleration_structure.raw),
1849 );
1850
1851 let local_acceleration_structure_infos;
1852 (
1853 acceleration_structure_infos,
1854 local_acceleration_structure_infos,
1855 ) = acceleration_structure_infos.extend_one(
1856 vk::WriteDescriptorSetAccelerationStructureKHR::default()
1857 .acceleration_structures(local_raw_acceleration_structures),
1858 );
1859
1860 write
1861 .descriptor_count(entry.count)
1862 .push_next(local_acceleration_structure_infos)
1863 }
1864 _ => unreachable!(),
1865 };
1866
1867 writes.push(write);
1868 }
1869
1870 unsafe { self.shared.raw.update_descriptor_sets(&writes, &[]) };
1871
1872 self.counters.bind_groups.add(1);
1873
1874 Ok(super::BindGroup { set })
1875 }
1876
1877 unsafe fn destroy_bind_group(&self, group: super::BindGroup) {
1878 unsafe {
1879 self.desc_allocator
1880 .lock()
1881 .free(&*self.shared, Some(group.set))
1882 };
1883
1884 self.counters.bind_groups.sub(1);
1885 }
1886
1887 unsafe fn create_shader_module(
1888 &self,
1889 desc: &crate::ShaderModuleDescriptor,
1890 shader: crate::ShaderInput,
1891 ) -> Result<super::ShaderModule, crate::ShaderError> {
1892 let spv = match shader {
1893 crate::ShaderInput::Naga(naga_shader) => {
1894 if self
1895 .shared
1896 .workarounds
1897 .contains(super::Workarounds::SEPARATE_ENTRY_POINTS)
1898 || !naga_shader.module.overrides.is_empty()
1899 {
1900 return Ok(super::ShaderModule::Intermediate {
1901 naga_shader,
1902 runtime_checks: desc.runtime_checks,
1903 });
1904 }
1905 let mut naga_options = self.naga_options.clone();
1906 naga_options.debug_info =
1907 naga_shader
1908 .debug_source
1909 .as_ref()
1910 .map(|d| naga::back::spv::DebugInfo {
1911 source_code: d.source_code.as_ref(),
1912 file_name: d.file_name.as_ref().into(),
1913 language: naga::back::spv::SourceLanguage::WGSL,
1914 });
1915 if !desc.runtime_checks.bounds_checks {
1916 naga_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
1917 index: naga::proc::BoundsCheckPolicy::Unchecked,
1918 buffer: naga::proc::BoundsCheckPolicy::Unchecked,
1919 image_load: naga::proc::BoundsCheckPolicy::Unchecked,
1920 binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
1921 };
1922 }
1923 Cow::Owned(
1924 naga::back::spv::write_vec(
1925 &naga_shader.module,
1926 &naga_shader.info,
1927 &naga_options,
1928 None,
1929 )
1930 .map_err(|e| crate::ShaderError::Compilation(format!("{e}")))?,
1931 )
1932 }
1933 crate::ShaderInput::Msl { .. } => {
1934 panic!("MSL_SHADER_PASSTHROUGH is not enabled for this backend")
1935 }
1936 crate::ShaderInput::Dxil { .. } | crate::ShaderInput::Hlsl { .. } => {
1937 panic!("`Features::HLSL_DXIL_SHADER_PASSTHROUGH` is not enabled")
1938 }
1939 crate::ShaderInput::SpirV(spv) => Cow::Borrowed(spv),
1940 };
1941
1942 let raw = self.create_shader_module_impl(&spv)?;
1943
1944 if let Some(label) = desc.label {
1945 unsafe { self.shared.set_object_name(raw, label) };
1946 }
1947
1948 self.counters.shader_modules.add(1);
1949
1950 Ok(super::ShaderModule::Raw(raw))
1951 }
1952
1953 unsafe fn destroy_shader_module(&self, module: super::ShaderModule) {
1954 match module {
1955 super::ShaderModule::Raw(raw) => {
1956 unsafe { self.shared.raw.destroy_shader_module(raw, None) };
1957 }
1958 super::ShaderModule::Intermediate { .. } => {}
1959 }
1960
1961 self.counters.shader_modules.sub(1);
1962 }
1963
1964 unsafe fn create_render_pipeline(
1965 &self,
1966 desc: &crate::RenderPipelineDescriptor<
1967 super::PipelineLayout,
1968 super::ShaderModule,
1969 super::PipelineCache,
1970 >,
1971 ) -> Result<super::RenderPipeline, crate::PipelineError> {
1972 let dynamic_states = [
1973 vk::DynamicState::VIEWPORT,
1974 vk::DynamicState::SCISSOR,
1975 vk::DynamicState::BLEND_CONSTANTS,
1976 vk::DynamicState::STENCIL_REFERENCE,
1977 ];
1978 let mut compatible_rp_key = super::RenderPassKey {
1979 sample_count: desc.multisample.count,
1980 multiview: desc.multiview,
1981 ..Default::default()
1982 };
1983 let mut stages = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new();
1984 let mut vertex_buffers = Vec::new();
1985 let mut vertex_attributes = Vec::new();
1986
1987 if let crate::VertexProcessor::Standard {
1988 vertex_buffers: desc_vertex_buffers,
1989 vertex_stage: _,
1990 } = &desc.vertex_processor
1991 {
1992 vertex_buffers = Vec::with_capacity(desc_vertex_buffers.len());
1993 for (i, vb) in desc_vertex_buffers.iter().enumerate() {
1994 vertex_buffers.push(vk::VertexInputBindingDescription {
1995 binding: i as u32,
1996 stride: vb.array_stride as u32,
1997 input_rate: match vb.step_mode {
1998 wgt::VertexStepMode::Vertex => vk::VertexInputRate::VERTEX,
1999 wgt::VertexStepMode::Instance => vk::VertexInputRate::INSTANCE,
2000 },
2001 });
2002 for at in vb.attributes {
2003 vertex_attributes.push(vk::VertexInputAttributeDescription {
2004 location: at.shader_location,
2005 binding: i as u32,
2006 format: conv::map_vertex_format(at.format),
2007 offset: at.offset as u32,
2008 });
2009 }
2010 }
2011 }
2012
2013 let vk_vertex_input = vk::PipelineVertexInputStateCreateInfo::default()
2014 .vertex_binding_descriptions(&vertex_buffers)
2015 .vertex_attribute_descriptions(&vertex_attributes);
2016
2017 let vk_input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
2018 .topology(conv::map_topology(desc.primitive.topology))
2019 .primitive_restart_enable(desc.primitive.strip_index_format.is_some());
2020
2021 let mut compiled_vs = None;
2022 let mut compiled_ms = None;
2023 let mut compiled_ts = None;
2024 match &desc.vertex_processor {
2025 crate::VertexProcessor::Standard {
2026 vertex_buffers: _,
2027 vertex_stage,
2028 } => {
2029 compiled_vs = Some(self.compile_stage(
2030 vertex_stage,
2031 naga::ShaderStage::Vertex,
2032 &desc.layout.binding_arrays,
2033 )?);
2034 stages.push(compiled_vs.as_ref().unwrap().create_info);
2035 }
2036 crate::VertexProcessor::Mesh {
2037 task_stage,
2038 mesh_stage,
2039 } => {
2040 if let Some(t) = task_stage.as_ref() {
2041 compiled_ts = Some(self.compile_stage(
2042 t,
2043 naga::ShaderStage::Task,
2044 &desc.layout.binding_arrays,
2045 )?);
2046 stages.push(compiled_ts.as_ref().unwrap().create_info);
2047 }
2048 compiled_ms = Some(self.compile_stage(
2049 mesh_stage,
2050 naga::ShaderStage::Mesh,
2051 &desc.layout.binding_arrays,
2052 )?);
2053 stages.push(compiled_ms.as_ref().unwrap().create_info);
2054 }
2055 }
2056 let compiled_fs = match desc.fragment_stage {
2057 Some(ref stage) => {
2058 let compiled = self.compile_stage(
2059 stage,
2060 naga::ShaderStage::Fragment,
2061 &desc.layout.binding_arrays,
2062 )?;
2063 stages.push(compiled.create_info);
2064 Some(compiled)
2065 }
2066 None => None,
2067 };
2068
2069 let mut vk_rasterization = vk::PipelineRasterizationStateCreateInfo::default()
2070 .polygon_mode(conv::map_polygon_mode(desc.primitive.polygon_mode))
2071 .front_face(conv::map_front_face(desc.primitive.front_face))
2072 .line_width(1.0)
2073 .depth_clamp_enable(desc.primitive.unclipped_depth);
2074 if let Some(face) = desc.primitive.cull_mode {
2075 vk_rasterization = vk_rasterization.cull_mode(conv::map_cull_face(face))
2076 }
2077 let mut vk_rasterization_conservative_state =
2078 vk::PipelineRasterizationConservativeStateCreateInfoEXT::default()
2079 .conservative_rasterization_mode(
2080 vk::ConservativeRasterizationModeEXT::OVERESTIMATE,
2081 );
2082 if desc.primitive.conservative {
2083 vk_rasterization = vk_rasterization.push_next(&mut vk_rasterization_conservative_state);
2084 }
2085
2086 let mut vk_depth_stencil = vk::PipelineDepthStencilStateCreateInfo::default();
2087 if let Some(ref ds) = desc.depth_stencil {
2088 let vk_format = self.shared.private_caps.map_texture_format(ds.format);
2089 let vk_layout = if ds.is_read_only(desc.primitive.cull_mode) {
2090 vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL
2091 } else {
2092 vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL
2093 };
2094 compatible_rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
2095 base: super::AttachmentKey::compatible(vk_format, vk_layout),
2096 stencil_ops: crate::AttachmentOps::all(),
2097 });
2098
2099 if ds.is_depth_enabled() {
2100 vk_depth_stencil = vk_depth_stencil
2101 .depth_test_enable(true)
2102 .depth_write_enable(ds.depth_write_enabled)
2103 .depth_compare_op(conv::map_comparison(ds.depth_compare));
2104 }
2105 if ds.stencil.is_enabled() {
2106 let s = &ds.stencil;
2107 let front = conv::map_stencil_face(&s.front, s.read_mask, s.write_mask);
2108 let back = conv::map_stencil_face(&s.back, s.read_mask, s.write_mask);
2109 vk_depth_stencil = vk_depth_stencil
2110 .stencil_test_enable(true)
2111 .front(front)
2112 .back(back);
2113 }
2114
2115 if ds.bias.is_enabled() {
2116 vk_rasterization = vk_rasterization
2117 .depth_bias_enable(true)
2118 .depth_bias_constant_factor(ds.bias.constant as f32)
2119 .depth_bias_clamp(ds.bias.clamp)
2120 .depth_bias_slope_factor(ds.bias.slope_scale);
2121 }
2122 }
2123
2124 let vk_viewport = vk::PipelineViewportStateCreateInfo::default()
2125 .flags(vk::PipelineViewportStateCreateFlags::empty())
2126 .scissor_count(1)
2127 .viewport_count(1);
2128
2129 let vk_sample_mask = [
2130 desc.multisample.mask as u32,
2131 (desc.multisample.mask >> 32) as u32,
2132 ];
2133 let vk_multisample = vk::PipelineMultisampleStateCreateInfo::default()
2134 .rasterization_samples(vk::SampleCountFlags::from_raw(desc.multisample.count))
2135 .alpha_to_coverage_enable(desc.multisample.alpha_to_coverage_enabled)
2136 .sample_mask(&vk_sample_mask);
2137
2138 let mut vk_attachments = Vec::with_capacity(desc.color_targets.len());
2139 for cat in desc.color_targets {
2140 let (key, attarchment) = if let Some(cat) = cat.as_ref() {
2141 let mut vk_attachment = vk::PipelineColorBlendAttachmentState::default()
2142 .color_write_mask(vk::ColorComponentFlags::from_raw(cat.write_mask.bits()));
2143 if let Some(ref blend) = cat.blend {
2144 let (color_op, color_src, color_dst) = conv::map_blend_component(&blend.color);
2145 let (alpha_op, alpha_src, alpha_dst) = conv::map_blend_component(&blend.alpha);
2146 vk_attachment = vk_attachment
2147 .blend_enable(true)
2148 .color_blend_op(color_op)
2149 .src_color_blend_factor(color_src)
2150 .dst_color_blend_factor(color_dst)
2151 .alpha_blend_op(alpha_op)
2152 .src_alpha_blend_factor(alpha_src)
2153 .dst_alpha_blend_factor(alpha_dst);
2154 }
2155
2156 let vk_format = self.shared.private_caps.map_texture_format(cat.format);
2157 (
2158 Some(super::ColorAttachmentKey {
2159 base: super::AttachmentKey::compatible(
2160 vk_format,
2161 vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
2162 ),
2163 resolve: None,
2164 }),
2165 vk_attachment,
2166 )
2167 } else {
2168 (None, vk::PipelineColorBlendAttachmentState::default())
2169 };
2170
2171 compatible_rp_key.colors.push(key);
2172 vk_attachments.push(attarchment);
2173 }
2174
2175 let vk_color_blend =
2176 vk::PipelineColorBlendStateCreateInfo::default().attachments(&vk_attachments);
2177
2178 let vk_dynamic_state =
2179 vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic_states);
2180
2181 let raw_pass = self.shared.make_render_pass(compatible_rp_key)?;
2182
2183 let vk_infos = [{
2184 vk::GraphicsPipelineCreateInfo::default()
2185 .layout(desc.layout.raw)
2186 .stages(&stages)
2187 .vertex_input_state(&vk_vertex_input)
2188 .input_assembly_state(&vk_input_assembly)
2189 .rasterization_state(&vk_rasterization)
2190 .viewport_state(&vk_viewport)
2191 .multisample_state(&vk_multisample)
2192 .depth_stencil_state(&vk_depth_stencil)
2193 .color_blend_state(&vk_color_blend)
2194 .dynamic_state(&vk_dynamic_state)
2195 .render_pass(raw_pass)
2196 }];
2197
2198 let pipeline_cache = desc
2199 .cache
2200 .map(|it| it.raw)
2201 .unwrap_or(vk::PipelineCache::null());
2202
2203 let mut raw_vec = {
2204 profiling::scope!("vkCreateGraphicsPipelines");
2205 unsafe {
2206 self.shared
2207 .raw
2208 .create_graphics_pipelines(pipeline_cache, &vk_infos, None)
2209 .map_err(|(_, e)| super::map_pipeline_err(e))
2210 }?
2211 };
2212
2213 let raw = raw_vec.pop().unwrap();
2214 if let Some(label) = desc.label {
2215 unsafe { self.shared.set_object_name(raw, label) };
2216 }
2217
2218 if let Some(CompiledStage {
2219 temp_raw_module: Some(raw_module),
2220 ..
2221 }) = compiled_vs
2222 {
2223 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2224 }
2225 if let Some(CompiledStage {
2226 temp_raw_module: Some(raw_module),
2227 ..
2228 }) = compiled_ts
2229 {
2230 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2231 }
2232 if let Some(CompiledStage {
2233 temp_raw_module: Some(raw_module),
2234 ..
2235 }) = compiled_ms
2236 {
2237 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2238 }
2239 if let Some(CompiledStage {
2240 temp_raw_module: Some(raw_module),
2241 ..
2242 }) = compiled_fs
2243 {
2244 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2245 }
2246
2247 self.counters.render_pipelines.add(1);
2248
2249 Ok(super::RenderPipeline { raw })
2250 }
2251
2252 unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) {
2253 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2254
2255 self.counters.render_pipelines.sub(1);
2256 }
2257
2258 unsafe fn create_compute_pipeline(
2259 &self,
2260 desc: &crate::ComputePipelineDescriptor<
2261 super::PipelineLayout,
2262 super::ShaderModule,
2263 super::PipelineCache,
2264 >,
2265 ) -> Result<super::ComputePipeline, crate::PipelineError> {
2266 let compiled = self.compile_stage(
2267 &desc.stage,
2268 naga::ShaderStage::Compute,
2269 &desc.layout.binding_arrays,
2270 )?;
2271
2272 let vk_infos = [{
2273 vk::ComputePipelineCreateInfo::default()
2274 .layout(desc.layout.raw)
2275 .stage(compiled.create_info)
2276 }];
2277
2278 let pipeline_cache = desc
2279 .cache
2280 .map(|it| it.raw)
2281 .unwrap_or(vk::PipelineCache::null());
2282
2283 let mut raw_vec = {
2284 profiling::scope!("vkCreateComputePipelines");
2285 unsafe {
2286 self.shared
2287 .raw
2288 .create_compute_pipelines(pipeline_cache, &vk_infos, None)
2289 .map_err(|(_, e)| super::map_pipeline_err(e))
2290 }?
2291 };
2292
2293 let raw = raw_vec.pop().unwrap();
2294 if let Some(label) = desc.label {
2295 unsafe { self.shared.set_object_name(raw, label) };
2296 }
2297
2298 if let Some(raw_module) = compiled.temp_raw_module {
2299 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2300 }
2301
2302 self.counters.compute_pipelines.add(1);
2303
2304 Ok(super::ComputePipeline { raw })
2305 }
2306
2307 unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) {
2308 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2309
2310 self.counters.compute_pipelines.sub(1);
2311 }
2312
2313 unsafe fn create_pipeline_cache(
2314 &self,
2315 desc: &crate::PipelineCacheDescriptor<'_>,
2316 ) -> Result<super::PipelineCache, crate::PipelineCacheError> {
2317 let mut info = vk::PipelineCacheCreateInfo::default();
2318 if let Some(data) = desc.data {
2319 info = info.initial_data(data)
2320 }
2321 profiling::scope!("vkCreatePipelineCache");
2322 let raw = unsafe { self.shared.raw.create_pipeline_cache(&info, None) }
2323 .map_err(super::map_host_device_oom_err)?;
2324
2325 Ok(super::PipelineCache { raw })
2326 }
2327 fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> {
2328 Some(self.shared.pipeline_cache_validation_key)
2329 }
2330 unsafe fn destroy_pipeline_cache(&self, cache: super::PipelineCache) {
2331 unsafe { self.shared.raw.destroy_pipeline_cache(cache.raw, None) }
2332 }
2333 unsafe fn create_query_set(
2334 &self,
2335 desc: &wgt::QuerySetDescriptor<crate::Label>,
2336 ) -> Result<super::QuerySet, crate::DeviceError> {
2337 self.error_if_would_oom_on_resource_allocation(true, desc.count as u64 * 256)?;
2340
2341 let (vk_type, pipeline_statistics) = match desc.ty {
2342 wgt::QueryType::Occlusion => (
2343 vk::QueryType::OCCLUSION,
2344 vk::QueryPipelineStatisticFlags::empty(),
2345 ),
2346 wgt::QueryType::PipelineStatistics(statistics) => (
2347 vk::QueryType::PIPELINE_STATISTICS,
2348 conv::map_pipeline_statistics(statistics),
2349 ),
2350 wgt::QueryType::Timestamp => (
2351 vk::QueryType::TIMESTAMP,
2352 vk::QueryPipelineStatisticFlags::empty(),
2353 ),
2354 };
2355
2356 let vk_info = vk::QueryPoolCreateInfo::default()
2357 .query_type(vk_type)
2358 .query_count(desc.count)
2359 .pipeline_statistics(pipeline_statistics);
2360
2361 let raw = unsafe { self.shared.raw.create_query_pool(&vk_info, None) }
2362 .map_err(super::map_host_device_oom_err)?;
2363 if let Some(label) = desc.label {
2364 unsafe { self.shared.set_object_name(raw, label) };
2365 }
2366
2367 self.counters.query_sets.add(1);
2368
2369 Ok(super::QuerySet { raw })
2370 }
2371
2372 unsafe fn destroy_query_set(&self, set: super::QuerySet) {
2373 unsafe { self.shared.raw.destroy_query_pool(set.raw, None) };
2374
2375 self.counters.query_sets.sub(1);
2376 }
2377
2378 unsafe fn create_fence(&self) -> Result<super::Fence, crate::DeviceError> {
2379 self.counters.fences.add(1);
2380
2381 Ok(if self.shared.private_caps.timeline_semaphores {
2382 let mut sem_type_info =
2383 vk::SemaphoreTypeCreateInfo::default().semaphore_type(vk::SemaphoreType::TIMELINE);
2384 let vk_info = vk::SemaphoreCreateInfo::default().push_next(&mut sem_type_info);
2385 let raw = unsafe { self.shared.raw.create_semaphore(&vk_info, None) }
2386 .map_err(super::map_host_device_oom_err)?;
2387
2388 super::Fence::TimelineSemaphore(raw)
2389 } else {
2390 super::Fence::FencePool {
2391 last_completed: 0,
2392 active: Vec::new(),
2393 free: Vec::new(),
2394 }
2395 })
2396 }
2397 unsafe fn destroy_fence(&self, fence: super::Fence) {
2398 match fence {
2399 super::Fence::TimelineSemaphore(raw) => {
2400 unsafe { self.shared.raw.destroy_semaphore(raw, None) };
2401 }
2402 super::Fence::FencePool {
2403 active,
2404 free,
2405 last_completed: _,
2406 } => {
2407 for (_, raw) in active {
2408 unsafe { self.shared.raw.destroy_fence(raw, None) };
2409 }
2410 for raw in free {
2411 unsafe { self.shared.raw.destroy_fence(raw, None) };
2412 }
2413 }
2414 }
2415
2416 self.counters.fences.sub(1);
2417 }
2418 unsafe fn get_fence_value(
2419 &self,
2420 fence: &super::Fence,
2421 ) -> Result<crate::FenceValue, crate::DeviceError> {
2422 fence.get_latest(
2423 &self.shared.raw,
2424 self.shared.extension_fns.timeline_semaphore.as_ref(),
2425 )
2426 }
2427 unsafe fn wait(
2428 &self,
2429 fence: &super::Fence,
2430 wait_value: crate::FenceValue,
2431 timeout_ms: u32,
2432 ) -> Result<bool, crate::DeviceError> {
2433 let timeout_ns = timeout_ms as u64 * super::MILLIS_TO_NANOS;
2434 self.shared.wait_for_fence(fence, wait_value, timeout_ns)
2435 }
2436
2437 unsafe fn start_graphics_debugger_capture(&self) -> bool {
2438 #[cfg(feature = "renderdoc")]
2439 {
2440 let raw_vk_instance =
2442 vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2443 let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2444 unsafe {
2445 self.render_doc
2446 .start_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2447 }
2448 }
2449 #[cfg(not(feature = "renderdoc"))]
2450 false
2451 }
2452 unsafe fn stop_graphics_debugger_capture(&self) {
2453 #[cfg(feature = "renderdoc")]
2454 {
2455 let raw_vk_instance =
2457 vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2458 let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2459
2460 unsafe {
2461 self.render_doc
2462 .end_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2463 }
2464 }
2465 }
2466
2467 unsafe fn pipeline_cache_get_data(&self, cache: &super::PipelineCache) -> Option<Vec<u8>> {
2468 let data = unsafe { self.raw_device().get_pipeline_cache_data(cache.raw) };
2469 data.ok()
2470 }
2471
2472 unsafe fn get_acceleration_structure_build_sizes<'a>(
2473 &self,
2474 desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>,
2475 ) -> crate::AccelerationStructureBuildSizes {
2476 const CAPACITY: usize = 8;
2477
2478 let ray_tracing_functions = self
2479 .shared
2480 .extension_fns
2481 .ray_tracing
2482 .as_ref()
2483 .expect("Feature `RAY_TRACING` not enabled");
2484
2485 let (geometries, primitive_counts) = match *desc.entries {
2486 crate::AccelerationStructureEntries::Instances(ref instances) => {
2487 let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default();
2488
2489 let geometry = vk::AccelerationStructureGeometryKHR::default()
2490 .geometry_type(vk::GeometryTypeKHR::INSTANCES)
2491 .geometry(vk::AccelerationStructureGeometryDataKHR {
2492 instances: instance_data,
2493 });
2494
2495 (
2496 smallvec::smallvec![geometry],
2497 smallvec::smallvec![instances.count],
2498 )
2499 }
2500 crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
2501 let mut primitive_counts =
2502 smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2503 let mut geometries = smallvec::SmallVec::<
2504 [vk::AccelerationStructureGeometryKHR; CAPACITY],
2505 >::with_capacity(in_geometries.len());
2506
2507 for triangles in in_geometries {
2508 let mut triangle_data =
2509 vk::AccelerationStructureGeometryTrianglesDataKHR::default()
2510 .index_type(vk::IndexType::NONE_KHR)
2511 .vertex_format(conv::map_vertex_format(triangles.vertex_format))
2512 .max_vertex(triangles.vertex_count)
2513 .vertex_stride(triangles.vertex_stride)
2514 .transform_data(vk::DeviceOrHostAddressConstKHR {
2524 device_address: if desc
2525 .flags
2526 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
2527 {
2528 unsafe {
2529 ray_tracing_functions
2530 .buffer_device_address
2531 .get_buffer_device_address(
2532 &vk::BufferDeviceAddressInfo::default().buffer(
2533 triangles
2534 .transform
2535 .as_ref()
2536 .unwrap()
2537 .buffer
2538 .raw,
2539 ),
2540 )
2541 }
2542 } else {
2543 0
2544 },
2545 });
2546
2547 let pritive_count = if let Some(ref indices) = triangles.indices {
2548 triangle_data =
2549 triangle_data.index_type(conv::map_index_format(indices.format));
2550 indices.count / 3
2551 } else {
2552 triangles.vertex_count
2553 };
2554
2555 let geometry = vk::AccelerationStructureGeometryKHR::default()
2556 .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
2557 .geometry(vk::AccelerationStructureGeometryDataKHR {
2558 triangles: triangle_data,
2559 })
2560 .flags(conv::map_acceleration_structure_geometry_flags(
2561 triangles.flags,
2562 ));
2563
2564 geometries.push(geometry);
2565 primitive_counts.push(pritive_count);
2566 }
2567 (geometries, primitive_counts)
2568 }
2569 crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
2570 let mut primitive_counts =
2571 smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2572 let mut geometries = smallvec::SmallVec::<
2573 [vk::AccelerationStructureGeometryKHR; CAPACITY],
2574 >::with_capacity(in_geometries.len());
2575 for aabb in in_geometries {
2576 let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
2577 .stride(aabb.stride);
2578
2579 let geometry = vk::AccelerationStructureGeometryKHR::default()
2580 .geometry_type(vk::GeometryTypeKHR::AABBS)
2581 .geometry(vk::AccelerationStructureGeometryDataKHR { aabbs: aabbs_data })
2582 .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
2583
2584 geometries.push(geometry);
2585 primitive_counts.push(aabb.count);
2586 }
2587 (geometries, primitive_counts)
2588 }
2589 };
2590
2591 let ty = match *desc.entries {
2592 crate::AccelerationStructureEntries::Instances(_) => {
2593 vk::AccelerationStructureTypeKHR::TOP_LEVEL
2594 }
2595 _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
2596 };
2597
2598 let geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
2599 .ty(ty)
2600 .flags(conv::map_acceleration_structure_flags(desc.flags))
2601 .geometries(&geometries);
2602
2603 let mut raw = Default::default();
2604 unsafe {
2605 ray_tracing_functions
2606 .acceleration_structure
2607 .get_acceleration_structure_build_sizes(
2608 vk::AccelerationStructureBuildTypeKHR::DEVICE,
2609 &geometry_info,
2610 &primitive_counts,
2611 &mut raw,
2612 )
2613 }
2614
2615 crate::AccelerationStructureBuildSizes {
2616 acceleration_structure_size: raw.acceleration_structure_size,
2617 update_scratch_size: raw.update_scratch_size,
2618 build_scratch_size: raw.build_scratch_size,
2619 }
2620 }
2621
2622 unsafe fn get_acceleration_structure_device_address(
2623 &self,
2624 acceleration_structure: &super::AccelerationStructure,
2625 ) -> wgt::BufferAddress {
2626 let ray_tracing_functions = self
2627 .shared
2628 .extension_fns
2629 .ray_tracing
2630 .as_ref()
2631 .expect("Feature `RAY_TRACING` not enabled");
2632
2633 unsafe {
2634 ray_tracing_functions
2635 .acceleration_structure
2636 .get_acceleration_structure_device_address(
2637 &vk::AccelerationStructureDeviceAddressInfoKHR::default()
2638 .acceleration_structure(acceleration_structure.raw),
2639 )
2640 }
2641 }
2642
2643 unsafe fn create_acceleration_structure(
2644 &self,
2645 desc: &crate::AccelerationStructureDescriptor,
2646 ) -> Result<super::AccelerationStructure, crate::DeviceError> {
2647 let ray_tracing_functions = self
2648 .shared
2649 .extension_fns
2650 .ray_tracing
2651 .as_ref()
2652 .expect("Feature `RAY_TRACING` not enabled");
2653
2654 let vk_buffer_info = vk::BufferCreateInfo::default()
2655 .size(desc.size)
2656 .usage(
2657 vk::BufferUsageFlags::ACCELERATION_STRUCTURE_STORAGE_KHR
2658 | vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS,
2659 )
2660 .sharing_mode(vk::SharingMode::EXCLUSIVE);
2661
2662 unsafe {
2663 let raw_buffer = self
2664 .shared
2665 .raw
2666 .create_buffer(&vk_buffer_info, None)
2667 .map_err(super::map_host_device_oom_and_ioca_err)?;
2668 let req = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
2669
2670 self.error_if_would_oom_on_resource_allocation(false, req.size)
2671 .inspect_err(|_| {
2672 self.shared.raw.destroy_buffer(raw_buffer, None);
2673 })?;
2674
2675 let block = self
2676 .mem_allocator
2677 .lock()
2678 .alloc(
2679 &*self.shared,
2680 gpu_alloc::Request {
2681 size: req.size,
2682 align_mask: req.alignment - 1,
2683 usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
2684 memory_types: req.memory_type_bits & self.valid_ash_memory_types,
2685 },
2686 )
2687 .inspect_err(|_| {
2688 self.shared.raw.destroy_buffer(raw_buffer, None);
2689 })?;
2690
2691 self.shared
2692 .raw
2693 .bind_buffer_memory(raw_buffer, *block.memory(), block.offset())
2694 .map_err(super::map_host_device_oom_and_ioca_err)
2695 .inspect_err(|_| {
2696 self.shared.raw.destroy_buffer(raw_buffer, None);
2697 })?;
2698
2699 if let Some(label) = desc.label {
2700 self.shared.set_object_name(raw_buffer, label);
2701 }
2702
2703 let vk_info = vk::AccelerationStructureCreateInfoKHR::default()
2704 .buffer(raw_buffer)
2705 .offset(0)
2706 .size(desc.size)
2707 .ty(conv::map_acceleration_structure_format(desc.format));
2708
2709 let raw_acceleration_structure = ray_tracing_functions
2710 .acceleration_structure
2711 .create_acceleration_structure(&vk_info, None)
2712 .map_err(super::map_host_oom_and_ioca_err)
2713 .inspect_err(|_| {
2714 self.shared.raw.destroy_buffer(raw_buffer, None);
2715 })?;
2716
2717 if let Some(label) = desc.label {
2718 self.shared
2719 .set_object_name(raw_acceleration_structure, label);
2720 }
2721
2722 let pool = if desc.allow_compaction {
2723 let vk_info = vk::QueryPoolCreateInfo::default()
2724 .query_type(vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR)
2725 .query_count(1);
2726
2727 let raw = self
2728 .shared
2729 .raw
2730 .create_query_pool(&vk_info, None)
2731 .map_err(super::map_host_device_oom_err)
2732 .inspect_err(|_| {
2733 ray_tracing_functions
2734 .acceleration_structure
2735 .destroy_acceleration_structure(raw_acceleration_structure, None);
2736 self.shared.raw.destroy_buffer(raw_buffer, None);
2737 })?;
2738 Some(raw)
2739 } else {
2740 None
2741 };
2742
2743 Ok(super::AccelerationStructure {
2744 raw: raw_acceleration_structure,
2745 buffer: raw_buffer,
2746 block: Mutex::new(block),
2747 compacted_size_query: pool,
2748 })
2749 }
2750 }
2751
2752 unsafe fn destroy_acceleration_structure(
2753 &self,
2754 acceleration_structure: super::AccelerationStructure,
2755 ) {
2756 let ray_tracing_functions = self
2757 .shared
2758 .extension_fns
2759 .ray_tracing
2760 .as_ref()
2761 .expect("Feature `RAY_TRACING` not enabled");
2762
2763 unsafe {
2764 ray_tracing_functions
2765 .acceleration_structure
2766 .destroy_acceleration_structure(acceleration_structure.raw, None);
2767 self.shared
2768 .raw
2769 .destroy_buffer(acceleration_structure.buffer, None);
2770 self.mem_allocator
2771 .lock()
2772 .dealloc(&*self.shared, acceleration_structure.block.into_inner());
2773 if let Some(query) = acceleration_structure.compacted_size_query {
2774 self.shared.raw.destroy_query_pool(query, None)
2775 }
2776 }
2777 }
2778
2779 fn get_internal_counters(&self) -> wgt::HalCounters {
2780 self.counters
2781 .memory_allocations
2782 .set(self.shared.memory_allocations_counter.read());
2783
2784 self.counters.as_ref().clone()
2785 }
2786
2787 fn tlas_instance_to_bytes(&self, instance: TlasInstance) -> Vec<u8> {
2788 const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
2789 let temp = RawTlasInstance {
2790 transform: instance.transform,
2791 custom_data_and_mask: (instance.custom_data & MAX_U24)
2792 | (u32::from(instance.mask) << 24),
2793 shader_binding_table_record_offset_and_flags: 0,
2794 acceleration_structure_reference: instance.blas_address,
2795 };
2796 bytemuck::bytes_of(&temp).to_vec()
2797 }
2798
2799 fn check_if_oom(&self) -> Result<(), crate::DeviceError> {
2800 let Some(threshold) = self
2801 .shared
2802 .instance
2803 .memory_budget_thresholds
2804 .for_device_loss
2805 else {
2806 return Ok(());
2807 };
2808
2809 if !self
2810 .shared
2811 .enabled_extensions
2812 .contains(&ext::memory_budget::NAME)
2813 {
2814 return Ok(());
2815 }
2816
2817 let get_physical_device_properties = self
2818 .shared
2819 .instance
2820 .get_physical_device_properties
2821 .as_ref()
2822 .unwrap();
2823
2824 let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
2825
2826 let mut memory_properties =
2827 vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
2828
2829 unsafe {
2830 get_physical_device_properties.get_physical_device_memory_properties2(
2831 self.shared.physical_device,
2832 &mut memory_properties,
2833 );
2834 }
2835
2836 let memory_properties = memory_properties.memory_properties;
2837
2838 for i in 0..memory_properties.memory_heap_count {
2839 let heap_usage = memory_budget_properties.heap_usage[i as usize];
2840 let heap_budget = memory_budget_properties.heap_budget[i as usize];
2841
2842 if heap_usage >= heap_budget / 100 * threshold as u64 {
2843 return Err(crate::DeviceError::OutOfMemory);
2844 }
2845 }
2846
2847 Ok(())
2848 }
2849}
2850
2851impl super::DeviceShared {
2852 pub(super) fn new_binary_semaphore(
2853 &self,
2854 name: &str,
2855 ) -> Result<vk::Semaphore, crate::DeviceError> {
2856 unsafe {
2857 let semaphore = self
2858 .raw
2859 .create_semaphore(&vk::SemaphoreCreateInfo::default(), None)
2860 .map_err(super::map_host_device_oom_err)?;
2861
2862 self.set_object_name(semaphore, name);
2863
2864 Ok(semaphore)
2865 }
2866 }
2867
2868 pub(super) fn wait_for_fence(
2869 &self,
2870 fence: &super::Fence,
2871 wait_value: crate::FenceValue,
2872 timeout_ns: u64,
2873 ) -> Result<bool, crate::DeviceError> {
2874 profiling::scope!("Device::wait");
2875 match *fence {
2876 super::Fence::TimelineSemaphore(raw) => {
2877 let semaphores = [raw];
2878 let values = [wait_value];
2879 let vk_info = vk::SemaphoreWaitInfo::default()
2880 .semaphores(&semaphores)
2881 .values(&values);
2882 let result = match self.extension_fns.timeline_semaphore {
2883 Some(super::ExtensionFn::Extension(ref ext)) => unsafe {
2884 ext.wait_semaphores(&vk_info, timeout_ns)
2885 },
2886 Some(super::ExtensionFn::Promoted) => unsafe {
2887 self.raw.wait_semaphores(&vk_info, timeout_ns)
2888 },
2889 None => unreachable!(),
2890 };
2891 match result {
2892 Ok(()) => Ok(true),
2893 Err(vk::Result::TIMEOUT) => Ok(false),
2894 Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2895 }
2896 }
2897 super::Fence::FencePool {
2898 last_completed,
2899 ref active,
2900 free: _,
2901 } => {
2902 if wait_value <= last_completed {
2903 Ok(true)
2904 } else {
2905 match active.iter().find(|&&(value, _)| value >= wait_value) {
2906 Some(&(_, raw)) => {
2907 match unsafe { self.raw.wait_for_fences(&[raw], true, timeout_ns) } {
2908 Ok(()) => Ok(true),
2909 Err(vk::Result::TIMEOUT) => Ok(false),
2910 Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2911 }
2912 }
2913 None => {
2914 crate::hal_usage_error(format!(
2915 "no signals reached value {wait_value}"
2916 ));
2917 }
2918 }
2919 }
2920 }
2921 }
2922 }
2923}
2924
2925impl From<gpu_alloc::AllocationError> for crate::DeviceError {
2926 fn from(error: gpu_alloc::AllocationError) -> Self {
2927 use gpu_alloc::AllocationError as Ae;
2928 match error {
2929 Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::TooManyObjects => Self::OutOfMemory,
2930 Ae::NoCompatibleMemoryTypes => crate::hal_usage_error(error),
2931 }
2932 }
2933}
2934impl From<gpu_alloc::MapError> for crate::DeviceError {
2935 fn from(error: gpu_alloc::MapError) -> Self {
2936 use gpu_alloc::MapError as Me;
2937 match error {
2938 Me::OutOfDeviceMemory | Me::OutOfHostMemory | Me::MapFailed => Self::OutOfMemory,
2939 Me::NonHostVisible | Me::AlreadyMapped => crate::hal_usage_error(error),
2940 }
2941 }
2942}
2943impl From<gpu_descriptor::AllocationError> for crate::DeviceError {
2944 fn from(error: gpu_descriptor::AllocationError) -> Self {
2945 use gpu_descriptor::AllocationError as Ae;
2946 match error {
2947 Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::Fragmentation => Self::OutOfMemory,
2948 }
2949 }
2950}
2951
2952fn handle_unexpected(err: vk::Result) -> ! {
2959 panic!("Unexpected Vulkan error: `{err}`")
2960}
2961
2962struct ImageWithoutMemory {
2963 raw: vk::Image,
2964 requirements: vk::MemoryRequirements,
2965 copy_size: crate::CopyExtent,
2966}