1use alloc::{borrow::ToOwned as _, collections::BTreeMap, ffi::CString, sync::Arc, vec::Vec};
2use core::{
3 ffi::CStr,
4 mem::{self, MaybeUninit},
5 num::NonZeroU32,
6 ptr,
7};
8
9use arrayvec::ArrayVec;
10use ash::{ext, khr, vk};
11use hashbrown::hash_map::Entry;
12use parking_lot::Mutex;
13
14use super::{conv, RawTlasInstance};
15use crate::TlasInstance;
16
17impl super::DeviceShared {
18 pub(super) unsafe fn set_object_name(&self, object: impl vk::Handle, name: &str) {
36 let Some(extension) = self.extension_fns.debug_utils.as_ref() else {
37 return;
38 };
39
40 let mut buffer: [u8; 64] = [0u8; 64];
43 let buffer_vec: Vec<u8>;
44
45 let name_bytes = if name.len() < buffer.len() {
47 buffer[..name.len()].copy_from_slice(name.as_bytes());
49 buffer[name.len()] = 0;
51 &buffer[..name.len() + 1]
52 } else {
53 buffer_vec = name
56 .as_bytes()
57 .iter()
58 .cloned()
59 .chain(core::iter::once(0))
60 .collect();
61 &buffer_vec
62 };
63
64 let name = CStr::from_bytes_until_nul(name_bytes).expect("We have added a null byte");
65
66 let _result = unsafe {
67 extension.set_debug_utils_object_name(
68 &vk::DebugUtilsObjectNameInfoEXT::default()
69 .object_handle(object)
70 .object_name(name),
71 )
72 };
73 }
74
75 pub fn make_render_pass(
76 &self,
77 key: super::RenderPassKey,
78 ) -> Result<vk::RenderPass, crate::DeviceError> {
79 Ok(match self.render_passes.lock().entry(key) {
80 Entry::Occupied(e) => *e.get(),
81 Entry::Vacant(e) => {
82 let super::RenderPassKey {
83 ref colors,
84 ref depth_stencil,
85 sample_count,
86 multiview,
87 } = *e.key();
88
89 let mut vk_attachments = Vec::new();
90 let mut color_refs = Vec::with_capacity(colors.len());
91 let mut resolve_refs = Vec::with_capacity(color_refs.capacity());
92 let mut ds_ref = None;
93 let samples = vk::SampleCountFlags::from_raw(sample_count);
94 let unused = vk::AttachmentReference {
95 attachment: vk::ATTACHMENT_UNUSED,
96 layout: vk::ImageLayout::UNDEFINED,
97 };
98 for cat in colors.iter() {
99 let (color_ref, resolve_ref) =
100 if let Some(super::ColorAttachmentKey { base, resolve }) = cat {
101 let super::AttachmentKey {
102 format,
103 layout,
104 ops,
105 } = *base;
106
107 let color_ref = vk::AttachmentReference {
108 attachment: vk_attachments.len() as u32,
109 layout,
110 };
111 vk_attachments.push({
112 let (load_op, store_op) = conv::map_attachment_ops(ops);
113 vk::AttachmentDescription::default()
114 .format(format)
115 .samples(samples)
116 .load_op(load_op)
117 .store_op(store_op)
118 .initial_layout(layout)
119 .final_layout(layout)
120 });
121 let resolve_ref = if let Some(rat) = resolve {
122 let super::AttachmentKey {
123 format,
124 layout,
125 ops,
126 } = *rat;
127
128 let (load_op, store_op) = conv::map_attachment_ops(ops);
129 let vk_attachment = vk::AttachmentDescription::default()
130 .format(format)
131 .samples(vk::SampleCountFlags::TYPE_1)
132 .load_op(load_op)
133 .store_op(store_op)
134 .initial_layout(layout)
135 .final_layout(layout);
136 vk_attachments.push(vk_attachment);
137
138 vk::AttachmentReference {
139 attachment: vk_attachments.len() as u32 - 1,
140 layout,
141 }
142 } else {
143 unused
144 };
145
146 (color_ref, resolve_ref)
147 } else {
148 (unused, unused)
149 };
150
151 color_refs.push(color_ref);
152 resolve_refs.push(resolve_ref);
153 }
154
155 if let Some(ds) = depth_stencil {
156 let super::DepthStencilAttachmentKey {
157 ref base,
158 stencil_ops,
159 } = *ds;
160
161 let super::AttachmentKey {
162 format,
163 layout,
164 ops,
165 } = *base;
166
167 ds_ref = Some(vk::AttachmentReference {
168 attachment: vk_attachments.len() as u32,
169 layout,
170 });
171 let (load_op, store_op) = conv::map_attachment_ops(ops);
172 let (stencil_load_op, stencil_store_op) = conv::map_attachment_ops(stencil_ops);
173 let vk_attachment = vk::AttachmentDescription::default()
174 .format(format)
175 .samples(samples)
176 .load_op(load_op)
177 .store_op(store_op)
178 .stencil_load_op(stencil_load_op)
179 .stencil_store_op(stencil_store_op)
180 .initial_layout(layout)
181 .final_layout(layout);
182 vk_attachments.push(vk_attachment);
183 }
184
185 let vk_subpasses = [{
186 let mut vk_subpass = vk::SubpassDescription::default()
187 .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
188 .color_attachments(&color_refs)
189 .resolve_attachments(&resolve_refs);
190
191 if self
192 .workarounds
193 .contains(super::Workarounds::EMPTY_RESOLVE_ATTACHMENT_LISTS)
194 && resolve_refs.is_empty()
195 {
196 vk_subpass.p_resolve_attachments = ptr::null();
197 }
198
199 if let Some(ref reference) = ds_ref {
200 vk_subpass = vk_subpass.depth_stencil_attachment(reference)
201 }
202 vk_subpass
203 }];
204
205 let mut vk_info = vk::RenderPassCreateInfo::default()
206 .attachments(&vk_attachments)
207 .subpasses(&vk_subpasses);
208
209 let mut multiview_info;
210 let mask;
211 if let Some(multiview) = multiview {
212 assert!(multiview.get() <= 8);
214 assert!(multiview.get() > 1);
215
216 mask = [(1 << multiview.get()) - 1];
220
221 multiview_info = vk::RenderPassMultiviewCreateInfoKHR::default()
223 .view_masks(&mask)
224 .correlation_masks(&mask);
225 vk_info = vk_info.push_next(&mut multiview_info);
226 }
227
228 let raw = unsafe {
229 self.raw
230 .create_render_pass(&vk_info, None)
231 .map_err(super::map_host_device_oom_err)?
232 };
233
234 *e.insert(raw)
235 }
236 })
237 }
238
239 fn make_memory_ranges<'a, I: 'a + Iterator<Item = crate::MemoryRange>>(
240 &self,
241 buffer: &'a super::Buffer,
242 ranges: I,
243 ) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
244 let block = buffer.block.as_ref()?.lock();
245 let mask = self.private_caps.non_coherent_map_mask;
246 Some(ranges.map(move |range| {
247 vk::MappedMemoryRange::default()
248 .memory(*block.memory())
249 .offset((block.offset() + range.start) & !mask)
250 .size((range.end - range.start + mask) & !mask)
251 }))
252 }
253}
254
255impl gpu_alloc::MemoryDevice<vk::DeviceMemory> for super::DeviceShared {
256 unsafe fn allocate_memory(
257 &self,
258 size: u64,
259 memory_type: u32,
260 flags: gpu_alloc::AllocationFlags,
261 ) -> Result<vk::DeviceMemory, gpu_alloc::OutOfMemory> {
262 let mut info = vk::MemoryAllocateInfo::default()
263 .allocation_size(size)
264 .memory_type_index(memory_type);
265
266 let mut info_flags;
267
268 if flags.contains(gpu_alloc::AllocationFlags::DEVICE_ADDRESS) {
269 info_flags = vk::MemoryAllocateFlagsInfo::default()
270 .flags(vk::MemoryAllocateFlags::DEVICE_ADDRESS);
271 info = info.push_next(&mut info_flags);
272 }
273
274 match unsafe { self.raw.allocate_memory(&info, None) } {
275 Ok(memory) => {
276 self.memory_allocations_counter.add(1);
277 Ok(memory)
278 }
279 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
280 Err(gpu_alloc::OutOfMemory::OutOfDeviceMemory)
281 }
282 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
283 Err(gpu_alloc::OutOfMemory::OutOfHostMemory)
284 }
285 Err(err) => handle_unexpected(err),
290 }
291 }
292
293 unsafe fn deallocate_memory(&self, memory: vk::DeviceMemory) {
294 self.memory_allocations_counter.sub(1);
295
296 unsafe { self.raw.free_memory(memory, None) };
297 }
298
299 unsafe fn map_memory(
300 &self,
301 memory: &mut vk::DeviceMemory,
302 offset: u64,
303 size: u64,
304 ) -> Result<ptr::NonNull<u8>, gpu_alloc::DeviceMapError> {
305 match unsafe {
306 self.raw
307 .map_memory(*memory, offset, size, vk::MemoryMapFlags::empty())
308 } {
309 Ok(ptr) => Ok(ptr::NonNull::new(ptr.cast::<u8>())
310 .expect("Pointer to memory mapping must not be null")),
311 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
312 Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory)
313 }
314 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
315 Err(gpu_alloc::DeviceMapError::OutOfHostMemory)
316 }
317 Err(vk::Result::ERROR_MEMORY_MAP_FAILED) => Err(gpu_alloc::DeviceMapError::MapFailed),
318 Err(err) => handle_unexpected(err),
319 }
320 }
321
322 unsafe fn unmap_memory(&self, memory: &mut vk::DeviceMemory) {
323 unsafe { self.raw.unmap_memory(*memory) };
324 }
325
326 unsafe fn invalidate_memory_ranges(
327 &self,
328 _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
329 ) -> Result<(), gpu_alloc::OutOfMemory> {
330 unimplemented!()
332 }
333
334 unsafe fn flush_memory_ranges(
335 &self,
336 _ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
337 ) -> Result<(), gpu_alloc::OutOfMemory> {
338 unimplemented!()
340 }
341}
342
343impl
344 gpu_descriptor::DescriptorDevice<vk::DescriptorSetLayout, vk::DescriptorPool, vk::DescriptorSet>
345 for super::DeviceShared
346{
347 unsafe fn create_descriptor_pool(
348 &self,
349 descriptor_count: &gpu_descriptor::DescriptorTotalCount,
350 max_sets: u32,
351 flags: gpu_descriptor::DescriptorPoolCreateFlags,
352 ) -> Result<vk::DescriptorPool, gpu_descriptor::CreatePoolError> {
353 let unfiltered_counts = [
355 (vk::DescriptorType::SAMPLER, descriptor_count.sampler),
356 (
357 vk::DescriptorType::SAMPLED_IMAGE,
358 descriptor_count.sampled_image,
359 ),
360 (
361 vk::DescriptorType::STORAGE_IMAGE,
362 descriptor_count.storage_image,
363 ),
364 (
365 vk::DescriptorType::UNIFORM_BUFFER,
366 descriptor_count.uniform_buffer,
367 ),
368 (
369 vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC,
370 descriptor_count.uniform_buffer_dynamic,
371 ),
372 (
373 vk::DescriptorType::STORAGE_BUFFER,
374 descriptor_count.storage_buffer,
375 ),
376 (
377 vk::DescriptorType::STORAGE_BUFFER_DYNAMIC,
378 descriptor_count.storage_buffer_dynamic,
379 ),
380 (
381 vk::DescriptorType::ACCELERATION_STRUCTURE_KHR,
382 descriptor_count.acceleration_structure,
383 ),
384 ];
385
386 let filtered_counts = unfiltered_counts
387 .iter()
388 .cloned()
389 .filter(|&(_, count)| count != 0)
390 .map(|(ty, count)| vk::DescriptorPoolSize {
391 ty,
392 descriptor_count: count,
393 })
394 .collect::<ArrayVec<_, 8>>();
395
396 let mut vk_flags =
397 if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND) {
398 vk::DescriptorPoolCreateFlags::UPDATE_AFTER_BIND
399 } else {
400 vk::DescriptorPoolCreateFlags::empty()
401 };
402 if flags.contains(gpu_descriptor::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET) {
403 vk_flags |= vk::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET;
404 }
405 let vk_info = vk::DescriptorPoolCreateInfo::default()
406 .max_sets(max_sets)
407 .flags(vk_flags)
408 .pool_sizes(&filtered_counts);
409
410 match unsafe { self.raw.create_descriptor_pool(&vk_info, None) } {
411 Ok(pool) => Ok(pool),
412 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
413 Err(gpu_descriptor::CreatePoolError::OutOfHostMemory)
414 }
415 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
416 Err(gpu_descriptor::CreatePoolError::OutOfDeviceMemory)
417 }
418 Err(vk::Result::ERROR_FRAGMENTATION) => {
419 Err(gpu_descriptor::CreatePoolError::Fragmentation)
420 }
421 Err(err) => handle_unexpected(err),
422 }
423 }
424
425 unsafe fn destroy_descriptor_pool(&self, pool: vk::DescriptorPool) {
426 unsafe { self.raw.destroy_descriptor_pool(pool, None) }
427 }
428
429 unsafe fn alloc_descriptor_sets<'a>(
430 &self,
431 pool: &mut vk::DescriptorPool,
432 layouts: impl ExactSizeIterator<Item = &'a vk::DescriptorSetLayout>,
433 sets: &mut impl Extend<vk::DescriptorSet>,
434 ) -> Result<(), gpu_descriptor::DeviceAllocationError> {
435 let result = unsafe {
436 self.raw.allocate_descriptor_sets(
437 &vk::DescriptorSetAllocateInfo::default()
438 .descriptor_pool(*pool)
439 .set_layouts(
440 &smallvec::SmallVec::<[vk::DescriptorSetLayout; 32]>::from_iter(
441 layouts.cloned(),
442 ),
443 ),
444 )
445 };
446
447 match result {
448 Ok(vk_sets) => {
449 sets.extend(vk_sets);
450 Ok(())
451 }
452 Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY)
453 | Err(vk::Result::ERROR_OUT_OF_POOL_MEMORY) => {
454 Err(gpu_descriptor::DeviceAllocationError::OutOfHostMemory)
455 }
456 Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
457 Err(gpu_descriptor::DeviceAllocationError::OutOfDeviceMemory)
458 }
459 Err(vk::Result::ERROR_FRAGMENTED_POOL) => {
460 Err(gpu_descriptor::DeviceAllocationError::FragmentedPool)
461 }
462 Err(err) => handle_unexpected(err),
463 }
464 }
465
466 unsafe fn dealloc_descriptor_sets<'a>(
467 &self,
468 pool: &mut vk::DescriptorPool,
469 sets: impl Iterator<Item = vk::DescriptorSet>,
470 ) {
471 let result = unsafe {
472 self.raw.free_descriptor_sets(
473 *pool,
474 &smallvec::SmallVec::<[vk::DescriptorSet; 32]>::from_iter(sets),
475 )
476 };
477 match result {
478 Ok(()) => {}
479 Err(err) => handle_unexpected(err),
480 }
481 }
482}
483
484struct CompiledStage {
485 create_info: vk::PipelineShaderStageCreateInfo<'static>,
486 _entry_point: CString,
487 temp_raw_module: Option<vk::ShaderModule>,
488}
489
490impl super::Device {
491 pub(super) unsafe fn create_swapchain(
492 &self,
493 surface: &super::Surface,
494 config: &crate::SurfaceConfiguration,
495 provided_old_swapchain: Option<super::Swapchain>,
496 ) -> Result<super::Swapchain, crate::SurfaceError> {
497 profiling::scope!("Device::create_swapchain");
498 let functor = khr::swapchain::Device::new(&surface.instance.raw, &self.shared.raw);
499
500 let old_swapchain = match provided_old_swapchain {
501 Some(osc) => osc.raw,
502 None => vk::SwapchainKHR::null(),
503 };
504
505 let color_space = if config.format == wgt::TextureFormat::Rgba16Float {
506 vk::ColorSpaceKHR::EXTENDED_SRGB_LINEAR_EXT
509 } else {
510 vk::ColorSpaceKHR::SRGB_NONLINEAR
511 };
512
513 let original_format = self.shared.private_caps.map_texture_format(config.format);
514 let mut raw_flags = vk::SwapchainCreateFlagsKHR::empty();
515 let mut raw_view_formats: Vec<vk::Format> = vec![];
516 if !config.view_formats.is_empty() {
517 raw_flags |= vk::SwapchainCreateFlagsKHR::MUTABLE_FORMAT;
518 raw_view_formats = config
519 .view_formats
520 .iter()
521 .map(|f| self.shared.private_caps.map_texture_format(*f))
522 .collect();
523 raw_view_formats.push(original_format);
524 }
525
526 let mut info = vk::SwapchainCreateInfoKHR::default()
527 .flags(raw_flags)
528 .surface(surface.raw)
529 .min_image_count(config.maximum_frame_latency + 1) .image_format(original_format)
531 .image_color_space(color_space)
532 .image_extent(vk::Extent2D {
533 width: config.extent.width,
534 height: config.extent.height,
535 })
536 .image_array_layers(config.extent.depth_or_array_layers)
537 .image_usage(conv::map_texture_usage(config.usage))
538 .image_sharing_mode(vk::SharingMode::EXCLUSIVE)
539 .pre_transform(vk::SurfaceTransformFlagsKHR::IDENTITY)
540 .composite_alpha(conv::map_composite_alpha_mode(config.composite_alpha_mode))
541 .present_mode(conv::map_present_mode(config.present_mode))
542 .clipped(true)
543 .old_swapchain(old_swapchain);
544
545 let mut format_list_info = vk::ImageFormatListCreateInfo::default();
546 if !raw_view_formats.is_empty() {
547 format_list_info = format_list_info.view_formats(&raw_view_formats);
548 info = info.push_next(&mut format_list_info);
549 }
550
551 let result = {
552 profiling::scope!("vkCreateSwapchainKHR");
553 unsafe { functor.create_swapchain(&info, None) }
554 };
555
556 if old_swapchain != vk::SwapchainKHR::null() {
558 unsafe { functor.destroy_swapchain(old_swapchain, None) }
559 }
560
561 let raw = match result {
562 Ok(swapchain) => swapchain,
563 Err(error) => {
564 return Err(match error {
565 vk::Result::ERROR_SURFACE_LOST_KHR
566 | vk::Result::ERROR_INITIALIZATION_FAILED => crate::SurfaceError::Lost,
567 vk::Result::ERROR_NATIVE_WINDOW_IN_USE_KHR => {
568 crate::SurfaceError::Other("Native window is in use")
569 }
570 other => super::map_host_device_oom_and_lost_err(other).into(),
573 });
574 }
575 };
576
577 let images =
578 unsafe { functor.get_swapchain_images(raw) }.map_err(super::map_host_device_oom_err)?;
579
580 let acquire_semaphores = (0..=images.len())
583 .map(|i| {
584 super::SwapchainAcquireSemaphore::new(&self.shared, i)
585 .map(Mutex::new)
586 .map(Arc::new)
587 })
588 .collect::<Result<Vec<_>, _>>()?;
589
590 let present_semaphores = (0..=images.len())
591 .map(|i| Arc::new(Mutex::new(super::SwapchainPresentSemaphores::new(i))))
592 .collect::<Vec<_>>();
593
594 Ok(super::Swapchain {
595 raw,
596 functor,
597 device: Arc::clone(&self.shared),
598 images,
599 config: config.clone(),
600 acquire_semaphores,
601 next_acquire_index: 0,
602 present_semaphores,
603 next_present_time: None,
604 })
605 }
606
607 pub unsafe fn texture_from_raw(
614 &self,
615 vk_image: vk::Image,
616 desc: &crate::TextureDescriptor,
617 drop_callback: Option<crate::DropCallback>,
618 ) -> super::Texture {
619 let mut raw_flags = vk::ImageCreateFlags::empty();
620 let mut view_formats = vec![];
621 for tf in desc.view_formats.iter() {
622 if *tf == desc.format {
623 continue;
624 }
625 view_formats.push(*tf);
626 }
627 if !view_formats.is_empty() {
628 raw_flags |=
629 vk::ImageCreateFlags::MUTABLE_FORMAT | vk::ImageCreateFlags::EXTENDED_USAGE;
630 view_formats.push(desc.format)
631 }
632 if desc.format.is_multi_planar_format() {
633 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
634 }
635
636 let identity = self.shared.texture_identity_factory.next();
637
638 let drop_guard = crate::DropGuard::from_option(drop_callback);
639
640 super::Texture {
641 raw: vk_image,
642 drop_guard,
643 external_memory: None,
644 block: None,
645 format: desc.format,
646 copy_size: desc.copy_extent(),
647 identity,
648 }
649 }
650
651 #[cfg(windows)]
652 fn find_memory_type_index(
653 &self,
654 type_bits_req: u32,
655 flags_req: vk::MemoryPropertyFlags,
656 ) -> Option<usize> {
657 let mem_properties = unsafe {
658 self.shared
659 .instance
660 .raw
661 .get_physical_device_memory_properties(self.shared.physical_device)
662 };
663
664 for (i, mem_ty) in mem_properties.memory_types_as_slice().iter().enumerate() {
666 let types_bits = 1 << i;
667 let is_required_memory_type = type_bits_req & types_bits != 0;
668 let has_required_properties = mem_ty.property_flags & flags_req == flags_req;
669 if is_required_memory_type && has_required_properties {
670 return Some(i);
671 }
672 }
673
674 None
675 }
676
677 fn create_image_without_memory(
678 &self,
679 desc: &crate::TextureDescriptor,
680 external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
681 ) -> Result<ImageWithoutMemory, crate::DeviceError> {
682 let copy_size = desc.copy_extent();
683
684 let mut raw_flags = vk::ImageCreateFlags::empty();
685 if desc.dimension == wgt::TextureDimension::D3
686 && desc.usage.contains(wgt::TextureUses::COLOR_TARGET)
687 {
688 raw_flags |= vk::ImageCreateFlags::TYPE_2D_ARRAY_COMPATIBLE;
689 }
690 if desc.is_cube_compatible() {
691 raw_flags |= vk::ImageCreateFlags::CUBE_COMPATIBLE;
692 }
693
694 let original_format = self.shared.private_caps.map_texture_format(desc.format);
695 let mut vk_view_formats = vec![];
696 if !desc.view_formats.is_empty() {
697 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
698
699 if self.shared.private_caps.image_format_list {
700 vk_view_formats = desc
701 .view_formats
702 .iter()
703 .map(|f| self.shared.private_caps.map_texture_format(*f))
704 .collect();
705 vk_view_formats.push(original_format)
706 }
707 }
708 if desc.format.is_multi_planar_format() {
709 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
710 }
711
712 let mut vk_info = vk::ImageCreateInfo::default()
713 .flags(raw_flags)
714 .image_type(conv::map_texture_dimension(desc.dimension))
715 .format(original_format)
716 .extent(conv::map_copy_extent(©_size))
717 .mip_levels(desc.mip_level_count)
718 .array_layers(desc.array_layer_count())
719 .samples(vk::SampleCountFlags::from_raw(desc.sample_count))
720 .tiling(vk::ImageTiling::OPTIMAL)
721 .usage(conv::map_texture_usage(desc.usage))
722 .sharing_mode(vk::SharingMode::EXCLUSIVE)
723 .initial_layout(vk::ImageLayout::UNDEFINED);
724
725 let mut format_list_info = vk::ImageFormatListCreateInfo::default();
726 if !vk_view_formats.is_empty() {
727 format_list_info = format_list_info.view_formats(&vk_view_formats);
728 vk_info = vk_info.push_next(&mut format_list_info);
729 }
730
731 if let Some(ext_info) = external_memory_image_create_info {
732 vk_info = vk_info.push_next(ext_info);
733 }
734
735 let raw = unsafe { self.shared.raw.create_image(&vk_info, None) }.map_err(map_err)?;
736 fn map_err(err: vk::Result) -> crate::DeviceError {
737 super::map_host_device_oom_and_ioca_err(err)
740 }
741 let req = unsafe { self.shared.raw.get_image_memory_requirements(raw) };
742
743 Ok(ImageWithoutMemory {
744 raw,
745 requirements: req,
746 copy_size,
747 })
748 }
749
750 #[cfg(windows)]
756 pub unsafe fn texture_from_d3d11_shared_handle(
757 &self,
758 d3d11_shared_handle: windows::Win32::Foundation::HANDLE,
759 desc: &crate::TextureDescriptor,
760 ) -> Result<super::Texture, crate::DeviceError> {
761 if !self
762 .shared
763 .features
764 .contains(wgt::Features::VULKAN_EXTERNAL_MEMORY_WIN32)
765 {
766 log::error!("Vulkan driver does not support VK_KHR_external_memory_win32");
767 return Err(crate::DeviceError::Unexpected);
768 }
769
770 let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
771 .handle_types(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE);
772
773 let image =
774 self.create_image_without_memory(desc, Some(&mut external_memory_image_info))?;
775
776 let mut dedicated_allocate_info =
779 vk::MemoryDedicatedAllocateInfo::default().image(image.raw);
780
781 let mut import_memory_info = vk::ImportMemoryWin32HandleInfoKHR::default()
782 .handle_type(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE)
783 .handle(d3d11_shared_handle.0 as _);
784 #[allow(clippy::unnecessary_mut_passed)]
786 {
787 import_memory_info.p_next = <*const _>::cast(&mut dedicated_allocate_info);
788 }
789
790 let mem_type_index = self
791 .find_memory_type_index(
792 image.requirements.memory_type_bits,
793 vk::MemoryPropertyFlags::DEVICE_LOCAL,
794 )
795 .ok_or(crate::DeviceError::Unexpected)?;
796
797 let memory_allocate_info = vk::MemoryAllocateInfo::default()
798 .allocation_size(image.requirements.size)
799 .memory_type_index(mem_type_index as _)
800 .push_next(&mut import_memory_info);
801 let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
802 .map_err(super::map_host_device_oom_err)?;
803
804 unsafe { self.shared.raw.bind_image_memory(image.raw, memory, 0) }
805 .map_err(super::map_host_device_oom_err)?;
806
807 if let Some(label) = desc.label {
808 unsafe { self.shared.set_object_name(image.raw, label) };
809 }
810
811 let identity = self.shared.texture_identity_factory.next();
812
813 self.counters.textures.add(1);
814
815 Ok(super::Texture {
816 raw: image.raw,
817 drop_guard: None,
818 external_memory: Some(memory),
819 block: None,
820 format: desc.format,
821 copy_size: image.copy_size,
822 identity,
823 })
824 }
825
826 fn create_shader_module_impl(
827 &self,
828 spv: &[u32],
829 label: &crate::Label<'_>,
830 ) -> Result<vk::ShaderModule, crate::DeviceError> {
831 let vk_info = vk::ShaderModuleCreateInfo::default()
832 .flags(vk::ShaderModuleCreateFlags::empty())
833 .code(spv);
834
835 let raw = unsafe {
836 profiling::scope!("vkCreateShaderModule");
837 self.shared
838 .raw
839 .create_shader_module(&vk_info, None)
840 .map_err(map_err)?
841 };
842 fn map_err(err: vk::Result) -> crate::DeviceError {
843 super::map_host_device_oom_err(err)
846 }
847
848 if let Some(label) = label {
849 unsafe { self.shared.set_object_name(raw, label) };
850 }
851
852 Ok(raw)
853 }
854
855 fn compile_stage(
856 &self,
857 stage: &crate::ProgrammableStage<super::ShaderModule>,
858 naga_stage: naga::ShaderStage,
859 binding_map: &naga::back::spv::BindingMap,
860 ) -> Result<CompiledStage, crate::PipelineError> {
861 let stage_flags = crate::auxil::map_naga_stage(naga_stage);
862 let vk_module = match *stage.module {
863 super::ShaderModule::Raw(raw) => raw,
864 super::ShaderModule::Intermediate {
865 ref naga_shader,
866 runtime_checks,
867 } => {
868 let pipeline_options = naga::back::spv::PipelineOptions {
869 entry_point: stage.entry_point.to_owned(),
870 shader_stage: naga_stage,
871 };
872 let needs_temp_options = !runtime_checks.bounds_checks
873 || !runtime_checks.force_loop_bounding
874 || !binding_map.is_empty()
875 || naga_shader.debug_source.is_some()
876 || !stage.zero_initialize_workgroup_memory;
877 let mut temp_options;
878 let options = if needs_temp_options {
879 temp_options = self.naga_options.clone();
880 if !runtime_checks.bounds_checks {
881 temp_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
882 index: naga::proc::BoundsCheckPolicy::Unchecked,
883 buffer: naga::proc::BoundsCheckPolicy::Unchecked,
884 image_load: naga::proc::BoundsCheckPolicy::Unchecked,
885 binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
886 };
887 }
888 if !runtime_checks.force_loop_bounding {
889 temp_options.force_loop_bounding = false;
890 }
891 if !binding_map.is_empty() {
892 temp_options.binding_map = binding_map.clone();
893 }
894
895 if let Some(ref debug) = naga_shader.debug_source {
896 temp_options.debug_info = Some(naga::back::spv::DebugInfo {
897 source_code: &debug.source_code,
898 file_name: debug.file_name.as_ref().into(),
899 language: naga::back::spv::SourceLanguage::WGSL,
900 })
901 }
902 if !stage.zero_initialize_workgroup_memory {
903 temp_options.zero_initialize_workgroup_memory =
904 naga::back::spv::ZeroInitializeWorkgroupMemoryMode::None;
905 }
906
907 &temp_options
908 } else {
909 &self.naga_options
910 };
911
912 let (module, info) = naga::back::pipeline_constants::process_overrides(
913 &naga_shader.module,
914 &naga_shader.info,
915 Some((naga_stage, stage.entry_point)),
916 stage.constants,
917 )
918 .map_err(|e| {
919 crate::PipelineError::PipelineConstants(stage_flags, format!("{e}"))
920 })?;
921
922 let spv = {
923 profiling::scope!("naga::spv::write_vec");
924 naga::back::spv::write_vec(&module, &info, options, Some(&pipeline_options))
925 }
926 .map_err(|e| crate::PipelineError::Linkage(stage_flags, format!("{e}")))?;
927 self.create_shader_module_impl(&spv, &None)?
928 }
929 };
930
931 let mut flags = vk::PipelineShaderStageCreateFlags::empty();
932 if self.shared.features.contains(wgt::Features::SUBGROUP) {
933 flags |= vk::PipelineShaderStageCreateFlags::ALLOW_VARYING_SUBGROUP_SIZE
934 }
935
936 let entry_point = CString::new(stage.entry_point).unwrap();
937 let mut create_info = vk::PipelineShaderStageCreateInfo::default()
938 .flags(flags)
939 .stage(conv::map_shader_stage(stage_flags))
940 .module(vk_module);
941
942 create_info.p_name = entry_point.as_ptr();
944
945 Ok(CompiledStage {
946 create_info,
947 _entry_point: entry_point,
948 temp_raw_module: match *stage.module {
949 super::ShaderModule::Raw(_) => None,
950 super::ShaderModule::Intermediate { .. } => Some(vk_module),
951 },
952 })
953 }
954
955 pub fn queue_family_index(&self) -> u32 {
961 self.shared.family_index
962 }
963
964 pub fn queue_index(&self) -> u32 {
965 self.shared.queue_index
966 }
967
968 pub fn raw_device(&self) -> &ash::Device {
969 &self.shared.raw
970 }
971
972 pub fn raw_physical_device(&self) -> vk::PhysicalDevice {
973 self.shared.physical_device
974 }
975
976 pub fn raw_queue(&self) -> vk::Queue {
977 self.shared.raw_queue
978 }
979
980 pub fn enabled_device_extensions(&self) -> &[&'static CStr] {
981 &self.shared.enabled_extensions
982 }
983
984 pub fn shared_instance(&self) -> &super::InstanceShared {
985 &self.shared.instance
986 }
987
988 fn error_if_would_oom_on_resource_allocation(
989 &self,
990 needs_host_access: bool,
991 size: u64,
992 ) -> Result<(), crate::DeviceError> {
993 let Some(threshold) = self
994 .shared
995 .instance
996 .memory_budget_thresholds
997 .for_resource_creation
998 else {
999 return Ok(());
1000 };
1001
1002 if !self
1003 .shared
1004 .enabled_extensions
1005 .contains(&ext::memory_budget::NAME)
1006 {
1007 return Ok(());
1008 }
1009
1010 let get_physical_device_properties = self
1011 .shared
1012 .instance
1013 .get_physical_device_properties
1014 .as_ref()
1015 .unwrap();
1016
1017 let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
1018
1019 let mut memory_properties =
1020 vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
1021
1022 unsafe {
1023 get_physical_device_properties.get_physical_device_memory_properties2(
1024 self.shared.physical_device,
1025 &mut memory_properties,
1026 );
1027 }
1028
1029 let mut host_visible_heaps = [false; vk::MAX_MEMORY_HEAPS];
1030 let mut device_local_heaps = [false; vk::MAX_MEMORY_HEAPS];
1031
1032 let memory_properties = memory_properties.memory_properties;
1033
1034 for i in 0..memory_properties.memory_type_count {
1035 let memory_type = memory_properties.memory_types[i as usize];
1036 let flags = memory_type.property_flags;
1037
1038 if flags.intersects(
1039 vk::MemoryPropertyFlags::LAZILY_ALLOCATED | vk::MemoryPropertyFlags::PROTECTED,
1040 ) {
1041 continue; }
1043
1044 if flags.contains(vk::MemoryPropertyFlags::HOST_VISIBLE) {
1045 host_visible_heaps[memory_type.heap_index as usize] = true;
1046 }
1047
1048 if flags.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL) {
1049 device_local_heaps[memory_type.heap_index as usize] = true;
1050 }
1051 }
1052
1053 let heaps = if needs_host_access {
1054 host_visible_heaps
1055 } else {
1056 device_local_heaps
1057 };
1058
1059 for (i, check) in heaps.iter().enumerate() {
1064 if !check {
1065 continue;
1066 }
1067
1068 let heap_usage = memory_budget_properties.heap_usage[i];
1069 let heap_budget = memory_budget_properties.heap_budget[i];
1070
1071 if heap_usage + size >= heap_budget / 100 * threshold as u64 {
1072 return Err(crate::DeviceError::OutOfMemory);
1073 }
1074 }
1075
1076 Ok(())
1077 }
1078}
1079
1080impl crate::Device for super::Device {
1081 type A = super::Api;
1082
1083 unsafe fn create_buffer(
1084 &self,
1085 desc: &crate::BufferDescriptor,
1086 ) -> Result<super::Buffer, crate::DeviceError> {
1087 let vk_info = vk::BufferCreateInfo::default()
1088 .size(desc.size)
1089 .usage(conv::map_buffer_usage(desc.usage))
1090 .sharing_mode(vk::SharingMode::EXCLUSIVE);
1091
1092 let raw = unsafe {
1093 self.shared
1094 .raw
1095 .create_buffer(&vk_info, None)
1096 .map_err(super::map_host_device_oom_and_ioca_err)?
1097 };
1098 let req = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
1099
1100 let mut alloc_usage = if desc
1101 .usage
1102 .intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE)
1103 {
1104 let mut flags = gpu_alloc::UsageFlags::HOST_ACCESS;
1105 flags.set(
1107 gpu_alloc::UsageFlags::DOWNLOAD,
1108 desc.usage.contains(wgt::BufferUses::MAP_READ),
1109 );
1110 flags.set(
1111 gpu_alloc::UsageFlags::UPLOAD,
1112 desc.usage.contains(wgt::BufferUses::MAP_WRITE),
1113 );
1114 flags
1115 } else {
1116 gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS
1117 };
1118 alloc_usage.set(
1119 gpu_alloc::UsageFlags::TRANSIENT,
1120 desc.memory_flags.contains(crate::MemoryFlags::TRANSIENT),
1121 );
1122
1123 let needs_host_access = alloc_usage.contains(gpu_alloc::UsageFlags::HOST_ACCESS);
1124
1125 self.error_if_would_oom_on_resource_allocation(needs_host_access, req.size)
1126 .inspect_err(|_| {
1127 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1128 })?;
1129
1130 let alignment_mask = req.alignment - 1;
1131
1132 let block = unsafe {
1133 self.mem_allocator.lock().alloc(
1134 &*self.shared,
1135 gpu_alloc::Request {
1136 size: req.size,
1137 align_mask: alignment_mask,
1138 usage: alloc_usage,
1139 memory_types: req.memory_type_bits & self.valid_ash_memory_types,
1140 },
1141 )
1142 }
1143 .inspect_err(|_| {
1144 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1145 })?;
1146
1147 unsafe {
1148 self.shared
1149 .raw
1150 .bind_buffer_memory(raw, *block.memory(), block.offset())
1151 }
1152 .map_err(super::map_host_device_oom_and_ioca_err)
1153 .inspect_err(|_| {
1154 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1155 })?;
1156
1157 if let Some(label) = desc.label {
1158 unsafe { self.shared.set_object_name(raw, label) };
1159 }
1160
1161 self.counters.buffer_memory.add(block.size() as isize);
1162 self.counters.buffers.add(1);
1163
1164 Ok(super::Buffer {
1165 raw,
1166 block: Some(Mutex::new(super::BufferMemoryBacking::Managed(block))),
1167 })
1168 }
1169 unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
1170 unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1171 if let Some(block) = buffer.block {
1172 let block = block.into_inner();
1173 self.counters.buffer_memory.sub(block.size() as isize);
1174 match block {
1175 super::BufferMemoryBacking::Managed(block) => unsafe {
1176 self.mem_allocator.lock().dealloc(&*self.shared, block)
1177 },
1178 super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
1179 self.shared.raw.free_memory(memory, None);
1180 },
1181 }
1182 }
1183
1184 self.counters.buffers.sub(1);
1185 }
1186
1187 unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) {
1188 self.counters.buffers.add(1);
1189 }
1190
1191 unsafe fn map_buffer(
1192 &self,
1193 buffer: &super::Buffer,
1194 range: crate::MemoryRange,
1195 ) -> Result<crate::BufferMapping, crate::DeviceError> {
1196 if let Some(ref block) = buffer.block {
1197 let size = range.end - range.start;
1198 let mut block = block.lock();
1199 if let super::BufferMemoryBacking::Managed(ref mut block) = *block {
1200 let ptr = unsafe { block.map(&*self.shared, range.start, size as usize)? };
1201 let is_coherent = block
1202 .props()
1203 .contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT);
1204 Ok(crate::BufferMapping { ptr, is_coherent })
1205 } else {
1206 crate::hal_usage_error("tried to map externally created buffer")
1207 }
1208 } else {
1209 crate::hal_usage_error("tried to map external buffer")
1210 }
1211 }
1212 unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
1213 if let Some(ref block) = buffer.block {
1214 match &mut *block.lock() {
1215 super::BufferMemoryBacking::Managed(block) => unsafe { block.unmap(&*self.shared) },
1216 super::BufferMemoryBacking::VulkanMemory { .. } => {
1217 crate::hal_usage_error("tried to unmap externally created buffer")
1218 }
1219 };
1220 } else {
1221 crate::hal_usage_error("tried to unmap external buffer")
1222 }
1223 }
1224
1225 unsafe fn flush_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1226 where
1227 I: Iterator<Item = crate::MemoryRange>,
1228 {
1229 if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1230 unsafe {
1231 self.shared
1232 .raw
1233 .flush_mapped_memory_ranges(
1234 &smallvec::SmallVec::<[vk::MappedMemoryRange; 32]>::from_iter(vk_ranges),
1235 )
1236 }
1237 .unwrap();
1238 }
1239 }
1240 unsafe fn invalidate_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1241 where
1242 I: Iterator<Item = crate::MemoryRange>,
1243 {
1244 if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1245 unsafe {
1246 self.shared
1247 .raw
1248 .invalidate_mapped_memory_ranges(&smallvec::SmallVec::<
1249 [vk::MappedMemoryRange; 32],
1250 >::from_iter(vk_ranges))
1251 }
1252 .unwrap();
1253 }
1254 }
1255
1256 unsafe fn create_texture(
1257 &self,
1258 desc: &crate::TextureDescriptor,
1259 ) -> Result<super::Texture, crate::DeviceError> {
1260 let image = self.create_image_without_memory(desc, None)?;
1261
1262 self.error_if_would_oom_on_resource_allocation(false, image.requirements.size)
1263 .inspect_err(|_| {
1264 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1265 })?;
1266
1267 let block = unsafe {
1268 self.mem_allocator.lock().alloc(
1269 &*self.shared,
1270 gpu_alloc::Request {
1271 size: image.requirements.size,
1272 align_mask: image.requirements.alignment - 1,
1273 usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
1274 memory_types: image.requirements.memory_type_bits & self.valid_ash_memory_types,
1275 },
1276 )
1277 }
1278 .inspect_err(|_| {
1279 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1280 })?;
1281
1282 self.counters.texture_memory.add(block.size() as isize);
1283
1284 unsafe {
1285 self.shared
1286 .raw
1287 .bind_image_memory(image.raw, *block.memory(), block.offset())
1288 }
1289 .map_err(super::map_host_device_oom_err)
1290 .inspect_err(|_| {
1291 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1292 })?;
1293
1294 if let Some(label) = desc.label {
1295 unsafe { self.shared.set_object_name(image.raw, label) };
1296 }
1297
1298 let identity = self.shared.texture_identity_factory.next();
1299
1300 self.counters.textures.add(1);
1301
1302 Ok(super::Texture {
1303 raw: image.raw,
1304 drop_guard: None,
1305 external_memory: None,
1306 block: Some(block),
1307 format: desc.format,
1308 copy_size: image.copy_size,
1309 identity,
1310 })
1311 }
1312 unsafe fn destroy_texture(&self, texture: super::Texture) {
1313 if texture.drop_guard.is_none() {
1314 unsafe { self.shared.raw.destroy_image(texture.raw, None) };
1315 }
1316 if let Some(memory) = texture.external_memory {
1317 unsafe { self.shared.raw.free_memory(memory, None) };
1318 }
1319 if let Some(block) = texture.block {
1320 self.counters.texture_memory.sub(block.size() as isize);
1321
1322 unsafe { self.mem_allocator.lock().dealloc(&*self.shared, block) };
1323 }
1324
1325 self.counters.textures.sub(1);
1326 }
1327
1328 unsafe fn add_raw_texture(&self, _texture: &super::Texture) {
1329 self.counters.textures.add(1);
1330 }
1331
1332 unsafe fn create_texture_view(
1333 &self,
1334 texture: &super::Texture,
1335 desc: &crate::TextureViewDescriptor,
1336 ) -> Result<super::TextureView, crate::DeviceError> {
1337 let subresource_range = conv::map_subresource_range(&desc.range, texture.format);
1338 let raw_format = self.shared.private_caps.map_texture_format(desc.format);
1339 let mut vk_info = vk::ImageViewCreateInfo::default()
1340 .flags(vk::ImageViewCreateFlags::empty())
1341 .image(texture.raw)
1342 .view_type(conv::map_view_dimension(desc.dimension))
1343 .format(raw_format)
1344 .subresource_range(subresource_range);
1345 let layers =
1346 NonZeroU32::new(subresource_range.layer_count).expect("Unexpected zero layer count");
1347
1348 let mut image_view_info;
1349 if self.shared.private_caps.image_view_usage && !desc.usage.is_empty() {
1350 image_view_info =
1351 vk::ImageViewUsageCreateInfo::default().usage(conv::map_texture_usage(desc.usage));
1352 vk_info = vk_info.push_next(&mut image_view_info);
1353 }
1354
1355 let raw = unsafe { self.shared.raw.create_image_view(&vk_info, None) }
1356 .map_err(super::map_host_device_oom_and_ioca_err)?;
1357
1358 if let Some(label) = desc.label {
1359 unsafe { self.shared.set_object_name(raw, label) };
1360 }
1361
1362 let identity = self.shared.texture_view_identity_factory.next();
1363
1364 self.counters.texture_views.add(1);
1365
1366 Ok(super::TextureView {
1367 raw_texture: texture.raw,
1368 raw,
1369 layers,
1370 format: desc.format,
1371 raw_format,
1372 base_mip_level: desc.range.base_mip_level,
1373 dimension: desc.dimension,
1374 texture_identity: texture.identity,
1375 view_identity: identity,
1376 })
1377 }
1378 unsafe fn destroy_texture_view(&self, view: super::TextureView) {
1379 unsafe { self.shared.raw.destroy_image_view(view.raw, None) };
1380
1381 self.counters.texture_views.sub(1);
1382 }
1383
1384 unsafe fn create_sampler(
1385 &self,
1386 desc: &crate::SamplerDescriptor,
1387 ) -> Result<super::Sampler, crate::DeviceError> {
1388 let mut create_info = vk::SamplerCreateInfo::default()
1389 .flags(vk::SamplerCreateFlags::empty())
1390 .mag_filter(conv::map_filter_mode(desc.mag_filter))
1391 .min_filter(conv::map_filter_mode(desc.min_filter))
1392 .mipmap_mode(conv::map_mip_filter_mode(desc.mipmap_filter))
1393 .address_mode_u(conv::map_address_mode(desc.address_modes[0]))
1394 .address_mode_v(conv::map_address_mode(desc.address_modes[1]))
1395 .address_mode_w(conv::map_address_mode(desc.address_modes[2]))
1396 .min_lod(desc.lod_clamp.start)
1397 .max_lod(desc.lod_clamp.end);
1398
1399 if let Some(fun) = desc.compare {
1400 create_info = create_info
1401 .compare_enable(true)
1402 .compare_op(conv::map_comparison(fun));
1403 }
1404
1405 if desc.anisotropy_clamp != 1 {
1406 create_info = create_info
1409 .anisotropy_enable(true)
1410 .max_anisotropy(desc.anisotropy_clamp as f32);
1411 }
1412
1413 if let Some(color) = desc.border_color {
1414 create_info = create_info.border_color(conv::map_border_color(color));
1415 }
1416
1417 let mut sampler_cache_guard = self.shared.sampler_cache.lock();
1418
1419 let raw = sampler_cache_guard.create_sampler(&self.shared.raw, create_info)?;
1420
1421 if let Some(label) = desc.label {
1425 unsafe { self.shared.set_object_name(raw, label) };
1428 }
1429
1430 drop(sampler_cache_guard);
1431
1432 self.counters.samplers.add(1);
1433
1434 Ok(super::Sampler { raw, create_info })
1435 }
1436 unsafe fn destroy_sampler(&self, sampler: super::Sampler) {
1437 self.shared.sampler_cache.lock().destroy_sampler(
1438 &self.shared.raw,
1439 sampler.create_info,
1440 sampler.raw,
1441 );
1442
1443 self.counters.samplers.sub(1);
1444 }
1445
1446 unsafe fn create_command_encoder(
1447 &self,
1448 desc: &crate::CommandEncoderDescriptor<super::Queue>,
1449 ) -> Result<super::CommandEncoder, crate::DeviceError> {
1450 let vk_info = vk::CommandPoolCreateInfo::default()
1451 .queue_family_index(desc.queue.family_index)
1452 .flags(vk::CommandPoolCreateFlags::TRANSIENT);
1453
1454 let raw = unsafe {
1455 self.shared
1456 .raw
1457 .create_command_pool(&vk_info, None)
1458 .map_err(super::map_host_device_oom_err)?
1459 };
1460
1461 self.counters.command_encoders.add(1);
1462
1463 Ok(super::CommandEncoder {
1464 raw,
1465 device: Arc::clone(&self.shared),
1466 active: vk::CommandBuffer::null(),
1467 bind_point: vk::PipelineBindPoint::default(),
1468 temp: super::Temp::default(),
1469 free: Vec::new(),
1470 discarded: Vec::new(),
1471 rpass_debug_marker_active: false,
1472 end_of_pass_timer_query: None,
1473 framebuffers: Default::default(),
1474 temp_texture_views: Default::default(),
1475 counters: Arc::clone(&self.counters),
1476 })
1477 }
1478
1479 unsafe fn create_bind_group_layout(
1480 &self,
1481 desc: &crate::BindGroupLayoutDescriptor,
1482 ) -> Result<super::BindGroupLayout, crate::DeviceError> {
1483 let mut desc_count = gpu_descriptor::DescriptorTotalCount::default();
1484 let mut types = Vec::new();
1485 for entry in desc.entries {
1486 let count = entry.count.map_or(1, |c| c.get());
1487 if entry.binding as usize >= types.len() {
1488 types.resize(
1489 entry.binding as usize + 1,
1490 (vk::DescriptorType::INPUT_ATTACHMENT, 0),
1491 );
1492 }
1493 types[entry.binding as usize] = (
1494 conv::map_binding_type(entry.ty),
1495 entry.count.map_or(1, |c| c.get()),
1496 );
1497
1498 match entry.ty {
1499 wgt::BindingType::Buffer {
1500 ty,
1501 has_dynamic_offset,
1502 ..
1503 } => match ty {
1504 wgt::BufferBindingType::Uniform => {
1505 if has_dynamic_offset {
1506 desc_count.uniform_buffer_dynamic += count;
1507 } else {
1508 desc_count.uniform_buffer += count;
1509 }
1510 }
1511 wgt::BufferBindingType::Storage { .. } => {
1512 if has_dynamic_offset {
1513 desc_count.storage_buffer_dynamic += count;
1514 } else {
1515 desc_count.storage_buffer += count;
1516 }
1517 }
1518 },
1519 wgt::BindingType::Sampler { .. } => {
1520 desc_count.sampler += count;
1521 }
1522 wgt::BindingType::Texture { .. } => {
1523 desc_count.sampled_image += count;
1524 }
1525 wgt::BindingType::StorageTexture { .. } => {
1526 desc_count.storage_image += count;
1527 }
1528 wgt::BindingType::AccelerationStructure { .. } => {
1529 desc_count.acceleration_structure += count;
1530 }
1531 wgt::BindingType::ExternalTexture => unimplemented!(),
1532 }
1533 }
1534
1535 let vk_bindings = desc
1537 .entries
1538 .iter()
1539 .map(|entry| vk::DescriptorSetLayoutBinding {
1540 binding: entry.binding,
1541 descriptor_type: types[entry.binding as usize].0,
1542 descriptor_count: types[entry.binding as usize].1,
1543 stage_flags: conv::map_shader_stage(entry.visibility),
1544 p_immutable_samplers: ptr::null(),
1545 _marker: Default::default(),
1546 })
1547 .collect::<Vec<_>>();
1548
1549 let binding_arrays: Vec<_> = desc
1550 .entries
1551 .iter()
1552 .enumerate()
1553 .filter_map(|(idx, entry)| entry.count.map(|count| (idx as u32, count)))
1554 .collect();
1555
1556 let vk_info = vk::DescriptorSetLayoutCreateInfo::default()
1557 .bindings(&vk_bindings)
1558 .flags(if !binding_arrays.is_empty() {
1559 vk::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND_POOL
1560 } else {
1561 vk::DescriptorSetLayoutCreateFlags::empty()
1562 });
1563
1564 let partially_bound = desc
1565 .flags
1566 .contains(crate::BindGroupLayoutFlags::PARTIALLY_BOUND);
1567
1568 let binding_flag_vec = desc
1569 .entries
1570 .iter()
1571 .map(|entry| {
1572 let mut flags = vk::DescriptorBindingFlags::empty();
1573
1574 if partially_bound && entry.count.is_some() {
1575 flags |= vk::DescriptorBindingFlags::PARTIALLY_BOUND;
1576 }
1577
1578 if entry.count.is_some() {
1579 flags |= vk::DescriptorBindingFlags::UPDATE_AFTER_BIND;
1580 }
1581
1582 flags
1583 })
1584 .collect::<Vec<_>>();
1585
1586 let mut binding_flag_info = vk::DescriptorSetLayoutBindingFlagsCreateInfo::default()
1587 .binding_flags(&binding_flag_vec);
1588
1589 let vk_info = vk_info.push_next(&mut binding_flag_info);
1590
1591 let raw = unsafe {
1592 self.shared
1593 .raw
1594 .create_descriptor_set_layout(&vk_info, None)
1595 .map_err(super::map_host_device_oom_err)?
1596 };
1597
1598 if let Some(label) = desc.label {
1599 unsafe { self.shared.set_object_name(raw, label) };
1600 }
1601
1602 self.counters.bind_group_layouts.add(1);
1603
1604 Ok(super::BindGroupLayout {
1605 raw,
1606 desc_count,
1607 types: types.into_boxed_slice(),
1608 binding_arrays,
1609 })
1610 }
1611 unsafe fn destroy_bind_group_layout(&self, bg_layout: super::BindGroupLayout) {
1612 unsafe {
1613 self.shared
1614 .raw
1615 .destroy_descriptor_set_layout(bg_layout.raw, None)
1616 };
1617
1618 self.counters.bind_group_layouts.sub(1);
1619 }
1620
1621 unsafe fn create_pipeline_layout(
1622 &self,
1623 desc: &crate::PipelineLayoutDescriptor<super::BindGroupLayout>,
1624 ) -> Result<super::PipelineLayout, crate::DeviceError> {
1625 let vk_set_layouts = desc
1627 .bind_group_layouts
1628 .iter()
1629 .map(|bgl| bgl.raw)
1630 .collect::<Vec<_>>();
1631 let vk_push_constant_ranges = desc
1632 .push_constant_ranges
1633 .iter()
1634 .map(|pcr| vk::PushConstantRange {
1635 stage_flags: conv::map_shader_stage(pcr.stages),
1636 offset: pcr.range.start,
1637 size: pcr.range.end - pcr.range.start,
1638 })
1639 .collect::<Vec<_>>();
1640
1641 let vk_info = vk::PipelineLayoutCreateInfo::default()
1642 .flags(vk::PipelineLayoutCreateFlags::empty())
1643 .set_layouts(&vk_set_layouts)
1644 .push_constant_ranges(&vk_push_constant_ranges);
1645
1646 let raw = {
1647 profiling::scope!("vkCreatePipelineLayout");
1648 unsafe {
1649 self.shared
1650 .raw
1651 .create_pipeline_layout(&vk_info, None)
1652 .map_err(super::map_host_device_oom_err)?
1653 }
1654 };
1655
1656 if let Some(label) = desc.label {
1657 unsafe { self.shared.set_object_name(raw, label) };
1658 }
1659
1660 let mut binding_arrays = BTreeMap::new();
1661 for (group, &layout) in desc.bind_group_layouts.iter().enumerate() {
1662 for &(binding, binding_array_size) in &layout.binding_arrays {
1663 binding_arrays.insert(
1664 naga::ResourceBinding {
1665 group: group as u32,
1666 binding,
1667 },
1668 naga::back::spv::BindingInfo {
1669 binding_array_size: Some(binding_array_size.get()),
1670 },
1671 );
1672 }
1673 }
1674
1675 self.counters.pipeline_layouts.add(1);
1676
1677 Ok(super::PipelineLayout {
1678 raw,
1679 binding_arrays,
1680 })
1681 }
1682 unsafe fn destroy_pipeline_layout(&self, pipeline_layout: super::PipelineLayout) {
1683 unsafe {
1684 self.shared
1685 .raw
1686 .destroy_pipeline_layout(pipeline_layout.raw, None)
1687 };
1688
1689 self.counters.pipeline_layouts.sub(1);
1690 }
1691
1692 unsafe fn create_bind_group(
1693 &self,
1694 desc: &crate::BindGroupDescriptor<
1695 super::BindGroupLayout,
1696 super::Buffer,
1697 super::Sampler,
1698 super::TextureView,
1699 super::AccelerationStructure,
1700 >,
1701 ) -> Result<super::BindGroup, crate::DeviceError> {
1702 let contains_binding_arrays = !desc.layout.binding_arrays.is_empty();
1703
1704 let desc_set_layout_flags = if contains_binding_arrays {
1705 gpu_descriptor::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND
1706 } else {
1707 gpu_descriptor::DescriptorSetLayoutCreateFlags::empty()
1708 };
1709
1710 let mut vk_sets = unsafe {
1711 self.desc_allocator.lock().allocate(
1712 &*self.shared,
1713 &desc.layout.raw,
1714 desc_set_layout_flags,
1715 &desc.layout.desc_count,
1716 1,
1717 )?
1718 };
1719
1720 let set = vk_sets.pop().unwrap();
1721 if let Some(label) = desc.label {
1722 unsafe { self.shared.set_object_name(*set.raw(), label) };
1723 }
1724
1725 struct ExtendStack<'a, T> {
1732 remainder: &'a mut [MaybeUninit<T>],
1733 }
1734
1735 impl<'a, T> ExtendStack<'a, T> {
1736 fn from_vec_capacity(vec: &'a mut Vec<T>) -> Self {
1737 Self {
1738 remainder: vec.spare_capacity_mut(),
1739 }
1740 }
1741
1742 fn extend_one(self, value: T) -> (Self, &'a mut T) {
1743 let (to_init, remainder) = self.remainder.split_first_mut().unwrap();
1744 let init = to_init.write(value);
1745 (Self { remainder }, init)
1746 }
1747
1748 fn extend(
1749 self,
1750 iter: impl IntoIterator<Item = T> + ExactSizeIterator,
1751 ) -> (Self, &'a mut [T]) {
1752 let (to_init, remainder) = self.remainder.split_at_mut(iter.len());
1753
1754 for (value, to_init) in iter.into_iter().zip(to_init.iter_mut()) {
1755 to_init.write(value);
1756 }
1757
1758 let init = {
1761 unsafe { mem::transmute::<&mut [MaybeUninit<T>], &mut [T]>(to_init) }
1768 };
1769 (Self { remainder }, init)
1770 }
1771 }
1772
1773 let mut writes = Vec::with_capacity(desc.entries.len());
1774 let mut buffer_infos = Vec::with_capacity(desc.buffers.len());
1775 let mut buffer_infos = ExtendStack::from_vec_capacity(&mut buffer_infos);
1776 let mut image_infos = Vec::with_capacity(desc.samplers.len() + desc.textures.len());
1777 let mut image_infos = ExtendStack::from_vec_capacity(&mut image_infos);
1778 let mut acceleration_structure_infos =
1783 Vec::with_capacity(desc.acceleration_structures.len());
1784 let mut acceleration_structure_infos =
1785 ExtendStack::from_vec_capacity(&mut acceleration_structure_infos);
1786 let mut raw_acceleration_structures =
1787 Vec::with_capacity(desc.acceleration_structures.len());
1788 let mut raw_acceleration_structures =
1789 ExtendStack::from_vec_capacity(&mut raw_acceleration_structures);
1790 for entry in desc.entries {
1791 let (ty, size) = desc.layout.types[entry.binding as usize];
1792 if size == 0 {
1793 continue; }
1795 let mut write = vk::WriteDescriptorSet::default()
1796 .dst_set(*set.raw())
1797 .dst_binding(entry.binding)
1798 .descriptor_type(ty);
1799
1800 write = match ty {
1801 vk::DescriptorType::SAMPLER => {
1802 let start = entry.resource_index;
1803 let end = start + entry.count;
1804 let local_image_infos;
1805 (image_infos, local_image_infos) =
1806 image_infos.extend(desc.samplers[start as usize..end as usize].iter().map(
1807 |sampler| vk::DescriptorImageInfo::default().sampler(sampler.raw),
1808 ));
1809 write.image_info(local_image_infos)
1810 }
1811 vk::DescriptorType::SAMPLED_IMAGE | vk::DescriptorType::STORAGE_IMAGE => {
1812 let start = entry.resource_index;
1813 let end = start + entry.count;
1814 let local_image_infos;
1815 (image_infos, local_image_infos) =
1816 image_infos.extend(desc.textures[start as usize..end as usize].iter().map(
1817 |binding| {
1818 let layout =
1819 conv::derive_image_layout(binding.usage, binding.view.format);
1820 vk::DescriptorImageInfo::default()
1821 .image_view(binding.view.raw)
1822 .image_layout(layout)
1823 },
1824 ));
1825 write.image_info(local_image_infos)
1826 }
1827 vk::DescriptorType::UNIFORM_BUFFER
1828 | vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC
1829 | vk::DescriptorType::STORAGE_BUFFER
1830 | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => {
1831 let start = entry.resource_index;
1832 let end = start + entry.count;
1833 let local_buffer_infos;
1834 (buffer_infos, local_buffer_infos) =
1835 buffer_infos.extend(desc.buffers[start as usize..end as usize].iter().map(
1836 |binding| {
1837 vk::DescriptorBufferInfo::default()
1838 .buffer(binding.buffer.raw)
1839 .offset(binding.offset)
1840 .range(
1841 binding.size.map_or(vk::WHOLE_SIZE, wgt::BufferSize::get),
1842 )
1843 },
1844 ));
1845 write.buffer_info(local_buffer_infos)
1846 }
1847 vk::DescriptorType::ACCELERATION_STRUCTURE_KHR => {
1848 let start = entry.resource_index;
1849 let end = start + entry.count;
1850
1851 let local_raw_acceleration_structures;
1852 (
1853 raw_acceleration_structures,
1854 local_raw_acceleration_structures,
1855 ) = raw_acceleration_structures.extend(
1856 desc.acceleration_structures[start as usize..end as usize]
1857 .iter()
1858 .map(|acceleration_structure| acceleration_structure.raw),
1859 );
1860
1861 let local_acceleration_structure_infos;
1862 (
1863 acceleration_structure_infos,
1864 local_acceleration_structure_infos,
1865 ) = acceleration_structure_infos.extend_one(
1866 vk::WriteDescriptorSetAccelerationStructureKHR::default()
1867 .acceleration_structures(local_raw_acceleration_structures),
1868 );
1869
1870 write
1871 .descriptor_count(entry.count)
1872 .push_next(local_acceleration_structure_infos)
1873 }
1874 _ => unreachable!(),
1875 };
1876
1877 writes.push(write);
1878 }
1879
1880 unsafe { self.shared.raw.update_descriptor_sets(&writes, &[]) };
1881
1882 self.counters.bind_groups.add(1);
1883
1884 Ok(super::BindGroup { set })
1885 }
1886
1887 unsafe fn destroy_bind_group(&self, group: super::BindGroup) {
1888 unsafe {
1889 self.desc_allocator
1890 .lock()
1891 .free(&*self.shared, Some(group.set))
1892 };
1893
1894 self.counters.bind_groups.sub(1);
1895 }
1896
1897 unsafe fn create_shader_module(
1898 &self,
1899 desc: &crate::ShaderModuleDescriptor,
1900 shader: crate::ShaderInput,
1901 ) -> Result<super::ShaderModule, crate::ShaderError> {
1902 let shader_module = match shader {
1903 crate::ShaderInput::Naga(naga_shader)
1904 if self
1905 .shared
1906 .workarounds
1907 .contains(super::Workarounds::SEPARATE_ENTRY_POINTS)
1908 || !naga_shader.module.overrides.is_empty() =>
1909 {
1910 super::ShaderModule::Intermediate {
1911 naga_shader,
1912 runtime_checks: desc.runtime_checks,
1913 }
1914 }
1915 crate::ShaderInput::Naga(naga_shader) => {
1916 let mut naga_options = self.naga_options.clone();
1917 naga_options.debug_info =
1918 naga_shader
1919 .debug_source
1920 .as_ref()
1921 .map(|d| naga::back::spv::DebugInfo {
1922 source_code: d.source_code.as_ref(),
1923 file_name: d.file_name.as_ref().into(),
1924 language: naga::back::spv::SourceLanguage::WGSL,
1925 });
1926 if !desc.runtime_checks.bounds_checks {
1927 naga_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
1928 index: naga::proc::BoundsCheckPolicy::Unchecked,
1929 buffer: naga::proc::BoundsCheckPolicy::Unchecked,
1930 image_load: naga::proc::BoundsCheckPolicy::Unchecked,
1931 binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
1932 };
1933 }
1934 let spv = naga::back::spv::write_vec(
1935 &naga_shader.module,
1936 &naga_shader.info,
1937 &naga_options,
1938 None,
1939 )
1940 .map_err(|e| crate::ShaderError::Compilation(format!("{e}")))?;
1941 super::ShaderModule::Raw(self.create_shader_module_impl(&spv, &desc.label)?)
1942 }
1943 crate::ShaderInput::SpirV(data) => {
1944 super::ShaderModule::Raw(self.create_shader_module_impl(data, &desc.label)?)
1945 }
1946 crate::ShaderInput::Msl { .. }
1947 | crate::ShaderInput::Dxil { .. }
1948 | crate::ShaderInput::Hlsl { .. }
1949 | crate::ShaderInput::Glsl { .. } => unreachable!(),
1950 };
1951
1952 self.counters.shader_modules.add(1);
1953
1954 Ok(shader_module)
1955 }
1956
1957 unsafe fn destroy_shader_module(&self, module: super::ShaderModule) {
1958 match module {
1959 super::ShaderModule::Raw(raw) => {
1960 unsafe { self.shared.raw.destroy_shader_module(raw, None) };
1961 }
1962 super::ShaderModule::Intermediate { .. } => {}
1963 }
1964
1965 self.counters.shader_modules.sub(1);
1966 }
1967
1968 unsafe fn create_render_pipeline(
1969 &self,
1970 desc: &crate::RenderPipelineDescriptor<
1971 super::PipelineLayout,
1972 super::ShaderModule,
1973 super::PipelineCache,
1974 >,
1975 ) -> Result<super::RenderPipeline, crate::PipelineError> {
1976 let dynamic_states = [
1977 vk::DynamicState::VIEWPORT,
1978 vk::DynamicState::SCISSOR,
1979 vk::DynamicState::BLEND_CONSTANTS,
1980 vk::DynamicState::STENCIL_REFERENCE,
1981 ];
1982 let mut compatible_rp_key = super::RenderPassKey {
1983 sample_count: desc.multisample.count,
1984 multiview: desc.multiview,
1985 ..Default::default()
1986 };
1987 let mut stages = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new();
1988 let mut vertex_buffers = Vec::new();
1989 let mut vertex_attributes = Vec::new();
1990
1991 if let crate::VertexProcessor::Standard {
1992 vertex_buffers: desc_vertex_buffers,
1993 vertex_stage: _,
1994 } = &desc.vertex_processor
1995 {
1996 vertex_buffers = Vec::with_capacity(desc_vertex_buffers.len());
1997 for (i, vb) in desc_vertex_buffers.iter().enumerate() {
1998 vertex_buffers.push(vk::VertexInputBindingDescription {
1999 binding: i as u32,
2000 stride: vb.array_stride as u32,
2001 input_rate: match vb.step_mode {
2002 wgt::VertexStepMode::Vertex => vk::VertexInputRate::VERTEX,
2003 wgt::VertexStepMode::Instance => vk::VertexInputRate::INSTANCE,
2004 },
2005 });
2006 for at in vb.attributes {
2007 vertex_attributes.push(vk::VertexInputAttributeDescription {
2008 location: at.shader_location,
2009 binding: i as u32,
2010 format: conv::map_vertex_format(at.format),
2011 offset: at.offset as u32,
2012 });
2013 }
2014 }
2015 }
2016
2017 let vk_vertex_input = vk::PipelineVertexInputStateCreateInfo::default()
2018 .vertex_binding_descriptions(&vertex_buffers)
2019 .vertex_attribute_descriptions(&vertex_attributes);
2020
2021 let vk_input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
2022 .topology(conv::map_topology(desc.primitive.topology))
2023 .primitive_restart_enable(desc.primitive.strip_index_format.is_some());
2024
2025 let mut compiled_vs = None;
2026 let mut compiled_ms = None;
2027 let mut compiled_ts = None;
2028 match &desc.vertex_processor {
2029 crate::VertexProcessor::Standard {
2030 vertex_buffers: _,
2031 vertex_stage,
2032 } => {
2033 compiled_vs = Some(self.compile_stage(
2034 vertex_stage,
2035 naga::ShaderStage::Vertex,
2036 &desc.layout.binding_arrays,
2037 )?);
2038 stages.push(compiled_vs.as_ref().unwrap().create_info);
2039 }
2040 crate::VertexProcessor::Mesh {
2041 task_stage,
2042 mesh_stage,
2043 } => {
2044 if let Some(t) = task_stage.as_ref() {
2045 compiled_ts = Some(self.compile_stage(
2046 t,
2047 naga::ShaderStage::Task,
2048 &desc.layout.binding_arrays,
2049 )?);
2050 stages.push(compiled_ts.as_ref().unwrap().create_info);
2051 }
2052 compiled_ms = Some(self.compile_stage(
2053 mesh_stage,
2054 naga::ShaderStage::Mesh,
2055 &desc.layout.binding_arrays,
2056 )?);
2057 stages.push(compiled_ms.as_ref().unwrap().create_info);
2058 }
2059 }
2060 let compiled_fs = match desc.fragment_stage {
2061 Some(ref stage) => {
2062 let compiled = self.compile_stage(
2063 stage,
2064 naga::ShaderStage::Fragment,
2065 &desc.layout.binding_arrays,
2066 )?;
2067 stages.push(compiled.create_info);
2068 Some(compiled)
2069 }
2070 None => None,
2071 };
2072
2073 let mut vk_rasterization = vk::PipelineRasterizationStateCreateInfo::default()
2074 .polygon_mode(conv::map_polygon_mode(desc.primitive.polygon_mode))
2075 .front_face(conv::map_front_face(desc.primitive.front_face))
2076 .line_width(1.0)
2077 .depth_clamp_enable(desc.primitive.unclipped_depth);
2078 if let Some(face) = desc.primitive.cull_mode {
2079 vk_rasterization = vk_rasterization.cull_mode(conv::map_cull_face(face))
2080 }
2081 let mut vk_rasterization_conservative_state =
2082 vk::PipelineRasterizationConservativeStateCreateInfoEXT::default()
2083 .conservative_rasterization_mode(
2084 vk::ConservativeRasterizationModeEXT::OVERESTIMATE,
2085 );
2086 if desc.primitive.conservative {
2087 vk_rasterization = vk_rasterization.push_next(&mut vk_rasterization_conservative_state);
2088 }
2089
2090 let mut vk_depth_stencil = vk::PipelineDepthStencilStateCreateInfo::default();
2091 if let Some(ref ds) = desc.depth_stencil {
2092 let vk_format = self.shared.private_caps.map_texture_format(ds.format);
2093 let vk_layout = if ds.is_read_only(desc.primitive.cull_mode) {
2094 vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL
2095 } else {
2096 vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL
2097 };
2098 compatible_rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
2099 base: super::AttachmentKey::compatible(vk_format, vk_layout),
2100 stencil_ops: crate::AttachmentOps::all(),
2101 });
2102
2103 if ds.is_depth_enabled() {
2104 vk_depth_stencil = vk_depth_stencil
2105 .depth_test_enable(true)
2106 .depth_write_enable(ds.depth_write_enabled)
2107 .depth_compare_op(conv::map_comparison(ds.depth_compare));
2108 }
2109 if ds.stencil.is_enabled() {
2110 let s = &ds.stencil;
2111 let front = conv::map_stencil_face(&s.front, s.read_mask, s.write_mask);
2112 let back = conv::map_stencil_face(&s.back, s.read_mask, s.write_mask);
2113 vk_depth_stencil = vk_depth_stencil
2114 .stencil_test_enable(true)
2115 .front(front)
2116 .back(back);
2117 }
2118
2119 if ds.bias.is_enabled() {
2120 vk_rasterization = vk_rasterization
2121 .depth_bias_enable(true)
2122 .depth_bias_constant_factor(ds.bias.constant as f32)
2123 .depth_bias_clamp(ds.bias.clamp)
2124 .depth_bias_slope_factor(ds.bias.slope_scale);
2125 }
2126 }
2127
2128 let vk_viewport = vk::PipelineViewportStateCreateInfo::default()
2129 .flags(vk::PipelineViewportStateCreateFlags::empty())
2130 .scissor_count(1)
2131 .viewport_count(1);
2132
2133 let vk_sample_mask = [
2134 desc.multisample.mask as u32,
2135 (desc.multisample.mask >> 32) as u32,
2136 ];
2137 let vk_multisample = vk::PipelineMultisampleStateCreateInfo::default()
2138 .rasterization_samples(vk::SampleCountFlags::from_raw(desc.multisample.count))
2139 .alpha_to_coverage_enable(desc.multisample.alpha_to_coverage_enabled)
2140 .sample_mask(&vk_sample_mask);
2141
2142 let mut vk_attachments = Vec::with_capacity(desc.color_targets.len());
2143 for cat in desc.color_targets {
2144 let (key, attarchment) = if let Some(cat) = cat.as_ref() {
2145 let mut vk_attachment = vk::PipelineColorBlendAttachmentState::default()
2146 .color_write_mask(vk::ColorComponentFlags::from_raw(cat.write_mask.bits()));
2147 if let Some(ref blend) = cat.blend {
2148 let (color_op, color_src, color_dst) = conv::map_blend_component(&blend.color);
2149 let (alpha_op, alpha_src, alpha_dst) = conv::map_blend_component(&blend.alpha);
2150 vk_attachment = vk_attachment
2151 .blend_enable(true)
2152 .color_blend_op(color_op)
2153 .src_color_blend_factor(color_src)
2154 .dst_color_blend_factor(color_dst)
2155 .alpha_blend_op(alpha_op)
2156 .src_alpha_blend_factor(alpha_src)
2157 .dst_alpha_blend_factor(alpha_dst);
2158 }
2159
2160 let vk_format = self.shared.private_caps.map_texture_format(cat.format);
2161 (
2162 Some(super::ColorAttachmentKey {
2163 base: super::AttachmentKey::compatible(
2164 vk_format,
2165 vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
2166 ),
2167 resolve: None,
2168 }),
2169 vk_attachment,
2170 )
2171 } else {
2172 (None, vk::PipelineColorBlendAttachmentState::default())
2173 };
2174
2175 compatible_rp_key.colors.push(key);
2176 vk_attachments.push(attarchment);
2177 }
2178
2179 let vk_color_blend =
2180 vk::PipelineColorBlendStateCreateInfo::default().attachments(&vk_attachments);
2181
2182 let vk_dynamic_state =
2183 vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic_states);
2184
2185 let raw_pass = self.shared.make_render_pass(compatible_rp_key)?;
2186
2187 let vk_infos = [{
2188 vk::GraphicsPipelineCreateInfo::default()
2189 .layout(desc.layout.raw)
2190 .stages(&stages)
2191 .vertex_input_state(&vk_vertex_input)
2192 .input_assembly_state(&vk_input_assembly)
2193 .rasterization_state(&vk_rasterization)
2194 .viewport_state(&vk_viewport)
2195 .multisample_state(&vk_multisample)
2196 .depth_stencil_state(&vk_depth_stencil)
2197 .color_blend_state(&vk_color_blend)
2198 .dynamic_state(&vk_dynamic_state)
2199 .render_pass(raw_pass)
2200 }];
2201
2202 let pipeline_cache = desc
2203 .cache
2204 .map(|it| it.raw)
2205 .unwrap_or(vk::PipelineCache::null());
2206
2207 let mut raw_vec = {
2208 profiling::scope!("vkCreateGraphicsPipelines");
2209 unsafe {
2210 self.shared
2211 .raw
2212 .create_graphics_pipelines(pipeline_cache, &vk_infos, None)
2213 .map_err(|(_, e)| super::map_pipeline_err(e))
2214 }?
2215 };
2216
2217 let raw = raw_vec.pop().unwrap();
2218 if let Some(label) = desc.label {
2219 unsafe { self.shared.set_object_name(raw, label) };
2220 }
2221
2222 if let Some(CompiledStage {
2223 temp_raw_module: Some(raw_module),
2224 ..
2225 }) = compiled_vs
2226 {
2227 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2228 }
2229 if let Some(CompiledStage {
2230 temp_raw_module: Some(raw_module),
2231 ..
2232 }) = compiled_ts
2233 {
2234 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2235 }
2236 if let Some(CompiledStage {
2237 temp_raw_module: Some(raw_module),
2238 ..
2239 }) = compiled_ms
2240 {
2241 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2242 }
2243 if let Some(CompiledStage {
2244 temp_raw_module: Some(raw_module),
2245 ..
2246 }) = compiled_fs
2247 {
2248 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2249 }
2250
2251 self.counters.render_pipelines.add(1);
2252
2253 Ok(super::RenderPipeline { raw })
2254 }
2255
2256 unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) {
2257 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2258
2259 self.counters.render_pipelines.sub(1);
2260 }
2261
2262 unsafe fn create_compute_pipeline(
2263 &self,
2264 desc: &crate::ComputePipelineDescriptor<
2265 super::PipelineLayout,
2266 super::ShaderModule,
2267 super::PipelineCache,
2268 >,
2269 ) -> Result<super::ComputePipeline, crate::PipelineError> {
2270 let compiled = self.compile_stage(
2271 &desc.stage,
2272 naga::ShaderStage::Compute,
2273 &desc.layout.binding_arrays,
2274 )?;
2275
2276 let vk_infos = [{
2277 vk::ComputePipelineCreateInfo::default()
2278 .layout(desc.layout.raw)
2279 .stage(compiled.create_info)
2280 }];
2281
2282 let pipeline_cache = desc
2283 .cache
2284 .map(|it| it.raw)
2285 .unwrap_or(vk::PipelineCache::null());
2286
2287 let mut raw_vec = {
2288 profiling::scope!("vkCreateComputePipelines");
2289 unsafe {
2290 self.shared
2291 .raw
2292 .create_compute_pipelines(pipeline_cache, &vk_infos, None)
2293 .map_err(|(_, e)| super::map_pipeline_err(e))
2294 }?
2295 };
2296
2297 let raw = raw_vec.pop().unwrap();
2298 if let Some(label) = desc.label {
2299 unsafe { self.shared.set_object_name(raw, label) };
2300 }
2301
2302 if let Some(raw_module) = compiled.temp_raw_module {
2303 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2304 }
2305
2306 self.counters.compute_pipelines.add(1);
2307
2308 Ok(super::ComputePipeline { raw })
2309 }
2310
2311 unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) {
2312 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2313
2314 self.counters.compute_pipelines.sub(1);
2315 }
2316
2317 unsafe fn create_pipeline_cache(
2318 &self,
2319 desc: &crate::PipelineCacheDescriptor<'_>,
2320 ) -> Result<super::PipelineCache, crate::PipelineCacheError> {
2321 let mut info = vk::PipelineCacheCreateInfo::default();
2322 if let Some(data) = desc.data {
2323 info = info.initial_data(data)
2324 }
2325 profiling::scope!("vkCreatePipelineCache");
2326 let raw = unsafe { self.shared.raw.create_pipeline_cache(&info, None) }
2327 .map_err(super::map_host_device_oom_err)?;
2328
2329 Ok(super::PipelineCache { raw })
2330 }
2331 fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> {
2332 Some(self.shared.pipeline_cache_validation_key)
2333 }
2334 unsafe fn destroy_pipeline_cache(&self, cache: super::PipelineCache) {
2335 unsafe { self.shared.raw.destroy_pipeline_cache(cache.raw, None) }
2336 }
2337 unsafe fn create_query_set(
2338 &self,
2339 desc: &wgt::QuerySetDescriptor<crate::Label>,
2340 ) -> Result<super::QuerySet, crate::DeviceError> {
2341 self.error_if_would_oom_on_resource_allocation(true, desc.count as u64 * 256)?;
2344
2345 let (vk_type, pipeline_statistics) = match desc.ty {
2346 wgt::QueryType::Occlusion => (
2347 vk::QueryType::OCCLUSION,
2348 vk::QueryPipelineStatisticFlags::empty(),
2349 ),
2350 wgt::QueryType::PipelineStatistics(statistics) => (
2351 vk::QueryType::PIPELINE_STATISTICS,
2352 conv::map_pipeline_statistics(statistics),
2353 ),
2354 wgt::QueryType::Timestamp => (
2355 vk::QueryType::TIMESTAMP,
2356 vk::QueryPipelineStatisticFlags::empty(),
2357 ),
2358 };
2359
2360 let vk_info = vk::QueryPoolCreateInfo::default()
2361 .query_type(vk_type)
2362 .query_count(desc.count)
2363 .pipeline_statistics(pipeline_statistics);
2364
2365 let raw = unsafe { self.shared.raw.create_query_pool(&vk_info, None) }
2366 .map_err(super::map_host_device_oom_err)?;
2367 if let Some(label) = desc.label {
2368 unsafe { self.shared.set_object_name(raw, label) };
2369 }
2370
2371 self.counters.query_sets.add(1);
2372
2373 Ok(super::QuerySet { raw })
2374 }
2375
2376 unsafe fn destroy_query_set(&self, set: super::QuerySet) {
2377 unsafe { self.shared.raw.destroy_query_pool(set.raw, None) };
2378
2379 self.counters.query_sets.sub(1);
2380 }
2381
2382 unsafe fn create_fence(&self) -> Result<super::Fence, crate::DeviceError> {
2383 self.counters.fences.add(1);
2384
2385 Ok(if self.shared.private_caps.timeline_semaphores {
2386 let mut sem_type_info =
2387 vk::SemaphoreTypeCreateInfo::default().semaphore_type(vk::SemaphoreType::TIMELINE);
2388 let vk_info = vk::SemaphoreCreateInfo::default().push_next(&mut sem_type_info);
2389 let raw = unsafe { self.shared.raw.create_semaphore(&vk_info, None) }
2390 .map_err(super::map_host_device_oom_err)?;
2391
2392 super::Fence::TimelineSemaphore(raw)
2393 } else {
2394 super::Fence::FencePool {
2395 last_completed: 0,
2396 active: Vec::new(),
2397 free: Vec::new(),
2398 }
2399 })
2400 }
2401 unsafe fn destroy_fence(&self, fence: super::Fence) {
2402 match fence {
2403 super::Fence::TimelineSemaphore(raw) => {
2404 unsafe { self.shared.raw.destroy_semaphore(raw, None) };
2405 }
2406 super::Fence::FencePool {
2407 active,
2408 free,
2409 last_completed: _,
2410 } => {
2411 for (_, raw) in active {
2412 unsafe { self.shared.raw.destroy_fence(raw, None) };
2413 }
2414 for raw in free {
2415 unsafe { self.shared.raw.destroy_fence(raw, None) };
2416 }
2417 }
2418 }
2419
2420 self.counters.fences.sub(1);
2421 }
2422 unsafe fn get_fence_value(
2423 &self,
2424 fence: &super::Fence,
2425 ) -> Result<crate::FenceValue, crate::DeviceError> {
2426 fence.get_latest(
2427 &self.shared.raw,
2428 self.shared.extension_fns.timeline_semaphore.as_ref(),
2429 )
2430 }
2431 unsafe fn wait(
2432 &self,
2433 fence: &super::Fence,
2434 wait_value: crate::FenceValue,
2435 timeout_ms: u32,
2436 ) -> Result<bool, crate::DeviceError> {
2437 let timeout_ns = timeout_ms as u64 * super::MILLIS_TO_NANOS;
2438 self.shared.wait_for_fence(fence, wait_value, timeout_ns)
2439 }
2440
2441 unsafe fn start_graphics_debugger_capture(&self) -> bool {
2442 #[cfg(feature = "renderdoc")]
2443 {
2444 let raw_vk_instance =
2446 vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2447 let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2448 unsafe {
2449 self.render_doc
2450 .start_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2451 }
2452 }
2453 #[cfg(not(feature = "renderdoc"))]
2454 false
2455 }
2456 unsafe fn stop_graphics_debugger_capture(&self) {
2457 #[cfg(feature = "renderdoc")]
2458 {
2459 let raw_vk_instance =
2461 vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2462 let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2463
2464 unsafe {
2465 self.render_doc
2466 .end_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2467 }
2468 }
2469 }
2470
2471 unsafe fn pipeline_cache_get_data(&self, cache: &super::PipelineCache) -> Option<Vec<u8>> {
2472 let data = unsafe { self.raw_device().get_pipeline_cache_data(cache.raw) };
2473 data.ok()
2474 }
2475
2476 unsafe fn get_acceleration_structure_build_sizes<'a>(
2477 &self,
2478 desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>,
2479 ) -> crate::AccelerationStructureBuildSizes {
2480 const CAPACITY: usize = 8;
2481
2482 let ray_tracing_functions = self
2483 .shared
2484 .extension_fns
2485 .ray_tracing
2486 .as_ref()
2487 .expect("Feature `RAY_TRACING` not enabled");
2488
2489 let (geometries, primitive_counts) = match *desc.entries {
2490 crate::AccelerationStructureEntries::Instances(ref instances) => {
2491 let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default();
2492
2493 let geometry = vk::AccelerationStructureGeometryKHR::default()
2494 .geometry_type(vk::GeometryTypeKHR::INSTANCES)
2495 .geometry(vk::AccelerationStructureGeometryDataKHR {
2496 instances: instance_data,
2497 });
2498
2499 (
2500 smallvec::smallvec![geometry],
2501 smallvec::smallvec![instances.count],
2502 )
2503 }
2504 crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
2505 let mut primitive_counts =
2506 smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2507 let mut geometries = smallvec::SmallVec::<
2508 [vk::AccelerationStructureGeometryKHR; CAPACITY],
2509 >::with_capacity(in_geometries.len());
2510
2511 for triangles in in_geometries {
2512 let mut triangle_data =
2513 vk::AccelerationStructureGeometryTrianglesDataKHR::default()
2514 .index_type(vk::IndexType::NONE_KHR)
2515 .vertex_format(conv::map_vertex_format(triangles.vertex_format))
2516 .max_vertex(triangles.vertex_count)
2517 .vertex_stride(triangles.vertex_stride)
2518 .transform_data(vk::DeviceOrHostAddressConstKHR {
2528 device_address: if desc
2529 .flags
2530 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
2531 {
2532 unsafe {
2533 ray_tracing_functions
2534 .buffer_device_address
2535 .get_buffer_device_address(
2536 &vk::BufferDeviceAddressInfo::default().buffer(
2537 triangles
2538 .transform
2539 .as_ref()
2540 .unwrap()
2541 .buffer
2542 .raw,
2543 ),
2544 )
2545 }
2546 } else {
2547 0
2548 },
2549 });
2550
2551 let pritive_count = if let Some(ref indices) = triangles.indices {
2552 triangle_data =
2553 triangle_data.index_type(conv::map_index_format(indices.format));
2554 indices.count / 3
2555 } else {
2556 triangles.vertex_count / 3
2557 };
2558
2559 let geometry = vk::AccelerationStructureGeometryKHR::default()
2560 .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
2561 .geometry(vk::AccelerationStructureGeometryDataKHR {
2562 triangles: triangle_data,
2563 })
2564 .flags(conv::map_acceleration_structure_geometry_flags(
2565 triangles.flags,
2566 ));
2567
2568 geometries.push(geometry);
2569 primitive_counts.push(pritive_count);
2570 }
2571 (geometries, primitive_counts)
2572 }
2573 crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
2574 let mut primitive_counts =
2575 smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2576 let mut geometries = smallvec::SmallVec::<
2577 [vk::AccelerationStructureGeometryKHR; CAPACITY],
2578 >::with_capacity(in_geometries.len());
2579 for aabb in in_geometries {
2580 let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
2581 .stride(aabb.stride);
2582
2583 let geometry = vk::AccelerationStructureGeometryKHR::default()
2584 .geometry_type(vk::GeometryTypeKHR::AABBS)
2585 .geometry(vk::AccelerationStructureGeometryDataKHR { aabbs: aabbs_data })
2586 .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
2587
2588 geometries.push(geometry);
2589 primitive_counts.push(aabb.count);
2590 }
2591 (geometries, primitive_counts)
2592 }
2593 };
2594
2595 let ty = match *desc.entries {
2596 crate::AccelerationStructureEntries::Instances(_) => {
2597 vk::AccelerationStructureTypeKHR::TOP_LEVEL
2598 }
2599 _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
2600 };
2601
2602 let geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
2603 .ty(ty)
2604 .flags(conv::map_acceleration_structure_flags(desc.flags))
2605 .geometries(&geometries);
2606
2607 let mut raw = Default::default();
2608 unsafe {
2609 ray_tracing_functions
2610 .acceleration_structure
2611 .get_acceleration_structure_build_sizes(
2612 vk::AccelerationStructureBuildTypeKHR::DEVICE,
2613 &geometry_info,
2614 &primitive_counts,
2615 &mut raw,
2616 )
2617 }
2618
2619 crate::AccelerationStructureBuildSizes {
2620 acceleration_structure_size: raw.acceleration_structure_size,
2621 update_scratch_size: raw.update_scratch_size,
2622 build_scratch_size: raw.build_scratch_size,
2623 }
2624 }
2625
2626 unsafe fn get_acceleration_structure_device_address(
2627 &self,
2628 acceleration_structure: &super::AccelerationStructure,
2629 ) -> wgt::BufferAddress {
2630 let ray_tracing_functions = self
2631 .shared
2632 .extension_fns
2633 .ray_tracing
2634 .as_ref()
2635 .expect("Feature `RAY_TRACING` not enabled");
2636
2637 unsafe {
2638 ray_tracing_functions
2639 .acceleration_structure
2640 .get_acceleration_structure_device_address(
2641 &vk::AccelerationStructureDeviceAddressInfoKHR::default()
2642 .acceleration_structure(acceleration_structure.raw),
2643 )
2644 }
2645 }
2646
2647 unsafe fn create_acceleration_structure(
2648 &self,
2649 desc: &crate::AccelerationStructureDescriptor,
2650 ) -> Result<super::AccelerationStructure, crate::DeviceError> {
2651 let ray_tracing_functions = self
2652 .shared
2653 .extension_fns
2654 .ray_tracing
2655 .as_ref()
2656 .expect("Feature `RAY_TRACING` not enabled");
2657
2658 let vk_buffer_info = vk::BufferCreateInfo::default()
2659 .size(desc.size)
2660 .usage(
2661 vk::BufferUsageFlags::ACCELERATION_STRUCTURE_STORAGE_KHR
2662 | vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS,
2663 )
2664 .sharing_mode(vk::SharingMode::EXCLUSIVE);
2665
2666 unsafe {
2667 let raw_buffer = self
2668 .shared
2669 .raw
2670 .create_buffer(&vk_buffer_info, None)
2671 .map_err(super::map_host_device_oom_and_ioca_err)?;
2672 let req = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
2673
2674 self.error_if_would_oom_on_resource_allocation(false, req.size)
2675 .inspect_err(|_| {
2676 self.shared.raw.destroy_buffer(raw_buffer, None);
2677 })?;
2678
2679 let block = self
2680 .mem_allocator
2681 .lock()
2682 .alloc(
2683 &*self.shared,
2684 gpu_alloc::Request {
2685 size: req.size,
2686 align_mask: req.alignment - 1,
2687 usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
2688 memory_types: req.memory_type_bits & self.valid_ash_memory_types,
2689 },
2690 )
2691 .inspect_err(|_| {
2692 self.shared.raw.destroy_buffer(raw_buffer, None);
2693 })?;
2694
2695 self.shared
2696 .raw
2697 .bind_buffer_memory(raw_buffer, *block.memory(), block.offset())
2698 .map_err(super::map_host_device_oom_and_ioca_err)
2699 .inspect_err(|_| {
2700 self.shared.raw.destroy_buffer(raw_buffer, None);
2701 })?;
2702
2703 if let Some(label) = desc.label {
2704 self.shared.set_object_name(raw_buffer, label);
2705 }
2706
2707 let vk_info = vk::AccelerationStructureCreateInfoKHR::default()
2708 .buffer(raw_buffer)
2709 .offset(0)
2710 .size(desc.size)
2711 .ty(conv::map_acceleration_structure_format(desc.format));
2712
2713 let raw_acceleration_structure = ray_tracing_functions
2714 .acceleration_structure
2715 .create_acceleration_structure(&vk_info, None)
2716 .map_err(super::map_host_oom_and_ioca_err)
2717 .inspect_err(|_| {
2718 self.shared.raw.destroy_buffer(raw_buffer, None);
2719 })?;
2720
2721 if let Some(label) = desc.label {
2722 self.shared
2723 .set_object_name(raw_acceleration_structure, label);
2724 }
2725
2726 let pool = if desc.allow_compaction {
2727 let vk_info = vk::QueryPoolCreateInfo::default()
2728 .query_type(vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR)
2729 .query_count(1);
2730
2731 let raw = self
2732 .shared
2733 .raw
2734 .create_query_pool(&vk_info, None)
2735 .map_err(super::map_host_device_oom_err)
2736 .inspect_err(|_| {
2737 ray_tracing_functions
2738 .acceleration_structure
2739 .destroy_acceleration_structure(raw_acceleration_structure, None);
2740 self.shared.raw.destroy_buffer(raw_buffer, None);
2741 })?;
2742 Some(raw)
2743 } else {
2744 None
2745 };
2746
2747 Ok(super::AccelerationStructure {
2748 raw: raw_acceleration_structure,
2749 buffer: raw_buffer,
2750 block: Mutex::new(block),
2751 compacted_size_query: pool,
2752 })
2753 }
2754 }
2755
2756 unsafe fn destroy_acceleration_structure(
2757 &self,
2758 acceleration_structure: super::AccelerationStructure,
2759 ) {
2760 let ray_tracing_functions = self
2761 .shared
2762 .extension_fns
2763 .ray_tracing
2764 .as_ref()
2765 .expect("Feature `RAY_TRACING` not enabled");
2766
2767 unsafe {
2768 ray_tracing_functions
2769 .acceleration_structure
2770 .destroy_acceleration_structure(acceleration_structure.raw, None);
2771 self.shared
2772 .raw
2773 .destroy_buffer(acceleration_structure.buffer, None);
2774 self.mem_allocator
2775 .lock()
2776 .dealloc(&*self.shared, acceleration_structure.block.into_inner());
2777 if let Some(query) = acceleration_structure.compacted_size_query {
2778 self.shared.raw.destroy_query_pool(query, None)
2779 }
2780 }
2781 }
2782
2783 fn get_internal_counters(&self) -> wgt::HalCounters {
2784 self.counters
2785 .memory_allocations
2786 .set(self.shared.memory_allocations_counter.read());
2787
2788 self.counters.as_ref().clone()
2789 }
2790
2791 fn tlas_instance_to_bytes(&self, instance: TlasInstance) -> Vec<u8> {
2792 const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
2793 let temp = RawTlasInstance {
2794 transform: instance.transform,
2795 custom_data_and_mask: (instance.custom_data & MAX_U24)
2796 | (u32::from(instance.mask) << 24),
2797 shader_binding_table_record_offset_and_flags: 0,
2798 acceleration_structure_reference: instance.blas_address,
2799 };
2800 bytemuck::bytes_of(&temp).to_vec()
2801 }
2802
2803 fn check_if_oom(&self) -> Result<(), crate::DeviceError> {
2804 let Some(threshold) = self
2805 .shared
2806 .instance
2807 .memory_budget_thresholds
2808 .for_device_loss
2809 else {
2810 return Ok(());
2811 };
2812
2813 if !self
2814 .shared
2815 .enabled_extensions
2816 .contains(&ext::memory_budget::NAME)
2817 {
2818 return Ok(());
2819 }
2820
2821 let get_physical_device_properties = self
2822 .shared
2823 .instance
2824 .get_physical_device_properties
2825 .as_ref()
2826 .unwrap();
2827
2828 let mut memory_budget_properties = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
2829
2830 let mut memory_properties =
2831 vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut memory_budget_properties);
2832
2833 unsafe {
2834 get_physical_device_properties.get_physical_device_memory_properties2(
2835 self.shared.physical_device,
2836 &mut memory_properties,
2837 );
2838 }
2839
2840 let memory_properties = memory_properties.memory_properties;
2841
2842 for i in 0..memory_properties.memory_heap_count {
2843 let heap_usage = memory_budget_properties.heap_usage[i as usize];
2844 let heap_budget = memory_budget_properties.heap_budget[i as usize];
2845
2846 if heap_usage >= heap_budget / 100 * threshold as u64 {
2847 return Err(crate::DeviceError::OutOfMemory);
2848 }
2849 }
2850
2851 Ok(())
2852 }
2853}
2854
2855impl super::DeviceShared {
2856 pub(super) fn new_binary_semaphore(
2857 &self,
2858 name: &str,
2859 ) -> Result<vk::Semaphore, crate::DeviceError> {
2860 unsafe {
2861 let semaphore = self
2862 .raw
2863 .create_semaphore(&vk::SemaphoreCreateInfo::default(), None)
2864 .map_err(super::map_host_device_oom_err)?;
2865
2866 self.set_object_name(semaphore, name);
2867
2868 Ok(semaphore)
2869 }
2870 }
2871
2872 pub(super) fn wait_for_fence(
2873 &self,
2874 fence: &super::Fence,
2875 wait_value: crate::FenceValue,
2876 timeout_ns: u64,
2877 ) -> Result<bool, crate::DeviceError> {
2878 profiling::scope!("Device::wait");
2879 match *fence {
2880 super::Fence::TimelineSemaphore(raw) => {
2881 let semaphores = [raw];
2882 let values = [wait_value];
2883 let vk_info = vk::SemaphoreWaitInfo::default()
2884 .semaphores(&semaphores)
2885 .values(&values);
2886 let result = match self.extension_fns.timeline_semaphore {
2887 Some(super::ExtensionFn::Extension(ref ext)) => unsafe {
2888 ext.wait_semaphores(&vk_info, timeout_ns)
2889 },
2890 Some(super::ExtensionFn::Promoted) => unsafe {
2891 self.raw.wait_semaphores(&vk_info, timeout_ns)
2892 },
2893 None => unreachable!(),
2894 };
2895 match result {
2896 Ok(()) => Ok(true),
2897 Err(vk::Result::TIMEOUT) => Ok(false),
2898 Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2899 }
2900 }
2901 super::Fence::FencePool {
2902 last_completed,
2903 ref active,
2904 free: _,
2905 } => {
2906 if wait_value <= last_completed {
2907 Ok(true)
2908 } else {
2909 match active.iter().find(|&&(value, _)| value >= wait_value) {
2910 Some(&(_, raw)) => {
2911 match unsafe { self.raw.wait_for_fences(&[raw], true, timeout_ns) } {
2912 Ok(()) => Ok(true),
2913 Err(vk::Result::TIMEOUT) => Ok(false),
2914 Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
2915 }
2916 }
2917 None => {
2918 crate::hal_usage_error(format!(
2919 "no signals reached value {wait_value}"
2920 ));
2921 }
2922 }
2923 }
2924 }
2925 }
2926 }
2927}
2928
2929impl From<gpu_alloc::AllocationError> for crate::DeviceError {
2930 fn from(error: gpu_alloc::AllocationError) -> Self {
2931 use gpu_alloc::AllocationError as Ae;
2932 match error {
2933 Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::TooManyObjects => Self::OutOfMemory,
2934 Ae::NoCompatibleMemoryTypes => crate::hal_usage_error(error),
2935 }
2936 }
2937}
2938impl From<gpu_alloc::MapError> for crate::DeviceError {
2939 fn from(error: gpu_alloc::MapError) -> Self {
2940 use gpu_alloc::MapError as Me;
2941 match error {
2942 Me::OutOfDeviceMemory | Me::OutOfHostMemory | Me::MapFailed => Self::OutOfMemory,
2943 Me::NonHostVisible | Me::AlreadyMapped => crate::hal_usage_error(error),
2944 }
2945 }
2946}
2947impl From<gpu_descriptor::AllocationError> for crate::DeviceError {
2948 fn from(error: gpu_descriptor::AllocationError) -> Self {
2949 use gpu_descriptor::AllocationError as Ae;
2950 match error {
2951 Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::Fragmentation => Self::OutOfMemory,
2952 }
2953 }
2954}
2955
2956fn handle_unexpected(err: vk::Result) -> ! {
2963 panic!("Unexpected Vulkan error: `{err}`")
2964}
2965
2966struct ImageWithoutMemory {
2967 raw: vk::Image,
2968 requirements: vk::MemoryRequirements,
2969 copy_size: crate::CopyExtent,
2970}