1use alloc::{borrow::ToOwned as _, collections::BTreeMap, ffi::CString, sync::Arc, vec::Vec};
2use core::{
3 ffi::CStr,
4 mem::{self, MaybeUninit},
5 num::NonZeroU32,
6 ptr,
7 time::Duration,
8};
9
10use arrayvec::ArrayVec;
11use ash::{ext, vk};
12use hashbrown::hash_map::Entry;
13use wgpu_sync::{Mutex, RwLock};
14
15use super::{conv, descriptor::DescriptorCounts, RawTlasInstance};
16use crate::TlasInstance;
17
18impl super::DeviceShared {
19 pub(super) unsafe fn set_object_name(&self, object: impl vk::Handle, name: &str) {
37 let Some(extension) = self.extension_fns.debug_utils.as_ref() else {
38 return;
39 };
40
41 let mut buffer: [u8; 64] = [0u8; 64];
44 let buffer_vec: Vec<u8>;
45
46 let name_bytes = if name.len() < buffer.len() {
48 buffer[..name.len()].copy_from_slice(name.as_bytes());
50 buffer[name.len()] = 0;
52 &buffer[..name.len() + 1]
53 } else {
54 buffer_vec = name
57 .as_bytes()
58 .iter()
59 .cloned()
60 .chain(core::iter::once(0))
61 .collect();
62 &buffer_vec
63 };
64
65 let name = CStr::from_bytes_until_nul(name_bytes).expect("We have added a null byte");
66
67 let _result = unsafe {
68 extension.set_debug_utils_object_name(
69 &vk::DebugUtilsObjectNameInfoEXT::default()
70 .object_handle(object)
71 .object_name(name),
72 )
73 };
74 }
75
76 pub fn make_render_pass(
77 &self,
78 key: super::RenderPassKey,
79 ) -> Result<vk::RenderPass, crate::DeviceError> {
80 Ok(match self.render_passes.lock().entry(key) {
81 Entry::Occupied(e) => *e.get(),
82 Entry::Vacant(e) => {
83 let super::RenderPassKey {
84 ref colors,
85 ref depth_stencil,
86 sample_count,
87 multiview_mask,
88 depth_read_only,
89 stencil_read_only,
90 } = *e.key();
91
92 let mut vk_attachments = Vec::new();
93 let mut color_refs = Vec::with_capacity(colors.len());
94 let mut resolve_refs = Vec::with_capacity(color_refs.capacity());
95 let mut ds_ref = None;
96 let samples = vk::SampleCountFlags::from_raw(sample_count);
97 let unused = vk::AttachmentReference {
98 attachment: vk::ATTACHMENT_UNUSED,
99 layout: vk::ImageLayout::UNDEFINED,
100 };
101 for cat in colors.iter() {
102 let (color_ref, resolve_ref) =
103 if let Some(super::ColorAttachmentKey { base, resolve }) = cat {
104 let super::AttachmentKey {
105 format,
106 layout,
107 ops,
108 } = *base;
109
110 let color_ref = vk::AttachmentReference {
111 attachment: vk_attachments.len() as u32,
112 layout,
113 };
114 vk_attachments.push({
115 let (load_op, store_op) = conv::map_attachment_ops(ops);
116 vk::AttachmentDescription::default()
117 .format(format)
118 .samples(samples)
119 .load_op(load_op)
120 .store_op(store_op)
121 .initial_layout(layout)
122 .final_layout(layout)
123 });
124 let resolve_ref = if let Some(rat) = resolve {
125 let super::AttachmentKey {
126 format,
127 layout,
128 ops,
129 } = *rat;
130
131 let (load_op, store_op) = conv::map_attachment_ops(ops);
132 let vk_attachment = vk::AttachmentDescription::default()
133 .format(format)
134 .samples(vk::SampleCountFlags::TYPE_1)
135 .load_op(load_op)
136 .store_op(store_op)
137 .initial_layout(layout)
138 .final_layout(layout);
139 vk_attachments.push(vk_attachment);
140
141 vk::AttachmentReference {
142 attachment: vk_attachments.len() as u32 - 1,
143 layout,
144 }
145 } else {
146 unused
147 };
148
149 (color_ref, resolve_ref)
150 } else {
151 (unused, unused)
152 };
153
154 color_refs.push(color_ref);
155 resolve_refs.push(resolve_ref);
156 }
157
158 if let Some(ds) = depth_stencil {
159 let super::DepthStencilAttachmentKey {
160 ref base,
161 stencil_ops,
162 } = *ds;
163
164 let super::AttachmentKey {
165 format,
166 layout,
167 ops,
168 } = *base;
169
170 ds_ref = Some(vk::AttachmentReference {
171 attachment: vk_attachments.len() as u32,
172 layout,
173 });
174 let (load_op, mut store_op) = conv::map_attachment_ops(ops);
175 let (stencil_load_op, mut stencil_store_op) =
176 conv::map_attachment_ops(stencil_ops);
177
178 let store_op_for_read_only = if self.private_caps.store_op_none {
179 vk::AttachmentStoreOp::NONE
180 } else {
181 vk::AttachmentStoreOp::STORE
182 };
183 if depth_read_only {
184 store_op = store_op_for_read_only;
185 }
186 if stencil_read_only {
187 stencil_store_op = store_op_for_read_only;
188 }
189
190 let vk_attachment = vk::AttachmentDescription::default()
191 .format(format)
192 .samples(samples)
193 .load_op(load_op)
194 .store_op(store_op)
195 .stencil_load_op(stencil_load_op)
196 .stencil_store_op(stencil_store_op)
197 .initial_layout(layout)
198 .final_layout(layout);
199 vk_attachments.push(vk_attachment);
200 }
201
202 let vk_subpasses = [{
203 let mut vk_subpass = vk::SubpassDescription::default()
204 .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
205 .color_attachments(&color_refs)
206 .resolve_attachments(&resolve_refs);
207
208 if self
209 .workarounds
210 .contains(super::Workarounds::EMPTY_RESOLVE_ATTACHMENT_LISTS)
211 && resolve_refs.is_empty()
212 {
213 vk_subpass.p_resolve_attachments = ptr::null();
214 }
215
216 if let Some(ref reference) = ds_ref {
217 vk_subpass = vk_subpass.depth_stencil_attachment(reference)
218 }
219 vk_subpass
220 }];
221
222 let mut vk_info = vk::RenderPassCreateInfo::default()
223 .attachments(&vk_attachments)
224 .subpasses(&vk_subpasses);
225
226 let mut multiview_info;
227 let mask;
228 if let Some(multiview_mask) = multiview_mask {
229 mask = [multiview_mask.get()];
230
231 multiview_info = vk::RenderPassMultiviewCreateInfoKHR::default()
233 .view_masks(&mask)
234 .correlation_masks(&mask);
235 vk_info = vk_info.push_next(&mut multiview_info);
236 }
237
238 let raw = unsafe {
239 self.raw
240 .create_render_pass(&vk_info, None)
241 .map_err(super::map_host_device_oom_err)?
242 };
243
244 *e.insert(raw)
245 }
246 })
247 }
248
249 fn make_memory_ranges<'a, I: 'a + Iterator<Item = crate::MemoryRange>>(
250 &self,
251 buffer: &'a super::Buffer,
252 ranges: I,
253 ) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
254 let super::BufferOwnership::Managed(ref allocation) = buffer.ownership else {
255 return None;
256 };
257 let allocation = allocation.lock();
258 let mask = self.private_caps.non_coherent_map_mask;
259 Some(ranges.map(move |range| {
260 vk::MappedMemoryRange::default()
261 .memory(allocation.memory())
262 .offset((allocation.offset() + range.start) & !mask)
263 .size((range.end - range.start + mask) & !mask)
264 }))
265 }
266}
267
268struct CompiledStage {
269 create_info: vk::PipelineShaderStageCreateInfo<'static>,
270 _entry_point: CString,
271 temp_raw_module: Option<vk::ShaderModule>,
272}
273
274struct MemoryProperties {
275 base: vk::PhysicalDeviceMemoryProperties,
276 heap_budget: ArrayVec<vk::DeviceSize, { vk::MAX_MEMORY_HEAPS }>,
277 heap_usage: ArrayVec<vk::DeviceSize, { vk::MAX_MEMORY_HEAPS }>,
278}
279
280impl MemoryProperties {
281 fn types(&self) -> &[vk::MemoryType] {
282 let count = self.base.memory_type_count as usize;
283 &self.base.memory_types[0..count]
284 }
285
286 fn heaps(&self) -> &[vk::MemoryHeap] {
287 let count = self.base.memory_heap_count as usize;
288 &self.base.memory_heaps[0..count]
289 }
290
291 fn heap_budget(&self) -> &[vk::DeviceSize] {
292 let count = self.base.memory_heap_count as usize;
293 &self.heap_budget[0..count]
294 }
295
296 fn heap_usage(&self) -> &[vk::DeviceSize] {
297 let count = self.base.memory_heap_count as usize;
298 &self.heap_usage[0..count]
299 }
300}
301
302impl super::Device {
303 pub unsafe fn texture_from_raw(
313 &self,
314 vk_image: vk::Image,
315 desc: &crate::TextureDescriptor,
316 drop_callback: Option<crate::DropCallback>,
317 memory: super::TextureMemory,
318 ) -> super::Texture {
319 let identity = self.shared.texture_identity_factory.next();
320 let drop_guard = crate::DropGuard::from_option(drop_callback);
321
322 if let Some(label) = desc.label {
323 unsafe { self.shared.set_object_name(vk_image, label) };
324 }
325
326 super::Texture {
327 raw: vk_image,
328 drop_guard,
329 memory,
330 format: desc.format,
331 copy_size: desc.copy_extent(),
332 identity,
333 }
334 }
335
336 fn find_memory_type_index(
337 &self,
338 type_bits_req: u32,
339 flags_req: vk::MemoryPropertyFlags,
340 ) -> Option<usize> {
341 let mem_properties = unsafe {
342 self.shared
343 .instance
344 .raw
345 .get_physical_device_memory_properties(self.shared.physical_device)
346 };
347
348 for (i, mem_ty) in mem_properties.memory_types_as_slice().iter().enumerate() {
350 let types_bits = 1 << i;
351 let is_required_memory_type = type_bits_req & types_bits != 0;
352 let has_required_properties = mem_ty.property_flags & flags_req == flags_req;
353 if is_required_memory_type && has_required_properties {
354 return Some(i);
355 }
356 }
357
358 None
359 }
360
361 fn create_image_without_memory(
362 &self,
363 desc: &crate::TextureDescriptor,
364 external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
365 ) -> Result<ImageWithoutMemory, crate::DeviceError> {
366 self.create_image_without_memory_with_tiling(
367 desc,
368 vk::ImageTiling::OPTIMAL,
369 external_memory_image_create_info,
370 None,
371 )
372 }
373
374 fn create_image_without_memory_with_tiling(
375 &self,
376 desc: &crate::TextureDescriptor,
377 tiling: vk::ImageTiling,
378 external_memory_image_create_info: Option<&mut vk::ExternalMemoryImageCreateInfo>,
379 drm_modifier_info: Option<&mut vk::ImageDrmFormatModifierExplicitCreateInfoEXT>,
380 ) -> Result<ImageWithoutMemory, crate::DeviceError> {
381 let copy_size = desc.copy_extent();
382
383 let mut raw_flags = vk::ImageCreateFlags::empty();
384 if desc.dimension == wgt::TextureDimension::D3
385 && desc.usage.contains(wgt::TextureUses::COLOR_TARGET)
386 {
387 raw_flags |= vk::ImageCreateFlags::TYPE_2D_ARRAY_COMPATIBLE;
388 }
389 if desc.is_cube_compatible() {
390 raw_flags |= vk::ImageCreateFlags::CUBE_COMPATIBLE;
391 }
392
393 let original_format = self.shared.private_caps.map_texture_format(desc.format);
394 let mut vk_view_formats = vec![];
395 if !desc.view_formats.is_empty() {
396 raw_flags |= vk::ImageCreateFlags::MUTABLE_FORMAT;
397
398 if self.shared.private_caps.image_format_list {
399 vk_view_formats = desc
400 .view_formats
401 .iter()
402 .map(|f| self.shared.private_caps.map_texture_format(*f))
403 .collect();
404 vk_view_formats.push(original_format)
405 }
406 }
407 if desc.format.is_multi_planar_format() {
408 raw_flags |=
409 vk::ImageCreateFlags::MUTABLE_FORMAT | vk::ImageCreateFlags::EXTENDED_USAGE;
410 }
411
412 let mut usage = conv::map_texture_usage(desc.usage);
413 if !usage.intersects(conv::VIEWABLE_IMAGE_USAGES)
418 && external_memory_image_create_info.is_none()
419 && super::adapter::supports_format(
420 &self.shared.instance.raw,
421 self.shared.physical_device,
422 original_format,
423 tiling,
424 vk::FormatFeatureFlags::SAMPLED_IMAGE,
425 )
426 {
427 usage |= vk::ImageUsageFlags::SAMPLED;
428 }
429
430 let mut vk_info = vk::ImageCreateInfo::default()
431 .flags(raw_flags)
432 .image_type(conv::map_texture_dimension(desc.dimension))
433 .format(original_format)
434 .extent(conv::map_copy_extent(©_size))
435 .mip_levels(desc.mip_level_count)
436 .array_layers(desc.array_layer_count())
437 .samples(vk::SampleCountFlags::from_raw(desc.sample_count))
438 .tiling(tiling)
439 .usage(usage)
440 .sharing_mode(vk::SharingMode::EXCLUSIVE)
441 .initial_layout(vk::ImageLayout::UNDEFINED);
442
443 let mut format_list_info = vk::ImageFormatListCreateInfo::default();
444 if !vk_view_formats.is_empty() {
445 format_list_info = format_list_info.view_formats(&vk_view_formats);
446 vk_info = vk_info.push_next(&mut format_list_info);
447 }
448
449 if let Some(ext_info) = external_memory_image_create_info {
450 vk_info = vk_info.push_next(ext_info);
451 }
452
453 if let Some(drm_info) = drm_modifier_info {
454 vk_info = vk_info.push_next(drm_info);
455 }
456
457 let raw = unsafe { self.shared.raw.create_image(&vk_info, None) }.map_err(map_err)?;
458 fn map_err(err: vk::Result) -> crate::DeviceError {
459 super::map_host_device_oom_and_ioca_err(err)
462 }
463 let mut req = unsafe { self.shared.raw.get_image_memory_requirements(raw) };
464
465 if desc.usage.contains(wgt::TextureUses::TRANSIENT) {
466 let mem_type_index = self.find_memory_type_index(
467 req.memory_type_bits,
468 vk::MemoryPropertyFlags::LAZILY_ALLOCATED,
469 );
470 if let Some(mem_type_index) = mem_type_index {
471 req.memory_type_bits = 1 << mem_type_index;
472 }
473 }
474
475 Ok(ImageWithoutMemory {
476 raw,
477 requirements: req,
478 })
479 }
480
481 #[cfg(windows)]
487 pub unsafe fn texture_from_d3d11_shared_handle(
488 &self,
489 d3d11_shared_handle: windows::Win32::Foundation::HANDLE,
490 desc: &crate::TextureDescriptor,
491 ) -> Result<super::Texture, crate::DeviceError> {
492 if !self
493 .shared
494 .enabled_extensions
495 .contains(&ash::khr::external_memory_win32::NAME)
496 {
497 log::error!("Vulkan driver does not support VK_KHR_external_memory_win32");
498 return Err(crate::DeviceError::Unexpected);
499 }
500
501 let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
502 .handle_types(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE);
503
504 let image =
505 self.create_image_without_memory(desc, Some(&mut external_memory_image_info))?;
506
507 let mut dedicated_allocate_info =
510 vk::MemoryDedicatedAllocateInfo::default().image(image.raw);
511
512 let mut import_memory_info = vk::ImportMemoryWin32HandleInfoKHR::default()
513 .handle_type(vk::ExternalMemoryHandleTypeFlags::D3D11_TEXTURE)
514 .handle(d3d11_shared_handle.0 as _);
515 #[allow(clippy::unnecessary_mut_passed)]
517 {
518 import_memory_info.p_next = <*const _>::cast(&mut dedicated_allocate_info);
519 }
520
521 let mem_type_index = self
522 .find_memory_type_index(
523 image.requirements.memory_type_bits,
524 vk::MemoryPropertyFlags::DEVICE_LOCAL,
525 )
526 .ok_or(crate::DeviceError::Unexpected)?;
527
528 let memory_allocate_info = vk::MemoryAllocateInfo::default()
529 .allocation_size(image.requirements.size)
530 .memory_type_index(mem_type_index as _)
531 .push_next(&mut import_memory_info);
532 let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
533 .map_err(super::map_host_device_oom_err)?;
534
535 unsafe { self.shared.raw.bind_image_memory(image.raw, memory, 0) }
536 .map_err(super::map_host_device_oom_err)?;
537
538 Ok(unsafe {
539 self.texture_from_raw(
540 image.raw,
541 desc,
542 None,
543 super::TextureMemory::Dedicated(memory),
544 )
545 })
546 }
547
548 #[cfg(unix)]
559 pub unsafe fn texture_from_dmabuf_fd(
560 &self,
561 fd: std::os::unix::io::OwnedFd,
562 desc: &crate::TextureDescriptor,
563 drm_modifier: u64,
564 stride: u64,
565 offset: u64,
566 ) -> Result<super::Texture, crate::DeviceError> {
567 use std::os::unix::io::IntoRawFd;
568
569 let enabled_extensions = &self.shared.enabled_extensions;
570 if !enabled_extensions.contains(&ext::external_memory_dma_buf::NAME)
571 || !enabled_extensions.contains(&ext::image_drm_format_modifier::NAME)
572 {
573 log::error!(
574 "Vulkan driver does not support VK_EXT_external_memory_dma_buf \
575 or VK_EXT_image_drm_format_modifier"
576 );
577 return Err(crate::DeviceError::Unexpected);
578 }
579
580 let external_memory_fd_fn = self
581 .shared
582 .extension_fns
583 .external_memory_fd
584 .as_ref()
585 .ok_or_else(|| {
586 log::error!("VK_KHR_external_memory_fd extension not loaded");
587 crate::DeviceError::Unexpected
588 })?;
589
590 let mut external_memory_image_info = vk::ExternalMemoryImageCreateInfo::default()
591 .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
592
593 let plane_layout = vk::SubresourceLayout {
594 offset,
595 row_pitch: stride,
596 size: 0,
597 array_pitch: 0,
598 depth_pitch: 0,
599 };
600 let mut drm_modifier_info = vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default()
601 .drm_format_modifier(drm_modifier)
602 .plane_layouts(core::slice::from_ref(&plane_layout));
603
604 let image = self.create_image_without_memory_with_tiling(
605 desc,
606 vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT,
607 Some(&mut external_memory_image_info),
608 Some(&mut drm_modifier_info),
609 )?;
610
611 let fd_raw = fd.into_raw_fd();
614
615 let result = self.import_dmabuf_memory(
616 external_memory_fd_fn,
617 fd_raw,
618 image.raw,
619 &image.requirements,
620 );
621
622 match result {
623 Ok(memory) => Ok(unsafe {
624 self.texture_from_raw(
625 image.raw,
626 desc,
627 None,
628 super::TextureMemory::Dedicated(memory),
629 )
630 }),
631 Err(e) => {
632 unsafe { self.shared.raw.destroy_image(image.raw, None) };
634 Err(e)
635 }
636 }
637 }
638
639 #[cfg(unix)]
644 fn import_dmabuf_memory(
645 &self,
646 external_memory_fd_fn: &ash::khr::external_memory_fd::Device,
647 fd_raw: i32,
648 image: vk::Image,
649 requirements: &vk::MemoryRequirements,
650 ) -> Result<vk::DeviceMemory, crate::DeviceError> {
651 let mut fd_props = vk::MemoryFdPropertiesKHR::default();
652 unsafe {
653 external_memory_fd_fn.get_memory_fd_properties(
654 vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
655 fd_raw,
656 &mut fd_props,
657 )
658 }
659 .map_err(|e| {
660 unsafe { libc::close(fd_raw) };
661 super::map_host_device_oom_err(e)
662 })?;
663
664 let mem_type_index = self
665 .find_memory_type_index(
666 requirements.memory_type_bits & fd_props.memory_type_bits,
667 vk::MemoryPropertyFlags::empty(),
668 )
669 .ok_or_else(|| {
670 unsafe { libc::close(fd_raw) };
671 crate::DeviceError::Unexpected
672 })?;
673
674 let mut dedicated_allocate_info = vk::MemoryDedicatedAllocateInfo::default().image(image);
675
676 let mut import_memory_info = vk::ImportMemoryFdInfoKHR::default()
677 .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT)
678 .fd(fd_raw);
679
680 let memory_allocate_info = vk::MemoryAllocateInfo::default()
681 .allocation_size(requirements.size)
682 .memory_type_index(mem_type_index as _)
683 .push_next(&mut import_memory_info)
684 .push_next(&mut dedicated_allocate_info);
685
686 let memory = unsafe { self.shared.raw.allocate_memory(&memory_allocate_info, None) }
689 .map_err(|e| {
690 unsafe { libc::close(fd_raw) };
691 super::map_host_device_oom_err(e)
692 })?;
693
694 unsafe { self.shared.raw.bind_image_memory(image, memory, 0) }.map_err(|e| {
696 unsafe { self.shared.raw.free_memory(memory, None) };
697 super::map_host_device_oom_err(e)
698 })?;
699
700 Ok(memory)
701 }
702
703 fn create_shader_module_impl(
704 &self,
705 spv: &[u32],
706 label: &crate::Label<'_>,
707 ) -> Result<vk::ShaderModule, crate::DeviceError> {
708 let vk_info = vk::ShaderModuleCreateInfo::default()
709 .flags(vk::ShaderModuleCreateFlags::empty())
710 .code(spv);
711
712 let raw = unsafe {
713 profiling::scope!("vkCreateShaderModule");
714 self.shared
715 .raw
716 .create_shader_module(&vk_info, None)
717 .map_err(map_err)?
718 };
719 fn map_err(err: vk::Result) -> crate::DeviceError {
720 super::map_host_device_oom_err(err)
723 }
724
725 if let Some(label) = label {
726 unsafe { self.shared.set_object_name(raw, label) };
727 }
728
729 Ok(raw)
730 }
731
732 fn compile_stage(
733 &self,
734 stage: &crate::ProgrammableStage<super::ShaderModule>,
735 naga_stage: naga::ShaderStage,
736 binding_map: &naga::back::spv::BindingMap,
737 ) -> Result<CompiledStage, crate::PipelineError> {
738 let stage_flags = crate::auxil::map_naga_stage(naga_stage);
739 let vk_module = match *stage.module {
740 super::ShaderModule::Raw(raw) => raw,
741 super::ShaderModule::Intermediate {
742 ref naga_shader,
743 runtime_checks,
744 } => {
745 let pipeline_options = naga::back::spv::PipelineOptions {
746 entry_point: stage.entry_point.to_owned(),
747 shader_stage: naga_stage,
748 };
749 let polyfill_zero_init = matches!(
750 naga_stage,
751 naga::ShaderStage::Task | naga::ShaderStage::Mesh
752 ) && self
753 .shared
754 .workarounds
755 .contains(super::Workarounds::POLYFILL_MESH_TASK_WORKGROUP_ZERO_INIT)
756 && self.naga_options.zero_initialize_workgroup_memory
757 == naga::back::spv::ZeroInitializeWorkgroupMemoryMode::Native;
758 let needs_temp_options = polyfill_zero_init
759 || !runtime_checks.bounds_checks
760 || !runtime_checks.force_loop_bounding
761 || !runtime_checks.ray_query_initialization_tracking
762 || !binding_map.is_empty()
763 || naga_shader.debug_source.is_some()
764 || !stage.zero_initialize_workgroup_memory
765 || !runtime_checks.task_shader_dispatch_tracking
766 || !runtime_checks.mesh_shader_primitive_indices_clamp
767 || !runtime_checks.int_div_checks;
768
769 let mut temp_options;
770 let options = if needs_temp_options {
771 temp_options = self.naga_options.clone();
772 if !runtime_checks.bounds_checks {
773 temp_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
774 index: naga::proc::BoundsCheckPolicy::Unchecked,
775 buffer: naga::proc::BoundsCheckPolicy::Unchecked,
776 image_load: naga::proc::BoundsCheckPolicy::Unchecked,
777 binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
778 };
779 }
780 if !runtime_checks.force_loop_bounding {
781 temp_options.force_loop_bounding = false;
782 }
783 if !runtime_checks.ray_query_initialization_tracking {
784 temp_options.ray_query_initialization_tracking = false;
785 }
786 if !binding_map.is_empty() {
787 temp_options.binding_map = binding_map.clone();
788 }
789
790 if let Some(ref debug) = naga_shader.debug_source {
791 temp_options.debug_info = Some(naga::back::spv::DebugInfo {
792 source_code: &debug.source_code,
793 file_name: debug.file_name.as_ref(),
794 language: naga::back::spv::SourceLanguage::WGSL,
795 })
796 }
797 if !stage.zero_initialize_workgroup_memory {
798 temp_options.zero_initialize_workgroup_memory =
799 naga::back::spv::ZeroInitializeWorkgroupMemoryMode::None;
800 } else if polyfill_zero_init {
801 temp_options.zero_initialize_workgroup_memory =
802 naga::back::spv::ZeroInitializeWorkgroupMemoryMode::Polyfill;
803 }
804 if !runtime_checks.task_shader_dispatch_tracking {
805 temp_options.task_dispatch_limits = None;
806 }
807 temp_options.mesh_shader_primitive_indices_clamp =
808 runtime_checks.mesh_shader_primitive_indices_clamp;
809 temp_options.emit_int_div_checks = runtime_checks.int_div_checks;
810
811 &temp_options
812 } else {
813 &self.naga_options
814 };
815
816 let (module, info) = naga::back::pipeline_constants::process_overrides(
817 &naga_shader.module,
818 &naga_shader.info,
819 Some((naga_stage, stage.entry_point)),
820 stage.constants,
821 )
822 .map_err(|e| {
823 crate::PipelineError::PipelineConstants(stage_flags, format!("{e}"))
824 })?;
825
826 let spv = {
827 profiling::scope!("naga::spv::write_vec");
828 naga::back::spv::write_vec(&module, &info, options, Some(&pipeline_options))
829 }
830 .map_err(|e| crate::PipelineError::Linkage(stage_flags, format!("{e}")))?;
831 self.create_shader_module_impl(&spv, &None)?
832 }
833 };
834
835 let mut flags = vk::PipelineShaderStageCreateFlags::empty();
836 if self.shared.features.contains(wgt::Features::SUBGROUP) {
837 flags |= vk::PipelineShaderStageCreateFlags::ALLOW_VARYING_SUBGROUP_SIZE
838 }
839
840 let entry_point = CString::new(stage.entry_point).unwrap();
841 let mut create_info = vk::PipelineShaderStageCreateInfo::default()
842 .flags(flags)
843 .stage(conv::map_shader_stage(stage_flags))
844 .module(vk_module);
845
846 create_info.p_name = entry_point.as_ptr();
848
849 Ok(CompiledStage {
850 create_info,
851 _entry_point: entry_point,
852 temp_raw_module: match *stage.module {
853 super::ShaderModule::Raw(_) => None,
854 super::ShaderModule::Intermediate { .. } => Some(vk_module),
855 },
856 })
857 }
858
859 pub fn queue_family_index(&self) -> u32 {
865 self.shared.family_index
866 }
867
868 pub fn queue_index(&self) -> u32 {
869 self.shared.queue_index
870 }
871
872 pub fn raw_device(&self) -> &ash::Device {
873 &self.shared.raw
874 }
875
876 pub fn raw_physical_device(&self) -> vk::PhysicalDevice {
877 self.shared.physical_device
878 }
879
880 pub fn raw_queue(&self) -> vk::Queue {
881 self.shared.raw_queue
882 }
883
884 pub fn enabled_device_extensions(&self) -> &[&'static CStr] {
885 &self.shared.enabled_extensions
886 }
887
888 pub fn shared_instance(&self) -> &super::InstanceShared {
889 &self.shared.instance
890 }
891
892 fn get_memory_properties(&self) -> Option<MemoryProperties> {
893 if !self
894 .shared
895 .enabled_extensions
896 .contains(&ext::memory_budget::NAME)
897 {
898 return None;
899 }
900
901 let get_physical_device_properties = self
902 .shared
903 .instance
904 .get_physical_device_properties
905 .as_ref()
906 .unwrap();
907
908 let mut budget = vk::PhysicalDeviceMemoryBudgetPropertiesEXT::default();
909 let mut props = vk::PhysicalDeviceMemoryProperties2::default().push_next(&mut budget);
910
911 unsafe {
912 get_physical_device_properties
913 .get_physical_device_memory_properties2(self.shared.physical_device, &mut props);
914 }
915
916 let vk::PhysicalDeviceMemoryProperties2 {
917 memory_properties, ..
918 } = props;
919
920 let mut heap_budget = ArrayVec::from(budget.heap_budget);
921 let mut heap_usage = ArrayVec::from(budget.heap_usage);
922 heap_budget.truncate(memory_properties.memory_heap_count as usize);
923 heap_usage.truncate(memory_properties.memory_heap_count as usize);
924
925 Some(MemoryProperties {
926 base: memory_properties,
927 heap_budget,
928 heap_usage,
929 })
930 }
931
932 fn error_if_would_oom_on_resource_allocation(
941 &self,
942 location: gpu_allocator::MemoryLocation,
943 requirements: &vk::MemoryRequirements,
944 ) -> Result<(), crate::DeviceError> {
945 use gpu_allocator::MemoryLocation;
946
947 let Some(threshold) = self
948 .shared
949 .instance
950 .memory_budget_thresholds
951 .for_resource_creation
952 else {
953 return Ok(());
954 };
955
956 let Some(memory_properties) = self.get_memory_properties() else {
957 return Ok(());
958 };
959
960 let preferred_flags = match location {
961 MemoryLocation::GpuOnly => vk::MemoryPropertyFlags::DEVICE_LOCAL,
962 MemoryLocation::CpuToGpu => {
963 vk::MemoryPropertyFlags::HOST_VISIBLE
964 | vk::MemoryPropertyFlags::HOST_COHERENT
965 | vk::MemoryPropertyFlags::DEVICE_LOCAL
966 }
967 MemoryLocation::GpuToCpu => {
968 vk::MemoryPropertyFlags::HOST_VISIBLE
969 | vk::MemoryPropertyFlags::HOST_COHERENT
970 | vk::MemoryPropertyFlags::HOST_CACHED
971 }
972 MemoryLocation::Unknown => vk::MemoryPropertyFlags::empty(),
973 };
974
975 let mut selected_heap = memory_properties
976 .types()
977 .iter()
978 .enumerate()
979 .find(|(i, ty)| {
980 (1 << i) & requirements.memory_type_bits != 0
981 && ty.property_flags.contains(preferred_flags)
982 });
983
984 if selected_heap.is_none() {
985 let required_flags = match location {
986 MemoryLocation::GpuOnly => vk::MemoryPropertyFlags::DEVICE_LOCAL,
987 MemoryLocation::CpuToGpu | MemoryLocation::GpuToCpu => {
988 vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT
989 }
990 MemoryLocation::Unknown => vk::MemoryPropertyFlags::empty(),
991 };
992 selected_heap = memory_properties
993 .types()
994 .iter()
995 .enumerate()
996 .find(|(i, ty)| {
997 (1 << i) & requirements.memory_type_bits != 0
998 && ty.property_flags.contains(required_flags)
999 });
1000 }
1001
1002 if let Some((_, ty)) = selected_heap {
1003 let i = ty.heap_index as usize;
1004 let heap_usage = memory_properties.heap_usage()[i];
1005 let heap_budget = memory_properties.heap_budget()[i];
1006 if heap_usage + requirements.size < heap_budget / 100 * threshold as u64 {
1007 Ok(())
1008 } else {
1009 log::warn!(
1010 "Allocation would result in an OOM condition\n\
1011 Request: {requirements:?}\n\
1012 Heap {index} had {heap_usage}B used of {heap_budget}B total before this request.",
1013 index = ty.heap_index,
1014 );
1015 Err(crate::DeviceError::OutOfMemory)
1016 }
1017 } else {
1018 log::warn!("Failed to find a suitable heap for {requirements:?}");
1019 Err(crate::DeviceError::OutOfMemory)
1020 }
1021 }
1022}
1023
1024impl crate::Device for super::Device {
1025 type A = super::Api;
1026
1027 unsafe fn create_buffer(
1028 &self,
1029 desc: &crate::BufferDescriptor,
1030 ) -> Result<(super::Buffer, wgt::BufferAddress), crate::DeviceError> {
1031 let vk_info = vk::BufferCreateInfo::default()
1032 .size(desc.size)
1033 .usage(conv::map_buffer_usage(desc.usage))
1034 .sharing_mode(vk::SharingMode::EXCLUSIVE);
1035
1036 let raw = unsafe {
1037 self.shared
1038 .raw
1039 .create_buffer(&vk_info, None)
1040 .map_err(super::map_host_device_oom_and_ioca_err)?
1041 };
1042
1043 let mut requirements = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
1044
1045 let is_cpu_read = desc.usage.contains(wgt::BufferUses::MAP_READ);
1046 let is_cpu_write = desc.usage.contains(wgt::BufferUses::MAP_WRITE);
1047
1048 let location = match (is_cpu_read, is_cpu_write) {
1049 (true, true) => gpu_allocator::MemoryLocation::CpuToGpu,
1050 (true, false) => gpu_allocator::MemoryLocation::GpuToCpu,
1051 (false, true) => gpu_allocator::MemoryLocation::CpuToGpu,
1052 (false, false) => gpu_allocator::MemoryLocation::GpuOnly,
1053 };
1054
1055 self.error_if_would_oom_on_resource_allocation(location, &requirements)
1056 .inspect_err(|_| {
1057 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1058 })?;
1059
1060 let name = desc.label.unwrap_or("Unlabeled buffer");
1061
1062 if desc
1063 .usage
1064 .contains(wgt::BufferUses::ACCELERATION_STRUCTURE_SCRATCH)
1065 {
1066 requirements.alignment = requirements
1068 .alignment
1069 .max(self.shared.private_caps.scratch_buffer_alignment as u64);
1070 }
1071
1072 if desc
1073 .usage
1074 .contains(wgt::BufferUses::RAY_TRACING_PIPELINE_SHADER_DATA)
1075 {
1076 requirements.alignment = requirements
1077 .alignment
1078 .max(self.shared.private_caps.shader_binding_table_alignment as u64);
1079 }
1080
1081 let allocation = self
1082 .mem_allocator
1083 .allocate(
1084 super::MemoryPool::from_memory_flags(desc.memory_flags),
1085 &gpu_allocator::vulkan::AllocationCreateDesc {
1086 name,
1087 requirements: vk::MemoryRequirements {
1088 memory_type_bits: requirements.memory_type_bits
1089 & self.valid_ash_memory_types,
1090 ..requirements
1091 },
1092 location,
1093 linear: true, allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
1095 },
1096 )
1097 .inspect_err(|_| {
1098 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1099 })?;
1100
1101 unsafe {
1102 self.shared
1103 .raw
1104 .bind_buffer_memory(raw, allocation.memory(), allocation.offset())
1105 }
1106 .map_err(super::map_host_device_oom_and_ioca_err)
1107 .inspect_err(|_| {
1108 unsafe { self.shared.raw.destroy_buffer(raw, None) };
1109 })?;
1110
1111 if let Some(label) = desc.label {
1112 unsafe { self.shared.set_object_name(raw, label) };
1113 }
1114
1115 self.counters.buffer_memory.add(allocation.size() as isize);
1116 self.counters.buffers.add(1);
1117
1118 Ok((
1119 super::Buffer {
1120 raw,
1121 ownership: super::BufferOwnership::Managed(Mutex::new(
1122 super::BufferMemoryBacking::Managed(allocation),
1123 )),
1124 },
1125 desc.size,
1126 ))
1127 }
1128 unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
1129 match buffer.ownership {
1130 super::BufferOwnership::Managed(allocation) => {
1131 unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1132 let allocation = allocation.into_inner();
1133 self.counters.buffer_memory.sub(allocation.size() as isize);
1134 match allocation {
1135 super::BufferMemoryBacking::Managed(allocation) => {
1136 let result = self.mem_allocator.free(allocation);
1137 if let Err(err) = result {
1138 log::warn!("Failed to free buffer allocation: {err}");
1139 }
1140 }
1141 super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
1142 self.shared.raw.free_memory(memory, None);
1143 },
1144 }
1145 }
1146 super::BufferOwnership::RawHandle => {
1147 unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
1148 }
1149 super::BufferOwnership::External(_drop_guard) => {
1150 }
1153 }
1154
1155 self.counters.buffers.sub(1);
1156 }
1157
1158 unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) {
1159 self.counters.buffers.add(1);
1160 }
1161
1162 unsafe fn map_buffer(
1163 &self,
1164 buffer: &super::Buffer,
1165 range: crate::MemoryRange,
1166 ) -> Result<crate::BufferMapping, crate::DeviceError> {
1167 let super::BufferOwnership::Managed(ref allocation) = buffer.ownership else {
1168 crate::hal_usage_error("tried to map external buffer")
1169 };
1170 let mut allocation = allocation.lock();
1171 let super::BufferMemoryBacking::Managed(ref mut allocation) = *allocation else {
1172 crate::hal_usage_error("tried to map externally created buffer")
1173 };
1174 let is_coherent = allocation
1175 .memory_properties()
1176 .contains(vk::MemoryPropertyFlags::HOST_COHERENT);
1177 Ok(crate::BufferMapping {
1178 ptr: unsafe {
1179 allocation
1180 .mapped_ptr()
1181 .unwrap()
1182 .cast()
1183 .offset(range.start as isize)
1184 },
1185 is_coherent,
1186 })
1187 }
1188
1189 unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
1190 match buffer.ownership {
1191 super::BufferOwnership::Managed(_) => {
1192 }
1194 super::BufferOwnership::RawHandle | super::BufferOwnership::External(_) => {
1195 crate::hal_usage_error("tried to unmap external buffer")
1196 }
1197 }
1198 }
1199
1200 unsafe fn flush_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1201 where
1202 I: Iterator<Item = crate::MemoryRange>,
1203 {
1204 if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1205 unsafe {
1206 self.shared
1207 .raw
1208 .flush_mapped_memory_ranges(
1209 &smallvec::SmallVec::<[vk::MappedMemoryRange; 32]>::from_iter(vk_ranges),
1210 )
1211 }
1212 .unwrap();
1213 }
1214 }
1215 unsafe fn invalidate_mapped_ranges<I>(&self, buffer: &super::Buffer, ranges: I)
1216 where
1217 I: Iterator<Item = crate::MemoryRange>,
1218 {
1219 if let Some(vk_ranges) = self.shared.make_memory_ranges(buffer, ranges) {
1220 unsafe {
1221 self.shared
1222 .raw
1223 .invalidate_mapped_memory_ranges(&smallvec::SmallVec::<
1224 [vk::MappedMemoryRange; 32],
1225 >::from_iter(vk_ranges))
1226 }
1227 .unwrap();
1228 }
1229 }
1230
1231 unsafe fn create_texture(
1232 &self,
1233 desc: &crate::TextureDescriptor,
1234 ) -> Result<super::Texture, crate::DeviceError> {
1235 let image = self.create_image_without_memory(desc, None)?;
1236
1237 self.error_if_would_oom_on_resource_allocation(
1238 gpu_allocator::MemoryLocation::GpuOnly,
1239 &image.requirements,
1240 )
1241 .inspect_err(|_| {
1242 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1243 })?;
1244
1245 let name = desc.label.unwrap_or("Unlabeled texture");
1246
1247 let allocation = self
1248 .mem_allocator
1249 .allocate(
1250 super::MemoryPool::from_memory_flags(desc.memory_flags),
1251 &gpu_allocator::vulkan::AllocationCreateDesc {
1252 name,
1253 requirements: vk::MemoryRequirements {
1254 memory_type_bits: image.requirements.memory_type_bits
1255 & self.valid_ash_memory_types,
1256 ..image.requirements
1257 },
1258 location: gpu_allocator::MemoryLocation::GpuOnly,
1259 linear: false,
1260 allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
1261 },
1262 )
1263 .inspect_err(|_| {
1264 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1265 })?;
1266
1267 unsafe {
1268 self.shared
1269 .raw
1270 .bind_image_memory(image.raw, allocation.memory(), allocation.offset())
1271 }
1272 .map_err(super::map_host_device_oom_err)
1273 .inspect_err(|_| {
1274 unsafe { self.shared.raw.destroy_image(image.raw, None) };
1275 })?;
1276
1277 self.counters.texture_memory.add(allocation.size() as isize);
1278 self.counters.textures.add(1);
1279
1280 Ok(unsafe {
1281 self.texture_from_raw(
1282 image.raw,
1283 desc,
1284 None,
1285 super::TextureMemory::Allocation(allocation),
1286 )
1287 })
1288 }
1289
1290 unsafe fn destroy_texture(&self, texture: super::Texture) {
1291 if texture.drop_guard.is_none() {
1292 unsafe { self.shared.raw.destroy_image(texture.raw, None) };
1293 }
1294
1295 match texture.memory {
1296 super::TextureMemory::Allocation(allocation) => {
1297 self.counters.texture_memory.sub(allocation.size() as isize);
1298 let result = self.mem_allocator.free(allocation);
1299 if let Err(err) = result {
1300 log::warn!("Failed to free texture allocation: {err}");
1301 }
1302 }
1303 super::TextureMemory::Dedicated(memory) => unsafe {
1304 self.shared.raw.free_memory(memory, None);
1305 },
1306 super::TextureMemory::External => {}
1307 }
1308
1309 self.counters.textures.sub(1);
1310 }
1311
1312 unsafe fn add_raw_texture(&self, _texture: &super::Texture) {
1313 self.counters.textures.add(1);
1314 }
1315
1316 unsafe fn create_texture_view(
1317 &self,
1318 texture: &super::Texture,
1319 desc: &crate::TextureViewDescriptor,
1320 ) -> Result<super::TextureView, crate::DeviceError> {
1321 let mut swizzle = desc.swizzle;
1322
1323 if texture.format.is_depth_stencil_format()
1331 && !self.shared.private_caps.depth_stencil_swizzle_one_support
1332 {
1333 for component in [
1334 &mut swizzle.r,
1335 &mut swizzle.g,
1336 &mut swizzle.b,
1337 &mut swizzle.a,
1338 ] {
1339 if *component == wgt::ComponentSwizzle::One {
1340 *component = wgt::ComponentSwizzle::A;
1341 }
1342 }
1343 }
1344
1345 let subresource_range = conv::map_subresource_range(&desc.range, texture.format);
1346 let raw_format = self.shared.private_caps.map_texture_format(desc.format);
1347 let mut vk_info = vk::ImageViewCreateInfo::default()
1348 .flags(vk::ImageViewCreateFlags::empty())
1349 .image(texture.raw)
1350 .view_type(conv::map_view_dimension(desc.dimension))
1351 .format(raw_format)
1352 .subresource_range(subresource_range)
1353 .components(conv::map_texture_component_swizzle(swizzle));
1354 let layers =
1355 NonZeroU32::new(subresource_range.layer_count).expect("Unexpected zero layer count");
1356
1357 let mut image_view_info;
1358 if self.shared.private_caps.image_view_usage && !desc.usage.is_empty() {
1359 image_view_info =
1360 vk::ImageViewUsageCreateInfo::default().usage(conv::map_texture_usage(desc.usage));
1361 vk_info = vk_info.push_next(&mut image_view_info);
1362 }
1363
1364 let raw = unsafe { self.shared.raw.create_image_view(&vk_info, None) }
1365 .map_err(super::map_host_device_oom_and_ioca_err)?;
1366
1367 if let Some(label) = desc.label {
1368 unsafe { self.shared.set_object_name(raw, label) };
1369 }
1370
1371 let identity = self.shared.texture_view_identity_factory.next();
1372
1373 self.counters.texture_views.add(1);
1374
1375 Ok(super::TextureView {
1376 raw_texture: texture.raw,
1377 raw,
1378 _layers: layers,
1379 format: desc.format,
1380 raw_format,
1381 base_mip_level: desc.range.base_mip_level,
1382 dimension: desc.dimension,
1383 texture_identity: texture.identity,
1384 view_identity: identity,
1385 })
1386 }
1387 unsafe fn destroy_texture_view(&self, view: super::TextureView) {
1388 unsafe { self.shared.raw.destroy_image_view(view.raw, None) };
1389
1390 self.counters.texture_views.sub(1);
1391 }
1392
1393 unsafe fn create_sampler(
1394 &self,
1395 desc: &crate::SamplerDescriptor,
1396 ) -> Result<super::Sampler, crate::DeviceError> {
1397 let mut create_info = vk::SamplerCreateInfo::default()
1398 .flags(vk::SamplerCreateFlags::empty())
1399 .mag_filter(conv::map_filter_mode(desc.mag_filter))
1400 .min_filter(conv::map_filter_mode(desc.min_filter))
1401 .mipmap_mode(conv::map_mip_filter_mode(desc.mipmap_filter))
1402 .address_mode_u(conv::map_address_mode(desc.address_modes[0]))
1403 .address_mode_v(conv::map_address_mode(desc.address_modes[1]))
1404 .address_mode_w(conv::map_address_mode(desc.address_modes[2]))
1405 .min_lod(desc.lod_clamp.start)
1406 .max_lod(desc.lod_clamp.end);
1407
1408 if let Some(fun) = desc.compare {
1409 create_info = create_info
1410 .compare_enable(true)
1411 .compare_op(conv::map_comparison(fun));
1412 }
1413
1414 if desc.anisotropy_clamp != 1 {
1415 create_info = create_info
1418 .anisotropy_enable(true)
1419 .max_anisotropy(desc.anisotropy_clamp as f32);
1420 }
1421
1422 if let Some(color) = desc.border_color {
1423 create_info = create_info.border_color(conv::map_border_color(color));
1424 }
1425
1426 let mut sampler_cache_guard = self.shared.sampler_cache.lock();
1427
1428 let raw = sampler_cache_guard.create_sampler(&self.shared.raw, create_info)?;
1429
1430 if let Some(label) = desc.label {
1434 unsafe { self.shared.set_object_name(raw, label) };
1437 }
1438
1439 drop(sampler_cache_guard);
1440
1441 self.counters.samplers.add(1);
1442
1443 Ok(super::Sampler { raw, create_info })
1444 }
1445 unsafe fn destroy_sampler(&self, sampler: super::Sampler) {
1446 self.shared.sampler_cache.lock().destroy_sampler(
1447 &self.shared.raw,
1448 sampler.create_info,
1449 sampler.raw,
1450 );
1451
1452 self.counters.samplers.sub(1);
1453 }
1454
1455 unsafe fn create_command_encoder(
1456 &self,
1457 desc: &crate::CommandEncoderDescriptor<super::Queue>,
1458 ) -> Result<super::CommandEncoder, crate::DeviceError> {
1459 let vk_info = vk::CommandPoolCreateInfo::default()
1460 .queue_family_index(desc.queue.family_index)
1461 .flags(vk::CommandPoolCreateFlags::TRANSIENT);
1462
1463 let raw = unsafe {
1464 self.shared
1465 .raw
1466 .create_command_pool(&vk_info, None)
1467 .map_err(super::map_host_device_oom_err)?
1468 };
1469
1470 self.counters.command_encoders.add(1);
1471
1472 Ok(super::CommandEncoder {
1473 raw,
1474 device: Arc::clone(&self.shared),
1475 active: vk::CommandBuffer::null(),
1476 bind_point: vk::PipelineBindPoint::default(),
1477 temp: super::Temp::default(),
1478 free: Vec::new(),
1479 discarded: Vec::new(),
1480 rpass_debug_marker_active: false,
1481 end_of_pass_timer_query: None,
1482 framebuffers: Default::default(),
1483 temp_texture_views: Default::default(),
1484 counters: Arc::clone(&self.counters),
1485 current_pipeline_is_multiview: false,
1486 })
1487 }
1488
1489 unsafe fn create_bind_group_layout(
1490 &self,
1491 desc: &crate::BindGroupLayoutDescriptor,
1492 ) -> Result<super::BindGroupLayout, crate::DeviceError> {
1493 let mut vk_bindings = Vec::new();
1498 let mut binding_flags = Vec::new();
1499 let mut binding_map = Vec::new();
1500 let mut next_binding = 0;
1501 let mut contains_binding_arrays = false;
1502 let mut desc_count = DescriptorCounts::default();
1503 for entry in desc.entries {
1504 if entry.count.is_some() {
1505 contains_binding_arrays = true;
1506 }
1507
1508 let partially_bound = desc
1509 .flags
1510 .contains(crate::BindGroupLayoutFlags::PARTIALLY_BOUND);
1511 let mut flags = vk::DescriptorBindingFlags::empty();
1512 if partially_bound && entry.count.is_some() {
1513 flags |= vk::DescriptorBindingFlags::PARTIALLY_BOUND;
1514 }
1515 if entry.count.is_some() {
1516 flags |= vk::DescriptorBindingFlags::UPDATE_AFTER_BIND;
1517 }
1518
1519 let count = entry.count.map_or(1, |c| c.get());
1520 match entry.ty {
1521 wgt::BindingType::ExternalTexture => {
1522 assert!(
1523 count == 1 && entry.count.is_none() && flags.is_empty(),
1524 "ExternalTexture binding arrays are not supported"
1525 );
1526 vk_bindings.extend((next_binding..next_binding + 3).map(|binding| {
1527 vk::DescriptorSetLayoutBinding {
1528 binding,
1529 descriptor_type: vk::DescriptorType::SAMPLED_IMAGE,
1530 descriptor_count: 1,
1531 stage_flags: conv::map_shader_stage(entry.visibility),
1532 p_immutable_samplers: ptr::null(),
1533 _marker: Default::default(),
1534 }
1535 }));
1536 vk_bindings.push(vk::DescriptorSetLayoutBinding {
1537 binding: next_binding + 3,
1538 descriptor_type: vk::DescriptorType::UNIFORM_BUFFER,
1539 descriptor_count: 1,
1540 stage_flags: conv::map_shader_stage(entry.visibility),
1541 p_immutable_samplers: ptr::null(),
1542 _marker: Default::default(),
1543 });
1544 binding_flags.extend(core::iter::repeat_n(flags, 4));
1545 binding_map.push((
1546 entry.binding,
1547 super::BindingInfo {
1548 binding: next_binding,
1549 binding_array_size: None,
1550 external_texture: true,
1551 },
1552 ));
1553 next_binding += 4;
1554 }
1555 _ => {
1556 vk_bindings.push(vk::DescriptorSetLayoutBinding {
1557 binding: next_binding,
1558 descriptor_type: conv::map_binding_type(entry.ty),
1559 descriptor_count: count,
1560 stage_flags: conv::map_shader_stage(entry.visibility),
1561 p_immutable_samplers: ptr::null(),
1562 _marker: Default::default(),
1563 });
1564 binding_flags.push(flags);
1565 binding_map.push((
1566 entry.binding,
1567 super::BindingInfo {
1568 binding: next_binding,
1569 binding_array_size: entry.count,
1570 external_texture: false,
1571 },
1572 ));
1573 next_binding += 1;
1574 }
1575 }
1576
1577 match entry.ty {
1578 wgt::BindingType::Buffer {
1579 ty,
1580 has_dynamic_offset,
1581 ..
1582 } => match ty {
1583 wgt::BufferBindingType::Uniform => {
1584 if has_dynamic_offset {
1585 desc_count.uniform_buffer_dynamic += count;
1586 } else {
1587 desc_count.uniform_buffer += count;
1588 }
1589 }
1590 wgt::BufferBindingType::Storage { .. } => {
1591 if has_dynamic_offset {
1592 desc_count.storage_buffer_dynamic += count;
1593 } else {
1594 desc_count.storage_buffer += count;
1595 }
1596 }
1597 },
1598 wgt::BindingType::Sampler { .. } => {
1599 desc_count.sampler += count;
1600 }
1601 wgt::BindingType::Texture { .. } => {
1602 desc_count.sampled_image += count;
1603 }
1604 wgt::BindingType::StorageTexture { .. } => {
1605 desc_count.storage_image += count;
1606 }
1607 wgt::BindingType::AccelerationStructure { .. } => {
1608 desc_count.acceleration_structure += count;
1609 }
1610 wgt::BindingType::ExternalTexture => {
1611 desc_count.sampled_image += 3;
1612 desc_count.uniform_buffer += 1;
1613 }
1614 }
1615 }
1616
1617 let vk_info = vk::DescriptorSetLayoutCreateInfo::default()
1618 .bindings(&vk_bindings)
1619 .flags(if contains_binding_arrays {
1620 vk::DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND_POOL
1621 } else {
1622 vk::DescriptorSetLayoutCreateFlags::empty()
1623 });
1624
1625 let mut binding_flag_info =
1626 vk::DescriptorSetLayoutBindingFlagsCreateInfo::default().binding_flags(&binding_flags);
1627
1628 let vk_info = vk_info.push_next(&mut binding_flag_info);
1629
1630 let raw = unsafe {
1631 self.shared
1632 .raw
1633 .create_descriptor_set_layout(&vk_info, None)
1634 .map_err(super::map_host_device_oom_err)?
1635 };
1636
1637 if let Some(label) = desc.label {
1638 unsafe { self.shared.set_object_name(raw, label) };
1639 }
1640
1641 let layout = super::BindGroupLayout {
1642 raw,
1643 desc_count,
1644 entries: desc.entries.into(),
1645 binding_map,
1646 contains_binding_arrays,
1647 };
1648
1649 let result = self
1650 .desc_allocator
1651 .lock()
1652 .register_layout(&self.shared.raw, &layout);
1653 if let Err(err) = result {
1654 unsafe {
1655 self.shared
1656 .raw
1657 .destroy_descriptor_set_layout(layout.raw, None)
1658 };
1659 return Err(err);
1660 }
1661
1662 self.counters.bind_group_layouts.add(1);
1663
1664 Ok(layout)
1665 }
1666 unsafe fn destroy_bind_group_layout(&self, bg_layout: super::BindGroupLayout) {
1667 self.desc_allocator
1668 .lock()
1669 .unregister_layout(&self.shared.raw, &bg_layout);
1670
1671 unsafe {
1672 self.shared
1673 .raw
1674 .destroy_descriptor_set_layout(bg_layout.raw, None)
1675 };
1676
1677 self.counters.bind_group_layouts.sub(1);
1678 }
1679
1680 unsafe fn create_pipeline_layout(
1681 &self,
1682 desc: &crate::PipelineLayoutDescriptor<super::BindGroupLayout>,
1683 ) -> Result<super::PipelineLayout, crate::DeviceError> {
1684 let vk_set_layouts = desc
1686 .bind_group_layouts
1687 .iter()
1688 .map(|bgl| match bgl {
1689 Some(bgl) => bgl.raw,
1690 None => {
1691 self.shared.empty_descriptor_set_layout
1699 }
1700 })
1701 .collect::<Vec<_>>();
1702 let vk_immediates_ranges: Option<vk::PushConstantRange> = if desc.immediate_size != 0 {
1703 Some(vk::PushConstantRange {
1704 stage_flags: vk::ShaderStageFlags::ALL,
1705 offset: 0,
1706 size: desc.immediate_size,
1707 })
1708 } else {
1709 None
1710 };
1711
1712 let vk_info = vk::PipelineLayoutCreateInfo::default()
1713 .flags(vk::PipelineLayoutCreateFlags::empty())
1714 .set_layouts(&vk_set_layouts)
1715 .push_constant_ranges(vk_immediates_ranges.as_slice());
1716
1717 let raw = {
1718 profiling::scope!("vkCreatePipelineLayout");
1719 unsafe {
1720 self.shared
1721 .raw
1722 .create_pipeline_layout(&vk_info, None)
1723 .map_err(super::map_host_device_oom_err)?
1724 }
1725 };
1726
1727 if let Some(label) = desc.label {
1728 unsafe { self.shared.set_object_name(raw, label) };
1729 }
1730
1731 let mut binding_map = BTreeMap::new();
1732 for (group, layout) in desc.bind_group_layouts.iter().enumerate() {
1733 let Some(layout) = layout else {
1734 continue;
1735 };
1736
1737 for &(binding, binding_info) in &layout.binding_map {
1738 binding_map.insert(
1739 naga::ResourceBinding {
1740 group: group as u32,
1741 binding,
1742 },
1743 naga::back::spv::BindingInfo {
1744 descriptor_set: group as u32,
1745 binding: binding_info.binding,
1746 binding_array_size: binding_info.binding_array_size.map(NonZeroU32::get),
1747 external_texture: binding_info.external_texture,
1748 },
1749 );
1750 }
1751 }
1752
1753 self.counters.pipeline_layouts.add(1);
1754 Ok(super::PipelineLayout { raw, binding_map })
1755 }
1756 unsafe fn destroy_pipeline_layout(&self, pipeline_layout: super::PipelineLayout) {
1757 unsafe {
1758 self.shared
1759 .raw
1760 .destroy_pipeline_layout(pipeline_layout.raw, None)
1761 };
1762
1763 self.counters.pipeline_layouts.sub(1);
1764 }
1765
1766 unsafe fn create_bind_group(
1767 &self,
1768 desc: &crate::BindGroupDescriptor<
1769 super::BindGroupLayout,
1770 super::Buffer,
1771 super::Sampler,
1772 super::TextureView,
1773 super::AccelerationStructure,
1774 >,
1775 ) -> Result<super::BindGroup, crate::DeviceError> {
1776 let set = unsafe {
1777 self.desc_allocator
1778 .lock()
1779 .alloc(&self.shared.raw, desc.layout)?
1780 };
1781
1782 if let Some(label) = desc.label {
1783 unsafe { self.shared.set_object_name(set.raw(), label) };
1784 }
1785
1786 struct ExtendStack<'a, T> {
1793 remainder: &'a mut [MaybeUninit<T>],
1794 }
1795
1796 impl<'a, T> ExtendStack<'a, T> {
1797 fn from_vec_capacity(vec: &'a mut Vec<T>) -> Self {
1798 Self {
1799 remainder: vec.spare_capacity_mut(),
1800 }
1801 }
1802
1803 fn extend_one(self, value: T) -> (Self, &'a mut T) {
1804 let (to_init, remainder) = self.remainder.split_first_mut().unwrap();
1805 let init = to_init.write(value);
1806 (Self { remainder }, init)
1807 }
1808
1809 fn extend(
1810 self,
1811 iter: impl IntoIterator<Item = T> + ExactSizeIterator,
1812 ) -> (Self, &'a mut [T]) {
1813 let (to_init, remainder) = self.remainder.split_at_mut(iter.len());
1814
1815 for (value, to_init) in iter.into_iter().zip(to_init.iter_mut()) {
1816 to_init.write(value);
1817 }
1818
1819 let init = {
1822 unsafe { mem::transmute::<&mut [MaybeUninit<T>], &mut [T]>(to_init) }
1829 };
1830 (Self { remainder }, init)
1831 }
1832 }
1833
1834 let mut writes = Vec::with_capacity(desc.entries.len());
1835 let mut buffer_infos =
1836 Vec::with_capacity(desc.buffers.len() + desc.external_textures.len());
1837 let mut buffer_infos = ExtendStack::from_vec_capacity(&mut buffer_infos);
1838 let mut image_infos = Vec::with_capacity(
1839 desc.samplers.len() + desc.textures.len() + 3 * desc.external_textures.len(),
1840 );
1841 let mut image_infos = ExtendStack::from_vec_capacity(&mut image_infos);
1842 let mut acceleration_structure_infos =
1847 Vec::with_capacity(desc.acceleration_structures.len());
1848 let mut acceleration_structure_infos =
1849 ExtendStack::from_vec_capacity(&mut acceleration_structure_infos);
1850 let mut raw_acceleration_structures =
1851 Vec::with_capacity(desc.acceleration_structures.len());
1852 let mut raw_acceleration_structures =
1853 ExtendStack::from_vec_capacity(&mut raw_acceleration_structures);
1854
1855 let layout_and_entry_iter = desc.entries.iter().map(|entry| {
1856 let layout = desc
1857 .layout
1858 .entries
1859 .iter()
1860 .find(|layout_entry| layout_entry.binding == entry.binding)
1861 .expect("internal error: no layout entry found with binding slot");
1862 (layout, entry)
1863 });
1864 let mut next_binding = 0;
1865 for (layout, entry) in layout_and_entry_iter {
1866 let dst_set = set.raw();
1867
1868 fn add_image_descriptor_writes<'a>(
1869 mut infos: ExtendStack<'a, vk::DescriptorImageInfo>,
1870 writes: &mut Vec<vk::WriteDescriptorSet<'a>>,
1871 dst_set: vk::DescriptorSet,
1872 next_binding: &mut u32,
1873 add_infos: &'a [crate::TextureBinding<super::TextureView>],
1874 add_ty: vk::DescriptorType,
1875 ) -> ExtendStack<'a, vk::DescriptorImageInfo> {
1876 let local_image_infos;
1877 (infos, local_image_infos) = infos.extend(add_infos.iter().map(|binding| {
1878 let layout = conv::derive_image_layout(binding.usage, binding.view.format);
1879 vk::DescriptorImageInfo::default()
1880 .image_view(binding.view.raw)
1881 .image_layout(layout)
1882 }));
1883 writes.push(
1884 vk::WriteDescriptorSet::default()
1885 .dst_set(dst_set)
1886 .dst_binding(*next_binding)
1887 .descriptor_type(add_ty)
1888 .image_info(local_image_infos),
1889 );
1890 *next_binding += 1;
1891 infos
1892 }
1893
1894 fn add_buffer_descriptor_writes<'a>(
1895 mut infos: ExtendStack<'a, vk::DescriptorBufferInfo>,
1896 writes: &mut Vec<vk::WriteDescriptorSet<'a>>,
1897 dst_set: vk::DescriptorSet,
1898 next_binding: &mut u32,
1899 add_infos: &'a [crate::BufferBinding<super::Buffer, wgt::BufferSize>],
1900 add_ty: vk::DescriptorType,
1901 ) -> ExtendStack<'a, vk::DescriptorBufferInfo> {
1902 let local_buffer_infos;
1903 (infos, local_buffer_infos) = infos.extend(add_infos.iter().map(|binding| {
1904 vk::DescriptorBufferInfo::default()
1905 .buffer(binding.buffer.raw)
1906 .offset(binding.offset)
1907 .range(binding.size.get())
1908 }));
1909 writes.push(
1910 vk::WriteDescriptorSet::default()
1911 .dst_set(dst_set)
1912 .dst_binding(*next_binding)
1913 .descriptor_type(add_ty)
1914 .buffer_info(local_buffer_infos),
1915 );
1916 *next_binding += 1;
1917 infos
1918 }
1919
1920 match layout.ty {
1921 wgt::BindingType::Sampler(_) => {
1922 let start = entry.resource_index;
1923 let end = start + entry.count;
1924 let local_image_infos;
1925 (image_infos, local_image_infos) =
1926 image_infos.extend(desc.samplers[start as usize..end as usize].iter().map(
1927 |sampler| vk::DescriptorImageInfo::default().sampler(sampler.raw),
1928 ));
1929 writes.push(
1930 vk::WriteDescriptorSet::default()
1931 .dst_set(dst_set)
1932 .dst_binding(next_binding)
1933 .descriptor_type(conv::map_binding_type(layout.ty))
1934 .image_info(local_image_infos),
1935 );
1936 next_binding += 1;
1937 }
1938 wgt::BindingType::Texture { .. } | wgt::BindingType::StorageTexture { .. } => {
1939 let start = entry.resource_index;
1940 let end = start + entry.count;
1941 image_infos = add_image_descriptor_writes(
1942 image_infos,
1943 &mut writes,
1944 dst_set,
1945 &mut next_binding,
1946 &desc.textures[start as usize..end as usize],
1947 conv::map_binding_type(layout.ty),
1948 );
1949 }
1950 wgt::BindingType::Buffer { .. } => {
1951 let start = entry.resource_index;
1952 let end = start + entry.count;
1953 buffer_infos = add_buffer_descriptor_writes(
1954 buffer_infos,
1955 &mut writes,
1956 dst_set,
1957 &mut next_binding,
1958 &desc.buffers[start as usize..end as usize],
1959 conv::map_binding_type(layout.ty),
1960 );
1961 }
1962 wgt::BindingType::AccelerationStructure { .. } => {
1963 let start = entry.resource_index;
1964 let end = start + entry.count;
1965
1966 let local_raw_acceleration_structures;
1967 (
1968 raw_acceleration_structures,
1969 local_raw_acceleration_structures,
1970 ) = raw_acceleration_structures.extend(
1971 desc.acceleration_structures[start as usize..end as usize]
1972 .iter()
1973 .map(|acceleration_structure| acceleration_structure.raw),
1974 );
1975
1976 let local_acceleration_structure_infos;
1977 (
1978 acceleration_structure_infos,
1979 local_acceleration_structure_infos,
1980 ) = acceleration_structure_infos.extend_one(
1981 vk::WriteDescriptorSetAccelerationStructureKHR::default()
1982 .acceleration_structures(local_raw_acceleration_structures),
1983 );
1984
1985 writes.push(
1986 vk::WriteDescriptorSet::default()
1987 .dst_set(dst_set)
1988 .dst_binding(next_binding)
1989 .descriptor_type(conv::map_binding_type(layout.ty))
1990 .descriptor_count(entry.count)
1991 .push_next(local_acceleration_structure_infos),
1992 );
1993 next_binding += 1;
1994 }
1995 wgt::BindingType::ExternalTexture => {
1996 assert!(
1997 entry.count == 1,
1998 "ExternalTexture binding arrays are not supported"
1999 );
2000 let index = entry.resource_index as usize;
2001 for plane in &desc.external_textures[index].planes {
2002 image_infos = add_image_descriptor_writes(
2003 image_infos,
2004 &mut writes,
2005 dst_set,
2006 &mut next_binding,
2007 core::slice::from_ref(plane),
2008 vk::DescriptorType::SAMPLED_IMAGE,
2009 );
2010 }
2011 buffer_infos = add_buffer_descriptor_writes(
2012 buffer_infos,
2013 &mut writes,
2014 dst_set,
2015 &mut next_binding,
2016 core::slice::from_ref(&desc.external_textures[index].params),
2017 vk::DescriptorType::UNIFORM_BUFFER,
2018 )
2019 }
2020 }
2021 }
2022
2023 unsafe { self.shared.raw.update_descriptor_sets(&writes, &[]) };
2024
2025 self.counters.bind_groups.add(1);
2026
2027 Ok(super::BindGroup { set })
2028 }
2029
2030 unsafe fn destroy_bind_group(&self, group: super::BindGroup) {
2031 unsafe { self.desc_allocator.lock().free(&self.shared.raw, group.set) };
2032
2033 self.counters.bind_groups.sub(1);
2034 }
2035
2036 unsafe fn create_shader_module(
2037 &self,
2038 desc: &crate::ShaderModuleDescriptor,
2039 shader: crate::ShaderInput,
2040 ) -> Result<super::ShaderModule, crate::ShaderError> {
2041 let shader_module = match shader {
2042 crate::ShaderInput::Naga(naga_shader)
2043 if self
2044 .shared
2045 .workarounds
2046 .contains(super::Workarounds::SEPARATE_ENTRY_POINTS)
2047 || !naga_shader.module.overrides.is_empty() =>
2048 {
2049 super::ShaderModule::Intermediate {
2050 naga_shader,
2051 runtime_checks: desc.runtime_checks,
2052 }
2053 }
2054 crate::ShaderInput::Naga(naga_shader) => {
2055 let mut naga_options = self.naga_options.clone();
2056 naga_options.debug_info =
2057 naga_shader
2058 .debug_source
2059 .as_ref()
2060 .map(|d| naga::back::spv::DebugInfo {
2061 source_code: d.source_code.as_ref(),
2062 file_name: d.file_name.as_ref(),
2063 language: naga::back::spv::SourceLanguage::WGSL,
2064 });
2065 if !desc.runtime_checks.bounds_checks {
2066 naga_options.bounds_check_policies = naga::proc::BoundsCheckPolicies {
2067 index: naga::proc::BoundsCheckPolicy::Unchecked,
2068 buffer: naga::proc::BoundsCheckPolicy::Unchecked,
2069 image_load: naga::proc::BoundsCheckPolicy::Unchecked,
2070 binding_array: naga::proc::BoundsCheckPolicy::Unchecked,
2071 };
2072 }
2073 let spv = naga::back::spv::write_vec(
2074 &naga_shader.module,
2075 &naga_shader.info,
2076 &naga_options,
2077 None,
2078 )
2079 .map_err(|e| crate::ShaderError::Compilation(format!("{e}")))?;
2080 super::ShaderModule::Raw(self.create_shader_module_impl(&spv, &desc.label)?)
2081 }
2082 crate::ShaderInput::SpirV(data) => {
2083 super::ShaderModule::Raw(self.create_shader_module_impl(data, &desc.label)?)
2084 }
2085 crate::ShaderInput::MetalLib { .. }
2086 | crate::ShaderInput::Msl { .. }
2087 | crate::ShaderInput::Dxil { .. }
2088 | crate::ShaderInput::Hlsl { .. }
2089 | crate::ShaderInput::Glsl { .. } => unreachable!(),
2090 };
2091
2092 self.counters.shader_modules.add(1);
2093
2094 Ok(shader_module)
2095 }
2096
2097 unsafe fn destroy_shader_module(&self, module: super::ShaderModule) {
2098 match module {
2099 super::ShaderModule::Raw(raw) => {
2100 unsafe { self.shared.raw.destroy_shader_module(raw, None) };
2101 }
2102 super::ShaderModule::Intermediate { .. } => {}
2103 }
2104
2105 self.counters.shader_modules.sub(1);
2106 }
2107
2108 unsafe fn create_render_pipeline(
2109 &self,
2110 desc: &crate::RenderPipelineDescriptor<
2111 super::PipelineLayout,
2112 super::ShaderModule,
2113 super::PipelineCache,
2114 >,
2115 ) -> Result<super::RenderPipeline, crate::PipelineError> {
2116 let dynamic_states = [
2117 vk::DynamicState::VIEWPORT,
2118 vk::DynamicState::SCISSOR,
2119 vk::DynamicState::BLEND_CONSTANTS,
2120 vk::DynamicState::STENCIL_REFERENCE,
2121 ];
2122 let mut compatible_rp_key = super::RenderPassKey {
2123 sample_count: desc.multisample.count,
2124 multiview_mask: desc.multiview_mask,
2125 ..Default::default()
2126 };
2127 let mut stages = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new();
2128 let mut vertex_buffers = Vec::new();
2129 let mut vertex_attributes = Vec::new();
2130
2131 if let crate::VertexProcessor::Standard {
2132 vertex_buffers: desc_vertex_buffers,
2133 vertex_stage: _,
2134 } = &desc.vertex_processor
2135 {
2136 vertex_buffers = Vec::with_capacity(desc_vertex_buffers.len());
2137 for (i, vb) in desc_vertex_buffers.iter().enumerate() {
2138 let Some(vb) = vb else {
2139 continue;
2140 };
2141 vertex_buffers.push(vk::VertexInputBindingDescription {
2142 binding: i as u32,
2143 stride: vb.array_stride as u32,
2144 input_rate: match vb.step_mode {
2145 wgt::VertexStepMode::Vertex => vk::VertexInputRate::VERTEX,
2146 wgt::VertexStepMode::Instance => vk::VertexInputRate::INSTANCE,
2147 },
2148 });
2149 for at in vb.attributes {
2150 vertex_attributes.push(vk::VertexInputAttributeDescription {
2151 location: at.shader_location,
2152 binding: i as u32,
2153 format: conv::map_vertex_format(at.format),
2154 offset: at.offset as u32,
2155 });
2156 }
2157 }
2158 }
2159
2160 let vk_vertex_input = vk::PipelineVertexInputStateCreateInfo::default()
2161 .vertex_binding_descriptions(&vertex_buffers)
2162 .vertex_attribute_descriptions(&vertex_attributes);
2163
2164 let vk_input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
2165 .topology(conv::map_topology(desc.primitive.topology))
2166 .primitive_restart_enable(desc.primitive.strip_index_format.is_some());
2167
2168 let mut compiled_vs = None;
2169 let mut compiled_ms = None;
2170 let mut compiled_ts = None;
2171 match &desc.vertex_processor {
2172 crate::VertexProcessor::Standard {
2173 vertex_buffers: _,
2174 vertex_stage,
2175 } => {
2176 compiled_vs = Some(self.compile_stage(
2177 vertex_stage,
2178 naga::ShaderStage::Vertex,
2179 &desc.layout.binding_map,
2180 )?);
2181 stages.push(compiled_vs.as_ref().unwrap().create_info);
2182 }
2183 crate::VertexProcessor::Mesh {
2184 task_stage,
2185 mesh_stage,
2186 } => {
2187 if let Some(t) = task_stage.as_ref() {
2188 compiled_ts = Some(self.compile_stage(
2189 t,
2190 naga::ShaderStage::Task,
2191 &desc.layout.binding_map,
2192 )?);
2193 stages.push(compiled_ts.as_ref().unwrap().create_info);
2194 }
2195 compiled_ms = Some(self.compile_stage(
2196 mesh_stage,
2197 naga::ShaderStage::Mesh,
2198 &desc.layout.binding_map,
2199 )?);
2200 stages.push(compiled_ms.as_ref().unwrap().create_info);
2201 }
2202 }
2203 let compiled_fs = match desc.fragment_stage {
2204 Some(ref stage) => {
2205 let compiled = self.compile_stage(
2206 stage,
2207 naga::ShaderStage::Fragment,
2208 &desc.layout.binding_map,
2209 )?;
2210 stages.push(compiled.create_info);
2211 Some(compiled)
2212 }
2213 None => None,
2214 };
2215
2216 let mut vk_rasterization = vk::PipelineRasterizationStateCreateInfo::default()
2217 .polygon_mode(conv::map_polygon_mode(desc.primitive.polygon_mode))
2218 .front_face(conv::map_front_face(desc.primitive.front_face))
2219 .line_width(1.0)
2220 .depth_clamp_enable(desc.primitive.unclipped_depth);
2221 if let Some(face) = desc.primitive.cull_mode {
2222 vk_rasterization = vk_rasterization.cull_mode(conv::map_cull_face(face))
2223 }
2224 let mut vk_rasterization_conservative_state =
2225 vk::PipelineRasterizationConservativeStateCreateInfoEXT::default()
2226 .conservative_rasterization_mode(
2227 vk::ConservativeRasterizationModeEXT::OVERESTIMATE,
2228 );
2229 if desc.primitive.conservative {
2230 vk_rasterization = vk_rasterization.push_next(&mut vk_rasterization_conservative_state);
2231 }
2232
2233 let mut vk_depth_stencil = vk::PipelineDepthStencilStateCreateInfo::default();
2234 if let Some(ref ds) = desc.depth_stencil {
2235 (
2236 compatible_rp_key.depth_read_only,
2237 compatible_rp_key.stencil_read_only,
2238 ) = (
2239 ds.is_depth_read_only(),
2240 ds.is_stencil_read_only(desc.primitive.cull_mode),
2241 );
2242 let vk_format = self.shared.private_caps.map_texture_format(ds.format);
2243 let vk_layout = match (
2244 compatible_rp_key.depth_read_only,
2245 compatible_rp_key.stencil_read_only,
2246 ) {
2247 (true, true) => vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL,
2248 (true, false) => vk::ImageLayout::DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL,
2249 (false, true) => vk::ImageLayout::DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL,
2250 (false, false) => vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL,
2251 };
2252 compatible_rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
2253 base: super::AttachmentKey::compatible(vk_format, vk_layout),
2254 stencil_ops: crate::AttachmentOps::all(),
2255 });
2256
2257 if ds.is_depth_enabled() {
2258 vk_depth_stencil = vk_depth_stencil
2259 .depth_test_enable(true)
2260 .depth_write_enable(ds.depth_write_enabled.unwrap_or_default())
2261 .depth_compare_op(conv::map_comparison(ds.depth_compare.unwrap_or_default()));
2262 }
2263 if ds.stencil.is_enabled() {
2264 let s = &ds.stencil;
2265 let front = conv::map_stencil_face(&s.front, s.read_mask, s.write_mask);
2266 let back = conv::map_stencil_face(&s.back, s.read_mask, s.write_mask);
2267 vk_depth_stencil = vk_depth_stencil
2268 .stencil_test_enable(true)
2269 .front(front)
2270 .back(back);
2271 }
2272
2273 if ds.bias.is_enabled() {
2274 vk_rasterization = vk_rasterization
2275 .depth_bias_enable(true)
2276 .depth_bias_constant_factor(ds.bias.constant as f32)
2277 .depth_bias_clamp(ds.bias.clamp)
2278 .depth_bias_slope_factor(ds.bias.slope_scale);
2279 }
2280 }
2281
2282 let vk_viewport = vk::PipelineViewportStateCreateInfo::default()
2283 .flags(vk::PipelineViewportStateCreateFlags::empty())
2284 .scissor_count(1)
2285 .viewport_count(1);
2286
2287 let vk_sample_mask = [
2288 desc.multisample.mask as u32,
2289 (desc.multisample.mask >> 32) as u32,
2290 ];
2291 let vk_multisample = vk::PipelineMultisampleStateCreateInfo::default()
2292 .rasterization_samples(vk::SampleCountFlags::from_raw(desc.multisample.count))
2293 .alpha_to_coverage_enable(desc.multisample.alpha_to_coverage_enabled)
2294 .sample_mask(&vk_sample_mask);
2295
2296 let mut vk_attachments = Vec::with_capacity(desc.color_targets.len());
2297 for cat in desc.color_targets {
2298 let (key, attarchment) = if let Some(cat) = cat.as_ref() {
2299 let mut vk_attachment = vk::PipelineColorBlendAttachmentState::default()
2300 .color_write_mask(vk::ColorComponentFlags::from_raw(cat.write_mask.bits()));
2301 if let Some(ref blend) = cat.blend {
2302 let (color_op, color_src, color_dst) = conv::map_blend_component(&blend.color);
2303 let (alpha_op, alpha_src, alpha_dst) = conv::map_blend_component(&blend.alpha);
2304 vk_attachment = vk_attachment
2305 .blend_enable(true)
2306 .color_blend_op(color_op)
2307 .src_color_blend_factor(color_src)
2308 .dst_color_blend_factor(color_dst)
2309 .alpha_blend_op(alpha_op)
2310 .src_alpha_blend_factor(alpha_src)
2311 .dst_alpha_blend_factor(alpha_dst);
2312 }
2313
2314 let vk_format = self.shared.private_caps.map_texture_format(cat.format);
2315 (
2316 Some(super::ColorAttachmentKey {
2317 base: super::AttachmentKey::compatible(
2318 vk_format,
2319 vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
2320 ),
2321 resolve: None,
2322 }),
2323 vk_attachment,
2324 )
2325 } else {
2326 (None, vk::PipelineColorBlendAttachmentState::default())
2327 };
2328
2329 compatible_rp_key.colors.push(key);
2330 vk_attachments.push(attarchment);
2331 }
2332
2333 let vk_color_blend =
2334 vk::PipelineColorBlendStateCreateInfo::default().attachments(&vk_attachments);
2335
2336 let vk_dynamic_state =
2337 vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic_states);
2338
2339 let raw_pass = self.shared.make_render_pass(compatible_rp_key)?;
2340
2341 let vk_infos = [{
2342 vk::GraphicsPipelineCreateInfo::default()
2343 .layout(desc.layout.raw)
2344 .stages(&stages)
2345 .vertex_input_state(&vk_vertex_input)
2346 .input_assembly_state(&vk_input_assembly)
2347 .rasterization_state(&vk_rasterization)
2348 .viewport_state(&vk_viewport)
2349 .multisample_state(&vk_multisample)
2350 .depth_stencil_state(&vk_depth_stencil)
2351 .color_blend_state(&vk_color_blend)
2352 .dynamic_state(&vk_dynamic_state)
2353 .render_pass(raw_pass)
2354 }];
2355
2356 let pipeline_cache = desc
2357 .cache
2358 .map(|it| it.raw)
2359 .unwrap_or(vk::PipelineCache::null());
2360
2361 let mut raw_vec = {
2362 profiling::scope!("vkCreateGraphicsPipelines");
2363 unsafe {
2364 self.shared
2365 .raw
2366 .create_graphics_pipelines(pipeline_cache, &vk_infos, None)
2367 .map_err(|(_, e)| super::map_pipeline_err(e))
2368 }?
2369 };
2370
2371 let raw = raw_vec.pop().unwrap();
2372 if let Some(label) = desc.label {
2373 unsafe { self.shared.set_object_name(raw, label) };
2374 }
2375
2376 if let Some(CompiledStage {
2377 temp_raw_module: Some(raw_module),
2378 ..
2379 }) = compiled_vs
2380 {
2381 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2382 }
2383 if let Some(CompiledStage {
2384 temp_raw_module: Some(raw_module),
2385 ..
2386 }) = compiled_ts
2387 {
2388 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2389 }
2390 if let Some(CompiledStage {
2391 temp_raw_module: Some(raw_module),
2392 ..
2393 }) = compiled_ms
2394 {
2395 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2396 }
2397 if let Some(CompiledStage {
2398 temp_raw_module: Some(raw_module),
2399 ..
2400 }) = compiled_fs
2401 {
2402 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2403 }
2404
2405 self.counters.render_pipelines.add(1);
2406
2407 Ok(super::RenderPipeline {
2408 raw,
2409 is_multiview: desc.multiview_mask.is_some(),
2410 })
2411 }
2412
2413 unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) {
2414 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2415
2416 self.counters.render_pipelines.sub(1);
2417 }
2418
2419 unsafe fn create_compute_pipeline(
2420 &self,
2421 desc: &crate::ComputePipelineDescriptor<
2422 super::PipelineLayout,
2423 super::ShaderModule,
2424 super::PipelineCache,
2425 >,
2426 ) -> Result<super::ComputePipeline, crate::PipelineError> {
2427 let compiled = self.compile_stage(
2428 &desc.stage,
2429 naga::ShaderStage::Compute,
2430 &desc.layout.binding_map,
2431 )?;
2432
2433 let vk_infos = [{
2434 vk::ComputePipelineCreateInfo::default()
2435 .layout(desc.layout.raw)
2436 .stage(compiled.create_info)
2437 }];
2438
2439 let pipeline_cache = desc
2440 .cache
2441 .map(|it| it.raw)
2442 .unwrap_or(vk::PipelineCache::null());
2443
2444 let mut raw_vec = {
2445 profiling::scope!("vkCreateComputePipelines");
2446 unsafe {
2447 self.shared
2448 .raw
2449 .create_compute_pipelines(pipeline_cache, &vk_infos, None)
2450 .map_err(|(_, e)| super::map_pipeline_err(e))
2451 }?
2452 };
2453
2454 let raw = raw_vec.pop().unwrap();
2455 if let Some(label) = desc.label {
2456 unsafe { self.shared.set_object_name(raw, label) };
2457 }
2458
2459 if let Some(raw_module) = compiled.temp_raw_module {
2460 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2461 }
2462
2463 self.counters.compute_pipelines.add(1);
2464
2465 Ok(super::ComputePipeline { raw })
2466 }
2467
2468 unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) {
2469 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2470
2471 self.counters.compute_pipelines.sub(1);
2472 }
2473
2474 unsafe fn create_ray_tracing_pipeline(
2475 &self,
2476 desc: &crate::RayTracingPipelineDescriptor<
2477 super::PipelineLayout,
2478 super::ShaderModule,
2479 super::PipelineCache,
2480 >,
2481 ) -> Result<super::RayTracingPipeline, crate::PipelineError> {
2482 let mut stages = Vec::new();
2483 let mut groups = Vec::new();
2484
2485 let compiled_ray_gen = self.compile_stage(
2486 &desc.ray_generation,
2487 naga::ShaderStage::RayGeneration,
2488 &desc.layout.binding_map,
2489 )?;
2490
2491 groups.push(
2492 vk::RayTracingShaderGroupCreateInfoKHR::default()
2493 .closest_hit_shader(vk::SHADER_UNUSED_KHR)
2494 .any_hit_shader(vk::SHADER_UNUSED_KHR)
2495 .intersection_shader(vk::SHADER_UNUSED_KHR)
2496 .general_shader(0) .ty(vk::RayTracingShaderGroupTypeKHR::GENERAL),
2498 );
2499
2500 stages.push(compiled_ray_gen.create_info);
2501
2502 let compiled_miss = self.compile_stage(
2503 &desc.miss,
2504 naga::ShaderStage::Miss,
2505 &desc.layout.binding_map,
2506 )?;
2507
2508 groups.push(
2509 vk::RayTracingShaderGroupCreateInfoKHR::default()
2510 .closest_hit_shader(vk::SHADER_UNUSED_KHR)
2511 .any_hit_shader(vk::SHADER_UNUSED_KHR)
2512 .intersection_shader(vk::SHADER_UNUSED_KHR)
2513 .general_shader(1) .ty(vk::RayTracingShaderGroupTypeKHR::GENERAL),
2515 );
2516
2517 stages.push(compiled_miss.create_info);
2518
2519 let mut compiled_stages = Vec::new();
2522
2523 for group in desc.intersection {
2524 let compiled_closest_hits = self.compile_stage(
2525 &group.closest_hit,
2526 naga::ShaderStage::ClosestHit,
2527 &desc.layout.binding_map,
2528 )?;
2529
2530 let closest_idx = stages.len();
2531
2532 stages.push(compiled_closest_hits.create_info);
2533
2534 compiled_stages.push(compiled_closest_hits);
2535
2536 let mut raw_hit: vk::RayTracingShaderGroupCreateInfoKHR<'_> =
2537 vk::RayTracingShaderGroupCreateInfoKHR::default()
2538 .closest_hit_shader(closest_idx as _)
2539 .any_hit_shader(vk::SHADER_UNUSED_KHR)
2540 .intersection_shader(vk::SHADER_UNUSED_KHR)
2541 .general_shader(vk::SHADER_UNUSED_KHR)
2542 .ty(vk::RayTracingShaderGroupTypeKHR::TRIANGLES_HIT_GROUP);
2543
2544 if let Some(any_hit) = &group.any_hit {
2545 let compiled_any_hit = self.compile_stage(
2546 any_hit,
2547 naga::ShaderStage::AnyHit,
2548 &desc.layout.binding_map,
2549 )?;
2550
2551 let any_idx = stages.len();
2552
2553 stages.push(compiled_any_hit.create_info);
2554
2555 compiled_stages.push(compiled_any_hit);
2556
2557 raw_hit = raw_hit.any_hit_shader(any_idx as _);
2558 }
2559
2560 groups.push(raw_hit);
2561 }
2562
2563 let create_infos = [{
2564 vk::RayTracingPipelineCreateInfoKHR::default()
2565 .layout(desc.layout.raw)
2566 .max_pipeline_ray_recursion_depth(desc.max_recursion_depth)
2567 .stages(&stages)
2568 .groups(&groups)
2569 }];
2570
2571 let pipeline_cache = desc
2572 .cache
2573 .map(|it| it.raw)
2574 .unwrap_or(vk::PipelineCache::null());
2575
2576 let fns = self
2577 .shared
2578 .extension_fns
2579 .ray_tracing_pipelines
2580 .as_ref()
2581 .unwrap();
2582 let pipelines = unsafe {
2583 fns.create_ray_tracing_pipelines(
2584 vk::DeferredOperationKHR::null(),
2585 pipeline_cache,
2586 &create_infos,
2587 None,
2588 )
2589 .map_err(|(_, e)| super::map_pipeline_err(e))
2590 }?;
2591
2592 if let Some(raw_module) = compiled_ray_gen.temp_raw_module {
2593 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2594 }
2595
2596 if let Some(raw_module) = compiled_miss.temp_raw_module {
2597 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2598 }
2599
2600 for raw_module in compiled_stages
2601 .into_iter()
2602 .flat_map(|stage| stage.temp_raw_module)
2603 {
2604 unsafe { self.shared.raw.destroy_shader_module(raw_module, None) };
2605 }
2606
2607 self.counters.ray_tracing_pipelines.add(1);
2608
2609 Ok(super::RayTracingPipeline { raw: pipelines[0] })
2610 }
2611
2612 unsafe fn destroy_ray_tracing_pipeline(&self, pipeline: super::RayTracingPipeline) {
2613 unsafe { self.shared.raw.destroy_pipeline(pipeline.raw, None) };
2614
2615 self.counters.ray_tracing_pipelines.sub(1);
2616 }
2617
2618 unsafe fn get_raytracing_pipeline_group_data(
2619 &self,
2620 pipeline: &super::RayTracingPipeline,
2621 groups: core::ops::Range<u32>,
2622 ) -> Result<Vec<u8>, crate::DeviceError> {
2623 let fns = self
2624 .shared
2625 .extension_fns
2626 .ray_tracing_pipelines
2627 .as_ref()
2628 .unwrap();
2629
2630 let num = groups.end - groups.start;
2631
2632 unsafe {
2633 fns.get_ray_tracing_shader_group_handles(
2634 pipeline.raw,
2635 groups.start,
2636 num,
2637 (num * self
2638 .shared
2639 .private_caps
2640 .ray_tracing_pipeline_group_data_size) as usize,
2641 )
2642 }
2643 .map_err(super::map_host_device_oom_err)
2644 }
2645
2646 unsafe fn create_pipeline_cache(
2647 &self,
2648 desc: &crate::PipelineCacheDescriptor<'_>,
2649 ) -> Result<super::PipelineCache, crate::PipelineCacheError> {
2650 let mut info = vk::PipelineCacheCreateInfo::default();
2651 if let Some(data) = desc.data {
2652 info = info.initial_data(data)
2653 }
2654 profiling::scope!("vkCreatePipelineCache");
2655 let raw = unsafe { self.shared.raw.create_pipeline_cache(&info, None) }
2656 .map_err(super::map_host_device_oom_err)?;
2657
2658 Ok(super::PipelineCache { raw })
2659 }
2660 fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> {
2661 Some(self.shared.pipeline_cache_validation_key)
2662 }
2663 unsafe fn destroy_pipeline_cache(&self, cache: super::PipelineCache) {
2664 unsafe { self.shared.raw.destroy_pipeline_cache(cache.raw, None) }
2665 }
2666 unsafe fn create_query_set(
2667 &self,
2668 desc: &wgt::QuerySetDescriptor<crate::Label>,
2669 ) -> Result<super::QuerySet, crate::DeviceError> {
2670 self.error_if_would_oom_on_resource_allocation(
2675 gpu_allocator::MemoryLocation::GpuToCpu,
2676 &vk::MemoryRequirements {
2677 size: desc.count as u64 * 256,
2678 alignment: 256,
2679 memory_type_bits: self.valid_ash_memory_types,
2680 },
2681 )?;
2682
2683 let (vk_type, pipeline_statistics) = match desc.ty {
2684 wgt::QueryType::Occlusion => (
2685 vk::QueryType::OCCLUSION,
2686 vk::QueryPipelineStatisticFlags::empty(),
2687 ),
2688 wgt::QueryType::PipelineStatistics(statistics) => (
2689 vk::QueryType::PIPELINE_STATISTICS,
2690 conv::map_pipeline_statistics(statistics),
2691 ),
2692 wgt::QueryType::Timestamp => (
2693 vk::QueryType::TIMESTAMP,
2694 vk::QueryPipelineStatisticFlags::empty(),
2695 ),
2696 };
2697
2698 let vk_info = vk::QueryPoolCreateInfo::default()
2699 .query_type(vk_type)
2700 .query_count(desc.count)
2701 .pipeline_statistics(pipeline_statistics);
2702
2703 let raw = unsafe { self.shared.raw.create_query_pool(&vk_info, None) }
2704 .map_err(super::map_host_device_oom_err)?;
2705 if let Some(label) = desc.label {
2706 unsafe { self.shared.set_object_name(raw, label) };
2707 }
2708
2709 self.counters.query_sets.add(1);
2710
2711 Ok(super::QuerySet { raw })
2712 }
2713
2714 unsafe fn destroy_query_set(&self, set: super::QuerySet) {
2715 unsafe { self.shared.raw.destroy_query_pool(set.raw, None) };
2716
2717 self.counters.query_sets.sub(1);
2718 }
2719
2720 unsafe fn create_fence(&self) -> Result<super::Fence, crate::DeviceError> {
2721 self.counters.fences.add(1);
2722
2723 Ok(if self.shared.private_caps.timeline_semaphores {
2724 let mut sem_type_info =
2725 vk::SemaphoreTypeCreateInfo::default().semaphore_type(vk::SemaphoreType::TIMELINE);
2726 let vk_info = vk::SemaphoreCreateInfo::default().push_next(&mut sem_type_info);
2727 let raw = unsafe { self.shared.raw.create_semaphore(&vk_info, None) }
2728 .map_err(super::map_host_device_oom_err)?;
2729
2730 super::Fence::TimelineSemaphore(raw)
2731 } else {
2732 super::Fence::FencePool(RwLock::new(super::FencePool {
2733 last_completed: 0,
2734 active: Vec::new(),
2735 free: Vec::new(),
2736 }))
2737 })
2738 }
2739 unsafe fn destroy_fence(&self, fence: super::Fence) {
2740 match fence {
2741 super::Fence::TimelineSemaphore(raw) => {
2742 unsafe { self.shared.raw.destroy_semaphore(raw, None) };
2743 }
2744 super::Fence::FencePool(pool) => {
2745 let super::FencePool {
2746 active,
2747 free,
2748 last_completed: _,
2749 } = pool.into_inner();
2750
2751 for (_, raw) in active {
2752 unsafe {
2753 self.shared.raw.destroy_fence(Arc::into_inner(raw).expect("Fence should have its reference count be one by the end of each function"), None)
2754 };
2755 }
2756 for raw in free {
2757 unsafe { self.shared.raw.destroy_fence(raw, None) };
2758 }
2759 }
2760 }
2761
2762 self.counters.fences.sub(1);
2763 }
2764 unsafe fn get_fence_value(
2765 &self,
2766 fence: &super::Fence,
2767 ) -> Result<crate::FenceValue, crate::DeviceError> {
2768 fence.get_latest(
2769 &self.shared.raw,
2770 self.shared.extension_fns.timeline_semaphore.as_ref(),
2771 )
2772 }
2773 unsafe fn wait(
2774 &self,
2775 fence: &super::Fence,
2776 wait_value: crate::FenceValue,
2777 timeout: Option<Duration>,
2778 ) -> Result<bool, crate::DeviceError> {
2779 let timeout_ns = timeout
2780 .unwrap_or(Duration::MAX)
2781 .as_nanos()
2782 .min(u64::MAX as _) as u64;
2783 self.shared.wait_for_fence(fence, wait_value, timeout_ns)
2784 }
2785
2786 unsafe fn start_graphics_debugger_capture(&self) -> bool {
2787 #[cfg(feature = "renderdoc")]
2788 {
2789 let raw_vk_instance =
2791 vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2792 let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2793 unsafe {
2794 self.render_doc
2795 .start_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2796 }
2797 }
2798 #[cfg(not(feature = "renderdoc"))]
2799 false
2800 }
2801 unsafe fn stop_graphics_debugger_capture(&self) {
2802 #[cfg(feature = "renderdoc")]
2803 {
2804 let raw_vk_instance =
2806 vk::Handle::as_raw(self.shared.instance.raw.handle()) as *mut *mut _;
2807 let raw_vk_instance_dispatch_table = unsafe { *raw_vk_instance };
2808
2809 unsafe {
2810 self.render_doc
2811 .end_frame_capture(raw_vk_instance_dispatch_table, ptr::null_mut())
2812 }
2813 }
2814 }
2815
2816 unsafe fn pipeline_cache_get_data(&self, cache: &super::PipelineCache) -> Option<Vec<u8>> {
2817 let data = unsafe { self.raw_device().get_pipeline_cache_data(cache.raw) };
2818 data.ok()
2819 }
2820
2821 unsafe fn get_acceleration_structure_build_sizes<'a>(
2822 &self,
2823 desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>,
2824 ) -> crate::AccelerationStructureBuildSizes {
2825 const CAPACITY: usize = 8;
2826
2827 let ray_tracing_functions = self
2828 .shared
2829 .extension_fns
2830 .ray_tracing
2831 .as_ref()
2832 .expect("Feature `RAY_TRACING` not enabled");
2833
2834 let (geometries, primitive_counts) = match *desc.entries {
2835 crate::AccelerationStructureEntries::Instances(ref instances) => {
2836 let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default();
2837
2838 let geometry = vk::AccelerationStructureGeometryKHR::default()
2839 .geometry_type(vk::GeometryTypeKHR::INSTANCES)
2840 .geometry(vk::AccelerationStructureGeometryDataKHR {
2841 instances: instance_data,
2842 });
2843
2844 (
2845 smallvec::smallvec![geometry],
2846 smallvec::smallvec![instances.count],
2847 )
2848 }
2849 crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
2850 let mut primitive_counts =
2851 smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2852 let mut geometries = smallvec::SmallVec::<
2853 [vk::AccelerationStructureGeometryKHR; CAPACITY],
2854 >::with_capacity(in_geometries.len());
2855
2856 for triangles in in_geometries {
2857 let mut triangle_data =
2858 vk::AccelerationStructureGeometryTrianglesDataKHR::default()
2859 .index_type(vk::IndexType::NONE_KHR)
2860 .vertex_format(conv::map_vertex_format(triangles.vertex_format))
2861 .max_vertex(triangles.vertex_count)
2862 .vertex_stride(triangles.vertex_stride)
2863 .transform_data(vk::DeviceOrHostAddressConstKHR {
2873 device_address: if desc
2874 .flags
2875 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
2876 {
2877 unsafe {
2878 ray_tracing_functions
2879 .buffer_device_address
2880 .get_buffer_device_address(
2881 &vk::BufferDeviceAddressInfo::default().buffer(
2882 triangles
2883 .transform
2884 .as_ref()
2885 .unwrap()
2886 .buffer
2887 .raw,
2888 ),
2889 )
2890 }
2891 } else {
2892 0
2893 },
2894 });
2895
2896 let pritive_count = if let Some(ref indices) = triangles.indices {
2897 triangle_data =
2898 triangle_data.index_type(conv::map_index_format(indices.format));
2899 indices.count / 3
2900 } else {
2901 triangles.vertex_count / 3
2902 };
2903
2904 let geometry = vk::AccelerationStructureGeometryKHR::default()
2905 .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
2906 .geometry(vk::AccelerationStructureGeometryDataKHR {
2907 triangles: triangle_data,
2908 })
2909 .flags(conv::map_acceleration_structure_geometry_flags(
2910 triangles.flags,
2911 ));
2912
2913 geometries.push(geometry);
2914 primitive_counts.push(pritive_count);
2915 }
2916 (geometries, primitive_counts)
2917 }
2918 crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
2919 let mut primitive_counts =
2920 smallvec::SmallVec::<[u32; CAPACITY]>::with_capacity(in_geometries.len());
2921 let mut geometries = smallvec::SmallVec::<
2922 [vk::AccelerationStructureGeometryKHR; CAPACITY],
2923 >::with_capacity(in_geometries.len());
2924 for aabb in in_geometries {
2925 let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
2926 .stride(aabb.stride);
2927
2928 let geometry = vk::AccelerationStructureGeometryKHR::default()
2929 .geometry_type(vk::GeometryTypeKHR::AABBS)
2930 .geometry(vk::AccelerationStructureGeometryDataKHR { aabbs: aabbs_data })
2931 .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
2932
2933 geometries.push(geometry);
2934 primitive_counts.push(aabb.count);
2935 }
2936 (geometries, primitive_counts)
2937 }
2938 };
2939
2940 let ty = match *desc.entries {
2941 crate::AccelerationStructureEntries::Instances(_) => {
2942 vk::AccelerationStructureTypeKHR::TOP_LEVEL
2943 }
2944 _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
2945 };
2946
2947 let geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
2948 .ty(ty)
2949 .flags(conv::map_acceleration_structure_flags(desc.flags))
2950 .geometries(&geometries);
2951
2952 let mut raw = Default::default();
2953 unsafe {
2954 ray_tracing_functions
2955 .acceleration_structure
2956 .get_acceleration_structure_build_sizes(
2957 vk::AccelerationStructureBuildTypeKHR::DEVICE,
2958 &geometry_info,
2959 &primitive_counts,
2960 &mut raw,
2961 )
2962 }
2963
2964 crate::AccelerationStructureBuildSizes {
2965 acceleration_structure_size: raw.acceleration_structure_size,
2966 update_scratch_size: raw.update_scratch_size,
2967 build_scratch_size: raw.build_scratch_size,
2968 }
2969 }
2970
2971 unsafe fn get_acceleration_structure_device_address(
2972 &self,
2973 acceleration_structure: &super::AccelerationStructure,
2974 ) -> wgt::BufferAddress {
2975 let ray_tracing_functions = self
2976 .shared
2977 .extension_fns
2978 .ray_tracing
2979 .as_ref()
2980 .expect("Feature `RAY_TRACING` not enabled");
2981
2982 unsafe {
2983 ray_tracing_functions
2984 .acceleration_structure
2985 .get_acceleration_structure_device_address(
2986 &vk::AccelerationStructureDeviceAddressInfoKHR::default()
2987 .acceleration_structure(acceleration_structure.raw),
2988 )
2989 }
2990 }
2991
2992 unsafe fn create_acceleration_structure(
2993 &self,
2994 desc: &crate::AccelerationStructureDescriptor,
2995 ) -> Result<super::AccelerationStructure, crate::DeviceError> {
2996 let ray_tracing_functions = self
2997 .shared
2998 .extension_fns
2999 .ray_tracing
3000 .as_ref()
3001 .expect("Feature `RAY_TRACING` not enabled");
3002
3003 let vk_buffer_info = vk::BufferCreateInfo::default()
3004 .size(desc.size)
3005 .usage(
3006 vk::BufferUsageFlags::ACCELERATION_STRUCTURE_STORAGE_KHR
3007 | vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS,
3008 )
3009 .sharing_mode(vk::SharingMode::EXCLUSIVE);
3010
3011 unsafe {
3012 let raw_buffer = self
3013 .shared
3014 .raw
3015 .create_buffer(&vk_buffer_info, None)
3016 .map_err(super::map_host_device_oom_and_ioca_err)?;
3017
3018 let requirements = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
3019
3020 self.error_if_would_oom_on_resource_allocation(
3021 gpu_allocator::MemoryLocation::GpuOnly,
3022 &requirements,
3023 )
3024 .inspect_err(|_| {
3025 self.shared.raw.destroy_buffer(raw_buffer, None);
3026 })?;
3027
3028 let name = desc
3029 .label
3030 .unwrap_or("Unlabeled acceleration structure buffer");
3031
3032 let pool = if desc.allow_compaction {
3033 super::MemoryPool::Transient
3034 } else {
3035 super::MemoryPool::General
3036 };
3037 let allocation = self
3038 .mem_allocator
3039 .allocate(
3040 pool,
3041 &gpu_allocator::vulkan::AllocationCreateDesc {
3042 name,
3043 requirements,
3044 location: gpu_allocator::MemoryLocation::GpuOnly,
3045 linear: true, allocation_scheme:
3047 gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
3048 },
3049 )
3050 .inspect_err(|_| {
3051 self.shared.raw.destroy_buffer(raw_buffer, None);
3052 })?;
3053
3054 self.shared
3055 .raw
3056 .bind_buffer_memory(raw_buffer, allocation.memory(), allocation.offset())
3057 .map_err(super::map_host_device_oom_and_ioca_err)
3058 .inspect_err(|_| {
3059 self.shared.raw.destroy_buffer(raw_buffer, None);
3060 })?;
3061
3062 if let Some(label) = desc.label {
3063 self.shared.set_object_name(raw_buffer, label);
3064 }
3065
3066 let vk_info = vk::AccelerationStructureCreateInfoKHR::default()
3067 .buffer(raw_buffer)
3068 .offset(0)
3069 .size(desc.size)
3070 .ty(conv::map_acceleration_structure_format(desc.format));
3071
3072 let raw_acceleration_structure = ray_tracing_functions
3073 .acceleration_structure
3074 .create_acceleration_structure(&vk_info, None)
3075 .map_err(super::map_host_oom_and_ioca_err)
3076 .inspect_err(|_| {
3077 self.shared.raw.destroy_buffer(raw_buffer, None);
3078 })?;
3079
3080 if let Some(label) = desc.label {
3081 self.shared
3082 .set_object_name(raw_acceleration_structure, label);
3083 }
3084
3085 let pool = if desc.allow_compaction {
3086 let vk_info = vk::QueryPoolCreateInfo::default()
3087 .query_type(vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR)
3088 .query_count(1);
3089
3090 let raw = self
3091 .shared
3092 .raw
3093 .create_query_pool(&vk_info, None)
3094 .map_err(super::map_host_device_oom_err)
3095 .inspect_err(|_| {
3096 ray_tracing_functions
3097 .acceleration_structure
3098 .destroy_acceleration_structure(raw_acceleration_structure, None);
3099 self.shared.raw.destroy_buffer(raw_buffer, None);
3100 })?;
3101 Some(raw)
3102 } else {
3103 None
3104 };
3105
3106 Ok(super::AccelerationStructure {
3107 raw: raw_acceleration_structure,
3108 buffer: raw_buffer,
3109 allocation,
3110 compacted_size_query: pool,
3111 })
3112 }
3113 }
3114
3115 unsafe fn destroy_acceleration_structure(
3116 &self,
3117 acceleration_structure: super::AccelerationStructure,
3118 ) {
3119 let ray_tracing_functions = self
3120 .shared
3121 .extension_fns
3122 .ray_tracing
3123 .as_ref()
3124 .expect("Feature `RAY_TRACING` not enabled");
3125
3126 unsafe {
3127 ray_tracing_functions
3128 .acceleration_structure
3129 .destroy_acceleration_structure(acceleration_structure.raw, None);
3130 self.shared
3131 .raw
3132 .destroy_buffer(acceleration_structure.buffer, None);
3133 let result = self.mem_allocator.free(acceleration_structure.allocation);
3134 if let Err(err) = result {
3135 log::warn!("Failed to free buffer acceleration structure: {err}");
3136 }
3137 if let Some(query) = acceleration_structure.compacted_size_query {
3138 self.shared.raw.destroy_query_pool(query, None)
3139 }
3140 }
3141 }
3142
3143 fn get_internal_counters(&self) -> wgt::HalCounters {
3144 self.counters
3145 .memory_allocations
3146 .set(self.shared.memory_allocations_counter.read());
3147
3148 self.counters.as_ref().clone()
3149 }
3150
3151 fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
3152 Some(self.mem_allocator.generate_report())
3153 }
3154
3155 fn tlas_instance_to_bytes(&self, instance: TlasInstance, to_extend: &mut Vec<u8>) {
3156 const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
3157 let temp = RawTlasInstance {
3158 transform: instance.transform,
3159 custom_data_and_mask: (instance.custom_data & MAX_U24)
3160 | (u32::from(instance.mask) << 24),
3161 shader_binding_table_record_offset_and_flags: (instance
3162 .pipeline_intersection_data_offset
3163 & MAX_U24),
3164 acceleration_structure_reference: instance.blas_address,
3165 };
3166 to_extend.extend_from_slice(bytemuck::bytes_of(&temp))
3167 }
3168
3169 fn check_if_oom(&self) -> Result<(), crate::DeviceError> {
3170 let Some(threshold) = self
3171 .shared
3172 .instance
3173 .memory_budget_thresholds
3174 .for_device_loss
3175 else {
3176 return Ok(());
3177 };
3178
3179 let Some(memory_properties) = self.get_memory_properties() else {
3180 return Ok(());
3181 };
3182
3183 for i in 0..memory_properties.heaps().len() {
3184 let heap_usage = memory_properties.heap_usage()[i];
3185 let heap_budget = memory_properties.heap_budget()[i];
3186
3187 if heap_usage >= heap_budget / 100 * threshold as u64 {
3188 return Err(crate::DeviceError::OutOfMemory);
3189 }
3190 }
3191
3192 Ok(())
3193 }
3194}
3195
3196impl super::DeviceShared {
3197 pub(super) fn new_binary_semaphore(
3198 &self,
3199 name: &str,
3200 ) -> Result<vk::Semaphore, crate::DeviceError> {
3201 unsafe {
3202 let semaphore = self
3203 .raw
3204 .create_semaphore(&vk::SemaphoreCreateInfo::default(), None)
3205 .map_err(super::map_host_device_oom_err)?;
3206
3207 if !self
3208 .instance
3209 .flags
3210 .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS)
3211 {
3212 self.set_object_name(semaphore, name);
3213 }
3214
3215 Ok(semaphore)
3216 }
3217 }
3218
3219 pub(super) fn wait_for_fence(
3220 &self,
3221 fence: &super::Fence,
3222 wait_value: crate::FenceValue,
3223 timeout_ns: u64,
3224 ) -> Result<bool, crate::DeviceError> {
3225 profiling::scope!("Device::wait");
3226 match *fence {
3227 super::Fence::TimelineSemaphore(raw) => {
3228 let semaphores = [raw];
3229 let values = [wait_value];
3230 let vk_info = vk::SemaphoreWaitInfo::default()
3231 .semaphores(&semaphores)
3232 .values(&values);
3233 let result = match self.extension_fns.timeline_semaphore {
3234 Some(super::ExtensionFn::Extension(ref ext)) => unsafe {
3235 ext.wait_semaphores(&vk_info, timeout_ns)
3236 },
3237 Some(super::ExtensionFn::Promoted) => unsafe {
3238 self.raw.wait_semaphores(&vk_info, timeout_ns)
3239 },
3240 None => unreachable!(),
3241 };
3242 match result {
3243 Ok(()) => Ok(true),
3244 Err(vk::Result::TIMEOUT) => Ok(false),
3245 Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
3246 }
3247 }
3248 super::Fence::FencePool(ref pool) => {
3249 let pool = pool.read();
3250 let super::FencePool {
3251 last_completed,
3252 ref active,
3253 free: _,
3254 } = *pool;
3255 if wait_value <= last_completed {
3256 Ok(true)
3257 } else {
3258 match active.iter().find(|&&(value, _)| value >= wait_value) {
3259 Some((_, fence)) => {
3260 let fence = fence.clone();
3262 drop(pool);
3263 match unsafe {
3264 self.raw.wait_for_fences(
3265 core::slice::from_ref(&fence),
3266 true,
3267 timeout_ns,
3268 )
3269 } {
3270 Ok(()) => Ok(true),
3271 Err(vk::Result::TIMEOUT) => Ok(false),
3272 Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
3273 }
3274 }
3275 None => {
3276 crate::hal_usage_error(format!(
3277 "no signals reached value {wait_value}"
3278 ));
3279 }
3280 }
3281 }
3282 }
3283 }
3284 }
3285}
3286
3287struct ImageWithoutMemory {
3288 raw: vk::Image,
3289 requirements: vk::MemoryRequirements,
3290}