1use alloc::{
2 borrow::Cow,
3 boxed::Box,
4 string::{String, ToString as _},
5 sync::{Arc, Weak},
6 vec::Vec,
7};
8use core::{
9 fmt,
10 mem::{self, ManuallyDrop},
11 num::{NonZeroU32, NonZeroU64},
12};
13
14use arrayvec::ArrayVec;
15use bitflags::Flags;
16use scopeguard::{guard, ScopeGuard};
17use smallvec::SmallVec;
18use wgpu_sync::atomic::{AtomicBool, Ordering};
19use wgpu_sync::OnceCell;
20use wgt::{
21 error::WebGpuError, math::align_to, ColorWrites, DeviceLostReason, TextureFormat,
22 TextureSampleType, TextureViewDimension,
23};
24
25#[cfg(feature = "trace")]
26use crate::device::trace::{self, IntoTrace as _};
27use crate::{
28 api_log,
29 binding_model::{
30 self, BindGroup, BindGroupLateBufferBindingInfo, BindGroupLayout,
31 BindGroupLayoutEntryError, BindGroupLayoutState, BindGroupState, CreateBindGroupError,
32 CreateBindGroupLayoutError,
33 },
34 command, conv,
35 device::{
36 bgl, create_validator, life::WaitIdleError, map_buffer, stage_compiler::StageCompiler,
37 AttachmentData, BufferMapPendingClosure, DeviceLostInvocation, MapMode,
38 MissingDownlevelFlags, MissingFeatures, RenderPassContext,
39 },
40 error::ErrorSink,
41 hal_label,
42 init_tracker::{
43 BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
44 TextureInitTrackerAction,
45 },
46 instance::{Adapter, RequestDeviceError},
47 lock::{rank, Mutex, RwLock},
48 pipeline::{
49 self, shader_module_error_into_compilation_info, ColorStateError, PassthroughInterface,
50 },
51 pool::ResourcePool,
52 resource::{
53 self, Buffer, BufferState, ExternalTexture, ExternalTextureState, Labeled, ParentDevice,
54 QuerySet, QuerySetState, RawResourceAccess, ResourceState, Sampler, StagingBuffer, Texture,
55 TextureView, Tlas, TrackingData,
56 },
57 resource_log,
58 snatch::{SnatchGuard, SnatchLock, Snatchable},
59 timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
60 track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
61 validation::{self, check_color_attachment_count},
62 weak_vec::WeakVec,
63 FastHashMap, LabelHelpers,
64};
65
66use super::{
67 queue::Queue, DeviceDescriptor, DeviceError, DeviceLostClosure, UserClosures, ZERO_BUFFER_SIZE,
68};
69
70use wgpu_sync::atomic::AtomicU64;
71
72pub(crate) struct CommandIndices {
73 pub(crate) active_submission_index: hal::FenceValue,
81 pub(crate) next_acceleration_structure_build_command_index: u64,
82}
83
84#[repr(C)]
95#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
96pub struct ExternalTextureParams {
97 pub yuv_conversion_matrix: [f32; 16],
102
103 pub gamut_conversion_matrix: [f32; 12],
116
117 pub src_transfer_function: wgt::ExternalTextureTransferFunction,
121
122 pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
125
126 pub sample_transform: [f32; 6],
139
140 pub load_transform: [f32; 6],
156
157 pub size: [u32; 2],
170
171 pub num_planes: u32,
175 pub _padding: [u8; 4],
177}
178
179const _: () =
180 assert!(size_of::<ExternalTextureParams>() == wgpu_hal::EXTERNAL_TEXTURE_PARAMS_SIZE as usize);
181const EXTERNAL_TEXTURE_PARAMS_BUFFER_SIZE: usize = {
182 size_of::<ExternalTextureParams>()
183 .next_multiple_of(wgpu_hal::UNIVERSAL_BUFFER_SIZE_ALIGNMENT as usize)
184};
185
186impl ExternalTextureParams {
187 pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
188 let gamut_conversion_matrix = [
189 desc.gamut_conversion_matrix[0],
190 desc.gamut_conversion_matrix[1],
191 desc.gamut_conversion_matrix[2],
192 0.0, desc.gamut_conversion_matrix[3],
194 desc.gamut_conversion_matrix[4],
195 desc.gamut_conversion_matrix[5],
196 0.0, desc.gamut_conversion_matrix[6],
198 desc.gamut_conversion_matrix[7],
199 desc.gamut_conversion_matrix[8],
200 0.0, ];
202
203 Self {
204 yuv_conversion_matrix: desc.yuv_conversion_matrix,
205 gamut_conversion_matrix,
206 src_transfer_function: desc.src_transfer_function,
207 dst_transfer_function: desc.dst_transfer_function,
208 size: [desc.width, desc.height],
209 sample_transform: desc.sample_transform,
210 load_transform: desc.load_transform,
211 num_planes: desc.num_planes() as u32,
212 _padding: Default::default(),
213 }
214 }
215}
216
217pub(crate) struct DeferredBufferMapPendingClosures(wgpu_sync::Mutex<Vec<BufferMapPendingClosure>>);
220
221impl DeferredBufferMapPendingClosures {
222 pub(crate) fn new() -> Self {
223 Self(wgpu_sync::Mutex::new(Vec::new()))
224 }
225
226 pub(crate) fn push(&self, closure: BufferMapPendingClosure) {
227 self.0.lock().push(closure);
228 }
229
230 pub(crate) fn swap(&self, other: &mut Vec<BufferMapPendingClosure>) {
231 mem::swap(&mut *self.0.lock(), other)
232 }
233}
234
235pub struct Device {
238 pub(crate) queue: OnceCell<Weak<Queue>>,
239 pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
240 pub(crate) empty_bgl: ManuallyDrop<Box<dyn hal::DynBindGroupLayout>>,
241 label: String,
243
244 pub(crate) command_allocator: command::CommandAllocator,
245
246 pub(crate) command_indices: RwLock<CommandIndices>,
250
251 pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
261
262 pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
263 pub(crate) snatchable_lock: SnatchLock,
264
265 pub(crate) valid: AtomicBool,
277
278 pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
282
283 pub(crate) error_sink: ErrorSink,
284
285 pub(crate) trackers: Mutex<DeviceTracker>,
287 pub(crate) tracker_indices: TrackerIndexAllocators,
288 pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
290 pub(crate) alignments: hal::Alignments,
291 pub(crate) limits: wgt::Limits,
292 pub(crate) features: wgt::Features,
293 pub(crate) downlevel: wgt::DownlevelCapabilities,
294 pub(crate) ordered_buffer_usages: wgt::BufferUses,
299 pub(crate) ordered_texture_usages: wgt::TextureUses,
304 pub(crate) instance_flags: wgt::InstanceFlags,
305 pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
306 pub(crate) deferred_buffer_map_pending_closures: DeferredBufferMapPendingClosures,
308 pub(crate) usage_scopes: UsageScopePool,
309 pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
310 pub(crate) timestamp_normalizer: OnceCell<crate::timestamp_normalization::TimestampNormalizer>,
312 pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
317
318 raw: Box<dyn hal::DynDevice>,
327 pub(crate) adapter: Arc<Adapter>,
328 #[cfg(feature = "trace")]
329 pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
330}
331
332#[cfg(send_sync)]
333const _: () = {
334 unsafe impl Send for Device {}
336 unsafe impl Sync for Device {}
338
339 fn _device_fields_are_send_sync(obj: &Device) {
340 fn _check_bound<T: Send + Sync>(_: &T) {}
341 let Device {
342 queue,
343 zero_buffer,
344 empty_bgl,
345 label,
346 command_allocator,
347 command_indices,
348 last_successful_submission_index,
349 fence,
350 snatchable_lock,
351 valid,
352 device_lost_closure,
353 error_sink,
354 trackers,
355 tracker_indices,
356 bgl_pool,
357 alignments,
358 limits,
359 features,
360 downlevel,
361 ordered_buffer_usages,
362 ordered_texture_usages,
363 instance_flags,
364 deferred_destroy,
365 deferred_buffer_map_pending_closures,
366 usage_scopes,
367 indirect_validation,
368 timestamp_normalizer,
369 default_external_texture_params_buffer,
370 raw,
371 adapter,
372 #[cfg(feature = "trace")]
373 trace,
374 } = obj;
375 _check_bound(queue);
376 _check_bound(zero_buffer);
377 _check_bound(empty_bgl);
378 _check_bound(label);
379 _check_bound(command_allocator);
380 _check_bound(command_indices);
381 _check_bound(last_successful_submission_index);
382 _check_bound(fence);
383 _check_bound(snatchable_lock);
384 _check_bound(valid);
385 _check_bound(device_lost_closure);
386 _check_bound(error_sink);
387 _check_bound(trackers);
388 _check_bound(tracker_indices);
389 _check_bound(bgl_pool);
390 _check_bound(alignments);
391 _check_bound(limits);
392 _check_bound(features);
393 _check_bound(downlevel);
394 _check_bound(ordered_buffer_usages);
395 _check_bound(ordered_texture_usages);
396 _check_bound(instance_flags);
397 _check_bound(deferred_destroy);
398 _check_bound(deferred_buffer_map_pending_closures);
399 _check_bound(usage_scopes);
400 _check_bound(indirect_validation);
401 _check_bound(timestamp_normalizer);
402 _check_bound(default_external_texture_params_buffer);
403 _check_bound(raw);
404 _check_bound(adapter);
405 #[cfg(feature = "trace")]
406 _check_bound(trace);
407 }
408};
409
410pub(crate) enum DeferredDestroy {
411 TextureViews(WeakVec<TextureView>),
412 BindGroups(WeakVec<BindGroup>),
413}
414
415impl fmt::Debug for Device {
416 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417 f.debug_struct("Device")
418 .field("label", &self.label())
419 .field("limits", &self.limits)
420 .field("features", &self.features)
421 .field("downlevel", &self.downlevel)
422 .finish()
423 }
424}
425
426impl Drop for Device {
427 #[allow(trivial_casts)]
428 fn drop(&mut self) {
429 profiling::scope!("Device::drop");
430 api_log!("Device::drop {:?}", self as *const _);
431 resource_log!("Drop {}", self.error_ident());
432
433 let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
436 let empty_bgl = unsafe { ManuallyDrop::take(&mut self.empty_bgl) };
438 let default_external_texture_params_buffer =
441 unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
442 let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
444 if let Some(indirect_validation) = self.indirect_validation.take() {
445 indirect_validation.dispose(self.raw.as_ref());
446 }
447 if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
448 timestamp_normalizer.dispose(self.raw.as_ref());
449 }
450 unsafe {
451 self.raw.destroy_buffer(zero_buffer);
452 self.raw.destroy_bind_group_layout(empty_bgl);
453 self.raw
454 .destroy_buffer(default_external_texture_params_buffer);
455 self.raw.destroy_fence(fence);
456 }
457 }
458}
459
460impl Device {
461 pub fn features(&self) -> &wgt::Features {
462 &self.features
463 }
464
465 pub fn limits(&self) -> &wgt::Limits {
466 &self.limits
467 }
468
469 pub fn downlevel(&self) -> &wgt::DownlevelCapabilities {
470 &self.downlevel
471 }
472
473 pub fn adapter_info(&self) -> wgt::AdapterInfo {
474 self.adapter.get_info()
475 }
476}
477
478impl Device {
479 pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
480 self.raw.as_ref()
481 }
482
483 pub fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
484 if self.features.contains(feature) {
485 Ok(())
486 } else {
487 Err(MissingFeatures(feature))
488 }
489 }
490
491 pub(crate) fn require_downlevel_flags(
492 &self,
493 flags: wgt::DownlevelFlags,
494 ) -> Result<(), MissingDownlevelFlags> {
495 if self.downlevel.flags.contains(flags) {
496 Ok(())
497 } else {
498 Err(MissingDownlevelFlags(flags))
499 }
500 }
501
502 pub unsafe fn start_graphics_debugger_capture(&self) {
508 api_log!("Device::start_graphics_debugger_capture");
509
510 if !self.is_valid() {
511 return;
512 }
513 unsafe { self.raw().start_graphics_debugger_capture() };
514 }
515
516 pub unsafe fn stop_graphics_debugger_capture(&self) {
522 api_log!("Device::stop_graphics_debugger_capture");
523
524 if !self.is_valid() {
525 return;
526 }
527 unsafe { self.raw().stop_graphics_debugger_capture() };
528 }
529}
530
531impl Device {
532 pub(crate) fn new(
533 raw_device: Box<dyn hal::DynDevice>,
534 adapter: &Arc<Adapter>,
535 desc: &DeviceDescriptor,
536 instance_flags: wgt::InstanceFlags,
537 ) -> Result<Self, DeviceError> {
538 #[cfg(not(feature = "trace"))]
539 match &desc.trace {
540 wgt::Trace::Off => {}
541 _ => {
542 log::error!("wgpu-core feature 'trace' is not enabled");
543 }
544 };
545 #[cfg(feature = "trace")]
546 let trace: Option<Box<dyn trace::Trace + Send + Sync + 'static>> = match &desc.trace {
547 wgt::Trace::Off => None,
548 wgt::Trace::Directory(dir) => match trace::DiskTrace::new(dir.clone()) {
549 Ok(mut trace) => {
550 trace::Trace::add(
551 &mut trace,
552 trace::Action::Init {
553 desc: wgt::DeviceDescriptor {
554 trace: wgt::Trace::Off,
555 ..desc.clone()
556 },
557 backend: adapter.backend(),
558 },
559 );
560 Some(Box::new(trace))
561 }
562 Err(e) => {
563 log::error!("Unable to start a trace in '{dir:?}': {e}");
564 None
565 }
566 },
567 wgt::Trace::Memory => {
568 let mut trace = trace::MemoryTrace::new();
569 trace::Trace::add(
570 &mut trace,
571 trace::Action::Init {
572 desc: wgt::DeviceDescriptor {
573 trace: wgt::Trace::Off,
574 ..desc.clone()
575 },
576 backend: adapter.backend(),
577 },
578 );
579 Some(Box::new(trace))
580 }
581 t => {
584 log::error!("unimplemented wgpu_types::Trace variant {t:?}");
585 None
586 }
587 };
588
589 let ordered_buffer_usages = adapter.raw.adapter.get_ordered_buffer_usages();
590 let ordered_texture_usages = adapter.raw.adapter.get_ordered_texture_usages();
591
592 let raw = raw_device.as_ref();
596
597 let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
598 let fence = guard(fence, |fence| unsafe { raw.destroy_fence(fence) });
599
600 let command_allocator = command::CommandAllocator::new();
601
602 let rt_uses = if desc
603 .required_features
604 .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
605 {
606 wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
607 } else {
608 wgt::BufferUses::empty()
609 };
610
611 let zero_buffer = unsafe {
613 raw_device.create_buffer(&hal::BufferDescriptor {
614 label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
615 size: ZERO_BUFFER_SIZE,
616 usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
617 memory_flags: hal::MemoryFlags::empty(),
618 })
619 }
620 .map_err(DeviceError::from_hal)?
621 .0;
622 let zero_buffer = guard(zero_buffer, |buffer| unsafe { raw.destroy_buffer(buffer) });
623
624 let empty_bgl = unsafe {
625 raw_device.create_bind_group_layout(&hal::BindGroupLayoutDescriptor {
626 label: None,
627 flags: hal::BindGroupLayoutFlags::empty(),
628 entries: &[],
629 })
630 }
631 .map_err(DeviceError::from_hal)?;
632 let empty_bgl = guard(empty_bgl, |bgl| unsafe {
633 raw.destroy_bind_group_layout(bgl)
634 });
635
636 let default_external_texture_params_buffer = unsafe {
637 raw_device.create_buffer(&hal::BufferDescriptor {
638 label: hal_label(
639 Some("(wgpu internal) default external texture params buffer"),
640 instance_flags,
641 ),
642 size: EXTERNAL_TEXTURE_PARAMS_BUFFER_SIZE as _,
643 usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
644 memory_flags: hal::MemoryFlags::empty(),
645 })
646 }
647 .map_err(DeviceError::from_hal)?
648 .0;
649 let default_external_texture_params_buffer =
650 guard(default_external_texture_params_buffer, |buffer| unsafe {
651 raw.destroy_buffer(buffer)
652 });
653
654 let alignments = adapter.raw.capabilities.alignments.clone();
656 let downlevel = adapter.raw.capabilities.downlevel.clone();
657 let limits = &adapter.raw.capabilities.limits;
658
659 let enable_indirect_validation = instance_flags
660 .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
661 && downlevel.flags.contains(
662 wgt::DownlevelFlags::INDIRECT_EXECUTION | wgt::DownlevelFlags::COMPUTE_SHADERS,
663 )
664 && limits.max_storage_buffers_per_shader_stage >= 2;
665
666 let indirect_validation = if enable_indirect_validation {
667 let indirect_validation = crate::indirect_validation::IndirectValidation::new(
668 raw,
669 &desc.required_limits,
670 &desc.required_features,
671 instance_flags,
672 adapter.backend(),
673 )?;
674 Some(guard(indirect_validation, |indirect_validation| {
675 indirect_validation.dispose(raw)
676 }))
677 } else {
678 None
679 };
680
681 #[deny(clippy::question_mark_used)]
683 {
684 let zero_buffer = ScopeGuard::into_inner(zero_buffer);
685 let empty_bgl = ScopeGuard::into_inner(empty_bgl);
686 let default_external_texture_params_buffer =
687 ScopeGuard::into_inner(default_external_texture_params_buffer);
688 let fence = ScopeGuard::into_inner(fence);
689 let indirect_validation = indirect_validation.map(ScopeGuard::into_inner);
690
691 Ok(Self {
692 raw: raw_device,
693 adapter: adapter.clone(),
694 queue: OnceCell::new(),
695 zero_buffer: ManuallyDrop::new(zero_buffer),
696 empty_bgl: ManuallyDrop::new(empty_bgl),
697 default_external_texture_params_buffer: ManuallyDrop::new(
698 default_external_texture_params_buffer,
699 ),
700 label: desc.label.to_string(),
701 command_allocator,
702 command_indices: RwLock::new(
703 rank::DEVICE_COMMAND_INDICES,
704 CommandIndices {
705 active_submission_index: 0,
706 next_acceleration_structure_build_command_index: 1,
708 },
709 ),
710 last_successful_submission_index: AtomicU64::new(0),
711 fence: ManuallyDrop::new(fence),
712 snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
713 valid: AtomicBool::new(true),
714 device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
715 error_sink: ErrorSink::new(),
716 trackers: Mutex::new(
717 rank::DEVICE_TRACKERS,
718 DeviceTracker::new(ordered_buffer_usages, ordered_texture_usages),
719 ),
720 tracker_indices: TrackerIndexAllocators::new(),
721 bgl_pool: ResourcePool::new(),
722 #[cfg(feature = "trace")]
723 trace: Mutex::new(rank::DEVICE_TRACE, trace),
724 alignments,
725 limits: desc.required_limits.clone(),
726 features: desc.required_features,
727 downlevel,
728 ordered_buffer_usages,
729 ordered_texture_usages,
730 instance_flags,
731 deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
732 usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
733 timestamp_normalizer: OnceCell::new(),
734 indirect_validation,
735 deferred_buffer_map_pending_closures: DeferredBufferMapPendingClosures::new(),
736 })
737 }
738 }
739
740 fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
745 let data = ExternalTextureParams {
746 #[rustfmt::skip]
747 yuv_conversion_matrix: [
748 1.0, 0.0, 0.0, 0.0,
749 0.0, 1.0, 0.0, 0.0,
750 0.0, 0.0, 1.0, 0.0,
751 0.0, 0.0, 0.0, 1.0,
752 ],
753 #[rustfmt::skip]
754 gamut_conversion_matrix: [
755 1.0, 0.0, 0.0, 0.0,
756 0.0, 1.0, 0.0, 0.0,
757 0.0, 0.0, 1.0, 0.0,
758 ],
759 src_transfer_function: Default::default(),
760 dst_transfer_function: Default::default(),
761 size: [0, 0],
762 #[rustfmt::skip]
763 sample_transform: [
764 1.0, 0.0,
765 0.0, 1.0,
766 0.0, 0.0
767 ],
768 #[rustfmt::skip]
769 load_transform: [
770 1.0, 0.0,
771 0.0, 1.0,
772 0.0, 0.0
773 ],
774 num_planes: 1,
775 _padding: Default::default(),
776 };
777 let mut staging_buffer = StagingBuffer::new(
778 self,
779 wgt::BufferSize::new(EXTERNAL_TEXTURE_PARAMS_BUFFER_SIZE as _).unwrap(),
780 )?;
781 staging_buffer.write_with_zero_padding(bytemuck::bytes_of(&data));
782 let staging_buffer = staging_buffer.flush();
783
784 let params_buffer = self.default_external_texture_params_buffer.as_ref();
785 let queue = self.get_queue().unwrap();
786 let mut pending_writes = queue.pending_writes.lock();
787
788 unsafe {
789 pending_writes
790 .command_encoder
791 .transition_buffers(&[hal::BufferBarrier {
792 buffer: params_buffer,
793 usage: hal::StateTransition {
794 from: wgt::BufferUses::MAP_WRITE,
795 to: wgt::BufferUses::COPY_DST,
796 },
797 }]);
798 pending_writes.command_encoder.copy_buffer_to_buffer(
799 staging_buffer.raw(),
800 params_buffer,
801 &[hal::BufferCopy {
802 src_offset: 0,
803 dst_offset: 0,
804 size: staging_buffer.size,
805 }],
806 );
807 pending_writes.consume(staging_buffer);
808 pending_writes
809 .command_encoder
810 .transition_buffers(&[hal::BufferBarrier {
811 buffer: params_buffer,
812 usage: hal::StateTransition {
813 from: wgt::BufferUses::COPY_DST,
814 to: wgt::BufferUses::UNIFORM,
815 },
816 }]);
817 }
818
819 Ok(())
820 }
821
822 pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
823 let queue = self.get_queue().unwrap();
824
825 let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
826 self,
827 queue.get_raw_timestamp_period(),
828 )?;
829
830 self.timestamp_normalizer
831 .set(timestamp_normalizer)
832 .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
833
834 self.init_default_external_texture_params_buffer()?;
835
836 Ok(())
837 }
838
839 pub fn backend(&self) -> wgt::Backend {
841 self.adapter.backend()
842 }
843
844 pub fn adapter(&self) -> &Arc<Adapter> {
846 &self.adapter
847 }
848
849 pub fn is_valid(&self) -> bool {
860 self.valid.load(Ordering::Acquire)
861 }
862
863 pub fn check_is_valid(&self) -> Result<(), DeviceError> {
869 if self.is_valid() {
870 Ok(())
871 } else {
872 Err(DeviceError::Lost)
873 }
874 }
875
876 #[cfg(feature = "trace")]
880 pub fn take_trace(&self) -> Option<Box<dyn trace::Trace + Send + Sync + 'static>> {
881 self.trace.lock().take()
882 }
883
884 pub fn lose_if_oom(&self) {
891 let _ = self
892 .raw()
893 .check_if_oom()
894 .map_err(|e| self.handle_hal_error(e));
895 }
896
897 pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
898 match error {
899 hal::DeviceError::OutOfMemory
900 | hal::DeviceError::Lost
901 | hal::DeviceError::Unexpected => {
902 self.lose(&error.to_string());
903 }
904 }
905 DeviceError::from_hal(error)
906 }
907
908 pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
909 match error {
910 hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
911 error => self.handle_hal_error(error),
912 }
913 }
914
915 pub(crate) fn deferred_resource_destruction(&self) {
923 let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
925 for item in deferred_destroy {
926 match item {
927 DeferredDestroy::TextureViews(views) => {
928 for view in views {
929 let Some(view) = view.upgrade() else {
930 continue;
931 };
932 let Ok(view_state) = view.state() else {
933 continue;
934 };
935 let Some(raw_view) =
936 view_state.raw.snatch(&mut self.snatchable_lock.write())
937 else {
938 continue;
939 };
940
941 resource_log!("Destroy raw {}", view.error_ident());
942
943 unsafe {
944 self.raw().destroy_texture_view(raw_view);
945 }
946 }
947 }
948 DeferredDestroy::BindGroups(bind_groups) => {
949 for bind_group in bind_groups {
950 let Some(bind_group) = bind_group.upgrade() else {
951 continue;
952 };
953 let Ok(bind_group_state) = bind_group.state() else {
954 continue;
955 };
956 let Some(raw_bind_group) = bind_group_state
957 .raw
958 .snatch(&mut self.snatchable_lock.write())
959 else {
960 continue;
961 };
962
963 resource_log!("Destroy raw {}", bind_group.error_ident());
964
965 unsafe {
966 self.raw().destroy_bind_group(raw_bind_group);
967 }
968 }
969 }
970 }
971 }
972 }
973
974 pub fn get_queue(&self) -> Option<Arc<Queue>> {
975 self.queue.get().as_ref()?.upgrade()
976 }
977
978 pub fn set_queue(&self, queue: &Arc<Queue>) {
979 assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
980 }
981
982 pub fn poll(
984 &self,
985 poll_type: wgt::PollType<crate::SubmissionIndex>,
986 ) -> Result<wgt::PollStatus, WaitIdleError> {
987 api_log!("Device::poll {poll_type:?}");
988 let (user_closures, result) = self.poll_and_return_closures(poll_type);
989 user_closures.fire();
990 result
991 }
992
993 pub(crate) fn poll_and_return_closures(
1002 &self,
1003 poll_type: wgt::PollType<crate::SubmissionIndex>,
1004 ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
1005 let snatch_guard = self.snatchable_lock.read();
1006 let maintain_result = self.maintain(poll_type, snatch_guard);
1007
1008 self.lose_if_oom();
1009
1010 self.deferred_resource_destruction();
1013
1014 maintain_result
1015 }
1016
1017 pub(crate) fn maintain<'this>(
1030 &'this self,
1031 poll_type: wgt::PollType<crate::SubmissionIndex>,
1032 snatch_guard: SnatchGuard,
1033 ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
1034 profiling::scope!("Device::maintain");
1035
1036 let mut user_closures = UserClosures::default();
1037
1038 self.deferred_buffer_map_pending_closures
1039 .swap(&mut user_closures.mappings);
1040
1041 let wait_submission_index = match poll_type {
1043 wgt::PollType::Wait {
1044 submission_index: Some(submission_index),
1045 ..
1046 } => {
1047 let last_successful_submission_index = self
1048 .last_successful_submission_index
1049 .load(Ordering::Acquire);
1050
1051 if submission_index > last_successful_submission_index {
1052 let result = Err(WaitIdleError::WrongSubmissionIndex(
1053 submission_index,
1054 last_successful_submission_index,
1055 ));
1056
1057 return (user_closures, result);
1058 }
1059
1060 Some(submission_index)
1061 }
1062 wgt::PollType::Wait {
1063 submission_index: None,
1064 ..
1065 } => Some(
1066 self.last_successful_submission_index
1067 .load(Ordering::Acquire),
1068 ),
1069 wgt::PollType::Poll => None,
1070 };
1071
1072 let wait_succeeded = if let Some(target_submission_index) = wait_submission_index {
1076 log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
1077
1078 let wait_timeout = match poll_type {
1079 wgt::PollType::Wait { timeout, .. } => timeout,
1080 wgt::PollType::Poll => unreachable!(
1081 "`wait_submission_index` index for poll type `Poll` should be None"
1082 ),
1083 };
1084
1085 let wait_result = unsafe {
1086 self.raw()
1087 .wait(self.fence.as_ref(), target_submission_index, wait_timeout)
1088 };
1089
1090 match wait_result {
1091 Ok(succeeded) => Some(succeeded),
1092 Err(e) => {
1093 let hal_error: WaitIdleError = self.handle_hal_error(e).into();
1094 return (user_closures, Err(hal_error));
1095 }
1096 }
1097 } else {
1098 None
1099 };
1100
1101 let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
1104 let current_finished_submission = match fence_value_result {
1105 Ok(fence_value) => fence_value,
1106 Err(e) => {
1107 let hal_error: WaitIdleError = self.handle_hal_error(e).into();
1108 return (user_closures, Err(hal_error));
1109 }
1110 };
1111
1112 let device_valid = {
1130 let _command_indices_guard = self.command_indices.read();
1131 self.is_valid()
1132 };
1133
1134 let mut queue_empty = false;
1140 if let Some(queue) = self.get_queue() {
1141 let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
1142 (
1143 user_closures.submissions,
1144 user_closures.mappings,
1145 user_closures.blas_compact_ready,
1146 queue_empty,
1147 ) = queue_result;
1148 drop(snatch_guard);
1162 } else {
1163 drop(snatch_guard);
1164 };
1165
1166 let result = if queue_empty && wait_succeeded != Some(false) {
1181 if let Some(wait_submission_index) = wait_submission_index {
1182 assert!(
1185 current_finished_submission >= wait_submission_index,
1186 concat!(
1187 "If the queue is empty, the current submission index ",
1188 "({}) should be at least the wait submission index ({})",
1189 ),
1190 current_finished_submission,
1191 wait_submission_index,
1192 );
1193 } else {
1194 }
1196
1197 Ok(wgt::PollStatus::QueueEmpty)
1198 } else if let Some(wait_submission_index) = wait_submission_index {
1199 if current_finished_submission >= wait_submission_index {
1203 Ok(wgt::PollStatus::WaitSucceeded)
1204 } else {
1205 Err(WaitIdleError::Timeout)
1206 }
1207 } else {
1208 Ok(wgt::PollStatus::Poll)
1209 };
1210
1211 let mut should_release_gpu_resource = false;
1218 if !device_valid && queue_empty {
1219 should_release_gpu_resource = true;
1222
1223 if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
1226 user_closures
1227 .device_lost_invocations
1228 .push(DeviceLostInvocation {
1229 closure: device_lost_closure,
1230 reason: DeviceLostReason::Destroyed,
1231 message: String::new(),
1232 });
1233 }
1234 }
1235
1236 if should_release_gpu_resource {
1237 self.release_gpu_resources();
1238 }
1239
1240 (user_closures, result)
1241 }
1242
1243 pub fn create_buffer_inner(
1244 self: &Arc<Self>,
1245 desc: &resource::BufferDescriptor,
1246 ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1247 self.check_is_valid()?;
1248
1249 if desc.size > self.limits.max_buffer_size {
1250 return Err(resource::CreateBufferError::MaxBufferSize {
1251 requested: desc.size,
1252 maximum: self.limits.max_buffer_size,
1253 });
1254 }
1255
1256 if desc
1257 .usage
1258 .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
1259 {
1260 self.require_acceleration_structures()?;
1261 }
1262
1263 if desc.usage.contains(wgt::BufferUsages::INDEX)
1264 && desc.usage.contains(
1265 wgt::BufferUsages::VERTEX
1266 | wgt::BufferUsages::UNIFORM
1267 | wgt::BufferUsages::INDIRECT
1268 | wgt::BufferUsages::STORAGE,
1269 )
1270 {
1271 self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
1272 }
1273
1274 if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1275 return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
1276 }
1277
1278 if !self
1279 .features
1280 .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
1281 {
1282 use wgt::BufferUsages as Bu;
1283 let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
1284 && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
1285 let read_mismatch = desc.usage.contains(Bu::MAP_READ)
1286 && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
1287 if write_mismatch || read_mismatch {
1288 return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
1289 }
1290 }
1291
1292 let mut usage = conv::map_buffer_usage(desc.usage);
1293
1294 if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
1295 self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
1296 usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
1299 }
1300
1301 if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
1302 usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
1303 }
1304
1305 if desc.mapped_at_creation {
1306 if !desc.size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1307 return Err(resource::CreateBufferError::UnalignedSize);
1308 }
1309 if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1310 usage |= wgt::BufferUses::COPY_DST;
1312 }
1313 } else {
1314 usage |= wgt::BufferUses::COPY_DST;
1317 }
1318
1319 let padded_size = if desc.size == 0 {
1338 wgt::COPY_BUFFER_ALIGNMENT
1339 } else if desc
1340 .usage
1341 .intersects(wgt::BufferUsages::VERTEX | wgt::BufferUsages::INDEX)
1342 {
1343 align_to(desc.size, wgt::COPY_BUFFER_ALIGNMENT) + wgt::COPY_BUFFER_ALIGNMENT
1345 } else {
1346 align_to(desc.size, wgt::COPY_BUFFER_ALIGNMENT)
1347 };
1348 let tail_start = desc.size & !(wgt::COPY_BUFFER_ALIGNMENT - 1);
1349
1350 let hal_desc = hal::BufferDescriptor {
1351 label: desc.label.to_hal(self.instance_flags),
1352 size: padded_size,
1353 usage,
1354 memory_flags: hal::MemoryFlags::empty(),
1355 };
1356 let (buffer, final_size) = unsafe { self.raw().create_buffer(&hal_desc) }
1357 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1358
1359 debug_assert!(final_size >= padded_size);
1360
1361 let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1362 self.timestamp_normalizer
1365 .get()
1366 .unwrap()
1367 .create_normalization_bind_group(
1368 self,
1369 &*buffer,
1370 desc.label.as_deref(),
1371 wgt::BufferSize::new(hal_desc.size).unwrap(),
1372 desc.usage,
1373 )
1374 }?);
1375
1376 let indirect_validation_bind_groups =
1377 self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
1378
1379 let buffer = Buffer {
1380 state: ResourceState::Valid(BufferState {
1381 raw: Snatchable::new(buffer),
1382 }),
1383 device: self.clone(),
1384 usage: desc.usage,
1385 size: desc.size,
1386 initialization_status: RwLock::new(
1387 rank::BUFFER_INITIALIZATION_STATUS,
1388 BufferInitTracker::new(tail_start),
1389 ),
1390 map_state: Arc::new(RwLock::new(
1391 rank::BUFFER_MAP_STATE,
1392 resource::BufferMapState::Idle,
1393 )),
1394 label: desc.label.to_string(),
1395 tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1396 bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1397 timestamp_normalization_bind_group,
1398 indirect_validation_bind_groups,
1399 };
1400
1401 let buffer = Arc::new(buffer);
1402
1403 let init_buffer_tail = |buffer, snatch_guard| {
1404 let mapping = map_buffer(
1405 buffer,
1406 tail_start,
1407 final_size - tail_start,
1408 MapMode::Write,
1409 snatch_guard,
1410 )?;
1411 let raw = buffer
1412 .raw(snatch_guard)
1413 .expect("newly-created buffer cannot be destroyed");
1414 unsafe {
1415 mapping
1417 .ptr
1418 .write_bytes(0, (final_size - tail_start) as usize);
1419 if !mapping.is_coherent {
1420 #[allow(clippy::single_range_in_vec_init)]
1421 self.raw()
1422 .flush_mapped_ranges(raw, &[tail_start..final_size]);
1423 }
1424 self.raw().unmap_buffer(raw);
1425 }
1426 Ok::<_, resource::CreateBufferError>(())
1427 };
1428
1429 let buffer_use = if !desc.mapped_at_creation {
1430 if tail_start == final_size {
1431 } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1433 let snatch_guard = self.snatchable_lock.read();
1435 init_buffer_tail(&buffer, &snatch_guard)?;
1436 } else if let Some(queue) = self.get_queue() {
1437 let snatch_guard = self.snatchable_lock.read();
1439 let mut pending_writes = queue.pending_writes.lock();
1440 self.trackers
1441 .lock()
1442 .buffers
1443 .insert_single(&buffer, wgt::BufferUses::COPY_DST);
1444 pending_writes.clear_buffer(
1445 self,
1446 &buffer,
1447 tail_start..final_size,
1448 &snatch_guard,
1449 )?;
1450 return Ok(buffer);
1451 } else {
1452 }
1454 wgt::BufferUses::empty()
1455 } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1456 let snatch_guard = self.snatchable_lock.read();
1460 if tail_start != final_size {
1461 init_buffer_tail(&buffer, &snatch_guard)?;
1462 }
1463 let map_size = buffer.size;
1464 let mapping = if map_size == 0 {
1465 hal::BufferMapping {
1466 ptr: core::ptr::NonNull::dangling(),
1467 is_coherent: true,
1468 }
1469 } else {
1470 map_buffer(&buffer, 0, map_size, MapMode::Write, &snatch_guard)?
1471 };
1472 drop(snatch_guard);
1473 *buffer.map_state.write() = resource::BufferMapState::Active {
1474 mapping,
1475 range: 0..map_size,
1476 host: MapMode::Write,
1477 };
1478 wgt::BufferUses::MAP_WRITE
1479 } else {
1480 let mut staging_buffer =
1487 StagingBuffer::new(self, wgt::BufferSize::new(final_size).unwrap())?;
1488
1489 staging_buffer.write_zeros();
1492 buffer.initialization_status.write().drain(0..final_size);
1493
1494 *buffer.map_state.write() = resource::BufferMapState::Init { staging_buffer };
1495 wgt::BufferUses::COPY_DST
1496 };
1497
1498 self.trackers
1502 .lock()
1503 .buffers
1504 .insert_single(&buffer, buffer_use);
1505
1506 Ok(buffer)
1507 }
1508
1509 pub fn create_buffer(self: &Arc<Self>, desc: &resource::BufferDescriptor) -> Arc<Buffer> {
1510 profiling::scope!("Device::create_buffer");
1511
1512 let buffer = self.create_buffer_inner(desc).unwrap_or_else(|err| {
1513 self.handle_error(err, desc.label.as_deref(), "Device::create_buffer");
1514 Buffer::invalid(Arc::clone(self), desc)
1515 });
1516 #[cfg(feature = "trace")]
1517 if let Some(ref mut trace) = *self.trace.lock() {
1518 use trace::IntoTrace;
1519 let mut desc = desc.clone();
1520 let mapped_at_creation = mem::replace(&mut desc.mapped_at_creation, false);
1521 if mapped_at_creation && !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1522 desc.usage |= wgt::BufferUsages::COPY_DST;
1523 }
1524 trace.add(trace::Action::CreateBuffer(buffer.to_trace(), desc));
1525 }
1526 api_log!(
1527 "Device::create_buffer({:?}{}) -> {:?}",
1528 desc.label.as_deref().unwrap_or(""),
1529 if desc.mapped_at_creation {
1530 ", mapped_at_creation"
1531 } else {
1532 ""
1533 },
1534 Arc::as_ptr(&buffer)
1535 );
1536 buffer
1537 }
1538
1539 #[cfg(feature = "replay")]
1540 pub fn set_buffer_data(
1541 self: &Arc<Self>,
1542 buffer: &Arc<Buffer>,
1543 offset: wgt::BufferAddress,
1544 data: &[u8],
1545 mode: MapMode,
1546 ) -> resource::BufferAccessResult {
1547 use crate::resource::RawResourceAccess;
1548
1549 let device = &buffer.device;
1550
1551 device.check_is_valid()?;
1552 buffer.check_usage(wgt::BufferUsages::MAP_WRITE)?;
1553
1554 let last_submission = device
1555 .get_queue()
1556 .and_then(|queue| queue.lock_life().get_buffer_latest_submission_index(buffer));
1557
1558 if let Some(last_submission) = last_submission {
1559 device.wait_for_submit(last_submission)?;
1560 }
1561
1562 let snatch_guard = device.snatchable_lock.read();
1563 let raw_buf = buffer.try_raw(&snatch_guard)?;
1564
1565 if offset > buffer.size {
1566 return Err(resource::BufferAccessError::OutOfBoundsStartOffsetOverrun {
1567 index: offset,
1568 max: buffer.size,
1569 });
1570 } else if buffer.size - offset < u64::try_from(data.len()).unwrap() {
1571 return Err(resource::BufferAccessError::OutOfBoundsEndOffsetOverrun {
1572 index: offset,
1573 size: u64::try_from(data.len()).unwrap(),
1574 max: buffer.size,
1575 });
1576 }
1577
1578 let mapping = unsafe {
1579 device
1580 .raw()
1581 .map_buffer(raw_buf, offset..offset + u64::try_from(data.len()).unwrap())
1582 }
1583 .map_err(|e| device.handle_hal_error(e))?;
1584
1585 unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()) };
1586
1587 if !mapping.is_coherent {
1588 #[allow(clippy::single_range_in_vec_init)]
1589 unsafe {
1590 device
1591 .raw()
1592 .flush_mapped_ranges(raw_buf, &[offset..offset + data.len() as u64])
1593 };
1594 }
1595
1596 unsafe { device.raw().unmap_buffer(raw_buf) };
1597
1598 Ok(mode)
1599 }
1600
1601 pub unsafe fn create_texture_from_hal(
1609 self: &Arc<Self>,
1610 hal_texture: Box<dyn hal::DynTexture>,
1611 desc: &resource::TextureDescriptor,
1612 initial_state: wgt::TextureUses,
1613 cleared: bool,
1614 ) -> (Arc<Texture>, Option<resource::CreateTextureError>) {
1615 profiling::scope!("Device::create_texture_from_hal");
1616
1617 let (texture, error) =
1618 match self.create_texture_from_hal_inner(hal_texture, desc, initial_state, cleared) {
1619 Ok(texture) => (texture, None),
1620 Err(e) => (Texture::invalid(self, desc), Some(e)),
1621 };
1622
1623 #[cfg(feature = "trace")]
1626 if let Some(ref mut trace) = *self.trace.lock() {
1627 trace.add(trace::Action::CreateTexture(
1628 texture.to_trace(),
1629 desc.clone(),
1630 ));
1631 }
1632
1633 api_log!(
1634 "Device::create_texture({desc:?}) -> {:?}",
1635 Arc::as_ptr(&texture)
1636 );
1637
1638 (texture, error)
1639 }
1640
1641 pub(crate) fn create_texture_from_hal_inner(
1642 self: &Arc<Self>,
1643 hal_texture: Box<dyn hal::DynTexture>,
1644 desc: &resource::TextureDescriptor,
1645 initial_state: wgt::TextureUses,
1646 cleared: bool,
1647 ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1648 unsafe { self.raw().add_raw_texture(&*hal_texture) };
1653
1654 if let Err(error) = self.check_is_valid() {
1655 unsafe { self.raw().destroy_texture(hal_texture) };
1656 return Err(error.into());
1657 }
1658
1659 let format_features = match self.describe_format_features(desc.format) {
1660 Ok(format_features) => format_features,
1661 Err(error) => {
1662 unsafe { self.raw().destroy_texture(hal_texture) };
1663 return Err(resource::CreateTextureError::MissingFeatures(
1664 desc.format,
1665 error,
1666 ));
1667 }
1668 };
1669
1670 let texture = Texture::new(
1671 self,
1672 resource::TextureInner::Native { raw: hal_texture },
1673 conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1674 desc,
1675 format_features,
1676 resource::TextureClearMode::None,
1677 !cleared, self.resolve_texture_binding_view_dimension(desc),
1679 );
1680
1681 let texture = Arc::new(texture);
1682
1683 self.trackers
1684 .lock()
1685 .textures
1686 .insert_single(&texture, initial_state);
1687
1688 Ok(texture)
1689 }
1690
1691 pub unsafe fn create_buffer_from_hal(
1699 self: &Arc<Self>,
1700 hal_buffer: Box<dyn hal::DynBuffer>,
1701 desc: &resource::BufferDescriptor,
1702 ) -> (Arc<Buffer>, Option<resource::CreateBufferError>) {
1703 profiling::scope!("Device::create_buffer");
1704 let (buffer, error) = match unsafe { self.create_buffer_from_hal_inner(hal_buffer, desc) } {
1705 Ok(buffer) => (buffer, None),
1706 Err(e) => (Buffer::invalid(Arc::clone(self), desc), Some(e)),
1707 };
1708
1709 #[cfg(feature = "trace")]
1712 if let Some(trace) = self.trace.lock().as_mut() {
1713 use trace::IntoTrace;
1714 trace.add(trace::Action::CreateBuffer(buffer.to_trace(), desc.clone()));
1715 }
1716 api_log!("Device::create_buffer -> {:?}", Arc::as_ptr(&buffer));
1717 (buffer, error)
1718 }
1719
1720 pub(crate) unsafe fn create_buffer_from_hal_inner(
1724 self: &Arc<Self>,
1725 hal_buffer: Box<dyn hal::DynBuffer>,
1726 desc: &resource::BufferDescriptor,
1727 ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1728 let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1729 self.timestamp_normalizer
1732 .get()
1733 .unwrap()
1734 .create_normalization_bind_group(
1735 self,
1736 &*hal_buffer,
1737 desc.label.as_deref(),
1738 wgt::BufferSize::new(desc.size).unwrap(),
1739 desc.usage,
1740 )?
1741 });
1742
1743 let indirect_validation_bind_groups = self.create_indirect_validation_bind_groups(
1744 hal_buffer.as_ref(),
1745 desc.size,
1746 desc.usage,
1747 )?;
1748
1749 unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1750
1751 let buffer = Buffer {
1752 state: ResourceState::Valid(BufferState {
1753 raw: Snatchable::new(hal_buffer),
1754 }),
1755 device: self.clone(),
1756 usage: desc.usage,
1757 size: desc.size,
1758 initialization_status: RwLock::new(
1759 rank::BUFFER_INITIALIZATION_STATUS,
1760 BufferInitTracker::new(0),
1761 ),
1762 map_state: Arc::new(RwLock::new(
1763 rank::BUFFER_MAP_STATE,
1764 resource::BufferMapState::Idle,
1765 )),
1766 label: desc.label.to_string(),
1767 tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1768 bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1769 timestamp_normalization_bind_group,
1770 indirect_validation_bind_groups,
1771 };
1772
1773 let buffer = Arc::new(buffer);
1774
1775 self.trackers
1776 .lock()
1777 .buffers
1778 .insert_single(&buffer, wgt::BufferUses::empty());
1779
1780 Ok(buffer)
1781 }
1782
1783 fn create_indirect_validation_bind_groups(
1788 &self,
1789 raw_buffer: &dyn hal::DynBuffer,
1790 buffer_size: u64,
1791 usage: wgt::BufferUsages,
1792 ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1793 {
1794 if !usage.contains(wgt::BufferUsages::INDIRECT) {
1795 return Ok(Snatchable::empty());
1796 }
1797
1798 let Some(ref indirect_validation) = self.indirect_validation else {
1799 return Ok(Snatchable::empty());
1800 };
1801
1802 let bind_groups = crate::indirect_validation::BindGroups::new(
1803 indirect_validation,
1804 self,
1805 buffer_size,
1806 raw_buffer,
1807 )
1808 .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1809
1810 if let Some(bind_groups) = bind_groups {
1811 Ok(Snatchable::new(bind_groups))
1812 } else {
1813 Ok(Snatchable::empty())
1814 }
1815 }
1816
1817 pub fn validate_texture_descriptor(
1822 self: &Arc<Self>,
1823 desc: &resource::TextureDescriptor,
1824 ) -> Result<(), resource::CreateTextureError> {
1825 self.validate_texture_descriptor_inner(desc)?;
1826 Ok(())
1827 }
1828
1829 fn validate_texture_descriptor_inner(
1836 self: &Arc<Self>,
1837 desc: &resource::TextureDescriptor,
1838 ) -> Result<(wgt::TextureFormatFeatures, Vec<TextureFormat>), resource::CreateTextureError>
1839 {
1840 use resource::{
1841 CreateTextureError, TextureBindingViewDimensionError, TextureDimensionError,
1842 };
1843
1844 self.check_is_valid()?;
1845
1846 if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1847 return Err(CreateTextureError::InvalidUsage(desc.usage));
1848 }
1849
1850 conv::check_texture_dimension_size(
1851 desc.dimension,
1852 desc.size,
1853 desc.sample_count,
1854 &self.limits,
1855 )?;
1856
1857 if desc.dimension != wgt::TextureDimension::D2 {
1858 if desc.format.is_depth_stencil_format() {
1860 return Err(CreateTextureError::InvalidDepthDimension(
1861 desc.dimension,
1862 desc.format,
1863 ));
1864 }
1865 if desc
1867 .usage
1868 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1869 {
1870 return Err(CreateTextureError::InvalidDimensionUsages(
1871 wgt::TextureUsages::TRANSIENT_ATTACHMENT,
1872 desc.dimension,
1873 ));
1874 }
1875 }
1876
1877 if desc.dimension != wgt::TextureDimension::D2
1878 && desc.dimension != wgt::TextureDimension::D3
1879 {
1880 if desc.format.is_compressed() {
1882 return Err(CreateTextureError::InvalidCompressedDimension(
1883 desc.dimension,
1884 desc.format,
1885 ));
1886 }
1887
1888 if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1890 return Err(CreateTextureError::InvalidDimensionUsages(
1891 wgt::TextureUsages::RENDER_ATTACHMENT,
1892 desc.dimension,
1893 ));
1894 }
1895 }
1896
1897 if desc.format.is_compressed() {
1898 let (block_width, block_height) = desc.format.block_dimensions();
1899
1900 if !desc.size.width.is_multiple_of(block_width) {
1901 return Err(CreateTextureError::InvalidDimension(
1902 TextureDimensionError::NotMultipleOfBlockWidth {
1903 width: desc.size.width,
1904 block_width,
1905 format: desc.format,
1906 },
1907 ));
1908 }
1909
1910 if !desc.size.height.is_multiple_of(block_height) {
1911 return Err(CreateTextureError::InvalidDimension(
1912 TextureDimensionError::NotMultipleOfBlockHeight {
1913 height: desc.size.height,
1914 block_height,
1915 format: desc.format,
1916 },
1917 ));
1918 }
1919
1920 if desc.dimension == wgt::TextureDimension::D3 {
1921 if desc.format.is_bcn() {
1923 self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1924 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1925 } else if desc.format.is_astc() {
1926 self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1927 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1928 } else {
1929 return Err(CreateTextureError::InvalidCompressedDimension(
1930 desc.dimension,
1931 desc.format,
1932 ));
1933 }
1934 }
1935 }
1936
1937 let mips = desc.mip_level_count;
1938 let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1939 if mips == 0 || mips > max_levels_allowed {
1940 return Err(CreateTextureError::InvalidMipLevelCount {
1941 requested: mips,
1942 maximum: max_levels_allowed,
1943 });
1944 }
1945
1946 {
1947 let (mut width_multiple, mut height_multiple) = desc.format.size_multiple_requirement();
1948
1949 if desc.format.is_multi_planar_format() {
1950 width_multiple <<= desc.mip_level_count.saturating_sub(1);
1954 height_multiple <<= desc.mip_level_count.saturating_sub(1);
1955 }
1956
1957 if !desc.size.width.is_multiple_of(width_multiple) {
1958 return Err(CreateTextureError::InvalidDimension(
1959 TextureDimensionError::WidthNotMultipleOf {
1960 width: desc.size.width,
1961 multiple: width_multiple,
1962 format: desc.format,
1963 },
1964 ));
1965 }
1966
1967 if !desc.size.height.is_multiple_of(height_multiple) {
1968 return Err(CreateTextureError::InvalidDimension(
1969 TextureDimensionError::HeightNotMultipleOf {
1970 height: desc.size.height,
1971 multiple: height_multiple,
1972 format: desc.format,
1973 },
1974 ));
1975 }
1976 }
1977
1978 if desc
1979 .usage
1980 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1981 {
1982 if desc.usage
1983 != (wgt::TextureUsages::TRANSIENT_ATTACHMENT
1984 | wgt::TextureUsages::RENDER_ATTACHMENT)
1985 {
1986 return Err(CreateTextureError::InvalidTransientTextureUsage(desc.usage));
1987 }
1988
1989 if desc.mip_level_count != 1 {
1990 return Err(CreateTextureError::InvalidTransientTextureMipLevelCount(
1991 desc.mip_level_count,
1992 ));
1993 }
1994
1995 if desc.size.depth_or_array_layers != 1 {
1996 return Err(CreateTextureError::InvalidTransientTextureLayerCount(
1997 desc.size.depth_or_array_layers,
1998 ));
1999 }
2000
2001 if !desc.view_formats.is_empty() {
2002 return Err(CreateTextureError::InvalidTransientTextureViewFormats);
2003 }
2004 }
2005
2006 let format_features = self
2007 .describe_format_features(desc.format)
2008 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
2009
2010 if let Some(planes) = desc.format.planes() {
2011 for plane in 0..planes {
2012 let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
2013 let format = desc.format.aspect_specific_format(aspect).unwrap();
2014 self.require_features(format.required_features())
2015 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
2016 }
2017 }
2018
2019 if desc.sample_count > 1 {
2020 if desc.mip_level_count != 1 {
2026 return Err(CreateTextureError::InvalidMipLevelCount {
2027 requested: desc.mip_level_count,
2028 maximum: 1,
2029 });
2030 }
2031
2032 if desc.size.depth_or_array_layers != 1
2033 && !self.features.contains(wgt::Features::MULTISAMPLE_ARRAY)
2034 {
2035 return Err(CreateTextureError::InvalidDimension(
2036 TextureDimensionError::MultisampledDepthOrArrayLayer(
2037 desc.size.depth_or_array_layers,
2038 ),
2039 ));
2040 }
2041
2042 if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
2043 return Err(CreateTextureError::InvalidMultisampledStorageBinding);
2044 }
2045
2046 if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
2047 return Err(CreateTextureError::MultisampledNotRenderAttachment);
2048 }
2049
2050 if !format_features.flags.intersects(
2051 wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
2052 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
2053 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
2054 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
2055 ) {
2056 return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
2057 }
2058
2059 if !format_features
2060 .flags
2061 .sample_count_supported(desc.sample_count)
2062 {
2063 return Err(CreateTextureError::InvalidSampleCount(
2064 desc.sample_count,
2065 desc.format,
2066 desc.format
2067 .guaranteed_format_features(self.features)
2068 .flags
2069 .supported_sample_counts(),
2070 self.adapter
2071 .get_texture_format_features(desc.format)
2072 .flags
2073 .supported_sample_counts(),
2074 ));
2075 };
2076 }
2077
2078 let missing_allowed_usages = desc.usage - format_features.allowed_usages;
2079 if !missing_allowed_usages.is_empty() {
2080 let wgpu_allowed_usages = desc
2082 .format
2083 .guaranteed_format_features(self.features)
2084 .allowed_usages;
2085 let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
2086 return Err(CreateTextureError::InvalidFormatUsages(
2087 missing_allowed_usages,
2088 desc.format,
2089 wgpu_missing_usages.is_empty(),
2090 ));
2091 }
2092
2093 let mut hal_view_formats = Vec::new();
2094 for format in desc.view_formats.iter() {
2095 self.require_features(format.required_features())
2096 .map_err(|error| CreateTextureError::MissingFeatures(*format, error))?;
2097 if desc.format == *format {
2098 continue;
2099 }
2100 if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
2101 return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
2102 }
2103 hal_view_formats.push(*format);
2104 }
2105 if !hal_view_formats.is_empty() {
2106 self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
2107 }
2108
2109 if !self
2110 .downlevel
2111 .flags
2112 .contains(wgt::DownlevelFlags::ARBITRARY_BINDING_VIEW_DIMENSIONS)
2113 {
2114 if let Some(texture_binding_view_dimension) = desc.texture_binding_view_dimension {
2115 match texture_binding_view_dimension {
2116 TextureViewDimension::D2 => {
2117 if desc.size.depth_or_array_layers != 1 {
2118 return Err(CreateTextureError::InvalidTextureBindingViewDimension(
2119 TextureBindingViewDimensionError::InvalidLayerCount {
2120 dimension: texture_binding_view_dimension,
2121 required: 1,
2122 actual: desc.size.depth_or_array_layers,
2123 },
2124 ));
2125 }
2126 }
2127 TextureViewDimension::Cube => {
2128 if desc.size.depth_or_array_layers != 6 {
2129 return Err(CreateTextureError::InvalidTextureBindingViewDimension(
2130 TextureBindingViewDimensionError::InvalidLayerCount {
2131 dimension: texture_binding_view_dimension,
2132 required: 6,
2133 actual: desc.size.depth_or_array_layers,
2134 },
2135 ));
2136 }
2137 }
2138 TextureViewDimension::CubeArray => {
2139 self.require_downlevel_flags(wgt::DownlevelFlags::CUBE_ARRAY_TEXTURES)?;
2140 if !desc.size.depth_or_array_layers.is_multiple_of(6) {
2141 return Err(CreateTextureError::InvalidTextureBindingViewDimension(
2142 TextureBindingViewDimensionError::InvalidCubeArrayLayerCount(
2143 desc.size.depth_or_array_layers,
2144 ),
2145 ));
2146 }
2147 }
2148 _ => {}
2149 }
2150 }
2151 }
2152
2153 Ok((format_features, hal_view_formats))
2154 }
2155
2156 fn resolve_texture_binding_view_dimension(
2157 &self,
2158 desc: &resource::TextureDescriptor,
2159 ) -> Option<TextureViewDimension> {
2160 if self
2161 .downlevel
2162 .flags
2163 .contains(wgt::DownlevelFlags::ARBITRARY_BINDING_VIEW_DIMENSIONS)
2164 {
2165 None
2166 } else {
2167 Some(
2168 desc.texture_binding_view_dimension
2169 .unwrap_or_else(|| desc.default_view_dimension()),
2170 )
2171 }
2172 }
2173
2174 fn create_texture_inner(
2175 self: &Arc<Self>,
2176 desc: &resource::TextureDescriptor,
2177 ) -> Result<Arc<Texture>, resource::CreateTextureError> {
2178 let (format_features, hal_view_formats) = self.validate_texture_descriptor_inner(desc)?;
2179
2180 let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
2181
2182 let texture_binding_view_dimension = self.resolve_texture_binding_view_dimension(desc);
2183
2184 let hal_desc = hal::TextureDescriptor {
2185 label: desc.label.to_hal(self.instance_flags),
2186 size: desc.size,
2187 mip_level_count: desc.mip_level_count,
2188 sample_count: desc.sample_count,
2189 dimension: desc.dimension,
2190 format: desc.format,
2191 usage: hal_usage,
2192 memory_flags: hal::MemoryFlags::empty(),
2193 view_formats: hal_view_formats,
2194 texture_binding_view_dimension,
2195 };
2196
2197 let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
2198 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2199
2200 let clear_mode = if hal_usage
2201 .contains(wgt::TextureUses::DEPTH_WRITE | wgt::TextureUses::STENCIL_WRITE)
2202 || hal_usage.contains(wgt::TextureUses::COLOR_TARGET)
2203 && desc.dimension == wgt::TextureDimension::D2
2204 {
2205 let (is_color, usage) = if desc.format.is_depth_stencil_format() {
2206 (
2207 false,
2208 wgt::TextureUses::DEPTH_WRITE | wgt::TextureUses::STENCIL_WRITE,
2209 )
2210 } else {
2211 (true, wgt::TextureUses::COLOR_TARGET)
2212 };
2213
2214 let clear_label = hal_label(
2215 Some("(wgpu internal) clear texture view"),
2216 self.instance_flags,
2217 );
2218
2219 let clear_views = resource::ClearViews::new(
2220 clear_label,
2221 usage,
2222 desc.format,
2223 desc.mip_level_count,
2224 desc.size.depth_or_array_layers,
2225 |desc| {
2226 unsafe { self.raw().create_texture_view(raw_texture.as_ref(), desc) }
2227 .map_err(|e| self.handle_hal_error(e))
2228 },
2229 )?;
2230 resource::TextureClearMode::RenderPass {
2231 clear_views,
2232 is_color,
2233 }
2234 } else {
2235 resource::TextureClearMode::BufferCopy
2236 };
2237
2238 let texture = Texture::new(
2239 self,
2240 resource::TextureInner::Native { raw: raw_texture },
2241 hal_usage,
2242 desc,
2243 format_features,
2244 clear_mode,
2245 true,
2246 texture_binding_view_dimension,
2247 );
2248
2249 let texture = Arc::new(texture);
2250
2251 self.trackers
2252 .lock()
2253 .textures
2254 .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
2255
2256 Ok(texture)
2257 }
2258
2259 pub fn create_texture(self: &Arc<Self>, desc: &resource::TextureDescriptor) -> Arc<Texture> {
2261 profiling::scope!("Device::create_texture");
2262
2263 let texture = self.create_texture_inner(desc).unwrap_or_else(|err| {
2264 self.handle_error(err, desc.label.as_deref(), "Device::create_texture");
2265 Texture::invalid(self, desc)
2266 });
2267 api_log!(
2268 "Device::create_texture({desc:?}) -> {:?}",
2269 Arc::as_ptr(&texture)
2270 );
2271
2272 #[cfg(feature = "trace")]
2273 if let Some(ref mut trace) = *self.trace.lock() {
2274 use crate::device::trace::IntoTrace as _;
2275
2276 trace.add(trace::Action::CreateTexture(
2277 texture.to_trace(),
2278 desc.clone(),
2279 ));
2280 }
2281 texture
2282 }
2283
2284 pub fn create_texture_error(
2286 self: &Arc<Self>,
2287 desc: &resource::TextureDescriptor,
2288 ) -> Arc<Texture> {
2289 let texture = Texture::invalid(self, desc);
2290 #[cfg(feature = "trace")]
2291 if let Some(ref mut trace) = *self.trace.lock() {
2292 use crate::device::trace::IntoTrace as _;
2293
2294 trace.add(trace::Action::CreateTextureError(
2295 texture.to_trace(),
2296 desc.clone(),
2297 ));
2298 }
2299 texture
2300 }
2301
2302 pub fn create_external_texture(
2303 self: &Arc<Self>,
2304 desc: &resource::ExternalTextureDescriptor,
2305 planes: &[Arc<TextureView>],
2306 ) -> Arc<ExternalTexture> {
2307 profiling::scope!("Device::create_external_texture");
2308
2309 let external_texture = self
2310 .create_external_texture_inner(desc, planes)
2311 .unwrap_or_else(|err| {
2312 self.handle_error(
2313 err,
2314 desc.label.as_deref(),
2315 "Device::create_external_texture",
2316 );
2317 ExternalTexture::invalid(Arc::clone(self), desc)
2318 });
2319
2320 #[cfg(feature = "trace")]
2321 if let Some(ref mut trace) = *self.trace.lock() {
2322 use crate::device::trace;
2323 use trace::IntoTrace as _;
2324
2325 let planes = Box::from(
2326 planes
2327 .iter()
2328 .map(|plane| plane.to_trace())
2329 .collect::<Vec<_>>(),
2330 );
2331 trace.add(trace::Action::CreateExternalTexture {
2332 id: external_texture.to_trace(),
2333 desc: desc.clone(),
2334 planes,
2335 });
2336 }
2337
2338 api_log!(
2339 "Device::create_external_texture({desc:?}) -> {:?}",
2340 Arc::as_ptr(&external_texture)
2341 );
2342
2343 external_texture
2344 }
2345
2346 pub(crate) fn create_external_texture_inner(
2347 self: &Arc<Self>,
2348 desc: &resource::ExternalTextureDescriptor,
2349 planes: &[Arc<TextureView>],
2350 ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
2351 use resource::CreateExternalTextureError;
2352 self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
2353 self.check_is_valid()?;
2354
2355 if desc.num_planes() != planes.len() {
2356 return Err(CreateExternalTextureError::IncorrectPlaneCount {
2357 format: desc.format,
2358 expected: desc.num_planes(),
2359 provided: planes.len(),
2360 });
2361 }
2362
2363 let planes = planes
2364 .iter()
2365 .enumerate()
2366 .map(|(i, plane)| {
2367 plane.same_device(self)?;
2368
2369 if plane.samples != 1 {
2370 return Err(CreateExternalTextureError::InvalidPlaneMultisample(
2371 plane.samples,
2372 ));
2373 }
2374
2375 let sample_type = plane
2376 .desc
2377 .format
2378 .sample_type(Some(plane.desc.range.aspect), Some(self.features))
2379 .unwrap();
2380 if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
2381 return Err(CreateExternalTextureError::InvalidPlaneSampleType {
2382 format: plane.desc.format,
2383 sample_type,
2384 });
2385 }
2386
2387 if plane.desc.dimension != TextureViewDimension::D2 {
2388 return Err(CreateExternalTextureError::InvalidPlaneDimension(
2389 plane.desc.dimension,
2390 ));
2391 }
2392
2393 let expected_components = match desc.format {
2394 wgt::ExternalTextureFormat::Rgba => 4,
2395 wgt::ExternalTextureFormat::Nv12 => match i {
2396 0 => 1,
2397 1 => 2,
2398 _ => unreachable!(),
2399 },
2400 wgt::ExternalTextureFormat::Yu12 => 1,
2401 };
2402 if plane.desc.format.components() != expected_components {
2403 return Err(CreateExternalTextureError::InvalidPlaneFormat {
2404 format: desc.format,
2405 plane: i,
2406 expected: expected_components,
2407 provided: plane.desc.format,
2408 });
2409 }
2410
2411 plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
2412 Ok(plane.clone())
2413 })
2414 .collect::<Result<_, _>>()?;
2415
2416 let params_data = ExternalTextureParams::from_desc(desc);
2417 let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
2418 let params_desc = resource::BufferDescriptor {
2419 label: label.map(Cow::Owned),
2420 size: size_of_val(¶ms_data) as wgt::BufferAddress,
2421 usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
2422 mapped_at_creation: false,
2423 };
2424 let params = self.create_buffer_inner(¶ms_desc)?;
2425 self.get_queue().unwrap().write_buffer_inner(
2426 params.clone(),
2427 0,
2428 bytemuck::bytes_of(¶ms_data),
2429 )?;
2430
2431 let external_texture = ExternalTexture {
2432 state: ResourceState::Valid(ExternalTextureState { params }),
2433 device: self.clone(),
2434 planes,
2435 label: desc.label.to_string(),
2436 tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
2437 };
2438 let external_texture = Arc::new(external_texture);
2439
2440 Ok(external_texture)
2441 }
2442
2443 pub fn create_sampler(self: &Arc<Self>, desc: &resource::SamplerDescriptor) -> Arc<Sampler> {
2444 profiling::scope!("Device::create_sampler");
2445
2446 let sampler = self.create_sampler_inner(desc).unwrap_or_else(|err| {
2447 self.handle_error(err, desc.label.as_deref(), "Device::create_sampler");
2448 Sampler::invalid(Arc::clone(self), desc)
2449 });
2450
2451 #[cfg(feature = "trace")]
2452 if let Some(ref mut trace) = *self.trace.lock() {
2453 use crate::device::trace::{Action, IntoTrace as _};
2454 trace.add(Action::CreateSampler(sampler.to_trace(), desc.clone()));
2455 }
2456
2457 api_log!("Device::create_sampler -> {:?}", Arc::as_ptr(&sampler));
2458
2459 sampler
2460 }
2461
2462 pub(crate) fn create_sampler_inner(
2463 self: &Arc<Self>,
2464 desc: &resource::SamplerDescriptor,
2465 ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
2466 self.check_is_valid()?;
2467
2468 if desc
2469 .address_modes
2470 .iter()
2471 .any(|am| am == &wgt::AddressMode::ClampToBorder)
2472 {
2473 self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
2474 }
2475
2476 if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
2477 self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
2478 }
2479
2480 if desc.lod_min_clamp < 0.0 {
2481 return Err(resource::CreateSamplerError::InvalidLodMinClamp(
2482 desc.lod_min_clamp,
2483 ));
2484 }
2485 if desc.lod_max_clamp < desc.lod_min_clamp {
2486 return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
2487 lod_min_clamp: desc.lod_min_clamp,
2488 lod_max_clamp: desc.lod_max_clamp,
2489 });
2490 }
2491
2492 if desc.anisotropy_clamp < 1 {
2493 return Err(resource::CreateSamplerError::InvalidAnisotropy(
2494 desc.anisotropy_clamp,
2495 ));
2496 }
2497
2498 if desc.anisotropy_clamp != 1 {
2499 if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
2500 return Err(
2501 resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2502 filter_type: resource::SamplerFilterErrorType::MinFilter,
2503 filter_mode: desc.min_filter,
2504 anisotropic_clamp: desc.anisotropy_clamp,
2505 },
2506 );
2507 }
2508 if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
2509 return Err(
2510 resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2511 filter_type: resource::SamplerFilterErrorType::MagFilter,
2512 filter_mode: desc.mag_filter,
2513 anisotropic_clamp: desc.anisotropy_clamp,
2514 },
2515 );
2516 }
2517 if !matches!(desc.mipmap_filter, wgt::MipmapFilterMode::Linear) {
2518 return Err(
2519 resource::CreateSamplerError::InvalidMipmapFilterModeWithAnisotropy {
2520 filter_type: resource::SamplerFilterErrorType::MipmapFilter,
2521 filter_mode: desc.mipmap_filter,
2522 anisotropic_clamp: desc.anisotropy_clamp,
2523 },
2524 );
2525 }
2526 }
2527
2528 let anisotropy_clamp = if self
2529 .downlevel
2530 .flags
2531 .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
2532 {
2533 desc.anisotropy_clamp.min(16)
2535 } else {
2536 1
2538 };
2539
2540 let hal_desc = hal::SamplerDescriptor {
2543 label: desc.label.to_hal(self.instance_flags),
2544 address_modes: desc.address_modes,
2545 mag_filter: desc.mag_filter,
2546 min_filter: desc.min_filter,
2547 mipmap_filter: desc.mipmap_filter,
2548 lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
2549 compare: desc.compare,
2550 anisotropy_clamp,
2551 border_color: desc.border_color,
2552 };
2553
2554 let raw = unsafe { self.raw().create_sampler(&hal_desc) }
2555 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2556
2557 let sampler = Sampler {
2558 raw: ResourceState::Valid(raw),
2559 device: self.clone(),
2560 label: desc.label.to_string(),
2561 tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
2562 comparison: desc.compare.is_some(),
2563 filtering: desc.min_filter == wgt::FilterMode::Linear
2564 || desc.mag_filter == wgt::FilterMode::Linear
2565 || desc.mipmap_filter == wgt::MipmapFilterMode::Linear,
2566 };
2567
2568 let sampler = Arc::new(sampler);
2569
2570 Ok(sampler)
2571 }
2572
2573 pub fn create_shader_module<'a>(
2574 self: &Arc<Self>,
2575 desc: &pipeline::ShaderModuleDescriptor<'a>,
2576 source: pipeline::ShaderModuleSource<'a>,
2577 ) -> Arc<pipeline::ShaderModule> {
2578 profiling::scope!("Device::create_shader_module");
2579 #[cfg(feature = "trace")]
2580 let data = self.trace.lock().as_mut().map(|trace| {
2581 use crate::device::trace::DataKind;
2582
2583 match source {
2584 #[cfg(feature = "wgsl")]
2585 pipeline::ShaderModuleSource::Wgsl(ref code) => {
2586 trace.make_binary(DataKind::Wgsl, code.as_bytes())
2587 }
2588 #[cfg(feature = "glsl")]
2589 pipeline::ShaderModuleSource::Glsl(ref code, _) => {
2590 trace.make_binary(DataKind::Glsl, code.as_bytes())
2591 }
2592 #[cfg(feature = "spirv")]
2593 pipeline::ShaderModuleSource::SpirV(ref code, _) => {
2594 trace.make_binary(DataKind::Spv, bytemuck::cast_slice::<u32, u8>(code))
2595 }
2596 pipeline::ShaderModuleSource::Naga(ref module) => {
2597 let string =
2598 ron::ser::to_string_pretty(module, ron::ser::PrettyConfig::default())
2599 .unwrap();
2600 trace.make_binary(DataKind::Ron, string.as_bytes())
2601 }
2602 pipeline::ShaderModuleSource::Dummy(_) => {
2603 panic!("found `ShaderModuleSource::Dummy`")
2604 }
2605 }
2606 });
2607 let shader = self
2608 .create_shader_module_inner(desc, source)
2609 .unwrap_or_else(|e| {
2610 let shader = pipeline::ShaderModule::invalid(
2611 Arc::clone(self),
2612 desc.label.to_string(),
2613 shader_module_error_into_compilation_info(&e),
2614 );
2615 self.handle_error(e, desc.label.as_deref(), "Device::create_shader_module");
2616 shader
2617 });
2618 api_log!("Device::create_shader_module -> {:?}", Arc::as_ptr(&shader));
2619
2620 #[cfg(feature = "trace")]
2621 if let Some(data) = data {
2622 use crate::device::trace::IntoTrace as _;
2625 self.trace
2626 .lock()
2627 .as_mut()
2628 .expect("trace went away during create_shader_module?")
2629 .add(trace::Action::CreateShaderModule {
2630 id: shader.to_trace(),
2631 desc: desc.clone(),
2632 data,
2633 });
2634 };
2635 shader
2636 }
2637
2638 pub(crate) fn create_shader_module_inner<'a>(
2639 self: &Arc<Self>,
2640 desc: &pipeline::ShaderModuleDescriptor<'a>,
2641 source: pipeline::ShaderModuleSource<'a>,
2642 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2643 self.check_is_valid()?;
2644
2645 let (module, source) = match source {
2646 #[cfg(feature = "wgsl")]
2647 pipeline::ShaderModuleSource::Wgsl(code) => {
2648 profiling::scope!("naga::front::wgsl::parse");
2649 let capabilities = crate::device::features_to_naga_capabilities(
2650 self.features,
2651 self.downlevel.flags,
2652 );
2653 let mut options = naga::front::wgsl::Options::new();
2654 options.capabilities = capabilities;
2655 let mut frontend = naga::front::wgsl::Frontend::new_with_options(options);
2656 let module = frontend.parse(&code).map_err(|inner| {
2657 pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2658 source: code.to_string(),
2659 label: desc.label.as_ref().map(|l| l.to_string()),
2660 inner: Box::new(inner),
2661 })
2662 })?;
2663 (Cow::Owned(module), code.into_owned())
2664 }
2665 #[cfg(feature = "spirv")]
2666 pipeline::ShaderModuleSource::SpirV(spv, options) => {
2667 let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2668 profiling::scope!("naga::front::spv::Frontend");
2669 let module = parser.parse().map_err(|inner| {
2670 pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2671 source: String::new(),
2672 label: desc.label.as_ref().map(|l| l.to_string()),
2673 inner: Box::new(inner),
2674 })
2675 })?;
2676 (Cow::Owned(module), String::new())
2677 }
2678 #[cfg(feature = "glsl")]
2679 pipeline::ShaderModuleSource::Glsl(code, options) => {
2680 let mut parser = naga::front::glsl::Frontend::default();
2681 profiling::scope!("naga::front::glsl::Frontend.parse");
2682 let module = parser.parse(&options, &code).map_err(|inner| {
2683 pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2684 source: code.to_string(),
2685 label: desc.label.as_ref().map(|l| l.to_string()),
2686 inner: Box::new(inner),
2687 })
2688 })?;
2689 (Cow::Owned(module), code.into_owned())
2690 }
2691 pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2692 pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2693 };
2694 for (_, var) in module.global_variables.iter() {
2695 match var.binding {
2696 Some(br) if br.group >= self.limits.max_bind_groups => {
2697 return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2698 bind: br,
2699 group: br.group,
2700 limit: self.limits.max_bind_groups,
2701 });
2702 }
2703 _ => continue,
2704 };
2705 }
2706
2707 profiling::scope!("naga::validate");
2708 let debug_source =
2709 if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2710 Some(hal::DebugSource {
2711 file_name: Cow::Owned(
2712 desc.label
2713 .as_ref()
2714 .map_or("shader".to_string(), |l| l.to_string()),
2715 ),
2716 source_code: Cow::Owned(source.clone()),
2717 })
2718 } else {
2719 None
2720 };
2721
2722 let info = create_validator(
2723 self.features,
2724 self.downlevel.flags,
2725 naga::valid::ValidationFlags::all(),
2726 )
2727 .validate(&module)
2728 .map_err(|inner| {
2729 pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2730 source,
2731 label: desc.label.as_ref().map(|l| l.to_string()),
2732 inner,
2733 })
2734 })?;
2735
2736 let module = pipeline::ShaderModule {
2740 state: ResourceState::Valid(pipeline::ShaderModuleState::NagaModule {
2741 shader: hal::NagaShader {
2742 module,
2743 info,
2744 debug_source,
2745 },
2746 runtime_checks: desc.runtime_checks,
2747 }),
2748 device: self.clone(),
2749 label: desc.label.to_string(),
2750 compilation_info: wgt::CompilationInfo::default(),
2751 };
2752
2753 let module = Arc::new(module);
2754
2755 Ok(module)
2756 }
2757
2758 pub unsafe fn create_shader_module_passthrough<'a>(
2763 self: &Arc<Self>,
2764 desc: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2765 ) -> Arc<pipeline::ShaderModule> {
2766 profiling::scope!("Device::create_shader_module_passthrough");
2767
2768 let shader =
2769 unsafe { self.create_shader_module_passthrough_inner(desc) }.unwrap_or_else(|e| {
2770 let shader = pipeline::ShaderModule::invalid(
2771 Arc::clone(self),
2772 desc.label.to_string(),
2773 shader_module_error_into_compilation_info(&e),
2774 );
2775 self.handle_error(
2776 e,
2777 desc.label.as_deref(),
2778 "Device::create_shader_module_passthrough",
2779 );
2780 shader
2781 });
2782
2783 #[cfg(feature = "trace")]
2784 if let Some(ref mut trace) = *self.trace.lock() {
2785 use crate::device::trace::{DataKind, IntoTrace as _};
2786
2787 let mut file_names = Vec::new();
2788 for (data, kind) in [
2789 (
2790 desc.spirv.as_ref().map(|a| bytemuck::cast_slice(a)),
2791 DataKind::Spv,
2792 ),
2793 (desc.dxil.as_deref(), DataKind::Dxil),
2794 (desc.hlsl.as_ref().map(|a| a.as_bytes()), DataKind::Hlsl),
2795 (desc.metallib.as_deref(), DataKind::MetalLib),
2796 (desc.msl.as_ref().map(|a| a.as_bytes()), DataKind::Msl),
2797 (desc.glsl.as_ref().map(|a| a.as_bytes()), DataKind::Glsl),
2798 (desc.wgsl.as_ref().map(|a| a.as_bytes()), DataKind::Wgsl),
2799 ] {
2800 if let Some(data) = data {
2801 file_names.push(trace.make_binary(kind, data));
2802 }
2803 }
2804 trace.add(trace::Action::CreateShaderModulePassthrough {
2805 id: shader.to_trace(),
2806 data: file_names,
2807 label: desc.label.clone(),
2808 entry_points: desc.entry_points.clone(),
2809 });
2810 };
2811 api_log!(
2812 "Device::create_shader_module_spirv -> {:?}",
2813 Arc::as_ptr(&shader)
2814 );
2815 shader
2816 }
2817
2818 pub(crate) unsafe fn create_shader_module_passthrough_inner<'a>(
2819 self: &Arc<Self>,
2820 descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2821 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2822 self.check_is_valid()?;
2823 self.require_features(wgt::Features::PASSTHROUGH_SHADERS)?;
2824
2825 if (descriptor.dxil.is_some() || descriptor.glsl.is_some())
2827 && descriptor.entry_points.len() != 1
2828 {
2829 return Err(pipeline::CreateShaderModuleError::IncorrectPassthroughEntryPointCount);
2830 }
2831
2832 let entry_point_hashmap = || {
2833 descriptor
2834 .entry_points
2835 .iter()
2836 .map(|e| (e.name.to_string(), e.workgroup_size))
2837 .collect()
2838 };
2839
2840 let hal_shader = match self.backend() {
2841 wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2842 descriptor
2843 .spirv
2844 .as_ref()
2845 .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2846 ),
2847 wgt::Backend::Dx12 => {
2848 if let Some(dxil) = &descriptor.dxil {
2849 hal::ShaderInput::Dxil { shader: dxil }
2850 } else if let Some(hlsl) = &descriptor.hlsl {
2851 hal::ShaderInput::Hlsl { shader: hlsl }
2852 } else {
2853 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2854 }
2855 }
2856 wgt::Backend::Metal => {
2857 if let Some(metallib) = &descriptor.metallib {
2858 hal::ShaderInput::MetalLib {
2859 file: metallib,
2860 num_workgroups: entry_point_hashmap(),
2861 }
2862 } else if let Some(msl) = &descriptor.msl {
2863 hal::ShaderInput::Msl {
2864 shader: msl,
2865 num_workgroups: entry_point_hashmap(),
2866 }
2867 } else {
2868 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2869 }
2870 }
2871 wgt::Backend::Gl => hal::ShaderInput::Glsl {
2872 shader: descriptor
2873 .glsl
2874 .as_ref()
2875 .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2876 },
2877 wgt::Backend::Noop => {
2878 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2879 }
2880 wgt::Backend::BrowserWebGpu => unreachable!(),
2881 };
2882
2883 let hal_desc = hal::ShaderModuleDescriptor {
2884 label: descriptor.label.to_hal(self.instance_flags),
2885 runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2886 };
2887
2888 let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2889 Ok(raw) => raw,
2890 Err(error) => {
2891 return Err(match error {
2892 hal::ShaderError::Device(error) => {
2893 pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2894 }
2895 hal::ShaderError::Compilation(ref msg) => {
2896 log::error!("Shader error: {msg}");
2897 pipeline::CreateShaderModuleError::Generation
2898 }
2899 })
2900 }
2901 };
2902
2903 let module = pipeline::ShaderModule {
2904 state: ResourceState::Valid(pipeline::ShaderModuleState::Passthrough {
2905 raw,
2906 interface: PassthroughInterface {
2907 entry_point_names: descriptor
2908 .entry_points
2909 .iter()
2910 .map(|e| e.name.to_string())
2911 .collect(),
2912 },
2913 }),
2914 device: self.clone(),
2915 label: descriptor.label.to_string(),
2916 compilation_info: wgt::CompilationInfo::default(),
2917 };
2918
2919 Ok(Arc::new(module))
2920 }
2921
2922 pub fn create_command_encoder(
2923 self: &Arc<Self>,
2924 desc: &wgt::CommandEncoderDescriptor<crate::Label>,
2925 ) -> Arc<command::CommandEncoder> {
2926 profiling::scope!("Device::create_command_encoder");
2927
2928 let cmd_enc = self
2929 .create_command_encoder_inner(&desc.label)
2930 .unwrap_or_else(|err| {
2931 let error = err.clone().into();
2932 self.handle_error(
2933 err,
2934 desc.label.as_ref().map(|l| l.as_ref()),
2935 "Device::create_command_encoder",
2936 );
2937 command::CommandEncoder::new_invalid(self, &desc.label, error)
2938 });
2939
2940 api_log!(
2941 "Device::create_command_encoder -> {:?}",
2942 Arc::as_ptr(&cmd_enc)
2943 );
2944
2945 cmd_enc
2946 }
2947
2948 pub(crate) fn create_command_encoder_inner(
2949 self: &Arc<Self>,
2950 label: &crate::Label,
2951 ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2952 self.check_is_valid()?;
2953
2954 let queue = self.get_queue().unwrap();
2955
2956 let encoder = self
2957 .command_allocator
2958 .acquire_encoder(self.raw(), queue.raw())
2959 .map_err(|e| self.handle_hal_error(e))?;
2960
2961 let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2962
2963 let cmd_enc = Arc::new(cmd_enc);
2964
2965 Ok(cmd_enc)
2966 }
2967
2968 pub fn create_render_bundle_encoder(
2969 self: &Arc<Self>,
2970 desc: &command::RenderBundleEncoderDescriptor,
2971 ) -> Box<command::RenderBundleEncoder> {
2972 profiling::scope!("Device::create_render_bundle_encoder");
2973 api_log!("Device::create_render_bundle_encoder");
2974 Box::new(
2975 command::RenderBundleEncoder::new(self, desc).unwrap_or_else(|err| {
2976 self.handle_error(
2977 err,
2978 desc.label.as_deref(),
2979 "Device::create_render_bundle_encoder",
2980 );
2981 command::RenderBundleEncoder::dummy(self)
2982 }),
2983 )
2984 }
2985
2986 pub(super) fn make_late_sized_buffer_groups(
2989 shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2990 layout: &binding_model::PipelineLayout,
2991 ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2992 layout
2996 .bind_group_layouts
2997 .iter()
2998 .enumerate()
2999 .map(|(group_index, bgl)| {
3000 let Some(bgl) = bgl else {
3001 return pipeline::LateSizedBufferGroup::default();
3002 };
3003
3004 let shader_sizes = bgl
3005 .entries
3006 .values()
3007 .filter_map(|entry| match entry.ty {
3008 wgt::BindingType::Buffer {
3009 min_binding_size: None,
3010 ..
3011 } => {
3012 let rb = naga::ResourceBinding {
3013 group: group_index as u32,
3014 binding: entry.binding,
3015 };
3016 let shader_size =
3017 shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
3018 Some(shader_size)
3019 }
3020 _ => None,
3021 })
3022 .collect();
3023 pipeline::LateSizedBufferGroup { shader_sizes }
3024 })
3025 .collect()
3026 }
3027
3028 pub fn create_bind_group_layout<BGLE>(
3029 self: &Arc<Self>,
3030 desc: &binding_model::BindGroupLayoutDescriptor<BGLE>,
3031 ) -> Arc<BindGroupLayout>
3032 where
3033 BGLE: TryInto<wgt::BindGroupLayoutEntry> + Copy,
3034 CreateBindGroupLayoutError: From<<BGLE as TryInto<wgt::BindGroupLayoutEntry>>::Error>,
3035 BGLE: Into<binding_model::BindGroupLayoutEntry>,
3036 {
3037 profiling::scope!("Device::create_bind_group_layout");
3038
3039 let bgl = self
3040 .create_bind_group_layout_inner(desc)
3041 .unwrap_or_else(|err| {
3042 self.handle_error(
3043 err,
3044 desc.label.as_deref(),
3045 "Device::create_bind_group_layout",
3046 );
3047 BindGroupLayout::invalid(self, desc.label.to_string())
3048 });
3049
3050 #[cfg(feature = "trace")]
3051 if let Some(ref mut trace) = *self.trace.lock() {
3052 use crate::device::trace::IntoTrace;
3053
3054 trace.add(trace::Action::CreateBindGroupLayout(
3055 bgl.to_trace(),
3056 desc.to_trace(),
3057 ));
3058 }
3059 api_log!(
3060 "Device::create_bind_group_layout -> {:?}",
3061 Arc::as_ptr(&bgl)
3062 );
3063 bgl
3064 }
3065
3066 fn create_bind_group_layout_inner<BGLE>(
3067 self: &Arc<Device>,
3068 desc: &binding_model::BindGroupLayoutDescriptor<BGLE>,
3069 ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError>
3070 where
3071 BGLE: TryInto<wgt::BindGroupLayoutEntry> + Copy,
3072 CreateBindGroupLayoutError: From<<BGLE as TryInto<wgt::BindGroupLayoutEntry>>::Error>,
3073 {
3074 self.check_is_valid()?;
3075
3076 let entry_map = bgl::EntryMap::from_entries(&desc.entries)?;
3077
3078 let bgl_result = self.bgl_pool.get_or_init(entry_map, |entry_map| {
3079 let bgl =
3080 self.create_bind_group_layout_impl(&desc.label, entry_map, bgl::Origin::Pool)?;
3081 bgl.exclusive_pipeline
3082 .set(binding_model::ExclusivePipeline::None)
3083 .unwrap();
3084 Ok(bgl)
3085 });
3086
3087 match bgl_result {
3088 Ok(layout) => Ok(layout),
3089 Err(e) => Err(e),
3090 }
3091 }
3092
3093 fn create_bind_group_layout_impl(
3094 self: &Arc<Self>,
3095 label: &crate::Label,
3096 entry_map: bgl::EntryMap,
3097 origin: bgl::Origin,
3098 ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
3099 #[derive(PartialEq)]
3100 enum WritableStorage {
3101 Yes,
3102 No,
3103 }
3104
3105 for entry in entry_map.values() {
3106 if entry.binding >= self.limits.max_bindings_per_bind_group {
3107 return Err(CreateBindGroupLayoutError::InvalidBindingIndex {
3108 binding: entry.binding,
3109 maximum: self.limits.max_bindings_per_bind_group,
3110 });
3111 }
3112
3113 use wgt::BindingType as Bt;
3114
3115 let mut required_features = wgt::Features::empty();
3116 let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
3117 let (array_feature, writable_storage) = match entry.ty {
3118 Bt::Buffer {
3119 ty: wgt::BufferBindingType::Uniform,
3120 has_dynamic_offset: false,
3121 min_binding_size: _,
3122 } => (
3123 Some(wgt::Features::BUFFER_BINDING_ARRAY),
3124 WritableStorage::No,
3125 ),
3126 Bt::Buffer {
3127 ty: wgt::BufferBindingType::Uniform,
3128 has_dynamic_offset: true,
3129 min_binding_size: _,
3130 } => (
3131 Some(wgt::Features::BUFFER_BINDING_ARRAY),
3132 WritableStorage::No,
3133 ),
3134 Bt::Buffer {
3135 ty: wgt::BufferBindingType::Storage { read_only },
3136 ..
3137 } => (
3138 Some(
3139 wgt::Features::BUFFER_BINDING_ARRAY
3140 | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
3141 ),
3142 match read_only {
3143 true => WritableStorage::No,
3144 false => WritableStorage::Yes,
3145 },
3146 ),
3147 Bt::Sampler { .. } => (
3148 Some(wgt::Features::TEXTURE_BINDING_ARRAY),
3149 WritableStorage::No,
3150 ),
3151 Bt::Texture {
3152 multisampled: true,
3153 sample_type: TextureSampleType::Float { filterable: true },
3154 ..
3155 } => {
3156 return Err(CreateBindGroupLayoutError::Entry {
3157 binding: entry.binding,
3158 error:
3159 BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
3160 });
3161 }
3162 Bt::Texture {
3163 multisampled,
3164 view_dimension,
3165 ..
3166 } => {
3167 if multisampled && view_dimension != TextureViewDimension::D2 {
3168 return Err(CreateBindGroupLayoutError::Entry {
3169 binding: entry.binding,
3170 error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
3171 });
3172 }
3173
3174 (
3175 Some(wgt::Features::TEXTURE_BINDING_ARRAY),
3176 WritableStorage::No,
3177 )
3178 }
3179 Bt::StorageTexture {
3180 access,
3181 view_dimension,
3182 format,
3183 } => {
3184 use wgt::{StorageTextureAccess as Access, TextureFormatFeatureFlags as Flags};
3185
3186 match view_dimension {
3187 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
3188 return Err(CreateBindGroupLayoutError::Entry {
3189 binding: entry.binding,
3190 error: BindGroupLayoutEntryError::StorageTextureCube,
3191 })
3192 }
3193 _ => (),
3194 }
3195 match access {
3196 wgt::StorageTextureAccess::Atomic
3197 if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
3198 {
3199 return Err(CreateBindGroupLayoutError::Entry {
3200 binding: entry.binding,
3201 error: BindGroupLayoutEntryError::StorageTextureAtomic,
3202 });
3203 }
3204 _ => (),
3205 }
3206
3207 let format_features =
3208 self.describe_format_features(format).map_err(|error| {
3209 CreateBindGroupLayoutError::Entry {
3210 binding: entry.binding,
3211 error: BindGroupLayoutEntryError::MissingFeatures(error),
3212 }
3213 })?;
3214
3215 let required_feature_flag = match access {
3216 Access::WriteOnly => Flags::STORAGE_WRITE_ONLY,
3217 Access::ReadOnly => Flags::STORAGE_READ_ONLY,
3218 Access::ReadWrite => Flags::STORAGE_READ_WRITE,
3219 Access::Atomic => Flags::STORAGE_ATOMIC,
3220 };
3221
3222 if !format_features.flags.contains(required_feature_flag) {
3223 return Err(
3224 CreateBindGroupLayoutError::UnsupportedStorageTextureAccess {
3225 binding: entry.binding,
3226 access,
3227 format,
3228 },
3229 );
3230 }
3231
3232 (
3233 Some(
3234 wgt::Features::TEXTURE_BINDING_ARRAY
3235 | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
3236 ),
3237 match access {
3238 wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
3239 wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
3240 wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
3241 wgt::StorageTextureAccess::Atomic => {
3242 required_features |= wgt::Features::TEXTURE_ATOMIC;
3243 WritableStorage::Yes
3244 }
3245 },
3246 )
3247 }
3248 Bt::AccelerationStructure { vertex_return } => {
3249 self.require_acceleration_structures().map_err(|e| {
3250 CreateBindGroupLayoutError::Entry {
3251 binding: entry.binding,
3252 error: e.into(),
3253 }
3254 })?;
3255 if vertex_return {
3256 self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
3257 .map_err(|e| CreateBindGroupLayoutError::Entry {
3258 binding: entry.binding,
3259 error: e.into(),
3260 })?;
3261 }
3262 (
3263 Some(wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY),
3264 WritableStorage::No,
3265 )
3266 }
3267 Bt::ExternalTexture => {
3268 self.require_features(wgt::Features::EXTERNAL_TEXTURE)
3269 .map_err(|e| CreateBindGroupLayoutError::Entry {
3270 binding: entry.binding,
3271 error: e.into(),
3272 })?;
3273 (None, WritableStorage::No)
3274 }
3275 };
3276
3277 if entry.count.is_some() {
3279 required_features |= array_feature
3280 .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
3281 .map_err(|error| CreateBindGroupLayoutError::Entry {
3282 binding: entry.binding,
3283 error,
3284 })?;
3285 }
3286
3287 if entry.visibility.contains_unknown_bits() {
3288 return Err(CreateBindGroupLayoutError::InvalidVisibility(
3289 entry.visibility,
3290 ));
3291 }
3292
3293 if entry
3294 .visibility
3295 .intersects(wgt::ShaderStages::TASK | wgt::ShaderStages::MESH)
3296 {
3297 required_features |= wgt::Features::EXPERIMENTAL_MESH_SHADER;
3298 }
3299
3300 if entry.visibility.intersects(
3301 wgt::ShaderStages::RAY_GENERATION
3302 | wgt::ShaderStages::ANY_HIT
3303 | wgt::ShaderStages::CLOSEST_HIT
3304 | wgt::ShaderStages::MISS,
3305 ) {
3306 required_features |= wgt::Features::EXPERIMENTAL_RAY_TRACING_PIPELINES;
3307 }
3308
3309 if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
3310 if writable_storage == WritableStorage::Yes {
3311 required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
3312 }
3313 if let Bt::Buffer {
3314 ty: wgt::BufferBindingType::Storage { .. },
3315 ..
3316 } = entry.ty
3317 {
3318 required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
3319 }
3320 }
3321 if writable_storage == WritableStorage::Yes
3322 && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
3323 {
3324 required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
3325 }
3326
3327 self.require_features(required_features)
3328 .map_err(BindGroupLayoutEntryError::MissingFeatures)
3329 .map_err(|error| CreateBindGroupLayoutError::Entry {
3330 binding: entry.binding,
3331 error,
3332 })?;
3333 self.require_downlevel_flags(required_downlevel_flags)
3334 .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
3335 .map_err(|error| CreateBindGroupLayoutError::Entry {
3336 binding: entry.binding,
3337 error,
3338 })?;
3339 }
3340
3341 let bgl_flags = conv::bind_group_layout_flags(self.features);
3342
3343 let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
3344 let hal_desc = hal::BindGroupLayoutDescriptor {
3345 label: label.to_hal(self.instance_flags),
3346 flags: bgl_flags,
3347 entries: &hal_bindings,
3348 };
3349
3350 let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3351 for entry in entry_map.values() {
3352 count_validator.add_binding(entry);
3353 }
3354 count_validator
3357 .validate(&self.limits, self.instance_flags)
3358 .map_err(CreateBindGroupLayoutError::TooManyBindings)?;
3359
3360 count_validator.validate_binding_arrays()?;
3362
3363 let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
3364 .map_err(|e| self.handle_hal_error(e))?;
3365
3366 let bgl = BindGroupLayout {
3367 state: ResourceState::Valid(BindGroupLayoutState {
3368 raw: binding_model::RawBindGroupLayout::Owning(ManuallyDrop::new(raw)),
3369 origin,
3370 binding_count_validator: count_validator,
3371 }),
3372 device: self.clone(),
3373 entries: entry_map,
3374 exclusive_pipeline: OnceCell::new(),
3375 label: label.to_string(),
3376 };
3377
3378 let bgl = Arc::new(bgl);
3379
3380 Ok(bgl)
3381 }
3382
3383 fn create_buffer_binding<'a>(
3384 &self,
3385 bb: &'a binding_model::BufferBinding,
3386 binding: u32,
3387 decl: &wgt::BindGroupLayoutEntry,
3388 buffer_init_actions: &mut Vec<BufferInitTrackerAction>,
3389 dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
3390 late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
3391 used: &mut BindGroupStates,
3392 snatch_guard: &'a SnatchGuard<'a>,
3393 ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer, NonZeroU64>, CreateBindGroupError> {
3394 use crate::binding_model::CreateBindGroupError as Error;
3395
3396 let (binding_ty, dynamic, min_size) = match decl.ty {
3397 wgt::BindingType::Buffer {
3398 ty,
3399 has_dynamic_offset,
3400 min_binding_size,
3401 } => (ty, has_dynamic_offset, min_binding_size),
3402 _ => {
3403 return Err(Error::WrongBindingType {
3404 binding,
3405 actual: decl.ty,
3406 expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
3407 })
3408 }
3409 };
3410
3411 let (pub_usage, internal_use, range_limit) = match binding_ty {
3412 wgt::BufferBindingType::Uniform => (
3413 wgt::BufferUsages::UNIFORM,
3414 wgt::BufferUses::UNIFORM,
3415 self.limits.max_uniform_buffer_binding_size,
3416 ),
3417 wgt::BufferBindingType::Storage { read_only } => (
3418 wgt::BufferUsages::STORAGE,
3419 if read_only {
3420 wgt::BufferUses::STORAGE_READ_ONLY
3421 } else {
3422 wgt::BufferUses::STORAGE_READ_WRITE
3423 },
3424 self.limits.max_storage_buffer_binding_size,
3425 ),
3426 };
3427
3428 let (align, align_limit_name) =
3429 binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
3430 if !bb.offset.is_multiple_of(align as u64) {
3431 return Err(Error::UnalignedBufferOffset(
3432 bb.offset,
3433 align_limit_name,
3434 align,
3435 ));
3436 }
3437
3438 let buffer = &bb.buffer;
3439
3440 used.buffers.insert_single(buffer.clone(), internal_use);
3441
3442 buffer.check_is_valid()?;
3443 buffer.same_device(self)?;
3444
3445 buffer.check_usage(pub_usage)?;
3446
3447 let req_size = match bb.size.map(wgt::BufferSize::new) {
3448 Some(non_zero @ Some(_)) => non_zero,
3450 None => None,
3452 Some(None) => return Err(CreateBindGroupError::BindingZeroSize(buffer.error_ident())),
3454 };
3455 let bb = buffer.binding(bb.offset, req_size, snatch_guard)?;
3456 let bind_size = bb.size.get();
3457
3458 if matches!(binding_ty, wgt::BufferBindingType::Storage { .. })
3459 && bind_size % u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) != 0
3460 {
3461 return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
3462 alignment: wgt::STORAGE_BINDING_SIZE_ALIGNMENT,
3463 size: bind_size,
3464 });
3465 }
3466
3467 let bind_end = bb.offset + bind_size;
3468
3469 if bind_size > range_limit {
3470 return Err(Error::BufferRangeTooLarge {
3471 binding,
3472 given: bind_size,
3473 limit: range_limit,
3474 });
3475 }
3476
3477 if dynamic {
3479 dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
3480 binding_idx: binding,
3481 buffer_size: buffer.size,
3482 binding_range: bb.offset..bind_end,
3483 maximum_dynamic_offset: buffer.size - bind_end,
3484 binding_type: binding_ty,
3485 });
3486 }
3487
3488 if let Some(non_zero) = min_size {
3489 let min_size = non_zero.get();
3490 if min_size > bind_size {
3491 return Err(Error::BindingSizeTooSmall {
3492 buffer: buffer.error_ident(),
3493 actual: bind_size,
3494 min: min_size,
3495 });
3496 }
3497 } else {
3498 let late_size = wgt::BufferSize::new(bind_size)
3499 .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
3500 late_buffer_binding_sizes.insert(binding, late_size);
3501 }
3502
3503 assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
3506
3507 let init_range = if dynamic {
3508 0..buffer.size
3511 } else {
3512 let bounds_check_alignment = binding_model::buffer_binding_type_bounds_check_alignment(
3517 &self.alignments,
3518 binding_ty,
3519 );
3520 let visible_size = align_to(bind_size, bounds_check_alignment);
3521
3522 bb.offset..bb.offset + visible_size
3523 };
3524
3525 buffer_init_actions.extend(buffer.initialization_status.read().create_action(
3529 buffer,
3530 init_range,
3531 MemoryInitKind::NeedsInitializedMemory,
3532 ));
3533
3534 Ok(bb)
3535 }
3536
3537 fn create_sampler_binding<'a>(
3538 &self,
3539 used: &mut BindGroupStates,
3540 binding: u32,
3541 decl: &wgt::BindGroupLayoutEntry,
3542 sampler: &'a Arc<Sampler>,
3543 ) -> Result<&'a dyn hal::DynSampler, CreateBindGroupError> {
3544 use crate::binding_model::CreateBindGroupError as Error;
3545
3546 used.samplers.insert_single(sampler.clone());
3547
3548 sampler.same_device(self)?;
3549
3550 match decl.ty {
3551 wgt::BindingType::Sampler(ty) => {
3552 let (allowed_filtering, allowed_comparison) = match ty {
3553 wgt::SamplerBindingType::Filtering => (None, false),
3554 wgt::SamplerBindingType::NonFiltering => (Some(false), false),
3555 wgt::SamplerBindingType::Comparison => (None, true),
3556 };
3557 if let Some(allowed_filtering) = allowed_filtering {
3558 if allowed_filtering != sampler.filtering {
3559 return Err(Error::WrongSamplerFiltering {
3560 binding,
3561 layout_flt: allowed_filtering,
3562 sampler_flt: sampler.filtering,
3563 });
3564 }
3565 }
3566 if allowed_comparison != sampler.comparison {
3567 return Err(Error::WrongSamplerComparison {
3568 binding,
3569 layout_cmp: allowed_comparison,
3570 sampler_cmp: sampler.comparison,
3571 });
3572 }
3573 }
3574 _ => {
3575 return Err(Error::WrongBindingType {
3576 binding,
3577 actual: decl.ty,
3578 expected: "Sampler",
3579 })
3580 }
3581 }
3582
3583 Ok(sampler.raw()?)
3584 }
3585
3586 fn create_texture_binding<'a>(
3587 &self,
3588 binding: u32,
3589 decl: &wgt::BindGroupLayoutEntry,
3590 view: &'a Arc<TextureView>,
3591 used: &mut BindGroupStates,
3592 texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3593 snatch_guard: &'a SnatchGuard<'a>,
3594 ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, CreateBindGroupError> {
3595 view.check_valid()?;
3596 view.same_device(self)?;
3597
3598 let internal_use = self.texture_use_parameters(
3599 binding,
3600 decl,
3601 view,
3602 "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
3603 )?;
3604
3605 used.views.insert_single(view.clone(), internal_use);
3606
3607 let texture = &view.parent;
3608
3609 texture_init_actions.push(TextureInitTrackerAction {
3614 texture: texture.clone(),
3615 range: TextureInitRange {
3616 mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
3617 layer_range: view
3618 .desc
3619 .range
3620 .layer_range(texture.desc.array_layer_count()),
3621 },
3622 kind: MemoryInitKind::NeedsInitializedMemory,
3623 });
3624
3625 Ok(hal::TextureBinding {
3626 view: view.try_raw(snatch_guard)?,
3627 usage: internal_use,
3628 })
3629 }
3630
3631 fn create_tlas_binding<'a>(
3632 self: &Arc<Self>,
3633 used: &mut BindGroupStates,
3634 binding: u32,
3635 decl: &wgt::BindGroupLayoutEntry,
3636 tlas: &'a Arc<Tlas>,
3637 snatch_guard: &'a SnatchGuard<'a>,
3638 ) -> Result<&'a dyn hal::DynAccelerationStructure, CreateBindGroupError> {
3639 use crate::binding_model::CreateBindGroupError as Error;
3640
3641 used.acceleration_structures.insert_single(tlas.clone());
3642
3643 tlas.check_is_valid()?;
3644 tlas.same_device(self)?;
3645
3646 match decl.ty {
3647 wgt::BindingType::AccelerationStructure { vertex_return } => {
3648 if vertex_return
3649 && !tlas.flags.contains(
3650 wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
3651 )
3652 {
3653 return Err(Error::MissingTLASVertexReturn { binding });
3654 }
3655 }
3656 _ => {
3657 return Err(Error::WrongBindingType {
3658 binding,
3659 actual: decl.ty,
3660 expected: "Tlas",
3661 });
3662 }
3663 }
3664
3665 Ok(tlas.try_raw(snatch_guard)?)
3666 }
3667
3668 fn create_external_texture_binding<'a>(
3669 &'a self,
3670 binding: u32,
3671 decl: &wgt::BindGroupLayoutEntry,
3672 external_texture: &'a Arc<ExternalTexture>,
3673 texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3674 used: &mut BindGroupStates,
3675 snatch_guard: &'a SnatchGuard,
3676 ) -> Result<
3677 hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3678 CreateBindGroupError,
3679 > {
3680 use crate::binding_model::CreateBindGroupError as Error;
3681
3682 let external_texture_state = external_texture.state()?;
3683 external_texture.same_device(self)?;
3684
3685 used.external_textures
3686 .insert_single(external_texture.clone());
3687
3688 match decl.ty {
3689 wgt::BindingType::ExternalTexture => {}
3690 _ => {
3691 return Err(Error::WrongBindingType {
3692 binding,
3693 actual: decl.ty,
3694 expected: "ExternalTexture",
3695 });
3696 }
3697 }
3698
3699 let planes = (0..3)
3700 .map(|i| {
3701 let plane = external_texture
3705 .planes
3706 .get(i)
3707 .unwrap_or(&external_texture.planes[0]);
3708 let internal_use = wgt::TextureUses::RESOURCE;
3709 used.views.insert_single(plane.clone(), internal_use);
3710 let view = plane.try_raw(snatch_guard)?;
3711 Ok(hal::TextureBinding {
3712 view,
3713 usage: internal_use,
3714 })
3715 })
3716 .collect::<Result<Vec<_>, Error>>()?;
3719 let planes = planes.try_into().unwrap();
3720
3721 for plane in external_texture.planes.iter() {
3724 let texture = &plane.parent;
3725 texture_init_actions.push(TextureInitTrackerAction {
3726 texture: texture.clone(),
3727 range: TextureInitRange {
3728 mip_range: plane.desc.range.mip_range(texture.desc.mip_level_count),
3729 layer_range: plane
3730 .desc
3731 .range
3732 .layer_range(texture.desc.array_layer_count()),
3733 },
3734 kind: MemoryInitKind::NeedsInitializedMemory,
3735 });
3736 }
3737
3738 used.buffers.insert_single(
3739 external_texture_state.params.clone(),
3740 wgt::BufferUses::UNIFORM,
3741 );
3742 let params = external_texture_state
3743 .params
3744 .binding(0, None, snatch_guard)?;
3745
3746 Ok(hal::ExternalTextureBinding { planes, params })
3747 }
3748
3749 fn create_external_texture_binding_from_view<'a>(
3750 &'a self,
3751 binding: u32,
3752 decl: &wgt::BindGroupLayoutEntry,
3753 view: &'a Arc<TextureView>,
3754 texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3755 used: &mut BindGroupStates,
3756 snatch_guard: &'a SnatchGuard,
3757 ) -> Result<
3758 hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3759 CreateBindGroupError,
3760 > {
3761 use crate::binding_model::CreateBindGroupError as Error;
3762
3763 view.same_device(self)?;
3764 view.check_valid()?;
3765
3766 let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
3767 used.views.insert_single(view.clone(), internal_use);
3768
3769 match decl.ty {
3770 wgt::BindingType::ExternalTexture => {}
3771 _ => {
3772 return Err(Error::WrongBindingType {
3773 binding,
3774 actual: decl.ty,
3775 expected: "ExternalTexture",
3776 });
3777 }
3778 }
3779
3780 let planes = [
3782 hal::TextureBinding {
3783 view: view.try_raw(snatch_guard)?,
3784 usage: internal_use,
3785 },
3786 hal::TextureBinding {
3787 view: view.try_raw(snatch_guard)?,
3788 usage: internal_use,
3789 },
3790 hal::TextureBinding {
3791 view: view.try_raw(snatch_guard)?,
3792 usage: internal_use,
3793 },
3794 ];
3795 let params = hal::BufferBinding::new_unchecked(
3796 self.default_external_texture_params_buffer.as_ref(),
3797 0,
3798 NonZeroU64::new(size_of::<ExternalTextureParams>() as u64).unwrap(),
3799 );
3800
3801 texture_init_actions.push(TextureInitTrackerAction {
3802 texture: view.parent.clone(),
3803 range: TextureInitRange {
3804 mip_range: view.desc.range.mip_range(view.parent.desc.mip_level_count),
3805 layer_range: view
3806 .desc
3807 .range
3808 .layer_range(view.parent.desc.array_layer_count()),
3809 },
3810 kind: MemoryInitKind::NeedsInitializedMemory,
3811 });
3812
3813 Ok(hal::ExternalTextureBinding { planes, params })
3814 }
3815
3816 pub fn create_bind_group(
3817 self: &Arc<Self>,
3818 desc: &binding_model::BindGroupDescriptor,
3819 ) -> Arc<BindGroup> {
3820 profiling::scope!("Device::create_bind_group");
3821 #[cfg(feature = "trace")]
3822 let trace_desc = (&desc).to_trace();
3823
3824 let bind_group = self.create_bind_group_inner(desc).unwrap_or_else(|err| {
3825 self.handle_error(err, desc.label.as_deref(), "Device::create_bind_group");
3826 BindGroup::invalid(self.clone(), desc.label.to_string(), desc.layout.clone())
3827 });
3828
3829 #[cfg(feature = "trace")]
3830 if let Some(ref mut trace) = *self.trace.lock() {
3831 trace.add(trace::Action::CreateBindGroup(
3832 bind_group.to_trace(),
3833 trace_desc,
3834 ));
3835 }
3836
3837 api_log!(
3838 "Device::create_bind_group -> {:?}",
3839 Arc::as_ptr(&bind_group)
3840 );
3841
3842 bind_group
3843 }
3844
3845 pub fn create_bind_group_inner(
3848 self: &Arc<Self>,
3849 desc: &binding_model::BindGroupDescriptor,
3850 ) -> Result<Arc<BindGroup>, CreateBindGroupError> {
3851 use crate::binding_model::{BindingResource as Br, CreateBindGroupError as Error};
3852
3853 self.check_is_valid()?;
3854
3855 let layout = desc.layout.clone();
3856
3857 layout.same_device(self)?;
3858 layout.check_is_valid()?;
3859
3860 {
3861 let actual = desc.entries.len();
3864 let expected = layout.entries.len();
3865 if actual != expected {
3866 return Err(Error::BindingsNumMismatch { expected, actual });
3867 }
3868 }
3869
3870 let mut dynamic_binding_info = Vec::new();
3872 let mut late_buffer_binding_sizes = FastHashMap::default();
3876 let mut used = BindGroupStates::new();
3878
3879 let mut buffer_init_actions = Vec::new();
3880 let mut texture_init_actions = Vec::new();
3881 let snatch_guard = self.snatchable_lock.read();
3882 let mut hal_entries = SmallVec::<[_; 4]>::with_capacity(desc.entries.len());
3883 let mut hal_buffers = SmallVec::<[_; 4]>::new();
3884 let mut hal_samplers = SmallVec::<[_; 4]>::new();
3885 let mut hal_textures = SmallVec::<[_; 8]>::new();
3886 let mut hal_tlas_s = SmallVec::<[_; 1]>::new();
3887 let mut hal_external_textures = SmallVec::<[_; 1]>::new();
3888 for entry in desc.entries.iter() {
3889 let binding = entry.binding;
3890 let decl = layout
3892 .entries
3893 .get(binding)
3894 .ok_or(Error::MissingBindingDeclaration(binding))?;
3895 let (res_index, count) = match entry.resource {
3896 Br::Buffer(ref bb) => {
3897 let bb = self.create_buffer_binding(
3898 bb,
3899 binding,
3900 decl,
3901 &mut buffer_init_actions,
3902 &mut dynamic_binding_info,
3903 &mut late_buffer_binding_sizes,
3904 &mut used,
3905 &snatch_guard,
3906 )?;
3907
3908 let res_index = hal_buffers.len();
3909 hal_buffers.push(bb);
3910 (res_index, 1)
3911 }
3912 Br::BufferArray(ref bindings_array) => {
3913 let num_bindings = bindings_array.len();
3914 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3915
3916 let res_index = hal_buffers.len();
3917 for bb in bindings_array.iter() {
3918 let bb = self.create_buffer_binding(
3919 bb,
3920 binding,
3921 decl,
3922 &mut buffer_init_actions,
3923 &mut dynamic_binding_info,
3924 &mut late_buffer_binding_sizes,
3925 &mut used,
3926 &snatch_guard,
3927 )?;
3928 hal_buffers.push(bb);
3929 }
3930 (res_index, num_bindings)
3931 }
3932 Br::Sampler(ref sampler) => {
3933 let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3934
3935 let res_index = hal_samplers.len();
3936 hal_samplers.push(sampler);
3937 (res_index, 1)
3938 }
3939 Br::SamplerArray(ref samplers) => {
3940 let num_bindings = samplers.len();
3941 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3942
3943 let res_index = hal_samplers.len();
3944 for sampler in samplers.iter() {
3945 let sampler =
3946 self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3947
3948 hal_samplers.push(sampler);
3949 }
3950
3951 (res_index, num_bindings)
3952 }
3953 Br::TextureView(ref view) => match decl.ty {
3954 wgt::BindingType::ExternalTexture => {
3955 let et = self.create_external_texture_binding_from_view(
3956 binding,
3957 decl,
3958 view,
3959 &mut texture_init_actions,
3960 &mut used,
3961 &snatch_guard,
3962 )?;
3963 let res_index = hal_external_textures.len();
3964 hal_external_textures.push(et);
3965 (res_index, 1)
3966 }
3967 _ => {
3968 let tb = self.create_texture_binding(
3969 binding,
3970 decl,
3971 view,
3972 &mut used,
3973 &mut texture_init_actions,
3974 &snatch_guard,
3975 )?;
3976 let res_index = hal_textures.len();
3977 hal_textures.push(tb);
3978 (res_index, 1)
3979 }
3980 },
3981 Br::TextureViewArray(ref views) => {
3982 let num_bindings = views.len();
3983 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3984
3985 let res_index = hal_textures.len();
3986 for view in views.iter() {
3987 let tb = self.create_texture_binding(
3988 binding,
3989 decl,
3990 view,
3991 &mut used,
3992 &mut texture_init_actions,
3993 &snatch_guard,
3994 )?;
3995
3996 hal_textures.push(tb);
3997 }
3998
3999 (res_index, num_bindings)
4000 }
4001 Br::AccelerationStructure(ref tlas) => {
4002 let tlas =
4003 self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
4004 let res_index = hal_tlas_s.len();
4005 hal_tlas_s.push(tlas);
4006 (res_index, 1)
4007 }
4008 Br::AccelerationStructureArray(ref tlas_array) => {
4009 let num_bindings = tlas_array.len();
4013 Self::check_array_binding(self.features, decl.count, num_bindings)?;
4014
4015 let res_index = hal_tlas_s.len();
4016 for tlas in tlas_array.iter() {
4017 let tlas = self.create_tlas_binding(
4018 &mut used,
4019 binding,
4020 decl,
4021 tlas,
4022 &snatch_guard,
4023 )?;
4024 hal_tlas_s.push(tlas);
4025 }
4026 (res_index, num_bindings)
4027 }
4028 Br::ExternalTexture(ref et) => {
4029 let et = self.create_external_texture_binding(
4030 binding,
4031 decl,
4032 et,
4033 &mut texture_init_actions,
4034 &mut used,
4035 &snatch_guard,
4036 )?;
4037 let res_index = hal_external_textures.len();
4038 hal_external_textures.push(et);
4039 (res_index, 1)
4040 }
4041 };
4042
4043 hal_entries.push(hal::BindGroupEntry {
4044 binding,
4045 resource_index: res_index as u32,
4046 count: count as u32,
4047 });
4048 }
4049
4050 used.optimize();
4051
4052 hal_entries.sort_by_key(|entry| entry.binding);
4053 for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
4054 if a.binding == b.binding {
4055 return Err(Error::DuplicateBinding(a.binding));
4056 }
4057 }
4058
4059 dynamic_binding_info.sort_by_key(|i| i.binding_idx);
4060
4061 let hal_desc = hal::BindGroupDescriptor {
4062 label: desc.label.to_hal(self.instance_flags),
4063 layout: layout.try_raw()?,
4064 entries: &hal_entries,
4065 buffers: &hal_buffers,
4066 samplers: &hal_samplers,
4067 textures: &hal_textures,
4068 acceleration_structures: &hal_tlas_s,
4069 external_textures: &hal_external_textures,
4070 };
4071 let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
4072 .map_err(|e| self.handle_hal_error(e))?;
4073
4074 let late_buffer_binding_infos = layout
4076 .entries
4077 .indices()
4078 .flat_map(|binding| {
4079 let size = late_buffer_binding_sizes.get(&binding).cloned()?;
4080 Some(BindGroupLateBufferBindingInfo {
4081 binding_index: binding,
4082 size,
4083 })
4084 })
4085 .collect();
4086
4087 let bind_group = BindGroup {
4088 state: ResourceState::Valid(BindGroupState {
4089 raw: Snatchable::new(raw),
4090 }),
4091 device: self.clone(),
4092 layout,
4093 label: desc.label.to_string(),
4094 tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
4095 used,
4096 buffer_init_actions,
4097 texture_init_actions,
4098 dynamic_binding_info,
4099 late_buffer_binding_infos,
4100 };
4101
4102 let bind_group = Arc::new(bind_group);
4103
4104 let weak_ref = Arc::downgrade(&bind_group);
4105 for texture in bind_group.used.views.used_textures() {
4106 let mut bind_groups = texture.bind_groups.lock();
4107 bind_groups.push(weak_ref.clone());
4108 }
4109 for buffer in bind_group.used.buffers.used_resources() {
4110 let mut bind_groups = buffer.bind_groups.lock();
4111 bind_groups.push(weak_ref.clone());
4112 }
4113
4114 Ok(bind_group)
4115 }
4116
4117 fn check_array_binding(
4118 features: wgt::Features,
4119 count: Option<NonZeroU32>,
4120 num_bindings: usize,
4121 ) -> Result<(), CreateBindGroupError> {
4122 use super::binding_model::CreateBindGroupError as Error;
4123
4124 if let Some(count) = count {
4125 let count = count.get() as usize;
4126 if count < num_bindings {
4127 return Err(Error::BindingArrayPartialLengthMismatch {
4128 actual: num_bindings,
4129 expected: count,
4130 });
4131 }
4132 if count != num_bindings
4133 && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
4134 {
4135 return Err(Error::BindingArrayLengthMismatch {
4136 actual: num_bindings,
4137 expected: count,
4138 });
4139 }
4140 if num_bindings == 0 {
4141 return Err(Error::BindingArrayZeroLength);
4142 }
4143 } else {
4144 return Err(Error::SingleBindingExpected);
4145 };
4146
4147 Ok(())
4148 }
4149
4150 fn texture_use_parameters(
4151 &self,
4152 binding: u32,
4153 decl: &wgt::BindGroupLayoutEntry,
4154 view: &TextureView,
4155 expected: &'static str,
4156 ) -> Result<wgt::TextureUses, CreateBindGroupError> {
4157 use crate::binding_model::CreateBindGroupError as Error;
4158 if view
4159 .desc
4160 .aspects()
4161 .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
4162 {
4163 return Err(Error::DepthStencilAspect);
4164 }
4165 match decl.ty {
4166 wgt::BindingType::Texture {
4167 sample_type,
4168 view_dimension,
4169 multisampled,
4170 } => {
4171 use wgt::TextureSampleType as Tst;
4172 if multisampled != (view.samples != 1) {
4173 return Err(Error::InvalidTextureMultisample {
4174 binding,
4175 layout_multisampled: multisampled,
4176 view_samples: view.samples,
4177 });
4178 }
4179 let compat_sample_type = view
4180 .desc
4181 .format
4182 .sample_type(Some(view.desc.range.aspect), Some(self.features))
4183 .unwrap();
4184 match (sample_type, compat_sample_type) {
4185 (Tst::Uint, Tst::Uint) |
4186 (Tst::Sint, Tst::Sint) |
4187 (Tst::Depth, Tst::Depth) |
4188 (Tst::Float { filterable: false }, Tst::Float { .. }) |
4190 (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
4192 (Tst::Float { filterable: false }, Tst::Depth) => {}
4194 (Tst::Float { filterable: true }, Tst::Float { .. })
4199 if view.format_features.flags
4200 .contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
4201 _ => {
4202 return Err(Error::InvalidTextureSampleType {
4203 binding,
4204 layout_sample_type: sample_type,
4205 view_format: view.desc.format,
4206 view_sample_type: compat_sample_type,
4207 })
4208 }
4209 }
4210 if view_dimension != view.desc.dimension {
4211 return Err(Error::InvalidTextureDimension {
4212 binding,
4213 layout_dimension: view_dimension,
4214 view_dimension: view.desc.dimension,
4215 });
4216 }
4217 if let Some(texture_binding_view_dimension) =
4218 view.parent.texture_binding_view_dimension()
4219 {
4220 if texture_binding_view_dimension != view.desc.dimension {
4221 return Err(Error::InvalidTextureBindingViewDimension {
4222 binding,
4223 view_dimension: view.desc.dimension,
4224 texture_binding_view_dimension,
4225 });
4226 }
4227 }
4228 view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
4229 let depth_stencil_uses = if view.desc.aspects() == hal::FormatAspects::DEPTH {
4230 wgt::TextureUses::DEPTH_SAMPLED
4231 } else if view.desc.aspects() == hal::FormatAspects::STENCIL {
4232 wgt::TextureUses::STENCIL_SAMPLED
4233 } else {
4234 wgt::TextureUses::empty()
4235 };
4236 Ok(wgt::TextureUses::RESOURCE | depth_stencil_uses)
4237 }
4238 wgt::BindingType::StorageTexture {
4239 access,
4240 format,
4241 view_dimension,
4242 } => {
4243 if format != view.desc.format {
4244 return Err(Error::InvalidStorageTextureFormat {
4245 binding,
4246 layout_format: format,
4247 view_format: view.desc.format,
4248 });
4249 }
4250 if view_dimension != view.desc.dimension {
4251 return Err(Error::InvalidTextureDimension {
4252 binding,
4253 layout_dimension: view_dimension,
4254 view_dimension: view.desc.dimension,
4255 });
4256 }
4257 if let Some(texture_binding_view_dimension) =
4258 view.parent.texture_binding_view_dimension()
4259 {
4260 if texture_binding_view_dimension != view.desc.dimension {
4261 return Err(Error::InvalidTextureBindingViewDimension {
4262 binding,
4263 view_dimension: view.desc.dimension,
4264 texture_binding_view_dimension,
4265 });
4266 }
4267 }
4268
4269 let mip_level_count = view.selector.mips.end - view.selector.mips.start;
4270 if mip_level_count != 1 {
4271 return Err(Error::InvalidStorageTextureMipLevelCount {
4272 binding,
4273 mip_level_count,
4274 });
4275 }
4276
4277 if view.desc.swizzle != wgt::TextureComponentSwizzle::default() {
4278 return Err(Error::InvalidStorageTextureSwizzle {
4279 swizzle: view.desc.swizzle,
4280 });
4281 }
4282
4283 view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
4284
4285 Ok(match access {
4286 wgt::StorageTextureAccess::ReadOnly => wgt::TextureUses::STORAGE_READ_ONLY,
4287 wgt::StorageTextureAccess::WriteOnly => wgt::TextureUses::STORAGE_WRITE_ONLY,
4288 wgt::StorageTextureAccess::ReadWrite => wgt::TextureUses::STORAGE_READ_WRITE,
4289 wgt::StorageTextureAccess::Atomic => wgt::TextureUses::STORAGE_ATOMIC,
4290 })
4291 }
4292 wgt::BindingType::ExternalTexture => {
4293 if view.desc.dimension != TextureViewDimension::D2 {
4294 return Err(Error::InvalidTextureDimension {
4295 binding,
4296 layout_dimension: TextureViewDimension::D2,
4297 view_dimension: view.desc.dimension,
4298 });
4299 }
4300 let mip_level_count = view.selector.mips.end - view.selector.mips.start;
4301 if mip_level_count != 1 {
4302 return Err(Error::InvalidExternalTextureMipLevelCount {
4303 binding,
4304 mip_level_count,
4305 });
4306 }
4307 if view.desc.format != TextureFormat::Rgba8Unorm
4308 && view.desc.format != TextureFormat::Bgra8Unorm
4309 && view.desc.format != TextureFormat::Rgba16Float
4310 {
4311 return Err(Error::InvalidExternalTextureFormat {
4312 binding,
4313 format: view.desc.format,
4314 });
4315 }
4316 if view.samples != 1 {
4317 return Err(Error::InvalidTextureMultisample {
4318 binding,
4319 layout_multisampled: false,
4320 view_samples: view.samples,
4321 });
4322 }
4323
4324 view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
4325 Ok(wgt::TextureUses::RESOURCE)
4326 }
4327 _ => Err(Error::WrongBindingType {
4328 binding,
4329 actual: decl.ty,
4330 expected,
4331 }),
4332 }
4333 }
4334
4335 pub fn create_pipeline_layout(
4336 self: &Arc<Self>,
4337 desc: &binding_model::PipelineLayoutDescriptor,
4338 ) -> Arc<binding_model::PipelineLayout> {
4339 profiling::scope!("Device::create_pipeline_layout");
4340 let layout = self
4341 .create_pipeline_layout_impl(desc, false)
4342 .unwrap_or_else(|err| {
4343 self.handle_error(err, desc.label.as_deref(), "Device::create_pipeline_layout");
4344 binding_model::PipelineLayout::invalid(Arc::clone(self), desc.label.to_string())
4345 });
4346 #[cfg(feature = "trace")]
4347 if let Some(ref mut trace) = *self.trace.lock() {
4348 use crate::device::trace::IntoTrace;
4349 trace.add(trace::Action::CreatePipelineLayout(
4350 layout.to_trace(),
4351 desc.to_trace(),
4352 ));
4353 }
4354 api_log!(
4355 "Device::create_pipeline_layout -> {:?}",
4356 Arc::as_ptr(&layout)
4357 );
4358 layout
4359 }
4360
4361 fn create_pipeline_layout_impl(
4362 self: &Arc<Self>,
4363 desc: &binding_model::PipelineLayoutDescriptor,
4364 ignore_exclusive_pipeline_check: bool,
4365 ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
4366 use crate::binding_model::CreatePipelineLayoutError as Error;
4367
4368 self.check_is_valid()?;
4369
4370 let bind_group_layouts_count = desc.bind_group_layouts.len();
4371 let device_max_bind_groups = self.limits.max_bind_groups as usize;
4372 if bind_group_layouts_count > device_max_bind_groups {
4373 return Err(Error::TooManyGroups {
4374 actual: bind_group_layouts_count,
4375 max: device_max_bind_groups,
4376 });
4377 }
4378
4379 if desc.immediate_size != 0 {
4380 self.require_features(wgt::Features::IMMEDIATES)?;
4381 }
4382 if self.limits.max_immediate_size < desc.immediate_size {
4383 return Err(Error::ImmediateRangeTooLarge {
4384 size: desc.immediate_size,
4385 max: self.limits.max_immediate_size,
4386 });
4387 }
4388 if !desc
4389 .immediate_size
4390 .is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT)
4391 {
4392 return Err(Error::MisalignedImmediateSize {
4393 size: desc.immediate_size,
4394 });
4395 }
4396
4397 let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
4398
4399 for (index, bgl) in desc.bind_group_layouts.iter().enumerate() {
4400 let Some(bgl) = bgl else {
4401 continue;
4402 };
4403
4404 bgl.same_device(self)?;
4405
4406 if !ignore_exclusive_pipeline_check {
4407 let exclusive_pipeline = bgl.exclusive_pipeline.get().unwrap();
4408 if !matches!(exclusive_pipeline, binding_model::ExclusivePipeline::None) {
4409 return Err(Error::BglHasExclusivePipeline {
4410 index,
4411 pipeline: alloc::format!("{exclusive_pipeline}"),
4412 });
4413 }
4414 }
4415
4416 count_validator.merge(&bgl.state()?.binding_count_validator);
4417 }
4418
4419 count_validator
4420 .validate(&self.limits, self.instance_flags)
4421 .map_err(Error::TooManyBindings)?;
4422
4423 let buffers_and_acceleration_structures_in_vertex_stage =
4424 count_validator.buffers_and_acceleration_structures_in_vertex_stage();
4425
4426 let get_bgl_iter = || {
4427 desc.bind_group_layouts
4428 .iter()
4429 .map(|bgl| bgl.as_ref().filter(|bgl| !bgl.entries.is_empty()))
4430 };
4431
4432 let bind_group_layouts = get_bgl_iter()
4433 .map(|bgl| bgl.cloned())
4434 .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
4435
4436 let raw_bind_group_layouts = get_bgl_iter()
4437 .map(|bgl| bgl.map(|bgl| bgl.try_raw()).transpose())
4438 .collect::<Result<ArrayVec<_, { hal::MAX_BIND_GROUPS }>, _>>()?;
4439
4440 let additional_flags = if self.indirect_validation.is_some() {
4441 hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
4442 } else {
4443 hal::PipelineLayoutFlags::empty()
4444 };
4445
4446 let hal_desc = hal::PipelineLayoutDescriptor {
4447 label: desc.label.to_hal(self.instance_flags),
4448 flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
4449 | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
4450 | additional_flags,
4451 bind_group_layouts: &raw_bind_group_layouts,
4452 immediate_size: desc.immediate_size,
4453 };
4454
4455 let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
4456 .map_err(|e| self.handle_hal_error(e))?;
4457
4458 drop(raw_bind_group_layouts);
4459
4460 let layout = binding_model::PipelineLayout {
4461 raw: ResourceState::Valid(raw),
4462 device: self.clone(),
4463 label: desc.label.to_string(),
4464 bind_group_layouts,
4465 immediate_size: desc.immediate_size,
4466 buffers_and_acceleration_structures_in_vertex_stage,
4467 };
4468
4469 let layout = Arc::new(layout);
4470
4471 Ok(layout)
4472 }
4473
4474 pub(super) fn create_derived_pipeline_layout(
4475 self: &Arc<Self>,
4476 mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
4477 immediate_size: u32,
4478 ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
4479 let immediate_size = align_to(immediate_size, wgt::IMMEDIATE_DATA_ALIGNMENT);
4482
4483 while derived_group_layouts
4484 .last()
4485 .is_some_and(|map| map.is_empty())
4486 {
4487 derived_group_layouts.pop();
4488 }
4489
4490 let mut unique_bind_group_layouts = FastHashMap::default();
4491
4492 let bind_group_layouts = derived_group_layouts
4493 .into_iter()
4494 .map(|mut bgl_entry_map| {
4495 if bgl_entry_map.is_empty() {
4496 return Ok(None);
4497 }
4498
4499 bgl_entry_map.sort();
4500 match unique_bind_group_layouts.entry(bgl_entry_map) {
4501 hashbrown::hash_map::Entry::Occupied(v) => Ok(Some(Arc::clone(v.get()))),
4502 hashbrown::hash_map::Entry::Vacant(e) => {
4503 match self.create_bind_group_layout_impl(
4504 &None,
4505 e.key().clone(),
4506 bgl::Origin::Derived,
4507 ) {
4508 Ok(bgl) => {
4509 e.insert(bgl.clone());
4510 Ok(Some(bgl))
4511 }
4512 Err(e) => Err(e),
4513 }
4514 }
4515 }
4516 })
4517 .collect::<Result<Vec<_>, _>>()?;
4518
4519 let layout_desc = binding_model::PipelineLayoutDescriptor {
4520 label: None,
4521 bind_group_layouts: Cow::Owned(bind_group_layouts),
4522 immediate_size,
4523 };
4524
4525 let layout = self.create_pipeline_layout_impl(&layout_desc, true)?;
4526 Ok(layout)
4527 }
4528
4529 pub fn create_compute_pipeline(
4534 self: &Arc<Self>,
4535 desc: pipeline::ComputePipelineDescriptor,
4536 ) -> Arc<pipeline::ComputePipeline> {
4537 profiling::scope!("Device::create_compute_pipeline");
4538 let compute_pipeline = self
4539 .create_compute_pipeline_or_error_inner(desc.clone())
4540 .unwrap_or_else(|err| {
4541 if let pipeline::CreatePipelineError::Internal { stage, ref message } = err {
4542 log::error!("Shader translation error for stage {stage:?}: {message}");
4543 log::error!("Please report it to https://github.com/gfx-rs/wgpu");
4544 }
4545 self.handle_error(
4546 err,
4547 desc.label.as_deref(),
4548 "Device::create_compute_pipeline",
4549 );
4550
4551 pipeline::ComputePipeline::invalid(self.clone(), desc.label.to_string())
4552 });
4553 #[cfg(feature = "trace")]
4554 if let Some(ref mut trace) = *self.trace.lock() {
4555 use crate::device::trace;
4556 use crate::device::trace::IntoTrace;
4557 trace.add(trace::Action::CreateComputePipeline {
4558 id: compute_pipeline.to_trace(),
4559 desc: desc.to_trace(),
4560 });
4561 }
4562 api_log!(
4563 "Device::create_compute_pipeline -> {:?}",
4564 Arc::as_ptr(&compute_pipeline)
4565 );
4566 compute_pipeline
4567 }
4568
4569 pub fn create_compute_pipeline_or_error(
4575 self: &Arc<Self>,
4576 desc: pipeline::ComputePipelineDescriptor,
4577 ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreatePipelineError> {
4578 let label = desc.label.to_string();
4579 match self.create_compute_pipeline_or_error_inner(desc) {
4580 Err(err) if err.webgpu_error_type() == wgt::error::ErrorType::DeviceLost => {
4581 Ok(pipeline::ComputePipeline::invalid(self.clone(), label))
4582 }
4583 result => result,
4584 }
4585 }
4586
4587 fn create_compute_pipeline_or_error_inner(
4588 self: &Arc<Self>,
4589 desc: pipeline::ComputePipelineDescriptor,
4590 ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreatePipelineError> {
4591 self.check_is_valid()?;
4592
4593 self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
4594
4595 let is_auto_layout = desc.layout.is_none();
4596
4597 let pipeline_layout = match desc.layout {
4599 Some(pipeline_layout) => {
4600 pipeline_layout.same_device(self)?;
4601 pipeline_layout.check_valid()?;
4602 Some(pipeline_layout)
4603 }
4604 None => None,
4605 };
4606
4607 let mut compiler =
4608 StageCompiler::new(self, pipeline_layout, validation::StageIo::default(), None);
4609 let compute_stage =
4610 compiler.compile(&desc.stage, validation::ShaderStageForValidation::Compute)?;
4611
4612 let resources = compiler.resources;
4613
4614 let pipeline_layout = match resources.layout {
4615 validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4616 validation::BindingLayoutSource::Derived(entries) => {
4617 self.create_derived_pipeline_layout(entries, resources.immediates.size())?
4618 }
4619 };
4620
4621 let naga::valid::ImmediateUsage::Valid {
4622 slots: immediate_slots_required,
4623 size: _,
4624 } = resources.immediates
4625 else {
4626 unreachable!("Immediates exceeding maxImmediateSize should have been rejected");
4627 };
4628
4629 let late_sized_buffer_groups = Device::make_late_sized_buffer_groups(
4630 &resources.minimum_binding_sizes,
4631 &pipeline_layout,
4632 );
4633
4634 let cache = match desc.cache {
4635 Some(cache) => {
4636 cache.check_is_valid()?;
4637 cache.same_device(self)?;
4638 Some(cache)
4639 }
4640 None => None,
4641 };
4642
4643 let pipeline_desc = hal::ComputePipelineDescriptor {
4644 label: desc.label.to_hal(self.instance_flags),
4645 layout: pipeline_layout.raw()?,
4646 stage: compute_stage.to_hal(),
4647 cache: cache.as_ref().map(|it| it.raw()).transpose()?,
4648 };
4649
4650 let raw = unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }
4651 .map_err(|error| self.handle_hal_pipeline_error(error))?;
4652 drop(compute_stage);
4653
4654 let pipeline = pipeline::ComputePipeline {
4655 state: ResourceState::Valid(pipeline::ComputePipelineState {
4656 raw: ManuallyDrop::new(raw),
4657 layout: pipeline_layout.clone(),
4658 _shader_module: desc.stage.module,
4659 }),
4660 device: self.clone(),
4661 late_sized_buffer_groups,
4662 immediate_slots_required,
4663 label: desc.label.to_string(),
4664 tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
4665 };
4666
4667 let pipeline = Arc::new(pipeline);
4668
4669 if is_auto_layout {
4670 for bgl in pipeline_layout.bind_group_layouts.iter() {
4671 let Some(bgl) = bgl else {
4672 continue;
4673 };
4674
4675 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
4678 }
4679 }
4680
4681 Ok(pipeline)
4682 }
4683
4684 pub fn create_render_pipeline(
4689 self: &Arc<Self>,
4690 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4691 ) -> Arc<pipeline::RenderPipeline> {
4692 profiling::scope!("Device::create_render_pipeline");
4693
4694 let render_pipeline = self
4695 .create_render_pipeline_or_error_inner(desc.clone())
4696 .unwrap_or_else(|err| {
4697 if let pipeline::CreatePipelineError::Internal { stage, ref message } = err {
4698 log::error!("Shader translation error for stage {stage:?}: {message}");
4699 log::error!("Please report it to https://github.com/gfx-rs/wgpu");
4700 }
4701 self.handle_error(err, desc.label.as_deref(), "Device::create_render_pipeline");
4702 pipeline::RenderPipeline::invalid(self.clone(), desc.label.to_string())
4703 });
4704 #[cfg(feature = "trace")]
4705 if let Some(ref mut trace) = *self.trace.lock() {
4706 use crate::device::trace::IntoTrace;
4707 trace.add(trace::Action::CreateGeneralRenderPipeline {
4708 id: render_pipeline.to_trace(),
4709 desc: desc.to_trace(),
4710 });
4711 }
4712 api_log!(
4713 "Device::create_render_pipeline -> {:?}",
4714 Arc::as_ptr(&render_pipeline)
4715 );
4716 render_pipeline
4717 }
4718
4719 pub fn create_render_pipeline_or_error(
4725 self: &Arc<Self>,
4726 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4727 ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreatePipelineError> {
4728 let label = desc.label.to_string();
4729 match self.create_render_pipeline_or_error_inner(desc) {
4730 Err(e) if e.webgpu_error_type() == wgt::error::ErrorType::DeviceLost => Ok(
4731 pipeline::RenderPipeline::invalid(self.clone(), label.to_string()),
4732 ),
4733 result => result,
4734 }
4735 }
4736
4737 fn create_render_pipeline_or_error_inner(
4738 self: &Arc<Self>,
4739 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4740 ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreatePipelineError> {
4741 use wgt::TextureFormatFeatureFlags as Tfff;
4742
4743 self.check_is_valid()?;
4744
4745 let color_targets = desc
4746 .fragment
4747 .as_ref()
4748 .map_or(&[][..], |fragment| &fragment.targets);
4749 let depth_stencil_state = desc.depth_stencil.as_ref();
4750
4751 check_color_attachment_count(color_targets.len(), self.limits.max_color_attachments)?;
4752
4753 {
4754 let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
4755 color_targets.iter().filter_map(|x| x.as_ref()).collect();
4756 if !cts.is_empty() && {
4757 let first = &cts[0];
4758 cts[1..]
4759 .iter()
4760 .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
4761 } {
4762 self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
4763 }
4764 }
4765
4766 let mut io = validation::StageIo::default();
4767
4768 let mut vertex_steps;
4769 let mut hal_vertex_buffer_layouts;
4770 let mut total_attributes;
4771 let mut dual_source_blending = false;
4772 let mut has_depth_attachment = false;
4773 if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4774 if vertex.buffers.len() > self.limits.max_vertex_buffers as usize {
4775 return Err(pipeline::RenderError::TooManyVertexBuffers {
4776 given: vertex.buffers.len() as u32,
4777 limit: self.limits.max_vertex_buffers,
4778 }
4779 .into());
4780 }
4781
4782 vertex_steps = Vec::with_capacity(vertex.buffers.len());
4783 hal_vertex_buffer_layouts = Vec::with_capacity(vertex.buffers.len());
4784 total_attributes = 0;
4785 for (i, vb_state) in vertex.buffers.iter().enumerate() {
4786 let Some(vb_state) = vb_state else {
4787 vertex_steps.push(None);
4788 hal_vertex_buffer_layouts.push(None);
4789 continue;
4790 };
4791
4792 if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
4795 return Err(pipeline::RenderError::VertexStrideTooLarge {
4796 index: i as u32,
4797 given: vb_state.array_stride as u32,
4798 limit: self.limits.max_vertex_buffer_array_stride,
4799 }
4800 .into());
4801 }
4802 if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
4803 return Err(pipeline::RenderError::UnalignedVertexStride {
4804 index: i as u32,
4805 stride: vb_state.array_stride,
4806 }
4807 .into());
4808 }
4809
4810 let max_stride = if vb_state.array_stride == 0 {
4811 self.limits.max_vertex_buffer_array_stride as u64
4812 } else {
4813 vb_state.array_stride
4814 };
4815 let mut last_stride = 0;
4816 for attribute in vb_state.attributes.iter() {
4817 let attribute_stride = attribute.offset + attribute.format.size();
4818 if attribute_stride > max_stride {
4819 return Err(pipeline::RenderError::VertexAttributeStrideTooLarge {
4820 location: attribute.shader_location,
4821 given: attribute_stride as u32,
4822 limit: max_stride as u32,
4823 }
4824 .into());
4825 }
4826
4827 let required_offset_alignment = attribute.format.size().min(4);
4828 if attribute.offset % required_offset_alignment != 0 {
4829 return Err(pipeline::RenderError::InvalidVertexAttributeOffset {
4830 location: attribute.shader_location,
4831 offset: attribute.offset,
4832 }
4833 .into());
4834 }
4835
4836 if attribute.shader_location >= self.limits.max_vertex_attributes {
4837 return Err(pipeline::RenderError::VertexAttributeLocationTooLarge {
4838 given: attribute.shader_location,
4839 limit: self.limits.max_vertex_attributes,
4840 }
4841 .into());
4842 }
4843
4844 last_stride = last_stride.max(attribute_stride);
4845 }
4846
4847 vertex_steps.push(Some(pipeline::VertexStep {
4848 stride: vb_state.array_stride,
4849 last_stride,
4850 mode: vb_state.step_mode,
4851 }));
4852 hal_vertex_buffer_layouts.push(if vb_state.attributes.is_empty() {
4853 None
4854 } else {
4855 Some(hal::VertexBufferLayout {
4856 array_stride: vb_state.array_stride,
4857 step_mode: vb_state.step_mode,
4858 attributes: vb_state.attributes.as_ref(),
4859 })
4860 });
4861
4862 for attribute in vb_state.attributes.iter() {
4863 if attribute.offset >= 0x10000000 {
4864 return Err(pipeline::RenderError::InvalidVertexAttributeOffset {
4865 location: attribute.shader_location,
4866 offset: attribute.offset,
4867 }
4868 .into());
4869 }
4870
4871 if let wgt::VertexFormat::Float64
4872 | wgt::VertexFormat::Float64x2
4873 | wgt::VertexFormat::Float64x3
4874 | wgt::VertexFormat::Float64x4 = attribute.format
4875 {
4876 self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
4877 }
4878
4879 let previous = io.varyings.insert(
4880 attribute.shader_location,
4881 validation::InterfaceVar::vertex_attribute(attribute.format),
4882 );
4883
4884 if previous.is_some() {
4885 return Err(pipeline::RenderError::ShaderLocationClash(
4886 attribute.shader_location,
4887 )
4888 .into());
4889 }
4890 }
4891 total_attributes += vb_state.attributes.len();
4892 }
4893
4894 if total_attributes > self.limits.max_vertex_attributes as usize {
4895 return Err(pipeline::RenderError::TooManyVertexAttributes {
4896 given: total_attributes as u32,
4897 limit: self.limits.max_vertex_attributes,
4898 }
4899 .into());
4900 }
4901 } else {
4902 vertex_steps = Vec::new();
4903 hal_vertex_buffer_layouts = Vec::new();
4904 };
4905
4906 if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
4907 return Err(pipeline::RenderError::StripIndexFormatForNonStripTopology {
4908 strip_index_format: desc.primitive.strip_index_format,
4909 topology: desc.primitive.topology,
4910 }
4911 .into());
4912 }
4913
4914 if desc.primitive.unclipped_depth {
4915 self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
4916 }
4917
4918 if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
4919 self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
4920 }
4921 if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
4922 self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
4923 }
4924
4925 if desc.primitive.conservative {
4926 self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
4927 }
4928
4929 if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
4930 return Err(pipeline::RenderError::ConservativeRasterizationNonFillPolygonMode.into());
4931 }
4932
4933 let mut target_specified = false;
4934 let mut required_color_outputs = 0u64;
4935 const _: () = {
4936 assert!(hal::MAX_COLOR_ATTACHMENTS <= 64);
4937 };
4938
4939 for (i, cs) in color_targets.iter().enumerate() {
4940 if let Some(cs) = cs.as_ref() {
4941 target_specified = true;
4942 let error = 'error: {
4943 if cs.write_mask.contains_unknown_bits() {
4947 break 'error Some(ColorStateError::InvalidWriteMask(cs.write_mask));
4948 } else if cs.write_mask != ColorWrites::NONE {
4949 required_color_outputs |= 1 << i;
4950 }
4951
4952 let format_features = self.describe_format_features(cs.format)?;
4953 if !format_features
4954 .allowed_usages
4955 .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4956 {
4957 break 'error Some(ColorStateError::FormatNotRenderable(cs.format));
4958 }
4959 if cs.blend.is_some() && !format_features.flags.contains(Tfff::BLENDABLE) {
4960 break 'error Some(ColorStateError::FormatNotBlendable(cs.format));
4961 }
4962 if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
4963 break 'error Some(ColorStateError::FormatNotColor(cs.format));
4964 }
4965
4966 if desc.multisample.count > 1
4967 && !format_features
4968 .flags
4969 .sample_count_supported(desc.multisample.count)
4970 {
4971 break 'error Some(ColorStateError::InvalidSampleCount(
4972 desc.multisample.count,
4973 cs.format,
4974 cs.format
4975 .guaranteed_format_features(self.features)
4976 .flags
4977 .supported_sample_counts(),
4978 self.adapter
4979 .get_texture_format_features(cs.format)
4980 .flags
4981 .supported_sample_counts(),
4982 ));
4983 }
4984
4985 if let Some(blend_mode) = cs.blend {
4986 for component in [&blend_mode.color, &blend_mode.alpha] {
4987 for factor in [component.src_factor, component.dst_factor] {
4988 if factor.uses_second_blend_source() {
4989 self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
4990 if i == 0 {
4991 dual_source_blending = true;
4992 } else {
4993 break 'error Some(
4994 ColorStateError::BlendFactorOnUnsupportedTarget {
4995 factor,
4996 target: i as u32,
4997 },
4998 );
4999 }
5000 }
5001
5002 if [wgt::BlendOperation::Min, wgt::BlendOperation::Max]
5003 .contains(&component.operation)
5004 && factor != wgt::BlendFactor::One
5005 {
5006 break 'error Some(ColorStateError::InvalidMinMaxBlendFactor {
5007 factor,
5008 target: i as u32,
5009 });
5010 }
5011 }
5012 }
5013 }
5014
5015 break 'error None;
5016 };
5017 if let Some(e) = error {
5018 return Err(pipeline::RenderError::ColorState(i as u8, e).into());
5019 }
5020 }
5021 }
5022
5023 if dual_source_blending && color_targets.len() > 1 {
5024 return Err(
5025 pipeline::RenderError::DualSourceBlendingWithMultipleColorTargets {
5026 count: color_targets.len(),
5027 }
5028 .into(),
5029 );
5030 }
5031
5032 validation::validate_color_attachment_bytes_per_sample(
5033 color_targets.iter().flatten().map(|cs| cs.format),
5034 self.limits.max_color_attachment_bytes_per_sample,
5035 )
5036 .map_err(pipeline::RenderError::ColorAttachment)?;
5037
5038 if let Some(ds) = depth_stencil_state {
5039 target_specified = true;
5041 let error = 'error: {
5042 if !ds.format.is_depth_stencil_format() {
5043 break 'error Some(pipeline::DepthStencilStateError::FormatNotDepthOrStencil(
5046 ds.format,
5047 ));
5048 }
5049
5050 let format_features = self.describe_format_features(ds.format)?;
5051 if !format_features
5052 .allowed_usages
5053 .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
5054 {
5055 break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
5056 ds.format,
5057 ));
5058 }
5059
5060 let aspect = hal::FormatAspects::from(ds.format);
5061 if aspect.contains(hal::FormatAspects::DEPTH) {
5062 has_depth_attachment = true;
5063 } else if ds.is_depth_enabled() {
5064 break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
5065 }
5066 if has_depth_attachment {
5067 let Some(depth_write_enabled) = ds.depth_write_enabled else {
5068 break 'error Some(
5069 pipeline::DepthStencilStateError::MissingDepthWriteEnabled(ds.format),
5070 );
5071 };
5072
5073 let depth_compare_required = depth_write_enabled
5074 || ds.stencil.front.depth_fail_op != wgt::StencilOperation::Keep
5075 || ds.stencil.back.depth_fail_op != wgt::StencilOperation::Keep;
5076 if depth_compare_required && ds.depth_compare.is_none() {
5077 break 'error Some(pipeline::DepthStencilStateError::MissingDepthCompare(
5078 ds.format,
5079 ));
5080 }
5081 }
5082
5083 if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
5084 break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
5085 ds.format,
5086 ));
5087 }
5088 if desc.multisample.count > 1
5089 && !format_features
5090 .flags
5091 .sample_count_supported(desc.multisample.count)
5092 {
5093 break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
5094 desc.multisample.count,
5095 ds.format,
5096 ds.format
5097 .guaranteed_format_features(self.features)
5098 .flags
5099 .supported_sample_counts(),
5100 self.adapter
5101 .get_texture_format_features(ds.format)
5102 .flags
5103 .supported_sample_counts(),
5104 ));
5105 }
5106
5107 break 'error None;
5108 };
5109 if let Some(e) = error {
5110 return Err(pipeline::RenderError::DepthStencilState(e).into());
5111 }
5112
5113 if ds.bias.clamp != 0.0 {
5114 self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
5115 }
5116
5117 if (ds.bias.is_enabled() || ds.bias.clamp != 0.0)
5118 && !desc.primitive.topology.is_triangles()
5119 {
5120 return Err(pipeline::RenderError::DepthStencilState(
5121 pipeline::DepthStencilStateError::DepthBiasWithIncompatibleTopology(
5122 desc.primitive.topology,
5123 ),
5124 )
5125 .into());
5126 }
5127 }
5128
5129 if !target_specified {
5130 return Err(pipeline::RenderError::NoTargetSpecified.into());
5131 }
5132
5133 let is_auto_layout = desc.layout.is_none();
5134
5135 let pipeline_layout = match desc.layout {
5137 Some(pipeline_layout) => {
5138 pipeline_layout.same_device(self)?;
5139 pipeline_layout.check_valid()?;
5140 Some(pipeline_layout)
5141 }
5142 None => None,
5143 };
5144
5145 let samples = {
5146 let sc = desc.multisample.count;
5147 if sc == 0 || sc > 32 || !sc.is_power_of_two() {
5148 return Err(pipeline::RenderError::InvalidSampleCount(sc).into());
5149 }
5150 sc
5151 };
5152
5153 let mut compiler =
5155 StageCompiler::new(self, pipeline_layout, io, Some(desc.primitive.topology));
5156 let vertex_stage;
5157 let task_stage;
5158 let mesh_stage;
5159
5160 match desc.vertex {
5161 pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
5162 let stage = validation::ShaderStageForValidation::Vertex {
5163 topology: desc.primitive.topology,
5164 compare_function: desc.depth_stencil.as_ref().and_then(|d| d.depth_compare),
5165 };
5166 vertex_stage = Some(compiler.compile(&vertex.stage, stage)?);
5167 task_stage = None;
5168 mesh_stage = None;
5169 }
5170 pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
5171 self.require_features(wgt::Features::EXPERIMENTAL_MESH_SHADER)?;
5172
5173 vertex_stage = None;
5174 task_stage = match task {
5175 Some(task) => Some(
5176 compiler
5177 .compile(&task.stage, validation::ShaderStageForValidation::Task)?,
5178 ),
5179 None => None,
5180 };
5181 mesh_stage = Some(
5182 compiler.compile(&mesh.stage, validation::ShaderStageForValidation::Mesh)?,
5183 );
5184 }
5185 }
5186
5187 let fragment_stage = match desc.fragment {
5188 Some(ref fragment) => {
5189 let stage = validation::ShaderStageForValidation::Fragment {
5190 dual_source_blending,
5191 has_depth_attachment,
5192 };
5193 Some(compiler.compile(&fragment.stage, stage)?)
5194 }
5195 None => None,
5196 };
5197
5198 let StageCompiler {
5201 resources,
5202 io,
5203 validated: validated_stages,
5204 ..
5205 } = compiler;
5206
5207 let mut active_color_outputs = 0u64;
5208 if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
5209 for (i, output) in io.varyings.iter() {
5210 active_color_outputs |= 1 << i;
5211 match color_targets.get(*i as usize) {
5212 Some(Some(state)) => {
5213 validation::check_color_attachment_compatibility(state, output.ty)
5214 .map_err(|err| pipeline::RenderError::ColorState(*i as u8, err))?;
5215 }
5216 _ => {
5217 log::debug!(
5218 "The fragment stage {:?} output @location({}) values are ignored",
5219 fragment_stage
5220 .as_ref()
5221 .map_or("", |stage| stage.entry_point.as_str()),
5222 i
5223 );
5224 }
5225 }
5226 }
5227
5228 let missing_color_outputs = required_color_outputs & !active_color_outputs;
5229 if missing_color_outputs != 0 {
5230 return Err(pipeline::RenderError::ColorState(
5231 missing_color_outputs.trailing_zeros() as u8,
5232 ColorStateError::OutputNotPresent,
5233 )
5234 .into());
5235 }
5236 }
5237
5238 let pipeline_layout = match resources.layout {
5239 validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
5240 validation::BindingLayoutSource::Derived(entries) => {
5241 self.create_derived_pipeline_layout(entries, resources.immediates.size())?
5242 }
5243 };
5244
5245 let naga::valid::ImmediateUsage::Valid {
5246 slots: immediate_slots_required,
5247 size: _,
5248 } = resources.immediates
5249 else {
5250 unreachable!("Immediates exceeding maxImmediateSize should have been rejected");
5251 };
5252
5253 if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
5254 let bind_groups_plus_vertex_buffers =
5255 u32::try_from(pipeline_layout.bind_group_layouts.len() + vertex.buffers.len())
5256 .unwrap();
5257 if bind_groups_plus_vertex_buffers > self.limits.max_bind_groups_plus_vertex_buffers {
5258 return Err(pipeline::RenderError::TooManyBindGroupsPlusVertexBuffers {
5259 given: bind_groups_plus_vertex_buffers,
5260 limit: self.limits.max_bind_groups_plus_vertex_buffers,
5261 }
5262 .into());
5263 }
5264
5265 let given = pipeline_layout
5266 .buffers_and_acceleration_structures_in_vertex_stage
5267 .saturating_add(vertex.buffers.len() as u32);
5268 if !self
5269 .instance_flags
5270 .contains(wgt::InstanceFlags::STRICT_WEBGPU_COMPLIANCE)
5271 {
5272 let limit = self
5273 .limits
5274 .max_buffers_and_acceleration_structures_per_shader_stage;
5275 if given > limit {
5276 return Err(
5277 pipeline::RenderError::TooManyBuffersAndAccelerationStructuresInVertexStage {
5278 given,
5279 limit,
5280 }
5281 .into(),
5282 );
5283 }
5284 }
5285 }
5286
5287 if let Some(mv_mask) = desc.multiview_mask {
5289 self.require_features(wgt::Features::MULTIVIEW)?;
5290 if !(mv_mask.get() + 1).is_power_of_two() {
5291 self.require_features(wgt::Features::SELECTIVE_MULTIVIEW)?;
5292 }
5293 }
5294
5295 if !self
5296 .downlevel
5297 .flags
5298 .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
5299 {
5300 for (binding, size) in resources.minimum_binding_sizes.iter() {
5301 if size.get() % 16 != 0 {
5302 return Err(pipeline::RenderError::UnalignedShader {
5303 binding: binding.binding,
5304 group: binding.group,
5305 size: size.get(),
5306 }
5307 .into());
5308 }
5309 }
5310 }
5311
5312 let late_sized_buffer_groups = Device::make_late_sized_buffer_groups(
5313 &resources.minimum_binding_sizes,
5314 &pipeline_layout,
5315 );
5316
5317 let cache = match desc.cache {
5318 Some(cache) => {
5319 cache.check_is_valid()?;
5320 cache.same_device(self)?;
5321 Some(cache)
5322 }
5323 None => None,
5324 };
5325
5326 let is_mesh = mesh_stage.is_some();
5327 let has_task_shader = task_stage.is_some();
5328 let raw = {
5329 let pipeline_desc = hal::RenderPipelineDescriptor {
5330 label: desc.label.to_hal(self.instance_flags),
5331 layout: pipeline_layout.raw()?,
5332 vertex_processor: match vertex_stage {
5333 Some(ref vertex_stage) => hal::VertexProcessor::Standard {
5334 vertex_buffers: &hal_vertex_buffer_layouts,
5335 vertex_stage: vertex_stage.to_hal(),
5336 },
5337 None => hal::VertexProcessor::Mesh {
5338 task_stage: task_stage.as_ref().map(|stage| stage.to_hal()),
5339 mesh_stage: mesh_stage.as_ref().unwrap().to_hal(),
5340 },
5341 },
5342 primitive: desc.primitive,
5343 depth_stencil: desc.depth_stencil.clone(),
5344 multisample: desc.multisample,
5345 fragment_stage: fragment_stage.as_ref().map(|stage| stage.to_hal()),
5346 color_targets,
5347 multiview_mask: desc.multiview_mask,
5348 cache: cache.as_ref().map(|it| it.raw()).transpose()?,
5349 };
5350 unsafe { self.raw().create_render_pipeline(&pipeline_desc) }
5351 .map_err(|err| self.handle_hal_pipeline_error(err))?
5352 };
5353 drop((vertex_stage, task_stage, mesh_stage, fragment_stage));
5354
5355 let pass_context = RenderPassContext {
5356 attachments: AttachmentData {
5357 colors: color_targets
5358 .iter()
5359 .map(|state| state.as_ref().map(|s| s.format))
5360 .collect(),
5361 resolves: ArrayVec::new(),
5362 depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
5363 },
5364 sample_count: samples,
5365 multiview_mask: desc.multiview_mask,
5366 };
5367
5368 let mut flags = pipeline::PipelineFlags::empty();
5369 for state in color_targets.iter().filter_map(|s| s.as_ref()) {
5370 if let Some(ref bs) = state.blend {
5371 if bs.color.uses_constant() | bs.alpha.uses_constant() {
5372 flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
5373 }
5374 }
5375 }
5376 if let Some(ds) = depth_stencil_state.as_ref() {
5377 if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
5378 flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
5379 }
5380 if !ds.is_depth_read_only() {
5381 flags |= pipeline::PipelineFlags::WRITES_DEPTH;
5382 }
5383 if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
5384 flags |= pipeline::PipelineFlags::WRITES_STENCIL;
5385 }
5386 }
5387 let shader_modules = {
5388 let mut shader_modules = ArrayVec::new();
5389 match desc.vertex {
5390 pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
5391 shader_modules.push(vertex.stage.module)
5392 }
5393 pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
5394 if let Some(task) = task {
5395 shader_modules.push(task.stage.module);
5396 }
5397 shader_modules.push(mesh.stage.module);
5398 }
5399 }
5400 shader_modules.extend(desc.fragment.map(|f| f.stage.module));
5401 shader_modules
5402 };
5403
5404 let pipeline = pipeline::RenderPipeline {
5405 state: ResourceState::Valid(pipeline::RenderPipelineState {
5406 raw: ManuallyDrop::new(raw),
5407 layout: pipeline_layout.clone(),
5408 }),
5409 device: self.clone(),
5410 pass_context,
5411 _shader_modules: shader_modules,
5412 flags,
5413 topology: desc.primitive.topology,
5414 strip_index_format: desc.primitive.strip_index_format,
5415 vertex_steps,
5416 late_sized_buffer_groups,
5417 immediate_slots_required,
5418 label: desc.label.to_string(),
5419 tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
5420 is_mesh,
5421 has_task_shader,
5422 };
5423
5424 let pipeline = Arc::new(pipeline);
5425
5426 if is_auto_layout {
5427 for bgl in pipeline_layout.bind_group_layouts.iter() {
5428 let Some(bgl) = bgl else {
5429 continue;
5430 };
5431
5432 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
5435 }
5436 }
5437
5438 Ok(pipeline)
5439 }
5440
5441 pub(crate) fn handle_hal_pipeline_error(
5450 &self,
5451 error: hal::PipelineError,
5452 ) -> pipeline::CreatePipelineError {
5453 match error {
5454 hal::PipelineError::Device(error) => {
5455 pipeline::CreatePipelineError::Device(self.handle_hal_error(error))
5456 }
5457 other => pipeline::CreatePipelineError::from_hal_without_device_loss(other),
5458 }
5459 }
5460
5461 pub unsafe fn create_pipeline_cache(
5465 self: &Arc<Self>,
5466 desc: &pipeline::PipelineCacheDescriptor,
5467 ) -> (
5468 Arc<pipeline::PipelineCache>,
5469 Option<pipeline::CreatePipelineCacheError>,
5470 ) {
5471 profiling::scope!("Device::create_pipeline_cache");
5472 let (cache, error) = match unsafe { self.create_pipeline_cache_inner(desc) } {
5473 Ok(cache) => (cache, None),
5474 Err(e) => (
5475 pipeline::PipelineCache::invalid(self.clone(), desc),
5476 Some(e),
5477 ),
5478 };
5479 #[cfg(feature = "trace")]
5480 if let Some(ref mut trace) = *self.trace.lock() {
5481 use trace::IntoTrace;
5482 trace.add(trace::Action::CreatePipelineCache {
5483 id: cache.to_trace(),
5484 desc: desc.clone(),
5485 });
5486 }
5487 api_log!("Device::create_pipeline_cache -> {:?}", Arc::as_ptr(&cache));
5488 (cache, error)
5489 }
5490
5491 pub(crate) unsafe fn create_pipeline_cache_inner(
5495 self: &Arc<Self>,
5496 desc: &pipeline::PipelineCacheDescriptor,
5497 ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
5498 use crate::pipeline_cache;
5499
5500 self.check_is_valid()?;
5501
5502 self.require_features(wgt::Features::PIPELINE_CACHE)?;
5503 let data = if let Some((data, validation_key)) = desc
5504 .data
5505 .as_ref()
5506 .zip(self.raw().pipeline_cache_validation_key())
5507 {
5508 let data = pipeline_cache::validate_pipeline_cache(
5509 data,
5510 &self.adapter.raw.info,
5511 validation_key,
5512 );
5513 match data {
5514 Ok(data) => Some(data),
5515 Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
5516 Err(_) => None,
5518 }
5519 } else {
5520 None
5521 };
5522 let cache_desc = hal::PipelineCacheDescriptor {
5523 data,
5524 label: desc.label.to_hal(self.instance_flags),
5525 };
5526 let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
5527 Ok(raw) => raw,
5528 Err(e) => match e {
5529 hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
5530 },
5531 };
5532 let cache = pipeline::PipelineCache {
5533 device: self.clone(),
5534 label: desc.label.to_string(),
5535 raw: ResourceState::Valid(raw),
5537 };
5538
5539 let cache = Arc::new(cache);
5540
5541 Ok(cache)
5542 }
5543
5544 fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
5545 use wgt::TextureFormatFeatureFlags as tfsc;
5547 let mut format_features = self.adapter.get_texture_format_features(format);
5548 if (format == TextureFormat::R32Float
5549 || format == TextureFormat::Rg32Float
5550 || format == TextureFormat::Rgba32Float)
5551 && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
5552 {
5553 format_features.flags.set(tfsc::FILTERABLE, false);
5554 }
5555 format_features
5556 }
5557
5558 pub(crate) fn describe_format_features(
5559 &self,
5560 format: TextureFormat,
5561 ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
5562 self.require_features(format.required_features())?;
5563
5564 let using_device_features = self
5565 .features
5566 .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
5567 let downlevel = !self
5570 .downlevel
5571 .flags
5572 .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
5573
5574 if using_device_features || downlevel {
5575 Ok(self.get_texture_format_features(format))
5576 } else {
5577 Ok(format.guaranteed_format_features(self.features))
5578 }
5579 }
5580
5581 #[cfg(feature = "replay")]
5582 pub(crate) fn wait_for_submit(
5583 &self,
5584 submission_index: crate::SubmissionIndex,
5585 ) -> Result<(), DeviceError> {
5586 let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
5587 .map_err(|e| self.handle_hal_error(e))?;
5588 if last_done_index < submission_index {
5589 unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
5590 .map_err(|e| self.handle_hal_error(e))?;
5591 if let Some(queue) = self.get_queue() {
5592 let closures = queue.lock_life().triage_submissions(submission_index);
5593 assert!(
5594 closures.is_empty(),
5595 "wait_for_submit is not expected to work with closures"
5596 );
5597 }
5598 }
5599 Ok(())
5600 }
5601
5602 pub fn create_query_set(
5603 self: &Arc<Self>,
5604 desc: &resource::QuerySetDescriptor,
5605 ) -> Arc<QuerySet> {
5606 profiling::scope!("Device::create_query_set");
5607 let query_set = self.create_query_set_inner(desc).unwrap_or_else(|err| {
5608 self.handle_error(err, desc.label.as_deref(), "Device::create_query_set");
5609 QuerySet::invalid(Arc::clone(self), desc)
5610 });
5611 #[cfg(feature = "trace")]
5612 if let Some(ref mut trace) = *self.trace.lock() {
5613 use trace::IntoTrace;
5614 trace.add(trace::Action::CreateQuerySet {
5615 id: query_set.to_trace(),
5616 desc: desc.clone(),
5617 });
5618 }
5619 api_log!("Device::create_query_set -> {:?}", Arc::as_ptr(&query_set));
5620 query_set
5621 }
5622
5623 pub(crate) fn create_query_set_inner(
5624 self: &Arc<Self>,
5625 desc: &resource::QuerySetDescriptor,
5626 ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
5627 use resource::CreateQuerySetError as Error;
5628
5629 self.check_is_valid()?;
5630
5631 match desc.ty {
5632 wgt::QueryType::Occlusion => {}
5633 wgt::QueryType::Timestamp => {
5634 self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
5635 }
5636 wgt::QueryType::PipelineStatistics(..) => {
5637 self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
5638 }
5639 }
5640
5641 if desc.count == 0 {
5642 return Err(Error::ZeroCount);
5643 }
5644
5645 if desc.count > wgt::QUERY_SET_MAX_QUERIES {
5646 return Err(Error::TooManyQueries {
5647 count: desc.count,
5648 maximum: wgt::QUERY_SET_MAX_QUERIES,
5649 });
5650 }
5651
5652 let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
5653
5654 let raw = unsafe { self.raw().create_query_set(&hal_desc) }
5655 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
5656
5657 let query_set = QuerySet {
5658 state: ResourceState::Valid(QuerySetState {
5659 raw: Snatchable::new(raw),
5660 }),
5661 device: self.clone(),
5662 tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
5663 desc: desc.map_label(|l| l.to_string()),
5664 initialized_slots: Mutex::new(
5665 rank::QUERY_SET_INITIALIZED_SLOTS,
5666 bit_vec::BitVec::from_elem(desc.count as usize, false),
5667 ),
5668 };
5669
5670 let query_set = Arc::new(query_set);
5671
5672 Ok(query_set)
5673 }
5674
5675 pub(crate) fn lose(&self, message: &str) {
5676 self.valid.store(false, Ordering::Release);
5681
5682 if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
5684 device_lost_closure(DeviceLostReason::Unknown, message.to_string());
5685 }
5686
5687 }
5695
5696 fn release_gpu_resources(&self) {
5697 let trackers = self.trackers.lock();
5706 let buffers = trackers
5707 .buffers
5708 .used_resources()
5709 .flat_map(Weak::upgrade)
5710 .collect::<Vec<_>>();
5711 let textures = trackers
5712 .textures
5713 .used_resources()
5714 .flat_map(Weak::upgrade)
5715 .collect::<Vec<_>>();
5716 drop(trackers);
5717
5718 for buffer in buffers {
5719 buffer.destroy();
5720 }
5721 for texture in textures {
5722 texture.destroy();
5723 }
5724 }
5725
5726 pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
5727 UsageScope::new_pooled(
5728 &self.usage_scopes,
5729 &self.tracker_indices,
5730 self.ordered_buffer_usages,
5731 self.ordered_texture_usages,
5732 )
5733 }
5734
5735 pub fn set_device_lost_closure(&self, device_lost_closure: DeviceLostClosure) {
5737 self.device_lost_closure.lock().replace(device_lost_closure);
5738 }
5739
5740 pub fn destroy(self: &Arc<Self>) {
5741 api_log!("Device::destroy {:?}", Arc::as_ptr(self));
5742
5743 if !self.is_valid() {
5749 return;
5750 }
5751
5752 self.valid.store(false, Ordering::Release);
5760 }
5761
5762 pub fn get_internal_counters(&self) -> wgt::InternalCounters {
5763 wgt::InternalCounters {
5764 hal: self.get_hal_counters(),
5765 core: wgt::CoreCounters {},
5766 }
5767 }
5768
5769 pub fn get_hal_counters(&self) -> wgt::HalCounters {
5770 self.raw().get_internal_counters()
5771 }
5772
5773 pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
5774 self.raw().generate_allocator_report()
5775 }
5776}
5777
5778crate::impl_resource_type!(Device);
5779crate::impl_labeled!(Device);
5780crate::impl_storage_item!(Device);