Skip to main content

wgpu_core/device/
resource.rs

1use alloc::{
2    borrow::Cow,
3    boxed::Box,
4    string::{String, ToString as _},
5    sync::{Arc, Weak},
6    vec::Vec,
7};
8use core::{
9    fmt,
10    mem::{self, ManuallyDrop},
11    num::{NonZeroU32, NonZeroU64},
12};
13
14use arrayvec::ArrayVec;
15use bitflags::Flags;
16use scopeguard::{guard, ScopeGuard};
17use smallvec::SmallVec;
18use wgpu_sync::atomic::{AtomicBool, Ordering};
19use wgpu_sync::OnceCell;
20use wgt::{
21    error::WebGpuError, math::align_to, ColorWrites, DeviceLostReason, TextureFormat,
22    TextureSampleType, TextureViewDimension,
23};
24
25#[cfg(feature = "trace")]
26use crate::device::trace::{self, IntoTrace as _};
27use crate::{
28    api_log,
29    binding_model::{
30        self, BindGroup, BindGroupLateBufferBindingInfo, BindGroupLayout,
31        BindGroupLayoutEntryError, BindGroupLayoutState, BindGroupState, CreateBindGroupError,
32        CreateBindGroupLayoutError,
33    },
34    command, conv,
35    device::{
36        bgl, create_validator, life::WaitIdleError, map_buffer, stage_compiler::StageCompiler,
37        AttachmentData, BufferMapPendingClosure, DeviceLostInvocation, MapMode,
38        MissingDownlevelFlags, MissingFeatures, RenderPassContext,
39    },
40    error::ErrorSink,
41    hal_label,
42    init_tracker::{
43        BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
44        TextureInitTrackerAction,
45    },
46    instance::{Adapter, RequestDeviceError},
47    lock::{rank, Mutex, RwLock},
48    pipeline::{
49        self, shader_module_error_into_compilation_info, ColorStateError, PassthroughInterface,
50    },
51    pool::ResourcePool,
52    resource::{
53        self, Buffer, BufferState, ExternalTexture, ExternalTextureState, Labeled, ParentDevice,
54        QuerySet, QuerySetState, RawResourceAccess, ResourceState, Sampler, StagingBuffer, Texture,
55        TextureView, Tlas, TrackingData,
56    },
57    resource_log,
58    snatch::{SnatchGuard, SnatchLock, Snatchable},
59    timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
60    track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
61    validation::{self, check_color_attachment_count},
62    weak_vec::WeakVec,
63    FastHashMap, LabelHelpers,
64};
65
66use super::{
67    queue::Queue, DeviceDescriptor, DeviceError, DeviceLostClosure, UserClosures, ZERO_BUFFER_SIZE,
68};
69
70use wgpu_sync::atomic::AtomicU64;
71
72pub(crate) struct CommandIndices {
73    /// The index of the last command submission that was attempted.
74    ///
75    /// Note that `fence` may never be signalled with this value, if the command
76    /// submission failed. If you need to wait for everything running on a
77    /// `Queue` to complete, wait for [`last_successful_submission_index`].
78    ///
79    /// [`last_successful_submission_index`]: Device::last_successful_submission_index
80    pub(crate) active_submission_index: hal::FenceValue,
81    pub(crate) next_acceleration_structure_build_command_index: u64,
82}
83
84/// Parameters provided to shaders via a uniform buffer of the type
85/// [`NagaExternalTextureParams`], describing an [`ExternalTexture`] resource
86/// binding.
87///
88/// [`NagaExternalTextureParams`]: naga::SpecialTypes::external_texture_params
89/// [`ExternalTexture`]: binding_model::BindingResource::ExternalTexture
90//
91// This definition must match Naga's special type definition generated by
92// [`naga::Module::generate_external_texture_types`] and naga's
93// [`naga::common::ExternalTextureParameter`] enum.
94#[repr(C)]
95#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
96pub struct ExternalTextureParams {
97    /// 4x4 column-major matrix with which to convert sampled YCbCr values
98    /// to RGBA.
99    ///
100    /// This is ignored when `num_planes` is 1.
101    pub yuv_conversion_matrix: [f32; 16],
102
103    /// 3x3 column-major matrix to transform linear RGB values in the source
104    /// color space to linear RGB values in the destination color space. In
105    /// combination with [`Self::src_transfer_function`] and
106    /// [`Self::dst_transfer_function`] this can be used to ensure that
107    /// [`ImageSample`] and [`ImageLoad`] operations return values in the
108    /// desired destination color space rather than the source color space of
109    /// the underlying planes.
110    ///
111    /// Includes a padding element after each column.
112    ///
113    /// [`ImageSample`]: naga::ir::Expression::ImageSample
114    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
115    pub gamut_conversion_matrix: [f32; 12],
116
117    /// Transfer function for the source color space. The *inverse* of this
118    /// will be applied to decode non-linear RGB to linear RGB in the source
119    /// color space.
120    pub src_transfer_function: wgt::ExternalTextureTransferFunction,
121
122    /// Transfer function for the destination color space. This will be applied
123    /// to encode linear RGB to non-linear RGB in the destination color space.
124    pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
125
126    /// Transform to apply to [`ImageSample`] coordinates.
127    ///
128    /// This is a 3x2 column-major matrix representing an affine transform from
129    /// normalized texture coordinates to the normalized coordinates that should
130    /// be sampled from the external texture's underlying plane(s).
131    ///
132    /// This transform may scale, translate, flip, and rotate in 90-degree
133    /// increments, but the result of transforming the rectangle (0,0)..(1,1)
134    /// must be an axis-aligned rectangle that falls within the bounds of
135    /// (0,0)..(1,1).
136    ///
137    /// [`ImageSample`]: naga::ir::Expression::ImageSample
138    pub sample_transform: [f32; 6],
139
140    /// Transform to apply to [`ImageLoad`] coordinates.
141    ///
142    /// This is a 3x2 column-major matrix representing an affine transform from
143    /// non-normalized texel coordinates to the non-normalized coordinates of
144    /// the texel that should be loaded from the external texture's underlying
145    /// plane 0. For planes 1 and 2, if present, plane 0's coordinates are
146    /// scaled according to the textures' relative sizes.
147    ///
148    /// This transform may scale, translate, flip, and rotate in 90-degree
149    /// increments, but the result of transforming the rectangle (0,0)..[`size`]
150    /// must be an axis-aligned rectangle that falls within the bounds of
151    /// (0,0)..[`size`].
152    ///
153    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
154    /// [`size`]: Self::size
155    pub load_transform: [f32; 6],
156
157    /// Size of the external texture.
158    ///
159    /// This is the value that should be returned by size queries in shader
160    /// code; it does not necessarily match the dimensions of the underlying
161    /// texture(s). As a special case, if this is `[0, 0]`, the actual size of
162    /// plane 0 should be used instead.
163    ///
164    /// This must be consistent with [`sample_transform`]: it should be the size
165    /// in texels of the rectangle covered by the square (0,0)..(1,1) after
166    /// [`sample_transform`] has been applied to it.
167    ///
168    /// [`sample_transform`]: Self::sample_transform
169    pub size: [u32; 2],
170
171    /// Number of planes. 1 indicates a single RGBA plane. 2 indicates a Y
172    /// plane and an interleaved CbCr plane. 3 indicates separate Y, Cb, and Cr
173    /// planes.
174    pub num_planes: u32,
175    // Ensure the size of this struct matches the type generated by Naga.
176    pub _padding: [u8; 4],
177}
178
179const _: () =
180    assert!(size_of::<ExternalTextureParams>() == wgpu_hal::EXTERNAL_TEXTURE_PARAMS_SIZE as usize);
181const EXTERNAL_TEXTURE_PARAMS_BUFFER_SIZE: usize = {
182    size_of::<ExternalTextureParams>()
183        .next_multiple_of(wgpu_hal::UNIVERSAL_BUFFER_SIZE_ALIGNMENT as usize)
184};
185
186impl ExternalTextureParams {
187    pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
188        let gamut_conversion_matrix = [
189            desc.gamut_conversion_matrix[0],
190            desc.gamut_conversion_matrix[1],
191            desc.gamut_conversion_matrix[2],
192            0.0, // padding
193            desc.gamut_conversion_matrix[3],
194            desc.gamut_conversion_matrix[4],
195            desc.gamut_conversion_matrix[5],
196            0.0, // padding
197            desc.gamut_conversion_matrix[6],
198            desc.gamut_conversion_matrix[7],
199            desc.gamut_conversion_matrix[8],
200            0.0, // padding
201        ];
202
203        Self {
204            yuv_conversion_matrix: desc.yuv_conversion_matrix,
205            gamut_conversion_matrix,
206            src_transfer_function: desc.src_transfer_function,
207            dst_transfer_function: desc.dst_transfer_function,
208            size: [desc.width, desc.height],
209            sample_transform: desc.sample_transform,
210            load_transform: desc.load_transform,
211            num_planes: desc.num_planes() as u32,
212            _padding: Default::default(),
213        }
214    }
215}
216
217/// Because all operations are push/swap (no longlived lock),
218/// we can have mutex without lock rank
219pub(crate) struct DeferredBufferMapPendingClosures(wgpu_sync::Mutex<Vec<BufferMapPendingClosure>>);
220
221impl DeferredBufferMapPendingClosures {
222    pub(crate) fn new() -> Self {
223        Self(wgpu_sync::Mutex::new(Vec::new()))
224    }
225
226    pub(crate) fn push(&self, closure: BufferMapPendingClosure) {
227        self.0.lock().push(closure);
228    }
229
230    pub(crate) fn swap(&self, other: &mut Vec<BufferMapPendingClosure>) {
231        mem::swap(&mut *self.0.lock(), other)
232    }
233}
234
235/// Structure describing a logical device. Some members are internally mutable,
236/// stored behind mutexes.
237pub struct Device {
238    pub(crate) queue: OnceCell<Weak<Queue>>,
239    pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
240    pub(crate) empty_bgl: ManuallyDrop<Box<dyn hal::DynBindGroupLayout>>,
241    /// The `label` from the descriptor used to create the resource.
242    label: String,
243
244    pub(crate) command_allocator: command::CommandAllocator,
245
246    /// Holding this lock for writing also serializes the operations on the
247    /// device's [`hal::Queue`] that must be externally synchronized:
248    /// submission, presentation, waiting for idle, and surface configuration.
249    pub(crate) command_indices: RwLock<CommandIndices>,
250
251    /// The index of the last successful submission to this device's
252    /// [`hal::Queue`].
253    ///
254    /// Unlike [`active_submission_index`], which is incremented each time
255    /// submission is attempted, this is updated only when submission succeeds,
256    /// so waiting for this value won't hang waiting for work that was never
257    /// submitted.
258    ///
259    /// [`active_submission_index`]: CommandIndices::active_submission_index
260    pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
261
262    pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
263    pub(crate) snatchable_lock: SnatchLock,
264
265    /// Is this device valid? Valid is closely associated with "lose the device",
266    /// which can be triggered by various methods, including at the end of device
267    /// destroy, and by any GPU errors that cause us to no longer trust the state
268    /// of the device. Ideally we would like to fold valid into the storage of
269    /// the device itself (for example as an Error enum), but unfortunately we
270    /// need to continue to be able to retrieve the device in poll_devices to
271    /// determine if it can be dropped. If our internal accesses of devices were
272    /// done through ref-counted references and external accesses checked for
273    /// Error enums, we wouldn't need this. For now, we need it. All the call
274    /// sites where we check it are areas that should be revisited if we start
275    /// using ref-counted references for internal access.
276    pub(crate) valid: AtomicBool,
277
278    /// Closure to be called on "lose the device". This is invoked directly by
279    /// device.lose or by the UserCallbacks returned from maintain when the device
280    /// has been destroyed and its queues are empty.
281    pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
282
283    pub(crate) error_sink: ErrorSink,
284
285    /// Stores the state of buffers and textures.
286    pub(crate) trackers: Mutex<DeviceTracker>,
287    pub(crate) tracker_indices: TrackerIndexAllocators,
288    /// Pool of bind group layouts, allowing deduplication.
289    pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
290    pub(crate) alignments: hal::Alignments,
291    pub(crate) limits: wgt::Limits,
292    pub(crate) features: wgt::Features,
293    pub(crate) downlevel: wgt::DownlevelCapabilities,
294    /// Buffer uses listed here, are expected to be ordered by the underlying hardware.
295    /// If a usage is ordered, then if the buffer state doesn't change between draw calls,
296    /// there are no barriers needed for synchronization.
297    /// See the implementations of [`hal::Adapter::get_ordered_buffer_usages`] for hardware specific info
298    pub(crate) ordered_buffer_usages: wgt::BufferUses,
299    /// Texture uses listed here, are expected to be ordered by the underlying hardware.
300    /// If a usage is ordered, then if the buffer state doesn't change between draw calls,
301    /// there are no barriers needed for synchronization.
302    /// See the implementations of [`hal::Adapter::get_ordered_texture_usages`] for hardware specific info
303    pub(crate) ordered_texture_usages: wgt::TextureUses,
304    pub(crate) instance_flags: wgt::InstanceFlags,
305    pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
306    /// This closures were created in [`Buffer::drop`] where we do not run them to prevent locking problems.
307    pub(crate) deferred_buffer_map_pending_closures: DeferredBufferMapPendingClosures,
308    pub(crate) usage_scopes: UsageScopePool,
309    pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
310    // Optional so that we can late-initialize this after the queue is created.
311    pub(crate) timestamp_normalizer: OnceCell<crate::timestamp_normalization::TimestampNormalizer>,
312    /// Uniform buffer containing [`ExternalTextureParams`] with values such
313    /// that a [`TextureView`] bound to a [`wgt::BindingType::ExternalTexture`]
314    /// binding point will be rendered correctly. Intended to be used as the
315    /// [`hal::ExternalTextureBinding::params`] field.
316    pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
317
318    // Drop order matters!
319    //
320    //  - Any member whose drop might destroy hal resources needs to be dropped
321    //    before the device. Most hal resources are handled manually in
322    //    `Device::drop`, but this applies to `command_allocator`.
323    //  - The device must be dropped before the adapter.
324    //  - Any member whose drop might write into the trace must be dropped
325    //    before the trace.
326    raw: Box<dyn hal::DynDevice>,
327    pub(crate) adapter: Arc<Adapter>,
328    #[cfg(feature = "trace")]
329    pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
330}
331
332#[cfg(send_sync)]
333const _: () = {
334    // SAFETY: Bounds checked below
335    unsafe impl Send for Device {}
336    // SAFETY: Bounds checked below
337    unsafe impl Sync for Device {}
338
339    fn _device_fields_are_send_sync(obj: &Device) {
340        fn _check_bound<T: Send + Sync>(_: &T) {}
341        let Device {
342            queue,
343            zero_buffer,
344            empty_bgl,
345            label,
346            command_allocator,
347            command_indices,
348            last_successful_submission_index,
349            fence,
350            snatchable_lock,
351            valid,
352            device_lost_closure,
353            error_sink,
354            trackers,
355            tracker_indices,
356            bgl_pool,
357            alignments,
358            limits,
359            features,
360            downlevel,
361            ordered_buffer_usages,
362            ordered_texture_usages,
363            instance_flags,
364            deferred_destroy,
365            deferred_buffer_map_pending_closures,
366            usage_scopes,
367            indirect_validation,
368            timestamp_normalizer,
369            default_external_texture_params_buffer,
370            raw,
371            adapter,
372            #[cfg(feature = "trace")]
373            trace,
374        } = obj;
375        _check_bound(queue);
376        _check_bound(zero_buffer);
377        _check_bound(empty_bgl);
378        _check_bound(label);
379        _check_bound(command_allocator);
380        _check_bound(command_indices);
381        _check_bound(last_successful_submission_index);
382        _check_bound(fence);
383        _check_bound(snatchable_lock);
384        _check_bound(valid);
385        _check_bound(device_lost_closure);
386        _check_bound(error_sink);
387        _check_bound(trackers);
388        _check_bound(tracker_indices);
389        _check_bound(bgl_pool);
390        _check_bound(alignments);
391        _check_bound(limits);
392        _check_bound(features);
393        _check_bound(downlevel);
394        _check_bound(ordered_buffer_usages);
395        _check_bound(ordered_texture_usages);
396        _check_bound(instance_flags);
397        _check_bound(deferred_destroy);
398        _check_bound(deferred_buffer_map_pending_closures);
399        _check_bound(usage_scopes);
400        _check_bound(indirect_validation);
401        _check_bound(timestamp_normalizer);
402        _check_bound(default_external_texture_params_buffer);
403        _check_bound(raw);
404        _check_bound(adapter);
405        #[cfg(feature = "trace")]
406        _check_bound(trace);
407    }
408};
409
410pub(crate) enum DeferredDestroy {
411    TextureViews(WeakVec<TextureView>),
412    BindGroups(WeakVec<BindGroup>),
413}
414
415impl fmt::Debug for Device {
416    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417        f.debug_struct("Device")
418            .field("label", &self.label())
419            .field("limits", &self.limits)
420            .field("features", &self.features)
421            .field("downlevel", &self.downlevel)
422            .finish()
423    }
424}
425
426impl Drop for Device {
427    #[allow(trivial_casts)]
428    fn drop(&mut self) {
429        profiling::scope!("Device::drop");
430        api_log!("Device::drop {:?}", self as *const _);
431        resource_log!("Drop {}", self.error_ident());
432
433        // SAFETY: We are in the Drop impl and we don't use self.zero_buffer anymore after this
434        // point.
435        let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
436        // SAFETY: We are in the Drop impl and we don't use self.empty_bgl anymore after this point.
437        let empty_bgl = unsafe { ManuallyDrop::take(&mut self.empty_bgl) };
438        // SAFETY: We are in the Drop impl and we don't use
439        // self.default_external_texture_params_buffer anymore after this point.
440        let default_external_texture_params_buffer =
441            unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
442        // SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point.
443        let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
444        if let Some(indirect_validation) = self.indirect_validation.take() {
445            indirect_validation.dispose(self.raw.as_ref());
446        }
447        if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
448            timestamp_normalizer.dispose(self.raw.as_ref());
449        }
450        unsafe {
451            self.raw.destroy_buffer(zero_buffer);
452            self.raw.destroy_bind_group_layout(empty_bgl);
453            self.raw
454                .destroy_buffer(default_external_texture_params_buffer);
455            self.raw.destroy_fence(fence);
456        }
457    }
458}
459
460impl Device {
461    pub fn features(&self) -> &wgt::Features {
462        &self.features
463    }
464
465    pub fn limits(&self) -> &wgt::Limits {
466        &self.limits
467    }
468
469    pub fn downlevel(&self) -> &wgt::DownlevelCapabilities {
470        &self.downlevel
471    }
472
473    pub fn adapter_info(&self) -> wgt::AdapterInfo {
474        self.adapter.get_info()
475    }
476}
477
478impl Device {
479    pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
480        self.raw.as_ref()
481    }
482
483    pub fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
484        if self.features.contains(feature) {
485            Ok(())
486        } else {
487            Err(MissingFeatures(feature))
488        }
489    }
490
491    pub(crate) fn require_downlevel_flags(
492        &self,
493        flags: wgt::DownlevelFlags,
494    ) -> Result<(), MissingDownlevelFlags> {
495        if self.downlevel.flags.contains(flags) {
496            Ok(())
497        } else {
498            Err(MissingDownlevelFlags(flags))
499        }
500    }
501
502    /// # Safety
503    ///
504    /// - See [wgpu::Device::start_graphics_debugger_capture][api] for details the safety.
505    ///
506    /// [api]: ../../wgpu/struct.Device.html#method.start_graphics_debugger_capture
507    pub unsafe fn start_graphics_debugger_capture(&self) {
508        api_log!("Device::start_graphics_debugger_capture");
509
510        if !self.is_valid() {
511            return;
512        }
513        unsafe { self.raw().start_graphics_debugger_capture() };
514    }
515
516    /// # Safety
517    ///
518    /// - See [wgpu::Device::stop_graphics_debugger_capture][api] for details the safety.
519    ///
520    /// [api]: ../../wgpu/struct.Device.html#method.stop_graphics_debugger_capture
521    pub unsafe fn stop_graphics_debugger_capture(&self) {
522        api_log!("Device::stop_graphics_debugger_capture");
523
524        if !self.is_valid() {
525            return;
526        }
527        unsafe { self.raw().stop_graphics_debugger_capture() };
528    }
529}
530
531impl Device {
532    pub(crate) fn new(
533        raw_device: Box<dyn hal::DynDevice>,
534        adapter: &Arc<Adapter>,
535        desc: &DeviceDescriptor,
536        instance_flags: wgt::InstanceFlags,
537    ) -> Result<Self, DeviceError> {
538        #[cfg(not(feature = "trace"))]
539        match &desc.trace {
540            wgt::Trace::Off => {}
541            _ => {
542                log::error!("wgpu-core feature 'trace' is not enabled");
543            }
544        };
545        #[cfg(feature = "trace")]
546        let trace: Option<Box<dyn trace::Trace + Send + Sync + 'static>> = match &desc.trace {
547            wgt::Trace::Off => None,
548            wgt::Trace::Directory(dir) => match trace::DiskTrace::new(dir.clone()) {
549                Ok(mut trace) => {
550                    trace::Trace::add(
551                        &mut trace,
552                        trace::Action::Init {
553                            desc: wgt::DeviceDescriptor {
554                                trace: wgt::Trace::Off,
555                                ..desc.clone()
556                            },
557                            backend: adapter.backend(),
558                        },
559                    );
560                    Some(Box::new(trace))
561                }
562                Err(e) => {
563                    log::error!("Unable to start a trace in '{dir:?}': {e}");
564                    None
565                }
566            },
567            wgt::Trace::Memory => {
568                let mut trace = trace::MemoryTrace::new();
569                trace::Trace::add(
570                    &mut trace,
571                    trace::Action::Init {
572                        desc: wgt::DeviceDescriptor {
573                            trace: wgt::Trace::Off,
574                            ..desc.clone()
575                        },
576                        backend: adapter.backend(),
577                    },
578                );
579                Some(Box::new(trace))
580            }
581            // The enum is non_exhaustive, so we must have a fallback arm (that should be
582            // unreachable in practice).
583            t => {
584                log::error!("unimplemented wgpu_types::Trace variant {t:?}");
585                None
586            }
587        };
588
589        let ordered_buffer_usages = adapter.raw.adapter.get_ordered_buffer_usages();
590        let ordered_texture_usages = adapter.raw.adapter.get_ordered_texture_usages();
591
592        // Resources requiring explicit destruction ahead of the device are wrapped in
593        // `ScopeGuard`s, which are defused just before the `Device` takes ownership of the
594        // resources.
595        let raw = raw_device.as_ref();
596
597        let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
598        let fence = guard(fence, |fence| unsafe { raw.destroy_fence(fence) });
599
600        let command_allocator = command::CommandAllocator::new();
601
602        let rt_uses = if desc
603            .required_features
604            .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
605        {
606            wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
607        } else {
608            wgt::BufferUses::empty()
609        };
610
611        // Create zeroed buffer used for texture clears (and raytracing if required).
612        let zero_buffer = unsafe {
613            raw_device.create_buffer(&hal::BufferDescriptor {
614                label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
615                size: ZERO_BUFFER_SIZE,
616                usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
617                memory_flags: hal::MemoryFlags::empty(),
618            })
619        }
620        .map_err(DeviceError::from_hal)?
621        .0;
622        let zero_buffer = guard(zero_buffer, |buffer| unsafe { raw.destroy_buffer(buffer) });
623
624        let empty_bgl = unsafe {
625            raw_device.create_bind_group_layout(&hal::BindGroupLayoutDescriptor {
626                label: None,
627                flags: hal::BindGroupLayoutFlags::empty(),
628                entries: &[],
629            })
630        }
631        .map_err(DeviceError::from_hal)?;
632        let empty_bgl = guard(empty_bgl, |bgl| unsafe {
633            raw.destroy_bind_group_layout(bgl)
634        });
635
636        let default_external_texture_params_buffer = unsafe {
637            raw_device.create_buffer(&hal::BufferDescriptor {
638                label: hal_label(
639                    Some("(wgpu internal) default external texture params buffer"),
640                    instance_flags,
641                ),
642                size: EXTERNAL_TEXTURE_PARAMS_BUFFER_SIZE as _,
643                usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
644                memory_flags: hal::MemoryFlags::empty(),
645            })
646        }
647        .map_err(DeviceError::from_hal)?
648        .0;
649        let default_external_texture_params_buffer =
650            guard(default_external_texture_params_buffer, |buffer| unsafe {
651                raw.destroy_buffer(buffer)
652            });
653
654        // Cloned as we need them below anyway.
655        let alignments = adapter.raw.capabilities.alignments.clone();
656        let downlevel = adapter.raw.capabilities.downlevel.clone();
657        let limits = &adapter.raw.capabilities.limits;
658
659        let enable_indirect_validation = instance_flags
660            .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
661            && downlevel.flags.contains(
662                wgt::DownlevelFlags::INDIRECT_EXECUTION | wgt::DownlevelFlags::COMPUTE_SHADERS,
663            )
664            && limits.max_storage_buffers_per_shader_stage >= 2;
665
666        let indirect_validation = if enable_indirect_validation {
667            let indirect_validation = crate::indirect_validation::IndirectValidation::new(
668                raw,
669                &desc.required_limits,
670                &desc.required_features,
671                instance_flags,
672                adapter.backend(),
673            )?;
674            Some(guard(indirect_validation, |indirect_validation| {
675                indirect_validation.dispose(raw)
676            }))
677        } else {
678            None
679        };
680
681        // Error returns after we start consuming guards could bypass resource cleanup.
682        #[deny(clippy::question_mark_used)]
683        {
684            let zero_buffer = ScopeGuard::into_inner(zero_buffer);
685            let empty_bgl = ScopeGuard::into_inner(empty_bgl);
686            let default_external_texture_params_buffer =
687                ScopeGuard::into_inner(default_external_texture_params_buffer);
688            let fence = ScopeGuard::into_inner(fence);
689            let indirect_validation = indirect_validation.map(ScopeGuard::into_inner);
690
691            Ok(Self {
692                raw: raw_device,
693                adapter: adapter.clone(),
694                queue: OnceCell::new(),
695                zero_buffer: ManuallyDrop::new(zero_buffer),
696                empty_bgl: ManuallyDrop::new(empty_bgl),
697                default_external_texture_params_buffer: ManuallyDrop::new(
698                    default_external_texture_params_buffer,
699                ),
700                label: desc.label.to_string(),
701                command_allocator,
702                command_indices: RwLock::new(
703                    rank::DEVICE_COMMAND_INDICES,
704                    CommandIndices {
705                        active_submission_index: 0,
706                        // By starting at one, we can put the result in a NonZeroU64.
707                        next_acceleration_structure_build_command_index: 1,
708                    },
709                ),
710                last_successful_submission_index: AtomicU64::new(0),
711                fence: ManuallyDrop::new(fence),
712                snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
713                valid: AtomicBool::new(true),
714                device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
715                error_sink: ErrorSink::new(),
716                trackers: Mutex::new(
717                    rank::DEVICE_TRACKERS,
718                    DeviceTracker::new(ordered_buffer_usages, ordered_texture_usages),
719                ),
720                tracker_indices: TrackerIndexAllocators::new(),
721                bgl_pool: ResourcePool::new(),
722                #[cfg(feature = "trace")]
723                trace: Mutex::new(rank::DEVICE_TRACE, trace),
724                alignments,
725                limits: desc.required_limits.clone(),
726                features: desc.required_features,
727                downlevel,
728                ordered_buffer_usages,
729                ordered_texture_usages,
730                instance_flags,
731                deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
732                usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
733                timestamp_normalizer: OnceCell::new(),
734                indirect_validation,
735                deferred_buffer_map_pending_closures: DeferredBufferMapPendingClosures::new(),
736            })
737        }
738    }
739
740    /// Initializes [`Device::default_external_texture_params_buffer`] with
741    /// required values such that a [`TextureView`] bound to a
742    /// [`wgt::BindingType::ExternalTexture`] binding point will be rendered
743    /// correctly.
744    fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
745        let data = ExternalTextureParams {
746            #[rustfmt::skip]
747            yuv_conversion_matrix: [
748                1.0, 0.0, 0.0, 0.0,
749                0.0, 1.0, 0.0, 0.0,
750                0.0, 0.0, 1.0, 0.0,
751                0.0, 0.0, 0.0, 1.0,
752            ],
753            #[rustfmt::skip]
754            gamut_conversion_matrix: [
755                1.0, 0.0, 0.0, /* padding */ 0.0,
756                0.0, 1.0, 0.0, /* padding */ 0.0,
757                0.0, 0.0, 1.0, /* padding */ 0.0,
758            ],
759            src_transfer_function: Default::default(),
760            dst_transfer_function: Default::default(),
761            size: [0, 0],
762            #[rustfmt::skip]
763            sample_transform: [
764                1.0, 0.0,
765                0.0, 1.0,
766                0.0, 0.0
767            ],
768            #[rustfmt::skip]
769            load_transform: [
770                1.0, 0.0,
771                0.0, 1.0,
772                0.0, 0.0
773            ],
774            num_planes: 1,
775            _padding: Default::default(),
776        };
777        let mut staging_buffer = StagingBuffer::new(
778            self,
779            wgt::BufferSize::new(EXTERNAL_TEXTURE_PARAMS_BUFFER_SIZE as _).unwrap(),
780        )?;
781        staging_buffer.write_with_zero_padding(bytemuck::bytes_of(&data));
782        let staging_buffer = staging_buffer.flush();
783
784        let params_buffer = self.default_external_texture_params_buffer.as_ref();
785        let queue = self.get_queue().unwrap();
786        let mut pending_writes = queue.pending_writes.lock();
787
788        unsafe {
789            pending_writes
790                .command_encoder
791                .transition_buffers(&[hal::BufferBarrier {
792                    buffer: params_buffer,
793                    usage: hal::StateTransition {
794                        from: wgt::BufferUses::MAP_WRITE,
795                        to: wgt::BufferUses::COPY_DST,
796                    },
797                }]);
798            pending_writes.command_encoder.copy_buffer_to_buffer(
799                staging_buffer.raw(),
800                params_buffer,
801                &[hal::BufferCopy {
802                    src_offset: 0,
803                    dst_offset: 0,
804                    size: staging_buffer.size,
805                }],
806            );
807            pending_writes.consume(staging_buffer);
808            pending_writes
809                .command_encoder
810                .transition_buffers(&[hal::BufferBarrier {
811                    buffer: params_buffer,
812                    usage: hal::StateTransition {
813                        from: wgt::BufferUses::COPY_DST,
814                        to: wgt::BufferUses::UNIFORM,
815                    },
816                }]);
817        }
818
819        Ok(())
820    }
821
822    pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
823        let queue = self.get_queue().unwrap();
824
825        let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
826            self,
827            queue.get_raw_timestamp_period(),
828        )?;
829
830        self.timestamp_normalizer
831            .set(timestamp_normalizer)
832            .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
833
834        self.init_default_external_texture_params_buffer()?;
835
836        Ok(())
837    }
838
839    /// Returns the backend this device is using.
840    pub fn backend(&self) -> wgt::Backend {
841        self.adapter.backend()
842    }
843
844    /// Returns a reference to the adapter this device was created from.
845    pub fn adapter(&self) -> &Arc<Adapter> {
846        &self.adapter
847    }
848
849    /// Return true if `self` is still valid.
850    ///
851    /// Note that a `false` result here does *not* guarantee that no further activity will
852    /// occur on the `Device`. A [`Device`] can be marked invalid at any time, even while
853    /// locks are held, meaning that operations begun before the `Device` was marked
854    /// invalid will still generally run to completion. This is a consequence of
855    /// making [`Device::valid`] an `AtomicBool`. It could be avoided by making
856    /// [`Device::valid`] an ordinary `bool` field protected by the same locks that
857    /// all other operations acquire, but we use an `AtomicBool` because there
858    /// are many device validity checks and we want those to be cheap.
859    pub fn is_valid(&self) -> bool {
860        self.valid.load(Ordering::Acquire)
861    }
862
863    /// Return `Err(DeviceError)` if `self` is not valid. Otherwise, return `Ok(())`.
864    ///
865    /// Note that an `Err` result here does *not* guarantee that no further activity will
866    /// occur on the `Device`. See the documentation for [`is_valid`][Self::is_valid] for
867    /// details.
868    pub fn check_is_valid(&self) -> Result<(), DeviceError> {
869        if self.is_valid() {
870            Ok(())
871        } else {
872            Err(DeviceError::Lost)
873        }
874    }
875
876    /// Stop tracing and return the trace object.
877    ///
878    /// This is mostly useful for in-memory traces.
879    #[cfg(feature = "trace")]
880    pub fn take_trace(&self) -> Option<Box<dyn trace::Trace + Send + Sync + 'static>> {
881        self.trace.lock().take()
882    }
883
884    /// Checks that we are operating within the memory budget reported by the native APIs.
885    ///
886    /// If we are not, the device gets invalidated.
887    ///
888    /// The budget might fluctuate over the lifetime of the application, so it should be checked
889    /// somewhat frequently.
890    pub fn lose_if_oom(&self) {
891        let _ = self
892            .raw()
893            .check_if_oom()
894            .map_err(|e| self.handle_hal_error(e));
895    }
896
897    pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
898        match error {
899            hal::DeviceError::OutOfMemory
900            | hal::DeviceError::Lost
901            | hal::DeviceError::Unexpected => {
902                self.lose(&error.to_string());
903            }
904        }
905        DeviceError::from_hal(error)
906    }
907
908    pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
909        match error {
910            hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
911            error => self.handle_hal_error(error),
912        }
913    }
914
915    /// Run some destroy operations that were deferred.
916    ///
917    /// Destroying the resources requires taking a write lock on the device's snatch lock,
918    /// so a good reason for deferring resource destruction is when we don't know for sure
919    /// how risky it is to take the lock (typically, it shouldn't be taken from the drop
920    /// implementation of a reference-counted structure).
921    /// The snatch lock must not be held while this function is called.
922    pub(crate) fn deferred_resource_destruction(&self) {
923        // Note that the deferred_destroy list may contain duplicate entries.
924        let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
925        for item in deferred_destroy {
926            match item {
927                DeferredDestroy::TextureViews(views) => {
928                    for view in views {
929                        let Some(view) = view.upgrade() else {
930                            continue;
931                        };
932                        let Ok(view_state) = view.state() else {
933                            continue;
934                        };
935                        let Some(raw_view) =
936                            view_state.raw.snatch(&mut self.snatchable_lock.write())
937                        else {
938                            continue;
939                        };
940
941                        resource_log!("Destroy raw {}", view.error_ident());
942
943                        unsafe {
944                            self.raw().destroy_texture_view(raw_view);
945                        }
946                    }
947                }
948                DeferredDestroy::BindGroups(bind_groups) => {
949                    for bind_group in bind_groups {
950                        let Some(bind_group) = bind_group.upgrade() else {
951                            continue;
952                        };
953                        let Ok(bind_group_state) = bind_group.state() else {
954                            continue;
955                        };
956                        let Some(raw_bind_group) = bind_group_state
957                            .raw
958                            .snatch(&mut self.snatchable_lock.write())
959                        else {
960                            continue;
961                        };
962
963                        resource_log!("Destroy raw {}", bind_group.error_ident());
964
965                        unsafe {
966                            self.raw().destroy_bind_group(raw_bind_group);
967                        }
968                    }
969                }
970            }
971        }
972    }
973
974    pub fn get_queue(&self) -> Option<Arc<Queue>> {
975        self.queue.get().as_ref()?.upgrade()
976    }
977
978    pub fn set_queue(&self, queue: &Arc<Queue>) {
979        assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
980    }
981
982    /// Check device for freeable resources and completed buffer mappings.
983    pub fn poll(
984        &self,
985        poll_type: wgt::PollType<crate::SubmissionIndex>,
986    ) -> Result<wgt::PollStatus, WaitIdleError> {
987        api_log!("Device::poll {poll_type:?}");
988        let (user_closures, result) = self.poll_and_return_closures(poll_type);
989        user_closures.fire();
990        result
991    }
992
993    /// Poll the device, returning any `UserClosures` that need to be executed.
994    ///
995    /// The caller must invoke the `UserClosures` even if this function returns
996    /// an error. This is an internal helper, used by [`Device::poll`] and
997    /// [`Instance::poll_all_devices`], so that `poll_all_devices` can invoke
998    /// closures once after all devices have been polled.
999    ///
1000    /// [`Instance::poll_all_devices`]: crate::instance::Instance::poll_all_devices
1001    pub(crate) fn poll_and_return_closures(
1002        &self,
1003        poll_type: wgt::PollType<crate::SubmissionIndex>,
1004    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
1005        let snatch_guard = self.snatchable_lock.read();
1006        let maintain_result = self.maintain(poll_type, snatch_guard);
1007
1008        self.lose_if_oom();
1009
1010        // Some deferred destroys are scheduled in maintain so run this right after
1011        // to avoid holding on to them until the next device poll.
1012        self.deferred_resource_destruction();
1013
1014        maintain_result
1015    }
1016
1017    /// Check the current status of the GPU and process any submissions that have
1018    /// finished.
1019    ///
1020    /// The `poll_type` argument tells if this function should wait for a particular
1021    /// submission index to complete, or if it should just poll the current status.
1022    ///
1023    /// This will process _all_ completed submissions, even if the caller only asked
1024    /// us to poll to a given submission index.
1025    ///
1026    /// The returned [`UserClosures`] contains callbacks that need to be invoked informing
1027    /// the user about various things occurring. These happen and should be handled even
1028    /// if this function returns an error, hence they are outside of the result.
1029    pub(crate) fn maintain<'this>(
1030        &'this self,
1031        poll_type: wgt::PollType<crate::SubmissionIndex>,
1032        snatch_guard: SnatchGuard,
1033    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
1034        profiling::scope!("Device::maintain");
1035
1036        let mut user_closures = UserClosures::default();
1037
1038        self.deferred_buffer_map_pending_closures
1039            .swap(&mut user_closures.mappings);
1040
1041        // If a wait was requested, determine which submission index to wait for.
1042        let wait_submission_index = match poll_type {
1043            wgt::PollType::Wait {
1044                submission_index: Some(submission_index),
1045                ..
1046            } => {
1047                let last_successful_submission_index = self
1048                    .last_successful_submission_index
1049                    .load(Ordering::Acquire);
1050
1051                if submission_index > last_successful_submission_index {
1052                    let result = Err(WaitIdleError::WrongSubmissionIndex(
1053                        submission_index,
1054                        last_successful_submission_index,
1055                    ));
1056
1057                    return (user_closures, result);
1058                }
1059
1060                Some(submission_index)
1061            }
1062            wgt::PollType::Wait {
1063                submission_index: None,
1064                ..
1065            } => Some(
1066                self.last_successful_submission_index
1067                    .load(Ordering::Acquire),
1068            ),
1069            wgt::PollType::Poll => None,
1070        };
1071
1072        // If a target submission index was specified, wait for it, and set
1073        // `wait_succeeded` to `Some(bool)` indicating success or timeout. If
1074        // no target was specified, set `wait_succeeded` to `None`.
1075        let wait_succeeded = if let Some(target_submission_index) = wait_submission_index {
1076            log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
1077
1078            let wait_timeout = match poll_type {
1079                wgt::PollType::Wait { timeout, .. } => timeout,
1080                wgt::PollType::Poll => unreachable!(
1081                    "`wait_submission_index` index for poll type `Poll` should be None"
1082                ),
1083            };
1084
1085            let wait_result = unsafe {
1086                self.raw()
1087                    .wait(self.fence.as_ref(), target_submission_index, wait_timeout)
1088            };
1089
1090            match wait_result {
1091                Ok(succeeded) => Some(succeeded),
1092                Err(e) => {
1093                    let hal_error: WaitIdleError = self.handle_hal_error(e).into();
1094                    return (user_closures, Err(hal_error));
1095                }
1096            }
1097        } else {
1098            None
1099        };
1100
1101        // Get the currently finished submission index. This may be higher than the requested
1102        // wait, or it may be less than the requested wait if the wait failed.
1103        let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
1104        let current_finished_submission = match fence_value_result {
1105            Ok(fence_value) => fence_value,
1106            Err(e) => {
1107                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
1108                return (user_closures, Err(hal_error));
1109            }
1110        };
1111
1112        // When a device is marked invalid, we must destroy all its hal resources once its
1113        // queue is empty, by calling `release_gpu_resources`.
1114        //
1115        // However, checking device validity for this purpose is tricky. A device can be
1116        // marked invalid at any time, regardless of what locks are held. This means that,
1117        // even though queue submission does check device validity at the start (in
1118        // `Queue::allocate_submission`), the submission will proceed even if the
1119        // device gets marked invalid after that check. Thus, other threads can observe
1120        // the queue becoming non-empty even after they have observed the device to be
1121        // invalid.
1122        //
1123        // In this function, for `release_gpu_resources` to work as intended, we must be
1124        // sure that no further work can be submitted. Specifically, we must be sure that
1125        // if we see the device marked invalid, then so will any subsequent attempts to
1126        // submit work to the queue, causing them to fail. To accomplish this, it suffices
1127        // to hold the same lock while we call `is_valid` that `Queue::allocate_submission`
1128        // holds while it does the submission.
1129        let device_valid = {
1130            let _command_indices_guard = self.command_indices.read();
1131            self.is_valid()
1132        };
1133
1134        // Maintain all finished submissions on the queue, updating the relevant user closures and
1135        // collecting if the queue is empty.
1136        //
1137        // We don't use the result of the wait here, as we want to progress forward as far as
1138        // possible and the wait could have been for submissions that finished long ago.
1139        let mut queue_empty = false;
1140        if let Some(queue) = self.get_queue() {
1141            let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
1142            (
1143                user_closures.submissions,
1144                user_closures.mappings,
1145                user_closures.blas_compact_ready,
1146                queue_empty,
1147            ) = queue_result;
1148            // DEADLOCK PREVENTION: We must drop `snatch_guard` before `queue` goes out of scope.
1149            //
1150            // `Queue::drop` acquires the snatch guard. If we still hold it when `queue` is dropped
1151            // at the end of this block, we would deadlock. This can happen in the following
1152            // scenario:
1153            //
1154            // - Thread A calls `Device::maintain` while Thread B holds the last strong ref to the
1155            //   queue.
1156            // - Thread A calls `self.get_queue()`, obtaining a new strong ref, and enters this
1157            //   branch.
1158            // - Thread B drops its strong ref, making Thread A's ref the last one.
1159            // - When `queue` goes out of scope here, `Queue::drop` runs and tries to acquire the
1160            //   snatch guard — but Thread A (this thread) still holds it, causing a deadlock.
1161            drop(snatch_guard);
1162        } else {
1163            drop(snatch_guard);
1164        };
1165
1166        // Based on the queue empty status, and the current finished submission index, determine
1167        // the result of the poll.
1168        //
1169        // After a successful wait, `current_finished_submission` should match or exceed
1170        // the target. But after a timeout, more work may have finished before
1171        // `current_finished_submission` is read, so it could be on either side of the
1172        // target, and the queue can also become empty before `Queue::maintain`
1173        // computes `queue_empty`.
1174        //
1175        // We report as accurately as we can with the information available. In
1176        // particular, when our wait timed out but `queue_empty` comes back `true`, we
1177        // report `WaitSucceeded` if `current_finished_submission` reached the target,
1178        // and `Timeout` if it didn't. We don't want to risk reporting `QueueEmpty`
1179        // without actually having seen that the target submission is retired.
1180        let result = if queue_empty && wait_succeeded != Some(false) {
1181            if let Some(wait_submission_index) = wait_submission_index {
1182                // Sanity-check that we don't report `QueueEmpty` without
1183                // reaching the target submission index.
1184                assert!(
1185                    current_finished_submission >= wait_submission_index,
1186                    concat!(
1187                        "If the queue is empty, the current submission index ",
1188                        "({}) should be at least the wait submission index ({})",
1189                    ),
1190                    current_finished_submission,
1191                    wait_submission_index,
1192                );
1193            } else {
1194                // We didn't wait (passive poll), safe to report `QueueEmpty`.
1195            }
1196
1197            Ok(wgt::PollStatus::QueueEmpty)
1198        } else if let Some(wait_submission_index) = wait_submission_index {
1199            // This is theoretically possible to succeed more than checking on the poll result
1200            // as submissions could have finished in the time between the timeout resolving,
1201            // the thread getting scheduled again, and us checking the fence value.
1202            if current_finished_submission >= wait_submission_index {
1203                Ok(wgt::PollStatus::WaitSucceeded)
1204            } else {
1205                Err(WaitIdleError::Timeout)
1206            }
1207        } else {
1208            Ok(wgt::PollStatus::Poll)
1209        };
1210
1211        // Detect if we have been destroyed and now need to lose the device.
1212        //
1213        // If we are invalid (set at start of destroy) and our queue is empty,
1214        // and we have a DeviceLostClosure, return the closure to be called by
1215        // our caller. This will complete the steps for both destroy and for
1216        // "lose the device".
1217        let mut should_release_gpu_resource = false;
1218        if !device_valid && queue_empty {
1219            // We can release gpu resources associated with this device (but not
1220            // while holding the life_tracker lock).
1221            should_release_gpu_resource = true;
1222
1223            // If we have a DeviceLostClosure, build an invocation with the
1224            // reason DeviceLostReason::Destroyed and no message.
1225            if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
1226                user_closures
1227                    .device_lost_invocations
1228                    .push(DeviceLostInvocation {
1229                        closure: device_lost_closure,
1230                        reason: DeviceLostReason::Destroyed,
1231                        message: String::new(),
1232                    });
1233            }
1234        }
1235
1236        if should_release_gpu_resource {
1237            self.release_gpu_resources();
1238        }
1239
1240        (user_closures, result)
1241    }
1242
1243    pub fn create_buffer_inner(
1244        self: &Arc<Self>,
1245        desc: &resource::BufferDescriptor,
1246    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1247        self.check_is_valid()?;
1248
1249        if desc.size > self.limits.max_buffer_size {
1250            return Err(resource::CreateBufferError::MaxBufferSize {
1251                requested: desc.size,
1252                maximum: self.limits.max_buffer_size,
1253            });
1254        }
1255
1256        if desc
1257            .usage
1258            .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
1259        {
1260            self.require_acceleration_structures()?;
1261        }
1262
1263        if desc.usage.contains(wgt::BufferUsages::INDEX)
1264            && desc.usage.contains(
1265                wgt::BufferUsages::VERTEX
1266                    | wgt::BufferUsages::UNIFORM
1267                    | wgt::BufferUsages::INDIRECT
1268                    | wgt::BufferUsages::STORAGE,
1269            )
1270        {
1271            self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
1272        }
1273
1274        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1275            return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
1276        }
1277
1278        if !self
1279            .features
1280            .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
1281        {
1282            use wgt::BufferUsages as Bu;
1283            let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
1284                && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
1285            let read_mismatch = desc.usage.contains(Bu::MAP_READ)
1286                && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
1287            if write_mismatch || read_mismatch {
1288                return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
1289            }
1290        }
1291
1292        let mut usage = conv::map_buffer_usage(desc.usage);
1293
1294        if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
1295            self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
1296            // We are going to be reading from it, internally;
1297            // when validating the content of the buffer
1298            usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
1299        }
1300
1301        if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
1302            usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
1303        }
1304
1305        if desc.mapped_at_creation {
1306            if !desc.size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1307                return Err(resource::CreateBufferError::UnalignedSize);
1308            }
1309            if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1310                // we are going to be copying into it, internally
1311                usage |= wgt::BufferUses::COPY_DST;
1312            }
1313        } else {
1314            // We are required to zero out (initialize) all memory. This is done
1315            // on demand using clear_buffer which requires write transfer usage!
1316            usage |= wgt::BufferUses::COPY_DST;
1317        }
1318
1319        // The great thing about buffer sizes is there are so many to choose from!
1320        //  - The application may request an arbitrary byte-aligned size.
1321        //  - If the buffer supports usage as a vertex or index buffer, we must ensure there
1322        //    is a naturally aligned block of 4B at the end of the buffer. Zero-size bindings
1323        //    are not supported by hal and are redirected to this block by
1324        //    `resolve_vertex_or_index_binding_range`.
1325        //  - The overall buffer size must be a non-zero multiple of COPY_BUFFER_ALIGNMENT (4).
1326        //    In some cases additional platform-dependent requirements may apply. The actual
1327        //    allocated size of the buffer is returned by hal along with the buffer object from
1328        //    `create_buffer`. `wgpu-core` is responsible for ensuring that any region of the
1329        //    buffer, including hal-added padding, is initialized before it may be accessed.
1330        //
1331        // Because buffer writes for initialization operate at multiples of COPY_BUFFER_ALIGNMENT,
1332        // and because initialization tracking will never determine that it is necessary to
1333        // initialize a region outside the application-visible range, round the tracked
1334        // initialization range down to a multiple of COPY_BUFFER_ALIGNMENT (tail_start), and
1335        // eagerly zero-initialize from there to the buffer's true end.
1336
1337        let padded_size = if desc.size == 0 {
1338            wgt::COPY_BUFFER_ALIGNMENT
1339        } else if desc
1340            .usage
1341            .intersects(wgt::BufferUsages::VERTEX | wgt::BufferUsages::INDEX)
1342        {
1343            // We require 4B of padding with 4B alignment. See above.
1344            align_to(desc.size, wgt::COPY_BUFFER_ALIGNMENT) + wgt::COPY_BUFFER_ALIGNMENT
1345        } else {
1346            align_to(desc.size, wgt::COPY_BUFFER_ALIGNMENT)
1347        };
1348        let tail_start = desc.size & !(wgt::COPY_BUFFER_ALIGNMENT - 1);
1349
1350        let hal_desc = hal::BufferDescriptor {
1351            label: desc.label.to_hal(self.instance_flags),
1352            size: padded_size,
1353            usage,
1354            memory_flags: hal::MemoryFlags::empty(),
1355        };
1356        let (buffer, final_size) = unsafe { self.raw().create_buffer(&hal_desc) }
1357            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1358
1359        debug_assert!(final_size >= padded_size);
1360
1361        let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1362            // SAFETY: The size passed here must be 4B aligned, >= the application size, and
1363            // <= the buffer size.
1364            self.timestamp_normalizer
1365                .get()
1366                .unwrap()
1367                .create_normalization_bind_group(
1368                    self,
1369                    &*buffer,
1370                    desc.label.as_deref(),
1371                    wgt::BufferSize::new(hal_desc.size).unwrap(),
1372                    desc.usage,
1373                )
1374        }?);
1375
1376        let indirect_validation_bind_groups =
1377            self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
1378
1379        let buffer = Buffer {
1380            state: ResourceState::Valid(BufferState {
1381                raw: Snatchable::new(buffer),
1382            }),
1383            device: self.clone(),
1384            usage: desc.usage,
1385            size: desc.size,
1386            initialization_status: RwLock::new(
1387                rank::BUFFER_INITIALIZATION_STATUS,
1388                BufferInitTracker::new(tail_start),
1389            ),
1390            map_state: Arc::new(RwLock::new(
1391                rank::BUFFER_MAP_STATE,
1392                resource::BufferMapState::Idle,
1393            )),
1394            label: desc.label.to_string(),
1395            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1396            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1397            timestamp_normalization_bind_group,
1398            indirect_validation_bind_groups,
1399        };
1400
1401        let buffer = Arc::new(buffer);
1402
1403        let init_buffer_tail = |buffer, snatch_guard| {
1404            let mapping = map_buffer(
1405                buffer,
1406                tail_start,
1407                final_size - tail_start,
1408                MapMode::Write,
1409                snatch_guard,
1410            )?;
1411            let raw = buffer
1412                .raw(snatch_guard)
1413                .expect("newly-created buffer cannot be destroyed");
1414            unsafe {
1415                // SAFETY: The buffer tail is valid and aligned.
1416                mapping
1417                    .ptr
1418                    .write_bytes(0, (final_size - tail_start) as usize);
1419                if !mapping.is_coherent {
1420                    #[allow(clippy::single_range_in_vec_init)]
1421                    self.raw()
1422                        .flush_mapped_ranges(raw, &[tail_start..final_size]);
1423                }
1424                self.raw().unmap_buffer(raw);
1425            }
1426            Ok::<_, resource::CreateBufferError>(())
1427        };
1428
1429        let buffer_use = if !desc.mapped_at_creation {
1430            if tail_start == final_size {
1431                // No tail init necessary
1432            } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1433                // Tail init by mapping
1434                let snatch_guard = self.snatchable_lock.read();
1435                init_buffer_tail(&buffer, &snatch_guard)?;
1436            } else if let Some(queue) = self.get_queue() {
1437                // Schedule tail init in pending writes
1438                let snatch_guard = self.snatchable_lock.read();
1439                let mut pending_writes = queue.pending_writes.lock();
1440                self.trackers
1441                    .lock()
1442                    .buffers
1443                    .insert_single(&buffer, wgt::BufferUses::COPY_DST);
1444                pending_writes.clear_buffer(
1445                    self,
1446                    &buffer,
1447                    tail_start..final_size,
1448                    &snatch_guard,
1449                )?;
1450                return Ok(buffer);
1451            } else {
1452                // Queue is gone, buffer will not be used.
1453            }
1454            wgt::BufferUses::empty()
1455        } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1456            // Mapped-at-creation with MAP_WRITE. Tail init by mapping, then map the
1457            // application-visible portion of the buffer. Don't do both in the same
1458            // mapping, because the application shouldn't see the tail.
1459            let snatch_guard = self.snatchable_lock.read();
1460            if tail_start != final_size {
1461                init_buffer_tail(&buffer, &snatch_guard)?;
1462            }
1463            let map_size = buffer.size;
1464            let mapping = if map_size == 0 {
1465                hal::BufferMapping {
1466                    ptr: core::ptr::NonNull::dangling(),
1467                    is_coherent: true,
1468                }
1469            } else {
1470                map_buffer(&buffer, 0, map_size, MapMode::Write, &snatch_guard)?
1471            };
1472            drop(snatch_guard);
1473            *buffer.map_state.write() = resource::BufferMapState::Active {
1474                mapping,
1475                range: 0..map_size,
1476                host: MapMode::Write,
1477            };
1478            wgt::BufferUses::MAP_WRITE
1479        } else {
1480            // Mapped-at-creation without MAP_WRITE. Create and zero-initialize a staging
1481            // buffer for the entire buffer (including tail). It is okay to use a single
1482            // mapping, `get_mapped_range` will still only expose the application-visible
1483            // range. The application could corrupt the tail with an out-of-bounds write,
1484            // which may cause problems for its shaders if they read out-of-bounds, but
1485            // won't cause problems in wgpu.
1486            let mut staging_buffer =
1487                StagingBuffer::new(self, wgt::BufferSize::new(final_size).unwrap())?;
1488
1489            // Zero initialize memory and then mark the buffer as initialized
1490            // (it's guaranteed that this is the case by the time the buffer is usable)
1491            staging_buffer.write_zeros();
1492            buffer.initialization_status.write().drain(0..final_size);
1493
1494            *buffer.map_state.write() = resource::BufferMapState::Init { staging_buffer };
1495            wgt::BufferUses::COPY_DST
1496        };
1497
1498        // If we didn't add the buffer to the tracker already for initialization via
1499        // PendingWrites, do so now.
1500
1501        self.trackers
1502            .lock()
1503            .buffers
1504            .insert_single(&buffer, buffer_use);
1505
1506        Ok(buffer)
1507    }
1508
1509    pub fn create_buffer(self: &Arc<Self>, desc: &resource::BufferDescriptor) -> Arc<Buffer> {
1510        profiling::scope!("Device::create_buffer");
1511
1512        let buffer = self.create_buffer_inner(desc).unwrap_or_else(|err| {
1513            self.handle_error(err, desc.label.as_deref(), "Device::create_buffer");
1514            Buffer::invalid(Arc::clone(self), desc)
1515        });
1516        #[cfg(feature = "trace")]
1517        if let Some(ref mut trace) = *self.trace.lock() {
1518            use trace::IntoTrace;
1519            let mut desc = desc.clone();
1520            let mapped_at_creation = mem::replace(&mut desc.mapped_at_creation, false);
1521            if mapped_at_creation && !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1522                desc.usage |= wgt::BufferUsages::COPY_DST;
1523            }
1524            trace.add(trace::Action::CreateBuffer(buffer.to_trace(), desc));
1525        }
1526        api_log!(
1527            "Device::create_buffer({:?}{}) -> {:?}",
1528            desc.label.as_deref().unwrap_or(""),
1529            if desc.mapped_at_creation {
1530                ", mapped_at_creation"
1531            } else {
1532                ""
1533            },
1534            Arc::as_ptr(&buffer)
1535        );
1536        buffer
1537    }
1538
1539    #[cfg(feature = "replay")]
1540    pub fn set_buffer_data(
1541        self: &Arc<Self>,
1542        buffer: &Arc<Buffer>,
1543        offset: wgt::BufferAddress,
1544        data: &[u8],
1545        mode: MapMode,
1546    ) -> resource::BufferAccessResult {
1547        use crate::resource::RawResourceAccess;
1548
1549        let device = &buffer.device;
1550
1551        device.check_is_valid()?;
1552        buffer.check_usage(wgt::BufferUsages::MAP_WRITE)?;
1553
1554        let last_submission = device
1555            .get_queue()
1556            .and_then(|queue| queue.lock_life().get_buffer_latest_submission_index(buffer));
1557
1558        if let Some(last_submission) = last_submission {
1559            device.wait_for_submit(last_submission)?;
1560        }
1561
1562        let snatch_guard = device.snatchable_lock.read();
1563        let raw_buf = buffer.try_raw(&snatch_guard)?;
1564
1565        if offset > buffer.size {
1566            return Err(resource::BufferAccessError::OutOfBoundsStartOffsetOverrun {
1567                index: offset,
1568                max: buffer.size,
1569            });
1570        } else if buffer.size - offset < u64::try_from(data.len()).unwrap() {
1571            return Err(resource::BufferAccessError::OutOfBoundsEndOffsetOverrun {
1572                index: offset,
1573                size: u64::try_from(data.len()).unwrap(),
1574                max: buffer.size,
1575            });
1576        }
1577
1578        let mapping = unsafe {
1579            device
1580                .raw()
1581                .map_buffer(raw_buf, offset..offset + u64::try_from(data.len()).unwrap())
1582        }
1583        .map_err(|e| device.handle_hal_error(e))?;
1584
1585        unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()) };
1586
1587        if !mapping.is_coherent {
1588            #[allow(clippy::single_range_in_vec_init)]
1589            unsafe {
1590                device
1591                    .raw()
1592                    .flush_mapped_ranges(raw_buf, &[offset..offset + data.len() as u64])
1593            };
1594        }
1595
1596        unsafe { device.raw().unmap_buffer(raw_buf) };
1597
1598        Ok(mode)
1599    }
1600
1601    /// # Safety
1602    ///
1603    /// - `hal_texture` must be created from `device_id` corresponding raw handle.
1604    /// - `hal_texture` must be created respecting `desc`
1605    /// - `hal_texture` must be initialized
1606    /// - The `initial_state` must match the actual driver-side state of
1607    ///   the wrapped resource at the moment of wrap.
1608    pub unsafe fn create_texture_from_hal(
1609        self: &Arc<Self>,
1610        hal_texture: Box<dyn hal::DynTexture>,
1611        desc: &resource::TextureDescriptor,
1612        initial_state: wgt::TextureUses,
1613        cleared: bool,
1614    ) -> (Arc<Texture>, Option<resource::CreateTextureError>) {
1615        profiling::scope!("Device::create_texture_from_hal");
1616
1617        let (texture, error) =
1618            match self.create_texture_from_hal_inner(hal_texture, desc, initial_state, cleared) {
1619                Ok(texture) => (texture, None),
1620                Err(e) => (Texture::invalid(self, desc), Some(e)),
1621            };
1622
1623        // NB: Any change done through the raw texture handle will not be
1624        // recorded in the replay
1625        #[cfg(feature = "trace")]
1626        if let Some(ref mut trace) = *self.trace.lock() {
1627            trace.add(trace::Action::CreateTexture(
1628                texture.to_trace(),
1629                desc.clone(),
1630            ));
1631        }
1632
1633        api_log!(
1634            "Device::create_texture({desc:?}) -> {:?}",
1635            Arc::as_ptr(&texture)
1636        );
1637
1638        (texture, error)
1639    }
1640
1641    pub(crate) fn create_texture_from_hal_inner(
1642        self: &Arc<Self>,
1643        hal_texture: Box<dyn hal::DynTexture>,
1644        desc: &resource::TextureDescriptor,
1645        initial_state: wgt::TextureUses,
1646        cleared: bool,
1647    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1648        // Count the raw texture before validation so the error paths below can
1649        // hand it back through `destroy_texture`. Merely dropping a hal texture
1650        // does not release what it wraps — e.g. an imported WebGL handle would
1651        // stay registered in glow's resource tracker forever.
1652        unsafe { self.raw().add_raw_texture(&*hal_texture) };
1653
1654        if let Err(error) = self.check_is_valid() {
1655            unsafe { self.raw().destroy_texture(hal_texture) };
1656            return Err(error.into());
1657        }
1658
1659        let format_features = match self.describe_format_features(desc.format) {
1660            Ok(format_features) => format_features,
1661            Err(error) => {
1662                unsafe { self.raw().destroy_texture(hal_texture) };
1663                return Err(resource::CreateTextureError::MissingFeatures(
1664                    desc.format,
1665                    error,
1666                ));
1667            }
1668        };
1669
1670        let texture = Texture::new(
1671            self,
1672            resource::TextureInner::Native { raw: hal_texture },
1673            conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1674            desc,
1675            format_features,
1676            resource::TextureClearMode::None,
1677            !cleared, // inverted so it marks the tracker properly
1678            self.resolve_texture_binding_view_dimension(desc),
1679        );
1680
1681        let texture = Arc::new(texture);
1682
1683        self.trackers
1684            .lock()
1685            .textures
1686            .insert_single(&texture, initial_state);
1687
1688        Ok(texture)
1689    }
1690
1691    /// # Safety
1692    ///
1693    /// - `hal_buffer` must have been created on this device.
1694    /// - `hal_buffer` must have been created respecting `desc` (in particular, the size,
1695    ///   which must always be a multiple of 4, and may be subject to other requirements).
1696    /// - `hal_buffer` must be initialized.
1697    /// - `hal_buffer` must not have zero size.
1698    pub unsafe fn create_buffer_from_hal(
1699        self: &Arc<Self>,
1700        hal_buffer: Box<dyn hal::DynBuffer>,
1701        desc: &resource::BufferDescriptor,
1702    ) -> (Arc<Buffer>, Option<resource::CreateBufferError>) {
1703        profiling::scope!("Device::create_buffer");
1704        let (buffer, error) = match unsafe { self.create_buffer_from_hal_inner(hal_buffer, desc) } {
1705            Ok(buffer) => (buffer, None),
1706            Err(e) => (Buffer::invalid(Arc::clone(self), desc), Some(e)),
1707        };
1708
1709        // NB: Any change done through the raw buffer handle will not be
1710        // recorded in the replay
1711        #[cfg(feature = "trace")]
1712        if let Some(trace) = self.trace.lock().as_mut() {
1713            use trace::IntoTrace;
1714            trace.add(trace::Action::CreateBuffer(buffer.to_trace(), desc.clone()));
1715        }
1716        api_log!("Device::create_buffer -> {:?}", Arc::as_ptr(&buffer));
1717        (buffer, error)
1718    }
1719
1720    /// # Safety
1721    ///
1722    /// Same safety requirements as `create_buffer_from_hal`.
1723    pub(crate) unsafe fn create_buffer_from_hal_inner(
1724        self: &Arc<Self>,
1725        hal_buffer: Box<dyn hal::DynBuffer>,
1726        desc: &resource::BufferDescriptor,
1727    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1728        let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1729            // SAFETY: The size passed here must be 4B aligned, >= the application size, and
1730            // <= the buffer size.
1731            self.timestamp_normalizer
1732                .get()
1733                .unwrap()
1734                .create_normalization_bind_group(
1735                    self,
1736                    &*hal_buffer,
1737                    desc.label.as_deref(),
1738                    wgt::BufferSize::new(desc.size).unwrap(),
1739                    desc.usage,
1740                )?
1741        });
1742
1743        let indirect_validation_bind_groups = self.create_indirect_validation_bind_groups(
1744            hal_buffer.as_ref(),
1745            desc.size,
1746            desc.usage,
1747        )?;
1748
1749        unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1750
1751        let buffer = Buffer {
1752            state: ResourceState::Valid(BufferState {
1753                raw: Snatchable::new(hal_buffer),
1754            }),
1755            device: self.clone(),
1756            usage: desc.usage,
1757            size: desc.size,
1758            initialization_status: RwLock::new(
1759                rank::BUFFER_INITIALIZATION_STATUS,
1760                BufferInitTracker::new(0),
1761            ),
1762            map_state: Arc::new(RwLock::new(
1763                rank::BUFFER_MAP_STATE,
1764                resource::BufferMapState::Idle,
1765            )),
1766            label: desc.label.to_string(),
1767            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1768            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1769            timestamp_normalization_bind_group,
1770            indirect_validation_bind_groups,
1771        };
1772
1773        let buffer = Arc::new(buffer);
1774
1775        self.trackers
1776            .lock()
1777            .buffers
1778            .insert_single(&buffer, wgt::BufferUses::empty());
1779
1780        Ok(buffer)
1781    }
1782
1783    /// If needed, creates the bind groups for indirect validation shaders to read
1784    /// from `raw_buffer`.
1785    ///
1786    /// `buffer_size` is the user-requested size of the buffer.
1787    fn create_indirect_validation_bind_groups(
1788        &self,
1789        raw_buffer: &dyn hal::DynBuffer,
1790        buffer_size: u64,
1791        usage: wgt::BufferUsages,
1792    ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1793    {
1794        if !usage.contains(wgt::BufferUsages::INDIRECT) {
1795            return Ok(Snatchable::empty());
1796        }
1797
1798        let Some(ref indirect_validation) = self.indirect_validation else {
1799            return Ok(Snatchable::empty());
1800        };
1801
1802        let bind_groups = crate::indirect_validation::BindGroups::new(
1803            indirect_validation,
1804            self,
1805            buffer_size,
1806            raw_buffer,
1807        )
1808        .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1809
1810        if let Some(bind_groups) = bind_groups {
1811            Ok(Snatchable::new(bind_groups))
1812        } else {
1813            Ok(Snatchable::empty())
1814        }
1815    }
1816
1817    /// Validate a texture descriptor.
1818    ///
1819    /// This applies the same validation as [`Self::create_texture`], without
1820    /// actually creating a texture.
1821    pub fn validate_texture_descriptor(
1822        self: &Arc<Self>,
1823        desc: &resource::TextureDescriptor,
1824    ) -> Result<(), resource::CreateTextureError> {
1825        self.validate_texture_descriptor_inner(desc)?;
1826        Ok(())
1827    }
1828
1829    /// Validate a texture descriptor.
1830    ///
1831    /// Implements the [validating GPUTextureDescriptor] algorithm, with some
1832    /// `wgpu` extensions.
1833    ///
1834    /// [validating GPUTextureDescriptor]: https://www.w3.org/TR/webgpu/#abstract-opdef-validating-gputexturedescriptor
1835    fn validate_texture_descriptor_inner(
1836        self: &Arc<Self>,
1837        desc: &resource::TextureDescriptor,
1838    ) -> Result<(wgt::TextureFormatFeatures, Vec<TextureFormat>), resource::CreateTextureError>
1839    {
1840        use resource::{
1841            CreateTextureError, TextureBindingViewDimensionError, TextureDimensionError,
1842        };
1843
1844        self.check_is_valid()?;
1845
1846        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1847            return Err(CreateTextureError::InvalidUsage(desc.usage));
1848        }
1849
1850        conv::check_texture_dimension_size(
1851            desc.dimension,
1852            desc.size,
1853            desc.sample_count,
1854            &self.limits,
1855        )?;
1856
1857        if desc.dimension != wgt::TextureDimension::D2 {
1858            // Depth textures can only be 2D
1859            if desc.format.is_depth_stencil_format() {
1860                return Err(CreateTextureError::InvalidDepthDimension(
1861                    desc.dimension,
1862                    desc.format,
1863                ));
1864            }
1865            // Transient textures can only be 2D
1866            if desc
1867                .usage
1868                .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1869            {
1870                return Err(CreateTextureError::InvalidDimensionUsages(
1871                    wgt::TextureUsages::TRANSIENT_ATTACHMENT,
1872                    desc.dimension,
1873                ));
1874            }
1875        }
1876
1877        if desc.dimension != wgt::TextureDimension::D2
1878            && desc.dimension != wgt::TextureDimension::D3
1879        {
1880            // Compressed textures can only be 2D or 3D
1881            if desc.format.is_compressed() {
1882                return Err(CreateTextureError::InvalidCompressedDimension(
1883                    desc.dimension,
1884                    desc.format,
1885                ));
1886            }
1887
1888            // Renderable textures can only be 2D or 3D
1889            if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1890                return Err(CreateTextureError::InvalidDimensionUsages(
1891                    wgt::TextureUsages::RENDER_ATTACHMENT,
1892                    desc.dimension,
1893                ));
1894            }
1895        }
1896
1897        if desc.format.is_compressed() {
1898            let (block_width, block_height) = desc.format.block_dimensions();
1899
1900            if !desc.size.width.is_multiple_of(block_width) {
1901                return Err(CreateTextureError::InvalidDimension(
1902                    TextureDimensionError::NotMultipleOfBlockWidth {
1903                        width: desc.size.width,
1904                        block_width,
1905                        format: desc.format,
1906                    },
1907                ));
1908            }
1909
1910            if !desc.size.height.is_multiple_of(block_height) {
1911                return Err(CreateTextureError::InvalidDimension(
1912                    TextureDimensionError::NotMultipleOfBlockHeight {
1913                        height: desc.size.height,
1914                        block_height,
1915                        format: desc.format,
1916                    },
1917                ));
1918            }
1919
1920            if desc.dimension == wgt::TextureDimension::D3 {
1921                // Only BCn formats with Sliced 3D feature can be used for 3D textures
1922                if desc.format.is_bcn() {
1923                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1924                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1925                } else if desc.format.is_astc() {
1926                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1927                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1928                } else {
1929                    return Err(CreateTextureError::InvalidCompressedDimension(
1930                        desc.dimension,
1931                        desc.format,
1932                    ));
1933                }
1934            }
1935        }
1936
1937        let mips = desc.mip_level_count;
1938        let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1939        if mips == 0 || mips > max_levels_allowed {
1940            return Err(CreateTextureError::InvalidMipLevelCount {
1941                requested: mips,
1942                maximum: max_levels_allowed,
1943            });
1944        }
1945
1946        {
1947            let (mut width_multiple, mut height_multiple) = desc.format.size_multiple_requirement();
1948
1949            if desc.format.is_multi_planar_format() {
1950                // TODO(https://github.com/gfx-rs/wgpu/issues/8491): fix
1951                // `mip_level_size` calculation for these formats and relax this
1952                // restriction.
1953                width_multiple <<= desc.mip_level_count.saturating_sub(1);
1954                height_multiple <<= desc.mip_level_count.saturating_sub(1);
1955            }
1956
1957            if !desc.size.width.is_multiple_of(width_multiple) {
1958                return Err(CreateTextureError::InvalidDimension(
1959                    TextureDimensionError::WidthNotMultipleOf {
1960                        width: desc.size.width,
1961                        multiple: width_multiple,
1962                        format: desc.format,
1963                    },
1964                ));
1965            }
1966
1967            if !desc.size.height.is_multiple_of(height_multiple) {
1968                return Err(CreateTextureError::InvalidDimension(
1969                    TextureDimensionError::HeightNotMultipleOf {
1970                        height: desc.size.height,
1971                        multiple: height_multiple,
1972                        format: desc.format,
1973                    },
1974                ));
1975            }
1976        }
1977
1978        if desc
1979            .usage
1980            .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1981        {
1982            if desc.usage
1983                != (wgt::TextureUsages::TRANSIENT_ATTACHMENT
1984                    | wgt::TextureUsages::RENDER_ATTACHMENT)
1985            {
1986                return Err(CreateTextureError::InvalidTransientTextureUsage(desc.usage));
1987            }
1988
1989            if desc.mip_level_count != 1 {
1990                return Err(CreateTextureError::InvalidTransientTextureMipLevelCount(
1991                    desc.mip_level_count,
1992                ));
1993            }
1994
1995            if desc.size.depth_or_array_layers != 1 {
1996                return Err(CreateTextureError::InvalidTransientTextureLayerCount(
1997                    desc.size.depth_or_array_layers,
1998                ));
1999            }
2000
2001            if !desc.view_formats.is_empty() {
2002                return Err(CreateTextureError::InvalidTransientTextureViewFormats);
2003            }
2004        }
2005
2006        let format_features = self
2007            .describe_format_features(desc.format)
2008            .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
2009
2010        if let Some(planes) = desc.format.planes() {
2011            for plane in 0..planes {
2012                let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
2013                let format = desc.format.aspect_specific_format(aspect).unwrap();
2014                self.require_features(format.required_features())
2015                    .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
2016            }
2017        }
2018
2019        if desc.sample_count > 1 {
2020            // <https://www.w3.org/TR/2025/CRD-webgpu-20251120/#:~:text=If%20descriptor%2EsampleCount%20%3E%201>
2021            //
2022            // Note that there are also some checks related to the sample count
2023            // in [`conv::check_texture_dimension_size`].
2024
2025            if desc.mip_level_count != 1 {
2026                return Err(CreateTextureError::InvalidMipLevelCount {
2027                    requested: desc.mip_level_count,
2028                    maximum: 1,
2029                });
2030            }
2031
2032            if desc.size.depth_or_array_layers != 1
2033                && !self.features.contains(wgt::Features::MULTISAMPLE_ARRAY)
2034            {
2035                return Err(CreateTextureError::InvalidDimension(
2036                    TextureDimensionError::MultisampledDepthOrArrayLayer(
2037                        desc.size.depth_or_array_layers,
2038                    ),
2039                ));
2040            }
2041
2042            if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
2043                return Err(CreateTextureError::InvalidMultisampledStorageBinding);
2044            }
2045
2046            if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
2047                return Err(CreateTextureError::MultisampledNotRenderAttachment);
2048            }
2049
2050            if !format_features.flags.intersects(
2051                wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
2052                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
2053                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
2054                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
2055            ) {
2056                return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
2057            }
2058
2059            if !format_features
2060                .flags
2061                .sample_count_supported(desc.sample_count)
2062            {
2063                return Err(CreateTextureError::InvalidSampleCount(
2064                    desc.sample_count,
2065                    desc.format,
2066                    desc.format
2067                        .guaranteed_format_features(self.features)
2068                        .flags
2069                        .supported_sample_counts(),
2070                    self.adapter
2071                        .get_texture_format_features(desc.format)
2072                        .flags
2073                        .supported_sample_counts(),
2074                ));
2075            };
2076        }
2077
2078        let missing_allowed_usages = desc.usage - format_features.allowed_usages;
2079        if !missing_allowed_usages.is_empty() {
2080            // detect downlevel incompatibilities
2081            let wgpu_allowed_usages = desc
2082                .format
2083                .guaranteed_format_features(self.features)
2084                .allowed_usages;
2085            let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
2086            return Err(CreateTextureError::InvalidFormatUsages(
2087                missing_allowed_usages,
2088                desc.format,
2089                wgpu_missing_usages.is_empty(),
2090            ));
2091        }
2092
2093        let mut hal_view_formats = Vec::new();
2094        for format in desc.view_formats.iter() {
2095            self.require_features(format.required_features())
2096                .map_err(|error| CreateTextureError::MissingFeatures(*format, error))?;
2097            if desc.format == *format {
2098                continue;
2099            }
2100            if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
2101                return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
2102            }
2103            hal_view_formats.push(*format);
2104        }
2105        if !hal_view_formats.is_empty() {
2106            self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
2107        }
2108
2109        if !self
2110            .downlevel
2111            .flags
2112            .contains(wgt::DownlevelFlags::ARBITRARY_BINDING_VIEW_DIMENSIONS)
2113        {
2114            if let Some(texture_binding_view_dimension) = desc.texture_binding_view_dimension {
2115                match texture_binding_view_dimension {
2116                    TextureViewDimension::D2 => {
2117                        if desc.size.depth_or_array_layers != 1 {
2118                            return Err(CreateTextureError::InvalidTextureBindingViewDimension(
2119                                TextureBindingViewDimensionError::InvalidLayerCount {
2120                                    dimension: texture_binding_view_dimension,
2121                                    required: 1,
2122                                    actual: desc.size.depth_or_array_layers,
2123                                },
2124                            ));
2125                        }
2126                    }
2127                    TextureViewDimension::Cube => {
2128                        if desc.size.depth_or_array_layers != 6 {
2129                            return Err(CreateTextureError::InvalidTextureBindingViewDimension(
2130                                TextureBindingViewDimensionError::InvalidLayerCount {
2131                                    dimension: texture_binding_view_dimension,
2132                                    required: 6,
2133                                    actual: desc.size.depth_or_array_layers,
2134                                },
2135                            ));
2136                        }
2137                    }
2138                    TextureViewDimension::CubeArray => {
2139                        self.require_downlevel_flags(wgt::DownlevelFlags::CUBE_ARRAY_TEXTURES)?;
2140                        if !desc.size.depth_or_array_layers.is_multiple_of(6) {
2141                            return Err(CreateTextureError::InvalidTextureBindingViewDimension(
2142                                TextureBindingViewDimensionError::InvalidCubeArrayLayerCount(
2143                                    desc.size.depth_or_array_layers,
2144                                ),
2145                            ));
2146                        }
2147                    }
2148                    _ => {}
2149                }
2150            }
2151        }
2152
2153        Ok((format_features, hal_view_formats))
2154    }
2155
2156    fn resolve_texture_binding_view_dimension(
2157        &self,
2158        desc: &resource::TextureDescriptor,
2159    ) -> Option<TextureViewDimension> {
2160        if self
2161            .downlevel
2162            .flags
2163            .contains(wgt::DownlevelFlags::ARBITRARY_BINDING_VIEW_DIMENSIONS)
2164        {
2165            None
2166        } else {
2167            Some(
2168                desc.texture_binding_view_dimension
2169                    .unwrap_or_else(|| desc.default_view_dimension()),
2170            )
2171        }
2172    }
2173
2174    fn create_texture_inner(
2175        self: &Arc<Self>,
2176        desc: &resource::TextureDescriptor,
2177    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
2178        let (format_features, hal_view_formats) = self.validate_texture_descriptor_inner(desc)?;
2179
2180        let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
2181
2182        let texture_binding_view_dimension = self.resolve_texture_binding_view_dimension(desc);
2183
2184        let hal_desc = hal::TextureDescriptor {
2185            label: desc.label.to_hal(self.instance_flags),
2186            size: desc.size,
2187            mip_level_count: desc.mip_level_count,
2188            sample_count: desc.sample_count,
2189            dimension: desc.dimension,
2190            format: desc.format,
2191            usage: hal_usage,
2192            memory_flags: hal::MemoryFlags::empty(),
2193            view_formats: hal_view_formats,
2194            texture_binding_view_dimension,
2195        };
2196
2197        let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
2198            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2199
2200        let clear_mode = if hal_usage
2201            .contains(wgt::TextureUses::DEPTH_WRITE | wgt::TextureUses::STENCIL_WRITE)
2202            || hal_usage.contains(wgt::TextureUses::COLOR_TARGET)
2203                && desc.dimension == wgt::TextureDimension::D2
2204        {
2205            let (is_color, usage) = if desc.format.is_depth_stencil_format() {
2206                (
2207                    false,
2208                    wgt::TextureUses::DEPTH_WRITE | wgt::TextureUses::STENCIL_WRITE,
2209                )
2210            } else {
2211                (true, wgt::TextureUses::COLOR_TARGET)
2212            };
2213
2214            let clear_label = hal_label(
2215                Some("(wgpu internal) clear texture view"),
2216                self.instance_flags,
2217            );
2218
2219            let clear_views = resource::ClearViews::new(
2220                clear_label,
2221                usage,
2222                desc.format,
2223                desc.mip_level_count,
2224                desc.size.depth_or_array_layers,
2225                |desc| {
2226                    unsafe { self.raw().create_texture_view(raw_texture.as_ref(), desc) }
2227                        .map_err(|e| self.handle_hal_error(e))
2228                },
2229            )?;
2230            resource::TextureClearMode::RenderPass {
2231                clear_views,
2232                is_color,
2233            }
2234        } else {
2235            resource::TextureClearMode::BufferCopy
2236        };
2237
2238        let texture = Texture::new(
2239            self,
2240            resource::TextureInner::Native { raw: raw_texture },
2241            hal_usage,
2242            desc,
2243            format_features,
2244            clear_mode,
2245            true,
2246            texture_binding_view_dimension,
2247        );
2248
2249        let texture = Arc::new(texture);
2250
2251        self.trackers
2252            .lock()
2253            .textures
2254            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
2255
2256        Ok(texture)
2257    }
2258
2259    /// <https://www.w3.org/TR/webgpu/#dom-gpudevice-createtexture>
2260    pub fn create_texture(self: &Arc<Self>, desc: &resource::TextureDescriptor) -> Arc<Texture> {
2261        profiling::scope!("Device::create_texture");
2262
2263        let texture = self.create_texture_inner(desc).unwrap_or_else(|err| {
2264            self.handle_error(err, desc.label.as_deref(), "Device::create_texture");
2265            Texture::invalid(self, desc)
2266        });
2267        api_log!(
2268            "Device::create_texture({desc:?}) -> {:?}",
2269            Arc::as_ptr(&texture)
2270        );
2271
2272        #[cfg(feature = "trace")]
2273        if let Some(ref mut trace) = *self.trace.lock() {
2274            use crate::device::trace::IntoTrace as _;
2275
2276            trace.add(trace::Action::CreateTexture(
2277                texture.to_trace(),
2278                desc.clone(),
2279            ));
2280        }
2281        texture
2282    }
2283
2284    /// Creates a texture that is guaranteed to be invalid
2285    pub fn create_texture_error(
2286        self: &Arc<Self>,
2287        desc: &resource::TextureDescriptor,
2288    ) -> Arc<Texture> {
2289        let texture = Texture::invalid(self, desc);
2290        #[cfg(feature = "trace")]
2291        if let Some(ref mut trace) = *self.trace.lock() {
2292            use crate::device::trace::IntoTrace as _;
2293
2294            trace.add(trace::Action::CreateTextureError(
2295                texture.to_trace(),
2296                desc.clone(),
2297            ));
2298        }
2299        texture
2300    }
2301
2302    pub fn create_external_texture(
2303        self: &Arc<Self>,
2304        desc: &resource::ExternalTextureDescriptor,
2305        planes: &[Arc<TextureView>],
2306    ) -> Arc<ExternalTexture> {
2307        profiling::scope!("Device::create_external_texture");
2308
2309        let external_texture = self
2310            .create_external_texture_inner(desc, planes)
2311            .unwrap_or_else(|err| {
2312                self.handle_error(
2313                    err,
2314                    desc.label.as_deref(),
2315                    "Device::create_external_texture",
2316                );
2317                ExternalTexture::invalid(Arc::clone(self), desc)
2318            });
2319
2320        #[cfg(feature = "trace")]
2321        if let Some(ref mut trace) = *self.trace.lock() {
2322            use crate::device::trace;
2323            use trace::IntoTrace as _;
2324
2325            let planes = Box::from(
2326                planes
2327                    .iter()
2328                    .map(|plane| plane.to_trace())
2329                    .collect::<Vec<_>>(),
2330            );
2331            trace.add(trace::Action::CreateExternalTexture {
2332                id: external_texture.to_trace(),
2333                desc: desc.clone(),
2334                planes,
2335            });
2336        }
2337
2338        api_log!(
2339            "Device::create_external_texture({desc:?}) -> {:?}",
2340            Arc::as_ptr(&external_texture)
2341        );
2342
2343        external_texture
2344    }
2345
2346    pub(crate) fn create_external_texture_inner(
2347        self: &Arc<Self>,
2348        desc: &resource::ExternalTextureDescriptor,
2349        planes: &[Arc<TextureView>],
2350    ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
2351        use resource::CreateExternalTextureError;
2352        self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
2353        self.check_is_valid()?;
2354
2355        if desc.num_planes() != planes.len() {
2356            return Err(CreateExternalTextureError::IncorrectPlaneCount {
2357                format: desc.format,
2358                expected: desc.num_planes(),
2359                provided: planes.len(),
2360            });
2361        }
2362
2363        let planes = planes
2364            .iter()
2365            .enumerate()
2366            .map(|(i, plane)| {
2367                plane.same_device(self)?;
2368
2369                if plane.samples != 1 {
2370                    return Err(CreateExternalTextureError::InvalidPlaneMultisample(
2371                        plane.samples,
2372                    ));
2373                }
2374
2375                let sample_type = plane
2376                    .desc
2377                    .format
2378                    .sample_type(Some(plane.desc.range.aspect), Some(self.features))
2379                    .unwrap();
2380                if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
2381                    return Err(CreateExternalTextureError::InvalidPlaneSampleType {
2382                        format: plane.desc.format,
2383                        sample_type,
2384                    });
2385                }
2386
2387                if plane.desc.dimension != TextureViewDimension::D2 {
2388                    return Err(CreateExternalTextureError::InvalidPlaneDimension(
2389                        plane.desc.dimension,
2390                    ));
2391                }
2392
2393                let expected_components = match desc.format {
2394                    wgt::ExternalTextureFormat::Rgba => 4,
2395                    wgt::ExternalTextureFormat::Nv12 => match i {
2396                        0 => 1,
2397                        1 => 2,
2398                        _ => unreachable!(),
2399                    },
2400                    wgt::ExternalTextureFormat::Yu12 => 1,
2401                };
2402                if plane.desc.format.components() != expected_components {
2403                    return Err(CreateExternalTextureError::InvalidPlaneFormat {
2404                        format: desc.format,
2405                        plane: i,
2406                        expected: expected_components,
2407                        provided: plane.desc.format,
2408                    });
2409                }
2410
2411                plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
2412                Ok(plane.clone())
2413            })
2414            .collect::<Result<_, _>>()?;
2415
2416        let params_data = ExternalTextureParams::from_desc(desc);
2417        let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
2418        let params_desc = resource::BufferDescriptor {
2419            label: label.map(Cow::Owned),
2420            size: size_of_val(&params_data) as wgt::BufferAddress,
2421            usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
2422            mapped_at_creation: false,
2423        };
2424        let params = self.create_buffer_inner(&params_desc)?;
2425        self.get_queue().unwrap().write_buffer_inner(
2426            params.clone(),
2427            0,
2428            bytemuck::bytes_of(&params_data),
2429        )?;
2430
2431        let external_texture = ExternalTexture {
2432            state: ResourceState::Valid(ExternalTextureState { params }),
2433            device: self.clone(),
2434            planes,
2435            label: desc.label.to_string(),
2436            tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
2437        };
2438        let external_texture = Arc::new(external_texture);
2439
2440        Ok(external_texture)
2441    }
2442
2443    pub fn create_sampler(self: &Arc<Self>, desc: &resource::SamplerDescriptor) -> Arc<Sampler> {
2444        profiling::scope!("Device::create_sampler");
2445
2446        let sampler = self.create_sampler_inner(desc).unwrap_or_else(|err| {
2447            self.handle_error(err, desc.label.as_deref(), "Device::create_sampler");
2448            Sampler::invalid(Arc::clone(self), desc)
2449        });
2450
2451        #[cfg(feature = "trace")]
2452        if let Some(ref mut trace) = *self.trace.lock() {
2453            use crate::device::trace::{Action, IntoTrace as _};
2454            trace.add(Action::CreateSampler(sampler.to_trace(), desc.clone()));
2455        }
2456
2457        api_log!("Device::create_sampler -> {:?}", Arc::as_ptr(&sampler));
2458
2459        sampler
2460    }
2461
2462    pub(crate) fn create_sampler_inner(
2463        self: &Arc<Self>,
2464        desc: &resource::SamplerDescriptor,
2465    ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
2466        self.check_is_valid()?;
2467
2468        if desc
2469            .address_modes
2470            .iter()
2471            .any(|am| am == &wgt::AddressMode::ClampToBorder)
2472        {
2473            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
2474        }
2475
2476        if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
2477            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
2478        }
2479
2480        if desc.lod_min_clamp < 0.0 {
2481            return Err(resource::CreateSamplerError::InvalidLodMinClamp(
2482                desc.lod_min_clamp,
2483            ));
2484        }
2485        if desc.lod_max_clamp < desc.lod_min_clamp {
2486            return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
2487                lod_min_clamp: desc.lod_min_clamp,
2488                lod_max_clamp: desc.lod_max_clamp,
2489            });
2490        }
2491
2492        if desc.anisotropy_clamp < 1 {
2493            return Err(resource::CreateSamplerError::InvalidAnisotropy(
2494                desc.anisotropy_clamp,
2495            ));
2496        }
2497
2498        if desc.anisotropy_clamp != 1 {
2499            if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
2500                return Err(
2501                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2502                        filter_type: resource::SamplerFilterErrorType::MinFilter,
2503                        filter_mode: desc.min_filter,
2504                        anisotropic_clamp: desc.anisotropy_clamp,
2505                    },
2506                );
2507            }
2508            if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
2509                return Err(
2510                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2511                        filter_type: resource::SamplerFilterErrorType::MagFilter,
2512                        filter_mode: desc.mag_filter,
2513                        anisotropic_clamp: desc.anisotropy_clamp,
2514                    },
2515                );
2516            }
2517            if !matches!(desc.mipmap_filter, wgt::MipmapFilterMode::Linear) {
2518                return Err(
2519                    resource::CreateSamplerError::InvalidMipmapFilterModeWithAnisotropy {
2520                        filter_type: resource::SamplerFilterErrorType::MipmapFilter,
2521                        filter_mode: desc.mipmap_filter,
2522                        anisotropic_clamp: desc.anisotropy_clamp,
2523                    },
2524                );
2525            }
2526        }
2527
2528        let anisotropy_clamp = if self
2529            .downlevel
2530            .flags
2531            .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
2532        {
2533            // Clamp anisotropy clamp to [1, 16] per the wgpu-hal interface
2534            desc.anisotropy_clamp.min(16)
2535        } else {
2536            // If it isn't supported, set this unconditionally to 1
2537            1
2538        };
2539
2540        //TODO: check for wgt::DownlevelFlags::COMPARISON_SAMPLERS
2541
2542        let hal_desc = hal::SamplerDescriptor {
2543            label: desc.label.to_hal(self.instance_flags),
2544            address_modes: desc.address_modes,
2545            mag_filter: desc.mag_filter,
2546            min_filter: desc.min_filter,
2547            mipmap_filter: desc.mipmap_filter,
2548            lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
2549            compare: desc.compare,
2550            anisotropy_clamp,
2551            border_color: desc.border_color,
2552        };
2553
2554        let raw = unsafe { self.raw().create_sampler(&hal_desc) }
2555            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2556
2557        let sampler = Sampler {
2558            raw: ResourceState::Valid(raw),
2559            device: self.clone(),
2560            label: desc.label.to_string(),
2561            tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
2562            comparison: desc.compare.is_some(),
2563            filtering: desc.min_filter == wgt::FilterMode::Linear
2564                || desc.mag_filter == wgt::FilterMode::Linear
2565                || desc.mipmap_filter == wgt::MipmapFilterMode::Linear,
2566        };
2567
2568        let sampler = Arc::new(sampler);
2569
2570        Ok(sampler)
2571    }
2572
2573    pub fn create_shader_module<'a>(
2574        self: &Arc<Self>,
2575        desc: &pipeline::ShaderModuleDescriptor<'a>,
2576        source: pipeline::ShaderModuleSource<'a>,
2577    ) -> Arc<pipeline::ShaderModule> {
2578        profiling::scope!("Device::create_shader_module");
2579        #[cfg(feature = "trace")]
2580        let data = self.trace.lock().as_mut().map(|trace| {
2581            use crate::device::trace::DataKind;
2582
2583            match source {
2584                #[cfg(feature = "wgsl")]
2585                pipeline::ShaderModuleSource::Wgsl(ref code) => {
2586                    trace.make_binary(DataKind::Wgsl, code.as_bytes())
2587                }
2588                #[cfg(feature = "glsl")]
2589                pipeline::ShaderModuleSource::Glsl(ref code, _) => {
2590                    trace.make_binary(DataKind::Glsl, code.as_bytes())
2591                }
2592                #[cfg(feature = "spirv")]
2593                pipeline::ShaderModuleSource::SpirV(ref code, _) => {
2594                    trace.make_binary(DataKind::Spv, bytemuck::cast_slice::<u32, u8>(code))
2595                }
2596                pipeline::ShaderModuleSource::Naga(ref module) => {
2597                    let string =
2598                        ron::ser::to_string_pretty(module, ron::ser::PrettyConfig::default())
2599                            .unwrap();
2600                    trace.make_binary(DataKind::Ron, string.as_bytes())
2601                }
2602                pipeline::ShaderModuleSource::Dummy(_) => {
2603                    panic!("found `ShaderModuleSource::Dummy`")
2604                }
2605            }
2606        });
2607        let shader = self
2608            .create_shader_module_inner(desc, source)
2609            .unwrap_or_else(|e| {
2610                let shader = pipeline::ShaderModule::invalid(
2611                    Arc::clone(self),
2612                    desc.label.to_string(),
2613                    shader_module_error_into_compilation_info(&e),
2614                );
2615                self.handle_error(e, desc.label.as_deref(), "Device::create_shader_module");
2616                shader
2617            });
2618        api_log!("Device::create_shader_module -> {:?}", Arc::as_ptr(&shader));
2619
2620        #[cfg(feature = "trace")]
2621        if let Some(data) = data {
2622            // We don't need these two operations with the trace to be atomic.
2623
2624            use crate::device::trace::IntoTrace as _;
2625            self.trace
2626                .lock()
2627                .as_mut()
2628                .expect("trace went away during create_shader_module?")
2629                .add(trace::Action::CreateShaderModule {
2630                    id: shader.to_trace(),
2631                    desc: desc.clone(),
2632                    data,
2633                });
2634        };
2635        shader
2636    }
2637
2638    pub(crate) fn create_shader_module_inner<'a>(
2639        self: &Arc<Self>,
2640        desc: &pipeline::ShaderModuleDescriptor<'a>,
2641        source: pipeline::ShaderModuleSource<'a>,
2642    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2643        self.check_is_valid()?;
2644
2645        let (module, source) = match source {
2646            #[cfg(feature = "wgsl")]
2647            pipeline::ShaderModuleSource::Wgsl(code) => {
2648                profiling::scope!("naga::front::wgsl::parse");
2649                let capabilities = crate::device::features_to_naga_capabilities(
2650                    self.features,
2651                    self.downlevel.flags,
2652                );
2653                let mut options = naga::front::wgsl::Options::new();
2654                options.capabilities = capabilities;
2655                let mut frontend = naga::front::wgsl::Frontend::new_with_options(options);
2656                let module = frontend.parse(&code).map_err(|inner| {
2657                    pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2658                        source: code.to_string(),
2659                        label: desc.label.as_ref().map(|l| l.to_string()),
2660                        inner: Box::new(inner),
2661                    })
2662                })?;
2663                (Cow::Owned(module), code.into_owned())
2664            }
2665            #[cfg(feature = "spirv")]
2666            pipeline::ShaderModuleSource::SpirV(spv, options) => {
2667                let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2668                profiling::scope!("naga::front::spv::Frontend");
2669                let module = parser.parse().map_err(|inner| {
2670                    pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2671                        source: String::new(),
2672                        label: desc.label.as_ref().map(|l| l.to_string()),
2673                        inner: Box::new(inner),
2674                    })
2675                })?;
2676                (Cow::Owned(module), String::new())
2677            }
2678            #[cfg(feature = "glsl")]
2679            pipeline::ShaderModuleSource::Glsl(code, options) => {
2680                let mut parser = naga::front::glsl::Frontend::default();
2681                profiling::scope!("naga::front::glsl::Frontend.parse");
2682                let module = parser.parse(&options, &code).map_err(|inner| {
2683                    pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2684                        source: code.to_string(),
2685                        label: desc.label.as_ref().map(|l| l.to_string()),
2686                        inner: Box::new(inner),
2687                    })
2688                })?;
2689                (Cow::Owned(module), code.into_owned())
2690            }
2691            pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2692            pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2693        };
2694        for (_, var) in module.global_variables.iter() {
2695            match var.binding {
2696                Some(br) if br.group >= self.limits.max_bind_groups => {
2697                    return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2698                        bind: br,
2699                        group: br.group,
2700                        limit: self.limits.max_bind_groups,
2701                    });
2702                }
2703                _ => continue,
2704            };
2705        }
2706
2707        profiling::scope!("naga::validate");
2708        let debug_source =
2709            if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2710                Some(hal::DebugSource {
2711                    file_name: Cow::Owned(
2712                        desc.label
2713                            .as_ref()
2714                            .map_or("shader".to_string(), |l| l.to_string()),
2715                    ),
2716                    source_code: Cow::Owned(source.clone()),
2717                })
2718            } else {
2719                None
2720            };
2721
2722        let info = create_validator(
2723            self.features,
2724            self.downlevel.flags,
2725            naga::valid::ValidationFlags::all(),
2726        )
2727        .validate(&module)
2728        .map_err(|inner| {
2729            pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2730                source,
2731                label: desc.label.as_ref().map(|l| l.to_string()),
2732                inner,
2733            })
2734        })?;
2735
2736        // Defer creating the `wgpu_hal` shader module until pipeline creation,
2737        // when we can apply the pipeline's override values and validate the
2738        // result.
2739        let module = pipeline::ShaderModule {
2740            state: ResourceState::Valid(pipeline::ShaderModuleState::NagaModule {
2741                shader: hal::NagaShader {
2742                    module,
2743                    info,
2744                    debug_source,
2745                },
2746                runtime_checks: desc.runtime_checks,
2747            }),
2748            device: self.clone(),
2749            label: desc.label.to_string(),
2750            compilation_info: wgt::CompilationInfo::default(),
2751        };
2752
2753        let module = Arc::new(module);
2754
2755        Ok(module)
2756    }
2757
2758    /// # Safety
2759    ///
2760    /// This function passes source code or binary to the backend as-is and can potentially result in a
2761    /// driver crash.
2762    pub unsafe fn create_shader_module_passthrough<'a>(
2763        self: &Arc<Self>,
2764        desc: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2765    ) -> Arc<pipeline::ShaderModule> {
2766        profiling::scope!("Device::create_shader_module_passthrough");
2767
2768        let shader =
2769            unsafe { self.create_shader_module_passthrough_inner(desc) }.unwrap_or_else(|e| {
2770                let shader = pipeline::ShaderModule::invalid(
2771                    Arc::clone(self),
2772                    desc.label.to_string(),
2773                    shader_module_error_into_compilation_info(&e),
2774                );
2775                self.handle_error(
2776                    e,
2777                    desc.label.as_deref(),
2778                    "Device::create_shader_module_passthrough",
2779                );
2780                shader
2781            });
2782
2783        #[cfg(feature = "trace")]
2784        if let Some(ref mut trace) = *self.trace.lock() {
2785            use crate::device::trace::{DataKind, IntoTrace as _};
2786
2787            let mut file_names = Vec::new();
2788            for (data, kind) in [
2789                (
2790                    desc.spirv.as_ref().map(|a| bytemuck::cast_slice(a)),
2791                    DataKind::Spv,
2792                ),
2793                (desc.dxil.as_deref(), DataKind::Dxil),
2794                (desc.hlsl.as_ref().map(|a| a.as_bytes()), DataKind::Hlsl),
2795                (desc.metallib.as_deref(), DataKind::MetalLib),
2796                (desc.msl.as_ref().map(|a| a.as_bytes()), DataKind::Msl),
2797                (desc.glsl.as_ref().map(|a| a.as_bytes()), DataKind::Glsl),
2798                (desc.wgsl.as_ref().map(|a| a.as_bytes()), DataKind::Wgsl),
2799            ] {
2800                if let Some(data) = data {
2801                    file_names.push(trace.make_binary(kind, data));
2802                }
2803            }
2804            trace.add(trace::Action::CreateShaderModulePassthrough {
2805                id: shader.to_trace(),
2806                data: file_names,
2807                label: desc.label.clone(),
2808                entry_points: desc.entry_points.clone(),
2809            });
2810        };
2811        api_log!(
2812            "Device::create_shader_module_spirv -> {:?}",
2813            Arc::as_ptr(&shader)
2814        );
2815        shader
2816    }
2817
2818    pub(crate) unsafe fn create_shader_module_passthrough_inner<'a>(
2819        self: &Arc<Self>,
2820        descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2821    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2822        self.check_is_valid()?;
2823        self.require_features(wgt::Features::PASSTHROUGH_SHADERS)?;
2824
2825        // Mainly important for GLSL or SPIR-V or DXIL, which each take exactly 1 entry point.
2826        if (descriptor.dxil.is_some() || descriptor.glsl.is_some())
2827            && descriptor.entry_points.len() != 1
2828        {
2829            return Err(pipeline::CreateShaderModuleError::IncorrectPassthroughEntryPointCount);
2830        }
2831
2832        let entry_point_hashmap = || {
2833            descriptor
2834                .entry_points
2835                .iter()
2836                .map(|e| (e.name.to_string(), e.workgroup_size))
2837                .collect()
2838        };
2839
2840        let hal_shader = match self.backend() {
2841            wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2842                descriptor
2843                    .spirv
2844                    .as_ref()
2845                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2846            ),
2847            wgt::Backend::Dx12 => {
2848                if let Some(dxil) = &descriptor.dxil {
2849                    hal::ShaderInput::Dxil { shader: dxil }
2850                } else if let Some(hlsl) = &descriptor.hlsl {
2851                    hal::ShaderInput::Hlsl { shader: hlsl }
2852                } else {
2853                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2854                }
2855            }
2856            wgt::Backend::Metal => {
2857                if let Some(metallib) = &descriptor.metallib {
2858                    hal::ShaderInput::MetalLib {
2859                        file: metallib,
2860                        num_workgroups: entry_point_hashmap(),
2861                    }
2862                } else if let Some(msl) = &descriptor.msl {
2863                    hal::ShaderInput::Msl {
2864                        shader: msl,
2865                        num_workgroups: entry_point_hashmap(),
2866                    }
2867                } else {
2868                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2869                }
2870            }
2871            wgt::Backend::Gl => hal::ShaderInput::Glsl {
2872                shader: descriptor
2873                    .glsl
2874                    .as_ref()
2875                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2876            },
2877            wgt::Backend::Noop => {
2878                return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2879            }
2880            wgt::Backend::BrowserWebGpu => unreachable!(),
2881        };
2882
2883        let hal_desc = hal::ShaderModuleDescriptor {
2884            label: descriptor.label.to_hal(self.instance_flags),
2885            runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2886        };
2887
2888        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2889            Ok(raw) => raw,
2890            Err(error) => {
2891                return Err(match error {
2892                    hal::ShaderError::Device(error) => {
2893                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2894                    }
2895                    hal::ShaderError::Compilation(ref msg) => {
2896                        log::error!("Shader error: {msg}");
2897                        pipeline::CreateShaderModuleError::Generation
2898                    }
2899                })
2900            }
2901        };
2902
2903        let module = pipeline::ShaderModule {
2904            state: ResourceState::Valid(pipeline::ShaderModuleState::Passthrough {
2905                raw,
2906                interface: PassthroughInterface {
2907                    entry_point_names: descriptor
2908                        .entry_points
2909                        .iter()
2910                        .map(|e| e.name.to_string())
2911                        .collect(),
2912                },
2913            }),
2914            device: self.clone(),
2915            label: descriptor.label.to_string(),
2916            compilation_info: wgt::CompilationInfo::default(),
2917        };
2918
2919        Ok(Arc::new(module))
2920    }
2921
2922    pub fn create_command_encoder(
2923        self: &Arc<Self>,
2924        desc: &wgt::CommandEncoderDescriptor<crate::Label>,
2925    ) -> Arc<command::CommandEncoder> {
2926        profiling::scope!("Device::create_command_encoder");
2927
2928        let cmd_enc = self
2929            .create_command_encoder_inner(&desc.label)
2930            .unwrap_or_else(|err| {
2931                let error = err.clone().into();
2932                self.handle_error(
2933                    err,
2934                    desc.label.as_ref().map(|l| l.as_ref()),
2935                    "Device::create_command_encoder",
2936                );
2937                command::CommandEncoder::new_invalid(self, &desc.label, error)
2938            });
2939
2940        api_log!(
2941            "Device::create_command_encoder -> {:?}",
2942            Arc::as_ptr(&cmd_enc)
2943        );
2944
2945        cmd_enc
2946    }
2947
2948    pub(crate) fn create_command_encoder_inner(
2949        self: &Arc<Self>,
2950        label: &crate::Label,
2951    ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2952        self.check_is_valid()?;
2953
2954        let queue = self.get_queue().unwrap();
2955
2956        let encoder = self
2957            .command_allocator
2958            .acquire_encoder(self.raw(), queue.raw())
2959            .map_err(|e| self.handle_hal_error(e))?;
2960
2961        let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2962
2963        let cmd_enc = Arc::new(cmd_enc);
2964
2965        Ok(cmd_enc)
2966    }
2967
2968    pub fn create_render_bundle_encoder(
2969        self: &Arc<Self>,
2970        desc: &command::RenderBundleEncoderDescriptor,
2971    ) -> Box<command::RenderBundleEncoder> {
2972        profiling::scope!("Device::create_render_bundle_encoder");
2973        api_log!("Device::create_render_bundle_encoder");
2974        Box::new(
2975            command::RenderBundleEncoder::new(self, desc).unwrap_or_else(|err| {
2976                self.handle_error(
2977                    err,
2978                    desc.label.as_deref(),
2979                    "Device::create_render_bundle_encoder",
2980                );
2981                command::RenderBundleEncoder::dummy(self)
2982            }),
2983        )
2984    }
2985
2986    /// Generate information about late-validated buffer bindings for pipelines.
2987    //TODO: should this be combined with `get_introspection_bind_group_layouts` in some way?
2988    pub(super) fn make_late_sized_buffer_groups(
2989        shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2990        layout: &binding_model::PipelineLayout,
2991    ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2992        // Given the shader-required binding sizes and the pipeline layout,
2993        // return the filtered list of them in the layout order,
2994        // removing those with given `min_binding_size`.
2995        layout
2996            .bind_group_layouts
2997            .iter()
2998            .enumerate()
2999            .map(|(group_index, bgl)| {
3000                let Some(bgl) = bgl else {
3001                    return pipeline::LateSizedBufferGroup::default();
3002                };
3003
3004                let shader_sizes = bgl
3005                    .entries
3006                    .values()
3007                    .filter_map(|entry| match entry.ty {
3008                        wgt::BindingType::Buffer {
3009                            min_binding_size: None,
3010                            ..
3011                        } => {
3012                            let rb = naga::ResourceBinding {
3013                                group: group_index as u32,
3014                                binding: entry.binding,
3015                            };
3016                            let shader_size =
3017                                shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
3018                            Some(shader_size)
3019                        }
3020                        _ => None,
3021                    })
3022                    .collect();
3023                pipeline::LateSizedBufferGroup { shader_sizes }
3024            })
3025            .collect()
3026    }
3027
3028    pub fn create_bind_group_layout<BGLE>(
3029        self: &Arc<Self>,
3030        desc: &binding_model::BindGroupLayoutDescriptor<BGLE>,
3031    ) -> Arc<BindGroupLayout>
3032    where
3033        BGLE: TryInto<wgt::BindGroupLayoutEntry> + Copy,
3034        CreateBindGroupLayoutError: From<<BGLE as TryInto<wgt::BindGroupLayoutEntry>>::Error>,
3035        BGLE: Into<binding_model::BindGroupLayoutEntry>,
3036    {
3037        profiling::scope!("Device::create_bind_group_layout");
3038
3039        let bgl = self
3040            .create_bind_group_layout_inner(desc)
3041            .unwrap_or_else(|err| {
3042                self.handle_error(
3043                    err,
3044                    desc.label.as_deref(),
3045                    "Device::create_bind_group_layout",
3046                );
3047                BindGroupLayout::invalid(self, desc.label.to_string())
3048            });
3049
3050        #[cfg(feature = "trace")]
3051        if let Some(ref mut trace) = *self.trace.lock() {
3052            use crate::device::trace::IntoTrace;
3053
3054            trace.add(trace::Action::CreateBindGroupLayout(
3055                bgl.to_trace(),
3056                desc.to_trace(),
3057            ));
3058        }
3059        api_log!(
3060            "Device::create_bind_group_layout -> {:?}",
3061            Arc::as_ptr(&bgl)
3062        );
3063        bgl
3064    }
3065
3066    fn create_bind_group_layout_inner<BGLE>(
3067        self: &Arc<Device>,
3068        desc: &binding_model::BindGroupLayoutDescriptor<BGLE>,
3069    ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError>
3070    where
3071        BGLE: TryInto<wgt::BindGroupLayoutEntry> + Copy,
3072        CreateBindGroupLayoutError: From<<BGLE as TryInto<wgt::BindGroupLayoutEntry>>::Error>,
3073    {
3074        self.check_is_valid()?;
3075
3076        let entry_map = bgl::EntryMap::from_entries(&desc.entries)?;
3077
3078        let bgl_result = self.bgl_pool.get_or_init(entry_map, |entry_map| {
3079            let bgl =
3080                self.create_bind_group_layout_impl(&desc.label, entry_map, bgl::Origin::Pool)?;
3081            bgl.exclusive_pipeline
3082                .set(binding_model::ExclusivePipeline::None)
3083                .unwrap();
3084            Ok(bgl)
3085        });
3086
3087        match bgl_result {
3088            Ok(layout) => Ok(layout),
3089            Err(e) => Err(e),
3090        }
3091    }
3092
3093    fn create_bind_group_layout_impl(
3094        self: &Arc<Self>,
3095        label: &crate::Label,
3096        entry_map: bgl::EntryMap,
3097        origin: bgl::Origin,
3098    ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
3099        #[derive(PartialEq)]
3100        enum WritableStorage {
3101            Yes,
3102            No,
3103        }
3104
3105        for entry in entry_map.values() {
3106            if entry.binding >= self.limits.max_bindings_per_bind_group {
3107                return Err(CreateBindGroupLayoutError::InvalidBindingIndex {
3108                    binding: entry.binding,
3109                    maximum: self.limits.max_bindings_per_bind_group,
3110                });
3111            }
3112
3113            use wgt::BindingType as Bt;
3114
3115            let mut required_features = wgt::Features::empty();
3116            let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
3117            let (array_feature, writable_storage) = match entry.ty {
3118                Bt::Buffer {
3119                    ty: wgt::BufferBindingType::Uniform,
3120                    has_dynamic_offset: false,
3121                    min_binding_size: _,
3122                } => (
3123                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
3124                    WritableStorage::No,
3125                ),
3126                Bt::Buffer {
3127                    ty: wgt::BufferBindingType::Uniform,
3128                    has_dynamic_offset: true,
3129                    min_binding_size: _,
3130                } => (
3131                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
3132                    WritableStorage::No,
3133                ),
3134                Bt::Buffer {
3135                    ty: wgt::BufferBindingType::Storage { read_only },
3136                    ..
3137                } => (
3138                    Some(
3139                        wgt::Features::BUFFER_BINDING_ARRAY
3140                            | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
3141                    ),
3142                    match read_only {
3143                        true => WritableStorage::No,
3144                        false => WritableStorage::Yes,
3145                    },
3146                ),
3147                Bt::Sampler { .. } => (
3148                    Some(wgt::Features::TEXTURE_BINDING_ARRAY),
3149                    WritableStorage::No,
3150                ),
3151                Bt::Texture {
3152                    multisampled: true,
3153                    sample_type: TextureSampleType::Float { filterable: true },
3154                    ..
3155                } => {
3156                    return Err(CreateBindGroupLayoutError::Entry {
3157                        binding: entry.binding,
3158                        error:
3159                            BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
3160                    });
3161                }
3162                Bt::Texture {
3163                    multisampled,
3164                    view_dimension,
3165                    ..
3166                } => {
3167                    if multisampled && view_dimension != TextureViewDimension::D2 {
3168                        return Err(CreateBindGroupLayoutError::Entry {
3169                            binding: entry.binding,
3170                            error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
3171                        });
3172                    }
3173
3174                    (
3175                        Some(wgt::Features::TEXTURE_BINDING_ARRAY),
3176                        WritableStorage::No,
3177                    )
3178                }
3179                Bt::StorageTexture {
3180                    access,
3181                    view_dimension,
3182                    format,
3183                } => {
3184                    use wgt::{StorageTextureAccess as Access, TextureFormatFeatureFlags as Flags};
3185
3186                    match view_dimension {
3187                        TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
3188                            return Err(CreateBindGroupLayoutError::Entry {
3189                                binding: entry.binding,
3190                                error: BindGroupLayoutEntryError::StorageTextureCube,
3191                            })
3192                        }
3193                        _ => (),
3194                    }
3195                    match access {
3196                        wgt::StorageTextureAccess::Atomic
3197                            if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
3198                        {
3199                            return Err(CreateBindGroupLayoutError::Entry {
3200                                binding: entry.binding,
3201                                error: BindGroupLayoutEntryError::StorageTextureAtomic,
3202                            });
3203                        }
3204                        _ => (),
3205                    }
3206
3207                    let format_features =
3208                        self.describe_format_features(format).map_err(|error| {
3209                            CreateBindGroupLayoutError::Entry {
3210                                binding: entry.binding,
3211                                error: BindGroupLayoutEntryError::MissingFeatures(error),
3212                            }
3213                        })?;
3214
3215                    let required_feature_flag = match access {
3216                        Access::WriteOnly => Flags::STORAGE_WRITE_ONLY,
3217                        Access::ReadOnly => Flags::STORAGE_READ_ONLY,
3218                        Access::ReadWrite => Flags::STORAGE_READ_WRITE,
3219                        Access::Atomic => Flags::STORAGE_ATOMIC,
3220                    };
3221
3222                    if !format_features.flags.contains(required_feature_flag) {
3223                        return Err(
3224                            CreateBindGroupLayoutError::UnsupportedStorageTextureAccess {
3225                                binding: entry.binding,
3226                                access,
3227                                format,
3228                            },
3229                        );
3230                    }
3231
3232                    (
3233                        Some(
3234                            wgt::Features::TEXTURE_BINDING_ARRAY
3235                                | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
3236                        ),
3237                        match access {
3238                            wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
3239                            wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
3240                            wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
3241                            wgt::StorageTextureAccess::Atomic => {
3242                                required_features |= wgt::Features::TEXTURE_ATOMIC;
3243                                WritableStorage::Yes
3244                            }
3245                        },
3246                    )
3247                }
3248                Bt::AccelerationStructure { vertex_return } => {
3249                    self.require_acceleration_structures().map_err(|e| {
3250                        CreateBindGroupLayoutError::Entry {
3251                            binding: entry.binding,
3252                            error: e.into(),
3253                        }
3254                    })?;
3255                    if vertex_return {
3256                        self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
3257                            .map_err(|e| CreateBindGroupLayoutError::Entry {
3258                                binding: entry.binding,
3259                                error: e.into(),
3260                            })?;
3261                    }
3262                    (
3263                        Some(wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY),
3264                        WritableStorage::No,
3265                    )
3266                }
3267                Bt::ExternalTexture => {
3268                    self.require_features(wgt::Features::EXTERNAL_TEXTURE)
3269                        .map_err(|e| CreateBindGroupLayoutError::Entry {
3270                            binding: entry.binding,
3271                            error: e.into(),
3272                        })?;
3273                    (None, WritableStorage::No)
3274                }
3275            };
3276
3277            // Validate the count parameter
3278            if entry.count.is_some() {
3279                required_features |= array_feature
3280                    .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
3281                    .map_err(|error| CreateBindGroupLayoutError::Entry {
3282                        binding: entry.binding,
3283                        error,
3284                    })?;
3285            }
3286
3287            if entry.visibility.contains_unknown_bits() {
3288                return Err(CreateBindGroupLayoutError::InvalidVisibility(
3289                    entry.visibility,
3290                ));
3291            }
3292
3293            if entry
3294                .visibility
3295                .intersects(wgt::ShaderStages::TASK | wgt::ShaderStages::MESH)
3296            {
3297                required_features |= wgt::Features::EXPERIMENTAL_MESH_SHADER;
3298            }
3299
3300            if entry.visibility.intersects(
3301                wgt::ShaderStages::RAY_GENERATION
3302                    | wgt::ShaderStages::ANY_HIT
3303                    | wgt::ShaderStages::CLOSEST_HIT
3304                    | wgt::ShaderStages::MISS,
3305            ) {
3306                required_features |= wgt::Features::EXPERIMENTAL_RAY_TRACING_PIPELINES;
3307            }
3308
3309            if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
3310                if writable_storage == WritableStorage::Yes {
3311                    required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
3312                }
3313                if let Bt::Buffer {
3314                    ty: wgt::BufferBindingType::Storage { .. },
3315                    ..
3316                } = entry.ty
3317                {
3318                    required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
3319                }
3320            }
3321            if writable_storage == WritableStorage::Yes
3322                && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
3323            {
3324                required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
3325            }
3326
3327            self.require_features(required_features)
3328                .map_err(BindGroupLayoutEntryError::MissingFeatures)
3329                .map_err(|error| CreateBindGroupLayoutError::Entry {
3330                    binding: entry.binding,
3331                    error,
3332                })?;
3333            self.require_downlevel_flags(required_downlevel_flags)
3334                .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
3335                .map_err(|error| CreateBindGroupLayoutError::Entry {
3336                    binding: entry.binding,
3337                    error,
3338                })?;
3339        }
3340
3341        let bgl_flags = conv::bind_group_layout_flags(self.features);
3342
3343        let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
3344        let hal_desc = hal::BindGroupLayoutDescriptor {
3345            label: label.to_hal(self.instance_flags),
3346            flags: bgl_flags,
3347            entries: &hal_bindings,
3348        };
3349
3350        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3351        for entry in entry_map.values() {
3352            count_validator.add_binding(entry);
3353        }
3354        // If a single bind group layout violates limits, the pipeline layout is
3355        // definitely going to violate limits too, lets catch it now.
3356        count_validator
3357            .validate(&self.limits, self.instance_flags)
3358            .map_err(CreateBindGroupLayoutError::TooManyBindings)?;
3359
3360        // Validate that binding arrays don't conflict with dynamic offsets.
3361        count_validator.validate_binding_arrays()?;
3362
3363        let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
3364            .map_err(|e| self.handle_hal_error(e))?;
3365
3366        let bgl = BindGroupLayout {
3367            state: ResourceState::Valid(BindGroupLayoutState {
3368                raw: binding_model::RawBindGroupLayout::Owning(ManuallyDrop::new(raw)),
3369                origin,
3370                binding_count_validator: count_validator,
3371            }),
3372            device: self.clone(),
3373            entries: entry_map,
3374            exclusive_pipeline: OnceCell::new(),
3375            label: label.to_string(),
3376        };
3377
3378        let bgl = Arc::new(bgl);
3379
3380        Ok(bgl)
3381    }
3382
3383    fn create_buffer_binding<'a>(
3384        &self,
3385        bb: &'a binding_model::BufferBinding,
3386        binding: u32,
3387        decl: &wgt::BindGroupLayoutEntry,
3388        buffer_init_actions: &mut Vec<BufferInitTrackerAction>,
3389        dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
3390        late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
3391        used: &mut BindGroupStates,
3392        snatch_guard: &'a SnatchGuard<'a>,
3393    ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer, NonZeroU64>, CreateBindGroupError> {
3394        use crate::binding_model::CreateBindGroupError as Error;
3395
3396        let (binding_ty, dynamic, min_size) = match decl.ty {
3397            wgt::BindingType::Buffer {
3398                ty,
3399                has_dynamic_offset,
3400                min_binding_size,
3401            } => (ty, has_dynamic_offset, min_binding_size),
3402            _ => {
3403                return Err(Error::WrongBindingType {
3404                    binding,
3405                    actual: decl.ty,
3406                    expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
3407                })
3408            }
3409        };
3410
3411        let (pub_usage, internal_use, range_limit) = match binding_ty {
3412            wgt::BufferBindingType::Uniform => (
3413                wgt::BufferUsages::UNIFORM,
3414                wgt::BufferUses::UNIFORM,
3415                self.limits.max_uniform_buffer_binding_size,
3416            ),
3417            wgt::BufferBindingType::Storage { read_only } => (
3418                wgt::BufferUsages::STORAGE,
3419                if read_only {
3420                    wgt::BufferUses::STORAGE_READ_ONLY
3421                } else {
3422                    wgt::BufferUses::STORAGE_READ_WRITE
3423                },
3424                self.limits.max_storage_buffer_binding_size,
3425            ),
3426        };
3427
3428        let (align, align_limit_name) =
3429            binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
3430        if !bb.offset.is_multiple_of(align as u64) {
3431            return Err(Error::UnalignedBufferOffset(
3432                bb.offset,
3433                align_limit_name,
3434                align,
3435            ));
3436        }
3437
3438        let buffer = &bb.buffer;
3439
3440        used.buffers.insert_single(buffer.clone(), internal_use);
3441
3442        buffer.check_is_valid()?;
3443        buffer.same_device(self)?;
3444
3445        buffer.check_usage(pub_usage)?;
3446
3447        let req_size = match bb.size.map(wgt::BufferSize::new) {
3448            // Requested a non-zero size
3449            Some(non_zero @ Some(_)) => non_zero,
3450            // Requested size not specified
3451            None => None,
3452            // Requested zero size
3453            Some(None) => return Err(CreateBindGroupError::BindingZeroSize(buffer.error_ident())),
3454        };
3455        let bb = buffer.binding(bb.offset, req_size, snatch_guard)?;
3456        let bind_size = bb.size.get();
3457
3458        if matches!(binding_ty, wgt::BufferBindingType::Storage { .. })
3459            && bind_size % u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) != 0
3460        {
3461            return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
3462                alignment: wgt::STORAGE_BINDING_SIZE_ALIGNMENT,
3463                size: bind_size,
3464            });
3465        }
3466
3467        let bind_end = bb.offset + bind_size;
3468
3469        if bind_size > range_limit {
3470            return Err(Error::BufferRangeTooLarge {
3471                binding,
3472                given: bind_size,
3473                limit: range_limit,
3474            });
3475        }
3476
3477        // Record binding info for validating dynamic offsets
3478        if dynamic {
3479            dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
3480                binding_idx: binding,
3481                buffer_size: buffer.size,
3482                binding_range: bb.offset..bind_end,
3483                maximum_dynamic_offset: buffer.size - bind_end,
3484                binding_type: binding_ty,
3485            });
3486        }
3487
3488        if let Some(non_zero) = min_size {
3489            let min_size = non_zero.get();
3490            if min_size > bind_size {
3491                return Err(Error::BindingSizeTooSmall {
3492                    buffer: buffer.error_ident(),
3493                    actual: bind_size,
3494                    min: min_size,
3495                });
3496            }
3497        } else {
3498            let late_size = wgt::BufferSize::new(bind_size)
3499                .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
3500            late_buffer_binding_sizes.insert(binding, late_size);
3501        }
3502
3503        // This was checked against the device's alignment requirements above,
3504        // which should always be a multiple of `COPY_BUFFER_ALIGNMENT`.
3505        assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
3506
3507        let init_range = if dynamic {
3508            // We don't know what part of the buffer will be bound, so require that it
3509            // is fully initialized.
3510            0..buffer.size
3511        } else {
3512            // `wgpu_hal` only restricts shader access to bound buffer regions with
3513            // a certain resolution. For the sake of lazy initialization, round up
3514            // the size of the bound range to reflect how much of the buffer is
3515            // actually going to be visible to the shader.
3516            let bounds_check_alignment = binding_model::buffer_binding_type_bounds_check_alignment(
3517                &self.alignments,
3518                binding_ty,
3519            );
3520            let visible_size = align_to(bind_size, bounds_check_alignment);
3521
3522            bb.offset..bb.offset + visible_size
3523        };
3524
3525        // Once a buffer is initialized, nothing can cause the contents to revert to an
3526        // unknown state, so we do not record an init action in the bind group when that is
3527        // the case. (The same is not true of textures.)
3528        buffer_init_actions.extend(buffer.initialization_status.read().create_action(
3529            buffer,
3530            init_range,
3531            MemoryInitKind::NeedsInitializedMemory,
3532        ));
3533
3534        Ok(bb)
3535    }
3536
3537    fn create_sampler_binding<'a>(
3538        &self,
3539        used: &mut BindGroupStates,
3540        binding: u32,
3541        decl: &wgt::BindGroupLayoutEntry,
3542        sampler: &'a Arc<Sampler>,
3543    ) -> Result<&'a dyn hal::DynSampler, CreateBindGroupError> {
3544        use crate::binding_model::CreateBindGroupError as Error;
3545
3546        used.samplers.insert_single(sampler.clone());
3547
3548        sampler.same_device(self)?;
3549
3550        match decl.ty {
3551            wgt::BindingType::Sampler(ty) => {
3552                let (allowed_filtering, allowed_comparison) = match ty {
3553                    wgt::SamplerBindingType::Filtering => (None, false),
3554                    wgt::SamplerBindingType::NonFiltering => (Some(false), false),
3555                    wgt::SamplerBindingType::Comparison => (None, true),
3556                };
3557                if let Some(allowed_filtering) = allowed_filtering {
3558                    if allowed_filtering != sampler.filtering {
3559                        return Err(Error::WrongSamplerFiltering {
3560                            binding,
3561                            layout_flt: allowed_filtering,
3562                            sampler_flt: sampler.filtering,
3563                        });
3564                    }
3565                }
3566                if allowed_comparison != sampler.comparison {
3567                    return Err(Error::WrongSamplerComparison {
3568                        binding,
3569                        layout_cmp: allowed_comparison,
3570                        sampler_cmp: sampler.comparison,
3571                    });
3572                }
3573            }
3574            _ => {
3575                return Err(Error::WrongBindingType {
3576                    binding,
3577                    actual: decl.ty,
3578                    expected: "Sampler",
3579                })
3580            }
3581        }
3582
3583        Ok(sampler.raw()?)
3584    }
3585
3586    fn create_texture_binding<'a>(
3587        &self,
3588        binding: u32,
3589        decl: &wgt::BindGroupLayoutEntry,
3590        view: &'a Arc<TextureView>,
3591        used: &mut BindGroupStates,
3592        texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3593        snatch_guard: &'a SnatchGuard<'a>,
3594    ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, CreateBindGroupError> {
3595        view.check_valid()?;
3596        view.same_device(self)?;
3597
3598        let internal_use = self.texture_use_parameters(
3599            binding,
3600            decl,
3601            view,
3602            "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
3603        )?;
3604
3605        used.views.insert_single(view.clone(), internal_use);
3606
3607        let texture = &view.parent;
3608
3609        // Unlike buffers, textures contents can revert to being undefined if used as a
3610        // render attachment with `StoreOp::Discard`. Therefore, we must always register
3611        // the init action in the bind group, and check the initialization state every
3612        // time the binding is used.
3613        texture_init_actions.push(TextureInitTrackerAction {
3614            texture: texture.clone(),
3615            range: TextureInitRange {
3616                mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
3617                layer_range: view
3618                    .desc
3619                    .range
3620                    .layer_range(texture.desc.array_layer_count()),
3621            },
3622            kind: MemoryInitKind::NeedsInitializedMemory,
3623        });
3624
3625        Ok(hal::TextureBinding {
3626            view: view.try_raw(snatch_guard)?,
3627            usage: internal_use,
3628        })
3629    }
3630
3631    fn create_tlas_binding<'a>(
3632        self: &Arc<Self>,
3633        used: &mut BindGroupStates,
3634        binding: u32,
3635        decl: &wgt::BindGroupLayoutEntry,
3636        tlas: &'a Arc<Tlas>,
3637        snatch_guard: &'a SnatchGuard<'a>,
3638    ) -> Result<&'a dyn hal::DynAccelerationStructure, CreateBindGroupError> {
3639        use crate::binding_model::CreateBindGroupError as Error;
3640
3641        used.acceleration_structures.insert_single(tlas.clone());
3642
3643        tlas.check_is_valid()?;
3644        tlas.same_device(self)?;
3645
3646        match decl.ty {
3647            wgt::BindingType::AccelerationStructure { vertex_return } => {
3648                if vertex_return
3649                    && !tlas.flags.contains(
3650                        wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
3651                    )
3652                {
3653                    return Err(Error::MissingTLASVertexReturn { binding });
3654                }
3655            }
3656            _ => {
3657                return Err(Error::WrongBindingType {
3658                    binding,
3659                    actual: decl.ty,
3660                    expected: "Tlas",
3661                });
3662            }
3663        }
3664
3665        Ok(tlas.try_raw(snatch_guard)?)
3666    }
3667
3668    fn create_external_texture_binding<'a>(
3669        &'a self,
3670        binding: u32,
3671        decl: &wgt::BindGroupLayoutEntry,
3672        external_texture: &'a Arc<ExternalTexture>,
3673        texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3674        used: &mut BindGroupStates,
3675        snatch_guard: &'a SnatchGuard,
3676    ) -> Result<
3677        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3678        CreateBindGroupError,
3679    > {
3680        use crate::binding_model::CreateBindGroupError as Error;
3681
3682        let external_texture_state = external_texture.state()?;
3683        external_texture.same_device(self)?;
3684
3685        used.external_textures
3686            .insert_single(external_texture.clone());
3687
3688        match decl.ty {
3689            wgt::BindingType::ExternalTexture => {}
3690            _ => {
3691                return Err(Error::WrongBindingType {
3692                    binding,
3693                    actual: decl.ty,
3694                    expected: "ExternalTexture",
3695                });
3696            }
3697        }
3698
3699        let planes = (0..3)
3700            .map(|i| {
3701                // We always need 3 bindings. If we have fewer than 3 planes
3702                // just bind plane 0 multiple times. The shader will only
3703                // sample from valid planes anyway.
3704                let plane = external_texture
3705                    .planes
3706                    .get(i)
3707                    .unwrap_or(&external_texture.planes[0]);
3708                let internal_use = wgt::TextureUses::RESOURCE;
3709                used.views.insert_single(plane.clone(), internal_use);
3710                let view = plane.try_raw(snatch_guard)?;
3711                Ok(hal::TextureBinding {
3712                    view,
3713                    usage: internal_use,
3714                })
3715            })
3716            // We can remove this intermediate Vec by using
3717            // array::try_from_fn() above, once it stabilizes.
3718            .collect::<Result<Vec<_>, Error>>()?;
3719        let planes = planes.try_into().unwrap();
3720
3721        // Iterate over the distinct planes, rather than the three bindings produced
3722        // above, so that a plane repeated to fill the bindings is only recorded once.
3723        for plane in external_texture.planes.iter() {
3724            let texture = &plane.parent;
3725            texture_init_actions.push(TextureInitTrackerAction {
3726                texture: texture.clone(),
3727                range: TextureInitRange {
3728                    mip_range: plane.desc.range.mip_range(texture.desc.mip_level_count),
3729                    layer_range: plane
3730                        .desc
3731                        .range
3732                        .layer_range(texture.desc.array_layer_count()),
3733                },
3734                kind: MemoryInitKind::NeedsInitializedMemory,
3735            });
3736        }
3737
3738        used.buffers.insert_single(
3739            external_texture_state.params.clone(),
3740            wgt::BufferUses::UNIFORM,
3741        );
3742        let params = external_texture_state
3743            .params
3744            .binding(0, None, snatch_guard)?;
3745
3746        Ok(hal::ExternalTextureBinding { planes, params })
3747    }
3748
3749    fn create_external_texture_binding_from_view<'a>(
3750        &'a self,
3751        binding: u32,
3752        decl: &wgt::BindGroupLayoutEntry,
3753        view: &'a Arc<TextureView>,
3754        texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3755        used: &mut BindGroupStates,
3756        snatch_guard: &'a SnatchGuard,
3757    ) -> Result<
3758        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3759        CreateBindGroupError,
3760    > {
3761        use crate::binding_model::CreateBindGroupError as Error;
3762
3763        view.same_device(self)?;
3764        view.check_valid()?;
3765
3766        let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
3767        used.views.insert_single(view.clone(), internal_use);
3768
3769        match decl.ty {
3770            wgt::BindingType::ExternalTexture => {}
3771            _ => {
3772                return Err(Error::WrongBindingType {
3773                    binding,
3774                    actual: decl.ty,
3775                    expected: "ExternalTexture",
3776                });
3777            }
3778        }
3779
3780        // We need 3 bindings, so just repeat the same texture view 3 times.
3781        let planes = [
3782            hal::TextureBinding {
3783                view: view.try_raw(snatch_guard)?,
3784                usage: internal_use,
3785            },
3786            hal::TextureBinding {
3787                view: view.try_raw(snatch_guard)?,
3788                usage: internal_use,
3789            },
3790            hal::TextureBinding {
3791                view: view.try_raw(snatch_guard)?,
3792                usage: internal_use,
3793            },
3794        ];
3795        let params = hal::BufferBinding::new_unchecked(
3796            self.default_external_texture_params_buffer.as_ref(),
3797            0,
3798            NonZeroU64::new(size_of::<ExternalTextureParams>() as u64).unwrap(),
3799        );
3800
3801        texture_init_actions.push(TextureInitTrackerAction {
3802            texture: view.parent.clone(),
3803            range: TextureInitRange {
3804                mip_range: view.desc.range.mip_range(view.parent.desc.mip_level_count),
3805                layer_range: view
3806                    .desc
3807                    .range
3808                    .layer_range(view.parent.desc.array_layer_count()),
3809            },
3810            kind: MemoryInitKind::NeedsInitializedMemory,
3811        });
3812
3813        Ok(hal::ExternalTextureBinding { planes, params })
3814    }
3815
3816    pub fn create_bind_group(
3817        self: &Arc<Self>,
3818        desc: &binding_model::BindGroupDescriptor,
3819    ) -> Arc<BindGroup> {
3820        profiling::scope!("Device::create_bind_group");
3821        #[cfg(feature = "trace")]
3822        let trace_desc = (&desc).to_trace();
3823
3824        let bind_group = self.create_bind_group_inner(desc).unwrap_or_else(|err| {
3825            self.handle_error(err, desc.label.as_deref(), "Device::create_bind_group");
3826            BindGroup::invalid(self.clone(), desc.label.to_string(), desc.layout.clone())
3827        });
3828
3829        #[cfg(feature = "trace")]
3830        if let Some(ref mut trace) = *self.trace.lock() {
3831            trace.add(trace::Action::CreateBindGroup(
3832                bind_group.to_trace(),
3833                trace_desc,
3834            ));
3835        }
3836
3837        api_log!(
3838            "Device::create_bind_group -> {:?}",
3839            Arc::as_ptr(&bind_group)
3840        );
3841
3842        bind_group
3843    }
3844
3845    // This function expects the provided bind group layout to be resolved
3846    // (not passing a duplicate) beforehand.
3847    pub fn create_bind_group_inner(
3848        self: &Arc<Self>,
3849        desc: &binding_model::BindGroupDescriptor,
3850    ) -> Result<Arc<BindGroup>, CreateBindGroupError> {
3851        use crate::binding_model::{BindingResource as Br, CreateBindGroupError as Error};
3852
3853        self.check_is_valid()?;
3854
3855        let layout = desc.layout.clone();
3856
3857        layout.same_device(self)?;
3858        layout.check_is_valid()?;
3859
3860        {
3861            // Check that the number of entries in the descriptor matches
3862            // the number of entries in the layout.
3863            let actual = desc.entries.len();
3864            let expected = layout.entries.len();
3865            if actual != expected {
3866                return Err(Error::BindingsNumMismatch { expected, actual });
3867            }
3868        }
3869
3870        // Record binding info for dynamic offset validation
3871        let mut dynamic_binding_info = Vec::new();
3872        // Map of binding -> shader reflected size
3873        //Note: we can't collect into a vector right away because
3874        // it needs to be in BGL iteration order, not BG entry order.
3875        let mut late_buffer_binding_sizes = FastHashMap::default();
3876        // fill out the descriptors
3877        let mut used = BindGroupStates::new();
3878
3879        let mut buffer_init_actions = Vec::new();
3880        let mut texture_init_actions = Vec::new();
3881        let snatch_guard = self.snatchable_lock.read();
3882        let mut hal_entries = SmallVec::<[_; 4]>::with_capacity(desc.entries.len());
3883        let mut hal_buffers = SmallVec::<[_; 4]>::new();
3884        let mut hal_samplers = SmallVec::<[_; 4]>::new();
3885        let mut hal_textures = SmallVec::<[_; 8]>::new();
3886        let mut hal_tlas_s = SmallVec::<[_; 1]>::new();
3887        let mut hal_external_textures = SmallVec::<[_; 1]>::new();
3888        for entry in desc.entries.iter() {
3889            let binding = entry.binding;
3890            // Find the corresponding declaration in the layout
3891            let decl = layout
3892                .entries
3893                .get(binding)
3894                .ok_or(Error::MissingBindingDeclaration(binding))?;
3895            let (res_index, count) = match entry.resource {
3896                Br::Buffer(ref bb) => {
3897                    let bb = self.create_buffer_binding(
3898                        bb,
3899                        binding,
3900                        decl,
3901                        &mut buffer_init_actions,
3902                        &mut dynamic_binding_info,
3903                        &mut late_buffer_binding_sizes,
3904                        &mut used,
3905                        &snatch_guard,
3906                    )?;
3907
3908                    let res_index = hal_buffers.len();
3909                    hal_buffers.push(bb);
3910                    (res_index, 1)
3911                }
3912                Br::BufferArray(ref bindings_array) => {
3913                    let num_bindings = bindings_array.len();
3914                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3915
3916                    let res_index = hal_buffers.len();
3917                    for bb in bindings_array.iter() {
3918                        let bb = self.create_buffer_binding(
3919                            bb,
3920                            binding,
3921                            decl,
3922                            &mut buffer_init_actions,
3923                            &mut dynamic_binding_info,
3924                            &mut late_buffer_binding_sizes,
3925                            &mut used,
3926                            &snatch_guard,
3927                        )?;
3928                        hal_buffers.push(bb);
3929                    }
3930                    (res_index, num_bindings)
3931                }
3932                Br::Sampler(ref sampler) => {
3933                    let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3934
3935                    let res_index = hal_samplers.len();
3936                    hal_samplers.push(sampler);
3937                    (res_index, 1)
3938                }
3939                Br::SamplerArray(ref samplers) => {
3940                    let num_bindings = samplers.len();
3941                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3942
3943                    let res_index = hal_samplers.len();
3944                    for sampler in samplers.iter() {
3945                        let sampler =
3946                            self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3947
3948                        hal_samplers.push(sampler);
3949                    }
3950
3951                    (res_index, num_bindings)
3952                }
3953                Br::TextureView(ref view) => match decl.ty {
3954                    wgt::BindingType::ExternalTexture => {
3955                        let et = self.create_external_texture_binding_from_view(
3956                            binding,
3957                            decl,
3958                            view,
3959                            &mut texture_init_actions,
3960                            &mut used,
3961                            &snatch_guard,
3962                        )?;
3963                        let res_index = hal_external_textures.len();
3964                        hal_external_textures.push(et);
3965                        (res_index, 1)
3966                    }
3967                    _ => {
3968                        let tb = self.create_texture_binding(
3969                            binding,
3970                            decl,
3971                            view,
3972                            &mut used,
3973                            &mut texture_init_actions,
3974                            &snatch_guard,
3975                        )?;
3976                        let res_index = hal_textures.len();
3977                        hal_textures.push(tb);
3978                        (res_index, 1)
3979                    }
3980                },
3981                Br::TextureViewArray(ref views) => {
3982                    let num_bindings = views.len();
3983                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3984
3985                    let res_index = hal_textures.len();
3986                    for view in views.iter() {
3987                        let tb = self.create_texture_binding(
3988                            binding,
3989                            decl,
3990                            view,
3991                            &mut used,
3992                            &mut texture_init_actions,
3993                            &snatch_guard,
3994                        )?;
3995
3996                        hal_textures.push(tb);
3997                    }
3998
3999                    (res_index, num_bindings)
4000                }
4001                Br::AccelerationStructure(ref tlas) => {
4002                    let tlas =
4003                        self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
4004                    let res_index = hal_tlas_s.len();
4005                    hal_tlas_s.push(tlas);
4006                    (res_index, 1)
4007                }
4008                Br::AccelerationStructureArray(ref tlas_array) => {
4009                    // Feature validation for TLAS binding arrays happens at bind group layout
4010                    // creation time (mirroring other binding-array resource types). By the time we
4011                    // get here, `decl.count` has already been validated against device features.
4012                    let num_bindings = tlas_array.len();
4013                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
4014
4015                    let res_index = hal_tlas_s.len();
4016                    for tlas in tlas_array.iter() {
4017                        let tlas = self.create_tlas_binding(
4018                            &mut used,
4019                            binding,
4020                            decl,
4021                            tlas,
4022                            &snatch_guard,
4023                        )?;
4024                        hal_tlas_s.push(tlas);
4025                    }
4026                    (res_index, num_bindings)
4027                }
4028                Br::ExternalTexture(ref et) => {
4029                    let et = self.create_external_texture_binding(
4030                        binding,
4031                        decl,
4032                        et,
4033                        &mut texture_init_actions,
4034                        &mut used,
4035                        &snatch_guard,
4036                    )?;
4037                    let res_index = hal_external_textures.len();
4038                    hal_external_textures.push(et);
4039                    (res_index, 1)
4040                }
4041            };
4042
4043            hal_entries.push(hal::BindGroupEntry {
4044                binding,
4045                resource_index: res_index as u32,
4046                count: count as u32,
4047            });
4048        }
4049
4050        used.optimize();
4051
4052        hal_entries.sort_by_key(|entry| entry.binding);
4053        for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
4054            if a.binding == b.binding {
4055                return Err(Error::DuplicateBinding(a.binding));
4056            }
4057        }
4058
4059        dynamic_binding_info.sort_by_key(|i| i.binding_idx);
4060
4061        let hal_desc = hal::BindGroupDescriptor {
4062            label: desc.label.to_hal(self.instance_flags),
4063            layout: layout.try_raw()?,
4064            entries: &hal_entries,
4065            buffers: &hal_buffers,
4066            samplers: &hal_samplers,
4067            textures: &hal_textures,
4068            acceleration_structures: &hal_tlas_s,
4069            external_textures: &hal_external_textures,
4070        };
4071        let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
4072            .map_err(|e| self.handle_hal_error(e))?;
4073
4074        // collect in the order of BGL iteration
4075        let late_buffer_binding_infos = layout
4076            .entries
4077            .indices()
4078            .flat_map(|binding| {
4079                let size = late_buffer_binding_sizes.get(&binding).cloned()?;
4080                Some(BindGroupLateBufferBindingInfo {
4081                    binding_index: binding,
4082                    size,
4083                })
4084            })
4085            .collect();
4086
4087        let bind_group = BindGroup {
4088            state: ResourceState::Valid(BindGroupState {
4089                raw: Snatchable::new(raw),
4090            }),
4091            device: self.clone(),
4092            layout,
4093            label: desc.label.to_string(),
4094            tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
4095            used,
4096            buffer_init_actions,
4097            texture_init_actions,
4098            dynamic_binding_info,
4099            late_buffer_binding_infos,
4100        };
4101
4102        let bind_group = Arc::new(bind_group);
4103
4104        let weak_ref = Arc::downgrade(&bind_group);
4105        for texture in bind_group.used.views.used_textures() {
4106            let mut bind_groups = texture.bind_groups.lock();
4107            bind_groups.push(weak_ref.clone());
4108        }
4109        for buffer in bind_group.used.buffers.used_resources() {
4110            let mut bind_groups = buffer.bind_groups.lock();
4111            bind_groups.push(weak_ref.clone());
4112        }
4113
4114        Ok(bind_group)
4115    }
4116
4117    fn check_array_binding(
4118        features: wgt::Features,
4119        count: Option<NonZeroU32>,
4120        num_bindings: usize,
4121    ) -> Result<(), CreateBindGroupError> {
4122        use super::binding_model::CreateBindGroupError as Error;
4123
4124        if let Some(count) = count {
4125            let count = count.get() as usize;
4126            if count < num_bindings {
4127                return Err(Error::BindingArrayPartialLengthMismatch {
4128                    actual: num_bindings,
4129                    expected: count,
4130                });
4131            }
4132            if count != num_bindings
4133                && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
4134            {
4135                return Err(Error::BindingArrayLengthMismatch {
4136                    actual: num_bindings,
4137                    expected: count,
4138                });
4139            }
4140            if num_bindings == 0 {
4141                return Err(Error::BindingArrayZeroLength);
4142            }
4143        } else {
4144            return Err(Error::SingleBindingExpected);
4145        };
4146
4147        Ok(())
4148    }
4149
4150    fn texture_use_parameters(
4151        &self,
4152        binding: u32,
4153        decl: &wgt::BindGroupLayoutEntry,
4154        view: &TextureView,
4155        expected: &'static str,
4156    ) -> Result<wgt::TextureUses, CreateBindGroupError> {
4157        use crate::binding_model::CreateBindGroupError as Error;
4158        if view
4159            .desc
4160            .aspects()
4161            .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
4162        {
4163            return Err(Error::DepthStencilAspect);
4164        }
4165        match decl.ty {
4166            wgt::BindingType::Texture {
4167                sample_type,
4168                view_dimension,
4169                multisampled,
4170            } => {
4171                use wgt::TextureSampleType as Tst;
4172                if multisampled != (view.samples != 1) {
4173                    return Err(Error::InvalidTextureMultisample {
4174                        binding,
4175                        layout_multisampled: multisampled,
4176                        view_samples: view.samples,
4177                    });
4178                }
4179                let compat_sample_type = view
4180                    .desc
4181                    .format
4182                    .sample_type(Some(view.desc.range.aspect), Some(self.features))
4183                    .unwrap();
4184                match (sample_type, compat_sample_type) {
4185                    (Tst::Uint, Tst::Uint) |
4186                        (Tst::Sint, Tst::Sint) |
4187                        (Tst::Depth, Tst::Depth) |
4188                        // if we expect non-filterable, accept anything float
4189                        (Tst::Float { filterable: false }, Tst::Float { .. }) |
4190                        // if we expect filterable, require it
4191                        (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
4192                        // if we expect non-filterable, also accept depth
4193                        (Tst::Float { filterable: false }, Tst::Depth) => {}
4194                    // if we expect filterable, also accept Float that is defined as
4195                    // unfilterable if filterable feature is explicitly enabled (only hit
4196                    // if wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES is
4197                    // enabled)
4198                    (Tst::Float { filterable: true }, Tst::Float { .. })
4199                        if view.format_features.flags
4200                            .contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
4201                    _ => {
4202                        return Err(Error::InvalidTextureSampleType {
4203                            binding,
4204                            layout_sample_type: sample_type,
4205                            view_format: view.desc.format,
4206                            view_sample_type: compat_sample_type,
4207                        })
4208                    }
4209                }
4210                if view_dimension != view.desc.dimension {
4211                    return Err(Error::InvalidTextureDimension {
4212                        binding,
4213                        layout_dimension: view_dimension,
4214                        view_dimension: view.desc.dimension,
4215                    });
4216                }
4217                if let Some(texture_binding_view_dimension) =
4218                    view.parent.texture_binding_view_dimension()
4219                {
4220                    if texture_binding_view_dimension != view.desc.dimension {
4221                        return Err(Error::InvalidTextureBindingViewDimension {
4222                            binding,
4223                            view_dimension: view.desc.dimension,
4224                            texture_binding_view_dimension,
4225                        });
4226                    }
4227                }
4228                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
4229                let depth_stencil_uses = if view.desc.aspects() == hal::FormatAspects::DEPTH {
4230                    wgt::TextureUses::DEPTH_SAMPLED
4231                } else if view.desc.aspects() == hal::FormatAspects::STENCIL {
4232                    wgt::TextureUses::STENCIL_SAMPLED
4233                } else {
4234                    wgt::TextureUses::empty()
4235                };
4236                Ok(wgt::TextureUses::RESOURCE | depth_stencil_uses)
4237            }
4238            wgt::BindingType::StorageTexture {
4239                access,
4240                format,
4241                view_dimension,
4242            } => {
4243                if format != view.desc.format {
4244                    return Err(Error::InvalidStorageTextureFormat {
4245                        binding,
4246                        layout_format: format,
4247                        view_format: view.desc.format,
4248                    });
4249                }
4250                if view_dimension != view.desc.dimension {
4251                    return Err(Error::InvalidTextureDimension {
4252                        binding,
4253                        layout_dimension: view_dimension,
4254                        view_dimension: view.desc.dimension,
4255                    });
4256                }
4257                if let Some(texture_binding_view_dimension) =
4258                    view.parent.texture_binding_view_dimension()
4259                {
4260                    if texture_binding_view_dimension != view.desc.dimension {
4261                        return Err(Error::InvalidTextureBindingViewDimension {
4262                            binding,
4263                            view_dimension: view.desc.dimension,
4264                            texture_binding_view_dimension,
4265                        });
4266                    }
4267                }
4268
4269                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
4270                if mip_level_count != 1 {
4271                    return Err(Error::InvalidStorageTextureMipLevelCount {
4272                        binding,
4273                        mip_level_count,
4274                    });
4275                }
4276
4277                if view.desc.swizzle != wgt::TextureComponentSwizzle::default() {
4278                    return Err(Error::InvalidStorageTextureSwizzle {
4279                        swizzle: view.desc.swizzle,
4280                    });
4281                }
4282
4283                view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
4284
4285                Ok(match access {
4286                    wgt::StorageTextureAccess::ReadOnly => wgt::TextureUses::STORAGE_READ_ONLY,
4287                    wgt::StorageTextureAccess::WriteOnly => wgt::TextureUses::STORAGE_WRITE_ONLY,
4288                    wgt::StorageTextureAccess::ReadWrite => wgt::TextureUses::STORAGE_READ_WRITE,
4289                    wgt::StorageTextureAccess::Atomic => wgt::TextureUses::STORAGE_ATOMIC,
4290                })
4291            }
4292            wgt::BindingType::ExternalTexture => {
4293                if view.desc.dimension != TextureViewDimension::D2 {
4294                    return Err(Error::InvalidTextureDimension {
4295                        binding,
4296                        layout_dimension: TextureViewDimension::D2,
4297                        view_dimension: view.desc.dimension,
4298                    });
4299                }
4300                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
4301                if mip_level_count != 1 {
4302                    return Err(Error::InvalidExternalTextureMipLevelCount {
4303                        binding,
4304                        mip_level_count,
4305                    });
4306                }
4307                if view.desc.format != TextureFormat::Rgba8Unorm
4308                    && view.desc.format != TextureFormat::Bgra8Unorm
4309                    && view.desc.format != TextureFormat::Rgba16Float
4310                {
4311                    return Err(Error::InvalidExternalTextureFormat {
4312                        binding,
4313                        format: view.desc.format,
4314                    });
4315                }
4316                if view.samples != 1 {
4317                    return Err(Error::InvalidTextureMultisample {
4318                        binding,
4319                        layout_multisampled: false,
4320                        view_samples: view.samples,
4321                    });
4322                }
4323
4324                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
4325                Ok(wgt::TextureUses::RESOURCE)
4326            }
4327            _ => Err(Error::WrongBindingType {
4328                binding,
4329                actual: decl.ty,
4330                expected,
4331            }),
4332        }
4333    }
4334
4335    pub fn create_pipeline_layout(
4336        self: &Arc<Self>,
4337        desc: &binding_model::PipelineLayoutDescriptor,
4338    ) -> Arc<binding_model::PipelineLayout> {
4339        profiling::scope!("Device::create_pipeline_layout");
4340        let layout = self
4341            .create_pipeline_layout_impl(desc, false)
4342            .unwrap_or_else(|err| {
4343                self.handle_error(err, desc.label.as_deref(), "Device::create_pipeline_layout");
4344                binding_model::PipelineLayout::invalid(Arc::clone(self), desc.label.to_string())
4345            });
4346        #[cfg(feature = "trace")]
4347        if let Some(ref mut trace) = *self.trace.lock() {
4348            use crate::device::trace::IntoTrace;
4349            trace.add(trace::Action::CreatePipelineLayout(
4350                layout.to_trace(),
4351                desc.to_trace(),
4352            ));
4353        }
4354        api_log!(
4355            "Device::create_pipeline_layout -> {:?}",
4356            Arc::as_ptr(&layout)
4357        );
4358        layout
4359    }
4360
4361    fn create_pipeline_layout_impl(
4362        self: &Arc<Self>,
4363        desc: &binding_model::PipelineLayoutDescriptor,
4364        ignore_exclusive_pipeline_check: bool,
4365    ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
4366        use crate::binding_model::CreatePipelineLayoutError as Error;
4367
4368        self.check_is_valid()?;
4369
4370        let bind_group_layouts_count = desc.bind_group_layouts.len();
4371        let device_max_bind_groups = self.limits.max_bind_groups as usize;
4372        if bind_group_layouts_count > device_max_bind_groups {
4373            return Err(Error::TooManyGroups {
4374                actual: bind_group_layouts_count,
4375                max: device_max_bind_groups,
4376            });
4377        }
4378
4379        if desc.immediate_size != 0 {
4380            self.require_features(wgt::Features::IMMEDIATES)?;
4381        }
4382        if self.limits.max_immediate_size < desc.immediate_size {
4383            return Err(Error::ImmediateRangeTooLarge {
4384                size: desc.immediate_size,
4385                max: self.limits.max_immediate_size,
4386            });
4387        }
4388        if !desc
4389            .immediate_size
4390            .is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT)
4391        {
4392            return Err(Error::MisalignedImmediateSize {
4393                size: desc.immediate_size,
4394            });
4395        }
4396
4397        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
4398
4399        for (index, bgl) in desc.bind_group_layouts.iter().enumerate() {
4400            let Some(bgl) = bgl else {
4401                continue;
4402            };
4403
4404            bgl.same_device(self)?;
4405
4406            if !ignore_exclusive_pipeline_check {
4407                let exclusive_pipeline = bgl.exclusive_pipeline.get().unwrap();
4408                if !matches!(exclusive_pipeline, binding_model::ExclusivePipeline::None) {
4409                    return Err(Error::BglHasExclusivePipeline {
4410                        index,
4411                        pipeline: alloc::format!("{exclusive_pipeline}"),
4412                    });
4413                }
4414            }
4415
4416            count_validator.merge(&bgl.state()?.binding_count_validator);
4417        }
4418
4419        count_validator
4420            .validate(&self.limits, self.instance_flags)
4421            .map_err(Error::TooManyBindings)?;
4422
4423        let buffers_and_acceleration_structures_in_vertex_stage =
4424            count_validator.buffers_and_acceleration_structures_in_vertex_stage();
4425
4426        let get_bgl_iter = || {
4427            desc.bind_group_layouts
4428                .iter()
4429                .map(|bgl| bgl.as_ref().filter(|bgl| !bgl.entries.is_empty()))
4430        };
4431
4432        let bind_group_layouts = get_bgl_iter()
4433            .map(|bgl| bgl.cloned())
4434            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
4435
4436        let raw_bind_group_layouts = get_bgl_iter()
4437            .map(|bgl| bgl.map(|bgl| bgl.try_raw()).transpose())
4438            .collect::<Result<ArrayVec<_, { hal::MAX_BIND_GROUPS }>, _>>()?;
4439
4440        let additional_flags = if self.indirect_validation.is_some() {
4441            hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
4442        } else {
4443            hal::PipelineLayoutFlags::empty()
4444        };
4445
4446        let hal_desc = hal::PipelineLayoutDescriptor {
4447            label: desc.label.to_hal(self.instance_flags),
4448            flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
4449                | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
4450                | additional_flags,
4451            bind_group_layouts: &raw_bind_group_layouts,
4452            immediate_size: desc.immediate_size,
4453        };
4454
4455        let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
4456            .map_err(|e| self.handle_hal_error(e))?;
4457
4458        drop(raw_bind_group_layouts);
4459
4460        let layout = binding_model::PipelineLayout {
4461            raw: ResourceState::Valid(raw),
4462            device: self.clone(),
4463            label: desc.label.to_string(),
4464            bind_group_layouts,
4465            immediate_size: desc.immediate_size,
4466            buffers_and_acceleration_structures_in_vertex_stage,
4467        };
4468
4469        let layout = Arc::new(layout);
4470
4471        Ok(layout)
4472    }
4473
4474    pub(super) fn create_derived_pipeline_layout(
4475        self: &Arc<Self>,
4476        mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
4477        immediate_size: u32,
4478    ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
4479        // <https://gpuweb.github.io/gpuweb/#abstract-opdef-default-pipeline-layout>
4480        // Round up the immediate size for pipeline layout as it is required to be a multiple of 4
4481        let immediate_size = align_to(immediate_size, wgt::IMMEDIATE_DATA_ALIGNMENT);
4482
4483        while derived_group_layouts
4484            .last()
4485            .is_some_and(|map| map.is_empty())
4486        {
4487            derived_group_layouts.pop();
4488        }
4489
4490        let mut unique_bind_group_layouts = FastHashMap::default();
4491
4492        let bind_group_layouts = derived_group_layouts
4493            .into_iter()
4494            .map(|mut bgl_entry_map| {
4495                if bgl_entry_map.is_empty() {
4496                    return Ok(None);
4497                }
4498
4499                bgl_entry_map.sort();
4500                match unique_bind_group_layouts.entry(bgl_entry_map) {
4501                    hashbrown::hash_map::Entry::Occupied(v) => Ok(Some(Arc::clone(v.get()))),
4502                    hashbrown::hash_map::Entry::Vacant(e) => {
4503                        match self.create_bind_group_layout_impl(
4504                            &None,
4505                            e.key().clone(),
4506                            bgl::Origin::Derived,
4507                        ) {
4508                            Ok(bgl) => {
4509                                e.insert(bgl.clone());
4510                                Ok(Some(bgl))
4511                            }
4512                            Err(e) => Err(e),
4513                        }
4514                    }
4515                }
4516            })
4517            .collect::<Result<Vec<_>, _>>()?;
4518
4519        let layout_desc = binding_model::PipelineLayoutDescriptor {
4520            label: None,
4521            bind_group_layouts: Cow::Owned(bind_group_layouts),
4522            immediate_size,
4523        };
4524
4525        let layout = self.create_pipeline_layout_impl(&layout_desc, true)?;
4526        Ok(layout)
4527    }
4528
4529    /// Creates a compute pipeline. If the creation fails,
4530    /// it will handle error in device and return an invalid compute pipeline.
4531    ///
4532    /// Corresponds to [GPUDevice.createComputePipeline](https://www.w3.org/TR/webgpu/#dom-gpudevice-createcomputepipeline)
4533    pub fn create_compute_pipeline(
4534        self: &Arc<Self>,
4535        desc: pipeline::ComputePipelineDescriptor,
4536    ) -> Arc<pipeline::ComputePipeline> {
4537        profiling::scope!("Device::create_compute_pipeline");
4538        let compute_pipeline = self
4539            .create_compute_pipeline_or_error_inner(desc.clone())
4540            .unwrap_or_else(|err| {
4541                if let pipeline::CreatePipelineError::Internal { stage, ref message } = err {
4542                    log::error!("Shader translation error for stage {stage:?}: {message}");
4543                    log::error!("Please report it to https://github.com/gfx-rs/wgpu");
4544                }
4545                self.handle_error(
4546                    err,
4547                    desc.label.as_deref(),
4548                    "Device::create_compute_pipeline",
4549                );
4550
4551                pipeline::ComputePipeline::invalid(self.clone(), desc.label.to_string())
4552            });
4553        #[cfg(feature = "trace")]
4554        if let Some(ref mut trace) = *self.trace.lock() {
4555            use crate::device::trace;
4556            use crate::device::trace::IntoTrace;
4557            trace.add(trace::Action::CreateComputePipeline {
4558                id: compute_pipeline.to_trace(),
4559                desc: desc.to_trace(),
4560            });
4561        }
4562        api_log!(
4563            "Device::create_compute_pipeline -> {:?}",
4564            Arc::as_ptr(&compute_pipeline)
4565        );
4566        compute_pipeline
4567    }
4568
4569    /// Creates a compute pipeline without raising any error to device.
4570    /// Device lost errors will be mapped to invalid compute pipeline
4571    /// as required by specification.
4572    ///
4573    /// Corresponds to [GPUDevice.createComputePipelineAsync](https://www.w3.org/TR/webgpu/#dom-gpudevice-createcomputepipelineasync)
4574    pub fn create_compute_pipeline_or_error(
4575        self: &Arc<Self>,
4576        desc: pipeline::ComputePipelineDescriptor,
4577    ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreatePipelineError> {
4578        let label = desc.label.to_string();
4579        match self.create_compute_pipeline_or_error_inner(desc) {
4580            Err(err) if err.webgpu_error_type() == wgt::error::ErrorType::DeviceLost => {
4581                Ok(pipeline::ComputePipeline::invalid(self.clone(), label))
4582            }
4583            result => result,
4584        }
4585    }
4586
4587    fn create_compute_pipeline_or_error_inner(
4588        self: &Arc<Self>,
4589        desc: pipeline::ComputePipelineDescriptor,
4590    ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreatePipelineError> {
4591        self.check_is_valid()?;
4592
4593        self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
4594
4595        let is_auto_layout = desc.layout.is_none();
4596
4597        // Get the pipeline layout from the desc if it is provided.
4598        let pipeline_layout = match desc.layout {
4599            Some(pipeline_layout) => {
4600                pipeline_layout.same_device(self)?;
4601                pipeline_layout.check_valid()?;
4602                Some(pipeline_layout)
4603            }
4604            None => None,
4605        };
4606
4607        let mut compiler =
4608            StageCompiler::new(self, pipeline_layout, validation::StageIo::default(), None);
4609        let compute_stage =
4610            compiler.compile(&desc.stage, validation::ShaderStageForValidation::Compute)?;
4611
4612        let resources = compiler.resources;
4613
4614        let pipeline_layout = match resources.layout {
4615            validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4616            validation::BindingLayoutSource::Derived(entries) => {
4617                self.create_derived_pipeline_layout(entries, resources.immediates.size())?
4618            }
4619        };
4620
4621        let naga::valid::ImmediateUsage::Valid {
4622            slots: immediate_slots_required,
4623            size: _,
4624        } = resources.immediates
4625        else {
4626            unreachable!("Immediates exceeding maxImmediateSize should have been rejected");
4627        };
4628
4629        let late_sized_buffer_groups = Device::make_late_sized_buffer_groups(
4630            &resources.minimum_binding_sizes,
4631            &pipeline_layout,
4632        );
4633
4634        let cache = match desc.cache {
4635            Some(cache) => {
4636                cache.check_is_valid()?;
4637                cache.same_device(self)?;
4638                Some(cache)
4639            }
4640            None => None,
4641        };
4642
4643        let pipeline_desc = hal::ComputePipelineDescriptor {
4644            label: desc.label.to_hal(self.instance_flags),
4645            layout: pipeline_layout.raw()?,
4646            stage: compute_stage.to_hal(),
4647            cache: cache.as_ref().map(|it| it.raw()).transpose()?,
4648        };
4649
4650        let raw = unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }
4651            .map_err(|error| self.handle_hal_pipeline_error(error))?;
4652        drop(compute_stage);
4653
4654        let pipeline = pipeline::ComputePipeline {
4655            state: ResourceState::Valid(pipeline::ComputePipelineState {
4656                raw: ManuallyDrop::new(raw),
4657                layout: pipeline_layout.clone(),
4658                _shader_module: desc.stage.module,
4659            }),
4660            device: self.clone(),
4661            late_sized_buffer_groups,
4662            immediate_slots_required,
4663            label: desc.label.to_string(),
4664            tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
4665        };
4666
4667        let pipeline = Arc::new(pipeline);
4668
4669        if is_auto_layout {
4670            for bgl in pipeline_layout.bind_group_layouts.iter() {
4671                let Some(bgl) = bgl else {
4672                    continue;
4673                };
4674
4675                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the
4676                // result.
4677                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
4678            }
4679        }
4680
4681        Ok(pipeline)
4682    }
4683
4684    /// Creates a render pipeline. If the creation fails,
4685    /// it will handle error in device and return an invalid render pipeline.
4686    ///
4687    /// Corresponds to [GPUDevice.createRenderPipeline](https://www.w3.org/TR/webgpu/#dom-gpudevice-createrenderpipeline)
4688    pub fn create_render_pipeline(
4689        self: &Arc<Self>,
4690        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4691    ) -> Arc<pipeline::RenderPipeline> {
4692        profiling::scope!("Device::create_render_pipeline");
4693
4694        let render_pipeline = self
4695            .create_render_pipeline_or_error_inner(desc.clone())
4696            .unwrap_or_else(|err| {
4697                if let pipeline::CreatePipelineError::Internal { stage, ref message } = err {
4698                    log::error!("Shader translation error for stage {stage:?}: {message}");
4699                    log::error!("Please report it to https://github.com/gfx-rs/wgpu");
4700                }
4701                self.handle_error(err, desc.label.as_deref(), "Device::create_render_pipeline");
4702                pipeline::RenderPipeline::invalid(self.clone(), desc.label.to_string())
4703            });
4704        #[cfg(feature = "trace")]
4705        if let Some(ref mut trace) = *self.trace.lock() {
4706            use crate::device::trace::IntoTrace;
4707            trace.add(trace::Action::CreateGeneralRenderPipeline {
4708                id: render_pipeline.to_trace(),
4709                desc: desc.to_trace(),
4710            });
4711        }
4712        api_log!(
4713            "Device::create_render_pipeline -> {:?}",
4714            Arc::as_ptr(&render_pipeline)
4715        );
4716        render_pipeline
4717    }
4718
4719    /// Creates a render pipeline without raising any error to device.
4720    /// Device lost errors will be mapped to invalid render pipeline
4721    /// as required by specification.
4722    ///
4723    /// Corresponds to [GPUDevice.createRenderPipelineAsync](https://www.w3.org/TR/webgpu/#dom-gpudevice-createrenderpipelineasync)
4724    pub fn create_render_pipeline_or_error(
4725        self: &Arc<Self>,
4726        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4727    ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreatePipelineError> {
4728        let label = desc.label.to_string();
4729        match self.create_render_pipeline_or_error_inner(desc) {
4730            Err(e) if e.webgpu_error_type() == wgt::error::ErrorType::DeviceLost => Ok(
4731                pipeline::RenderPipeline::invalid(self.clone(), label.to_string()),
4732            ),
4733            result => result,
4734        }
4735    }
4736
4737    fn create_render_pipeline_or_error_inner(
4738        self: &Arc<Self>,
4739        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4740    ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreatePipelineError> {
4741        use wgt::TextureFormatFeatureFlags as Tfff;
4742
4743        self.check_is_valid()?;
4744
4745        let color_targets = desc
4746            .fragment
4747            .as_ref()
4748            .map_or(&[][..], |fragment| &fragment.targets);
4749        let depth_stencil_state = desc.depth_stencil.as_ref();
4750
4751        check_color_attachment_count(color_targets.len(), self.limits.max_color_attachments)?;
4752
4753        {
4754            let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
4755                color_targets.iter().filter_map(|x| x.as_ref()).collect();
4756            if !cts.is_empty() && {
4757                let first = &cts[0];
4758                cts[1..]
4759                    .iter()
4760                    .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
4761            } {
4762                self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
4763            }
4764        }
4765
4766        let mut io = validation::StageIo::default();
4767
4768        let mut vertex_steps;
4769        let mut hal_vertex_buffer_layouts;
4770        let mut total_attributes;
4771        let mut dual_source_blending = false;
4772        let mut has_depth_attachment = false;
4773        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4774            if vertex.buffers.len() > self.limits.max_vertex_buffers as usize {
4775                return Err(pipeline::RenderError::TooManyVertexBuffers {
4776                    given: vertex.buffers.len() as u32,
4777                    limit: self.limits.max_vertex_buffers,
4778                }
4779                .into());
4780            }
4781
4782            vertex_steps = Vec::with_capacity(vertex.buffers.len());
4783            hal_vertex_buffer_layouts = Vec::with_capacity(vertex.buffers.len());
4784            total_attributes = 0;
4785            for (i, vb_state) in vertex.buffers.iter().enumerate() {
4786                let Some(vb_state) = vb_state else {
4787                    vertex_steps.push(None);
4788                    hal_vertex_buffer_layouts.push(None);
4789                    continue;
4790                };
4791
4792                // https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpuvertexbufferlayout
4793
4794                if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
4795                    return Err(pipeline::RenderError::VertexStrideTooLarge {
4796                        index: i as u32,
4797                        given: vb_state.array_stride as u32,
4798                        limit: self.limits.max_vertex_buffer_array_stride,
4799                    }
4800                    .into());
4801                }
4802                if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
4803                    return Err(pipeline::RenderError::UnalignedVertexStride {
4804                        index: i as u32,
4805                        stride: vb_state.array_stride,
4806                    }
4807                    .into());
4808                }
4809
4810                let max_stride = if vb_state.array_stride == 0 {
4811                    self.limits.max_vertex_buffer_array_stride as u64
4812                } else {
4813                    vb_state.array_stride
4814                };
4815                let mut last_stride = 0;
4816                for attribute in vb_state.attributes.iter() {
4817                    let attribute_stride = attribute.offset + attribute.format.size();
4818                    if attribute_stride > max_stride {
4819                        return Err(pipeline::RenderError::VertexAttributeStrideTooLarge {
4820                            location: attribute.shader_location,
4821                            given: attribute_stride as u32,
4822                            limit: max_stride as u32,
4823                        }
4824                        .into());
4825                    }
4826
4827                    let required_offset_alignment = attribute.format.size().min(4);
4828                    if attribute.offset % required_offset_alignment != 0 {
4829                        return Err(pipeline::RenderError::InvalidVertexAttributeOffset {
4830                            location: attribute.shader_location,
4831                            offset: attribute.offset,
4832                        }
4833                        .into());
4834                    }
4835
4836                    if attribute.shader_location >= self.limits.max_vertex_attributes {
4837                        return Err(pipeline::RenderError::VertexAttributeLocationTooLarge {
4838                            given: attribute.shader_location,
4839                            limit: self.limits.max_vertex_attributes,
4840                        }
4841                        .into());
4842                    }
4843
4844                    last_stride = last_stride.max(attribute_stride);
4845                }
4846
4847                vertex_steps.push(Some(pipeline::VertexStep {
4848                    stride: vb_state.array_stride,
4849                    last_stride,
4850                    mode: vb_state.step_mode,
4851                }));
4852                hal_vertex_buffer_layouts.push(if vb_state.attributes.is_empty() {
4853                    None
4854                } else {
4855                    Some(hal::VertexBufferLayout {
4856                        array_stride: vb_state.array_stride,
4857                        step_mode: vb_state.step_mode,
4858                        attributes: vb_state.attributes.as_ref(),
4859                    })
4860                });
4861
4862                for attribute in vb_state.attributes.iter() {
4863                    if attribute.offset >= 0x10000000 {
4864                        return Err(pipeline::RenderError::InvalidVertexAttributeOffset {
4865                            location: attribute.shader_location,
4866                            offset: attribute.offset,
4867                        }
4868                        .into());
4869                    }
4870
4871                    if let wgt::VertexFormat::Float64
4872                    | wgt::VertexFormat::Float64x2
4873                    | wgt::VertexFormat::Float64x3
4874                    | wgt::VertexFormat::Float64x4 = attribute.format
4875                    {
4876                        self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
4877                    }
4878
4879                    let previous = io.varyings.insert(
4880                        attribute.shader_location,
4881                        validation::InterfaceVar::vertex_attribute(attribute.format),
4882                    );
4883
4884                    if previous.is_some() {
4885                        return Err(pipeline::RenderError::ShaderLocationClash(
4886                            attribute.shader_location,
4887                        )
4888                        .into());
4889                    }
4890                }
4891                total_attributes += vb_state.attributes.len();
4892            }
4893
4894            if total_attributes > self.limits.max_vertex_attributes as usize {
4895                return Err(pipeline::RenderError::TooManyVertexAttributes {
4896                    given: total_attributes as u32,
4897                    limit: self.limits.max_vertex_attributes,
4898                }
4899                .into());
4900            }
4901        } else {
4902            vertex_steps = Vec::new();
4903            hal_vertex_buffer_layouts = Vec::new();
4904        };
4905
4906        if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
4907            return Err(pipeline::RenderError::StripIndexFormatForNonStripTopology {
4908                strip_index_format: desc.primitive.strip_index_format,
4909                topology: desc.primitive.topology,
4910            }
4911            .into());
4912        }
4913
4914        if desc.primitive.unclipped_depth {
4915            self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
4916        }
4917
4918        if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
4919            self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
4920        }
4921        if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
4922            self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
4923        }
4924
4925        if desc.primitive.conservative {
4926            self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
4927        }
4928
4929        if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
4930            return Err(pipeline::RenderError::ConservativeRasterizationNonFillPolygonMode.into());
4931        }
4932
4933        let mut target_specified = false;
4934        let mut required_color_outputs = 0u64;
4935        const _: () = {
4936            assert!(hal::MAX_COLOR_ATTACHMENTS <= 64);
4937        };
4938
4939        for (i, cs) in color_targets.iter().enumerate() {
4940            if let Some(cs) = cs.as_ref() {
4941                target_specified = true;
4942                let error = 'error: {
4943                    // This is expected to be the operative check for illegal write mask
4944                    // values (larger than 15), because WebGPU requires that it be validated
4945                    // on the device timeline.
4946                    if cs.write_mask.contains_unknown_bits() {
4947                        break 'error Some(ColorStateError::InvalidWriteMask(cs.write_mask));
4948                    } else if cs.write_mask != ColorWrites::NONE {
4949                        required_color_outputs |= 1 << i;
4950                    }
4951
4952                    let format_features = self.describe_format_features(cs.format)?;
4953                    if !format_features
4954                        .allowed_usages
4955                        .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4956                    {
4957                        break 'error Some(ColorStateError::FormatNotRenderable(cs.format));
4958                    }
4959                    if cs.blend.is_some() && !format_features.flags.contains(Tfff::BLENDABLE) {
4960                        break 'error Some(ColorStateError::FormatNotBlendable(cs.format));
4961                    }
4962                    if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
4963                        break 'error Some(ColorStateError::FormatNotColor(cs.format));
4964                    }
4965
4966                    if desc.multisample.count > 1
4967                        && !format_features
4968                            .flags
4969                            .sample_count_supported(desc.multisample.count)
4970                    {
4971                        break 'error Some(ColorStateError::InvalidSampleCount(
4972                            desc.multisample.count,
4973                            cs.format,
4974                            cs.format
4975                                .guaranteed_format_features(self.features)
4976                                .flags
4977                                .supported_sample_counts(),
4978                            self.adapter
4979                                .get_texture_format_features(cs.format)
4980                                .flags
4981                                .supported_sample_counts(),
4982                        ));
4983                    }
4984
4985                    if let Some(blend_mode) = cs.blend {
4986                        for component in [&blend_mode.color, &blend_mode.alpha] {
4987                            for factor in [component.src_factor, component.dst_factor] {
4988                                if factor.uses_second_blend_source() {
4989                                    self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
4990                                    if i == 0 {
4991                                        dual_source_blending = true;
4992                                    } else {
4993                                        break 'error Some(
4994                                            ColorStateError::BlendFactorOnUnsupportedTarget {
4995                                                factor,
4996                                                target: i as u32,
4997                                            },
4998                                        );
4999                                    }
5000                                }
5001
5002                                if [wgt::BlendOperation::Min, wgt::BlendOperation::Max]
5003                                    .contains(&component.operation)
5004                                    && factor != wgt::BlendFactor::One
5005                                {
5006                                    break 'error Some(ColorStateError::InvalidMinMaxBlendFactor {
5007                                        factor,
5008                                        target: i as u32,
5009                                    });
5010                                }
5011                            }
5012                        }
5013                    }
5014
5015                    break 'error None;
5016                };
5017                if let Some(e) = error {
5018                    return Err(pipeline::RenderError::ColorState(i as u8, e).into());
5019                }
5020            }
5021        }
5022
5023        if dual_source_blending && color_targets.len() > 1 {
5024            return Err(
5025                pipeline::RenderError::DualSourceBlendingWithMultipleColorTargets {
5026                    count: color_targets.len(),
5027                }
5028                .into(),
5029            );
5030        }
5031
5032        validation::validate_color_attachment_bytes_per_sample(
5033            color_targets.iter().flatten().map(|cs| cs.format),
5034            self.limits.max_color_attachment_bytes_per_sample,
5035        )
5036        .map_err(pipeline::RenderError::ColorAttachment)?;
5037
5038        if let Some(ds) = depth_stencil_state {
5039            // See <https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpudepthstencilstate>.
5040            target_specified = true;
5041            let error = 'error: {
5042                if !ds.format.is_depth_stencil_format() {
5043                    // This error case is not redundant with the aspect check below when
5044                    // neither depth nor stencil is enabled at all.
5045                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepthOrStencil(
5046                        ds.format,
5047                    ));
5048                }
5049
5050                let format_features = self.describe_format_features(ds.format)?;
5051                if !format_features
5052                    .allowed_usages
5053                    .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
5054                {
5055                    break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
5056                        ds.format,
5057                    ));
5058                }
5059
5060                let aspect = hal::FormatAspects::from(ds.format);
5061                if aspect.contains(hal::FormatAspects::DEPTH) {
5062                    has_depth_attachment = true;
5063                } else if ds.is_depth_enabled() {
5064                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
5065                }
5066                if has_depth_attachment {
5067                    let Some(depth_write_enabled) = ds.depth_write_enabled else {
5068                        break 'error Some(
5069                            pipeline::DepthStencilStateError::MissingDepthWriteEnabled(ds.format),
5070                        );
5071                    };
5072
5073                    let depth_compare_required = depth_write_enabled
5074                        || ds.stencil.front.depth_fail_op != wgt::StencilOperation::Keep
5075                        || ds.stencil.back.depth_fail_op != wgt::StencilOperation::Keep;
5076                    if depth_compare_required && ds.depth_compare.is_none() {
5077                        break 'error Some(pipeline::DepthStencilStateError::MissingDepthCompare(
5078                            ds.format,
5079                        ));
5080                    }
5081                }
5082
5083                if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
5084                    break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
5085                        ds.format,
5086                    ));
5087                }
5088                if desc.multisample.count > 1
5089                    && !format_features
5090                        .flags
5091                        .sample_count_supported(desc.multisample.count)
5092                {
5093                    break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
5094                        desc.multisample.count,
5095                        ds.format,
5096                        ds.format
5097                            .guaranteed_format_features(self.features)
5098                            .flags
5099                            .supported_sample_counts(),
5100                        self.adapter
5101                            .get_texture_format_features(ds.format)
5102                            .flags
5103                            .supported_sample_counts(),
5104                    ));
5105                }
5106
5107                break 'error None;
5108            };
5109            if let Some(e) = error {
5110                return Err(pipeline::RenderError::DepthStencilState(e).into());
5111            }
5112
5113            if ds.bias.clamp != 0.0 {
5114                self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
5115            }
5116
5117            if (ds.bias.is_enabled() || ds.bias.clamp != 0.0)
5118                && !desc.primitive.topology.is_triangles()
5119            {
5120                return Err(pipeline::RenderError::DepthStencilState(
5121                    pipeline::DepthStencilStateError::DepthBiasWithIncompatibleTopology(
5122                        desc.primitive.topology,
5123                    ),
5124                )
5125                .into());
5126            }
5127        }
5128
5129        if !target_specified {
5130            return Err(pipeline::RenderError::NoTargetSpecified.into());
5131        }
5132
5133        let is_auto_layout = desc.layout.is_none();
5134
5135        // Get the pipeline layout from the desc if it is provided.
5136        let pipeline_layout = match desc.layout {
5137            Some(pipeline_layout) => {
5138                pipeline_layout.same_device(self)?;
5139                pipeline_layout.check_valid()?;
5140                Some(pipeline_layout)
5141            }
5142            None => None,
5143        };
5144
5145        let samples = {
5146            let sc = desc.multisample.count;
5147            if sc == 0 || sc > 32 || !sc.is_power_of_two() {
5148                return Err(pipeline::RenderError::InvalidSampleCount(sc).into());
5149            }
5150            sc
5151        };
5152
5153        // Validate and compile each programmable shader stage.
5154        let mut compiler =
5155            StageCompiler::new(self, pipeline_layout, io, Some(desc.primitive.topology));
5156        let vertex_stage;
5157        let task_stage;
5158        let mesh_stage;
5159
5160        match desc.vertex {
5161            pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
5162                let stage = validation::ShaderStageForValidation::Vertex {
5163                    topology: desc.primitive.topology,
5164                    compare_function: desc.depth_stencil.as_ref().and_then(|d| d.depth_compare),
5165                };
5166                vertex_stage = Some(compiler.compile(&vertex.stage, stage)?);
5167                task_stage = None;
5168                mesh_stage = None;
5169            }
5170            pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
5171                self.require_features(wgt::Features::EXPERIMENTAL_MESH_SHADER)?;
5172
5173                vertex_stage = None;
5174                task_stage = match task {
5175                    Some(task) => Some(
5176                        compiler
5177                            .compile(&task.stage, validation::ShaderStageForValidation::Task)?,
5178                    ),
5179                    None => None,
5180                };
5181                mesh_stage = Some(
5182                    compiler.compile(&mesh.stage, validation::ShaderStageForValidation::Mesh)?,
5183                );
5184            }
5185        }
5186
5187        let fragment_stage = match desc.fragment {
5188            Some(ref fragment) => {
5189                let stage = validation::ShaderStageForValidation::Fragment {
5190                    dual_source_blending,
5191                    has_depth_attachment,
5192                };
5193                Some(compiler.compile(&fragment.stage, stage)?)
5194            }
5195            None => None,
5196        };
5197
5198        // Extract the information the `StageCompiler` has accumulated about the
5199        // overall pipeline.
5200        let StageCompiler {
5201            resources,
5202            io,
5203            validated: validated_stages,
5204            ..
5205        } = compiler;
5206
5207        let mut active_color_outputs = 0u64;
5208        if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
5209            for (i, output) in io.varyings.iter() {
5210                active_color_outputs |= 1 << i;
5211                match color_targets.get(*i as usize) {
5212                    Some(Some(state)) => {
5213                        validation::check_color_attachment_compatibility(state, output.ty)
5214                            .map_err(|err| pipeline::RenderError::ColorState(*i as u8, err))?;
5215                    }
5216                    _ => {
5217                        log::debug!(
5218                            "The fragment stage {:?} output @location({}) values are ignored",
5219                            fragment_stage
5220                                .as_ref()
5221                                .map_or("", |stage| stage.entry_point.as_str()),
5222                            i
5223                        );
5224                    }
5225                }
5226            }
5227
5228            let missing_color_outputs = required_color_outputs & !active_color_outputs;
5229            if missing_color_outputs != 0 {
5230                return Err(pipeline::RenderError::ColorState(
5231                    missing_color_outputs.trailing_zeros() as u8,
5232                    ColorStateError::OutputNotPresent,
5233                )
5234                .into());
5235            }
5236        }
5237
5238        let pipeline_layout = match resources.layout {
5239            validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
5240            validation::BindingLayoutSource::Derived(entries) => {
5241                self.create_derived_pipeline_layout(entries, resources.immediates.size())?
5242            }
5243        };
5244
5245        let naga::valid::ImmediateUsage::Valid {
5246            slots: immediate_slots_required,
5247            size: _,
5248        } = resources.immediates
5249        else {
5250            unreachable!("Immediates exceeding maxImmediateSize should have been rejected");
5251        };
5252
5253        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
5254            let bind_groups_plus_vertex_buffers =
5255                u32::try_from(pipeline_layout.bind_group_layouts.len() + vertex.buffers.len())
5256                    .unwrap();
5257            if bind_groups_plus_vertex_buffers > self.limits.max_bind_groups_plus_vertex_buffers {
5258                return Err(pipeline::RenderError::TooManyBindGroupsPlusVertexBuffers {
5259                    given: bind_groups_plus_vertex_buffers,
5260                    limit: self.limits.max_bind_groups_plus_vertex_buffers,
5261                }
5262                .into());
5263            }
5264
5265            let given = pipeline_layout
5266                .buffers_and_acceleration_structures_in_vertex_stage
5267                .saturating_add(vertex.buffers.len() as u32);
5268            if !self
5269                .instance_flags
5270                .contains(wgt::InstanceFlags::STRICT_WEBGPU_COMPLIANCE)
5271            {
5272                let limit = self
5273                    .limits
5274                    .max_buffers_and_acceleration_structures_per_shader_stage;
5275                if given > limit {
5276                    return Err(
5277                        pipeline::RenderError::TooManyBuffersAndAccelerationStructuresInVertexStage {
5278                            given,
5279                            limit,
5280                        }
5281                        .into(),
5282                    );
5283                }
5284            }
5285        }
5286
5287        // Multiview is only supported if the feature is enabled
5288        if let Some(mv_mask) = desc.multiview_mask {
5289            self.require_features(wgt::Features::MULTIVIEW)?;
5290            if !(mv_mask.get() + 1).is_power_of_two() {
5291                self.require_features(wgt::Features::SELECTIVE_MULTIVIEW)?;
5292            }
5293        }
5294
5295        if !self
5296            .downlevel
5297            .flags
5298            .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
5299        {
5300            for (binding, size) in resources.minimum_binding_sizes.iter() {
5301                if size.get() % 16 != 0 {
5302                    return Err(pipeline::RenderError::UnalignedShader {
5303                        binding: binding.binding,
5304                        group: binding.group,
5305                        size: size.get(),
5306                    }
5307                    .into());
5308                }
5309            }
5310        }
5311
5312        let late_sized_buffer_groups = Device::make_late_sized_buffer_groups(
5313            &resources.minimum_binding_sizes,
5314            &pipeline_layout,
5315        );
5316
5317        let cache = match desc.cache {
5318            Some(cache) => {
5319                cache.check_is_valid()?;
5320                cache.same_device(self)?;
5321                Some(cache)
5322            }
5323            None => None,
5324        };
5325
5326        let is_mesh = mesh_stage.is_some();
5327        let has_task_shader = task_stage.is_some();
5328        let raw = {
5329            let pipeline_desc = hal::RenderPipelineDescriptor {
5330                label: desc.label.to_hal(self.instance_flags),
5331                layout: pipeline_layout.raw()?,
5332                vertex_processor: match vertex_stage {
5333                    Some(ref vertex_stage) => hal::VertexProcessor::Standard {
5334                        vertex_buffers: &hal_vertex_buffer_layouts,
5335                        vertex_stage: vertex_stage.to_hal(),
5336                    },
5337                    None => hal::VertexProcessor::Mesh {
5338                        task_stage: task_stage.as_ref().map(|stage| stage.to_hal()),
5339                        mesh_stage: mesh_stage.as_ref().unwrap().to_hal(),
5340                    },
5341                },
5342                primitive: desc.primitive,
5343                depth_stencil: desc.depth_stencil.clone(),
5344                multisample: desc.multisample,
5345                fragment_stage: fragment_stage.as_ref().map(|stage| stage.to_hal()),
5346                color_targets,
5347                multiview_mask: desc.multiview_mask,
5348                cache: cache.as_ref().map(|it| it.raw()).transpose()?,
5349            };
5350            unsafe { self.raw().create_render_pipeline(&pipeline_desc) }
5351                .map_err(|err| self.handle_hal_pipeline_error(err))?
5352        };
5353        drop((vertex_stage, task_stage, mesh_stage, fragment_stage));
5354
5355        let pass_context = RenderPassContext {
5356            attachments: AttachmentData {
5357                colors: color_targets
5358                    .iter()
5359                    .map(|state| state.as_ref().map(|s| s.format))
5360                    .collect(),
5361                resolves: ArrayVec::new(),
5362                depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
5363            },
5364            sample_count: samples,
5365            multiview_mask: desc.multiview_mask,
5366        };
5367
5368        let mut flags = pipeline::PipelineFlags::empty();
5369        for state in color_targets.iter().filter_map(|s| s.as_ref()) {
5370            if let Some(ref bs) = state.blend {
5371                if bs.color.uses_constant() | bs.alpha.uses_constant() {
5372                    flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
5373                }
5374            }
5375        }
5376        if let Some(ds) = depth_stencil_state.as_ref() {
5377            if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
5378                flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
5379            }
5380            if !ds.is_depth_read_only() {
5381                flags |= pipeline::PipelineFlags::WRITES_DEPTH;
5382            }
5383            if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
5384                flags |= pipeline::PipelineFlags::WRITES_STENCIL;
5385            }
5386        }
5387        let shader_modules = {
5388            let mut shader_modules = ArrayVec::new();
5389            match desc.vertex {
5390                pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
5391                    shader_modules.push(vertex.stage.module)
5392                }
5393                pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
5394                    if let Some(task) = task {
5395                        shader_modules.push(task.stage.module);
5396                    }
5397                    shader_modules.push(mesh.stage.module);
5398                }
5399            }
5400            shader_modules.extend(desc.fragment.map(|f| f.stage.module));
5401            shader_modules
5402        };
5403
5404        let pipeline = pipeline::RenderPipeline {
5405            state: ResourceState::Valid(pipeline::RenderPipelineState {
5406                raw: ManuallyDrop::new(raw),
5407                layout: pipeline_layout.clone(),
5408            }),
5409            device: self.clone(),
5410            pass_context,
5411            _shader_modules: shader_modules,
5412            flags,
5413            topology: desc.primitive.topology,
5414            strip_index_format: desc.primitive.strip_index_format,
5415            vertex_steps,
5416            late_sized_buffer_groups,
5417            immediate_slots_required,
5418            label: desc.label.to_string(),
5419            tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
5420            is_mesh,
5421            has_task_shader,
5422        };
5423
5424        let pipeline = Arc::new(pipeline);
5425
5426        if is_auto_layout {
5427            for bgl in pipeline_layout.bind_group_layouts.iter() {
5428                let Some(bgl) = bgl else {
5429                    continue;
5430                };
5431
5432                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the
5433                // result.
5434                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
5435            }
5436        }
5437
5438        Ok(pipeline)
5439    }
5440
5441    /// Handle a [`hal::PipelineError`].
5442    ///
5443    /// Turn `error` into a [`CreatePipelineError`] and return it.
5444    ///
5445    /// Pass all [`hal::DeviceError`]s through [`Device::handle_hal_error`],
5446    /// so that `self` is marked as lost when appropriate.
5447    ///
5448    /// [`CreatePipelineError`]: pipeline::CreatePipelineError
5449    pub(crate) fn handle_hal_pipeline_error(
5450        &self,
5451        error: hal::PipelineError,
5452    ) -> pipeline::CreatePipelineError {
5453        match error {
5454            hal::PipelineError::Device(error) => {
5455                pipeline::CreatePipelineError::Device(self.handle_hal_error(error))
5456            }
5457            other => pipeline::CreatePipelineError::from_hal_without_device_loss(other),
5458        }
5459    }
5460
5461    /// # Safety
5462    /// The `data` field on `desc` must have previously been returned from
5463    /// [`pipeline::PipelineCache::get_data`]
5464    pub unsafe fn create_pipeline_cache(
5465        self: &Arc<Self>,
5466        desc: &pipeline::PipelineCacheDescriptor,
5467    ) -> (
5468        Arc<pipeline::PipelineCache>,
5469        Option<pipeline::CreatePipelineCacheError>,
5470    ) {
5471        profiling::scope!("Device::create_pipeline_cache");
5472        let (cache, error) = match unsafe { self.create_pipeline_cache_inner(desc) } {
5473            Ok(cache) => (cache, None),
5474            Err(e) => (
5475                pipeline::PipelineCache::invalid(self.clone(), desc),
5476                Some(e),
5477            ),
5478        };
5479        #[cfg(feature = "trace")]
5480        if let Some(ref mut trace) = *self.trace.lock() {
5481            use trace::IntoTrace;
5482            trace.add(trace::Action::CreatePipelineCache {
5483                id: cache.to_trace(),
5484                desc: desc.clone(),
5485            });
5486        }
5487        api_log!("Device::create_pipeline_cache -> {:?}", Arc::as_ptr(&cache));
5488        (cache, error)
5489    }
5490
5491    /// # Safety
5492    /// The `data` field on `desc` must have previously been returned from
5493    /// [`pipeline::PipelineCache::get_data`]
5494    pub(crate) unsafe fn create_pipeline_cache_inner(
5495        self: &Arc<Self>,
5496        desc: &pipeline::PipelineCacheDescriptor,
5497    ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
5498        use crate::pipeline_cache;
5499
5500        self.check_is_valid()?;
5501
5502        self.require_features(wgt::Features::PIPELINE_CACHE)?;
5503        let data = if let Some((data, validation_key)) = desc
5504            .data
5505            .as_ref()
5506            .zip(self.raw().pipeline_cache_validation_key())
5507        {
5508            let data = pipeline_cache::validate_pipeline_cache(
5509                data,
5510                &self.adapter.raw.info,
5511                validation_key,
5512            );
5513            match data {
5514                Ok(data) => Some(data),
5515                Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
5516                // If the error was unavoidable and we are asked to fallback, do so
5517                Err(_) => None,
5518            }
5519        } else {
5520            None
5521        };
5522        let cache_desc = hal::PipelineCacheDescriptor {
5523            data,
5524            label: desc.label.to_hal(self.instance_flags),
5525        };
5526        let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
5527            Ok(raw) => raw,
5528            Err(e) => match e {
5529                hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
5530            },
5531        };
5532        let cache = pipeline::PipelineCache {
5533            device: self.clone(),
5534            label: desc.label.to_string(),
5535            // This would be none in the error condition, which we don't implement yet
5536            raw: ResourceState::Valid(raw),
5537        };
5538
5539        let cache = Arc::new(cache);
5540
5541        Ok(cache)
5542    }
5543
5544    fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
5545        // Variant of adapter.get_texture_format_features that takes device features into account
5546        use wgt::TextureFormatFeatureFlags as tfsc;
5547        let mut format_features = self.adapter.get_texture_format_features(format);
5548        if (format == TextureFormat::R32Float
5549            || format == TextureFormat::Rg32Float
5550            || format == TextureFormat::Rgba32Float)
5551            && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
5552        {
5553            format_features.flags.set(tfsc::FILTERABLE, false);
5554        }
5555        format_features
5556    }
5557
5558    pub(crate) fn describe_format_features(
5559        &self,
5560        format: TextureFormat,
5561    ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
5562        self.require_features(format.required_features())?;
5563
5564        let using_device_features = self
5565            .features
5566            .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
5567        // If we're running downlevel, we need to manually ask the backend what
5568        // we can use as we can't trust WebGPU.
5569        let downlevel = !self
5570            .downlevel
5571            .flags
5572            .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
5573
5574        if using_device_features || downlevel {
5575            Ok(self.get_texture_format_features(format))
5576        } else {
5577            Ok(format.guaranteed_format_features(self.features))
5578        }
5579    }
5580
5581    #[cfg(feature = "replay")]
5582    pub(crate) fn wait_for_submit(
5583        &self,
5584        submission_index: crate::SubmissionIndex,
5585    ) -> Result<(), DeviceError> {
5586        let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
5587            .map_err(|e| self.handle_hal_error(e))?;
5588        if last_done_index < submission_index {
5589            unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
5590                .map_err(|e| self.handle_hal_error(e))?;
5591            if let Some(queue) = self.get_queue() {
5592                let closures = queue.lock_life().triage_submissions(submission_index);
5593                assert!(
5594                    closures.is_empty(),
5595                    "wait_for_submit is not expected to work with closures"
5596                );
5597            }
5598        }
5599        Ok(())
5600    }
5601
5602    pub fn create_query_set(
5603        self: &Arc<Self>,
5604        desc: &resource::QuerySetDescriptor,
5605    ) -> Arc<QuerySet> {
5606        profiling::scope!("Device::create_query_set");
5607        let query_set = self.create_query_set_inner(desc).unwrap_or_else(|err| {
5608            self.handle_error(err, desc.label.as_deref(), "Device::create_query_set");
5609            QuerySet::invalid(Arc::clone(self), desc)
5610        });
5611        #[cfg(feature = "trace")]
5612        if let Some(ref mut trace) = *self.trace.lock() {
5613            use trace::IntoTrace;
5614            trace.add(trace::Action::CreateQuerySet {
5615                id: query_set.to_trace(),
5616                desc: desc.clone(),
5617            });
5618        }
5619        api_log!("Device::create_query_set -> {:?}", Arc::as_ptr(&query_set));
5620        query_set
5621    }
5622
5623    pub(crate) fn create_query_set_inner(
5624        self: &Arc<Self>,
5625        desc: &resource::QuerySetDescriptor,
5626    ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
5627        use resource::CreateQuerySetError as Error;
5628
5629        self.check_is_valid()?;
5630
5631        match desc.ty {
5632            wgt::QueryType::Occlusion => {}
5633            wgt::QueryType::Timestamp => {
5634                self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
5635            }
5636            wgt::QueryType::PipelineStatistics(..) => {
5637                self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
5638            }
5639        }
5640
5641        if desc.count == 0 {
5642            return Err(Error::ZeroCount);
5643        }
5644
5645        if desc.count > wgt::QUERY_SET_MAX_QUERIES {
5646            return Err(Error::TooManyQueries {
5647                count: desc.count,
5648                maximum: wgt::QUERY_SET_MAX_QUERIES,
5649            });
5650        }
5651
5652        let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
5653
5654        let raw = unsafe { self.raw().create_query_set(&hal_desc) }
5655            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
5656
5657        let query_set = QuerySet {
5658            state: ResourceState::Valid(QuerySetState {
5659                raw: Snatchable::new(raw),
5660            }),
5661            device: self.clone(),
5662            tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
5663            desc: desc.map_label(|l| l.to_string()),
5664            initialized_slots: Mutex::new(
5665                rank::QUERY_SET_INITIALIZED_SLOTS,
5666                bit_vec::BitVec::from_elem(desc.count as usize, false),
5667            ),
5668        };
5669
5670        let query_set = Arc::new(query_set);
5671
5672        Ok(query_set)
5673    }
5674
5675    pub(crate) fn lose(&self, message: &str) {
5676        // Follow the steps at https://gpuweb.github.io/gpuweb/#lose-the-device.
5677
5678        // Mark the device explicitly as invalid. This is checked in various
5679        // places to prevent new work from being submitted.
5680        self.valid.store(false, Ordering::Release);
5681
5682        // 1) Resolve the GPUDevice device.lost promise.
5683        if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
5684            device_lost_closure(DeviceLostReason::Unknown, message.to_string());
5685        }
5686
5687        // 2) Complete any outstanding mapAsync() steps.
5688        // 3) Complete any outstanding onSubmittedWorkDone() steps.
5689
5690        // These parts are passively accomplished by setting valid to false,
5691        // since that will prevent any new work from being added to the queues.
5692        // Future calls to poll_devices will continue to check the work queues
5693        // until they are cleared, and then drop the device.
5694    }
5695
5696    fn release_gpu_resources(&self) {
5697        // This is called when the device is lost, which makes every associated
5698        // resource invalid and unusable. This is an opportunity to release all of
5699        // the underlying gpu resources, even though the objects remain visible to
5700        // the user agent. We purge this memory naturally when resources have been
5701        // moved into the appropriate buckets, so this function just needs to
5702        // initiate movement into those buckets, and it can do that by calling
5703        // "destroy" on all the resources we know about.
5704
5705        let trackers = self.trackers.lock();
5706        let buffers = trackers
5707            .buffers
5708            .used_resources()
5709            .flat_map(Weak::upgrade)
5710            .collect::<Vec<_>>();
5711        let textures = trackers
5712            .textures
5713            .used_resources()
5714            .flat_map(Weak::upgrade)
5715            .collect::<Vec<_>>();
5716        drop(trackers);
5717
5718        for buffer in buffers {
5719            buffer.destroy();
5720        }
5721        for texture in textures {
5722            texture.destroy();
5723        }
5724    }
5725
5726    pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
5727        UsageScope::new_pooled(
5728            &self.usage_scopes,
5729            &self.tracker_indices,
5730            self.ordered_buffer_usages,
5731            self.ordered_texture_usages,
5732        )
5733    }
5734
5735    /// `device_lost_closure` might never be called.
5736    pub fn set_device_lost_closure(&self, device_lost_closure: DeviceLostClosure) {
5737        self.device_lost_closure.lock().replace(device_lost_closure);
5738    }
5739
5740    pub fn destroy(self: &Arc<Self>) {
5741        api_log!("Device::destroy {:?}", Arc::as_ptr(self));
5742
5743        // Follow the steps at
5744        // https://gpuweb.github.io/gpuweb/#dom-gpudevice-destroy.
5745        // It's legal to call destroy multiple times, but if the device
5746        // is already invalid, there's nothing more to do. There's also
5747        // no need to return an error.
5748        if !self.is_valid() {
5749            return;
5750        }
5751
5752        // The last part of destroy is to lose the device. The spec says
5753        // delay that until all "currently-enqueued operations on any
5754        // queue on this device are completed." This is accomplished by
5755        // setting valid to false, and then relying upon maintain to
5756        // check for empty queues and a DeviceLostClosure. At that time,
5757        // the DeviceLostClosure will be called with "destroyed" as the
5758        // reason.
5759        self.valid.store(false, Ordering::Release);
5760    }
5761
5762    pub fn get_internal_counters(&self) -> wgt::InternalCounters {
5763        wgt::InternalCounters {
5764            hal: self.get_hal_counters(),
5765            core: wgt::CoreCounters {},
5766        }
5767    }
5768
5769    pub fn get_hal_counters(&self) -> wgt::HalCounters {
5770        self.raw().get_internal_counters()
5771    }
5772
5773    pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
5774        self.raw().generate_allocator_report()
5775    }
5776}
5777
5778crate::impl_resource_type!(Device);
5779crate::impl_labeled!(Device);
5780crate::impl_storage_item!(Device);