wgpu_core/device/
resource.rs

1use alloc::{
2    borrow::Cow,
3    boxed::Box,
4    string::{String, ToString as _},
5    sync::{Arc, Weak},
6    vec::Vec,
7};
8use core::{
9    fmt,
10    mem::{self, ManuallyDrop},
11    num::NonZeroU32,
12    sync::atomic::{AtomicBool, Ordering},
13};
14use hal::ShouldBeNonZeroExt;
15
16use arrayvec::ArrayVec;
17use bitflags::Flags;
18use smallvec::SmallVec;
19use wgt::{
20    math::align_to, DeviceLostReason, TextureFormat, TextureSampleType, TextureSelector,
21    TextureViewDimension,
22};
23
24#[cfg(feature = "trace")]
25use crate::device::trace;
26use crate::{
27    binding_model::{self, BindGroup, BindGroupLayout, BindGroupLayoutEntryError},
28    command, conv,
29    device::{
30        bgl, create_validator, life::WaitIdleError, map_buffer, AttachmentData,
31        DeviceLostInvocation, HostMap, MissingDownlevelFlags, MissingFeatures, RenderPassContext,
32        CLEANUP_WAIT_MS,
33    },
34    hal_label,
35    init_tracker::{
36        BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
37        TextureInitTrackerAction,
38    },
39    instance::{Adapter, RequestDeviceError},
40    lock::{rank, Mutex, RwLock},
41    pipeline,
42    pool::ResourcePool,
43    resource::{
44        self, Buffer, ExternalTexture, Fallible, Labeled, ParentDevice, QuerySet,
45        RawResourceAccess, Sampler, StagingBuffer, Texture, TextureView,
46        TextureViewNotRenderableReason, Tlas, TrackingData,
47    },
48    resource_log,
49    snatch::{SnatchGuard, SnatchLock, Snatchable},
50    timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
51    track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
52    validation::{self, validate_color_attachment_bytes_per_sample},
53    weak_vec::WeakVec,
54    FastHashMap, LabelHelpers, OnceCellOrLock,
55};
56
57use super::{
58    queue::Queue, DeviceDescriptor, DeviceError, DeviceLostClosure, UserClosures,
59    ENTRYPOINT_FAILURE_ERROR, ZERO_BUFFER_SIZE,
60};
61
62#[cfg(supports_64bit_atomics)]
63use core::sync::atomic::AtomicU64;
64#[cfg(not(supports_64bit_atomics))]
65use portable_atomic::AtomicU64;
66
67pub(crate) struct CommandIndices {
68    /// The index of the last command submission that was attempted.
69    ///
70    /// Note that `fence` may never be signalled with this value, if the command
71    /// submission failed. If you need to wait for everything running on a
72    /// `Queue` to complete, wait for [`last_successful_submission_index`].
73    ///
74    /// [`last_successful_submission_index`]: Device::last_successful_submission_index
75    pub(crate) active_submission_index: hal::FenceValue,
76    pub(crate) next_acceleration_structure_build_command_index: u64,
77}
78
79/// Parameters provided to shaders via a uniform buffer of the type
80/// [`NagaExternalTextureParams`], describing an [`ExternalTexture`] resource
81/// binding.
82///
83/// [`NagaExternalTextureParams`]: naga::SpecialTypes::external_texture_params
84/// [`ExternalTexture`]: binding_model::BindingResource::ExternalTexture
85#[repr(C)]
86#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
87pub struct ExternalTextureParams {
88    /// 4x4 column-major matrix with which to convert sampled YCbCr values
89    /// to RGBA.
90    ///
91    /// This is ignored when `num_planes` is 1.
92    pub yuv_conversion_matrix: [f32; 16],
93
94    /// 3x3 column-major matrix to transform linear RGB values in the source
95    /// color space to linear RGB values in the destination color space. In
96    /// combination with [`Self::src_transfer_function`] and
97    /// [`Self::dst_transfer_function`] this can be used to ensure that
98    /// [`ImageSample`] and [`ImageLoad`] operations return values in the
99    /// desired destination color space rather than the source color space of
100    /// the underlying planes.
101    ///
102    /// Includes a padding element after each column.
103    ///
104    /// [`ImageSample`]: naga::ir::Expression::ImageSample
105    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
106    pub gamut_conversion_matrix: [f32; 12],
107
108    /// Transfer function for the source color space. The *inverse* of this
109    /// will be applied to decode non-linear RGB to linear RGB in the source
110    /// color space.
111    pub src_transfer_function: wgt::ExternalTextureTransferFunction,
112
113    /// Transfer function for the destination color space. This will be applied
114    /// to encode linear RGB to non-linear RGB in the destination color space.
115    pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
116
117    /// Transform to apply to [`ImageSample`] coordinates.
118    ///
119    /// This is a 3x2 column-major matrix representing an affine transform from
120    /// normalized texture coordinates to the normalized coordinates that should
121    /// be sampled from the external texture's underlying plane(s).
122    ///
123    /// This transform may scale, translate, flip, and rotate in 90-degree
124    /// increments, but the result of transforming the rectangle (0,0)..(1,1)
125    /// must be an axis-aligned rectangle that falls within the bounds of
126    /// (0,0)..(1,1).
127    ///
128    /// [`ImageSample`]: naga::ir::Expression::ImageSample
129    pub sample_transform: [f32; 6],
130
131    /// Transform to apply to [`ImageLoad`] coordinates.
132    ///
133    /// This is a 3x2 column-major matrix representing an affine transform from
134    /// non-normalized texel coordinates to the non-normalized coordinates of
135    /// the texel that should be loaded from the external texture's underlying
136    /// plane 0. For planes 1 and 2, if present, plane 0's coordinates are
137    /// scaled according to the textures' relative sizes.
138    ///
139    /// This transform may scale, translate, flip, and rotate in 90-degree
140    /// increments, but the result of transforming the rectangle (0,0)..[`size`]
141    /// must be an axis-aligned rectangle that falls within the bounds of
142    /// (0,0)..[`size`].
143    ///
144    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
145    /// [`size`]: Self::size
146    pub load_transform: [f32; 6],
147
148    /// Size of the external texture.
149    ///
150    /// This is the value that should be returned by size queries in shader
151    /// code; it does not necessarily match the dimensions of the underlying
152    /// texture(s). As a special case, if this is `[0, 0]`, the actual size of
153    /// plane 0 should be used instead.
154    ///
155    /// This must be consistent with [`sample_transform`]: it should be the size
156    /// in texels of the rectangle covered by the square (0,0)..(1,1) after
157    /// [`sample_transform`] has been applied to it.
158    ///
159    /// [`sample_transform`]: Self::sample_transform
160    pub size: [u32; 2],
161
162    /// Number of planes. 1 indicates a single RGBA plane. 2 indicates a Y
163    /// plane and an interleaved CbCr plane. 3 indicates separate Y, Cb, and Cr
164    /// planes.
165    pub num_planes: u32,
166    // Ensure the size of this struct matches the type generated by Naga.
167    pub _padding: [u8; 4],
168}
169
170impl ExternalTextureParams {
171    pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
172        let gamut_conversion_matrix = [
173            desc.gamut_conversion_matrix[0],
174            desc.gamut_conversion_matrix[1],
175            desc.gamut_conversion_matrix[2],
176            0.0, // padding
177            desc.gamut_conversion_matrix[3],
178            desc.gamut_conversion_matrix[4],
179            desc.gamut_conversion_matrix[5],
180            0.0, // padding
181            desc.gamut_conversion_matrix[6],
182            desc.gamut_conversion_matrix[7],
183            desc.gamut_conversion_matrix[8],
184            0.0, // padding
185        ];
186
187        Self {
188            yuv_conversion_matrix: desc.yuv_conversion_matrix,
189            gamut_conversion_matrix,
190            src_transfer_function: desc.src_transfer_function,
191            dst_transfer_function: desc.dst_transfer_function,
192            size: [desc.width, desc.height],
193            sample_transform: desc.sample_transform,
194            load_transform: desc.load_transform,
195            num_planes: desc.num_planes() as u32,
196            _padding: Default::default(),
197        }
198    }
199}
200
201/// Structure describing a logical device. Some members are internally mutable,
202/// stored behind mutexes.
203pub struct Device {
204    raw: Box<dyn hal::DynDevice>,
205    pub(crate) adapter: Arc<Adapter>,
206    pub(crate) queue: OnceCellOrLock<Weak<Queue>>,
207    pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
208    /// The `label` from the descriptor used to create the resource.
209    label: String,
210
211    pub(crate) command_allocator: command::CommandAllocator,
212
213    pub(crate) command_indices: RwLock<CommandIndices>,
214
215    /// The index of the last successful submission to this device's
216    /// [`hal::Queue`].
217    ///
218    /// Unlike [`active_submission_index`], which is incremented each time
219    /// submission is attempted, this is updated only when submission succeeds,
220    /// so waiting for this value won't hang waiting for work that was never
221    /// submitted.
222    ///
223    /// [`active_submission_index`]: CommandIndices::active_submission_index
224    pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
225
226    // NOTE: if both are needed, the `snatchable_lock` must be consistently acquired before the
227    // `fence` lock to avoid deadlocks.
228    pub(crate) fence: RwLock<ManuallyDrop<Box<dyn hal::DynFence>>>,
229    pub(crate) snatchable_lock: SnatchLock,
230
231    /// Is this device valid? Valid is closely associated with "lose the device",
232    /// which can be triggered by various methods, including at the end of device
233    /// destroy, and by any GPU errors that cause us to no longer trust the state
234    /// of the device. Ideally we would like to fold valid into the storage of
235    /// the device itself (for example as an Error enum), but unfortunately we
236    /// need to continue to be able to retrieve the device in poll_devices to
237    /// determine if it can be dropped. If our internal accesses of devices were
238    /// done through ref-counted references and external accesses checked for
239    /// Error enums, we wouldn't need this. For now, we need it. All the call
240    /// sites where we check it are areas that should be revisited if we start
241    /// using ref-counted references for internal access.
242    pub(crate) valid: AtomicBool,
243
244    /// Closure to be called on "lose the device". This is invoked directly by
245    /// device.lose or by the UserCallbacks returned from maintain when the device
246    /// has been destroyed and its queues are empty.
247    pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
248
249    /// Stores the state of buffers and textures.
250    pub(crate) trackers: Mutex<DeviceTracker>,
251    pub(crate) tracker_indices: TrackerIndexAllocators,
252    /// Pool of bind group layouts, allowing deduplication.
253    pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
254    pub(crate) alignments: hal::Alignments,
255    pub(crate) limits: wgt::Limits,
256    pub(crate) features: wgt::Features,
257    pub(crate) downlevel: wgt::DownlevelCapabilities,
258    pub(crate) instance_flags: wgt::InstanceFlags,
259    pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
260    pub(crate) usage_scopes: UsageScopePool,
261    pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
262    // Optional so that we can late-initialize this after the queue is created.
263    pub(crate) timestamp_normalizer:
264        OnceCellOrLock<crate::timestamp_normalization::TimestampNormalizer>,
265    /// Uniform buffer containing [`ExternalTextureParams`] with values such
266    /// that a [`TextureView`] bound to a [`wgt::BindingType::ExternalTexture`]
267    /// binding point will be rendered correctly. Intended to be used as the
268    /// [`hal::ExternalTextureBinding::params`] field.
269    pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
270    // needs to be dropped last
271    #[cfg(feature = "trace")]
272    pub(crate) trace: Mutex<Option<trace::Trace>>,
273}
274
275pub(crate) enum DeferredDestroy {
276    TextureViews(WeakVec<TextureView>),
277    BindGroups(WeakVec<BindGroup>),
278}
279
280impl fmt::Debug for Device {
281    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
282        f.debug_struct("Device")
283            .field("label", &self.label())
284            .field("limits", &self.limits)
285            .field("features", &self.features)
286            .field("downlevel", &self.downlevel)
287            .finish()
288    }
289}
290
291impl Drop for Device {
292    fn drop(&mut self) {
293        resource_log!("Drop {}", self.error_ident());
294
295        // SAFETY: We are in the Drop impl and we don't use self.zero_buffer anymore after this point.
296        let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
297        // SAFETY: We are in the Drop impl and we don't use
298        // self.default_external_texture_params_buffer anymore after this point.
299        let default_external_texture_params_buffer =
300            unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
301        // SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point.
302        let fence = unsafe { ManuallyDrop::take(&mut self.fence.write()) };
303        if let Some(indirect_validation) = self.indirect_validation.take() {
304            indirect_validation.dispose(self.raw.as_ref());
305        }
306        if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
307            timestamp_normalizer.dispose(self.raw.as_ref());
308        }
309        unsafe {
310            self.raw.destroy_buffer(zero_buffer);
311            self.raw
312                .destroy_buffer(default_external_texture_params_buffer);
313            self.raw.destroy_fence(fence);
314        }
315    }
316}
317
318impl Device {
319    pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
320        self.raw.as_ref()
321    }
322    pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
323        if self.features.contains(feature) {
324            Ok(())
325        } else {
326            Err(MissingFeatures(feature))
327        }
328    }
329
330    pub(crate) fn require_downlevel_flags(
331        &self,
332        flags: wgt::DownlevelFlags,
333    ) -> Result<(), MissingDownlevelFlags> {
334        if self.downlevel.flags.contains(flags) {
335            Ok(())
336        } else {
337            Err(MissingDownlevelFlags(flags))
338        }
339    }
340}
341
342impl Device {
343    pub(crate) fn new(
344        raw_device: Box<dyn hal::DynDevice>,
345        adapter: &Arc<Adapter>,
346        desc: &DeviceDescriptor,
347        instance_flags: wgt::InstanceFlags,
348    ) -> Result<Self, DeviceError> {
349        #[cfg(not(feature = "trace"))]
350        match &desc.trace {
351            wgt::Trace::Off => {}
352            _ => {
353                log::error!("wgpu-core feature 'trace' is not enabled");
354            }
355        };
356        #[cfg(feature = "trace")]
357        let trace_dir_name: Option<&std::path::PathBuf> = match &desc.trace {
358            wgt::Trace::Off => None,
359            wgt::Trace::Directory(d) => Some(d),
360            // The enum is non_exhaustive, so we must have a fallback arm (that should be
361            // unreachable in practice).
362            t => {
363                log::error!("unimplemented wgpu_types::Trace variant {t:?}");
364                None
365            }
366        };
367
368        let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
369
370        let command_allocator = command::CommandAllocator::new();
371
372        let rt_uses = if desc
373            .required_features
374            .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
375        {
376            wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
377        } else {
378            wgt::BufferUses::empty()
379        };
380
381        // Create zeroed buffer used for texture clears (and raytracing if required).
382        let zero_buffer = unsafe {
383            raw_device.create_buffer(&hal::BufferDescriptor {
384                label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
385                size: ZERO_BUFFER_SIZE,
386                usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
387                memory_flags: hal::MemoryFlags::empty(),
388            })
389        }
390        .map_err(DeviceError::from_hal)?;
391
392        let default_external_texture_params_buffer = unsafe {
393            raw_device.create_buffer(&hal::BufferDescriptor {
394                label: hal_label(
395                    Some("(wgpu internal) default external texture params buffer"),
396                    instance_flags,
397                ),
398                size: size_of::<ExternalTextureParams>() as _,
399                usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
400                memory_flags: hal::MemoryFlags::empty(),
401            })
402        }
403        .map_err(DeviceError::from_hal)?;
404
405        let alignments = adapter.raw.capabilities.alignments.clone();
406        let downlevel = adapter.raw.capabilities.downlevel.clone();
407
408        let enable_indirect_validation = instance_flags
409            .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
410            && downlevel
411                .flags
412                .contains(wgt::DownlevelFlags::INDIRECT_EXECUTION);
413
414        let indirect_validation = if enable_indirect_validation {
415            Some(crate::indirect_validation::IndirectValidation::new(
416                raw_device.as_ref(),
417                &desc.required_limits,
418                &desc.required_features,
419                adapter.backend(),
420            )?)
421        } else {
422            None
423        };
424
425        Ok(Self {
426            raw: raw_device,
427            adapter: adapter.clone(),
428            queue: OnceCellOrLock::new(),
429            zero_buffer: ManuallyDrop::new(zero_buffer),
430            default_external_texture_params_buffer: ManuallyDrop::new(
431                default_external_texture_params_buffer,
432            ),
433            label: desc.label.to_string(),
434            command_allocator,
435            command_indices: RwLock::new(
436                rank::DEVICE_COMMAND_INDICES,
437                CommandIndices {
438                    active_submission_index: 0,
439                    // By starting at one, we can put the result in a NonZeroU64.
440                    next_acceleration_structure_build_command_index: 1,
441                },
442            ),
443            last_successful_submission_index: AtomicU64::new(0),
444            fence: RwLock::new(rank::DEVICE_FENCE, ManuallyDrop::new(fence)),
445            snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
446            valid: AtomicBool::new(true),
447            device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
448            trackers: Mutex::new(rank::DEVICE_TRACKERS, DeviceTracker::new()),
449            tracker_indices: TrackerIndexAllocators::new(),
450            bgl_pool: ResourcePool::new(),
451            #[cfg(feature = "trace")]
452            trace: Mutex::new(
453                rank::DEVICE_TRACE,
454                trace_dir_name.and_then(|path| match trace::Trace::new(path.clone()) {
455                    Ok(mut trace) => {
456                        trace.add(trace::Action::Init {
457                            desc: wgt::DeviceDescriptor {
458                                trace: wgt::Trace::Off,
459                                ..desc.clone()
460                            },
461                            backend: adapter.backend(),
462                        });
463                        Some(trace)
464                    }
465                    Err(e) => {
466                        log::error!("Unable to start a trace in '{path:?}': {e}");
467                        None
468                    }
469                }),
470            ),
471            alignments,
472            limits: desc.required_limits.clone(),
473            features: desc.required_features,
474            downlevel,
475            instance_flags,
476            deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
477            usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
478            timestamp_normalizer: OnceCellOrLock::new(),
479            indirect_validation,
480        })
481    }
482
483    /// Initializes [`Device::default_external_texture_params_buffer`] with
484    /// required values such that a [`TextureView`] bound to a
485    /// [`wgt::BindingType::ExternalTexture`] binding point will be rendered
486    /// correctly.
487    fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
488        let data = ExternalTextureParams {
489            #[rustfmt::skip]
490            yuv_conversion_matrix: [
491                1.0, 0.0, 0.0, 0.0,
492                0.0, 1.0, 0.0, 0.0,
493                0.0, 0.0, 1.0, 0.0,
494                0.0, 0.0, 0.0, 1.0,
495            ],
496            #[rustfmt::skip]
497            gamut_conversion_matrix: [
498                1.0, 0.0, 0.0, /* padding */ 0.0,
499                0.0, 1.0, 0.0, /* padding */ 0.0,
500                0.0, 0.0, 1.0, /* padding */ 0.0,
501            ],
502            src_transfer_function: Default::default(),
503            dst_transfer_function: Default::default(),
504            size: [0, 0],
505            #[rustfmt::skip]
506            sample_transform: [
507                1.0, 0.0,
508                0.0, 1.0,
509                0.0, 0.0
510            ],
511            #[rustfmt::skip]
512            load_transform: [
513                1.0, 0.0,
514                0.0, 1.0,
515                0.0, 0.0
516            ],
517            num_planes: 1,
518            _padding: Default::default(),
519        };
520        let mut staging_buffer =
521            StagingBuffer::new(self, wgt::BufferSize::new(size_of_val(&data) as _).unwrap())?;
522        staging_buffer.write(bytemuck::bytes_of(&data));
523        let staging_buffer = staging_buffer.flush();
524
525        let params_buffer = self.default_external_texture_params_buffer.as_ref();
526        let queue = self.get_queue().unwrap();
527        let mut pending_writes = queue.pending_writes.lock();
528
529        unsafe {
530            pending_writes
531                .command_encoder
532                .transition_buffers(&[hal::BufferBarrier {
533                    buffer: params_buffer,
534                    usage: hal::StateTransition {
535                        from: wgt::BufferUses::MAP_WRITE,
536                        to: wgt::BufferUses::COPY_SRC,
537                    },
538                }]);
539            pending_writes.command_encoder.copy_buffer_to_buffer(
540                staging_buffer.raw(),
541                params_buffer,
542                &[hal::BufferCopy {
543                    src_offset: 0,
544                    dst_offset: 0,
545                    size: staging_buffer.size,
546                }],
547            );
548            pending_writes.consume(staging_buffer);
549            pending_writes
550                .command_encoder
551                .transition_buffers(&[hal::BufferBarrier {
552                    buffer: params_buffer,
553                    usage: hal::StateTransition {
554                        from: wgt::BufferUses::COPY_DST,
555                        to: wgt::BufferUses::UNIFORM,
556                    },
557                }]);
558        }
559
560        Ok(())
561    }
562
563    pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
564        let queue = self.get_queue().unwrap();
565
566        let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
567            self,
568            queue.get_timestamp_period(),
569        )?;
570
571        self.timestamp_normalizer
572            .set(timestamp_normalizer)
573            .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
574
575        self.init_default_external_texture_params_buffer()?;
576
577        Ok(())
578    }
579
580    /// Returns the backend this device is using.
581    pub fn backend(&self) -> wgt::Backend {
582        self.adapter.backend()
583    }
584
585    pub fn is_valid(&self) -> bool {
586        self.valid.load(Ordering::Acquire)
587    }
588
589    pub fn check_is_valid(&self) -> Result<(), DeviceError> {
590        if self.is_valid() {
591            Ok(())
592        } else {
593            Err(DeviceError::Lost)
594        }
595    }
596
597    /// Checks that we are operating within the memory budget reported by the native APIs.
598    ///
599    /// If we are not, the device gets invalidated.
600    ///
601    /// The budget might fluctuate over the lifetime of the application, so it should be checked
602    /// somewhat frequently.
603    pub fn lose_if_oom(&self) {
604        let _ = self
605            .raw()
606            .check_if_oom()
607            .map_err(|e| self.handle_hal_error(e));
608    }
609
610    pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
611        match error {
612            hal::DeviceError::OutOfMemory
613            | hal::DeviceError::Lost
614            | hal::DeviceError::Unexpected => {
615                self.lose(&error.to_string());
616            }
617        }
618        DeviceError::from_hal(error)
619    }
620
621    pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
622        match error {
623            hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
624            error => self.handle_hal_error(error),
625        }
626    }
627
628    /// Run some destroy operations that were deferred.
629    ///
630    /// Destroying the resources requires taking a write lock on the device's snatch lock,
631    /// so a good reason for deferring resource destruction is when we don't know for sure
632    /// how risky it is to take the lock (typically, it shouldn't be taken from the drop
633    /// implementation of a reference-counted structure).
634    /// The snatch lock must not be held while this function is called.
635    pub(crate) fn deferred_resource_destruction(&self) {
636        let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
637        for item in deferred_destroy {
638            match item {
639                DeferredDestroy::TextureViews(views) => {
640                    for view in views {
641                        let Some(view) = view.upgrade() else {
642                            continue;
643                        };
644                        let Some(raw_view) = view.raw.snatch(&mut self.snatchable_lock.write())
645                        else {
646                            continue;
647                        };
648
649                        resource_log!("Destroy raw {}", view.error_ident());
650
651                        unsafe {
652                            self.raw().destroy_texture_view(raw_view);
653                        }
654                    }
655                }
656                DeferredDestroy::BindGroups(bind_groups) => {
657                    for bind_group in bind_groups {
658                        let Some(bind_group) = bind_group.upgrade() else {
659                            continue;
660                        };
661                        let Some(raw_bind_group) =
662                            bind_group.raw.snatch(&mut self.snatchable_lock.write())
663                        else {
664                            continue;
665                        };
666
667                        resource_log!("Destroy raw {}", bind_group.error_ident());
668
669                        unsafe {
670                            self.raw().destroy_bind_group(raw_bind_group);
671                        }
672                    }
673                }
674            }
675        }
676    }
677
678    pub fn get_queue(&self) -> Option<Arc<Queue>> {
679        self.queue.get().as_ref()?.upgrade()
680    }
681
682    pub fn set_queue(&self, queue: &Arc<Queue>) {
683        assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
684    }
685
686    /// Check the current status of the GPU and process any submissions that have
687    /// finished.
688    ///
689    /// The `poll_type` argument tells if this function should wait for a particular
690    /// submission index to complete, or if it should just poll the current status.
691    ///
692    /// This will process _all_ completed submissions, even if the caller only asked
693    /// us to poll to a given submission index.
694    ///
695    /// Return a pair `(closures, result)`, where:
696    ///
697    /// - `closures` is a list of callbacks that need to be invoked informing the user
698    ///   about various things occurring. These happen and should be handled even if
699    ///   this function returns an error, hence they are outside of the result.
700    ///
701    /// - `results` is a boolean indicating the result of the wait operation, including
702    ///   if there was a timeout or a validation error.
703    pub(crate) fn maintain<'this>(
704        &'this self,
705        fence: crate::lock::RwLockReadGuard<ManuallyDrop<Box<dyn hal::DynFence>>>,
706        poll_type: wgt::PollType<crate::SubmissionIndex>,
707        snatch_guard: SnatchGuard,
708    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
709        profiling::scope!("Device::maintain");
710
711        let mut user_closures = UserClosures::default();
712
713        // If a wait was requested, determine which submission index to wait for.
714        let wait_submission_index = match poll_type {
715            wgt::PollType::WaitForSubmissionIndex(submission_index) => {
716                let last_successful_submission_index = self
717                    .last_successful_submission_index
718                    .load(Ordering::Acquire);
719
720                if submission_index > last_successful_submission_index {
721                    let result = Err(WaitIdleError::WrongSubmissionIndex(
722                        submission_index,
723                        last_successful_submission_index,
724                    ));
725
726                    return (user_closures, result);
727                }
728
729                Some(submission_index)
730            }
731            wgt::PollType::Wait => Some(
732                self.last_successful_submission_index
733                    .load(Ordering::Acquire),
734            ),
735            wgt::PollType::Poll => None,
736        };
737
738        // Wait for the submission index if requested.
739        if let Some(target_submission_index) = wait_submission_index {
740            log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
741
742            let wait_result = unsafe {
743                self.raw()
744                    .wait(fence.as_ref(), target_submission_index, CLEANUP_WAIT_MS)
745            };
746
747            // This error match is only about `DeviceErrors`. At this stage we do not care if
748            // the wait succeeded or not, and the `Ok(bool)`` variant is ignored.
749            if let Err(e) = wait_result {
750                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
751                return (user_closures, Err(hal_error));
752            }
753        }
754
755        // Get the currently finished submission index. This may be higher than the requested
756        // wait, or it may be less than the requested wait if the wait failed.
757        let fence_value_result = unsafe { self.raw().get_fence_value(fence.as_ref()) };
758        let current_finished_submission = match fence_value_result {
759            Ok(fence_value) => fence_value,
760            Err(e) => {
761                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
762                return (user_closures, Err(hal_error));
763            }
764        };
765
766        // Maintain all finished submissions on the queue, updating the relevant user closures and collecting if the queue is empty.
767        //
768        // We don't use the result of the wait here, as we want to progress forward as far as possible
769        // and the wait could have been for submissions that finished long ago.
770        let mut queue_empty = false;
771        if let Some(queue) = self.get_queue() {
772            let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
773            (
774                user_closures.submissions,
775                user_closures.mappings,
776                user_closures.blas_compact_ready,
777                queue_empty,
778            ) = queue_result
779        };
780
781        // Based on the queue empty status, and the current finished submission index, determine the result of the poll.
782        let result = if queue_empty {
783            if let Some(wait_submission_index) = wait_submission_index {
784                // Assert to ensure that if we received a queue empty status, the fence shows the correct value.
785                // This is defensive, as this should never be hit.
786                assert!(
787                    current_finished_submission >= wait_submission_index,
788                    "If the queue is empty, the current submission index ({current_finished_submission}) should be at least the wait submission index ({wait_submission_index})"
789                );
790            }
791
792            Ok(wgt::PollStatus::QueueEmpty)
793        } else if let Some(wait_submission_index) = wait_submission_index {
794            // This is theoretically possible to succeed more than checking on the poll result
795            // as submissions could have finished in the time between the timeout resolving,
796            // the thread getting scheduled again, and us checking the fence value.
797            if current_finished_submission >= wait_submission_index {
798                Ok(wgt::PollStatus::WaitSucceeded)
799            } else {
800                Err(WaitIdleError::Timeout)
801            }
802        } else {
803            Ok(wgt::PollStatus::Poll)
804        };
805
806        // Detect if we have been destroyed and now need to lose the device.
807        //
808        // If we are invalid (set at start of destroy) and our queue is empty,
809        // and we have a DeviceLostClosure, return the closure to be called by
810        // our caller. This will complete the steps for both destroy and for
811        // "lose the device".
812        let mut should_release_gpu_resource = false;
813        if !self.is_valid() && queue_empty {
814            // We can release gpu resources associated with this device (but not
815            // while holding the life_tracker lock).
816            should_release_gpu_resource = true;
817
818            // If we have a DeviceLostClosure, build an invocation with the
819            // reason DeviceLostReason::Destroyed and no message.
820            if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
821                user_closures
822                    .device_lost_invocations
823                    .push(DeviceLostInvocation {
824                        closure: device_lost_closure,
825                        reason: DeviceLostReason::Destroyed,
826                        message: String::new(),
827                    });
828            }
829        }
830
831        // Don't hold the locks while calling release_gpu_resources.
832        drop(fence);
833        drop(snatch_guard);
834
835        if should_release_gpu_resource {
836            self.release_gpu_resources();
837        }
838
839        (user_closures, result)
840    }
841
842    pub(crate) fn create_buffer(
843        self: &Arc<Self>,
844        desc: &resource::BufferDescriptor,
845    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
846        self.check_is_valid()?;
847
848        if desc.size > self.limits.max_buffer_size {
849            return Err(resource::CreateBufferError::MaxBufferSize {
850                requested: desc.size,
851                maximum: self.limits.max_buffer_size,
852            });
853        }
854
855        if desc
856            .usage
857            .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
858        {
859            self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)?;
860        }
861
862        if desc.usage.contains(wgt::BufferUsages::INDEX)
863            && desc.usage.contains(
864                wgt::BufferUsages::VERTEX
865                    | wgt::BufferUsages::UNIFORM
866                    | wgt::BufferUsages::INDIRECT
867                    | wgt::BufferUsages::STORAGE,
868            )
869        {
870            self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
871        }
872
873        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
874            return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
875        }
876
877        if !self
878            .features
879            .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
880        {
881            use wgt::BufferUsages as Bu;
882            let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
883                && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
884            let read_mismatch = desc.usage.contains(Bu::MAP_READ)
885                && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
886            if write_mismatch || read_mismatch {
887                return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
888            }
889        }
890
891        let mut usage = conv::map_buffer_usage(desc.usage);
892
893        if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
894            self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
895            // We are going to be reading from it, internally;
896            // when validating the content of the buffer
897            usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
898        }
899
900        if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
901            usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
902        }
903
904        if desc.mapped_at_creation {
905            if desc.size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
906                return Err(resource::CreateBufferError::UnalignedSize);
907            }
908            if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
909                // we are going to be copying into it, internally
910                usage |= wgt::BufferUses::COPY_DST;
911            }
912        } else {
913            // We are required to zero out (initialize) all memory. This is done
914            // on demand using clear_buffer which requires write transfer usage!
915            usage |= wgt::BufferUses::COPY_DST;
916        }
917
918        let actual_size = if desc.size == 0 {
919            wgt::COPY_BUFFER_ALIGNMENT
920        } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
921            // Bumping the size by 1 so that we can bind an empty range at the
922            // end of the buffer.
923            desc.size + 1
924        } else {
925            desc.size
926        };
927        let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
928        let aligned_size = if clear_remainder != 0 {
929            actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
930        } else {
931            actual_size
932        };
933
934        let hal_desc = hal::BufferDescriptor {
935            label: desc.label.to_hal(self.instance_flags),
936            size: aligned_size,
937            usage,
938            memory_flags: hal::MemoryFlags::empty(),
939        };
940        let buffer = unsafe { self.raw().create_buffer(&hal_desc) }
941            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
942
943        let timestamp_normalization_bind_group = Snatchable::new(unsafe {
944            // SAFETY: The size passed here must not overflow the buffer.
945            self.timestamp_normalizer
946                .get()
947                .unwrap()
948                .create_normalization_bind_group(
949                    self,
950                    &*buffer,
951                    desc.label.as_deref(),
952                    wgt::BufferSize::new(hal_desc.size).unwrap(),
953                    desc.usage,
954                )
955        }?);
956
957        let indirect_validation_bind_groups =
958            self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
959
960        let buffer = Buffer {
961            raw: Snatchable::new(buffer),
962            device: self.clone(),
963            usage: desc.usage,
964            size: desc.size,
965            initialization_status: RwLock::new(
966                rank::BUFFER_INITIALIZATION_STATUS,
967                BufferInitTracker::new(aligned_size),
968            ),
969            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
970            label: desc.label.to_string(),
971            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
972            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
973            timestamp_normalization_bind_group,
974            indirect_validation_bind_groups,
975        };
976
977        let buffer = Arc::new(buffer);
978
979        let buffer_use = if !desc.mapped_at_creation {
980            wgt::BufferUses::empty()
981        } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
982            // buffer is mappable, so we are just doing that at start
983            let map_size = buffer.size;
984            let mapping = if map_size == 0 {
985                hal::BufferMapping {
986                    ptr: core::ptr::NonNull::dangling(),
987                    is_coherent: true,
988                }
989            } else {
990                let snatch_guard: SnatchGuard = self.snatchable_lock.read();
991                map_buffer(&buffer, 0, map_size, HostMap::Write, &snatch_guard)?
992            };
993            *buffer.map_state.lock() = resource::BufferMapState::Active {
994                mapping,
995                range: 0..map_size,
996                host: HostMap::Write,
997            };
998            wgt::BufferUses::MAP_WRITE
999        } else {
1000            let mut staging_buffer =
1001                StagingBuffer::new(self, wgt::BufferSize::new(aligned_size).unwrap())?;
1002
1003            // Zero initialize memory and then mark the buffer as initialized
1004            // (it's guaranteed that this is the case by the time the buffer is usable)
1005            staging_buffer.write_zeros();
1006            buffer.initialization_status.write().drain(0..aligned_size);
1007
1008            *buffer.map_state.lock() = resource::BufferMapState::Init { staging_buffer };
1009            wgt::BufferUses::COPY_DST
1010        };
1011
1012        self.trackers
1013            .lock()
1014            .buffers
1015            .insert_single(&buffer, buffer_use);
1016
1017        Ok(buffer)
1018    }
1019
1020    pub(crate) fn create_texture_from_hal(
1021        self: &Arc<Self>,
1022        hal_texture: Box<dyn hal::DynTexture>,
1023        desc: &resource::TextureDescriptor,
1024    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1025        let format_features = self
1026            .describe_format_features(desc.format)
1027            .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error))?;
1028
1029        unsafe { self.raw().add_raw_texture(&*hal_texture) };
1030
1031        let texture = Texture::new(
1032            self,
1033            resource::TextureInner::Native { raw: hal_texture },
1034            conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1035            desc,
1036            format_features,
1037            resource::TextureClearMode::None,
1038            false,
1039        );
1040
1041        let texture = Arc::new(texture);
1042
1043        self.trackers
1044            .lock()
1045            .textures
1046            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1047
1048        Ok(texture)
1049    }
1050
1051    /// # Safety
1052    ///
1053    /// - `hal_buffer` must have been created on this device.
1054    /// - `hal_buffer` must have been created respecting `desc` (in particular, the size).
1055    /// - `hal_buffer` must be initialized.
1056    /// - `hal_buffer` must not have zero size.
1057    pub(crate) unsafe fn create_buffer_from_hal(
1058        self: &Arc<Self>,
1059        hal_buffer: Box<dyn hal::DynBuffer>,
1060        desc: &resource::BufferDescriptor,
1061    ) -> (Fallible<Buffer>, Option<resource::CreateBufferError>) {
1062        let timestamp_normalization_bind_group = unsafe {
1063            match self
1064                .timestamp_normalizer
1065                .get()
1066                .unwrap()
1067                .create_normalization_bind_group(
1068                    self,
1069                    &*hal_buffer,
1070                    desc.label.as_deref(),
1071                    wgt::BufferSize::new(desc.size).unwrap(),
1072                    desc.usage,
1073                ) {
1074                Ok(bg) => Snatchable::new(bg),
1075                Err(e) => {
1076                    return (
1077                        Fallible::Invalid(Arc::new(desc.label.to_string())),
1078                        Some(e.into()),
1079                    )
1080                }
1081            }
1082        };
1083
1084        let indirect_validation_bind_groups = match self.create_indirect_validation_bind_groups(
1085            hal_buffer.as_ref(),
1086            desc.size,
1087            desc.usage,
1088        ) {
1089            Ok(ok) => ok,
1090            Err(e) => return (Fallible::Invalid(Arc::new(desc.label.to_string())), Some(e)),
1091        };
1092
1093        unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1094
1095        let buffer = Buffer {
1096            raw: Snatchable::new(hal_buffer),
1097            device: self.clone(),
1098            usage: desc.usage,
1099            size: desc.size,
1100            initialization_status: RwLock::new(
1101                rank::BUFFER_INITIALIZATION_STATUS,
1102                BufferInitTracker::new(0),
1103            ),
1104            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1105            label: desc.label.to_string(),
1106            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1107            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1108            timestamp_normalization_bind_group,
1109            indirect_validation_bind_groups,
1110        };
1111
1112        let buffer = Arc::new(buffer);
1113
1114        self.trackers
1115            .lock()
1116            .buffers
1117            .insert_single(&buffer, wgt::BufferUses::empty());
1118
1119        (Fallible::Valid(buffer), None)
1120    }
1121
1122    fn create_indirect_validation_bind_groups(
1123        &self,
1124        raw_buffer: &dyn hal::DynBuffer,
1125        buffer_size: u64,
1126        usage: wgt::BufferUsages,
1127    ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1128    {
1129        if !usage.contains(wgt::BufferUsages::INDIRECT) {
1130            return Ok(Snatchable::empty());
1131        }
1132
1133        let Some(ref indirect_validation) = self.indirect_validation else {
1134            return Ok(Snatchable::empty());
1135        };
1136
1137        let bind_groups = crate::indirect_validation::BindGroups::new(
1138            indirect_validation,
1139            self,
1140            buffer_size,
1141            raw_buffer,
1142        )
1143        .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1144
1145        if let Some(bind_groups) = bind_groups {
1146            Ok(Snatchable::new(bind_groups))
1147        } else {
1148            Ok(Snatchable::empty())
1149        }
1150    }
1151
1152    pub(crate) fn create_texture(
1153        self: &Arc<Self>,
1154        desc: &resource::TextureDescriptor,
1155    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1156        use resource::{CreateTextureError, TextureDimensionError};
1157
1158        self.check_is_valid()?;
1159
1160        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1161            return Err(CreateTextureError::InvalidUsage(desc.usage));
1162        }
1163
1164        conv::check_texture_dimension_size(
1165            desc.dimension,
1166            desc.size,
1167            desc.sample_count,
1168            &self.limits,
1169        )?;
1170
1171        if desc.dimension != wgt::TextureDimension::D2 {
1172            // Depth textures can only be 2D
1173            if desc.format.is_depth_stencil_format() {
1174                return Err(CreateTextureError::InvalidDepthDimension(
1175                    desc.dimension,
1176                    desc.format,
1177                ));
1178            }
1179        }
1180
1181        if desc.dimension != wgt::TextureDimension::D2
1182            && desc.dimension != wgt::TextureDimension::D3
1183        {
1184            // Compressed textures can only be 2D or 3D
1185            if desc.format.is_compressed() {
1186                return Err(CreateTextureError::InvalidCompressedDimension(
1187                    desc.dimension,
1188                    desc.format,
1189                ));
1190            }
1191
1192            // Renderable textures can only be 2D or 3D
1193            if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1194                return Err(CreateTextureError::InvalidDimensionUsages(
1195                    wgt::TextureUsages::RENDER_ATTACHMENT,
1196                    desc.dimension,
1197                ));
1198            }
1199        }
1200
1201        if desc.format.is_compressed() {
1202            let (block_width, block_height) = desc.format.block_dimensions();
1203
1204            if desc.size.width % block_width != 0 {
1205                return Err(CreateTextureError::InvalidDimension(
1206                    TextureDimensionError::NotMultipleOfBlockWidth {
1207                        width: desc.size.width,
1208                        block_width,
1209                        format: desc.format,
1210                    },
1211                ));
1212            }
1213
1214            if desc.size.height % block_height != 0 {
1215                return Err(CreateTextureError::InvalidDimension(
1216                    TextureDimensionError::NotMultipleOfBlockHeight {
1217                        height: desc.size.height,
1218                        block_height,
1219                        format: desc.format,
1220                    },
1221                ));
1222            }
1223
1224            if desc.dimension == wgt::TextureDimension::D3 {
1225                // Only BCn formats with Sliced 3D feature can be used for 3D textures
1226                if desc.format.is_bcn() {
1227                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1228                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1229                } else if desc.format.is_astc() {
1230                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1231                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1232                } else {
1233                    return Err(CreateTextureError::InvalidCompressedDimension(
1234                        desc.dimension,
1235                        desc.format,
1236                    ));
1237                }
1238            }
1239        }
1240
1241        {
1242            let (width_multiple, height_multiple) = desc.format.size_multiple_requirement();
1243
1244            if desc.size.width % width_multiple != 0 {
1245                return Err(CreateTextureError::InvalidDimension(
1246                    TextureDimensionError::WidthNotMultipleOf {
1247                        width: desc.size.width,
1248                        multiple: width_multiple,
1249                        format: desc.format,
1250                    },
1251                ));
1252            }
1253
1254            if desc.size.height % height_multiple != 0 {
1255                return Err(CreateTextureError::InvalidDimension(
1256                    TextureDimensionError::HeightNotMultipleOf {
1257                        height: desc.size.height,
1258                        multiple: height_multiple,
1259                        format: desc.format,
1260                    },
1261                ));
1262            }
1263        }
1264
1265        let format_features = self
1266            .describe_format_features(desc.format)
1267            .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1268
1269        if desc.sample_count > 1 {
1270            if desc.mip_level_count != 1 {
1271                return Err(CreateTextureError::InvalidMipLevelCount {
1272                    requested: desc.mip_level_count,
1273                    maximum: 1,
1274                });
1275            }
1276
1277            if desc.size.depth_or_array_layers != 1 {
1278                return Err(CreateTextureError::InvalidDimension(
1279                    TextureDimensionError::MultisampledDepthOrArrayLayer(
1280                        desc.size.depth_or_array_layers,
1281                    ),
1282                ));
1283            }
1284
1285            if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
1286                return Err(CreateTextureError::InvalidMultisampledStorageBinding);
1287            }
1288
1289            if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1290                return Err(CreateTextureError::MultisampledNotRenderAttachment);
1291            }
1292
1293            if !format_features.flags.intersects(
1294                wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
1295                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
1296                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
1297                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
1298            ) {
1299                return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
1300            }
1301
1302            if !format_features
1303                .flags
1304                .sample_count_supported(desc.sample_count)
1305            {
1306                return Err(CreateTextureError::InvalidSampleCount(
1307                    desc.sample_count,
1308                    desc.format,
1309                    desc.format
1310                        .guaranteed_format_features(self.features)
1311                        .flags
1312                        .supported_sample_counts(),
1313                    self.adapter
1314                        .get_texture_format_features(desc.format)
1315                        .flags
1316                        .supported_sample_counts(),
1317                ));
1318            };
1319        }
1320
1321        let mips = desc.mip_level_count;
1322        let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1323        if mips == 0 || mips > max_levels_allowed {
1324            return Err(CreateTextureError::InvalidMipLevelCount {
1325                requested: mips,
1326                maximum: max_levels_allowed,
1327            });
1328        }
1329
1330        let missing_allowed_usages = desc.usage - format_features.allowed_usages;
1331        if !missing_allowed_usages.is_empty() {
1332            // detect downlevel incompatibilities
1333            let wgpu_allowed_usages = desc
1334                .format
1335                .guaranteed_format_features(self.features)
1336                .allowed_usages;
1337            let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
1338            return Err(CreateTextureError::InvalidFormatUsages(
1339                missing_allowed_usages,
1340                desc.format,
1341                wgpu_missing_usages.is_empty(),
1342            ));
1343        }
1344
1345        let mut hal_view_formats = Vec::new();
1346        for format in desc.view_formats.iter() {
1347            if desc.format == *format {
1348                continue;
1349            }
1350            if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
1351                return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
1352            }
1353            hal_view_formats.push(*format);
1354        }
1355        if !hal_view_formats.is_empty() {
1356            self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
1357        }
1358
1359        let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
1360
1361        let hal_desc = hal::TextureDescriptor {
1362            label: desc.label.to_hal(self.instance_flags),
1363            size: desc.size,
1364            mip_level_count: desc.mip_level_count,
1365            sample_count: desc.sample_count,
1366            dimension: desc.dimension,
1367            format: desc.format,
1368            usage: hal_usage,
1369            memory_flags: hal::MemoryFlags::empty(),
1370            view_formats: hal_view_formats,
1371        };
1372
1373        let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
1374            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1375
1376        let clear_mode = if hal_usage
1377            .intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE | wgt::TextureUses::COLOR_TARGET)
1378            && desc.dimension == wgt::TextureDimension::D2
1379        {
1380            let (is_color, usage) = if desc.format.is_depth_stencil_format() {
1381                (false, wgt::TextureUses::DEPTH_STENCIL_WRITE)
1382            } else {
1383                (true, wgt::TextureUses::COLOR_TARGET)
1384            };
1385
1386            let clear_label = hal_label(
1387                Some("(wgpu internal) clear texture view"),
1388                self.instance_flags,
1389            );
1390
1391            let mut clear_views = SmallVec::new();
1392            for mip_level in 0..desc.mip_level_count {
1393                for array_layer in 0..desc.size.depth_or_array_layers {
1394                    macro_rules! push_clear_view {
1395                        ($format:expr, $aspect:expr) => {
1396                            let desc = hal::TextureViewDescriptor {
1397                                label: clear_label,
1398                                format: $format,
1399                                dimension: TextureViewDimension::D2,
1400                                usage,
1401                                range: wgt::ImageSubresourceRange {
1402                                    aspect: $aspect,
1403                                    base_mip_level: mip_level,
1404                                    mip_level_count: Some(1),
1405                                    base_array_layer: array_layer,
1406                                    array_layer_count: Some(1),
1407                                },
1408                            };
1409                            clear_views.push(ManuallyDrop::new(
1410                                unsafe {
1411                                    self.raw().create_texture_view(raw_texture.as_ref(), &desc)
1412                                }
1413                                .map_err(|e| self.handle_hal_error(e))?,
1414                            ));
1415                        };
1416                    }
1417
1418                    if let Some(planes) = desc.format.planes() {
1419                        for plane in 0..planes {
1420                            let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1421                            let format = desc.format.aspect_specific_format(aspect).unwrap();
1422                            push_clear_view!(format, aspect);
1423                        }
1424                    } else {
1425                        push_clear_view!(desc.format, wgt::TextureAspect::All);
1426                    }
1427                }
1428            }
1429            resource::TextureClearMode::RenderPass {
1430                clear_views,
1431                is_color,
1432            }
1433        } else {
1434            resource::TextureClearMode::BufferCopy
1435        };
1436
1437        let texture = Texture::new(
1438            self,
1439            resource::TextureInner::Native { raw: raw_texture },
1440            hal_usage,
1441            desc,
1442            format_features,
1443            clear_mode,
1444            true,
1445        );
1446
1447        let texture = Arc::new(texture);
1448
1449        self.trackers
1450            .lock()
1451            .textures
1452            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1453
1454        Ok(texture)
1455    }
1456
1457    pub(crate) fn create_texture_view(
1458        self: &Arc<Self>,
1459        texture: &Arc<Texture>,
1460        desc: &resource::TextureViewDescriptor,
1461    ) -> Result<Arc<TextureView>, resource::CreateTextureViewError> {
1462        self.check_is_valid()?;
1463
1464        let snatch_guard = texture.device.snatchable_lock.read();
1465
1466        let texture_raw = texture.try_raw(&snatch_guard)?;
1467
1468        // resolve TextureViewDescriptor defaults
1469        // https://gpuweb.github.io/gpuweb/#abstract-opdef-resolving-gputextureviewdescriptor-defaults
1470        let resolved_format = desc.format.unwrap_or_else(|| {
1471            texture
1472                .desc
1473                .format
1474                .aspect_specific_format(desc.range.aspect)
1475                .unwrap_or(texture.desc.format)
1476        });
1477
1478        let resolved_dimension = desc
1479            .dimension
1480            .unwrap_or_else(|| match texture.desc.dimension {
1481                wgt::TextureDimension::D1 => TextureViewDimension::D1,
1482                wgt::TextureDimension::D2 => {
1483                    if texture.desc.array_layer_count() == 1 {
1484                        TextureViewDimension::D2
1485                    } else {
1486                        TextureViewDimension::D2Array
1487                    }
1488                }
1489                wgt::TextureDimension::D3 => TextureViewDimension::D3,
1490            });
1491
1492        let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| {
1493            texture
1494                .desc
1495                .mip_level_count
1496                .saturating_sub(desc.range.base_mip_level)
1497        });
1498
1499        let resolved_array_layer_count =
1500            desc.range
1501                .array_layer_count
1502                .unwrap_or_else(|| match resolved_dimension {
1503                    TextureViewDimension::D1
1504                    | TextureViewDimension::D2
1505                    | TextureViewDimension::D3 => 1,
1506                    TextureViewDimension::Cube => 6,
1507                    TextureViewDimension::D2Array | TextureViewDimension::CubeArray => texture
1508                        .desc
1509                        .array_layer_count()
1510                        .saturating_sub(desc.range.base_array_layer),
1511                });
1512
1513        let resolved_usage = {
1514            let usage = desc.usage.unwrap_or(wgt::TextureUsages::empty());
1515            if usage.is_empty() {
1516                texture.desc.usage
1517            } else if texture.desc.usage.contains(usage) {
1518                usage
1519            } else {
1520                return Err(resource::CreateTextureViewError::InvalidTextureViewUsage {
1521                    view: usage,
1522                    texture: texture.desc.usage,
1523                });
1524            }
1525        };
1526
1527        let format_features = self.describe_format_features(resolved_format)?;
1528        let allowed_format_usages = format_features.allowed_usages;
1529        if resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1530            && !allowed_format_usages.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1531        {
1532            return Err(
1533                resource::CreateTextureViewError::TextureViewFormatNotRenderable(resolved_format),
1534            );
1535        }
1536
1537        if resolved_usage.contains(wgt::TextureUsages::STORAGE_BINDING)
1538            && !allowed_format_usages.contains(wgt::TextureUsages::STORAGE_BINDING)
1539        {
1540            return Err(
1541                resource::CreateTextureViewError::TextureViewFormatNotStorage(resolved_format),
1542            );
1543        }
1544
1545        // validate TextureViewDescriptor
1546
1547        let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect);
1548        if aspects.is_empty() {
1549            return Err(resource::CreateTextureViewError::InvalidAspect {
1550                texture_format: texture.desc.format,
1551                requested_aspect: desc.range.aspect,
1552            });
1553        }
1554
1555        let format_is_good = if desc.range.aspect == wgt::TextureAspect::All {
1556            resolved_format == texture.desc.format
1557                || texture.desc.view_formats.contains(&resolved_format)
1558        } else {
1559            Some(resolved_format)
1560                == texture
1561                    .desc
1562                    .format
1563                    .aspect_specific_format(desc.range.aspect)
1564        };
1565        if !format_is_good {
1566            return Err(resource::CreateTextureViewError::FormatReinterpretation {
1567                texture: texture.desc.format,
1568                view: resolved_format,
1569            });
1570        }
1571
1572        // check if multisampled texture is seen as anything but 2D
1573        if texture.desc.sample_count > 1 && resolved_dimension != TextureViewDimension::D2 {
1574            return Err(
1575                resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension(
1576                    resolved_dimension,
1577                ),
1578            );
1579        }
1580
1581        // check if the dimension is compatible with the texture
1582        if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() {
1583            return Err(
1584                resource::CreateTextureViewError::InvalidTextureViewDimension {
1585                    view: resolved_dimension,
1586                    texture: texture.desc.dimension,
1587                },
1588            );
1589        }
1590
1591        match resolved_dimension {
1592            TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => {
1593                if resolved_array_layer_count != 1 {
1594                    return Err(resource::CreateTextureViewError::InvalidArrayLayerCount {
1595                        requested: resolved_array_layer_count,
1596                        dim: resolved_dimension,
1597                    });
1598                }
1599            }
1600            TextureViewDimension::Cube => {
1601                if resolved_array_layer_count != 6 {
1602                    return Err(
1603                        resource::CreateTextureViewError::InvalidCubemapTextureDepth {
1604                            depth: resolved_array_layer_count,
1605                        },
1606                    );
1607                }
1608            }
1609            TextureViewDimension::CubeArray => {
1610                if resolved_array_layer_count % 6 != 0 {
1611                    return Err(
1612                        resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth {
1613                            depth: resolved_array_layer_count,
1614                        },
1615                    );
1616                }
1617            }
1618            _ => {}
1619        }
1620
1621        match resolved_dimension {
1622            TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1623                if texture.desc.size.width != texture.desc.size.height {
1624                    return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize);
1625                }
1626            }
1627            _ => {}
1628        }
1629
1630        if resolved_mip_level_count == 0 {
1631            return Err(resource::CreateTextureViewError::ZeroMipLevelCount);
1632        }
1633
1634        let mip_level_end = desc
1635            .range
1636            .base_mip_level
1637            .saturating_add(resolved_mip_level_count);
1638
1639        let level_end = texture.desc.mip_level_count;
1640        if mip_level_end > level_end {
1641            return Err(resource::CreateTextureViewError::TooManyMipLevels {
1642                requested: mip_level_end,
1643                total: level_end,
1644            });
1645        }
1646
1647        if resolved_array_layer_count == 0 {
1648            return Err(resource::CreateTextureViewError::ZeroArrayLayerCount);
1649        }
1650
1651        let array_layer_end = desc
1652            .range
1653            .base_array_layer
1654            .saturating_add(resolved_array_layer_count);
1655
1656        let layer_end = texture.desc.array_layer_count();
1657        if array_layer_end > layer_end {
1658            return Err(resource::CreateTextureViewError::TooManyArrayLayers {
1659                requested: array_layer_end,
1660                total: layer_end,
1661            });
1662        };
1663
1664        // https://gpuweb.github.io/gpuweb/#abstract-opdef-renderable-texture-view
1665        let render_extent = 'error: {
1666            if !resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1667                break 'error Err(TextureViewNotRenderableReason::Usage(resolved_usage));
1668            }
1669
1670            let allowed_view_dimensions = [
1671                TextureViewDimension::D2,
1672                TextureViewDimension::D2Array,
1673                TextureViewDimension::D3,
1674            ];
1675            if !allowed_view_dimensions.contains(&resolved_dimension) {
1676                break 'error Err(TextureViewNotRenderableReason::Dimension(
1677                    resolved_dimension,
1678                ));
1679            }
1680
1681            if resolved_mip_level_count != 1 {
1682                break 'error Err(TextureViewNotRenderableReason::MipLevelCount(
1683                    resolved_mip_level_count,
1684                ));
1685            }
1686
1687            if resolved_array_layer_count != 1
1688                && !(self.features.contains(wgt::Features::MULTIVIEW))
1689            {
1690                break 'error Err(TextureViewNotRenderableReason::ArrayLayerCount(
1691                    resolved_array_layer_count,
1692                ));
1693            }
1694
1695            if aspects != hal::FormatAspects::from(texture.desc.format) {
1696                break 'error Err(TextureViewNotRenderableReason::Aspects(aspects));
1697            }
1698
1699            Ok(texture
1700                .desc
1701                .compute_render_extent(desc.range.base_mip_level))
1702        };
1703
1704        // filter the usages based on the other criteria
1705        let usage = {
1706            let resolved_hal_usage = conv::map_texture_usage(
1707                resolved_usage,
1708                resolved_format.into(),
1709                format_features.flags,
1710            );
1711            let mask_copy = !(wgt::TextureUses::COPY_SRC | wgt::TextureUses::COPY_DST);
1712            let mask_dimension = match resolved_dimension {
1713                TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1714                    wgt::TextureUses::RESOURCE
1715                }
1716                TextureViewDimension::D3 => {
1717                    wgt::TextureUses::RESOURCE
1718                        | wgt::TextureUses::STORAGE_READ_ONLY
1719                        | wgt::TextureUses::STORAGE_WRITE_ONLY
1720                        | wgt::TextureUses::STORAGE_READ_WRITE
1721                }
1722                _ => wgt::TextureUses::all(),
1723            };
1724            let mask_mip_level = if resolved_mip_level_count == 1 {
1725                wgt::TextureUses::all()
1726            } else {
1727                wgt::TextureUses::RESOURCE
1728            };
1729            resolved_hal_usage & mask_copy & mask_dimension & mask_mip_level
1730        };
1731
1732        // use the combined depth-stencil format for the view
1733        let format = if resolved_format.is_depth_stencil_component(texture.desc.format) {
1734            texture.desc.format
1735        } else {
1736            resolved_format
1737        };
1738
1739        let resolved_range = wgt::ImageSubresourceRange {
1740            aspect: desc.range.aspect,
1741            base_mip_level: desc.range.base_mip_level,
1742            mip_level_count: Some(resolved_mip_level_count),
1743            base_array_layer: desc.range.base_array_layer,
1744            array_layer_count: Some(resolved_array_layer_count),
1745        };
1746
1747        let hal_desc = hal::TextureViewDescriptor {
1748            label: desc.label.to_hal(self.instance_flags),
1749            format,
1750            dimension: resolved_dimension,
1751            usage,
1752            range: resolved_range,
1753        };
1754
1755        let raw = unsafe { self.raw().create_texture_view(texture_raw, &hal_desc) }
1756            .map_err(|e| self.handle_hal_error(e))?;
1757
1758        let selector = TextureSelector {
1759            mips: desc.range.base_mip_level..mip_level_end,
1760            layers: desc.range.base_array_layer..array_layer_end,
1761        };
1762
1763        let view = TextureView {
1764            raw: Snatchable::new(raw),
1765            parent: texture.clone(),
1766            device: self.clone(),
1767            desc: resource::HalTextureViewDescriptor {
1768                texture_format: texture.desc.format,
1769                format: resolved_format,
1770                dimension: resolved_dimension,
1771                usage: resolved_usage,
1772                range: resolved_range,
1773            },
1774            format_features: texture.format_features,
1775            render_extent,
1776            samples: texture.desc.sample_count,
1777            selector,
1778            label: desc.label.to_string(),
1779            tracking_data: TrackingData::new(self.tracker_indices.texture_views.clone()),
1780        };
1781
1782        let view = Arc::new(view);
1783
1784        {
1785            let mut views = texture.views.lock();
1786            views.push(Arc::downgrade(&view));
1787        }
1788
1789        Ok(view)
1790    }
1791
1792    pub(crate) fn create_external_texture(
1793        self: &Arc<Self>,
1794        desc: &resource::ExternalTextureDescriptor,
1795        planes: &[Arc<TextureView>],
1796    ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
1797        use resource::CreateExternalTextureError;
1798        self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
1799        self.check_is_valid()?;
1800
1801        if desc.num_planes() != planes.len() {
1802            return Err(CreateExternalTextureError::IncorrectPlaneCount {
1803                format: desc.format,
1804                expected: desc.num_planes(),
1805                provided: planes.len(),
1806            });
1807        }
1808
1809        let planes = planes
1810            .iter()
1811            .enumerate()
1812            .map(|(i, plane)| {
1813                if plane.samples != 1 {
1814                    return Err(CreateExternalTextureError::InvalidPlaneMultisample(
1815                        plane.samples,
1816                    ));
1817                }
1818
1819                let sample_type = plane
1820                    .desc
1821                    .format
1822                    .sample_type(Some(plane.desc.range.aspect), Some(self.features))
1823                    .unwrap();
1824                if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
1825                    return Err(CreateExternalTextureError::InvalidPlaneSampleType {
1826                        format: plane.desc.format,
1827                        sample_type,
1828                    });
1829                }
1830
1831                if plane.desc.dimension != TextureViewDimension::D2 {
1832                    return Err(CreateExternalTextureError::InvalidPlaneDimension(
1833                        plane.desc.dimension,
1834                    ));
1835                }
1836
1837                let expected_components = match desc.format {
1838                    wgt::ExternalTextureFormat::Rgba => 4,
1839                    wgt::ExternalTextureFormat::Nv12 => match i {
1840                        0 => 1,
1841                        1 => 2,
1842                        _ => unreachable!(),
1843                    },
1844                    wgt::ExternalTextureFormat::Yu12 => 1,
1845                };
1846                if plane.desc.format.components() != expected_components {
1847                    return Err(CreateExternalTextureError::InvalidPlaneFormat {
1848                        format: desc.format,
1849                        plane: i,
1850                        expected: expected_components,
1851                        provided: plane.desc.format,
1852                    });
1853                }
1854
1855                plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
1856                Ok(plane.clone())
1857            })
1858            .collect::<Result<_, _>>()?;
1859
1860        let params_data = ExternalTextureParams::from_desc(desc);
1861        let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
1862        let params_desc = resource::BufferDescriptor {
1863            label: label.map(Cow::Owned),
1864            size: size_of_val(&params_data) as wgt::BufferAddress,
1865            usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
1866            mapped_at_creation: false,
1867        };
1868        let params = self.create_buffer(&params_desc)?;
1869        self.get_queue().unwrap().write_buffer(
1870            Fallible::Valid(params.clone()),
1871            0,
1872            bytemuck::bytes_of(&params_data),
1873        )?;
1874
1875        let external_texture = ExternalTexture {
1876            device: self.clone(),
1877            planes,
1878            params,
1879            label: desc.label.to_string(),
1880            tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
1881        };
1882        let external_texture = Arc::new(external_texture);
1883
1884        Ok(external_texture)
1885    }
1886
1887    pub(crate) fn create_sampler(
1888        self: &Arc<Self>,
1889        desc: &resource::SamplerDescriptor,
1890    ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
1891        self.check_is_valid()?;
1892
1893        if desc
1894            .address_modes
1895            .iter()
1896            .any(|am| am == &wgt::AddressMode::ClampToBorder)
1897        {
1898            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
1899        }
1900
1901        if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
1902            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
1903        }
1904
1905        if desc.lod_min_clamp < 0.0 {
1906            return Err(resource::CreateSamplerError::InvalidLodMinClamp(
1907                desc.lod_min_clamp,
1908            ));
1909        }
1910        if desc.lod_max_clamp < desc.lod_min_clamp {
1911            return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
1912                lod_min_clamp: desc.lod_min_clamp,
1913                lod_max_clamp: desc.lod_max_clamp,
1914            });
1915        }
1916
1917        if desc.anisotropy_clamp < 1 {
1918            return Err(resource::CreateSamplerError::InvalidAnisotropy(
1919                desc.anisotropy_clamp,
1920            ));
1921        }
1922
1923        if desc.anisotropy_clamp != 1 {
1924            if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
1925                return Err(
1926                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
1927                        filter_type: resource::SamplerFilterErrorType::MinFilter,
1928                        filter_mode: desc.min_filter,
1929                        anisotropic_clamp: desc.anisotropy_clamp,
1930                    },
1931                );
1932            }
1933            if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
1934                return Err(
1935                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
1936                        filter_type: resource::SamplerFilterErrorType::MagFilter,
1937                        filter_mode: desc.mag_filter,
1938                        anisotropic_clamp: desc.anisotropy_clamp,
1939                    },
1940                );
1941            }
1942            if !matches!(desc.mipmap_filter, wgt::FilterMode::Linear) {
1943                return Err(
1944                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
1945                        filter_type: resource::SamplerFilterErrorType::MipmapFilter,
1946                        filter_mode: desc.mipmap_filter,
1947                        anisotropic_clamp: desc.anisotropy_clamp,
1948                    },
1949                );
1950            }
1951        }
1952
1953        let anisotropy_clamp = if self
1954            .downlevel
1955            .flags
1956            .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
1957        {
1958            // Clamp anisotropy clamp to [1, 16] per the wgpu-hal interface
1959            desc.anisotropy_clamp.min(16)
1960        } else {
1961            // If it isn't supported, set this unconditionally to 1
1962            1
1963        };
1964
1965        //TODO: check for wgt::DownlevelFlags::COMPARISON_SAMPLERS
1966
1967        let hal_desc = hal::SamplerDescriptor {
1968            label: desc.label.to_hal(self.instance_flags),
1969            address_modes: desc.address_modes,
1970            mag_filter: desc.mag_filter,
1971            min_filter: desc.min_filter,
1972            mipmap_filter: desc.mipmap_filter,
1973            lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
1974            compare: desc.compare,
1975            anisotropy_clamp,
1976            border_color: desc.border_color,
1977        };
1978
1979        let raw = unsafe { self.raw().create_sampler(&hal_desc) }
1980            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1981
1982        let sampler = Sampler {
1983            raw: ManuallyDrop::new(raw),
1984            device: self.clone(),
1985            label: desc.label.to_string(),
1986            tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
1987            comparison: desc.compare.is_some(),
1988            filtering: desc.min_filter == wgt::FilterMode::Linear
1989                || desc.mag_filter == wgt::FilterMode::Linear
1990                || desc.mipmap_filter == wgt::FilterMode::Linear,
1991        };
1992
1993        let sampler = Arc::new(sampler);
1994
1995        Ok(sampler)
1996    }
1997
1998    pub(crate) fn create_shader_module<'a>(
1999        self: &Arc<Self>,
2000        desc: &pipeline::ShaderModuleDescriptor<'a>,
2001        source: pipeline::ShaderModuleSource<'a>,
2002    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2003        self.check_is_valid()?;
2004
2005        let (module, source) = match source {
2006            #[cfg(feature = "wgsl")]
2007            pipeline::ShaderModuleSource::Wgsl(code) => {
2008                profiling::scope!("naga::front::wgsl::parse_str");
2009                let module = naga::front::wgsl::parse_str(&code).map_err(|inner| {
2010                    pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2011                        source: code.to_string(),
2012                        label: desc.label.as_ref().map(|l| l.to_string()),
2013                        inner: Box::new(inner),
2014                    })
2015                })?;
2016                (Cow::Owned(module), code.into_owned())
2017            }
2018            #[cfg(feature = "spirv")]
2019            pipeline::ShaderModuleSource::SpirV(spv, options) => {
2020                let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2021                profiling::scope!("naga::front::spv::Frontend");
2022                let module = parser.parse().map_err(|inner| {
2023                    pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2024                        source: String::new(),
2025                        label: desc.label.as_ref().map(|l| l.to_string()),
2026                        inner: Box::new(inner),
2027                    })
2028                })?;
2029                (Cow::Owned(module), String::new())
2030            }
2031            #[cfg(feature = "glsl")]
2032            pipeline::ShaderModuleSource::Glsl(code, options) => {
2033                let mut parser = naga::front::glsl::Frontend::default();
2034                profiling::scope!("naga::front::glsl::Frontend.parse");
2035                let module = parser.parse(&options, &code).map_err(|inner| {
2036                    pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2037                        source: code.to_string(),
2038                        label: desc.label.as_ref().map(|l| l.to_string()),
2039                        inner: Box::new(inner),
2040                    })
2041                })?;
2042                (Cow::Owned(module), code.into_owned())
2043            }
2044            pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2045            pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2046        };
2047        for (_, var) in module.global_variables.iter() {
2048            match var.binding {
2049                Some(br) if br.group >= self.limits.max_bind_groups => {
2050                    return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2051                        bind: br,
2052                        group: br.group,
2053                        limit: self.limits.max_bind_groups,
2054                    });
2055                }
2056                _ => continue,
2057            };
2058        }
2059
2060        profiling::scope!("naga::validate");
2061        let debug_source =
2062            if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2063                Some(hal::DebugSource {
2064                    file_name: Cow::Owned(
2065                        desc.label
2066                            .as_ref()
2067                            .map_or("shader".to_string(), |l| l.to_string()),
2068                    ),
2069                    source_code: Cow::Owned(source.clone()),
2070                })
2071            } else {
2072                None
2073            };
2074
2075        let info = create_validator(
2076            self.features,
2077            self.downlevel.flags,
2078            naga::valid::ValidationFlags::all(),
2079        )
2080        .validate(&module)
2081        .map_err(|inner| {
2082            pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2083                source,
2084                label: desc.label.as_ref().map(|l| l.to_string()),
2085                inner: Box::new(inner),
2086            })
2087        })?;
2088
2089        let interface = validation::Interface::new(&module, &info, self.limits.clone());
2090        let hal_shader = hal::ShaderInput::Naga(hal::NagaShader {
2091            module,
2092            info,
2093            debug_source,
2094        });
2095        let hal_desc = hal::ShaderModuleDescriptor {
2096            label: desc.label.to_hal(self.instance_flags),
2097            runtime_checks: desc.runtime_checks,
2098        };
2099        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2100            Ok(raw) => raw,
2101            Err(error) => {
2102                return Err(match error {
2103                    hal::ShaderError::Device(error) => {
2104                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2105                    }
2106                    hal::ShaderError::Compilation(ref msg) => {
2107                        log::error!("Shader error: {msg}");
2108                        pipeline::CreateShaderModuleError::Generation
2109                    }
2110                })
2111            }
2112        };
2113
2114        let module = pipeline::ShaderModule {
2115            raw: ManuallyDrop::new(raw),
2116            device: self.clone(),
2117            interface: Some(interface),
2118            label: desc.label.to_string(),
2119        };
2120
2121        let module = Arc::new(module);
2122
2123        Ok(module)
2124    }
2125
2126    #[allow(unused_unsafe)]
2127    pub(crate) unsafe fn create_shader_module_passthrough<'a>(
2128        self: &Arc<Self>,
2129        descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2130    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2131        self.check_is_valid()?;
2132        self.require_features(wgt::Features::EXPERIMENTAL_PASSTHROUGH_SHADERS)?;
2133
2134        // TODO: when we get to use if-let chains, this will be a little nicer!
2135
2136        log::info!("Backend: {}", self.backend());
2137        let hal_shader = match self.backend() {
2138            wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2139                descriptor
2140                    .spirv
2141                    .as_ref()
2142                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2143            ),
2144            wgt::Backend::Dx12 => {
2145                if let Some(dxil) = &descriptor.dxil {
2146                    hal::ShaderInput::Dxil {
2147                        shader: dxil,
2148                        entry_point: descriptor.entry_point.clone(),
2149                        num_workgroups: descriptor.num_workgroups,
2150                    }
2151                } else if let Some(hlsl) = &descriptor.hlsl {
2152                    hal::ShaderInput::Hlsl {
2153                        shader: hlsl,
2154                        entry_point: descriptor.entry_point.clone(),
2155                        num_workgroups: descriptor.num_workgroups,
2156                    }
2157                } else {
2158                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2159                }
2160            }
2161            wgt::Backend::Metal => hal::ShaderInput::Msl {
2162                shader: descriptor
2163                    .msl
2164                    .as_ref()
2165                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2166                entry_point: descriptor.entry_point.clone(),
2167                num_workgroups: descriptor.num_workgroups,
2168            },
2169            wgt::Backend::Gl => hal::ShaderInput::Glsl {
2170                shader: descriptor
2171                    .glsl
2172                    .as_ref()
2173                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2174                entry_point: descriptor.entry_point.clone(),
2175                num_workgroups: descriptor.num_workgroups,
2176            },
2177            wgt::Backend::Noop => {
2178                return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2179            }
2180            wgt::Backend::BrowserWebGpu => unreachable!(),
2181        };
2182
2183        let hal_desc = hal::ShaderModuleDescriptor {
2184            label: descriptor.label.to_hal(self.instance_flags),
2185            runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2186        };
2187
2188        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2189            Ok(raw) => raw,
2190            Err(error) => {
2191                return Err(match error {
2192                    hal::ShaderError::Device(error) => {
2193                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2194                    }
2195                    hal::ShaderError::Compilation(ref msg) => {
2196                        log::error!("Shader error: {msg}");
2197                        pipeline::CreateShaderModuleError::Generation
2198                    }
2199                })
2200            }
2201        };
2202
2203        let module = pipeline::ShaderModule {
2204            raw: ManuallyDrop::new(raw),
2205            device: self.clone(),
2206            interface: None,
2207            label: descriptor.label.to_string(),
2208        };
2209
2210        Ok(Arc::new(module))
2211    }
2212
2213    pub(crate) fn create_command_encoder(
2214        self: &Arc<Self>,
2215        label: &crate::Label,
2216    ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2217        self.check_is_valid()?;
2218
2219        let queue = self.get_queue().unwrap();
2220
2221        let encoder = self
2222            .command_allocator
2223            .acquire_encoder(self.raw(), queue.raw())
2224            .map_err(|e| self.handle_hal_error(e))?;
2225
2226        let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2227
2228        let cmd_enc = Arc::new(cmd_enc);
2229
2230        Ok(cmd_enc)
2231    }
2232
2233    /// Generate information about late-validated buffer bindings for pipelines.
2234    //TODO: should this be combined with `get_introspection_bind_group_layouts` in some way?
2235    fn make_late_sized_buffer_groups(
2236        shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2237        layout: &binding_model::PipelineLayout,
2238    ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2239        // Given the shader-required binding sizes and the pipeline layout,
2240        // return the filtered list of them in the layout order,
2241        // removing those with given `min_binding_size`.
2242        layout
2243            .bind_group_layouts
2244            .iter()
2245            .enumerate()
2246            .map(|(group_index, bgl)| pipeline::LateSizedBufferGroup {
2247                shader_sizes: bgl
2248                    .entries
2249                    .values()
2250                    .filter_map(|entry| match entry.ty {
2251                        wgt::BindingType::Buffer {
2252                            min_binding_size: None,
2253                            ..
2254                        } => {
2255                            let rb = naga::ResourceBinding {
2256                                group: group_index as u32,
2257                                binding: entry.binding,
2258                            };
2259                            let shader_size =
2260                                shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
2261                            Some(shader_size)
2262                        }
2263                        _ => None,
2264                    })
2265                    .collect(),
2266            })
2267            .collect()
2268    }
2269
2270    pub(crate) fn create_bind_group_layout(
2271        self: &Arc<Self>,
2272        label: &crate::Label,
2273        entry_map: bgl::EntryMap,
2274        origin: bgl::Origin,
2275    ) -> Result<Arc<BindGroupLayout>, binding_model::CreateBindGroupLayoutError> {
2276        #[derive(PartialEq)]
2277        enum WritableStorage {
2278            Yes,
2279            No,
2280        }
2281
2282        for entry in entry_map.values() {
2283            use wgt::BindingType as Bt;
2284
2285            let mut required_features = wgt::Features::empty();
2286            let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
2287            let (array_feature, writable_storage) = match entry.ty {
2288                Bt::Buffer {
2289                    ty: wgt::BufferBindingType::Uniform,
2290                    has_dynamic_offset: false,
2291                    min_binding_size: _,
2292                } => (
2293                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2294                    WritableStorage::No,
2295                ),
2296                Bt::Buffer {
2297                    ty: wgt::BufferBindingType::Uniform,
2298                    has_dynamic_offset: true,
2299                    min_binding_size: _,
2300                } => (
2301                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2302                    WritableStorage::No,
2303                ),
2304                Bt::Buffer {
2305                    ty: wgt::BufferBindingType::Storage { read_only },
2306                    ..
2307                } => (
2308                    Some(
2309                        wgt::Features::BUFFER_BINDING_ARRAY
2310                            | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2311                    ),
2312                    match read_only {
2313                        true => WritableStorage::No,
2314                        false => WritableStorage::Yes,
2315                    },
2316                ),
2317                Bt::Sampler { .. } => (
2318                    Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2319                    WritableStorage::No,
2320                ),
2321                Bt::Texture {
2322                    multisampled: true,
2323                    sample_type: TextureSampleType::Float { filterable: true },
2324                    ..
2325                } => {
2326                    return Err(binding_model::CreateBindGroupLayoutError::Entry {
2327                        binding: entry.binding,
2328                        error:
2329                            BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
2330                    });
2331                }
2332                Bt::Texture {
2333                    multisampled,
2334                    view_dimension,
2335                    ..
2336                } => {
2337                    if multisampled && view_dimension != TextureViewDimension::D2 {
2338                        return Err(binding_model::CreateBindGroupLayoutError::Entry {
2339                            binding: entry.binding,
2340                            error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
2341                        });
2342                    }
2343
2344                    (
2345                        Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2346                        WritableStorage::No,
2347                    )
2348                }
2349                Bt::StorageTexture {
2350                    access,
2351                    view_dimension,
2352                    format: _,
2353                } => {
2354                    match view_dimension {
2355                        TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2356                            return Err(binding_model::CreateBindGroupLayoutError::Entry {
2357                                binding: entry.binding,
2358                                error: BindGroupLayoutEntryError::StorageTextureCube,
2359                            })
2360                        }
2361                        _ => (),
2362                    }
2363                    match access {
2364                        wgt::StorageTextureAccess::Atomic
2365                            if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
2366                        {
2367                            return Err(binding_model::CreateBindGroupLayoutError::Entry {
2368                                binding: entry.binding,
2369                                error: BindGroupLayoutEntryError::StorageTextureAtomic,
2370                            });
2371                        }
2372                        _ => (),
2373                    }
2374                    (
2375                        Some(
2376                            wgt::Features::TEXTURE_BINDING_ARRAY
2377                                | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2378                        ),
2379                        match access {
2380                            wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
2381                            wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
2382                            wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
2383                            wgt::StorageTextureAccess::Atomic => {
2384                                required_features |= wgt::Features::TEXTURE_ATOMIC;
2385                                WritableStorage::Yes
2386                            }
2387                        },
2388                    )
2389                }
2390                Bt::AccelerationStructure { vertex_return } => {
2391                    self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)
2392                        .map_err(|e| binding_model::CreateBindGroupLayoutError::Entry {
2393                            binding: entry.binding,
2394                            error: e.into(),
2395                        })?;
2396                    if vertex_return {
2397                        self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
2398                            .map_err(|e| binding_model::CreateBindGroupLayoutError::Entry {
2399                                binding: entry.binding,
2400                                error: e.into(),
2401                            })?;
2402                    }
2403
2404                    (None, WritableStorage::No)
2405                }
2406                Bt::ExternalTexture => {
2407                    self.require_features(wgt::Features::EXTERNAL_TEXTURE)
2408                        .map_err(|e| binding_model::CreateBindGroupLayoutError::Entry {
2409                            binding: entry.binding,
2410                            error: e.into(),
2411                        })?;
2412                    (None, WritableStorage::No)
2413                }
2414            };
2415
2416            // Validate the count parameter
2417            if entry.count.is_some() {
2418                required_features |= array_feature
2419                    .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
2420                    .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry {
2421                        binding: entry.binding,
2422                        error,
2423                    })?;
2424            }
2425
2426            if entry.visibility.contains_unknown_bits() {
2427                return Err(
2428                    binding_model::CreateBindGroupLayoutError::InvalidVisibility(entry.visibility),
2429                );
2430            }
2431
2432            if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
2433                if writable_storage == WritableStorage::Yes {
2434                    required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
2435                }
2436                if let Bt::Buffer {
2437                    ty: wgt::BufferBindingType::Storage { .. },
2438                    ..
2439                } = entry.ty
2440                {
2441                    required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
2442                }
2443            }
2444            if writable_storage == WritableStorage::Yes
2445                && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
2446            {
2447                required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
2448            }
2449
2450            self.require_features(required_features)
2451                .map_err(BindGroupLayoutEntryError::MissingFeatures)
2452                .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry {
2453                    binding: entry.binding,
2454                    error,
2455                })?;
2456            self.require_downlevel_flags(required_downlevel_flags)
2457                .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
2458                .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry {
2459                    binding: entry.binding,
2460                    error,
2461                })?;
2462        }
2463
2464        let bgl_flags = conv::bind_group_layout_flags(self.features);
2465
2466        let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
2467        let hal_desc = hal::BindGroupLayoutDescriptor {
2468            label: label.to_hal(self.instance_flags),
2469            flags: bgl_flags,
2470            entries: &hal_bindings,
2471        };
2472
2473        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
2474        for entry in entry_map.values() {
2475            count_validator.add_binding(entry);
2476        }
2477        // If a single bind group layout violates limits, the pipeline layout is
2478        // definitely going to violate limits too, lets catch it now.
2479        count_validator
2480            .validate(&self.limits)
2481            .map_err(binding_model::CreateBindGroupLayoutError::TooManyBindings)?;
2482
2483        // Validate that binding arrays don't conflict with dynamic offsets.
2484        count_validator.validate_binding_arrays()?;
2485
2486        let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
2487            .map_err(|e| self.handle_hal_error(e))?;
2488
2489        let bgl = BindGroupLayout {
2490            raw: ManuallyDrop::new(raw),
2491            device: self.clone(),
2492            entries: entry_map,
2493            origin,
2494            exclusive_pipeline: OnceCellOrLock::new(),
2495            binding_count_validator: count_validator,
2496            label: label.to_string(),
2497        };
2498
2499        let bgl = Arc::new(bgl);
2500
2501        Ok(bgl)
2502    }
2503
2504    fn create_buffer_binding<'a>(
2505        &self,
2506        bb: &'a binding_model::ResolvedBufferBinding,
2507        binding: u32,
2508        decl: &wgt::BindGroupLayoutEntry,
2509        used_buffer_ranges: &mut Vec<BufferInitTrackerAction>,
2510        dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
2511        late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
2512        used: &mut BindGroupStates,
2513        snatch_guard: &'a SnatchGuard<'a>,
2514    ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer>, binding_model::CreateBindGroupError>
2515    {
2516        use crate::binding_model::CreateBindGroupError as Error;
2517
2518        let (binding_ty, dynamic, min_size) = match decl.ty {
2519            wgt::BindingType::Buffer {
2520                ty,
2521                has_dynamic_offset,
2522                min_binding_size,
2523            } => (ty, has_dynamic_offset, min_binding_size),
2524            _ => {
2525                return Err(Error::WrongBindingType {
2526                    binding,
2527                    actual: decl.ty,
2528                    expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
2529                })
2530            }
2531        };
2532
2533        let (pub_usage, internal_use, range_limit) = match binding_ty {
2534            wgt::BufferBindingType::Uniform => (
2535                wgt::BufferUsages::UNIFORM,
2536                wgt::BufferUses::UNIFORM,
2537                self.limits.max_uniform_buffer_binding_size,
2538            ),
2539            wgt::BufferBindingType::Storage { read_only } => (
2540                wgt::BufferUsages::STORAGE,
2541                if read_only {
2542                    wgt::BufferUses::STORAGE_READ_ONLY
2543                } else {
2544                    wgt::BufferUses::STORAGE_READ_WRITE
2545                },
2546                self.limits.max_storage_buffer_binding_size,
2547            ),
2548        };
2549
2550        let (align, align_limit_name) =
2551            binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
2552        if bb.offset % align as u64 != 0 {
2553            return Err(Error::UnalignedBufferOffset(
2554                bb.offset,
2555                align_limit_name,
2556                align,
2557            ));
2558        }
2559
2560        let buffer = &bb.buffer;
2561
2562        used.buffers.insert_single(buffer.clone(), internal_use);
2563
2564        buffer.same_device(self)?;
2565
2566        buffer.check_usage(pub_usage)?;
2567
2568        let (bb, bind_size) = buffer.binding(bb.offset, bb.size, snatch_guard)?;
2569
2570        if matches!(binding_ty, wgt::BufferBindingType::Storage { .. }) {
2571            let storage_buf_size_alignment = 4;
2572
2573            let aligned = bind_size % u64::from(storage_buf_size_alignment) == 0;
2574            if !aligned {
2575                return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
2576                    alignment: storage_buf_size_alignment,
2577                    size: bind_size,
2578                });
2579            }
2580        }
2581
2582        let bind_end = bb.offset + bind_size;
2583
2584        if bind_size > range_limit as u64 {
2585            return Err(Error::BufferRangeTooLarge {
2586                binding,
2587                given: bind_size as u32,
2588                limit: range_limit,
2589            });
2590        }
2591
2592        // Record binding info for validating dynamic offsets
2593        if dynamic {
2594            dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
2595                binding_idx: binding,
2596                buffer_size: buffer.size,
2597                binding_range: bb.offset..bind_end,
2598                maximum_dynamic_offset: buffer.size - bind_end,
2599                binding_type: binding_ty,
2600            });
2601        }
2602
2603        if let Some(non_zero) = min_size {
2604            let min_size = non_zero.get();
2605            if min_size > bind_size {
2606                return Err(Error::BindingSizeTooSmall {
2607                    buffer: buffer.error_ident(),
2608                    actual: bind_size,
2609                    min: min_size,
2610                });
2611            }
2612        } else {
2613            let late_size = wgt::BufferSize::new(bind_size)
2614                .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
2615            late_buffer_binding_sizes.insert(binding, late_size);
2616        }
2617
2618        // This was checked against the device's alignment requirements above,
2619        // which should always be a multiple of `COPY_BUFFER_ALIGNMENT`.
2620        assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
2621
2622        // `wgpu_hal` only restricts shader access to bound buffer regions with
2623        // a certain resolution. For the sake of lazy initialization, round up
2624        // the size of the bound range to reflect how much of the buffer is
2625        // actually going to be visible to the shader.
2626        let bounds_check_alignment =
2627            binding_model::buffer_binding_type_bounds_check_alignment(&self.alignments, binding_ty);
2628        let visible_size = align_to(bind_size, bounds_check_alignment);
2629
2630        used_buffer_ranges.extend(buffer.initialization_status.read().create_action(
2631            buffer,
2632            bb.offset..bb.offset + visible_size,
2633            MemoryInitKind::NeedsInitializedMemory,
2634        ));
2635
2636        Ok(bb)
2637    }
2638
2639    fn create_sampler_binding<'a>(
2640        &self,
2641        used: &mut BindGroupStates,
2642        binding: u32,
2643        decl: &wgt::BindGroupLayoutEntry,
2644        sampler: &'a Arc<Sampler>,
2645    ) -> Result<&'a dyn hal::DynSampler, binding_model::CreateBindGroupError> {
2646        use crate::binding_model::CreateBindGroupError as Error;
2647
2648        used.samplers.insert_single(sampler.clone());
2649
2650        sampler.same_device(self)?;
2651
2652        match decl.ty {
2653            wgt::BindingType::Sampler(ty) => {
2654                let (allowed_filtering, allowed_comparison) = match ty {
2655                    wgt::SamplerBindingType::Filtering => (None, false),
2656                    wgt::SamplerBindingType::NonFiltering => (Some(false), false),
2657                    wgt::SamplerBindingType::Comparison => (None, true),
2658                };
2659                if let Some(allowed_filtering) = allowed_filtering {
2660                    if allowed_filtering != sampler.filtering {
2661                        return Err(Error::WrongSamplerFiltering {
2662                            binding,
2663                            layout_flt: allowed_filtering,
2664                            sampler_flt: sampler.filtering,
2665                        });
2666                    }
2667                }
2668                if allowed_comparison != sampler.comparison {
2669                    return Err(Error::WrongSamplerComparison {
2670                        binding,
2671                        layout_cmp: allowed_comparison,
2672                        sampler_cmp: sampler.comparison,
2673                    });
2674                }
2675            }
2676            _ => {
2677                return Err(Error::WrongBindingType {
2678                    binding,
2679                    actual: decl.ty,
2680                    expected: "Sampler",
2681                })
2682            }
2683        }
2684
2685        Ok(sampler.raw())
2686    }
2687
2688    fn create_texture_binding<'a>(
2689        &self,
2690        binding: u32,
2691        decl: &wgt::BindGroupLayoutEntry,
2692        view: &'a Arc<TextureView>,
2693        used: &mut BindGroupStates,
2694        used_texture_ranges: &mut Vec<TextureInitTrackerAction>,
2695        snatch_guard: &'a SnatchGuard<'a>,
2696    ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, binding_model::CreateBindGroupError>
2697    {
2698        view.same_device(self)?;
2699
2700        let internal_use = self.texture_use_parameters(
2701            binding,
2702            decl,
2703            view,
2704            "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
2705        )?;
2706
2707        used.views.insert_single(view.clone(), internal_use);
2708
2709        let texture = &view.parent;
2710
2711        used_texture_ranges.push(TextureInitTrackerAction {
2712            texture: texture.clone(),
2713            range: TextureInitRange {
2714                mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
2715                layer_range: view
2716                    .desc
2717                    .range
2718                    .layer_range(texture.desc.array_layer_count()),
2719            },
2720            kind: MemoryInitKind::NeedsInitializedMemory,
2721        });
2722
2723        Ok(hal::TextureBinding {
2724            view: view.try_raw(snatch_guard)?,
2725            usage: internal_use,
2726        })
2727    }
2728
2729    fn create_tlas_binding<'a>(
2730        self: &Arc<Self>,
2731        used: &mut BindGroupStates,
2732        binding: u32,
2733        decl: &wgt::BindGroupLayoutEntry,
2734        tlas: &'a Arc<Tlas>,
2735        snatch_guard: &'a SnatchGuard<'a>,
2736    ) -> Result<&'a dyn hal::DynAccelerationStructure, binding_model::CreateBindGroupError> {
2737        use crate::binding_model::CreateBindGroupError as Error;
2738
2739        used.acceleration_structures.insert_single(tlas.clone());
2740
2741        tlas.same_device(self)?;
2742
2743        match decl.ty {
2744            wgt::BindingType::AccelerationStructure { vertex_return } => {
2745                if vertex_return
2746                    && !tlas.flags.contains(
2747                        wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
2748                    )
2749                {
2750                    return Err(Error::MissingTLASVertexReturn { binding });
2751                }
2752            }
2753            _ => {
2754                return Err(Error::WrongBindingType {
2755                    binding,
2756                    actual: decl.ty,
2757                    expected: "Tlas",
2758                });
2759            }
2760        }
2761
2762        Ok(tlas.try_raw(snatch_guard)?)
2763    }
2764
2765    fn create_external_texture_binding<'a>(
2766        &'a self,
2767        binding: u32,
2768        decl: &wgt::BindGroupLayoutEntry,
2769        external_texture: &'a Arc<ExternalTexture>,
2770        used: &mut BindGroupStates,
2771        snatch_guard: &'a SnatchGuard,
2772    ) -> Result<
2773        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
2774        binding_model::CreateBindGroupError,
2775    > {
2776        use crate::binding_model::CreateBindGroupError as Error;
2777
2778        external_texture.same_device(self)?;
2779
2780        used.external_textures
2781            .insert_single(external_texture.clone());
2782
2783        match decl.ty {
2784            wgt::BindingType::ExternalTexture => {}
2785            _ => {
2786                return Err(Error::WrongBindingType {
2787                    binding,
2788                    actual: decl.ty,
2789                    expected: "ExternalTexture",
2790                });
2791            }
2792        }
2793
2794        let planes = (0..3)
2795            .map(|i| {
2796                // We always need 3 bindings. If we have fewer than 3 planes
2797                // just bind plane 0 multiple times. The shader will only
2798                // sample from valid planes anyway.
2799                let plane = external_texture
2800                    .planes
2801                    .get(i)
2802                    .unwrap_or(&external_texture.planes[0]);
2803                let internal_use = wgt::TextureUses::RESOURCE;
2804                used.views.insert_single(plane.clone(), internal_use);
2805                let view = plane.try_raw(snatch_guard)?;
2806                Ok(hal::TextureBinding {
2807                    view,
2808                    usage: internal_use,
2809                })
2810            })
2811            // We can remove this intermediate Vec by using
2812            // array::try_from_fn() above, once it stabilizes.
2813            .collect::<Result<Vec<_>, Error>>()?;
2814        let planes = planes.try_into().unwrap();
2815
2816        used.buffers
2817            .insert_single(external_texture.params.clone(), wgt::BufferUses::UNIFORM);
2818        let params = external_texture.params.binding(0, None, snatch_guard)?.0;
2819
2820        Ok(hal::ExternalTextureBinding { planes, params })
2821    }
2822
2823    fn create_external_texture_binding_from_view<'a>(
2824        &'a self,
2825        binding: u32,
2826        decl: &wgt::BindGroupLayoutEntry,
2827        view: &'a Arc<TextureView>,
2828        used: &mut BindGroupStates,
2829        snatch_guard: &'a SnatchGuard,
2830    ) -> Result<
2831        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
2832        binding_model::CreateBindGroupError,
2833    > {
2834        use crate::binding_model::CreateBindGroupError as Error;
2835
2836        view.same_device(self)?;
2837
2838        let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
2839        used.views.insert_single(view.clone(), internal_use);
2840
2841        match decl.ty {
2842            wgt::BindingType::ExternalTexture => {}
2843            _ => {
2844                return Err(Error::WrongBindingType {
2845                    binding,
2846                    actual: decl.ty,
2847                    expected: "ExternalTexture",
2848                });
2849            }
2850        }
2851
2852        // We need 3 bindings, so just repeat the same texture view 3 times.
2853        let planes = [
2854            hal::TextureBinding {
2855                view: view.try_raw(snatch_guard)?,
2856                usage: internal_use,
2857            },
2858            hal::TextureBinding {
2859                view: view.try_raw(snatch_guard)?,
2860                usage: internal_use,
2861            },
2862            hal::TextureBinding {
2863                view: view.try_raw(snatch_guard)?,
2864                usage: internal_use,
2865            },
2866        ];
2867        let params = hal::BufferBinding::new_unchecked(
2868            self.default_external_texture_params_buffer.as_ref(),
2869            0,
2870            None,
2871        );
2872
2873        Ok(hal::ExternalTextureBinding { planes, params })
2874    }
2875
2876    // This function expects the provided bind group layout to be resolved
2877    // (not passing a duplicate) beforehand.
2878    pub(crate) fn create_bind_group(
2879        self: &Arc<Self>,
2880        desc: binding_model::ResolvedBindGroupDescriptor,
2881    ) -> Result<Arc<BindGroup>, binding_model::CreateBindGroupError> {
2882        use crate::binding_model::{CreateBindGroupError as Error, ResolvedBindingResource as Br};
2883
2884        let layout = desc.layout;
2885
2886        self.check_is_valid()?;
2887        layout.same_device(self)?;
2888
2889        {
2890            // Check that the number of entries in the descriptor matches
2891            // the number of entries in the layout.
2892            let actual = desc.entries.len();
2893            let expected = layout.entries.len();
2894            if actual != expected {
2895                return Err(Error::BindingsNumMismatch { expected, actual });
2896            }
2897        }
2898
2899        // TODO: arrayvec/smallvec, or re-use allocations
2900        // Record binding info for dynamic offset validation
2901        let mut dynamic_binding_info = Vec::new();
2902        // Map of binding -> shader reflected size
2903        //Note: we can't collect into a vector right away because
2904        // it needs to be in BGL iteration order, not BG entry order.
2905        let mut late_buffer_binding_sizes = FastHashMap::default();
2906        // fill out the descriptors
2907        let mut used = BindGroupStates::new();
2908
2909        let mut used_buffer_ranges = Vec::new();
2910        let mut used_texture_ranges = Vec::new();
2911        let mut hal_entries = Vec::with_capacity(desc.entries.len());
2912        let mut hal_buffers = Vec::new();
2913        let mut hal_samplers = Vec::new();
2914        let mut hal_textures = Vec::new();
2915        let mut hal_tlas_s = Vec::new();
2916        let mut hal_external_textures = Vec::new();
2917        let snatch_guard = self.snatchable_lock.read();
2918        for entry in desc.entries.iter() {
2919            let binding = entry.binding;
2920            // Find the corresponding declaration in the layout
2921            let decl = layout
2922                .entries
2923                .get(binding)
2924                .ok_or(Error::MissingBindingDeclaration(binding))?;
2925            let (res_index, count) = match entry.resource {
2926                Br::Buffer(ref bb) => {
2927                    let bb = self.create_buffer_binding(
2928                        bb,
2929                        binding,
2930                        decl,
2931                        &mut used_buffer_ranges,
2932                        &mut dynamic_binding_info,
2933                        &mut late_buffer_binding_sizes,
2934                        &mut used,
2935                        &snatch_guard,
2936                    )?;
2937
2938                    let res_index = hal_buffers.len();
2939                    hal_buffers.push(bb);
2940                    (res_index, 1)
2941                }
2942                Br::BufferArray(ref bindings_array) => {
2943                    let num_bindings = bindings_array.len();
2944                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
2945
2946                    let res_index = hal_buffers.len();
2947                    for bb in bindings_array.iter() {
2948                        let bb = self.create_buffer_binding(
2949                            bb,
2950                            binding,
2951                            decl,
2952                            &mut used_buffer_ranges,
2953                            &mut dynamic_binding_info,
2954                            &mut late_buffer_binding_sizes,
2955                            &mut used,
2956                            &snatch_guard,
2957                        )?;
2958                        hal_buffers.push(bb);
2959                    }
2960                    (res_index, num_bindings)
2961                }
2962                Br::Sampler(ref sampler) => {
2963                    let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
2964
2965                    let res_index = hal_samplers.len();
2966                    hal_samplers.push(sampler);
2967                    (res_index, 1)
2968                }
2969                Br::SamplerArray(ref samplers) => {
2970                    let num_bindings = samplers.len();
2971                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
2972
2973                    let res_index = hal_samplers.len();
2974                    for sampler in samplers.iter() {
2975                        let sampler =
2976                            self.create_sampler_binding(&mut used, binding, decl, sampler)?;
2977
2978                        hal_samplers.push(sampler);
2979                    }
2980
2981                    (res_index, num_bindings)
2982                }
2983                Br::TextureView(ref view) => match decl.ty {
2984                    wgt::BindingType::ExternalTexture => {
2985                        let et = self.create_external_texture_binding_from_view(
2986                            binding,
2987                            decl,
2988                            view,
2989                            &mut used,
2990                            &snatch_guard,
2991                        )?;
2992                        let res_index = hal_external_textures.len();
2993                        hal_external_textures.push(et);
2994                        (res_index, 1)
2995                    }
2996                    _ => {
2997                        let tb = self.create_texture_binding(
2998                            binding,
2999                            decl,
3000                            view,
3001                            &mut used,
3002                            &mut used_texture_ranges,
3003                            &snatch_guard,
3004                        )?;
3005                        let res_index = hal_textures.len();
3006                        hal_textures.push(tb);
3007                        (res_index, 1)
3008                    }
3009                },
3010                Br::TextureViewArray(ref views) => {
3011                    let num_bindings = views.len();
3012                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3013
3014                    let res_index = hal_textures.len();
3015                    for view in views.iter() {
3016                        let tb = self.create_texture_binding(
3017                            binding,
3018                            decl,
3019                            view,
3020                            &mut used,
3021                            &mut used_texture_ranges,
3022                            &snatch_guard,
3023                        )?;
3024
3025                        hal_textures.push(tb);
3026                    }
3027
3028                    (res_index, num_bindings)
3029                }
3030                Br::AccelerationStructure(ref tlas) => {
3031                    let tlas =
3032                        self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
3033                    let res_index = hal_tlas_s.len();
3034                    hal_tlas_s.push(tlas);
3035                    (res_index, 1)
3036                }
3037                Br::ExternalTexture(ref et) => {
3038                    let et = self.create_external_texture_binding(
3039                        binding,
3040                        decl,
3041                        et,
3042                        &mut used,
3043                        &snatch_guard,
3044                    )?;
3045                    let res_index = hal_external_textures.len();
3046                    hal_external_textures.push(et);
3047                    (res_index, 1)
3048                }
3049            };
3050
3051            hal_entries.push(hal::BindGroupEntry {
3052                binding,
3053                resource_index: res_index as u32,
3054                count: count as u32,
3055            });
3056        }
3057
3058        used.optimize();
3059
3060        hal_entries.sort_by_key(|entry| entry.binding);
3061        for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
3062            if a.binding == b.binding {
3063                return Err(Error::DuplicateBinding(a.binding));
3064            }
3065        }
3066        let hal_desc = hal::BindGroupDescriptor {
3067            label: desc.label.to_hal(self.instance_flags),
3068            layout: layout.raw(),
3069            entries: &hal_entries,
3070            buffers: &hal_buffers,
3071            samplers: &hal_samplers,
3072            textures: &hal_textures,
3073            acceleration_structures: &hal_tlas_s,
3074            external_textures: &hal_external_textures,
3075        };
3076        let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
3077            .map_err(|e| self.handle_hal_error(e))?;
3078
3079        // collect in the order of BGL iteration
3080        let late_buffer_binding_sizes = layout
3081            .entries
3082            .indices()
3083            .flat_map(|binding| late_buffer_binding_sizes.get(&binding).cloned())
3084            .collect();
3085
3086        let bind_group = BindGroup {
3087            raw: Snatchable::new(raw),
3088            device: self.clone(),
3089            layout,
3090            label: desc.label.to_string(),
3091            tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
3092            used,
3093            used_buffer_ranges,
3094            used_texture_ranges,
3095            dynamic_binding_info,
3096            late_buffer_binding_sizes,
3097        };
3098
3099        let bind_group = Arc::new(bind_group);
3100
3101        let weak_ref = Arc::downgrade(&bind_group);
3102        for range in &bind_group.used_texture_ranges {
3103            let mut bind_groups = range.texture.bind_groups.lock();
3104            bind_groups.push(weak_ref.clone());
3105        }
3106        for range in &bind_group.used_buffer_ranges {
3107            let mut bind_groups = range.buffer.bind_groups.lock();
3108            bind_groups.push(weak_ref.clone());
3109        }
3110
3111        Ok(bind_group)
3112    }
3113
3114    fn check_array_binding(
3115        features: wgt::Features,
3116        count: Option<NonZeroU32>,
3117        num_bindings: usize,
3118    ) -> Result<(), binding_model::CreateBindGroupError> {
3119        use super::binding_model::CreateBindGroupError as Error;
3120
3121        if let Some(count) = count {
3122            let count = count.get() as usize;
3123            if count < num_bindings {
3124                return Err(Error::BindingArrayPartialLengthMismatch {
3125                    actual: num_bindings,
3126                    expected: count,
3127                });
3128            }
3129            if count != num_bindings
3130                && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
3131            {
3132                return Err(Error::BindingArrayLengthMismatch {
3133                    actual: num_bindings,
3134                    expected: count,
3135                });
3136            }
3137            if num_bindings == 0 {
3138                return Err(Error::BindingArrayZeroLength);
3139            }
3140        } else {
3141            return Err(Error::SingleBindingExpected);
3142        };
3143
3144        Ok(())
3145    }
3146
3147    fn texture_use_parameters(
3148        &self,
3149        binding: u32,
3150        decl: &wgt::BindGroupLayoutEntry,
3151        view: &TextureView,
3152        expected: &'static str,
3153    ) -> Result<wgt::TextureUses, binding_model::CreateBindGroupError> {
3154        use crate::binding_model::CreateBindGroupError as Error;
3155        if view
3156            .desc
3157            .aspects()
3158            .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
3159        {
3160            return Err(Error::DepthStencilAspect);
3161        }
3162        match decl.ty {
3163            wgt::BindingType::Texture {
3164                sample_type,
3165                view_dimension,
3166                multisampled,
3167            } => {
3168                use wgt::TextureSampleType as Tst;
3169                if multisampled != (view.samples != 1) {
3170                    return Err(Error::InvalidTextureMultisample {
3171                        binding,
3172                        layout_multisampled: multisampled,
3173                        view_samples: view.samples,
3174                    });
3175                }
3176                let compat_sample_type = view
3177                    .desc
3178                    .format
3179                    .sample_type(Some(view.desc.range.aspect), Some(self.features))
3180                    .unwrap();
3181                match (sample_type, compat_sample_type) {
3182                    (Tst::Uint, Tst::Uint) |
3183                        (Tst::Sint, Tst::Sint) |
3184                        (Tst::Depth, Tst::Depth) |
3185                        // if we expect non-filterable, accept anything float
3186                        (Tst::Float { filterable: false }, Tst::Float { .. }) |
3187                        // if we expect filterable, require it
3188                        (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
3189                        // if we expect non-filterable, also accept depth
3190                        (Tst::Float { filterable: false }, Tst::Depth) => {}
3191                    // if we expect filterable, also accept Float that is defined as
3192                    // unfilterable if filterable feature is explicitly enabled (only hit
3193                    // if wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES is
3194                    // enabled)
3195                    (Tst::Float { filterable: true }, Tst::Float { .. }) if view.format_features.flags.contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
3196                    _ => {
3197                        return Err(Error::InvalidTextureSampleType {
3198                            binding,
3199                            layout_sample_type: sample_type,
3200                            view_format: view.desc.format,
3201                            view_sample_type: compat_sample_type,
3202                        })
3203                    }
3204                }
3205                if view_dimension != view.desc.dimension {
3206                    return Err(Error::InvalidTextureDimension {
3207                        binding,
3208                        layout_dimension: view_dimension,
3209                        view_dimension: view.desc.dimension,
3210                    });
3211                }
3212                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3213                Ok(wgt::TextureUses::RESOURCE)
3214            }
3215            wgt::BindingType::StorageTexture {
3216                access,
3217                format,
3218                view_dimension,
3219            } => {
3220                if format != view.desc.format {
3221                    return Err(Error::InvalidStorageTextureFormat {
3222                        binding,
3223                        layout_format: format,
3224                        view_format: view.desc.format,
3225                    });
3226                }
3227                if view_dimension != view.desc.dimension {
3228                    return Err(Error::InvalidTextureDimension {
3229                        binding,
3230                        layout_dimension: view_dimension,
3231                        view_dimension: view.desc.dimension,
3232                    });
3233                }
3234
3235                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3236                if mip_level_count != 1 {
3237                    return Err(Error::InvalidStorageTextureMipLevelCount {
3238                        binding,
3239                        mip_level_count,
3240                    });
3241                }
3242
3243                let internal_use = match access {
3244                    wgt::StorageTextureAccess::WriteOnly => {
3245                        if !view
3246                            .format_features
3247                            .flags
3248                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY)
3249                        {
3250                            return Err(Error::StorageWriteNotSupported(view.desc.format));
3251                        }
3252                        wgt::TextureUses::STORAGE_WRITE_ONLY
3253                    }
3254                    wgt::StorageTextureAccess::ReadOnly => {
3255                        if !view
3256                            .format_features
3257                            .flags
3258                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY)
3259                        {
3260                            return Err(Error::StorageReadNotSupported(view.desc.format));
3261                        }
3262                        wgt::TextureUses::STORAGE_READ_ONLY
3263                    }
3264                    wgt::StorageTextureAccess::ReadWrite => {
3265                        if !view
3266                            .format_features
3267                            .flags
3268                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE)
3269                        {
3270                            return Err(Error::StorageReadWriteNotSupported(view.desc.format));
3271                        }
3272
3273                        wgt::TextureUses::STORAGE_READ_WRITE
3274                    }
3275                    wgt::StorageTextureAccess::Atomic => {
3276                        if !view
3277                            .format_features
3278                            .flags
3279                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_ATOMIC)
3280                        {
3281                            return Err(Error::StorageAtomicNotSupported(view.desc.format));
3282                        }
3283
3284                        wgt::TextureUses::STORAGE_ATOMIC
3285                    }
3286                };
3287                view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
3288                Ok(internal_use)
3289            }
3290            wgt::BindingType::ExternalTexture => {
3291                if view.desc.dimension != TextureViewDimension::D2 {
3292                    return Err(Error::InvalidTextureDimension {
3293                        binding,
3294                        layout_dimension: TextureViewDimension::D2,
3295                        view_dimension: view.desc.dimension,
3296                    });
3297                }
3298                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3299                if mip_level_count != 1 {
3300                    return Err(Error::InvalidExternalTextureMipLevelCount {
3301                        binding,
3302                        mip_level_count,
3303                    });
3304                }
3305                if view.desc.format != TextureFormat::Rgba8Unorm
3306                    && view.desc.format != TextureFormat::Bgra8Unorm
3307                    && view.desc.format != TextureFormat::Rgba16Float
3308                {
3309                    return Err(Error::InvalidExternalTextureFormat {
3310                        binding,
3311                        format: view.desc.format,
3312                    });
3313                }
3314                if view.samples != 1 {
3315                    return Err(Error::InvalidTextureMultisample {
3316                        binding,
3317                        layout_multisampled: false,
3318                        view_samples: view.samples,
3319                    });
3320                }
3321
3322                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3323                Ok(wgt::TextureUses::RESOURCE)
3324            }
3325            _ => Err(Error::WrongBindingType {
3326                binding,
3327                actual: decl.ty,
3328                expected,
3329            }),
3330        }
3331    }
3332
3333    pub(crate) fn create_pipeline_layout(
3334        self: &Arc<Self>,
3335        desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3336    ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
3337        use crate::binding_model::CreatePipelineLayoutError as Error;
3338
3339        self.check_is_valid()?;
3340
3341        let bind_group_layouts_count = desc.bind_group_layouts.len();
3342        let device_max_bind_groups = self.limits.max_bind_groups as usize;
3343        if bind_group_layouts_count > device_max_bind_groups {
3344            return Err(Error::TooManyGroups {
3345                actual: bind_group_layouts_count,
3346                max: device_max_bind_groups,
3347            });
3348        }
3349
3350        if !desc.push_constant_ranges.is_empty() {
3351            self.require_features(wgt::Features::PUSH_CONSTANTS)?;
3352        }
3353
3354        let mut used_stages = wgt::ShaderStages::empty();
3355        for (index, pc) in desc.push_constant_ranges.iter().enumerate() {
3356            if pc.stages.intersects(used_stages) {
3357                return Err(Error::MoreThanOnePushConstantRangePerStage {
3358                    index,
3359                    provided: pc.stages,
3360                    intersected: pc.stages & used_stages,
3361                });
3362            }
3363            used_stages |= pc.stages;
3364
3365            let device_max_pc_size = self.limits.max_push_constant_size;
3366            if device_max_pc_size < pc.range.end {
3367                return Err(Error::PushConstantRangeTooLarge {
3368                    index,
3369                    range: pc.range.clone(),
3370                    max: device_max_pc_size,
3371                });
3372            }
3373
3374            if pc.range.start % wgt::PUSH_CONSTANT_ALIGNMENT != 0 {
3375                return Err(Error::MisalignedPushConstantRange {
3376                    index,
3377                    bound: pc.range.start,
3378                });
3379            }
3380            if pc.range.end % wgt::PUSH_CONSTANT_ALIGNMENT != 0 {
3381                return Err(Error::MisalignedPushConstantRange {
3382                    index,
3383                    bound: pc.range.end,
3384                });
3385            }
3386        }
3387
3388        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3389
3390        for bgl in desc.bind_group_layouts.iter() {
3391            bgl.same_device(self)?;
3392            count_validator.merge(&bgl.binding_count_validator);
3393        }
3394
3395        count_validator
3396            .validate(&self.limits)
3397            .map_err(Error::TooManyBindings)?;
3398
3399        let bind_group_layouts = desc
3400            .bind_group_layouts
3401            .iter()
3402            .cloned()
3403            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3404
3405        let raw_bind_group_layouts = desc
3406            .bind_group_layouts
3407            .iter()
3408            .map(|bgl| bgl.raw())
3409            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3410
3411        let additional_flags = if self.indirect_validation.is_some() {
3412            hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
3413        } else {
3414            hal::PipelineLayoutFlags::empty()
3415        };
3416
3417        let hal_desc = hal::PipelineLayoutDescriptor {
3418            label: desc.label.to_hal(self.instance_flags),
3419            flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
3420                | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
3421                | additional_flags,
3422            bind_group_layouts: &raw_bind_group_layouts,
3423            push_constant_ranges: desc.push_constant_ranges.as_ref(),
3424        };
3425
3426        let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
3427            .map_err(|e| self.handle_hal_error(e))?;
3428
3429        drop(raw_bind_group_layouts);
3430
3431        let layout = binding_model::PipelineLayout {
3432            raw: ManuallyDrop::new(raw),
3433            device: self.clone(),
3434            label: desc.label.to_string(),
3435            bind_group_layouts,
3436            push_constant_ranges: desc.push_constant_ranges.iter().cloned().collect(),
3437        };
3438
3439        let layout = Arc::new(layout);
3440
3441        Ok(layout)
3442    }
3443
3444    pub(crate) fn derive_pipeline_layout(
3445        self: &Arc<Self>,
3446        mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
3447    ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
3448        while derived_group_layouts
3449            .last()
3450            .is_some_and(|map| map.is_empty())
3451        {
3452            derived_group_layouts.pop();
3453        }
3454
3455        let mut unique_bind_group_layouts = FastHashMap::default();
3456
3457        let bind_group_layouts = derived_group_layouts
3458            .into_iter()
3459            .map(|mut bgl_entry_map| {
3460                bgl_entry_map.sort();
3461                match unique_bind_group_layouts.entry(bgl_entry_map) {
3462                    hashbrown::hash_map::Entry::Occupied(v) => Ok(Arc::clone(v.get())),
3463                    hashbrown::hash_map::Entry::Vacant(e) => {
3464                        match self.create_bind_group_layout(
3465                            &None,
3466                            e.key().clone(),
3467                            bgl::Origin::Derived,
3468                        ) {
3469                            Ok(bgl) => {
3470                                e.insert(bgl.clone());
3471                                Ok(bgl)
3472                            }
3473                            Err(e) => Err(e),
3474                        }
3475                    }
3476                }
3477            })
3478            .collect::<Result<Vec<_>, _>>()?;
3479
3480        let layout_desc = binding_model::ResolvedPipelineLayoutDescriptor {
3481            label: None,
3482            bind_group_layouts: Cow::Owned(bind_group_layouts),
3483            push_constant_ranges: Cow::Borrowed(&[]), //TODO?
3484        };
3485
3486        let layout = self.create_pipeline_layout(&layout_desc)?;
3487        Ok(layout)
3488    }
3489
3490    pub(crate) fn create_compute_pipeline(
3491        self: &Arc<Self>,
3492        desc: pipeline::ResolvedComputePipelineDescriptor,
3493    ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreateComputePipelineError> {
3494        self.check_is_valid()?;
3495
3496        self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
3497
3498        let shader_module = desc.stage.module;
3499
3500        shader_module.same_device(self)?;
3501
3502        let is_auto_layout = desc.layout.is_none();
3503
3504        // Get the pipeline layout from the desc if it is provided.
3505        let pipeline_layout = match desc.layout {
3506            Some(pipeline_layout) => {
3507                pipeline_layout.same_device(self)?;
3508                Some(pipeline_layout)
3509            }
3510            None => None,
3511        };
3512
3513        let mut binding_layout_source = match pipeline_layout {
3514            Some(ref pipeline_layout) => {
3515                validation::BindingLayoutSource::Provided(pipeline_layout.get_binding_maps())
3516            }
3517            None => validation::BindingLayoutSource::new_derived(&self.limits),
3518        };
3519        let mut shader_binding_sizes = FastHashMap::default();
3520        let io = validation::StageIo::default();
3521
3522        let final_entry_point_name;
3523
3524        {
3525            let stage = wgt::ShaderStages::COMPUTE;
3526
3527            final_entry_point_name = shader_module.finalize_entry_point_name(
3528                stage,
3529                desc.stage.entry_point.as_ref().map(|ep| ep.as_ref()),
3530            )?;
3531
3532            if let Some(ref interface) = shader_module.interface {
3533                let _ = interface.check_stage(
3534                    &mut binding_layout_source,
3535                    &mut shader_binding_sizes,
3536                    &final_entry_point_name,
3537                    stage,
3538                    io,
3539                    None,
3540                )?;
3541            }
3542        }
3543
3544        let pipeline_layout = match binding_layout_source {
3545            validation::BindingLayoutSource::Provided(_) => {
3546                drop(binding_layout_source);
3547                pipeline_layout.unwrap()
3548            }
3549            validation::BindingLayoutSource::Derived(entries) => {
3550                self.derive_pipeline_layout(entries)?
3551            }
3552        };
3553
3554        let late_sized_buffer_groups =
3555            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
3556
3557        let cache = match desc.cache {
3558            Some(cache) => {
3559                cache.same_device(self)?;
3560                Some(cache)
3561            }
3562            None => None,
3563        };
3564
3565        let pipeline_desc = hal::ComputePipelineDescriptor {
3566            label: desc.label.to_hal(self.instance_flags),
3567            layout: pipeline_layout.raw(),
3568            stage: hal::ProgrammableStage {
3569                module: shader_module.raw(),
3570                entry_point: final_entry_point_name.as_ref(),
3571                constants: &desc.stage.constants,
3572                zero_initialize_workgroup_memory: desc.stage.zero_initialize_workgroup_memory,
3573            },
3574            cache: cache.as_ref().map(|it| it.raw()),
3575        };
3576
3577        let raw =
3578            unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }.map_err(
3579                |err| match err {
3580                    hal::PipelineError::Device(error) => {
3581                        pipeline::CreateComputePipelineError::Device(self.handle_hal_error(error))
3582                    }
3583                    hal::PipelineError::Linkage(_stages, msg) => {
3584                        pipeline::CreateComputePipelineError::Internal(msg)
3585                    }
3586                    hal::PipelineError::EntryPoint(_stage) => {
3587                        pipeline::CreateComputePipelineError::Internal(
3588                            ENTRYPOINT_FAILURE_ERROR.to_string(),
3589                        )
3590                    }
3591                    hal::PipelineError::PipelineConstants(_stages, msg) => {
3592                        pipeline::CreateComputePipelineError::PipelineConstants(msg)
3593                    }
3594                },
3595            )?;
3596
3597        let pipeline = pipeline::ComputePipeline {
3598            raw: ManuallyDrop::new(raw),
3599            layout: pipeline_layout,
3600            device: self.clone(),
3601            _shader_module: shader_module,
3602            late_sized_buffer_groups,
3603            label: desc.label.to_string(),
3604            tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
3605        };
3606
3607        let pipeline = Arc::new(pipeline);
3608
3609        if is_auto_layout {
3610            for bgl in pipeline.layout.bind_group_layouts.iter() {
3611                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the result.
3612                let _ = bgl
3613                    .exclusive_pipeline
3614                    .set(binding_model::ExclusivePipeline::Compute(Arc::downgrade(
3615                        &pipeline,
3616                    )));
3617            }
3618        }
3619
3620        Ok(pipeline)
3621    }
3622
3623    pub(crate) fn create_render_pipeline(
3624        self: &Arc<Self>,
3625        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
3626    ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreateRenderPipelineError> {
3627        use wgt::TextureFormatFeatureFlags as Tfff;
3628
3629        self.check_is_valid()?;
3630
3631        let mut shader_binding_sizes = FastHashMap::default();
3632
3633        let num_attachments = desc.fragment.as_ref().map(|f| f.targets.len()).unwrap_or(0);
3634        let max_attachments = self.limits.max_color_attachments as usize;
3635        if num_attachments > max_attachments {
3636            return Err(pipeline::CreateRenderPipelineError::ColorAttachment(
3637                command::ColorAttachmentError::TooMany {
3638                    given: num_attachments,
3639                    limit: max_attachments,
3640                },
3641            ));
3642        }
3643
3644        let color_targets = desc
3645            .fragment
3646            .as_ref()
3647            .map_or(&[][..], |fragment| &fragment.targets);
3648        let depth_stencil_state = desc.depth_stencil.as_ref();
3649
3650        {
3651            let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
3652                color_targets.iter().filter_map(|x| x.as_ref()).collect();
3653            if !cts.is_empty() && {
3654                let first = &cts[0];
3655                cts[1..]
3656                    .iter()
3657                    .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
3658            } {
3659                self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
3660            }
3661        }
3662
3663        let mut io = validation::StageIo::default();
3664        let mut validated_stages = wgt::ShaderStages::empty();
3665
3666        let mut vertex_steps;
3667        let mut vertex_buffers;
3668        let mut total_attributes;
3669        let mut shader_expects_dual_source_blending = false;
3670        let mut pipeline_expects_dual_source_blending = false;
3671        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
3672            vertex_steps = Vec::with_capacity(vertex.buffers.len());
3673            vertex_buffers = Vec::with_capacity(vertex.buffers.len());
3674            total_attributes = 0;
3675            shader_expects_dual_source_blending = false;
3676            pipeline_expects_dual_source_blending = false;
3677            for (i, vb_state) in vertex.buffers.iter().enumerate() {
3678                // https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpuvertexbufferlayout
3679
3680                if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
3681                    return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge {
3682                        index: i as u32,
3683                        given: vb_state.array_stride as u32,
3684                        limit: self.limits.max_vertex_buffer_array_stride,
3685                    });
3686                }
3687                if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
3688                    return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride {
3689                        index: i as u32,
3690                        stride: vb_state.array_stride,
3691                    });
3692                }
3693
3694                let max_stride = if vb_state.array_stride == 0 {
3695                    self.limits.max_vertex_buffer_array_stride as u64
3696                } else {
3697                    vb_state.array_stride
3698                };
3699                let mut last_stride = 0;
3700                for attribute in vb_state.attributes.iter() {
3701                    let attribute_stride = attribute.offset + attribute.format.size();
3702                    if attribute_stride > max_stride {
3703                        return Err(
3704                            pipeline::CreateRenderPipelineError::VertexAttributeStrideTooLarge {
3705                                location: attribute.shader_location,
3706                                given: attribute_stride as u32,
3707                                limit: max_stride as u32,
3708                            },
3709                        );
3710                    }
3711
3712                    let required_offset_alignment = attribute.format.size().min(4);
3713                    if attribute.offset % required_offset_alignment != 0 {
3714                        return Err(
3715                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
3716                                location: attribute.shader_location,
3717                                offset: attribute.offset,
3718                            },
3719                        );
3720                    }
3721
3722                    if attribute.shader_location >= self.limits.max_vertex_attributes {
3723                        return Err(
3724                            pipeline::CreateRenderPipelineError::VertexAttributeLocationTooLarge {
3725                                given: attribute.shader_location,
3726                                limit: self.limits.max_vertex_attributes,
3727                            },
3728                        );
3729                    }
3730
3731                    last_stride = last_stride.max(attribute_stride);
3732                }
3733                vertex_steps.push(pipeline::VertexStep {
3734                    stride: vb_state.array_stride,
3735                    last_stride,
3736                    mode: vb_state.step_mode,
3737                });
3738                if vb_state.attributes.is_empty() {
3739                    continue;
3740                }
3741                vertex_buffers.push(hal::VertexBufferLayout {
3742                    array_stride: vb_state.array_stride,
3743                    step_mode: vb_state.step_mode,
3744                    attributes: vb_state.attributes.as_ref(),
3745                });
3746
3747                for attribute in vb_state.attributes.iter() {
3748                    if attribute.offset >= 0x10000000 {
3749                        return Err(
3750                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
3751                                location: attribute.shader_location,
3752                                offset: attribute.offset,
3753                            },
3754                        );
3755                    }
3756
3757                    if let wgt::VertexFormat::Float64
3758                    | wgt::VertexFormat::Float64x2
3759                    | wgt::VertexFormat::Float64x3
3760                    | wgt::VertexFormat::Float64x4 = attribute.format
3761                    {
3762                        self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
3763                    }
3764
3765                    let previous = io.insert(
3766                        attribute.shader_location,
3767                        validation::InterfaceVar::vertex_attribute(attribute.format),
3768                    );
3769
3770                    if previous.is_some() {
3771                        return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash(
3772                            attribute.shader_location,
3773                        ));
3774                    }
3775                }
3776                total_attributes += vb_state.attributes.len();
3777            }
3778
3779            if vertex_buffers.len() > self.limits.max_vertex_buffers as usize {
3780                return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers {
3781                    given: vertex_buffers.len() as u32,
3782                    limit: self.limits.max_vertex_buffers,
3783                });
3784            }
3785            if total_attributes > self.limits.max_vertex_attributes as usize {
3786                return Err(
3787                    pipeline::CreateRenderPipelineError::TooManyVertexAttributes {
3788                        given: total_attributes as u32,
3789                        limit: self.limits.max_vertex_attributes,
3790                    },
3791                );
3792            }
3793        } else {
3794            vertex_steps = Vec::new();
3795            vertex_buffers = Vec::new();
3796        };
3797
3798        if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
3799            return Err(
3800                pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology {
3801                    strip_index_format: desc.primitive.strip_index_format,
3802                    topology: desc.primitive.topology,
3803                },
3804            );
3805        }
3806
3807        if desc.primitive.unclipped_depth {
3808            self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
3809        }
3810
3811        if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
3812            self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
3813        }
3814        if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
3815            self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
3816        }
3817
3818        if desc.primitive.conservative {
3819            self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
3820        }
3821
3822        if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
3823            return Err(
3824                pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode,
3825            );
3826        }
3827
3828        let mut target_specified = false;
3829
3830        for (i, cs) in color_targets.iter().enumerate() {
3831            if let Some(cs) = cs.as_ref() {
3832                target_specified = true;
3833                let error = 'error: {
3834                    if cs.write_mask.contains_unknown_bits() {
3835                        break 'error Some(pipeline::ColorStateError::InvalidWriteMask(
3836                            cs.write_mask,
3837                        ));
3838                    }
3839
3840                    let format_features = self.describe_format_features(cs.format)?;
3841                    if !format_features
3842                        .allowed_usages
3843                        .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
3844                    {
3845                        break 'error Some(pipeline::ColorStateError::FormatNotRenderable(
3846                            cs.format,
3847                        ));
3848                    }
3849                    let blendable = format_features.flags.contains(Tfff::BLENDABLE);
3850                    let filterable = format_features.flags.contains(Tfff::FILTERABLE);
3851                    let adapter_specific = self
3852                        .features
3853                        .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
3854                    // according to WebGPU specifications the texture needs to be
3855                    // [`TextureFormatFeatureFlags::FILTERABLE`] if blending is set - use
3856                    // [`Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES`] to elude
3857                    // this limitation
3858                    if cs.blend.is_some() && (!blendable || (!filterable && !adapter_specific)) {
3859                        break 'error Some(pipeline::ColorStateError::FormatNotBlendable(
3860                            cs.format,
3861                        ));
3862                    }
3863                    if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
3864                        break 'error Some(pipeline::ColorStateError::FormatNotColor(cs.format));
3865                    }
3866
3867                    if desc.multisample.count > 1
3868                        && !format_features
3869                            .flags
3870                            .sample_count_supported(desc.multisample.count)
3871                    {
3872                        break 'error Some(pipeline::ColorStateError::InvalidSampleCount(
3873                            desc.multisample.count,
3874                            cs.format,
3875                            cs.format
3876                                .guaranteed_format_features(self.features)
3877                                .flags
3878                                .supported_sample_counts(),
3879                            self.adapter
3880                                .get_texture_format_features(cs.format)
3881                                .flags
3882                                .supported_sample_counts(),
3883                        ));
3884                    }
3885
3886                    if let Some(blend_mode) = cs.blend {
3887                        for factor in [
3888                            blend_mode.color.src_factor,
3889                            blend_mode.color.dst_factor,
3890                            blend_mode.alpha.src_factor,
3891                            blend_mode.alpha.dst_factor,
3892                        ] {
3893                            if factor.ref_second_blend_source() {
3894                                self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
3895                                if i == 0 {
3896                                    pipeline_expects_dual_source_blending = true;
3897                                    break;
3898                                } else {
3899                                    return Err(pipeline::CreateRenderPipelineError
3900                                        ::BlendFactorOnUnsupportedTarget { factor, target: i as u32 });
3901                                }
3902                            }
3903                        }
3904                    }
3905
3906                    break 'error None;
3907                };
3908                if let Some(e) = error {
3909                    return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e));
3910                }
3911            }
3912        }
3913
3914        let limit = self.limits.max_color_attachment_bytes_per_sample;
3915        let formats = color_targets
3916            .iter()
3917            .map(|cs| cs.as_ref().map(|cs| cs.format));
3918        if let Err(total) = validate_color_attachment_bytes_per_sample(formats, limit) {
3919            return Err(pipeline::CreateRenderPipelineError::ColorAttachment(
3920                command::ColorAttachmentError::TooManyBytesPerSample { total, limit },
3921            ));
3922        }
3923
3924        if let Some(ds) = depth_stencil_state {
3925            target_specified = true;
3926            let error = 'error: {
3927                let format_features = self.describe_format_features(ds.format)?;
3928                if !format_features
3929                    .allowed_usages
3930                    .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
3931                {
3932                    break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
3933                        ds.format,
3934                    ));
3935                }
3936
3937                let aspect = hal::FormatAspects::from(ds.format);
3938                if ds.is_depth_enabled() && !aspect.contains(hal::FormatAspects::DEPTH) {
3939                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
3940                }
3941                if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
3942                    break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
3943                        ds.format,
3944                    ));
3945                }
3946                if desc.multisample.count > 1
3947                    && !format_features
3948                        .flags
3949                        .sample_count_supported(desc.multisample.count)
3950                {
3951                    break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
3952                        desc.multisample.count,
3953                        ds.format,
3954                        ds.format
3955                            .guaranteed_format_features(self.features)
3956                            .flags
3957                            .supported_sample_counts(),
3958                        self.adapter
3959                            .get_texture_format_features(ds.format)
3960                            .flags
3961                            .supported_sample_counts(),
3962                    ));
3963                }
3964
3965                break 'error None;
3966            };
3967            if let Some(e) = error {
3968                return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e));
3969            }
3970
3971            if ds.bias.clamp != 0.0 {
3972                self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
3973            }
3974        }
3975
3976        if !target_specified {
3977            return Err(pipeline::CreateRenderPipelineError::NoTargetSpecified);
3978        }
3979
3980        let is_auto_layout = desc.layout.is_none();
3981
3982        // Get the pipeline layout from the desc if it is provided.
3983        let pipeline_layout = match desc.layout {
3984            Some(pipeline_layout) => {
3985                pipeline_layout.same_device(self)?;
3986                Some(pipeline_layout)
3987            }
3988            None => None,
3989        };
3990
3991        let mut binding_layout_source = match pipeline_layout {
3992            Some(ref pipeline_layout) => {
3993                validation::BindingLayoutSource::Provided(pipeline_layout.get_binding_maps())
3994            }
3995            None => validation::BindingLayoutSource::new_derived(&self.limits),
3996        };
3997
3998        let samples = {
3999            let sc = desc.multisample.count;
4000            if sc == 0 || sc > 32 || !sc.is_power_of_two() {
4001                return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc));
4002            }
4003            sc
4004        };
4005
4006        let mut vertex_stage = None;
4007        let mut task_stage = None;
4008        let mut mesh_stage = None;
4009        let mut _vertex_entry_point_name = String::new();
4010        let mut _task_entry_point_name = String::new();
4011        let mut _mesh_entry_point_name = String::new();
4012        match desc.vertex {
4013            pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
4014                vertex_stage = {
4015                    let stage_desc = &vertex.stage;
4016                    let stage = wgt::ShaderStages::VERTEX;
4017
4018                    let vertex_shader_module = &stage_desc.module;
4019                    vertex_shader_module.same_device(self)?;
4020
4021                    let stage_err =
4022                        |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4023
4024                    _vertex_entry_point_name = vertex_shader_module
4025                        .finalize_entry_point_name(
4026                            stage,
4027                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4028                        )
4029                        .map_err(stage_err)?;
4030
4031                    if let Some(ref interface) = vertex_shader_module.interface {
4032                        io = interface
4033                            .check_stage(
4034                                &mut binding_layout_source,
4035                                &mut shader_binding_sizes,
4036                                &_vertex_entry_point_name,
4037                                stage,
4038                                io,
4039                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4040                            )
4041                            .map_err(stage_err)?;
4042                        validated_stages |= stage;
4043                    }
4044                    Some(hal::ProgrammableStage {
4045                        module: vertex_shader_module.raw(),
4046                        entry_point: &_vertex_entry_point_name,
4047                        constants: &stage_desc.constants,
4048                        zero_initialize_workgroup_memory: stage_desc
4049                            .zero_initialize_workgroup_memory,
4050                    })
4051                };
4052            }
4053            pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
4054                task_stage = if let Some(task) = task {
4055                    let stage_desc = &task.stage;
4056                    let stage = wgt::ShaderStages::TASK;
4057                    let task_shader_module = &stage_desc.module;
4058                    task_shader_module.same_device(self)?;
4059
4060                    let stage_err =
4061                        |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4062
4063                    _task_entry_point_name = task_shader_module
4064                        .finalize_entry_point_name(
4065                            stage,
4066                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4067                        )
4068                        .map_err(stage_err)?;
4069
4070                    if let Some(ref interface) = task_shader_module.interface {
4071                        io = interface
4072                            .check_stage(
4073                                &mut binding_layout_source,
4074                                &mut shader_binding_sizes,
4075                                &_task_entry_point_name,
4076                                stage,
4077                                io,
4078                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4079                            )
4080                            .map_err(stage_err)?;
4081                        validated_stages |= stage;
4082                    }
4083                    Some(hal::ProgrammableStage {
4084                        module: task_shader_module.raw(),
4085                        entry_point: &_task_entry_point_name,
4086                        constants: &stage_desc.constants,
4087                        zero_initialize_workgroup_memory: stage_desc
4088                            .zero_initialize_workgroup_memory,
4089                    })
4090                } else {
4091                    None
4092                };
4093                mesh_stage = {
4094                    let stage_desc = &mesh.stage;
4095                    let stage = wgt::ShaderStages::MESH;
4096                    let mesh_shader_module = &stage_desc.module;
4097                    mesh_shader_module.same_device(self)?;
4098
4099                    let stage_err =
4100                        |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4101
4102                    _mesh_entry_point_name = mesh_shader_module
4103                        .finalize_entry_point_name(
4104                            stage,
4105                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4106                        )
4107                        .map_err(stage_err)?;
4108
4109                    if let Some(ref interface) = mesh_shader_module.interface {
4110                        io = interface
4111                            .check_stage(
4112                                &mut binding_layout_source,
4113                                &mut shader_binding_sizes,
4114                                &_mesh_entry_point_name,
4115                                stage,
4116                                io,
4117                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4118                            )
4119                            .map_err(stage_err)?;
4120                        validated_stages |= stage;
4121                    }
4122                    Some(hal::ProgrammableStage {
4123                        module: mesh_shader_module.raw(),
4124                        entry_point: &_mesh_entry_point_name,
4125                        constants: &stage_desc.constants,
4126                        zero_initialize_workgroup_memory: stage_desc
4127                            .zero_initialize_workgroup_memory,
4128                    })
4129                };
4130            }
4131        }
4132
4133        let fragment_entry_point_name;
4134        let fragment_stage = match desc.fragment {
4135            Some(ref fragment_state) => {
4136                let stage = wgt::ShaderStages::FRAGMENT;
4137
4138                let shader_module = &fragment_state.stage.module;
4139                shader_module.same_device(self)?;
4140
4141                let stage_err = |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4142
4143                fragment_entry_point_name = shader_module
4144                    .finalize_entry_point_name(
4145                        stage,
4146                        fragment_state
4147                            .stage
4148                            .entry_point
4149                            .as_ref()
4150                            .map(|ep| ep.as_ref()),
4151                    )
4152                    .map_err(stage_err)?;
4153
4154                if validated_stages == wgt::ShaderStages::VERTEX {
4155                    if let Some(ref interface) = shader_module.interface {
4156                        io = interface
4157                            .check_stage(
4158                                &mut binding_layout_source,
4159                                &mut shader_binding_sizes,
4160                                &fragment_entry_point_name,
4161                                stage,
4162                                io,
4163                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4164                            )
4165                            .map_err(stage_err)?;
4166                        validated_stages |= stage;
4167                    }
4168                }
4169
4170                if let Some(ref interface) = shader_module.interface {
4171                    shader_expects_dual_source_blending = interface
4172                        .fragment_uses_dual_source_blending(&fragment_entry_point_name)
4173                        .map_err(|error| pipeline::CreateRenderPipelineError::Stage {
4174                            stage,
4175                            error,
4176                        })?;
4177                }
4178
4179                Some(hal::ProgrammableStage {
4180                    module: shader_module.raw(),
4181                    entry_point: &fragment_entry_point_name,
4182                    constants: &fragment_state.stage.constants,
4183                    zero_initialize_workgroup_memory: fragment_state
4184                        .stage
4185                        .zero_initialize_workgroup_memory,
4186                })
4187            }
4188            None => None,
4189        };
4190
4191        if !pipeline_expects_dual_source_blending && shader_expects_dual_source_blending {
4192            return Err(
4193                pipeline::CreateRenderPipelineError::ShaderExpectsPipelineToUseDualSourceBlending,
4194            );
4195        }
4196        if pipeline_expects_dual_source_blending && !shader_expects_dual_source_blending {
4197            return Err(
4198                pipeline::CreateRenderPipelineError::PipelineExpectsShaderToUseDualSourceBlending,
4199            );
4200        }
4201
4202        if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
4203            for (i, output) in io.iter() {
4204                match color_targets.get(*i as usize) {
4205                    Some(Some(state)) => {
4206                        validation::check_texture_format(state.format, &output.ty).map_err(
4207                            |pipeline| {
4208                                pipeline::CreateRenderPipelineError::ColorState(
4209                                    *i as u8,
4210                                    pipeline::ColorStateError::IncompatibleFormat {
4211                                        pipeline,
4212                                        shader: output.ty,
4213                                    },
4214                                )
4215                            },
4216                        )?;
4217                    }
4218                    _ => {
4219                        log::warn!(
4220                            "The fragment stage {:?} output @location({}) values are ignored",
4221                            fragment_stage
4222                                .as_ref()
4223                                .map_or("", |stage| stage.entry_point),
4224                            i
4225                        );
4226                    }
4227                }
4228            }
4229        }
4230        let last_stage = match desc.fragment {
4231            Some(_) => wgt::ShaderStages::FRAGMENT,
4232            None => wgt::ShaderStages::VERTEX,
4233        };
4234        if is_auto_layout && !validated_stages.contains(last_stage) {
4235            return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into());
4236        }
4237
4238        let pipeline_layout = match binding_layout_source {
4239            validation::BindingLayoutSource::Provided(_) => {
4240                drop(binding_layout_source);
4241                pipeline_layout.unwrap()
4242            }
4243            validation::BindingLayoutSource::Derived(entries) => {
4244                self.derive_pipeline_layout(entries)?
4245            }
4246        };
4247
4248        // Multiview is only supported if the feature is enabled
4249        if desc.multiview.is_some() {
4250            self.require_features(wgt::Features::MULTIVIEW)?;
4251        }
4252
4253        if !self
4254            .downlevel
4255            .flags
4256            .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
4257        {
4258            for (binding, size) in shader_binding_sizes.iter() {
4259                if size.get() % 16 != 0 {
4260                    return Err(pipeline::CreateRenderPipelineError::UnalignedShader {
4261                        binding: binding.binding,
4262                        group: binding.group,
4263                        size: size.get(),
4264                    });
4265                }
4266            }
4267        }
4268
4269        let late_sized_buffer_groups =
4270            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4271
4272        let cache = match desc.cache {
4273            Some(cache) => {
4274                cache.same_device(self)?;
4275                Some(cache)
4276            }
4277            None => None,
4278        };
4279
4280        let is_mesh = mesh_stage.is_some();
4281        let raw = {
4282            let pipeline_desc = hal::RenderPipelineDescriptor {
4283                label: desc.label.to_hal(self.instance_flags),
4284                layout: pipeline_layout.raw(),
4285                vertex_processor: match vertex_stage {
4286                    Some(vertex_stage) => hal::VertexProcessor::Standard {
4287                        vertex_buffers: &vertex_buffers,
4288                        vertex_stage,
4289                    },
4290                    None => hal::VertexProcessor::Mesh {
4291                        task_stage,
4292                        mesh_stage: mesh_stage.unwrap(),
4293                    },
4294                },
4295                primitive: desc.primitive,
4296                depth_stencil: desc.depth_stencil.clone(),
4297                multisample: desc.multisample,
4298                fragment_stage,
4299                color_targets,
4300                multiview: desc.multiview,
4301                cache: cache.as_ref().map(|it| it.raw()),
4302            };
4303            unsafe { self.raw().create_render_pipeline(&pipeline_desc) }.map_err(
4304                |err| match err {
4305                    hal::PipelineError::Device(error) => {
4306                        pipeline::CreateRenderPipelineError::Device(self.handle_hal_error(error))
4307                    }
4308                    hal::PipelineError::Linkage(stage, msg) => {
4309                        pipeline::CreateRenderPipelineError::Internal { stage, error: msg }
4310                    }
4311                    hal::PipelineError::EntryPoint(stage) => {
4312                        pipeline::CreateRenderPipelineError::Internal {
4313                            stage: hal::auxil::map_naga_stage(stage),
4314                            error: ENTRYPOINT_FAILURE_ERROR.to_string(),
4315                        }
4316                    }
4317                    hal::PipelineError::PipelineConstants(stage, error) => {
4318                        pipeline::CreateRenderPipelineError::PipelineConstants { stage, error }
4319                    }
4320                },
4321            )?
4322        };
4323
4324        let pass_context = RenderPassContext {
4325            attachments: AttachmentData {
4326                colors: color_targets
4327                    .iter()
4328                    .map(|state| state.as_ref().map(|s| s.format))
4329                    .collect(),
4330                resolves: ArrayVec::new(),
4331                depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
4332            },
4333            sample_count: samples,
4334            multiview: desc.multiview,
4335        };
4336
4337        let mut flags = pipeline::PipelineFlags::empty();
4338        for state in color_targets.iter().filter_map(|s| s.as_ref()) {
4339            if let Some(ref bs) = state.blend {
4340                if bs.color.uses_constant() | bs.alpha.uses_constant() {
4341                    flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
4342                }
4343            }
4344        }
4345        if let Some(ds) = depth_stencil_state.as_ref() {
4346            if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
4347                flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
4348            }
4349            if !ds.is_depth_read_only() {
4350                flags |= pipeline::PipelineFlags::WRITES_DEPTH;
4351            }
4352            if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
4353                flags |= pipeline::PipelineFlags::WRITES_STENCIL;
4354            }
4355        }
4356        let shader_modules = {
4357            let mut shader_modules = ArrayVec::new();
4358            match desc.vertex {
4359                pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
4360                    shader_modules.push(vertex.stage.module)
4361                }
4362                pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
4363                    if let Some(task) = task {
4364                        shader_modules.push(task.stage.module);
4365                    }
4366                    shader_modules.push(mesh.stage.module);
4367                }
4368            }
4369            shader_modules.extend(desc.fragment.map(|f| f.stage.module));
4370            shader_modules
4371        };
4372
4373        let pipeline = pipeline::RenderPipeline {
4374            raw: ManuallyDrop::new(raw),
4375            layout: pipeline_layout,
4376            device: self.clone(),
4377            pass_context,
4378            _shader_modules: shader_modules,
4379            flags,
4380            strip_index_format: desc.primitive.strip_index_format,
4381            vertex_steps,
4382            late_sized_buffer_groups,
4383            label: desc.label.to_string(),
4384            tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
4385            is_mesh,
4386        };
4387
4388        let pipeline = Arc::new(pipeline);
4389
4390        if is_auto_layout {
4391            for bgl in pipeline.layout.bind_group_layouts.iter() {
4392                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the result.
4393                let _ = bgl
4394                    .exclusive_pipeline
4395                    .set(binding_model::ExclusivePipeline::Render(Arc::downgrade(
4396                        &pipeline,
4397                    )));
4398            }
4399        }
4400
4401        Ok(pipeline)
4402    }
4403
4404    /// # Safety
4405    /// The `data` field on `desc` must have previously been returned from [`crate::global::Global::pipeline_cache_get_data`]
4406    pub unsafe fn create_pipeline_cache(
4407        self: &Arc<Self>,
4408        desc: &pipeline::PipelineCacheDescriptor,
4409    ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
4410        use crate::pipeline_cache;
4411
4412        self.check_is_valid()?;
4413
4414        self.require_features(wgt::Features::PIPELINE_CACHE)?;
4415        let data = if let Some((data, validation_key)) = desc
4416            .data
4417            .as_ref()
4418            .zip(self.raw().pipeline_cache_validation_key())
4419        {
4420            let data = pipeline_cache::validate_pipeline_cache(
4421                data,
4422                &self.adapter.raw.info,
4423                validation_key,
4424            );
4425            match data {
4426                Ok(data) => Some(data),
4427                Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
4428                // If the error was unavoidable and we are asked to fallback, do so
4429                Err(_) => None,
4430            }
4431        } else {
4432            None
4433        };
4434        let cache_desc = hal::PipelineCacheDescriptor {
4435            data,
4436            label: desc.label.to_hal(self.instance_flags),
4437        };
4438        let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
4439            Ok(raw) => raw,
4440            Err(e) => match e {
4441                hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
4442            },
4443        };
4444        let cache = pipeline::PipelineCache {
4445            device: self.clone(),
4446            label: desc.label.to_string(),
4447            // This would be none in the error condition, which we don't implement yet
4448            raw: ManuallyDrop::new(raw),
4449        };
4450
4451        let cache = Arc::new(cache);
4452
4453        Ok(cache)
4454    }
4455
4456    fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
4457        // Variant of adapter.get_texture_format_features that takes device features into account
4458        use wgt::TextureFormatFeatureFlags as tfsc;
4459        let mut format_features = self.adapter.get_texture_format_features(format);
4460        if (format == TextureFormat::R32Float
4461            || format == TextureFormat::Rg32Float
4462            || format == TextureFormat::Rgba32Float)
4463            && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
4464        {
4465            format_features.flags.set(tfsc::FILTERABLE, false);
4466        }
4467        format_features
4468    }
4469
4470    fn describe_format_features(
4471        &self,
4472        format: TextureFormat,
4473    ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
4474        self.require_features(format.required_features())?;
4475
4476        let using_device_features = self
4477            .features
4478            .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
4479        // If we're running downlevel, we need to manually ask the backend what
4480        // we can use as we can't trust WebGPU.
4481        let downlevel = !self
4482            .downlevel
4483            .flags
4484            .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
4485
4486        if using_device_features || downlevel {
4487            Ok(self.get_texture_format_features(format))
4488        } else {
4489            Ok(format.guaranteed_format_features(self.features))
4490        }
4491    }
4492
4493    #[cfg(feature = "replay")]
4494    pub(crate) fn wait_for_submit(
4495        &self,
4496        submission_index: crate::SubmissionIndex,
4497    ) -> Result<(), DeviceError> {
4498        let fence = self.fence.read();
4499        let last_done_index = unsafe { self.raw().get_fence_value(fence.as_ref()) }
4500            .map_err(|e| self.handle_hal_error(e))?;
4501        if last_done_index < submission_index {
4502            unsafe { self.raw().wait(fence.as_ref(), submission_index, !0) }
4503                .map_err(|e| self.handle_hal_error(e))?;
4504            drop(fence);
4505            if let Some(queue) = self.get_queue() {
4506                let closures = queue.lock_life().triage_submissions(submission_index);
4507                assert!(
4508                    closures.is_empty(),
4509                    "wait_for_submit is not expected to work with closures"
4510                );
4511            }
4512        }
4513        Ok(())
4514    }
4515
4516    pub(crate) fn create_query_set(
4517        self: &Arc<Self>,
4518        desc: &resource::QuerySetDescriptor,
4519    ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
4520        use resource::CreateQuerySetError as Error;
4521
4522        self.check_is_valid()?;
4523
4524        match desc.ty {
4525            wgt::QueryType::Occlusion => {}
4526            wgt::QueryType::Timestamp => {
4527                self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
4528            }
4529            wgt::QueryType::PipelineStatistics(..) => {
4530                self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
4531            }
4532        }
4533
4534        if desc.count == 0 {
4535            return Err(Error::ZeroCount);
4536        }
4537
4538        if desc.count > wgt::QUERY_SET_MAX_QUERIES {
4539            return Err(Error::TooManyQueries {
4540                count: desc.count,
4541                maximum: wgt::QUERY_SET_MAX_QUERIES,
4542            });
4543        }
4544
4545        let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
4546
4547        let raw = unsafe { self.raw().create_query_set(&hal_desc) }
4548            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
4549
4550        let query_set = QuerySet {
4551            raw: ManuallyDrop::new(raw),
4552            device: self.clone(),
4553            label: desc.label.to_string(),
4554            tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
4555            desc: desc.map_label(|_| ()),
4556        };
4557
4558        let query_set = Arc::new(query_set);
4559
4560        Ok(query_set)
4561    }
4562
4563    fn lose(&self, message: &str) {
4564        // Follow the steps at https://gpuweb.github.io/gpuweb/#lose-the-device.
4565
4566        // Mark the device explicitly as invalid. This is checked in various
4567        // places to prevent new work from being submitted.
4568        self.valid.store(false, Ordering::Release);
4569
4570        // 1) Resolve the GPUDevice device.lost promise.
4571        if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
4572            device_lost_closure(DeviceLostReason::Unknown, message.to_string());
4573        }
4574
4575        // 2) Complete any outstanding mapAsync() steps.
4576        // 3) Complete any outstanding onSubmittedWorkDone() steps.
4577
4578        // These parts are passively accomplished by setting valid to false,
4579        // since that will prevent any new work from being added to the queues.
4580        // Future calls to poll_devices will continue to check the work queues
4581        // until they are cleared, and then drop the device.
4582    }
4583
4584    fn release_gpu_resources(&self) {
4585        // This is called when the device is lost, which makes every associated
4586        // resource invalid and unusable. This is an opportunity to release all of
4587        // the underlying gpu resources, even though the objects remain visible to
4588        // the user agent. We purge this memory naturally when resources have been
4589        // moved into the appropriate buckets, so this function just needs to
4590        // initiate movement into those buckets, and it can do that by calling
4591        // "destroy" on all the resources we know about.
4592
4593        // During these iterations, we discard all errors. We don't care!
4594        let trackers = self.trackers.lock();
4595        for buffer in trackers.buffers.used_resources() {
4596            if let Some(buffer) = Weak::upgrade(buffer) {
4597                buffer.destroy();
4598            }
4599        }
4600        for texture in trackers.textures.used_resources() {
4601            if let Some(texture) = Weak::upgrade(texture) {
4602                texture.destroy();
4603            }
4604        }
4605    }
4606
4607    pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
4608        UsageScope::new_pooled(&self.usage_scopes, &self.tracker_indices)
4609    }
4610
4611    pub fn get_hal_counters(&self) -> wgt::HalCounters {
4612        self.raw().get_internal_counters()
4613    }
4614
4615    pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
4616        self.raw().generate_allocator_report()
4617    }
4618}
4619
4620crate::impl_resource_type!(Device);
4621crate::impl_labeled!(Device);
4622crate::impl_storage_item!(Device);