wgpu_core/device/
resource.rs

1use alloc::{
2    borrow::Cow,
3    boxed::Box,
4    string::{String, ToString as _},
5    sync::{Arc, Weak},
6    vec::Vec,
7};
8use core::{
9    fmt,
10    mem::{self, ManuallyDrop},
11    num::NonZeroU32,
12    sync::atomic::{AtomicBool, Ordering},
13};
14use hal::ShouldBeNonZeroExt;
15
16use arrayvec::ArrayVec;
17use bitflags::Flags;
18use smallvec::SmallVec;
19use wgt::{
20    math::align_to, DeviceLostReason, TextureFormat, TextureSampleType, TextureSelector,
21    TextureViewDimension,
22};
23
24#[cfg(feature = "trace")]
25use crate::device::trace;
26use crate::{
27    binding_model::{self, BindGroup, BindGroupLayout, BindGroupLayoutEntryError},
28    command, conv,
29    device::{
30        bgl, create_validator, life::WaitIdleError, map_buffer, AttachmentData,
31        DeviceLostInvocation, HostMap, MissingDownlevelFlags, MissingFeatures, RenderPassContext,
32        CLEANUP_WAIT_MS,
33    },
34    hal_label,
35    init_tracker::{
36        BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
37        TextureInitTrackerAction,
38    },
39    instance::{Adapter, RequestDeviceError},
40    lock::{rank, Mutex, RwLock},
41    pipeline,
42    pool::ResourcePool,
43    resource::{
44        self, Buffer, ExternalTexture, Fallible, Labeled, ParentDevice, QuerySet,
45        RawResourceAccess, Sampler, StagingBuffer, Texture, TextureView,
46        TextureViewNotRenderableReason, Tlas, TrackingData,
47    },
48    resource_log,
49    snatch::{SnatchGuard, SnatchLock, Snatchable},
50    timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
51    track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
52    validation::{self, validate_color_attachment_bytes_per_sample},
53    weak_vec::WeakVec,
54    FastHashMap, LabelHelpers, OnceCellOrLock,
55};
56
57use super::{
58    queue::Queue, DeviceDescriptor, DeviceError, DeviceLostClosure, UserClosures,
59    ENTRYPOINT_FAILURE_ERROR, ZERO_BUFFER_SIZE,
60};
61
62#[cfg(supports_64bit_atomics)]
63use core::sync::atomic::AtomicU64;
64#[cfg(not(supports_64bit_atomics))]
65use portable_atomic::AtomicU64;
66
67pub(crate) struct CommandIndices {
68    /// The index of the last command submission that was attempted.
69    ///
70    /// Note that `fence` may never be signalled with this value, if the command
71    /// submission failed. If you need to wait for everything running on a
72    /// `Queue` to complete, wait for [`last_successful_submission_index`].
73    ///
74    /// [`last_successful_submission_index`]: Device::last_successful_submission_index
75    pub(crate) active_submission_index: hal::FenceValue,
76    pub(crate) next_acceleration_structure_build_command_index: u64,
77}
78
79/// Parameters provided to shaders via a uniform buffer of the type
80/// [`NagaExternalTextureParams`], describing an [`ExternalTexture`] resource
81/// binding.
82///
83/// [`NagaExternalTextureParams`]: naga::SpecialTypes::external_texture_params
84/// [`ExternalTexture`]: binding_model::BindingResource::ExternalTexture
85#[repr(C)]
86#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
87pub struct ExternalTextureParams {
88    /// 4x4 column-major matrix with which to convert sampled YCbCr values
89    /// to RGBA.
90    ///
91    /// This is ignored when `num_planes` is 1.
92    pub yuv_conversion_matrix: [f32; 16],
93
94    /// 3x3 column-major matrix to transform linear RGB values in the source
95    /// color space to linear RGB values in the destination color space. In
96    /// combination with [`Self::src_transfer_function`] and
97    /// [`Self::dst_transfer_function`] this can be used to ensure that
98    /// [`ImageSample`] and [`ImageLoad`] operations return values in the
99    /// desired destination color space rather than the source color space of
100    /// the underlying planes.
101    ///
102    /// Includes a padding element after each column.
103    ///
104    /// [`ImageSample`]: naga::ir::Expression::ImageSample
105    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
106    pub gamut_conversion_matrix: [f32; 12],
107
108    /// Transfer function for the source color space. The *inverse* of this
109    /// will be applied to decode non-linear RGB to linear RGB in the source
110    /// color space.
111    pub src_transfer_function: wgt::ExternalTextureTransferFunction,
112
113    /// Transfer function for the destination color space. This will be applied
114    /// to encode linear RGB to non-linear RGB in the destination color space.
115    pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
116
117    /// Transform to apply to [`ImageSample`] coordinates.
118    ///
119    /// This is a 3x2 column-major matrix representing an affine transform from
120    /// normalized texture coordinates to the normalized coordinates that should
121    /// be sampled from the external texture's underlying plane(s).
122    ///
123    /// This transform may scale, translate, flip, and rotate in 90-degree
124    /// increments, but the result of transforming the rectangle (0,0)..(1,1)
125    /// must be an axis-aligned rectangle that falls within the bounds of
126    /// (0,0)..(1,1).
127    ///
128    /// [`ImageSample`]: naga::ir::Expression::ImageSample
129    pub sample_transform: [f32; 6],
130
131    /// Transform to apply to [`ImageLoad`] coordinates.
132    ///
133    /// This is a 3x2 column-major matrix representing an affine transform from
134    /// non-normalized texel coordinates to the non-normalized coordinates of
135    /// the texel that should be loaded from the external texture's underlying
136    /// plane 0. For planes 1 and 2, if present, plane 0's coordinates are
137    /// scaled according to the textures' relative sizes.
138    ///
139    /// This transform may scale, translate, flip, and rotate in 90-degree
140    /// increments, but the result of transforming the rectangle (0,0)..[`size`]
141    /// must be an axis-aligned rectangle that falls within the bounds of
142    /// (0,0)..[`size`].
143    ///
144    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
145    /// [`size`]: Self::size
146    pub load_transform: [f32; 6],
147
148    /// Size of the external texture.
149    ///
150    /// This is the value that should be returned by size queries in shader
151    /// code; it does not necessarily match the dimensions of the underlying
152    /// texture(s). As a special case, if this is `[0, 0]`, the actual size of
153    /// plane 0 should be used instead.
154    ///
155    /// This must be consistent with [`sample_transform`]: it should be the size
156    /// in texels of the rectangle covered by the square (0,0)..(1,1) after
157    /// [`sample_transform`] has been applied to it.
158    ///
159    /// [`sample_transform`]: Self::sample_transform
160    pub size: [u32; 2],
161
162    /// Number of planes. 1 indicates a single RGBA plane. 2 indicates a Y
163    /// plane and an interleaved CbCr plane. 3 indicates separate Y, Cb, and Cr
164    /// planes.
165    pub num_planes: u32,
166}
167
168impl ExternalTextureParams {
169    pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
170        let gamut_conversion_matrix = [
171            desc.gamut_conversion_matrix[0],
172            desc.gamut_conversion_matrix[1],
173            desc.gamut_conversion_matrix[2],
174            0.0, // padding
175            desc.gamut_conversion_matrix[3],
176            desc.gamut_conversion_matrix[4],
177            desc.gamut_conversion_matrix[5],
178            0.0, // padding
179            desc.gamut_conversion_matrix[6],
180            desc.gamut_conversion_matrix[7],
181            desc.gamut_conversion_matrix[8],
182            0.0, // padding
183        ];
184
185        Self {
186            yuv_conversion_matrix: desc.yuv_conversion_matrix,
187            gamut_conversion_matrix,
188            src_transfer_function: desc.src_transfer_function,
189            dst_transfer_function: desc.dst_transfer_function,
190            size: [desc.width, desc.height],
191            sample_transform: desc.sample_transform,
192            load_transform: desc.load_transform,
193            num_planes: desc.num_planes() as u32,
194        }
195    }
196}
197
198/// Structure describing a logical device. Some members are internally mutable,
199/// stored behind mutexes.
200pub struct Device {
201    raw: Box<dyn hal::DynDevice>,
202    pub(crate) adapter: Arc<Adapter>,
203    pub(crate) queue: OnceCellOrLock<Weak<Queue>>,
204    pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
205    /// The `label` from the descriptor used to create the resource.
206    label: String,
207
208    pub(crate) command_allocator: command::CommandAllocator,
209
210    pub(crate) command_indices: RwLock<CommandIndices>,
211
212    /// The index of the last successful submission to this device's
213    /// [`hal::Queue`].
214    ///
215    /// Unlike [`active_submission_index`], which is incremented each time
216    /// submission is attempted, this is updated only when submission succeeds,
217    /// so waiting for this value won't hang waiting for work that was never
218    /// submitted.
219    ///
220    /// [`active_submission_index`]: CommandIndices::active_submission_index
221    pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
222
223    // NOTE: if both are needed, the `snatchable_lock` must be consistently acquired before the
224    // `fence` lock to avoid deadlocks.
225    pub(crate) fence: RwLock<ManuallyDrop<Box<dyn hal::DynFence>>>,
226    pub(crate) snatchable_lock: SnatchLock,
227
228    /// Is this device valid? Valid is closely associated with "lose the device",
229    /// which can be triggered by various methods, including at the end of device
230    /// destroy, and by any GPU errors that cause us to no longer trust the state
231    /// of the device. Ideally we would like to fold valid into the storage of
232    /// the device itself (for example as an Error enum), but unfortunately we
233    /// need to continue to be able to retrieve the device in poll_devices to
234    /// determine if it can be dropped. If our internal accesses of devices were
235    /// done through ref-counted references and external accesses checked for
236    /// Error enums, we wouldn't need this. For now, we need it. All the call
237    /// sites where we check it are areas that should be revisited if we start
238    /// using ref-counted references for internal access.
239    pub(crate) valid: AtomicBool,
240
241    /// Closure to be called on "lose the device". This is invoked directly by
242    /// device.lose or by the UserCallbacks returned from maintain when the device
243    /// has been destroyed and its queues are empty.
244    pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
245
246    /// Stores the state of buffers and textures.
247    pub(crate) trackers: Mutex<DeviceTracker>,
248    pub(crate) tracker_indices: TrackerIndexAllocators,
249    /// Pool of bind group layouts, allowing deduplication.
250    pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
251    pub(crate) alignments: hal::Alignments,
252    pub(crate) limits: wgt::Limits,
253    pub(crate) features: wgt::Features,
254    pub(crate) downlevel: wgt::DownlevelCapabilities,
255    pub(crate) instance_flags: wgt::InstanceFlags,
256    pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
257    pub(crate) usage_scopes: UsageScopePool,
258    pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
259    // Optional so that we can late-initialize this after the queue is created.
260    pub(crate) timestamp_normalizer:
261        OnceCellOrLock<crate::timestamp_normalization::TimestampNormalizer>,
262    /// Uniform buffer containing [`ExternalTextureParams`] with values such
263    /// that a [`TextureView`] bound to a [`wgt::BindingType::ExternalTexture`]
264    /// binding point will be rendered correctly. Intended to be used as the
265    /// [`hal::ExternalTextureBinding::params`] field.
266    pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
267    // needs to be dropped last
268    #[cfg(feature = "trace")]
269    pub(crate) trace: Mutex<Option<trace::Trace>>,
270}
271
272pub(crate) enum DeferredDestroy {
273    TextureViews(WeakVec<TextureView>),
274    BindGroups(WeakVec<BindGroup>),
275}
276
277impl fmt::Debug for Device {
278    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
279        f.debug_struct("Device")
280            .field("label", &self.label())
281            .field("limits", &self.limits)
282            .field("features", &self.features)
283            .field("downlevel", &self.downlevel)
284            .finish()
285    }
286}
287
288impl Drop for Device {
289    fn drop(&mut self) {
290        resource_log!("Drop {}", self.error_ident());
291
292        // SAFETY: We are in the Drop impl and we don't use self.zero_buffer anymore after this point.
293        let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
294        // SAFETY: We are in the Drop impl and we don't use
295        // self.default_external_texture_params_buffer anymore after this point.
296        let default_external_texture_params_buffer =
297            unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
298        // SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point.
299        let fence = unsafe { ManuallyDrop::take(&mut self.fence.write()) };
300        if let Some(indirect_validation) = self.indirect_validation.take() {
301            indirect_validation.dispose(self.raw.as_ref());
302        }
303        if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
304            timestamp_normalizer.dispose(self.raw.as_ref());
305        }
306        unsafe {
307            self.raw.destroy_buffer(zero_buffer);
308            self.raw
309                .destroy_buffer(default_external_texture_params_buffer);
310            self.raw.destroy_fence(fence);
311        }
312    }
313}
314
315impl Device {
316    pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
317        self.raw.as_ref()
318    }
319    pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
320        if self.features.contains(feature) {
321            Ok(())
322        } else {
323            Err(MissingFeatures(feature))
324        }
325    }
326
327    pub(crate) fn require_downlevel_flags(
328        &self,
329        flags: wgt::DownlevelFlags,
330    ) -> Result<(), MissingDownlevelFlags> {
331        if self.downlevel.flags.contains(flags) {
332            Ok(())
333        } else {
334            Err(MissingDownlevelFlags(flags))
335        }
336    }
337}
338
339impl Device {
340    pub(crate) fn new(
341        raw_device: Box<dyn hal::DynDevice>,
342        adapter: &Arc<Adapter>,
343        desc: &DeviceDescriptor,
344        instance_flags: wgt::InstanceFlags,
345    ) -> Result<Self, DeviceError> {
346        #[cfg(not(feature = "trace"))]
347        match &desc.trace {
348            wgt::Trace::Off => {}
349            _ => {
350                log::error!("wgpu-core feature 'trace' is not enabled");
351            }
352        };
353        #[cfg(feature = "trace")]
354        let trace_dir_name: Option<&std::path::PathBuf> = match &desc.trace {
355            wgt::Trace::Off => None,
356            wgt::Trace::Directory(d) => Some(d),
357            // The enum is non_exhaustive, so we must have a fallback arm (that should be
358            // unreachable in practice).
359            t => {
360                log::error!("unimplemented wgpu_types::Trace variant {t:?}");
361                None
362            }
363        };
364
365        let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
366
367        let command_allocator = command::CommandAllocator::new();
368
369        let rt_uses = if desc
370            .required_features
371            .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
372        {
373            wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
374        } else {
375            wgt::BufferUses::empty()
376        };
377
378        // Create zeroed buffer used for texture clears (and raytracing if required).
379        let zero_buffer = unsafe {
380            raw_device.create_buffer(&hal::BufferDescriptor {
381                label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
382                size: ZERO_BUFFER_SIZE,
383                usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
384                memory_flags: hal::MemoryFlags::empty(),
385            })
386        }
387        .map_err(DeviceError::from_hal)?;
388
389        let default_external_texture_params_buffer = unsafe {
390            raw_device.create_buffer(&hal::BufferDescriptor {
391                label: hal_label(
392                    Some("(wgpu internal) default external texture params buffer"),
393                    instance_flags,
394                ),
395                size: size_of::<ExternalTextureParams>() as _,
396                usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
397                memory_flags: hal::MemoryFlags::empty(),
398            })
399        }
400        .map_err(DeviceError::from_hal)?;
401
402        let alignments = adapter.raw.capabilities.alignments.clone();
403        let downlevel = adapter.raw.capabilities.downlevel.clone();
404
405        let enable_indirect_validation = instance_flags
406            .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
407            && downlevel
408                .flags
409                .contains(wgt::DownlevelFlags::INDIRECT_EXECUTION);
410
411        let indirect_validation = if enable_indirect_validation {
412            Some(crate::indirect_validation::IndirectValidation::new(
413                raw_device.as_ref(),
414                &desc.required_limits,
415                &desc.required_features,
416                adapter.backend(),
417            )?)
418        } else {
419            None
420        };
421
422        Ok(Self {
423            raw: raw_device,
424            adapter: adapter.clone(),
425            queue: OnceCellOrLock::new(),
426            zero_buffer: ManuallyDrop::new(zero_buffer),
427            default_external_texture_params_buffer: ManuallyDrop::new(
428                default_external_texture_params_buffer,
429            ),
430            label: desc.label.to_string(),
431            command_allocator,
432            command_indices: RwLock::new(
433                rank::DEVICE_COMMAND_INDICES,
434                CommandIndices {
435                    active_submission_index: 0,
436                    // By starting at one, we can put the result in a NonZeroU64.
437                    next_acceleration_structure_build_command_index: 1,
438                },
439            ),
440            last_successful_submission_index: AtomicU64::new(0),
441            fence: RwLock::new(rank::DEVICE_FENCE, ManuallyDrop::new(fence)),
442            snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
443            valid: AtomicBool::new(true),
444            device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
445            trackers: Mutex::new(rank::DEVICE_TRACKERS, DeviceTracker::new()),
446            tracker_indices: TrackerIndexAllocators::new(),
447            bgl_pool: ResourcePool::new(),
448            #[cfg(feature = "trace")]
449            trace: Mutex::new(
450                rank::DEVICE_TRACE,
451                trace_dir_name.and_then(|path| match trace::Trace::new(path.clone()) {
452                    Ok(mut trace) => {
453                        trace.add(trace::Action::Init {
454                            desc: wgt::DeviceDescriptor {
455                                trace: wgt::Trace::Off,
456                                ..desc.clone()
457                            },
458                            backend: adapter.backend(),
459                        });
460                        Some(trace)
461                    }
462                    Err(e) => {
463                        log::error!("Unable to start a trace in '{path:?}': {e}");
464                        None
465                    }
466                }),
467            ),
468            alignments,
469            limits: desc.required_limits.clone(),
470            features: desc.required_features,
471            downlevel,
472            instance_flags,
473            deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
474            usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
475            timestamp_normalizer: OnceCellOrLock::new(),
476            indirect_validation,
477        })
478    }
479
480    /// Initializes [`Device::default_external_texture_params_buffer`] with
481    /// required values such that a [`TextureView`] bound to a
482    /// [`wgt::BindingType::ExternalTexture`] binding point will be rendered
483    /// correctly.
484    fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
485        let data = ExternalTextureParams {
486            #[rustfmt::skip]
487            yuv_conversion_matrix: [
488                1.0, 0.0, 0.0, 0.0,
489                0.0, 1.0, 0.0, 0.0,
490                0.0, 0.0, 1.0, 0.0,
491                0.0, 0.0, 0.0, 1.0,
492            ],
493            #[rustfmt::skip]
494            gamut_conversion_matrix: [
495                1.0, 0.0, 0.0, /* padding */ 0.0,
496                0.0, 1.0, 0.0, /* padding */ 0.0,
497                0.0, 0.0, 1.0, /* padding */ 0.0,
498            ],
499            src_transfer_function: Default::default(),
500            dst_transfer_function: Default::default(),
501            size: [0, 0],
502            #[rustfmt::skip]
503            sample_transform: [
504                1.0, 0.0,
505                0.0, 1.0,
506                0.0, 0.0
507            ],
508            #[rustfmt::skip]
509            load_transform: [
510                1.0, 0.0,
511                0.0, 1.0,
512                0.0, 0.0
513            ],
514            num_planes: 1,
515        };
516        let mut staging_buffer =
517            StagingBuffer::new(self, wgt::BufferSize::new(size_of_val(&data) as _).unwrap())?;
518        staging_buffer.write(bytemuck::bytes_of(&data));
519        let staging_buffer = staging_buffer.flush();
520
521        let params_buffer = self.default_external_texture_params_buffer.as_ref();
522        let queue = self.get_queue().unwrap();
523        let mut pending_writes = queue.pending_writes.lock();
524
525        unsafe {
526            pending_writes
527                .command_encoder
528                .transition_buffers(&[hal::BufferBarrier {
529                    buffer: params_buffer,
530                    usage: hal::StateTransition {
531                        from: wgt::BufferUses::MAP_WRITE,
532                        to: wgt::BufferUses::COPY_SRC,
533                    },
534                }]);
535            pending_writes.command_encoder.copy_buffer_to_buffer(
536                staging_buffer.raw(),
537                params_buffer,
538                &[hal::BufferCopy {
539                    src_offset: 0,
540                    dst_offset: 0,
541                    size: staging_buffer.size,
542                }],
543            );
544            pending_writes.consume(staging_buffer);
545            pending_writes
546                .command_encoder
547                .transition_buffers(&[hal::BufferBarrier {
548                    buffer: params_buffer,
549                    usage: hal::StateTransition {
550                        from: wgt::BufferUses::COPY_DST,
551                        to: wgt::BufferUses::UNIFORM,
552                    },
553                }]);
554        }
555
556        Ok(())
557    }
558
559    pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
560        let queue = self.get_queue().unwrap();
561
562        let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
563            self,
564            queue.get_timestamp_period(),
565        )?;
566
567        self.timestamp_normalizer
568            .set(timestamp_normalizer)
569            .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
570
571        self.init_default_external_texture_params_buffer()?;
572
573        Ok(())
574    }
575
576    /// Returns the backend this device is using.
577    pub fn backend(&self) -> wgt::Backend {
578        self.adapter.backend()
579    }
580
581    pub fn is_valid(&self) -> bool {
582        self.valid.load(Ordering::Acquire)
583    }
584
585    pub fn check_is_valid(&self) -> Result<(), DeviceError> {
586        if self.is_valid() {
587            Ok(())
588        } else {
589            Err(DeviceError::Lost)
590        }
591    }
592
593    /// Checks that we are operating within the memory budget reported by the native APIs.
594    ///
595    /// If we are not, the device gets invalidated.
596    ///
597    /// The budget might fluctuate over the lifetime of the application, so it should be checked
598    /// somewhat frequently.
599    pub fn lose_if_oom(&self) {
600        let _ = self
601            .raw()
602            .check_if_oom()
603            .map_err(|e| self.handle_hal_error(e));
604    }
605
606    pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
607        match error {
608            hal::DeviceError::OutOfMemory
609            | hal::DeviceError::Lost
610            | hal::DeviceError::Unexpected => {
611                self.lose(&error.to_string());
612            }
613        }
614        DeviceError::from_hal(error)
615    }
616
617    pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
618        match error {
619            hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
620            error => self.handle_hal_error(error),
621        }
622    }
623
624    /// Run some destroy operations that were deferred.
625    ///
626    /// Destroying the resources requires taking a write lock on the device's snatch lock,
627    /// so a good reason for deferring resource destruction is when we don't know for sure
628    /// how risky it is to take the lock (typically, it shouldn't be taken from the drop
629    /// implementation of a reference-counted structure).
630    /// The snatch lock must not be held while this function is called.
631    pub(crate) fn deferred_resource_destruction(&self) {
632        let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
633        for item in deferred_destroy {
634            match item {
635                DeferredDestroy::TextureViews(views) => {
636                    for view in views {
637                        let Some(view) = view.upgrade() else {
638                            continue;
639                        };
640                        let Some(raw_view) = view.raw.snatch(&mut self.snatchable_lock.write())
641                        else {
642                            continue;
643                        };
644
645                        resource_log!("Destroy raw {}", view.error_ident());
646
647                        unsafe {
648                            self.raw().destroy_texture_view(raw_view);
649                        }
650                    }
651                }
652                DeferredDestroy::BindGroups(bind_groups) => {
653                    for bind_group in bind_groups {
654                        let Some(bind_group) = bind_group.upgrade() else {
655                            continue;
656                        };
657                        let Some(raw_bind_group) =
658                            bind_group.raw.snatch(&mut self.snatchable_lock.write())
659                        else {
660                            continue;
661                        };
662
663                        resource_log!("Destroy raw {}", bind_group.error_ident());
664
665                        unsafe {
666                            self.raw().destroy_bind_group(raw_bind_group);
667                        }
668                    }
669                }
670            }
671        }
672    }
673
674    pub fn get_queue(&self) -> Option<Arc<Queue>> {
675        self.queue.get().as_ref()?.upgrade()
676    }
677
678    pub fn set_queue(&self, queue: &Arc<Queue>) {
679        assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
680    }
681
682    /// Check the current status of the GPU and process any submissions that have
683    /// finished.
684    ///
685    /// The `poll_type` argument tells if this function should wait for a particular
686    /// submission index to complete, or if it should just poll the current status.
687    ///
688    /// This will process _all_ completed submissions, even if the caller only asked
689    /// us to poll to a given submission index.
690    ///
691    /// Return a pair `(closures, result)`, where:
692    ///
693    /// - `closures` is a list of callbacks that need to be invoked informing the user
694    ///   about various things occurring. These happen and should be handled even if
695    ///   this function returns an error, hence they are outside of the result.
696    ///
697    /// - `results` is a boolean indicating the result of the wait operation, including
698    ///   if there was a timeout or a validation error.
699    pub(crate) fn maintain<'this>(
700        &'this self,
701        fence: crate::lock::RwLockReadGuard<ManuallyDrop<Box<dyn hal::DynFence>>>,
702        poll_type: wgt::PollType<crate::SubmissionIndex>,
703        snatch_guard: SnatchGuard,
704    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
705        profiling::scope!("Device::maintain");
706
707        let mut user_closures = UserClosures::default();
708
709        // If a wait was requested, determine which submission index to wait for.
710        let wait_submission_index = match poll_type {
711            wgt::PollType::WaitForSubmissionIndex(submission_index) => {
712                let last_successful_submission_index = self
713                    .last_successful_submission_index
714                    .load(Ordering::Acquire);
715
716                if submission_index > last_successful_submission_index {
717                    let result = Err(WaitIdleError::WrongSubmissionIndex(
718                        submission_index,
719                        last_successful_submission_index,
720                    ));
721
722                    return (user_closures, result);
723                }
724
725                Some(submission_index)
726            }
727            wgt::PollType::Wait => Some(
728                self.last_successful_submission_index
729                    .load(Ordering::Acquire),
730            ),
731            wgt::PollType::Poll => None,
732        };
733
734        // Wait for the submission index if requested.
735        if let Some(target_submission_index) = wait_submission_index {
736            log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
737
738            let wait_result = unsafe {
739                self.raw()
740                    .wait(fence.as_ref(), target_submission_index, CLEANUP_WAIT_MS)
741            };
742
743            // This error match is only about `DeviceErrors`. At this stage we do not care if
744            // the wait succeeded or not, and the `Ok(bool)`` variant is ignored.
745            if let Err(e) = wait_result {
746                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
747                return (user_closures, Err(hal_error));
748            }
749        }
750
751        // Get the currently finished submission index. This may be higher than the requested
752        // wait, or it may be less than the requested wait if the wait failed.
753        let fence_value_result = unsafe { self.raw().get_fence_value(fence.as_ref()) };
754        let current_finished_submission = match fence_value_result {
755            Ok(fence_value) => fence_value,
756            Err(e) => {
757                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
758                return (user_closures, Err(hal_error));
759            }
760        };
761
762        // Maintain all finished submissions on the queue, updating the relevant user closures and collecting if the queue is empty.
763        //
764        // We don't use the result of the wait here, as we want to progress forward as far as possible
765        // and the wait could have been for submissions that finished long ago.
766        let mut queue_empty = false;
767        if let Some(queue) = self.get_queue() {
768            let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
769            (
770                user_closures.submissions,
771                user_closures.mappings,
772                user_closures.blas_compact_ready,
773                queue_empty,
774            ) = queue_result
775        };
776
777        // Based on the queue empty status, and the current finished submission index, determine the result of the poll.
778        let result = if queue_empty {
779            if let Some(wait_submission_index) = wait_submission_index {
780                // Assert to ensure that if we received a queue empty status, the fence shows the correct value.
781                // This is defensive, as this should never be hit.
782                assert!(
783                    current_finished_submission >= wait_submission_index,
784                    "If the queue is empty, the current submission index ({current_finished_submission}) should be at least the wait submission index ({wait_submission_index})"
785                );
786            }
787
788            Ok(wgt::PollStatus::QueueEmpty)
789        } else if let Some(wait_submission_index) = wait_submission_index {
790            // This is theoretically possible to succeed more than checking on the poll result
791            // as submissions could have finished in the time between the timeout resolving,
792            // the thread getting scheduled again, and us checking the fence value.
793            if current_finished_submission >= wait_submission_index {
794                Ok(wgt::PollStatus::WaitSucceeded)
795            } else {
796                Err(WaitIdleError::Timeout)
797            }
798        } else {
799            Ok(wgt::PollStatus::Poll)
800        };
801
802        // Detect if we have been destroyed and now need to lose the device.
803        //
804        // If we are invalid (set at start of destroy) and our queue is empty,
805        // and we have a DeviceLostClosure, return the closure to be called by
806        // our caller. This will complete the steps for both destroy and for
807        // "lose the device".
808        let mut should_release_gpu_resource = false;
809        if !self.is_valid() && queue_empty {
810            // We can release gpu resources associated with this device (but not
811            // while holding the life_tracker lock).
812            should_release_gpu_resource = true;
813
814            // If we have a DeviceLostClosure, build an invocation with the
815            // reason DeviceLostReason::Destroyed and no message.
816            if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
817                user_closures
818                    .device_lost_invocations
819                    .push(DeviceLostInvocation {
820                        closure: device_lost_closure,
821                        reason: DeviceLostReason::Destroyed,
822                        message: String::new(),
823                    });
824            }
825        }
826
827        // Don't hold the locks while calling release_gpu_resources.
828        drop(fence);
829        drop(snatch_guard);
830
831        if should_release_gpu_resource {
832            self.release_gpu_resources();
833        }
834
835        (user_closures, result)
836    }
837
838    pub(crate) fn create_buffer(
839        self: &Arc<Self>,
840        desc: &resource::BufferDescriptor,
841    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
842        self.check_is_valid()?;
843
844        if desc.size > self.limits.max_buffer_size {
845            return Err(resource::CreateBufferError::MaxBufferSize {
846                requested: desc.size,
847                maximum: self.limits.max_buffer_size,
848            });
849        }
850
851        if desc
852            .usage
853            .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
854        {
855            self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)?;
856        }
857
858        if desc.usage.contains(wgt::BufferUsages::INDEX)
859            && desc.usage.contains(
860                wgt::BufferUsages::VERTEX
861                    | wgt::BufferUsages::UNIFORM
862                    | wgt::BufferUsages::INDIRECT
863                    | wgt::BufferUsages::STORAGE,
864            )
865        {
866            self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
867        }
868
869        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
870            return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
871        }
872
873        if !self
874            .features
875            .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
876        {
877            use wgt::BufferUsages as Bu;
878            let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
879                && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
880            let read_mismatch = desc.usage.contains(Bu::MAP_READ)
881                && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
882            if write_mismatch || read_mismatch {
883                return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
884            }
885        }
886
887        let mut usage = conv::map_buffer_usage(desc.usage);
888
889        if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
890            self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
891            // We are going to be reading from it, internally;
892            // when validating the content of the buffer
893            usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
894        }
895
896        if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
897            usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
898        }
899
900        if desc.mapped_at_creation {
901            if desc.size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
902                return Err(resource::CreateBufferError::UnalignedSize);
903            }
904            if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
905                // we are going to be copying into it, internally
906                usage |= wgt::BufferUses::COPY_DST;
907            }
908        } else {
909            // We are required to zero out (initialize) all memory. This is done
910            // on demand using clear_buffer which requires write transfer usage!
911            usage |= wgt::BufferUses::COPY_DST;
912        }
913
914        let actual_size = if desc.size == 0 {
915            wgt::COPY_BUFFER_ALIGNMENT
916        } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
917            // Bumping the size by 1 so that we can bind an empty range at the
918            // end of the buffer.
919            desc.size + 1
920        } else {
921            desc.size
922        };
923        let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
924        let aligned_size = if clear_remainder != 0 {
925            actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
926        } else {
927            actual_size
928        };
929
930        let hal_desc = hal::BufferDescriptor {
931            label: desc.label.to_hal(self.instance_flags),
932            size: aligned_size,
933            usage,
934            memory_flags: hal::MemoryFlags::empty(),
935        };
936        let buffer = unsafe { self.raw().create_buffer(&hal_desc) }
937            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
938
939        let timestamp_normalization_bind_group = Snatchable::new(unsafe {
940            // SAFETY: The size passed here must not overflow the buffer.
941            self.timestamp_normalizer
942                .get()
943                .unwrap()
944                .create_normalization_bind_group(
945                    self,
946                    &*buffer,
947                    desc.label.as_deref(),
948                    wgt::BufferSize::new(hal_desc.size).unwrap(),
949                    desc.usage,
950                )
951        }?);
952
953        let indirect_validation_bind_groups =
954            self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
955
956        let buffer = Buffer {
957            raw: Snatchable::new(buffer),
958            device: self.clone(),
959            usage: desc.usage,
960            size: desc.size,
961            initialization_status: RwLock::new(
962                rank::BUFFER_INITIALIZATION_STATUS,
963                BufferInitTracker::new(aligned_size),
964            ),
965            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
966            label: desc.label.to_string(),
967            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
968            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
969            timestamp_normalization_bind_group,
970            indirect_validation_bind_groups,
971        };
972
973        let buffer = Arc::new(buffer);
974
975        let buffer_use = if !desc.mapped_at_creation {
976            wgt::BufferUses::empty()
977        } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
978            // buffer is mappable, so we are just doing that at start
979            let map_size = buffer.size;
980            let mapping = if map_size == 0 {
981                hal::BufferMapping {
982                    ptr: core::ptr::NonNull::dangling(),
983                    is_coherent: true,
984                }
985            } else {
986                let snatch_guard: SnatchGuard = self.snatchable_lock.read();
987                map_buffer(&buffer, 0, map_size, HostMap::Write, &snatch_guard)?
988            };
989            *buffer.map_state.lock() = resource::BufferMapState::Active {
990                mapping,
991                range: 0..map_size,
992                host: HostMap::Write,
993            };
994            wgt::BufferUses::MAP_WRITE
995        } else {
996            let mut staging_buffer =
997                StagingBuffer::new(self, wgt::BufferSize::new(aligned_size).unwrap())?;
998
999            // Zero initialize memory and then mark the buffer as initialized
1000            // (it's guaranteed that this is the case by the time the buffer is usable)
1001            staging_buffer.write_zeros();
1002            buffer.initialization_status.write().drain(0..aligned_size);
1003
1004            *buffer.map_state.lock() = resource::BufferMapState::Init { staging_buffer };
1005            wgt::BufferUses::COPY_DST
1006        };
1007
1008        self.trackers
1009            .lock()
1010            .buffers
1011            .insert_single(&buffer, buffer_use);
1012
1013        Ok(buffer)
1014    }
1015
1016    pub(crate) fn create_texture_from_hal(
1017        self: &Arc<Self>,
1018        hal_texture: Box<dyn hal::DynTexture>,
1019        desc: &resource::TextureDescriptor,
1020    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1021        let format_features = self
1022            .describe_format_features(desc.format)
1023            .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error))?;
1024
1025        unsafe { self.raw().add_raw_texture(&*hal_texture) };
1026
1027        let texture = Texture::new(
1028            self,
1029            resource::TextureInner::Native { raw: hal_texture },
1030            conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1031            desc,
1032            format_features,
1033            resource::TextureClearMode::None,
1034            false,
1035        );
1036
1037        let texture = Arc::new(texture);
1038
1039        self.trackers
1040            .lock()
1041            .textures
1042            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1043
1044        Ok(texture)
1045    }
1046
1047    /// # Safety
1048    ///
1049    /// - `hal_buffer` must have been created on this device.
1050    /// - `hal_buffer` must have been created respecting `desc` (in particular, the size).
1051    /// - `hal_buffer` must be initialized.
1052    /// - `hal_buffer` must not have zero size.
1053    pub(crate) unsafe fn create_buffer_from_hal(
1054        self: &Arc<Self>,
1055        hal_buffer: Box<dyn hal::DynBuffer>,
1056        desc: &resource::BufferDescriptor,
1057    ) -> (Fallible<Buffer>, Option<resource::CreateBufferError>) {
1058        let timestamp_normalization_bind_group = unsafe {
1059            match self
1060                .timestamp_normalizer
1061                .get()
1062                .unwrap()
1063                .create_normalization_bind_group(
1064                    self,
1065                    &*hal_buffer,
1066                    desc.label.as_deref(),
1067                    wgt::BufferSize::new(desc.size).unwrap(),
1068                    desc.usage,
1069                ) {
1070                Ok(bg) => Snatchable::new(bg),
1071                Err(e) => {
1072                    return (
1073                        Fallible::Invalid(Arc::new(desc.label.to_string())),
1074                        Some(e.into()),
1075                    )
1076                }
1077            }
1078        };
1079
1080        let indirect_validation_bind_groups = match self.create_indirect_validation_bind_groups(
1081            hal_buffer.as_ref(),
1082            desc.size,
1083            desc.usage,
1084        ) {
1085            Ok(ok) => ok,
1086            Err(e) => return (Fallible::Invalid(Arc::new(desc.label.to_string())), Some(e)),
1087        };
1088
1089        unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1090
1091        let buffer = Buffer {
1092            raw: Snatchable::new(hal_buffer),
1093            device: self.clone(),
1094            usage: desc.usage,
1095            size: desc.size,
1096            initialization_status: RwLock::new(
1097                rank::BUFFER_INITIALIZATION_STATUS,
1098                BufferInitTracker::new(0),
1099            ),
1100            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1101            label: desc.label.to_string(),
1102            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1103            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1104            timestamp_normalization_bind_group,
1105            indirect_validation_bind_groups,
1106        };
1107
1108        let buffer = Arc::new(buffer);
1109
1110        self.trackers
1111            .lock()
1112            .buffers
1113            .insert_single(&buffer, wgt::BufferUses::empty());
1114
1115        (Fallible::Valid(buffer), None)
1116    }
1117
1118    fn create_indirect_validation_bind_groups(
1119        &self,
1120        raw_buffer: &dyn hal::DynBuffer,
1121        buffer_size: u64,
1122        usage: wgt::BufferUsages,
1123    ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1124    {
1125        if !usage.contains(wgt::BufferUsages::INDIRECT) {
1126            return Ok(Snatchable::empty());
1127        }
1128
1129        let Some(ref indirect_validation) = self.indirect_validation else {
1130            return Ok(Snatchable::empty());
1131        };
1132
1133        let bind_groups = crate::indirect_validation::BindGroups::new(
1134            indirect_validation,
1135            self,
1136            buffer_size,
1137            raw_buffer,
1138        )
1139        .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1140
1141        if let Some(bind_groups) = bind_groups {
1142            Ok(Snatchable::new(bind_groups))
1143        } else {
1144            Ok(Snatchable::empty())
1145        }
1146    }
1147
1148    pub(crate) fn create_texture(
1149        self: &Arc<Self>,
1150        desc: &resource::TextureDescriptor,
1151    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1152        use resource::{CreateTextureError, TextureDimensionError};
1153
1154        self.check_is_valid()?;
1155
1156        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1157            return Err(CreateTextureError::InvalidUsage(desc.usage));
1158        }
1159
1160        conv::check_texture_dimension_size(
1161            desc.dimension,
1162            desc.size,
1163            desc.sample_count,
1164            &self.limits,
1165        )?;
1166
1167        if desc.dimension != wgt::TextureDimension::D2 {
1168            // Depth textures can only be 2D
1169            if desc.format.is_depth_stencil_format() {
1170                return Err(CreateTextureError::InvalidDepthDimension(
1171                    desc.dimension,
1172                    desc.format,
1173                ));
1174            }
1175        }
1176
1177        if desc.dimension != wgt::TextureDimension::D2
1178            && desc.dimension != wgt::TextureDimension::D3
1179        {
1180            // Compressed textures can only be 2D or 3D
1181            if desc.format.is_compressed() {
1182                return Err(CreateTextureError::InvalidCompressedDimension(
1183                    desc.dimension,
1184                    desc.format,
1185                ));
1186            }
1187
1188            // Renderable textures can only be 2D or 3D
1189            if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1190                return Err(CreateTextureError::InvalidDimensionUsages(
1191                    wgt::TextureUsages::RENDER_ATTACHMENT,
1192                    desc.dimension,
1193                ));
1194            }
1195        }
1196
1197        if desc.format.is_compressed() {
1198            let (block_width, block_height) = desc.format.block_dimensions();
1199
1200            if desc.size.width % block_width != 0 {
1201                return Err(CreateTextureError::InvalidDimension(
1202                    TextureDimensionError::NotMultipleOfBlockWidth {
1203                        width: desc.size.width,
1204                        block_width,
1205                        format: desc.format,
1206                    },
1207                ));
1208            }
1209
1210            if desc.size.height % block_height != 0 {
1211                return Err(CreateTextureError::InvalidDimension(
1212                    TextureDimensionError::NotMultipleOfBlockHeight {
1213                        height: desc.size.height,
1214                        block_height,
1215                        format: desc.format,
1216                    },
1217                ));
1218            }
1219
1220            if desc.dimension == wgt::TextureDimension::D3 {
1221                // Only BCn formats with Sliced 3D feature can be used for 3D textures
1222                if desc.format.is_bcn() {
1223                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1224                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1225                } else if desc.format.is_astc() {
1226                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1227                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1228                } else {
1229                    return Err(CreateTextureError::InvalidCompressedDimension(
1230                        desc.dimension,
1231                        desc.format,
1232                    ));
1233                }
1234            }
1235        }
1236
1237        {
1238            let (width_multiple, height_multiple) = desc.format.size_multiple_requirement();
1239
1240            if desc.size.width % width_multiple != 0 {
1241                return Err(CreateTextureError::InvalidDimension(
1242                    TextureDimensionError::WidthNotMultipleOf {
1243                        width: desc.size.width,
1244                        multiple: width_multiple,
1245                        format: desc.format,
1246                    },
1247                ));
1248            }
1249
1250            if desc.size.height % height_multiple != 0 {
1251                return Err(CreateTextureError::InvalidDimension(
1252                    TextureDimensionError::HeightNotMultipleOf {
1253                        height: desc.size.height,
1254                        multiple: height_multiple,
1255                        format: desc.format,
1256                    },
1257                ));
1258            }
1259        }
1260
1261        let format_features = self
1262            .describe_format_features(desc.format)
1263            .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1264
1265        if desc.sample_count > 1 {
1266            if desc.mip_level_count != 1 {
1267                return Err(CreateTextureError::InvalidMipLevelCount {
1268                    requested: desc.mip_level_count,
1269                    maximum: 1,
1270                });
1271            }
1272
1273            if desc.size.depth_or_array_layers != 1 {
1274                return Err(CreateTextureError::InvalidDimension(
1275                    TextureDimensionError::MultisampledDepthOrArrayLayer(
1276                        desc.size.depth_or_array_layers,
1277                    ),
1278                ));
1279            }
1280
1281            if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
1282                return Err(CreateTextureError::InvalidMultisampledStorageBinding);
1283            }
1284
1285            if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1286                return Err(CreateTextureError::MultisampledNotRenderAttachment);
1287            }
1288
1289            if !format_features.flags.intersects(
1290                wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
1291                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
1292                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
1293                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
1294            ) {
1295                return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
1296            }
1297
1298            if !format_features
1299                .flags
1300                .sample_count_supported(desc.sample_count)
1301            {
1302                return Err(CreateTextureError::InvalidSampleCount(
1303                    desc.sample_count,
1304                    desc.format,
1305                    desc.format
1306                        .guaranteed_format_features(self.features)
1307                        .flags
1308                        .supported_sample_counts(),
1309                    self.adapter
1310                        .get_texture_format_features(desc.format)
1311                        .flags
1312                        .supported_sample_counts(),
1313                ));
1314            };
1315        }
1316
1317        let mips = desc.mip_level_count;
1318        let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1319        if mips == 0 || mips > max_levels_allowed {
1320            return Err(CreateTextureError::InvalidMipLevelCount {
1321                requested: mips,
1322                maximum: max_levels_allowed,
1323            });
1324        }
1325
1326        let missing_allowed_usages = desc.usage - format_features.allowed_usages;
1327        if !missing_allowed_usages.is_empty() {
1328            // detect downlevel incompatibilities
1329            let wgpu_allowed_usages = desc
1330                .format
1331                .guaranteed_format_features(self.features)
1332                .allowed_usages;
1333            let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
1334            return Err(CreateTextureError::InvalidFormatUsages(
1335                missing_allowed_usages,
1336                desc.format,
1337                wgpu_missing_usages.is_empty(),
1338            ));
1339        }
1340
1341        let mut hal_view_formats = Vec::new();
1342        for format in desc.view_formats.iter() {
1343            if desc.format == *format {
1344                continue;
1345            }
1346            if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
1347                return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
1348            }
1349            hal_view_formats.push(*format);
1350        }
1351        if !hal_view_formats.is_empty() {
1352            self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
1353        }
1354
1355        let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
1356
1357        let hal_desc = hal::TextureDescriptor {
1358            label: desc.label.to_hal(self.instance_flags),
1359            size: desc.size,
1360            mip_level_count: desc.mip_level_count,
1361            sample_count: desc.sample_count,
1362            dimension: desc.dimension,
1363            format: desc.format,
1364            usage: hal_usage,
1365            memory_flags: hal::MemoryFlags::empty(),
1366            view_formats: hal_view_formats,
1367        };
1368
1369        let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
1370            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1371
1372        let clear_mode = if hal_usage
1373            .intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE | wgt::TextureUses::COLOR_TARGET)
1374            && desc.dimension == wgt::TextureDimension::D2
1375        {
1376            let (is_color, usage) = if desc.format.is_depth_stencil_format() {
1377                (false, wgt::TextureUses::DEPTH_STENCIL_WRITE)
1378            } else {
1379                (true, wgt::TextureUses::COLOR_TARGET)
1380            };
1381
1382            let clear_label = hal_label(
1383                Some("(wgpu internal) clear texture view"),
1384                self.instance_flags,
1385            );
1386
1387            let mut clear_views = SmallVec::new();
1388            for mip_level in 0..desc.mip_level_count {
1389                for array_layer in 0..desc.size.depth_or_array_layers {
1390                    macro_rules! push_clear_view {
1391                        ($format:expr, $aspect:expr) => {
1392                            let desc = hal::TextureViewDescriptor {
1393                                label: clear_label,
1394                                format: $format,
1395                                dimension: TextureViewDimension::D2,
1396                                usage,
1397                                range: wgt::ImageSubresourceRange {
1398                                    aspect: $aspect,
1399                                    base_mip_level: mip_level,
1400                                    mip_level_count: Some(1),
1401                                    base_array_layer: array_layer,
1402                                    array_layer_count: Some(1),
1403                                },
1404                            };
1405                            clear_views.push(ManuallyDrop::new(
1406                                unsafe {
1407                                    self.raw().create_texture_view(raw_texture.as_ref(), &desc)
1408                                }
1409                                .map_err(|e| self.handle_hal_error(e))?,
1410                            ));
1411                        };
1412                    }
1413
1414                    if let Some(planes) = desc.format.planes() {
1415                        for plane in 0..planes {
1416                            let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1417                            let format = desc.format.aspect_specific_format(aspect).unwrap();
1418                            push_clear_view!(format, aspect);
1419                        }
1420                    } else {
1421                        push_clear_view!(desc.format, wgt::TextureAspect::All);
1422                    }
1423                }
1424            }
1425            resource::TextureClearMode::RenderPass {
1426                clear_views,
1427                is_color,
1428            }
1429        } else {
1430            resource::TextureClearMode::BufferCopy
1431        };
1432
1433        let texture = Texture::new(
1434            self,
1435            resource::TextureInner::Native { raw: raw_texture },
1436            hal_usage,
1437            desc,
1438            format_features,
1439            clear_mode,
1440            true,
1441        );
1442
1443        let texture = Arc::new(texture);
1444
1445        self.trackers
1446            .lock()
1447            .textures
1448            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1449
1450        Ok(texture)
1451    }
1452
1453    pub(crate) fn create_texture_view(
1454        self: &Arc<Self>,
1455        texture: &Arc<Texture>,
1456        desc: &resource::TextureViewDescriptor,
1457    ) -> Result<Arc<TextureView>, resource::CreateTextureViewError> {
1458        self.check_is_valid()?;
1459
1460        let snatch_guard = texture.device.snatchable_lock.read();
1461
1462        let texture_raw = texture.try_raw(&snatch_guard)?;
1463
1464        // resolve TextureViewDescriptor defaults
1465        // https://gpuweb.github.io/gpuweb/#abstract-opdef-resolving-gputextureviewdescriptor-defaults
1466        let resolved_format = desc.format.unwrap_or_else(|| {
1467            texture
1468                .desc
1469                .format
1470                .aspect_specific_format(desc.range.aspect)
1471                .unwrap_or(texture.desc.format)
1472        });
1473
1474        let resolved_dimension = desc
1475            .dimension
1476            .unwrap_or_else(|| match texture.desc.dimension {
1477                wgt::TextureDimension::D1 => TextureViewDimension::D1,
1478                wgt::TextureDimension::D2 => {
1479                    if texture.desc.array_layer_count() == 1 {
1480                        TextureViewDimension::D2
1481                    } else {
1482                        TextureViewDimension::D2Array
1483                    }
1484                }
1485                wgt::TextureDimension::D3 => TextureViewDimension::D3,
1486            });
1487
1488        let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| {
1489            texture
1490                .desc
1491                .mip_level_count
1492                .saturating_sub(desc.range.base_mip_level)
1493        });
1494
1495        let resolved_array_layer_count =
1496            desc.range
1497                .array_layer_count
1498                .unwrap_or_else(|| match resolved_dimension {
1499                    TextureViewDimension::D1
1500                    | TextureViewDimension::D2
1501                    | TextureViewDimension::D3 => 1,
1502                    TextureViewDimension::Cube => 6,
1503                    TextureViewDimension::D2Array | TextureViewDimension::CubeArray => texture
1504                        .desc
1505                        .array_layer_count()
1506                        .saturating_sub(desc.range.base_array_layer),
1507                });
1508
1509        let resolved_usage = {
1510            let usage = desc.usage.unwrap_or(wgt::TextureUsages::empty());
1511            if usage.is_empty() {
1512                texture.desc.usage
1513            } else if texture.desc.usage.contains(usage) {
1514                usage
1515            } else {
1516                return Err(resource::CreateTextureViewError::InvalidTextureViewUsage {
1517                    view: usage,
1518                    texture: texture.desc.usage,
1519                });
1520            }
1521        };
1522
1523        let format_features = self.describe_format_features(resolved_format)?;
1524        let allowed_format_usages = format_features.allowed_usages;
1525        if resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1526            && !allowed_format_usages.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1527        {
1528            return Err(
1529                resource::CreateTextureViewError::TextureViewFormatNotRenderable(resolved_format),
1530            );
1531        }
1532
1533        if resolved_usage.contains(wgt::TextureUsages::STORAGE_BINDING)
1534            && !allowed_format_usages.contains(wgt::TextureUsages::STORAGE_BINDING)
1535        {
1536            return Err(
1537                resource::CreateTextureViewError::TextureViewFormatNotStorage(resolved_format),
1538            );
1539        }
1540
1541        // validate TextureViewDescriptor
1542
1543        let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect);
1544        if aspects.is_empty() {
1545            return Err(resource::CreateTextureViewError::InvalidAspect {
1546                texture_format: texture.desc.format,
1547                requested_aspect: desc.range.aspect,
1548            });
1549        }
1550
1551        let format_is_good = if desc.range.aspect == wgt::TextureAspect::All {
1552            resolved_format == texture.desc.format
1553                || texture.desc.view_formats.contains(&resolved_format)
1554        } else {
1555            Some(resolved_format)
1556                == texture
1557                    .desc
1558                    .format
1559                    .aspect_specific_format(desc.range.aspect)
1560        };
1561        if !format_is_good {
1562            return Err(resource::CreateTextureViewError::FormatReinterpretation {
1563                texture: texture.desc.format,
1564                view: resolved_format,
1565            });
1566        }
1567
1568        // check if multisampled texture is seen as anything but 2D
1569        if texture.desc.sample_count > 1 && resolved_dimension != TextureViewDimension::D2 {
1570            return Err(
1571                resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension(
1572                    resolved_dimension,
1573                ),
1574            );
1575        }
1576
1577        // check if the dimension is compatible with the texture
1578        if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() {
1579            return Err(
1580                resource::CreateTextureViewError::InvalidTextureViewDimension {
1581                    view: resolved_dimension,
1582                    texture: texture.desc.dimension,
1583                },
1584            );
1585        }
1586
1587        match resolved_dimension {
1588            TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => {
1589                if resolved_array_layer_count != 1 {
1590                    return Err(resource::CreateTextureViewError::InvalidArrayLayerCount {
1591                        requested: resolved_array_layer_count,
1592                        dim: resolved_dimension,
1593                    });
1594                }
1595            }
1596            TextureViewDimension::Cube => {
1597                if resolved_array_layer_count != 6 {
1598                    return Err(
1599                        resource::CreateTextureViewError::InvalidCubemapTextureDepth {
1600                            depth: resolved_array_layer_count,
1601                        },
1602                    );
1603                }
1604            }
1605            TextureViewDimension::CubeArray => {
1606                if resolved_array_layer_count % 6 != 0 {
1607                    return Err(
1608                        resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth {
1609                            depth: resolved_array_layer_count,
1610                        },
1611                    );
1612                }
1613            }
1614            _ => {}
1615        }
1616
1617        match resolved_dimension {
1618            TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1619                if texture.desc.size.width != texture.desc.size.height {
1620                    return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize);
1621                }
1622            }
1623            _ => {}
1624        }
1625
1626        if resolved_mip_level_count == 0 {
1627            return Err(resource::CreateTextureViewError::ZeroMipLevelCount);
1628        }
1629
1630        let mip_level_end = desc
1631            .range
1632            .base_mip_level
1633            .saturating_add(resolved_mip_level_count);
1634
1635        let level_end = texture.desc.mip_level_count;
1636        if mip_level_end > level_end {
1637            return Err(resource::CreateTextureViewError::TooManyMipLevels {
1638                requested: mip_level_end,
1639                total: level_end,
1640            });
1641        }
1642
1643        if resolved_array_layer_count == 0 {
1644            return Err(resource::CreateTextureViewError::ZeroArrayLayerCount);
1645        }
1646
1647        let array_layer_end = desc
1648            .range
1649            .base_array_layer
1650            .saturating_add(resolved_array_layer_count);
1651
1652        let layer_end = texture.desc.array_layer_count();
1653        if array_layer_end > layer_end {
1654            return Err(resource::CreateTextureViewError::TooManyArrayLayers {
1655                requested: array_layer_end,
1656                total: layer_end,
1657            });
1658        };
1659
1660        // https://gpuweb.github.io/gpuweb/#abstract-opdef-renderable-texture-view
1661        let render_extent = 'error: {
1662            if !resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1663                break 'error Err(TextureViewNotRenderableReason::Usage(resolved_usage));
1664            }
1665
1666            let allowed_view_dimensions = [
1667                TextureViewDimension::D2,
1668                TextureViewDimension::D2Array,
1669                TextureViewDimension::D3,
1670            ];
1671            if !allowed_view_dimensions.contains(&resolved_dimension) {
1672                break 'error Err(TextureViewNotRenderableReason::Dimension(
1673                    resolved_dimension,
1674                ));
1675            }
1676
1677            if resolved_mip_level_count != 1 {
1678                break 'error Err(TextureViewNotRenderableReason::MipLevelCount(
1679                    resolved_mip_level_count,
1680                ));
1681            }
1682
1683            if resolved_array_layer_count != 1
1684                && !(self.features.contains(wgt::Features::MULTIVIEW))
1685            {
1686                break 'error Err(TextureViewNotRenderableReason::ArrayLayerCount(
1687                    resolved_array_layer_count,
1688                ));
1689            }
1690
1691            if aspects != hal::FormatAspects::from(texture.desc.format) {
1692                break 'error Err(TextureViewNotRenderableReason::Aspects(aspects));
1693            }
1694
1695            Ok(texture
1696                .desc
1697                .compute_render_extent(desc.range.base_mip_level))
1698        };
1699
1700        // filter the usages based on the other criteria
1701        let usage = {
1702            let resolved_hal_usage = conv::map_texture_usage(
1703                resolved_usage,
1704                resolved_format.into(),
1705                format_features.flags,
1706            );
1707            let mask_copy = !(wgt::TextureUses::COPY_SRC | wgt::TextureUses::COPY_DST);
1708            let mask_dimension = match resolved_dimension {
1709                TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1710                    wgt::TextureUses::RESOURCE
1711                }
1712                TextureViewDimension::D3 => {
1713                    wgt::TextureUses::RESOURCE
1714                        | wgt::TextureUses::STORAGE_READ_ONLY
1715                        | wgt::TextureUses::STORAGE_WRITE_ONLY
1716                        | wgt::TextureUses::STORAGE_READ_WRITE
1717                }
1718                _ => wgt::TextureUses::all(),
1719            };
1720            let mask_mip_level = if resolved_mip_level_count == 1 {
1721                wgt::TextureUses::all()
1722            } else {
1723                wgt::TextureUses::RESOURCE
1724            };
1725            resolved_hal_usage & mask_copy & mask_dimension & mask_mip_level
1726        };
1727
1728        // use the combined depth-stencil format for the view
1729        let format = if resolved_format.is_depth_stencil_component(texture.desc.format) {
1730            texture.desc.format
1731        } else {
1732            resolved_format
1733        };
1734
1735        let resolved_range = wgt::ImageSubresourceRange {
1736            aspect: desc.range.aspect,
1737            base_mip_level: desc.range.base_mip_level,
1738            mip_level_count: Some(resolved_mip_level_count),
1739            base_array_layer: desc.range.base_array_layer,
1740            array_layer_count: Some(resolved_array_layer_count),
1741        };
1742
1743        let hal_desc = hal::TextureViewDescriptor {
1744            label: desc.label.to_hal(self.instance_flags),
1745            format,
1746            dimension: resolved_dimension,
1747            usage,
1748            range: resolved_range,
1749        };
1750
1751        let raw = unsafe { self.raw().create_texture_view(texture_raw, &hal_desc) }
1752            .map_err(|e| self.handle_hal_error(e))?;
1753
1754        let selector = TextureSelector {
1755            mips: desc.range.base_mip_level..mip_level_end,
1756            layers: desc.range.base_array_layer..array_layer_end,
1757        };
1758
1759        let view = TextureView {
1760            raw: Snatchable::new(raw),
1761            parent: texture.clone(),
1762            device: self.clone(),
1763            desc: resource::HalTextureViewDescriptor {
1764                texture_format: texture.desc.format,
1765                format: resolved_format,
1766                dimension: resolved_dimension,
1767                usage: resolved_usage,
1768                range: resolved_range,
1769            },
1770            format_features: texture.format_features,
1771            render_extent,
1772            samples: texture.desc.sample_count,
1773            selector,
1774            label: desc.label.to_string(),
1775            tracking_data: TrackingData::new(self.tracker_indices.texture_views.clone()),
1776        };
1777
1778        let view = Arc::new(view);
1779
1780        {
1781            let mut views = texture.views.lock();
1782            views.push(Arc::downgrade(&view));
1783        }
1784
1785        Ok(view)
1786    }
1787
1788    pub(crate) fn create_external_texture(
1789        self: &Arc<Self>,
1790        desc: &resource::ExternalTextureDescriptor,
1791        planes: &[Arc<TextureView>],
1792    ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
1793        use resource::CreateExternalTextureError;
1794        self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
1795        self.check_is_valid()?;
1796
1797        if desc.num_planes() != planes.len() {
1798            return Err(CreateExternalTextureError::IncorrectPlaneCount {
1799                format: desc.format,
1800                expected: desc.num_planes(),
1801                provided: planes.len(),
1802            });
1803        }
1804
1805        let planes = planes
1806            .iter()
1807            .enumerate()
1808            .map(|(i, plane)| {
1809                if plane.samples != 1 {
1810                    return Err(CreateExternalTextureError::InvalidPlaneMultisample(
1811                        plane.samples,
1812                    ));
1813                }
1814
1815                let sample_type = plane
1816                    .desc
1817                    .format
1818                    .sample_type(Some(plane.desc.range.aspect), Some(self.features))
1819                    .unwrap();
1820                if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
1821                    return Err(CreateExternalTextureError::InvalidPlaneSampleType {
1822                        format: plane.desc.format,
1823                        sample_type,
1824                    });
1825                }
1826
1827                if plane.desc.dimension != TextureViewDimension::D2 {
1828                    return Err(CreateExternalTextureError::InvalidPlaneDimension(
1829                        plane.desc.dimension,
1830                    ));
1831                }
1832
1833                let expected_components = match desc.format {
1834                    wgt::ExternalTextureFormat::Rgba => 4,
1835                    wgt::ExternalTextureFormat::Nv12 => match i {
1836                        0 => 1,
1837                        1 => 2,
1838                        _ => unreachable!(),
1839                    },
1840                    wgt::ExternalTextureFormat::Yu12 => 1,
1841                };
1842                if plane.desc.format.components() != expected_components {
1843                    return Err(CreateExternalTextureError::InvalidPlaneFormat {
1844                        format: desc.format,
1845                        plane: i,
1846                        expected: expected_components,
1847                        provided: plane.desc.format,
1848                    });
1849                }
1850
1851                plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
1852                Ok(plane.clone())
1853            })
1854            .collect::<Result<_, _>>()?;
1855
1856        let params_data = ExternalTextureParams::from_desc(desc);
1857        let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
1858        let params_desc = resource::BufferDescriptor {
1859            label: label.map(Cow::Owned),
1860            size: size_of_val(&params_data) as wgt::BufferAddress,
1861            usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
1862            mapped_at_creation: false,
1863        };
1864        let params = self.create_buffer(&params_desc)?;
1865        self.get_queue().unwrap().write_buffer(
1866            Fallible::Valid(params.clone()),
1867            0,
1868            bytemuck::bytes_of(&params_data),
1869        )?;
1870
1871        let external_texture = ExternalTexture {
1872            device: self.clone(),
1873            planes,
1874            params,
1875            label: desc.label.to_string(),
1876            tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
1877        };
1878        let external_texture = Arc::new(external_texture);
1879
1880        Ok(external_texture)
1881    }
1882
1883    pub(crate) fn create_sampler(
1884        self: &Arc<Self>,
1885        desc: &resource::SamplerDescriptor,
1886    ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
1887        self.check_is_valid()?;
1888
1889        if desc
1890            .address_modes
1891            .iter()
1892            .any(|am| am == &wgt::AddressMode::ClampToBorder)
1893        {
1894            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
1895        }
1896
1897        if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
1898            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
1899        }
1900
1901        if desc.lod_min_clamp < 0.0 {
1902            return Err(resource::CreateSamplerError::InvalidLodMinClamp(
1903                desc.lod_min_clamp,
1904            ));
1905        }
1906        if desc.lod_max_clamp < desc.lod_min_clamp {
1907            return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
1908                lod_min_clamp: desc.lod_min_clamp,
1909                lod_max_clamp: desc.lod_max_clamp,
1910            });
1911        }
1912
1913        if desc.anisotropy_clamp < 1 {
1914            return Err(resource::CreateSamplerError::InvalidAnisotropy(
1915                desc.anisotropy_clamp,
1916            ));
1917        }
1918
1919        if desc.anisotropy_clamp != 1 {
1920            if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
1921                return Err(
1922                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
1923                        filter_type: resource::SamplerFilterErrorType::MinFilter,
1924                        filter_mode: desc.min_filter,
1925                        anisotropic_clamp: desc.anisotropy_clamp,
1926                    },
1927                );
1928            }
1929            if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
1930                return Err(
1931                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
1932                        filter_type: resource::SamplerFilterErrorType::MagFilter,
1933                        filter_mode: desc.mag_filter,
1934                        anisotropic_clamp: desc.anisotropy_clamp,
1935                    },
1936                );
1937            }
1938            if !matches!(desc.mipmap_filter, wgt::FilterMode::Linear) {
1939                return Err(
1940                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
1941                        filter_type: resource::SamplerFilterErrorType::MipmapFilter,
1942                        filter_mode: desc.mipmap_filter,
1943                        anisotropic_clamp: desc.anisotropy_clamp,
1944                    },
1945                );
1946            }
1947        }
1948
1949        let anisotropy_clamp = if self
1950            .downlevel
1951            .flags
1952            .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
1953        {
1954            // Clamp anisotropy clamp to [1, 16] per the wgpu-hal interface
1955            desc.anisotropy_clamp.min(16)
1956        } else {
1957            // If it isn't supported, set this unconditionally to 1
1958            1
1959        };
1960
1961        //TODO: check for wgt::DownlevelFlags::COMPARISON_SAMPLERS
1962
1963        let hal_desc = hal::SamplerDescriptor {
1964            label: desc.label.to_hal(self.instance_flags),
1965            address_modes: desc.address_modes,
1966            mag_filter: desc.mag_filter,
1967            min_filter: desc.min_filter,
1968            mipmap_filter: desc.mipmap_filter,
1969            lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
1970            compare: desc.compare,
1971            anisotropy_clamp,
1972            border_color: desc.border_color,
1973        };
1974
1975        let raw = unsafe { self.raw().create_sampler(&hal_desc) }
1976            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1977
1978        let sampler = Sampler {
1979            raw: ManuallyDrop::new(raw),
1980            device: self.clone(),
1981            label: desc.label.to_string(),
1982            tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
1983            comparison: desc.compare.is_some(),
1984            filtering: desc.min_filter == wgt::FilterMode::Linear
1985                || desc.mag_filter == wgt::FilterMode::Linear
1986                || desc.mipmap_filter == wgt::FilterMode::Linear,
1987        };
1988
1989        let sampler = Arc::new(sampler);
1990
1991        Ok(sampler)
1992    }
1993
1994    pub(crate) fn create_shader_module<'a>(
1995        self: &Arc<Self>,
1996        desc: &pipeline::ShaderModuleDescriptor<'a>,
1997        source: pipeline::ShaderModuleSource<'a>,
1998    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
1999        self.check_is_valid()?;
2000
2001        let (module, source) = match source {
2002            #[cfg(feature = "wgsl")]
2003            pipeline::ShaderModuleSource::Wgsl(code) => {
2004                profiling::scope!("naga::front::wgsl::parse_str");
2005                let module = naga::front::wgsl::parse_str(&code).map_err(|inner| {
2006                    pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2007                        source: code.to_string(),
2008                        label: desc.label.as_ref().map(|l| l.to_string()),
2009                        inner: Box::new(inner),
2010                    })
2011                })?;
2012                (Cow::Owned(module), code.into_owned())
2013            }
2014            #[cfg(feature = "spirv")]
2015            pipeline::ShaderModuleSource::SpirV(spv, options) => {
2016                let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2017                profiling::scope!("naga::front::spv::Frontend");
2018                let module = parser.parse().map_err(|inner| {
2019                    pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2020                        source: String::new(),
2021                        label: desc.label.as_ref().map(|l| l.to_string()),
2022                        inner: Box::new(inner),
2023                    })
2024                })?;
2025                (Cow::Owned(module), String::new())
2026            }
2027            #[cfg(feature = "glsl")]
2028            pipeline::ShaderModuleSource::Glsl(code, options) => {
2029                let mut parser = naga::front::glsl::Frontend::default();
2030                profiling::scope!("naga::front::glsl::Frontend.parse");
2031                let module = parser.parse(&options, &code).map_err(|inner| {
2032                    pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2033                        source: code.to_string(),
2034                        label: desc.label.as_ref().map(|l| l.to_string()),
2035                        inner: Box::new(inner),
2036                    })
2037                })?;
2038                (Cow::Owned(module), code.into_owned())
2039            }
2040            pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2041            pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2042        };
2043        for (_, var) in module.global_variables.iter() {
2044            match var.binding {
2045                Some(br) if br.group >= self.limits.max_bind_groups => {
2046                    return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2047                        bind: br,
2048                        group: br.group,
2049                        limit: self.limits.max_bind_groups,
2050                    });
2051                }
2052                _ => continue,
2053            };
2054        }
2055
2056        profiling::scope!("naga::validate");
2057        let debug_source =
2058            if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2059                Some(hal::DebugSource {
2060                    file_name: Cow::Owned(
2061                        desc.label
2062                            .as_ref()
2063                            .map_or("shader".to_string(), |l| l.to_string()),
2064                    ),
2065                    source_code: Cow::Owned(source.clone()),
2066                })
2067            } else {
2068                None
2069            };
2070
2071        let info = create_validator(
2072            self.features,
2073            self.downlevel.flags,
2074            naga::valid::ValidationFlags::all(),
2075        )
2076        .validate(&module)
2077        .map_err(|inner| {
2078            pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2079                source,
2080                label: desc.label.as_ref().map(|l| l.to_string()),
2081                inner: Box::new(inner),
2082            })
2083        })?;
2084
2085        let interface = validation::Interface::new(&module, &info, self.limits.clone());
2086        let hal_shader = hal::ShaderInput::Naga(hal::NagaShader {
2087            module,
2088            info,
2089            debug_source,
2090        });
2091        let hal_desc = hal::ShaderModuleDescriptor {
2092            label: desc.label.to_hal(self.instance_flags),
2093            runtime_checks: desc.runtime_checks,
2094        };
2095        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2096            Ok(raw) => raw,
2097            Err(error) => {
2098                return Err(match error {
2099                    hal::ShaderError::Device(error) => {
2100                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2101                    }
2102                    hal::ShaderError::Compilation(ref msg) => {
2103                        log::error!("Shader error: {msg}");
2104                        pipeline::CreateShaderModuleError::Generation
2105                    }
2106                })
2107            }
2108        };
2109
2110        let module = pipeline::ShaderModule {
2111            raw: ManuallyDrop::new(raw),
2112            device: self.clone(),
2113            interface: Some(interface),
2114            label: desc.label.to_string(),
2115        };
2116
2117        let module = Arc::new(module);
2118
2119        Ok(module)
2120    }
2121
2122    #[allow(unused_unsafe)]
2123    pub(crate) unsafe fn create_shader_module_passthrough<'a>(
2124        self: &Arc<Self>,
2125        descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2126    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2127        self.check_is_valid()?;
2128        let hal_shader = match descriptor {
2129            pipeline::ShaderModuleDescriptorPassthrough::SpirV(inner) => {
2130                self.require_features(wgt::Features::SPIRV_SHADER_PASSTHROUGH)?;
2131                hal::ShaderInput::SpirV(&inner.source)
2132            }
2133            pipeline::ShaderModuleDescriptorPassthrough::Msl(inner) => {
2134                self.require_features(wgt::Features::MSL_SHADER_PASSTHROUGH)?;
2135                hal::ShaderInput::Msl {
2136                    shader: inner.source.to_string(),
2137                    entry_point: inner.entry_point.to_string(),
2138                    num_workgroups: inner.num_workgroups,
2139                }
2140            }
2141            pipeline::ShaderModuleDescriptorPassthrough::Dxil(inner) => {
2142                self.require_features(wgt::Features::HLSL_DXIL_SHADER_PASSTHROUGH)?;
2143                hal::ShaderInput::Dxil {
2144                    shader: inner.source,
2145                    entry_point: inner.entry_point.clone(),
2146                    num_workgroups: inner.num_workgroups,
2147                }
2148            }
2149            pipeline::ShaderModuleDescriptorPassthrough::Hlsl(inner) => {
2150                self.require_features(wgt::Features::HLSL_DXIL_SHADER_PASSTHROUGH)?;
2151                hal::ShaderInput::Hlsl {
2152                    shader: inner.source,
2153                    entry_point: inner.entry_point.clone(),
2154                    num_workgroups: inner.num_workgroups,
2155                }
2156            }
2157        };
2158
2159        let hal_desc = hal::ShaderModuleDescriptor {
2160            label: descriptor.label().to_hal(self.instance_flags),
2161            runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2162        };
2163
2164        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2165            Ok(raw) => raw,
2166            Err(error) => {
2167                return Err(match error {
2168                    hal::ShaderError::Device(error) => {
2169                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2170                    }
2171                    hal::ShaderError::Compilation(ref msg) => {
2172                        log::error!("Shader error: {msg}");
2173                        pipeline::CreateShaderModuleError::Generation
2174                    }
2175                })
2176            }
2177        };
2178
2179        let module = pipeline::ShaderModule {
2180            raw: ManuallyDrop::new(raw),
2181            device: self.clone(),
2182            interface: None,
2183            label: descriptor.label().to_string(),
2184        };
2185
2186        Ok(Arc::new(module))
2187    }
2188
2189    pub(crate) fn create_command_encoder(
2190        self: &Arc<Self>,
2191        label: &crate::Label,
2192    ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2193        self.check_is_valid()?;
2194
2195        let queue = self.get_queue().unwrap();
2196
2197        let encoder = self
2198            .command_allocator
2199            .acquire_encoder(self.raw(), queue.raw())
2200            .map_err(|e| self.handle_hal_error(e))?;
2201
2202        let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2203
2204        let cmd_enc = Arc::new(cmd_enc);
2205
2206        Ok(cmd_enc)
2207    }
2208
2209    /// Generate information about late-validated buffer bindings for pipelines.
2210    //TODO: should this be combined with `get_introspection_bind_group_layouts` in some way?
2211    fn make_late_sized_buffer_groups(
2212        shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2213        layout: &binding_model::PipelineLayout,
2214    ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2215        // Given the shader-required binding sizes and the pipeline layout,
2216        // return the filtered list of them in the layout order,
2217        // removing those with given `min_binding_size`.
2218        layout
2219            .bind_group_layouts
2220            .iter()
2221            .enumerate()
2222            .map(|(group_index, bgl)| pipeline::LateSizedBufferGroup {
2223                shader_sizes: bgl
2224                    .entries
2225                    .values()
2226                    .filter_map(|entry| match entry.ty {
2227                        wgt::BindingType::Buffer {
2228                            min_binding_size: None,
2229                            ..
2230                        } => {
2231                            let rb = naga::ResourceBinding {
2232                                group: group_index as u32,
2233                                binding: entry.binding,
2234                            };
2235                            let shader_size =
2236                                shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
2237                            Some(shader_size)
2238                        }
2239                        _ => None,
2240                    })
2241                    .collect(),
2242            })
2243            .collect()
2244    }
2245
2246    pub(crate) fn create_bind_group_layout(
2247        self: &Arc<Self>,
2248        label: &crate::Label,
2249        entry_map: bgl::EntryMap,
2250        origin: bgl::Origin,
2251    ) -> Result<Arc<BindGroupLayout>, binding_model::CreateBindGroupLayoutError> {
2252        #[derive(PartialEq)]
2253        enum WritableStorage {
2254            Yes,
2255            No,
2256        }
2257
2258        for entry in entry_map.values() {
2259            use wgt::BindingType as Bt;
2260
2261            let mut required_features = wgt::Features::empty();
2262            let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
2263            let (array_feature, writable_storage) = match entry.ty {
2264                Bt::Buffer {
2265                    ty: wgt::BufferBindingType::Uniform,
2266                    has_dynamic_offset: false,
2267                    min_binding_size: _,
2268                } => (
2269                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2270                    WritableStorage::No,
2271                ),
2272                Bt::Buffer {
2273                    ty: wgt::BufferBindingType::Uniform,
2274                    has_dynamic_offset: true,
2275                    min_binding_size: _,
2276                } => (
2277                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2278                    WritableStorage::No,
2279                ),
2280                Bt::Buffer {
2281                    ty: wgt::BufferBindingType::Storage { read_only },
2282                    ..
2283                } => (
2284                    Some(
2285                        wgt::Features::BUFFER_BINDING_ARRAY
2286                            | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2287                    ),
2288                    match read_only {
2289                        true => WritableStorage::No,
2290                        false => WritableStorage::Yes,
2291                    },
2292                ),
2293                Bt::Sampler { .. } => (
2294                    Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2295                    WritableStorage::No,
2296                ),
2297                Bt::Texture {
2298                    multisampled: true,
2299                    sample_type: TextureSampleType::Float { filterable: true },
2300                    ..
2301                } => {
2302                    return Err(binding_model::CreateBindGroupLayoutError::Entry {
2303                        binding: entry.binding,
2304                        error:
2305                            BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
2306                    });
2307                }
2308                Bt::Texture {
2309                    multisampled,
2310                    view_dimension,
2311                    ..
2312                } => {
2313                    if multisampled && view_dimension != TextureViewDimension::D2 {
2314                        return Err(binding_model::CreateBindGroupLayoutError::Entry {
2315                            binding: entry.binding,
2316                            error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
2317                        });
2318                    }
2319
2320                    (
2321                        Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2322                        WritableStorage::No,
2323                    )
2324                }
2325                Bt::StorageTexture {
2326                    access,
2327                    view_dimension,
2328                    format: _,
2329                } => {
2330                    match view_dimension {
2331                        TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2332                            return Err(binding_model::CreateBindGroupLayoutError::Entry {
2333                                binding: entry.binding,
2334                                error: BindGroupLayoutEntryError::StorageTextureCube,
2335                            })
2336                        }
2337                        _ => (),
2338                    }
2339                    match access {
2340                        wgt::StorageTextureAccess::Atomic
2341                            if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
2342                        {
2343                            return Err(binding_model::CreateBindGroupLayoutError::Entry {
2344                                binding: entry.binding,
2345                                error: BindGroupLayoutEntryError::StorageTextureAtomic,
2346                            });
2347                        }
2348                        _ => (),
2349                    }
2350                    (
2351                        Some(
2352                            wgt::Features::TEXTURE_BINDING_ARRAY
2353                                | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2354                        ),
2355                        match access {
2356                            wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
2357                            wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
2358                            wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
2359                            wgt::StorageTextureAccess::Atomic => {
2360                                required_features |= wgt::Features::TEXTURE_ATOMIC;
2361                                WritableStorage::Yes
2362                            }
2363                        },
2364                    )
2365                }
2366                Bt::AccelerationStructure { vertex_return } => {
2367                    self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)
2368                        .map_err(|e| binding_model::CreateBindGroupLayoutError::Entry {
2369                            binding: entry.binding,
2370                            error: e.into(),
2371                        })?;
2372                    if vertex_return {
2373                        self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
2374                            .map_err(|e| binding_model::CreateBindGroupLayoutError::Entry {
2375                                binding: entry.binding,
2376                                error: e.into(),
2377                            })?;
2378                    }
2379
2380                    (None, WritableStorage::No)
2381                }
2382                Bt::ExternalTexture => {
2383                    self.require_features(wgt::Features::EXTERNAL_TEXTURE)
2384                        .map_err(|e| binding_model::CreateBindGroupLayoutError::Entry {
2385                            binding: entry.binding,
2386                            error: e.into(),
2387                        })?;
2388                    (None, WritableStorage::No)
2389                }
2390            };
2391
2392            // Validate the count parameter
2393            if entry.count.is_some() {
2394                required_features |= array_feature
2395                    .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
2396                    .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry {
2397                        binding: entry.binding,
2398                        error,
2399                    })?;
2400            }
2401
2402            if entry.visibility.contains_unknown_bits() {
2403                return Err(
2404                    binding_model::CreateBindGroupLayoutError::InvalidVisibility(entry.visibility),
2405                );
2406            }
2407
2408            if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
2409                if writable_storage == WritableStorage::Yes {
2410                    required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
2411                }
2412                if let Bt::Buffer {
2413                    ty: wgt::BufferBindingType::Storage { .. },
2414                    ..
2415                } = entry.ty
2416                {
2417                    required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
2418                }
2419            }
2420            if writable_storage == WritableStorage::Yes
2421                && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
2422            {
2423                required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
2424            }
2425
2426            self.require_features(required_features)
2427                .map_err(BindGroupLayoutEntryError::MissingFeatures)
2428                .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry {
2429                    binding: entry.binding,
2430                    error,
2431                })?;
2432            self.require_downlevel_flags(required_downlevel_flags)
2433                .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
2434                .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry {
2435                    binding: entry.binding,
2436                    error,
2437                })?;
2438        }
2439
2440        let bgl_flags = conv::bind_group_layout_flags(self.features);
2441
2442        let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
2443        let hal_desc = hal::BindGroupLayoutDescriptor {
2444            label: label.to_hal(self.instance_flags),
2445            flags: bgl_flags,
2446            entries: &hal_bindings,
2447        };
2448
2449        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
2450        for entry in entry_map.values() {
2451            count_validator.add_binding(entry);
2452        }
2453        // If a single bind group layout violates limits, the pipeline layout is
2454        // definitely going to violate limits too, lets catch it now.
2455        count_validator
2456            .validate(&self.limits)
2457            .map_err(binding_model::CreateBindGroupLayoutError::TooManyBindings)?;
2458
2459        // Validate that binding arrays don't conflict with dynamic offsets.
2460        count_validator.validate_binding_arrays()?;
2461
2462        let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
2463            .map_err(|e| self.handle_hal_error(e))?;
2464
2465        let bgl = BindGroupLayout {
2466            raw: ManuallyDrop::new(raw),
2467            device: self.clone(),
2468            entries: entry_map,
2469            origin,
2470            exclusive_pipeline: OnceCellOrLock::new(),
2471            binding_count_validator: count_validator,
2472            label: label.to_string(),
2473        };
2474
2475        let bgl = Arc::new(bgl);
2476
2477        Ok(bgl)
2478    }
2479
2480    fn create_buffer_binding<'a>(
2481        &self,
2482        bb: &'a binding_model::ResolvedBufferBinding,
2483        binding: u32,
2484        decl: &wgt::BindGroupLayoutEntry,
2485        used_buffer_ranges: &mut Vec<BufferInitTrackerAction>,
2486        dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
2487        late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
2488        used: &mut BindGroupStates,
2489        snatch_guard: &'a SnatchGuard<'a>,
2490    ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer>, binding_model::CreateBindGroupError>
2491    {
2492        use crate::binding_model::CreateBindGroupError as Error;
2493
2494        let (binding_ty, dynamic, min_size) = match decl.ty {
2495            wgt::BindingType::Buffer {
2496                ty,
2497                has_dynamic_offset,
2498                min_binding_size,
2499            } => (ty, has_dynamic_offset, min_binding_size),
2500            _ => {
2501                return Err(Error::WrongBindingType {
2502                    binding,
2503                    actual: decl.ty,
2504                    expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
2505                })
2506            }
2507        };
2508
2509        let (pub_usage, internal_use, range_limit) = match binding_ty {
2510            wgt::BufferBindingType::Uniform => (
2511                wgt::BufferUsages::UNIFORM,
2512                wgt::BufferUses::UNIFORM,
2513                self.limits.max_uniform_buffer_binding_size,
2514            ),
2515            wgt::BufferBindingType::Storage { read_only } => (
2516                wgt::BufferUsages::STORAGE,
2517                if read_only {
2518                    wgt::BufferUses::STORAGE_READ_ONLY
2519                } else {
2520                    wgt::BufferUses::STORAGE_READ_WRITE
2521                },
2522                self.limits.max_storage_buffer_binding_size,
2523            ),
2524        };
2525
2526        let (align, align_limit_name) =
2527            binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
2528        if bb.offset % align as u64 != 0 {
2529            return Err(Error::UnalignedBufferOffset(
2530                bb.offset,
2531                align_limit_name,
2532                align,
2533            ));
2534        }
2535
2536        let buffer = &bb.buffer;
2537
2538        used.buffers.insert_single(buffer.clone(), internal_use);
2539
2540        buffer.same_device(self)?;
2541
2542        buffer.check_usage(pub_usage)?;
2543
2544        let (bb, bind_size) = buffer.binding(bb.offset, bb.size, snatch_guard)?;
2545
2546        if matches!(binding_ty, wgt::BufferBindingType::Storage { .. }) {
2547            let storage_buf_size_alignment = 4;
2548
2549            let aligned = bind_size % u64::from(storage_buf_size_alignment) == 0;
2550            if !aligned {
2551                return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
2552                    alignment: storage_buf_size_alignment,
2553                    size: bind_size,
2554                });
2555            }
2556        }
2557
2558        let bind_end = bb.offset + bind_size;
2559
2560        if bind_size > range_limit as u64 {
2561            return Err(Error::BufferRangeTooLarge {
2562                binding,
2563                given: bind_size as u32,
2564                limit: range_limit,
2565            });
2566        }
2567
2568        // Record binding info for validating dynamic offsets
2569        if dynamic {
2570            dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
2571                binding_idx: binding,
2572                buffer_size: buffer.size,
2573                binding_range: bb.offset..bind_end,
2574                maximum_dynamic_offset: buffer.size - bind_end,
2575                binding_type: binding_ty,
2576            });
2577        }
2578
2579        if let Some(non_zero) = min_size {
2580            let min_size = non_zero.get();
2581            if min_size > bind_size {
2582                return Err(Error::BindingSizeTooSmall {
2583                    buffer: buffer.error_ident(),
2584                    actual: bind_size,
2585                    min: min_size,
2586                });
2587            }
2588        } else {
2589            let late_size = wgt::BufferSize::new(bind_size)
2590                .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
2591            late_buffer_binding_sizes.insert(binding, late_size);
2592        }
2593
2594        // This was checked against the device's alignment requirements above,
2595        // which should always be a multiple of `COPY_BUFFER_ALIGNMENT`.
2596        assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
2597
2598        // `wgpu_hal` only restricts shader access to bound buffer regions with
2599        // a certain resolution. For the sake of lazy initialization, round up
2600        // the size of the bound range to reflect how much of the buffer is
2601        // actually going to be visible to the shader.
2602        let bounds_check_alignment =
2603            binding_model::buffer_binding_type_bounds_check_alignment(&self.alignments, binding_ty);
2604        let visible_size = align_to(bind_size, bounds_check_alignment);
2605
2606        used_buffer_ranges.extend(buffer.initialization_status.read().create_action(
2607            buffer,
2608            bb.offset..bb.offset + visible_size,
2609            MemoryInitKind::NeedsInitializedMemory,
2610        ));
2611
2612        Ok(bb)
2613    }
2614
2615    fn create_sampler_binding<'a>(
2616        &self,
2617        used: &mut BindGroupStates,
2618        binding: u32,
2619        decl: &wgt::BindGroupLayoutEntry,
2620        sampler: &'a Arc<Sampler>,
2621    ) -> Result<&'a dyn hal::DynSampler, binding_model::CreateBindGroupError> {
2622        use crate::binding_model::CreateBindGroupError as Error;
2623
2624        used.samplers.insert_single(sampler.clone());
2625
2626        sampler.same_device(self)?;
2627
2628        match decl.ty {
2629            wgt::BindingType::Sampler(ty) => {
2630                let (allowed_filtering, allowed_comparison) = match ty {
2631                    wgt::SamplerBindingType::Filtering => (None, false),
2632                    wgt::SamplerBindingType::NonFiltering => (Some(false), false),
2633                    wgt::SamplerBindingType::Comparison => (None, true),
2634                };
2635                if let Some(allowed_filtering) = allowed_filtering {
2636                    if allowed_filtering != sampler.filtering {
2637                        return Err(Error::WrongSamplerFiltering {
2638                            binding,
2639                            layout_flt: allowed_filtering,
2640                            sampler_flt: sampler.filtering,
2641                        });
2642                    }
2643                }
2644                if allowed_comparison != sampler.comparison {
2645                    return Err(Error::WrongSamplerComparison {
2646                        binding,
2647                        layout_cmp: allowed_comparison,
2648                        sampler_cmp: sampler.comparison,
2649                    });
2650                }
2651            }
2652            _ => {
2653                return Err(Error::WrongBindingType {
2654                    binding,
2655                    actual: decl.ty,
2656                    expected: "Sampler",
2657                })
2658            }
2659        }
2660
2661        Ok(sampler.raw())
2662    }
2663
2664    fn create_texture_binding<'a>(
2665        &self,
2666        binding: u32,
2667        decl: &wgt::BindGroupLayoutEntry,
2668        view: &'a Arc<TextureView>,
2669        used: &mut BindGroupStates,
2670        used_texture_ranges: &mut Vec<TextureInitTrackerAction>,
2671        snatch_guard: &'a SnatchGuard<'a>,
2672    ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, binding_model::CreateBindGroupError>
2673    {
2674        view.same_device(self)?;
2675
2676        let internal_use = self.texture_use_parameters(
2677            binding,
2678            decl,
2679            view,
2680            "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
2681        )?;
2682
2683        used.views.insert_single(view.clone(), internal_use);
2684
2685        let texture = &view.parent;
2686
2687        used_texture_ranges.push(TextureInitTrackerAction {
2688            texture: texture.clone(),
2689            range: TextureInitRange {
2690                mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
2691                layer_range: view
2692                    .desc
2693                    .range
2694                    .layer_range(texture.desc.array_layer_count()),
2695            },
2696            kind: MemoryInitKind::NeedsInitializedMemory,
2697        });
2698
2699        Ok(hal::TextureBinding {
2700            view: view.try_raw(snatch_guard)?,
2701            usage: internal_use,
2702        })
2703    }
2704
2705    fn create_tlas_binding<'a>(
2706        self: &Arc<Self>,
2707        used: &mut BindGroupStates,
2708        binding: u32,
2709        decl: &wgt::BindGroupLayoutEntry,
2710        tlas: &'a Arc<Tlas>,
2711        snatch_guard: &'a SnatchGuard<'a>,
2712    ) -> Result<&'a dyn hal::DynAccelerationStructure, binding_model::CreateBindGroupError> {
2713        use crate::binding_model::CreateBindGroupError as Error;
2714
2715        used.acceleration_structures.insert_single(tlas.clone());
2716
2717        tlas.same_device(self)?;
2718
2719        match decl.ty {
2720            wgt::BindingType::AccelerationStructure { vertex_return } => {
2721                if vertex_return
2722                    && !tlas.flags.contains(
2723                        wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
2724                    )
2725                {
2726                    return Err(Error::MissingTLASVertexReturn { binding });
2727                }
2728            }
2729            _ => {
2730                return Err(Error::WrongBindingType {
2731                    binding,
2732                    actual: decl.ty,
2733                    expected: "Tlas",
2734                });
2735            }
2736        }
2737
2738        Ok(tlas.try_raw(snatch_guard)?)
2739    }
2740
2741    fn create_external_texture_binding<'a>(
2742        &'a self,
2743        binding: u32,
2744        decl: &wgt::BindGroupLayoutEntry,
2745        external_texture: &'a Arc<ExternalTexture>,
2746        used: &mut BindGroupStates,
2747        snatch_guard: &'a SnatchGuard,
2748    ) -> Result<
2749        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
2750        binding_model::CreateBindGroupError,
2751    > {
2752        use crate::binding_model::CreateBindGroupError as Error;
2753
2754        external_texture.same_device(self)?;
2755
2756        used.external_textures
2757            .insert_single(external_texture.clone());
2758
2759        match decl.ty {
2760            wgt::BindingType::ExternalTexture => {}
2761            _ => {
2762                return Err(Error::WrongBindingType {
2763                    binding,
2764                    actual: decl.ty,
2765                    expected: "ExternalTexture",
2766                });
2767            }
2768        }
2769
2770        let planes = (0..3)
2771            .map(|i| {
2772                // We always need 3 bindings. If we have fewer than 3 planes
2773                // just bind plane 0 multiple times. The shader will only
2774                // sample from valid planes anyway.
2775                let plane = external_texture
2776                    .planes
2777                    .get(i)
2778                    .unwrap_or(&external_texture.planes[0]);
2779                let internal_use = wgt::TextureUses::RESOURCE;
2780                used.views.insert_single(plane.clone(), internal_use);
2781                let view = plane.try_raw(snatch_guard)?;
2782                Ok(hal::TextureBinding {
2783                    view,
2784                    usage: internal_use,
2785                })
2786            })
2787            // We can remove this intermediate Vec by using
2788            // array::try_from_fn() above, once it stabilizes.
2789            .collect::<Result<Vec<_>, Error>>()?;
2790        let planes = planes.try_into().unwrap();
2791
2792        used.buffers
2793            .insert_single(external_texture.params.clone(), wgt::BufferUses::UNIFORM);
2794        let params = external_texture.params.binding(0, None, snatch_guard)?.0;
2795
2796        Ok(hal::ExternalTextureBinding { planes, params })
2797    }
2798
2799    fn create_external_texture_binding_from_view<'a>(
2800        &'a self,
2801        binding: u32,
2802        decl: &wgt::BindGroupLayoutEntry,
2803        view: &'a Arc<TextureView>,
2804        used: &mut BindGroupStates,
2805        snatch_guard: &'a SnatchGuard,
2806    ) -> Result<
2807        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
2808        binding_model::CreateBindGroupError,
2809    > {
2810        use crate::binding_model::CreateBindGroupError as Error;
2811
2812        view.same_device(self)?;
2813
2814        let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
2815        used.views.insert_single(view.clone(), internal_use);
2816
2817        match decl.ty {
2818            wgt::BindingType::ExternalTexture => {}
2819            _ => {
2820                return Err(Error::WrongBindingType {
2821                    binding,
2822                    actual: decl.ty,
2823                    expected: "ExternalTexture",
2824                });
2825            }
2826        }
2827
2828        // We need 3 bindings, so just repeat the same texture view 3 times.
2829        let planes = [
2830            hal::TextureBinding {
2831                view: view.try_raw(snatch_guard)?,
2832                usage: internal_use,
2833            },
2834            hal::TextureBinding {
2835                view: view.try_raw(snatch_guard)?,
2836                usage: internal_use,
2837            },
2838            hal::TextureBinding {
2839                view: view.try_raw(snatch_guard)?,
2840                usage: internal_use,
2841            },
2842        ];
2843        let params = hal::BufferBinding::new_unchecked(
2844            self.default_external_texture_params_buffer.as_ref(),
2845            0,
2846            None,
2847        );
2848
2849        Ok(hal::ExternalTextureBinding { planes, params })
2850    }
2851
2852    // This function expects the provided bind group layout to be resolved
2853    // (not passing a duplicate) beforehand.
2854    pub(crate) fn create_bind_group(
2855        self: &Arc<Self>,
2856        desc: binding_model::ResolvedBindGroupDescriptor,
2857    ) -> Result<Arc<BindGroup>, binding_model::CreateBindGroupError> {
2858        use crate::binding_model::{CreateBindGroupError as Error, ResolvedBindingResource as Br};
2859
2860        let layout = desc.layout;
2861
2862        self.check_is_valid()?;
2863        layout.same_device(self)?;
2864
2865        {
2866            // Check that the number of entries in the descriptor matches
2867            // the number of entries in the layout.
2868            let actual = desc.entries.len();
2869            let expected = layout.entries.len();
2870            if actual != expected {
2871                return Err(Error::BindingsNumMismatch { expected, actual });
2872            }
2873        }
2874
2875        // TODO: arrayvec/smallvec, or re-use allocations
2876        // Record binding info for dynamic offset validation
2877        let mut dynamic_binding_info = Vec::new();
2878        // Map of binding -> shader reflected size
2879        //Note: we can't collect into a vector right away because
2880        // it needs to be in BGL iteration order, not BG entry order.
2881        let mut late_buffer_binding_sizes = FastHashMap::default();
2882        // fill out the descriptors
2883        let mut used = BindGroupStates::new();
2884
2885        let mut used_buffer_ranges = Vec::new();
2886        let mut used_texture_ranges = Vec::new();
2887        let mut hal_entries = Vec::with_capacity(desc.entries.len());
2888        let mut hal_buffers = Vec::new();
2889        let mut hal_samplers = Vec::new();
2890        let mut hal_textures = Vec::new();
2891        let mut hal_tlas_s = Vec::new();
2892        let mut hal_external_textures = Vec::new();
2893        let snatch_guard = self.snatchable_lock.read();
2894        for entry in desc.entries.iter() {
2895            let binding = entry.binding;
2896            // Find the corresponding declaration in the layout
2897            let decl = layout
2898                .entries
2899                .get(binding)
2900                .ok_or(Error::MissingBindingDeclaration(binding))?;
2901            let (res_index, count) = match entry.resource {
2902                Br::Buffer(ref bb) => {
2903                    let bb = self.create_buffer_binding(
2904                        bb,
2905                        binding,
2906                        decl,
2907                        &mut used_buffer_ranges,
2908                        &mut dynamic_binding_info,
2909                        &mut late_buffer_binding_sizes,
2910                        &mut used,
2911                        &snatch_guard,
2912                    )?;
2913
2914                    let res_index = hal_buffers.len();
2915                    hal_buffers.push(bb);
2916                    (res_index, 1)
2917                }
2918                Br::BufferArray(ref bindings_array) => {
2919                    let num_bindings = bindings_array.len();
2920                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
2921
2922                    let res_index = hal_buffers.len();
2923                    for bb in bindings_array.iter() {
2924                        let bb = self.create_buffer_binding(
2925                            bb,
2926                            binding,
2927                            decl,
2928                            &mut used_buffer_ranges,
2929                            &mut dynamic_binding_info,
2930                            &mut late_buffer_binding_sizes,
2931                            &mut used,
2932                            &snatch_guard,
2933                        )?;
2934                        hal_buffers.push(bb);
2935                    }
2936                    (res_index, num_bindings)
2937                }
2938                Br::Sampler(ref sampler) => {
2939                    let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
2940
2941                    let res_index = hal_samplers.len();
2942                    hal_samplers.push(sampler);
2943                    (res_index, 1)
2944                }
2945                Br::SamplerArray(ref samplers) => {
2946                    let num_bindings = samplers.len();
2947                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
2948
2949                    let res_index = hal_samplers.len();
2950                    for sampler in samplers.iter() {
2951                        let sampler =
2952                            self.create_sampler_binding(&mut used, binding, decl, sampler)?;
2953
2954                        hal_samplers.push(sampler);
2955                    }
2956
2957                    (res_index, num_bindings)
2958                }
2959                Br::TextureView(ref view) => match decl.ty {
2960                    wgt::BindingType::ExternalTexture => {
2961                        let et = self.create_external_texture_binding_from_view(
2962                            binding,
2963                            decl,
2964                            view,
2965                            &mut used,
2966                            &snatch_guard,
2967                        )?;
2968                        let res_index = hal_external_textures.len();
2969                        hal_external_textures.push(et);
2970                        (res_index, 1)
2971                    }
2972                    _ => {
2973                        let tb = self.create_texture_binding(
2974                            binding,
2975                            decl,
2976                            view,
2977                            &mut used,
2978                            &mut used_texture_ranges,
2979                            &snatch_guard,
2980                        )?;
2981                        let res_index = hal_textures.len();
2982                        hal_textures.push(tb);
2983                        (res_index, 1)
2984                    }
2985                },
2986                Br::TextureViewArray(ref views) => {
2987                    let num_bindings = views.len();
2988                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
2989
2990                    let res_index = hal_textures.len();
2991                    for view in views.iter() {
2992                        let tb = self.create_texture_binding(
2993                            binding,
2994                            decl,
2995                            view,
2996                            &mut used,
2997                            &mut used_texture_ranges,
2998                            &snatch_guard,
2999                        )?;
3000
3001                        hal_textures.push(tb);
3002                    }
3003
3004                    (res_index, num_bindings)
3005                }
3006                Br::AccelerationStructure(ref tlas) => {
3007                    let tlas =
3008                        self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
3009                    let res_index = hal_tlas_s.len();
3010                    hal_tlas_s.push(tlas);
3011                    (res_index, 1)
3012                }
3013                Br::ExternalTexture(ref et) => {
3014                    let et = self.create_external_texture_binding(
3015                        binding,
3016                        decl,
3017                        et,
3018                        &mut used,
3019                        &snatch_guard,
3020                    )?;
3021                    let res_index = hal_external_textures.len();
3022                    hal_external_textures.push(et);
3023                    (res_index, 1)
3024                }
3025            };
3026
3027            hal_entries.push(hal::BindGroupEntry {
3028                binding,
3029                resource_index: res_index as u32,
3030                count: count as u32,
3031            });
3032        }
3033
3034        used.optimize();
3035
3036        hal_entries.sort_by_key(|entry| entry.binding);
3037        for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
3038            if a.binding == b.binding {
3039                return Err(Error::DuplicateBinding(a.binding));
3040            }
3041        }
3042        let hal_desc = hal::BindGroupDescriptor {
3043            label: desc.label.to_hal(self.instance_flags),
3044            layout: layout.raw(),
3045            entries: &hal_entries,
3046            buffers: &hal_buffers,
3047            samplers: &hal_samplers,
3048            textures: &hal_textures,
3049            acceleration_structures: &hal_tlas_s,
3050            external_textures: &hal_external_textures,
3051        };
3052        let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
3053            .map_err(|e| self.handle_hal_error(e))?;
3054
3055        // collect in the order of BGL iteration
3056        let late_buffer_binding_sizes = layout
3057            .entries
3058            .indices()
3059            .flat_map(|binding| late_buffer_binding_sizes.get(&binding).cloned())
3060            .collect();
3061
3062        let bind_group = BindGroup {
3063            raw: Snatchable::new(raw),
3064            device: self.clone(),
3065            layout,
3066            label: desc.label.to_string(),
3067            tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
3068            used,
3069            used_buffer_ranges,
3070            used_texture_ranges,
3071            dynamic_binding_info,
3072            late_buffer_binding_sizes,
3073        };
3074
3075        let bind_group = Arc::new(bind_group);
3076
3077        let weak_ref = Arc::downgrade(&bind_group);
3078        for range in &bind_group.used_texture_ranges {
3079            let mut bind_groups = range.texture.bind_groups.lock();
3080            bind_groups.push(weak_ref.clone());
3081        }
3082        for range in &bind_group.used_buffer_ranges {
3083            let mut bind_groups = range.buffer.bind_groups.lock();
3084            bind_groups.push(weak_ref.clone());
3085        }
3086
3087        Ok(bind_group)
3088    }
3089
3090    fn check_array_binding(
3091        features: wgt::Features,
3092        count: Option<NonZeroU32>,
3093        num_bindings: usize,
3094    ) -> Result<(), binding_model::CreateBindGroupError> {
3095        use super::binding_model::CreateBindGroupError as Error;
3096
3097        if let Some(count) = count {
3098            let count = count.get() as usize;
3099            if count < num_bindings {
3100                return Err(Error::BindingArrayPartialLengthMismatch {
3101                    actual: num_bindings,
3102                    expected: count,
3103                });
3104            }
3105            if count != num_bindings
3106                && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
3107            {
3108                return Err(Error::BindingArrayLengthMismatch {
3109                    actual: num_bindings,
3110                    expected: count,
3111                });
3112            }
3113            if num_bindings == 0 {
3114                return Err(Error::BindingArrayZeroLength);
3115            }
3116        } else {
3117            return Err(Error::SingleBindingExpected);
3118        };
3119
3120        Ok(())
3121    }
3122
3123    fn texture_use_parameters(
3124        &self,
3125        binding: u32,
3126        decl: &wgt::BindGroupLayoutEntry,
3127        view: &TextureView,
3128        expected: &'static str,
3129    ) -> Result<wgt::TextureUses, binding_model::CreateBindGroupError> {
3130        use crate::binding_model::CreateBindGroupError as Error;
3131        if view
3132            .desc
3133            .aspects()
3134            .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
3135        {
3136            return Err(Error::DepthStencilAspect);
3137        }
3138        match decl.ty {
3139            wgt::BindingType::Texture {
3140                sample_type,
3141                view_dimension,
3142                multisampled,
3143            } => {
3144                use wgt::TextureSampleType as Tst;
3145                if multisampled != (view.samples != 1) {
3146                    return Err(Error::InvalidTextureMultisample {
3147                        binding,
3148                        layout_multisampled: multisampled,
3149                        view_samples: view.samples,
3150                    });
3151                }
3152                let compat_sample_type = view
3153                    .desc
3154                    .format
3155                    .sample_type(Some(view.desc.range.aspect), Some(self.features))
3156                    .unwrap();
3157                match (sample_type, compat_sample_type) {
3158                    (Tst::Uint, Tst::Uint) |
3159                        (Tst::Sint, Tst::Sint) |
3160                        (Tst::Depth, Tst::Depth) |
3161                        // if we expect non-filterable, accept anything float
3162                        (Tst::Float { filterable: false }, Tst::Float { .. }) |
3163                        // if we expect filterable, require it
3164                        (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
3165                        // if we expect non-filterable, also accept depth
3166                        (Tst::Float { filterable: false }, Tst::Depth) => {}
3167                    // if we expect filterable, also accept Float that is defined as
3168                    // unfilterable if filterable feature is explicitly enabled (only hit
3169                    // if wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES is
3170                    // enabled)
3171                    (Tst::Float { filterable: true }, Tst::Float { .. }) if view.format_features.flags.contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
3172                    _ => {
3173                        return Err(Error::InvalidTextureSampleType {
3174                            binding,
3175                            layout_sample_type: sample_type,
3176                            view_format: view.desc.format,
3177                            view_sample_type: compat_sample_type,
3178                        })
3179                    }
3180                }
3181                if view_dimension != view.desc.dimension {
3182                    return Err(Error::InvalidTextureDimension {
3183                        binding,
3184                        layout_dimension: view_dimension,
3185                        view_dimension: view.desc.dimension,
3186                    });
3187                }
3188                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3189                Ok(wgt::TextureUses::RESOURCE)
3190            }
3191            wgt::BindingType::StorageTexture {
3192                access,
3193                format,
3194                view_dimension,
3195            } => {
3196                if format != view.desc.format {
3197                    return Err(Error::InvalidStorageTextureFormat {
3198                        binding,
3199                        layout_format: format,
3200                        view_format: view.desc.format,
3201                    });
3202                }
3203                if view_dimension != view.desc.dimension {
3204                    return Err(Error::InvalidTextureDimension {
3205                        binding,
3206                        layout_dimension: view_dimension,
3207                        view_dimension: view.desc.dimension,
3208                    });
3209                }
3210
3211                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3212                if mip_level_count != 1 {
3213                    return Err(Error::InvalidStorageTextureMipLevelCount {
3214                        binding,
3215                        mip_level_count,
3216                    });
3217                }
3218
3219                let internal_use = match access {
3220                    wgt::StorageTextureAccess::WriteOnly => {
3221                        if !view
3222                            .format_features
3223                            .flags
3224                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY)
3225                        {
3226                            return Err(Error::StorageWriteNotSupported(view.desc.format));
3227                        }
3228                        wgt::TextureUses::STORAGE_WRITE_ONLY
3229                    }
3230                    wgt::StorageTextureAccess::ReadOnly => {
3231                        if !view
3232                            .format_features
3233                            .flags
3234                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY)
3235                        {
3236                            return Err(Error::StorageReadNotSupported(view.desc.format));
3237                        }
3238                        wgt::TextureUses::STORAGE_READ_ONLY
3239                    }
3240                    wgt::StorageTextureAccess::ReadWrite => {
3241                        if !view
3242                            .format_features
3243                            .flags
3244                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE)
3245                        {
3246                            return Err(Error::StorageReadWriteNotSupported(view.desc.format));
3247                        }
3248
3249                        wgt::TextureUses::STORAGE_READ_WRITE
3250                    }
3251                    wgt::StorageTextureAccess::Atomic => {
3252                        if !view
3253                            .format_features
3254                            .flags
3255                            .contains(wgt::TextureFormatFeatureFlags::STORAGE_ATOMIC)
3256                        {
3257                            return Err(Error::StorageAtomicNotSupported(view.desc.format));
3258                        }
3259
3260                        wgt::TextureUses::STORAGE_ATOMIC
3261                    }
3262                };
3263                view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
3264                Ok(internal_use)
3265            }
3266            wgt::BindingType::ExternalTexture => {
3267                if view.desc.dimension != TextureViewDimension::D2 {
3268                    return Err(Error::InvalidTextureDimension {
3269                        binding,
3270                        layout_dimension: TextureViewDimension::D2,
3271                        view_dimension: view.desc.dimension,
3272                    });
3273                }
3274                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3275                if mip_level_count != 1 {
3276                    return Err(Error::InvalidExternalTextureMipLevelCount {
3277                        binding,
3278                        mip_level_count,
3279                    });
3280                }
3281                if view.desc.format != TextureFormat::Rgba8Unorm
3282                    && view.desc.format != TextureFormat::Bgra8Unorm
3283                    && view.desc.format != TextureFormat::Rgba16Float
3284                {
3285                    return Err(Error::InvalidExternalTextureFormat {
3286                        binding,
3287                        format: view.desc.format,
3288                    });
3289                }
3290                if view.samples != 1 {
3291                    return Err(Error::InvalidTextureMultisample {
3292                        binding,
3293                        layout_multisampled: false,
3294                        view_samples: view.samples,
3295                    });
3296                }
3297
3298                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3299                Ok(wgt::TextureUses::RESOURCE)
3300            }
3301            _ => Err(Error::WrongBindingType {
3302                binding,
3303                actual: decl.ty,
3304                expected,
3305            }),
3306        }
3307    }
3308
3309    pub(crate) fn create_pipeline_layout(
3310        self: &Arc<Self>,
3311        desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3312    ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
3313        use crate::binding_model::CreatePipelineLayoutError as Error;
3314
3315        self.check_is_valid()?;
3316
3317        let bind_group_layouts_count = desc.bind_group_layouts.len();
3318        let device_max_bind_groups = self.limits.max_bind_groups as usize;
3319        if bind_group_layouts_count > device_max_bind_groups {
3320            return Err(Error::TooManyGroups {
3321                actual: bind_group_layouts_count,
3322                max: device_max_bind_groups,
3323            });
3324        }
3325
3326        if !desc.push_constant_ranges.is_empty() {
3327            self.require_features(wgt::Features::PUSH_CONSTANTS)?;
3328        }
3329
3330        let mut used_stages = wgt::ShaderStages::empty();
3331        for (index, pc) in desc.push_constant_ranges.iter().enumerate() {
3332            if pc.stages.intersects(used_stages) {
3333                return Err(Error::MoreThanOnePushConstantRangePerStage {
3334                    index,
3335                    provided: pc.stages,
3336                    intersected: pc.stages & used_stages,
3337                });
3338            }
3339            used_stages |= pc.stages;
3340
3341            let device_max_pc_size = self.limits.max_push_constant_size;
3342            if device_max_pc_size < pc.range.end {
3343                return Err(Error::PushConstantRangeTooLarge {
3344                    index,
3345                    range: pc.range.clone(),
3346                    max: device_max_pc_size,
3347                });
3348            }
3349
3350            if pc.range.start % wgt::PUSH_CONSTANT_ALIGNMENT != 0 {
3351                return Err(Error::MisalignedPushConstantRange {
3352                    index,
3353                    bound: pc.range.start,
3354                });
3355            }
3356            if pc.range.end % wgt::PUSH_CONSTANT_ALIGNMENT != 0 {
3357                return Err(Error::MisalignedPushConstantRange {
3358                    index,
3359                    bound: pc.range.end,
3360                });
3361            }
3362        }
3363
3364        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3365
3366        for bgl in desc.bind_group_layouts.iter() {
3367            bgl.same_device(self)?;
3368            count_validator.merge(&bgl.binding_count_validator);
3369        }
3370
3371        count_validator
3372            .validate(&self.limits)
3373            .map_err(Error::TooManyBindings)?;
3374
3375        let bind_group_layouts = desc
3376            .bind_group_layouts
3377            .iter()
3378            .cloned()
3379            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3380
3381        let raw_bind_group_layouts = desc
3382            .bind_group_layouts
3383            .iter()
3384            .map(|bgl| bgl.raw())
3385            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3386
3387        let additional_flags = if self.indirect_validation.is_some() {
3388            hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
3389        } else {
3390            hal::PipelineLayoutFlags::empty()
3391        };
3392
3393        let hal_desc = hal::PipelineLayoutDescriptor {
3394            label: desc.label.to_hal(self.instance_flags),
3395            flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
3396                | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
3397                | additional_flags,
3398            bind_group_layouts: &raw_bind_group_layouts,
3399            push_constant_ranges: desc.push_constant_ranges.as_ref(),
3400        };
3401
3402        let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
3403            .map_err(|e| self.handle_hal_error(e))?;
3404
3405        drop(raw_bind_group_layouts);
3406
3407        let layout = binding_model::PipelineLayout {
3408            raw: ManuallyDrop::new(raw),
3409            device: self.clone(),
3410            label: desc.label.to_string(),
3411            bind_group_layouts,
3412            push_constant_ranges: desc.push_constant_ranges.iter().cloned().collect(),
3413        };
3414
3415        let layout = Arc::new(layout);
3416
3417        Ok(layout)
3418    }
3419
3420    pub(crate) fn derive_pipeline_layout(
3421        self: &Arc<Self>,
3422        mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
3423    ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
3424        while derived_group_layouts
3425            .last()
3426            .is_some_and(|map| map.is_empty())
3427        {
3428            derived_group_layouts.pop();
3429        }
3430
3431        let mut unique_bind_group_layouts = FastHashMap::default();
3432
3433        let bind_group_layouts = derived_group_layouts
3434            .into_iter()
3435            .map(|mut bgl_entry_map| {
3436                bgl_entry_map.sort();
3437                match unique_bind_group_layouts.entry(bgl_entry_map) {
3438                    hashbrown::hash_map::Entry::Occupied(v) => Ok(Arc::clone(v.get())),
3439                    hashbrown::hash_map::Entry::Vacant(e) => {
3440                        match self.create_bind_group_layout(
3441                            &None,
3442                            e.key().clone(),
3443                            bgl::Origin::Derived,
3444                        ) {
3445                            Ok(bgl) => {
3446                                e.insert(bgl.clone());
3447                                Ok(bgl)
3448                            }
3449                            Err(e) => Err(e),
3450                        }
3451                    }
3452                }
3453            })
3454            .collect::<Result<Vec<_>, _>>()?;
3455
3456        let layout_desc = binding_model::ResolvedPipelineLayoutDescriptor {
3457            label: None,
3458            bind_group_layouts: Cow::Owned(bind_group_layouts),
3459            push_constant_ranges: Cow::Borrowed(&[]), //TODO?
3460        };
3461
3462        let layout = self.create_pipeline_layout(&layout_desc)?;
3463        Ok(layout)
3464    }
3465
3466    pub(crate) fn create_compute_pipeline(
3467        self: &Arc<Self>,
3468        desc: pipeline::ResolvedComputePipelineDescriptor,
3469    ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreateComputePipelineError> {
3470        self.check_is_valid()?;
3471
3472        self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
3473
3474        let shader_module = desc.stage.module;
3475
3476        shader_module.same_device(self)?;
3477
3478        let is_auto_layout = desc.layout.is_none();
3479
3480        // Get the pipeline layout from the desc if it is provided.
3481        let pipeline_layout = match desc.layout {
3482            Some(pipeline_layout) => {
3483                pipeline_layout.same_device(self)?;
3484                Some(pipeline_layout)
3485            }
3486            None => None,
3487        };
3488
3489        let mut binding_layout_source = match pipeline_layout {
3490            Some(ref pipeline_layout) => {
3491                validation::BindingLayoutSource::Provided(pipeline_layout.get_binding_maps())
3492            }
3493            None => validation::BindingLayoutSource::new_derived(&self.limits),
3494        };
3495        let mut shader_binding_sizes = FastHashMap::default();
3496        let io = validation::StageIo::default();
3497
3498        let final_entry_point_name;
3499
3500        {
3501            let stage = wgt::ShaderStages::COMPUTE;
3502
3503            final_entry_point_name = shader_module.finalize_entry_point_name(
3504                stage,
3505                desc.stage.entry_point.as_ref().map(|ep| ep.as_ref()),
3506            )?;
3507
3508            if let Some(ref interface) = shader_module.interface {
3509                let _ = interface.check_stage(
3510                    &mut binding_layout_source,
3511                    &mut shader_binding_sizes,
3512                    &final_entry_point_name,
3513                    stage,
3514                    io,
3515                    None,
3516                )?;
3517            }
3518        }
3519
3520        let pipeline_layout = match binding_layout_source {
3521            validation::BindingLayoutSource::Provided(_) => {
3522                drop(binding_layout_source);
3523                pipeline_layout.unwrap()
3524            }
3525            validation::BindingLayoutSource::Derived(entries) => {
3526                self.derive_pipeline_layout(entries)?
3527            }
3528        };
3529
3530        let late_sized_buffer_groups =
3531            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
3532
3533        let cache = match desc.cache {
3534            Some(cache) => {
3535                cache.same_device(self)?;
3536                Some(cache)
3537            }
3538            None => None,
3539        };
3540
3541        let pipeline_desc = hal::ComputePipelineDescriptor {
3542            label: desc.label.to_hal(self.instance_flags),
3543            layout: pipeline_layout.raw(),
3544            stage: hal::ProgrammableStage {
3545                module: shader_module.raw(),
3546                entry_point: final_entry_point_name.as_ref(),
3547                constants: &desc.stage.constants,
3548                zero_initialize_workgroup_memory: desc.stage.zero_initialize_workgroup_memory,
3549            },
3550            cache: cache.as_ref().map(|it| it.raw()),
3551        };
3552
3553        let raw =
3554            unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }.map_err(
3555                |err| match err {
3556                    hal::PipelineError::Device(error) => {
3557                        pipeline::CreateComputePipelineError::Device(self.handle_hal_error(error))
3558                    }
3559                    hal::PipelineError::Linkage(_stages, msg) => {
3560                        pipeline::CreateComputePipelineError::Internal(msg)
3561                    }
3562                    hal::PipelineError::EntryPoint(_stage) => {
3563                        pipeline::CreateComputePipelineError::Internal(
3564                            ENTRYPOINT_FAILURE_ERROR.to_string(),
3565                        )
3566                    }
3567                    hal::PipelineError::PipelineConstants(_stages, msg) => {
3568                        pipeline::CreateComputePipelineError::PipelineConstants(msg)
3569                    }
3570                },
3571            )?;
3572
3573        let pipeline = pipeline::ComputePipeline {
3574            raw: ManuallyDrop::new(raw),
3575            layout: pipeline_layout,
3576            device: self.clone(),
3577            _shader_module: shader_module,
3578            late_sized_buffer_groups,
3579            label: desc.label.to_string(),
3580            tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
3581        };
3582
3583        let pipeline = Arc::new(pipeline);
3584
3585        if is_auto_layout {
3586            for bgl in pipeline.layout.bind_group_layouts.iter() {
3587                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the result.
3588                let _ = bgl
3589                    .exclusive_pipeline
3590                    .set(binding_model::ExclusivePipeline::Compute(Arc::downgrade(
3591                        &pipeline,
3592                    )));
3593            }
3594        }
3595
3596        Ok(pipeline)
3597    }
3598
3599    pub(crate) fn create_render_pipeline(
3600        self: &Arc<Self>,
3601        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
3602    ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreateRenderPipelineError> {
3603        use wgt::TextureFormatFeatureFlags as Tfff;
3604
3605        self.check_is_valid()?;
3606
3607        let mut shader_binding_sizes = FastHashMap::default();
3608
3609        let num_attachments = desc.fragment.as_ref().map(|f| f.targets.len()).unwrap_or(0);
3610        let max_attachments = self.limits.max_color_attachments as usize;
3611        if num_attachments > max_attachments {
3612            return Err(pipeline::CreateRenderPipelineError::ColorAttachment(
3613                command::ColorAttachmentError::TooMany {
3614                    given: num_attachments,
3615                    limit: max_attachments,
3616                },
3617            ));
3618        }
3619
3620        let color_targets = desc
3621            .fragment
3622            .as_ref()
3623            .map_or(&[][..], |fragment| &fragment.targets);
3624        let depth_stencil_state = desc.depth_stencil.as_ref();
3625
3626        {
3627            let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
3628                color_targets.iter().filter_map(|x| x.as_ref()).collect();
3629            if !cts.is_empty() && {
3630                let first = &cts[0];
3631                cts[1..]
3632                    .iter()
3633                    .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
3634            } {
3635                self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
3636            }
3637        }
3638
3639        let mut io = validation::StageIo::default();
3640        let mut validated_stages = wgt::ShaderStages::empty();
3641
3642        let mut vertex_steps;
3643        let mut vertex_buffers;
3644        let mut total_attributes;
3645        let mut shader_expects_dual_source_blending = false;
3646        let mut pipeline_expects_dual_source_blending = false;
3647        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
3648            vertex_steps = Vec::with_capacity(vertex.buffers.len());
3649            vertex_buffers = Vec::with_capacity(vertex.buffers.len());
3650            total_attributes = 0;
3651            shader_expects_dual_source_blending = false;
3652            pipeline_expects_dual_source_blending = false;
3653            for (i, vb_state) in vertex.buffers.iter().enumerate() {
3654                // https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpuvertexbufferlayout
3655
3656                if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
3657                    return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge {
3658                        index: i as u32,
3659                        given: vb_state.array_stride as u32,
3660                        limit: self.limits.max_vertex_buffer_array_stride,
3661                    });
3662                }
3663                if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
3664                    return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride {
3665                        index: i as u32,
3666                        stride: vb_state.array_stride,
3667                    });
3668                }
3669
3670                let max_stride = if vb_state.array_stride == 0 {
3671                    self.limits.max_vertex_buffer_array_stride as u64
3672                } else {
3673                    vb_state.array_stride
3674                };
3675                let mut last_stride = 0;
3676                for attribute in vb_state.attributes.iter() {
3677                    let attribute_stride = attribute.offset + attribute.format.size();
3678                    if attribute_stride > max_stride {
3679                        return Err(
3680                            pipeline::CreateRenderPipelineError::VertexAttributeStrideTooLarge {
3681                                location: attribute.shader_location,
3682                                given: attribute_stride as u32,
3683                                limit: max_stride as u32,
3684                            },
3685                        );
3686                    }
3687
3688                    let required_offset_alignment = attribute.format.size().min(4);
3689                    if attribute.offset % required_offset_alignment != 0 {
3690                        return Err(
3691                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
3692                                location: attribute.shader_location,
3693                                offset: attribute.offset,
3694                            },
3695                        );
3696                    }
3697
3698                    if attribute.shader_location >= self.limits.max_vertex_attributes {
3699                        return Err(
3700                            pipeline::CreateRenderPipelineError::VertexAttributeLocationTooLarge {
3701                                given: attribute.shader_location,
3702                                limit: self.limits.max_vertex_attributes,
3703                            },
3704                        );
3705                    }
3706
3707                    last_stride = last_stride.max(attribute_stride);
3708                }
3709                vertex_steps.push(pipeline::VertexStep {
3710                    stride: vb_state.array_stride,
3711                    last_stride,
3712                    mode: vb_state.step_mode,
3713                });
3714                if vb_state.attributes.is_empty() {
3715                    continue;
3716                }
3717                vertex_buffers.push(hal::VertexBufferLayout {
3718                    array_stride: vb_state.array_stride,
3719                    step_mode: vb_state.step_mode,
3720                    attributes: vb_state.attributes.as_ref(),
3721                });
3722
3723                for attribute in vb_state.attributes.iter() {
3724                    if attribute.offset >= 0x10000000 {
3725                        return Err(
3726                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
3727                                location: attribute.shader_location,
3728                                offset: attribute.offset,
3729                            },
3730                        );
3731                    }
3732
3733                    if let wgt::VertexFormat::Float64
3734                    | wgt::VertexFormat::Float64x2
3735                    | wgt::VertexFormat::Float64x3
3736                    | wgt::VertexFormat::Float64x4 = attribute.format
3737                    {
3738                        self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
3739                    }
3740
3741                    let previous = io.insert(
3742                        attribute.shader_location,
3743                        validation::InterfaceVar::vertex_attribute(attribute.format),
3744                    );
3745
3746                    if previous.is_some() {
3747                        return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash(
3748                            attribute.shader_location,
3749                        ));
3750                    }
3751                }
3752                total_attributes += vb_state.attributes.len();
3753            }
3754
3755            if vertex_buffers.len() > self.limits.max_vertex_buffers as usize {
3756                return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers {
3757                    given: vertex_buffers.len() as u32,
3758                    limit: self.limits.max_vertex_buffers,
3759                });
3760            }
3761            if total_attributes > self.limits.max_vertex_attributes as usize {
3762                return Err(
3763                    pipeline::CreateRenderPipelineError::TooManyVertexAttributes {
3764                        given: total_attributes as u32,
3765                        limit: self.limits.max_vertex_attributes,
3766                    },
3767                );
3768            }
3769        } else {
3770            vertex_steps = Vec::new();
3771            vertex_buffers = Vec::new();
3772        };
3773
3774        if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
3775            return Err(
3776                pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology {
3777                    strip_index_format: desc.primitive.strip_index_format,
3778                    topology: desc.primitive.topology,
3779                },
3780            );
3781        }
3782
3783        if desc.primitive.unclipped_depth {
3784            self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
3785        }
3786
3787        if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
3788            self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
3789        }
3790        if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
3791            self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
3792        }
3793
3794        if desc.primitive.conservative {
3795            self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
3796        }
3797
3798        if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
3799            return Err(
3800                pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode,
3801            );
3802        }
3803
3804        let mut target_specified = false;
3805
3806        for (i, cs) in color_targets.iter().enumerate() {
3807            if let Some(cs) = cs.as_ref() {
3808                target_specified = true;
3809                let error = 'error: {
3810                    if cs.write_mask.contains_unknown_bits() {
3811                        break 'error Some(pipeline::ColorStateError::InvalidWriteMask(
3812                            cs.write_mask,
3813                        ));
3814                    }
3815
3816                    let format_features = self.describe_format_features(cs.format)?;
3817                    if !format_features
3818                        .allowed_usages
3819                        .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
3820                    {
3821                        break 'error Some(pipeline::ColorStateError::FormatNotRenderable(
3822                            cs.format,
3823                        ));
3824                    }
3825                    let blendable = format_features.flags.contains(Tfff::BLENDABLE);
3826                    let filterable = format_features.flags.contains(Tfff::FILTERABLE);
3827                    let adapter_specific = self
3828                        .features
3829                        .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
3830                    // according to WebGPU specifications the texture needs to be
3831                    // [`TextureFormatFeatureFlags::FILTERABLE`] if blending is set - use
3832                    // [`Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES`] to elude
3833                    // this limitation
3834                    if cs.blend.is_some() && (!blendable || (!filterable && !adapter_specific)) {
3835                        break 'error Some(pipeline::ColorStateError::FormatNotBlendable(
3836                            cs.format,
3837                        ));
3838                    }
3839                    if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
3840                        break 'error Some(pipeline::ColorStateError::FormatNotColor(cs.format));
3841                    }
3842
3843                    if desc.multisample.count > 1
3844                        && !format_features
3845                            .flags
3846                            .sample_count_supported(desc.multisample.count)
3847                    {
3848                        break 'error Some(pipeline::ColorStateError::InvalidSampleCount(
3849                            desc.multisample.count,
3850                            cs.format,
3851                            cs.format
3852                                .guaranteed_format_features(self.features)
3853                                .flags
3854                                .supported_sample_counts(),
3855                            self.adapter
3856                                .get_texture_format_features(cs.format)
3857                                .flags
3858                                .supported_sample_counts(),
3859                        ));
3860                    }
3861
3862                    if let Some(blend_mode) = cs.blend {
3863                        for factor in [
3864                            blend_mode.color.src_factor,
3865                            blend_mode.color.dst_factor,
3866                            blend_mode.alpha.src_factor,
3867                            blend_mode.alpha.dst_factor,
3868                        ] {
3869                            if factor.ref_second_blend_source() {
3870                                self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
3871                                if i == 0 {
3872                                    pipeline_expects_dual_source_blending = true;
3873                                    break;
3874                                } else {
3875                                    return Err(pipeline::CreateRenderPipelineError
3876                                        ::BlendFactorOnUnsupportedTarget { factor, target: i as u32 });
3877                                }
3878                            }
3879                        }
3880                    }
3881
3882                    break 'error None;
3883                };
3884                if let Some(e) = error {
3885                    return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e));
3886                }
3887            }
3888        }
3889
3890        let limit = self.limits.max_color_attachment_bytes_per_sample;
3891        let formats = color_targets
3892            .iter()
3893            .map(|cs| cs.as_ref().map(|cs| cs.format));
3894        if let Err(total) = validate_color_attachment_bytes_per_sample(formats, limit) {
3895            return Err(pipeline::CreateRenderPipelineError::ColorAttachment(
3896                command::ColorAttachmentError::TooManyBytesPerSample { total, limit },
3897            ));
3898        }
3899
3900        if let Some(ds) = depth_stencil_state {
3901            target_specified = true;
3902            let error = 'error: {
3903                let format_features = self.describe_format_features(ds.format)?;
3904                if !format_features
3905                    .allowed_usages
3906                    .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
3907                {
3908                    break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
3909                        ds.format,
3910                    ));
3911                }
3912
3913                let aspect = hal::FormatAspects::from(ds.format);
3914                if ds.is_depth_enabled() && !aspect.contains(hal::FormatAspects::DEPTH) {
3915                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
3916                }
3917                if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
3918                    break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
3919                        ds.format,
3920                    ));
3921                }
3922                if desc.multisample.count > 1
3923                    && !format_features
3924                        .flags
3925                        .sample_count_supported(desc.multisample.count)
3926                {
3927                    break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
3928                        desc.multisample.count,
3929                        ds.format,
3930                        ds.format
3931                            .guaranteed_format_features(self.features)
3932                            .flags
3933                            .supported_sample_counts(),
3934                        self.adapter
3935                            .get_texture_format_features(ds.format)
3936                            .flags
3937                            .supported_sample_counts(),
3938                    ));
3939                }
3940
3941                break 'error None;
3942            };
3943            if let Some(e) = error {
3944                return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e));
3945            }
3946
3947            if ds.bias.clamp != 0.0 {
3948                self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
3949            }
3950        }
3951
3952        if !target_specified {
3953            return Err(pipeline::CreateRenderPipelineError::NoTargetSpecified);
3954        }
3955
3956        let is_auto_layout = desc.layout.is_none();
3957
3958        // Get the pipeline layout from the desc if it is provided.
3959        let pipeline_layout = match desc.layout {
3960            Some(pipeline_layout) => {
3961                pipeline_layout.same_device(self)?;
3962                Some(pipeline_layout)
3963            }
3964            None => None,
3965        };
3966
3967        let mut binding_layout_source = match pipeline_layout {
3968            Some(ref pipeline_layout) => {
3969                validation::BindingLayoutSource::Provided(pipeline_layout.get_binding_maps())
3970            }
3971            None => validation::BindingLayoutSource::new_derived(&self.limits),
3972        };
3973
3974        let samples = {
3975            let sc = desc.multisample.count;
3976            if sc == 0 || sc > 32 || !sc.is_power_of_two() {
3977                return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc));
3978            }
3979            sc
3980        };
3981
3982        let mut vertex_stage = None;
3983        let mut task_stage = None;
3984        let mut mesh_stage = None;
3985        let mut _vertex_entry_point_name = String::new();
3986        let mut _task_entry_point_name = String::new();
3987        let mut _mesh_entry_point_name = String::new();
3988        match desc.vertex {
3989            pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
3990                vertex_stage = {
3991                    let stage_desc = &vertex.stage;
3992                    let stage = wgt::ShaderStages::VERTEX;
3993
3994                    let vertex_shader_module = &stage_desc.module;
3995                    vertex_shader_module.same_device(self)?;
3996
3997                    let stage_err =
3998                        |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
3999
4000                    _vertex_entry_point_name = vertex_shader_module
4001                        .finalize_entry_point_name(
4002                            stage,
4003                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4004                        )
4005                        .map_err(stage_err)?;
4006
4007                    if let Some(ref interface) = vertex_shader_module.interface {
4008                        io = interface
4009                            .check_stage(
4010                                &mut binding_layout_source,
4011                                &mut shader_binding_sizes,
4012                                &_vertex_entry_point_name,
4013                                stage,
4014                                io,
4015                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4016                            )
4017                            .map_err(stage_err)?;
4018                        validated_stages |= stage;
4019                    }
4020                    Some(hal::ProgrammableStage {
4021                        module: vertex_shader_module.raw(),
4022                        entry_point: &_vertex_entry_point_name,
4023                        constants: &stage_desc.constants,
4024                        zero_initialize_workgroup_memory: stage_desc
4025                            .zero_initialize_workgroup_memory,
4026                    })
4027                };
4028            }
4029            pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
4030                task_stage = if let Some(task) = task {
4031                    let stage_desc = &task.stage;
4032                    let stage = wgt::ShaderStages::TASK;
4033                    let task_shader_module = &stage_desc.module;
4034                    task_shader_module.same_device(self)?;
4035
4036                    let stage_err =
4037                        |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4038
4039                    _task_entry_point_name = task_shader_module
4040                        .finalize_entry_point_name(
4041                            stage,
4042                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4043                        )
4044                        .map_err(stage_err)?;
4045
4046                    if let Some(ref interface) = task_shader_module.interface {
4047                        io = interface
4048                            .check_stage(
4049                                &mut binding_layout_source,
4050                                &mut shader_binding_sizes,
4051                                &_task_entry_point_name,
4052                                stage,
4053                                io,
4054                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4055                            )
4056                            .map_err(stage_err)?;
4057                        validated_stages |= stage;
4058                    }
4059                    Some(hal::ProgrammableStage {
4060                        module: task_shader_module.raw(),
4061                        entry_point: &_task_entry_point_name,
4062                        constants: &stage_desc.constants,
4063                        zero_initialize_workgroup_memory: stage_desc
4064                            .zero_initialize_workgroup_memory,
4065                    })
4066                } else {
4067                    None
4068                };
4069                mesh_stage = {
4070                    let stage_desc = &mesh.stage;
4071                    let stage = wgt::ShaderStages::MESH;
4072                    let mesh_shader_module = &stage_desc.module;
4073                    mesh_shader_module.same_device(self)?;
4074
4075                    let stage_err =
4076                        |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4077
4078                    _mesh_entry_point_name = mesh_shader_module
4079                        .finalize_entry_point_name(
4080                            stage,
4081                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4082                        )
4083                        .map_err(stage_err)?;
4084
4085                    if let Some(ref interface) = mesh_shader_module.interface {
4086                        io = interface
4087                            .check_stage(
4088                                &mut binding_layout_source,
4089                                &mut shader_binding_sizes,
4090                                &_mesh_entry_point_name,
4091                                stage,
4092                                io,
4093                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4094                            )
4095                            .map_err(stage_err)?;
4096                        validated_stages |= stage;
4097                    }
4098                    Some(hal::ProgrammableStage {
4099                        module: mesh_shader_module.raw(),
4100                        entry_point: &_mesh_entry_point_name,
4101                        constants: &stage_desc.constants,
4102                        zero_initialize_workgroup_memory: stage_desc
4103                            .zero_initialize_workgroup_memory,
4104                    })
4105                };
4106            }
4107        }
4108
4109        let fragment_entry_point_name;
4110        let fragment_stage = match desc.fragment {
4111            Some(ref fragment_state) => {
4112                let stage = wgt::ShaderStages::FRAGMENT;
4113
4114                let shader_module = &fragment_state.stage.module;
4115                shader_module.same_device(self)?;
4116
4117                let stage_err = |error| pipeline::CreateRenderPipelineError::Stage { stage, error };
4118
4119                fragment_entry_point_name = shader_module
4120                    .finalize_entry_point_name(
4121                        stage,
4122                        fragment_state
4123                            .stage
4124                            .entry_point
4125                            .as_ref()
4126                            .map(|ep| ep.as_ref()),
4127                    )
4128                    .map_err(stage_err)?;
4129
4130                if validated_stages == wgt::ShaderStages::VERTEX {
4131                    if let Some(ref interface) = shader_module.interface {
4132                        io = interface
4133                            .check_stage(
4134                                &mut binding_layout_source,
4135                                &mut shader_binding_sizes,
4136                                &fragment_entry_point_name,
4137                                stage,
4138                                io,
4139                                desc.depth_stencil.as_ref().map(|d| d.depth_compare),
4140                            )
4141                            .map_err(stage_err)?;
4142                        validated_stages |= stage;
4143                    }
4144                }
4145
4146                if let Some(ref interface) = shader_module.interface {
4147                    shader_expects_dual_source_blending = interface
4148                        .fragment_uses_dual_source_blending(&fragment_entry_point_name)
4149                        .map_err(|error| pipeline::CreateRenderPipelineError::Stage {
4150                            stage,
4151                            error,
4152                        })?;
4153                }
4154
4155                Some(hal::ProgrammableStage {
4156                    module: shader_module.raw(),
4157                    entry_point: &fragment_entry_point_name,
4158                    constants: &fragment_state.stage.constants,
4159                    zero_initialize_workgroup_memory: fragment_state
4160                        .stage
4161                        .zero_initialize_workgroup_memory,
4162                })
4163            }
4164            None => None,
4165        };
4166
4167        if !pipeline_expects_dual_source_blending && shader_expects_dual_source_blending {
4168            return Err(
4169                pipeline::CreateRenderPipelineError::ShaderExpectsPipelineToUseDualSourceBlending,
4170            );
4171        }
4172        if pipeline_expects_dual_source_blending && !shader_expects_dual_source_blending {
4173            return Err(
4174                pipeline::CreateRenderPipelineError::PipelineExpectsShaderToUseDualSourceBlending,
4175            );
4176        }
4177
4178        if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
4179            for (i, output) in io.iter() {
4180                match color_targets.get(*i as usize) {
4181                    Some(Some(state)) => {
4182                        validation::check_texture_format(state.format, &output.ty).map_err(
4183                            |pipeline| {
4184                                pipeline::CreateRenderPipelineError::ColorState(
4185                                    *i as u8,
4186                                    pipeline::ColorStateError::IncompatibleFormat {
4187                                        pipeline,
4188                                        shader: output.ty,
4189                                    },
4190                                )
4191                            },
4192                        )?;
4193                    }
4194                    _ => {
4195                        log::warn!(
4196                            "The fragment stage {:?} output @location({}) values are ignored",
4197                            fragment_stage
4198                                .as_ref()
4199                                .map_or("", |stage| stage.entry_point),
4200                            i
4201                        );
4202                    }
4203                }
4204            }
4205        }
4206        let last_stage = match desc.fragment {
4207            Some(_) => wgt::ShaderStages::FRAGMENT,
4208            None => wgt::ShaderStages::VERTEX,
4209        };
4210        if is_auto_layout && !validated_stages.contains(last_stage) {
4211            return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into());
4212        }
4213
4214        let pipeline_layout = match binding_layout_source {
4215            validation::BindingLayoutSource::Provided(_) => {
4216                drop(binding_layout_source);
4217                pipeline_layout.unwrap()
4218            }
4219            validation::BindingLayoutSource::Derived(entries) => {
4220                self.derive_pipeline_layout(entries)?
4221            }
4222        };
4223
4224        // Multiview is only supported if the feature is enabled
4225        if desc.multiview.is_some() {
4226            self.require_features(wgt::Features::MULTIVIEW)?;
4227        }
4228
4229        if !self
4230            .downlevel
4231            .flags
4232            .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
4233        {
4234            for (binding, size) in shader_binding_sizes.iter() {
4235                if size.get() % 16 != 0 {
4236                    return Err(pipeline::CreateRenderPipelineError::UnalignedShader {
4237                        binding: binding.binding,
4238                        group: binding.group,
4239                        size: size.get(),
4240                    });
4241                }
4242            }
4243        }
4244
4245        let late_sized_buffer_groups =
4246            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4247
4248        let cache = match desc.cache {
4249            Some(cache) => {
4250                cache.same_device(self)?;
4251                Some(cache)
4252            }
4253            None => None,
4254        };
4255
4256        let is_mesh = mesh_stage.is_some();
4257        let raw = {
4258            let pipeline_desc = hal::RenderPipelineDescriptor {
4259                label: desc.label.to_hal(self.instance_flags),
4260                layout: pipeline_layout.raw(),
4261                vertex_processor: match vertex_stage {
4262                    Some(vertex_stage) => hal::VertexProcessor::Standard {
4263                        vertex_buffers: &vertex_buffers,
4264                        vertex_stage,
4265                    },
4266                    None => hal::VertexProcessor::Mesh {
4267                        task_stage,
4268                        mesh_stage: mesh_stage.unwrap(),
4269                    },
4270                },
4271                primitive: desc.primitive,
4272                depth_stencil: desc.depth_stencil.clone(),
4273                multisample: desc.multisample,
4274                fragment_stage,
4275                color_targets,
4276                multiview: desc.multiview,
4277                cache: cache.as_ref().map(|it| it.raw()),
4278            };
4279            unsafe { self.raw().create_render_pipeline(&pipeline_desc) }.map_err(
4280                |err| match err {
4281                    hal::PipelineError::Device(error) => {
4282                        pipeline::CreateRenderPipelineError::Device(self.handle_hal_error(error))
4283                    }
4284                    hal::PipelineError::Linkage(stage, msg) => {
4285                        pipeline::CreateRenderPipelineError::Internal { stage, error: msg }
4286                    }
4287                    hal::PipelineError::EntryPoint(stage) => {
4288                        pipeline::CreateRenderPipelineError::Internal {
4289                            stage: hal::auxil::map_naga_stage(stage),
4290                            error: ENTRYPOINT_FAILURE_ERROR.to_string(),
4291                        }
4292                    }
4293                    hal::PipelineError::PipelineConstants(stage, error) => {
4294                        pipeline::CreateRenderPipelineError::PipelineConstants { stage, error }
4295                    }
4296                },
4297            )?
4298        };
4299
4300        let pass_context = RenderPassContext {
4301            attachments: AttachmentData {
4302                colors: color_targets
4303                    .iter()
4304                    .map(|state| state.as_ref().map(|s| s.format))
4305                    .collect(),
4306                resolves: ArrayVec::new(),
4307                depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
4308            },
4309            sample_count: samples,
4310            multiview: desc.multiview,
4311        };
4312
4313        let mut flags = pipeline::PipelineFlags::empty();
4314        for state in color_targets.iter().filter_map(|s| s.as_ref()) {
4315            if let Some(ref bs) = state.blend {
4316                if bs.color.uses_constant() | bs.alpha.uses_constant() {
4317                    flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
4318                }
4319            }
4320        }
4321        if let Some(ds) = depth_stencil_state.as_ref() {
4322            if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
4323                flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
4324            }
4325            if !ds.is_depth_read_only() {
4326                flags |= pipeline::PipelineFlags::WRITES_DEPTH;
4327            }
4328            if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
4329                flags |= pipeline::PipelineFlags::WRITES_STENCIL;
4330            }
4331        }
4332        let shader_modules = {
4333            let mut shader_modules = ArrayVec::new();
4334            match desc.vertex {
4335                pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
4336                    shader_modules.push(vertex.stage.module)
4337                }
4338                pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
4339                    if let Some(task) = task {
4340                        shader_modules.push(task.stage.module);
4341                    }
4342                    shader_modules.push(mesh.stage.module);
4343                }
4344            }
4345            shader_modules.extend(desc.fragment.map(|f| f.stage.module));
4346            shader_modules
4347        };
4348
4349        let pipeline = pipeline::RenderPipeline {
4350            raw: ManuallyDrop::new(raw),
4351            layout: pipeline_layout,
4352            device: self.clone(),
4353            pass_context,
4354            _shader_modules: shader_modules,
4355            flags,
4356            strip_index_format: desc.primitive.strip_index_format,
4357            vertex_steps,
4358            late_sized_buffer_groups,
4359            label: desc.label.to_string(),
4360            tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
4361            is_mesh,
4362        };
4363
4364        let pipeline = Arc::new(pipeline);
4365
4366        if is_auto_layout {
4367            for bgl in pipeline.layout.bind_group_layouts.iter() {
4368                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the result.
4369                let _ = bgl
4370                    .exclusive_pipeline
4371                    .set(binding_model::ExclusivePipeline::Render(Arc::downgrade(
4372                        &pipeline,
4373                    )));
4374            }
4375        }
4376
4377        Ok(pipeline)
4378    }
4379
4380    /// # Safety
4381    /// The `data` field on `desc` must have previously been returned from [`crate::global::Global::pipeline_cache_get_data`]
4382    pub unsafe fn create_pipeline_cache(
4383        self: &Arc<Self>,
4384        desc: &pipeline::PipelineCacheDescriptor,
4385    ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
4386        use crate::pipeline_cache;
4387
4388        self.check_is_valid()?;
4389
4390        self.require_features(wgt::Features::PIPELINE_CACHE)?;
4391        let data = if let Some((data, validation_key)) = desc
4392            .data
4393            .as_ref()
4394            .zip(self.raw().pipeline_cache_validation_key())
4395        {
4396            let data = pipeline_cache::validate_pipeline_cache(
4397                data,
4398                &self.adapter.raw.info,
4399                validation_key,
4400            );
4401            match data {
4402                Ok(data) => Some(data),
4403                Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
4404                // If the error was unavoidable and we are asked to fallback, do so
4405                Err(_) => None,
4406            }
4407        } else {
4408            None
4409        };
4410        let cache_desc = hal::PipelineCacheDescriptor {
4411            data,
4412            label: desc.label.to_hal(self.instance_flags),
4413        };
4414        let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
4415            Ok(raw) => raw,
4416            Err(e) => match e {
4417                hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
4418            },
4419        };
4420        let cache = pipeline::PipelineCache {
4421            device: self.clone(),
4422            label: desc.label.to_string(),
4423            // This would be none in the error condition, which we don't implement yet
4424            raw: ManuallyDrop::new(raw),
4425        };
4426
4427        let cache = Arc::new(cache);
4428
4429        Ok(cache)
4430    }
4431
4432    fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
4433        // Variant of adapter.get_texture_format_features that takes device features into account
4434        use wgt::TextureFormatFeatureFlags as tfsc;
4435        let mut format_features = self.adapter.get_texture_format_features(format);
4436        if (format == TextureFormat::R32Float
4437            || format == TextureFormat::Rg32Float
4438            || format == TextureFormat::Rgba32Float)
4439            && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
4440        {
4441            format_features.flags.set(tfsc::FILTERABLE, false);
4442        }
4443        format_features
4444    }
4445
4446    fn describe_format_features(
4447        &self,
4448        format: TextureFormat,
4449    ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
4450        self.require_features(format.required_features())?;
4451
4452        let using_device_features = self
4453            .features
4454            .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
4455        // If we're running downlevel, we need to manually ask the backend what
4456        // we can use as we can't trust WebGPU.
4457        let downlevel = !self
4458            .downlevel
4459            .flags
4460            .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
4461
4462        if using_device_features || downlevel {
4463            Ok(self.get_texture_format_features(format))
4464        } else {
4465            Ok(format.guaranteed_format_features(self.features))
4466        }
4467    }
4468
4469    #[cfg(feature = "replay")]
4470    pub(crate) fn wait_for_submit(
4471        &self,
4472        submission_index: crate::SubmissionIndex,
4473    ) -> Result<(), DeviceError> {
4474        let fence = self.fence.read();
4475        let last_done_index = unsafe { self.raw().get_fence_value(fence.as_ref()) }
4476            .map_err(|e| self.handle_hal_error(e))?;
4477        if last_done_index < submission_index {
4478            unsafe { self.raw().wait(fence.as_ref(), submission_index, !0) }
4479                .map_err(|e| self.handle_hal_error(e))?;
4480            drop(fence);
4481            if let Some(queue) = self.get_queue() {
4482                let closures = queue.lock_life().triage_submissions(submission_index);
4483                assert!(
4484                    closures.is_empty(),
4485                    "wait_for_submit is not expected to work with closures"
4486                );
4487            }
4488        }
4489        Ok(())
4490    }
4491
4492    pub(crate) fn create_query_set(
4493        self: &Arc<Self>,
4494        desc: &resource::QuerySetDescriptor,
4495    ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
4496        use resource::CreateQuerySetError as Error;
4497
4498        self.check_is_valid()?;
4499
4500        match desc.ty {
4501            wgt::QueryType::Occlusion => {}
4502            wgt::QueryType::Timestamp => {
4503                self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
4504            }
4505            wgt::QueryType::PipelineStatistics(..) => {
4506                self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
4507            }
4508        }
4509
4510        if desc.count == 0 {
4511            return Err(Error::ZeroCount);
4512        }
4513
4514        if desc.count > wgt::QUERY_SET_MAX_QUERIES {
4515            return Err(Error::TooManyQueries {
4516                count: desc.count,
4517                maximum: wgt::QUERY_SET_MAX_QUERIES,
4518            });
4519        }
4520
4521        let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
4522
4523        let raw = unsafe { self.raw().create_query_set(&hal_desc) }
4524            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
4525
4526        let query_set = QuerySet {
4527            raw: ManuallyDrop::new(raw),
4528            device: self.clone(),
4529            label: desc.label.to_string(),
4530            tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
4531            desc: desc.map_label(|_| ()),
4532        };
4533
4534        let query_set = Arc::new(query_set);
4535
4536        Ok(query_set)
4537    }
4538
4539    fn lose(&self, message: &str) {
4540        // Follow the steps at https://gpuweb.github.io/gpuweb/#lose-the-device.
4541
4542        // Mark the device explicitly as invalid. This is checked in various
4543        // places to prevent new work from being submitted.
4544        self.valid.store(false, Ordering::Release);
4545
4546        // 1) Resolve the GPUDevice device.lost promise.
4547        if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
4548            device_lost_closure(DeviceLostReason::Unknown, message.to_string());
4549        }
4550
4551        // 2) Complete any outstanding mapAsync() steps.
4552        // 3) Complete any outstanding onSubmittedWorkDone() steps.
4553
4554        // These parts are passively accomplished by setting valid to false,
4555        // since that will prevent any new work from being added to the queues.
4556        // Future calls to poll_devices will continue to check the work queues
4557        // until they are cleared, and then drop the device.
4558    }
4559
4560    fn release_gpu_resources(&self) {
4561        // This is called when the device is lost, which makes every associated
4562        // resource invalid and unusable. This is an opportunity to release all of
4563        // the underlying gpu resources, even though the objects remain visible to
4564        // the user agent. We purge this memory naturally when resources have been
4565        // moved into the appropriate buckets, so this function just needs to
4566        // initiate movement into those buckets, and it can do that by calling
4567        // "destroy" on all the resources we know about.
4568
4569        // During these iterations, we discard all errors. We don't care!
4570        let trackers = self.trackers.lock();
4571        for buffer in trackers.buffers.used_resources() {
4572            if let Some(buffer) = Weak::upgrade(buffer) {
4573                buffer.destroy();
4574            }
4575        }
4576        for texture in trackers.textures.used_resources() {
4577            if let Some(texture) = Weak::upgrade(texture) {
4578                texture.destroy();
4579            }
4580        }
4581    }
4582
4583    pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
4584        UsageScope::new_pooled(&self.usage_scopes, &self.tracker_indices)
4585    }
4586
4587    pub fn get_hal_counters(&self) -> wgt::HalCounters {
4588        self.raw().get_internal_counters()
4589    }
4590
4591    pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
4592        self.raw().generate_allocator_report()
4593    }
4594}
4595
4596crate::impl_resource_type!(Device);
4597crate::impl_labeled!(Device);
4598crate::impl_storage_item!(Device);