wgpu_core/
resource.rs

1use alloc::{borrow::Cow, borrow::ToOwned as _, boxed::Box, string::String, sync::Arc, vec::Vec};
2use core::{
3    borrow::Borrow,
4    fmt,
5    mem::{self, size_of, ManuallyDrop},
6    num::NonZeroU64,
7    ops::Range,
8    ptr::NonNull,
9};
10use smallvec::SmallVec;
11use thiserror::Error;
12use wgt::{
13    error::{ErrorType, WebGpuError},
14    TextureSelector,
15};
16
17#[cfg(feature = "trace")]
18use crate::device::trace;
19use crate::{
20    binding_model::{BindGroup, BindingError},
21    device::{
22        queue, resource::DeferredDestroy, BufferMapPendingClosure, Device, DeviceError,
23        DeviceMismatch, HostMap, MissingDownlevelFlags, MissingFeatures,
24    },
25    hal_label,
26    init_tracker::{BufferInitTracker, TextureInitTracker},
27    lock::{rank, Mutex, RwLock},
28    ray_tracing::{BlasCompactReadyPendingClosure, BlasPrepareCompactError},
29    resource_log,
30    snatch::{SnatchGuard, Snatchable},
31    timestamp_normalization::TimestampNormalizationBindGroup,
32    track::{SharedTrackerIndexAllocator, TrackerIndex},
33    weak_vec::WeakVec,
34    Label, LabelHelpers, SubmissionIndex,
35};
36
37/// Information about the wgpu-core resource.
38///
39/// Each type representing a `wgpu-core` resource, like [`Device`],
40/// [`Buffer`], etc., contains a `ResourceInfo` which contains
41/// its latest submission index and label.
42///
43/// A resource may need to be retained for any of several reasons:
44/// and any lifetime logic will be handled by `Arc<Resource>` refcount
45///
46/// - The user may hold a reference to it (via a `wgpu::Buffer`, say).
47///
48/// - Other resources may depend on it (a texture view's backing
49///   texture, for example).
50///
51/// - It may be used by commands sent to the GPU that have not yet
52///   finished execution.
53///
54/// [`Device`]: crate::device::resource::Device
55/// [`Buffer`]: crate::resource::Buffer
56#[derive(Debug)]
57pub(crate) struct TrackingData {
58    tracker_index: TrackerIndex,
59    tracker_indices: Arc<SharedTrackerIndexAllocator>,
60}
61
62impl Drop for TrackingData {
63    fn drop(&mut self) {
64        self.tracker_indices.free(self.tracker_index);
65    }
66}
67
68impl TrackingData {
69    pub(crate) fn new(tracker_indices: Arc<SharedTrackerIndexAllocator>) -> Self {
70        Self {
71            tracker_index: tracker_indices.alloc(),
72            tracker_indices,
73        }
74    }
75
76    pub(crate) fn tracker_index(&self) -> TrackerIndex {
77        self.tracker_index
78    }
79}
80
81#[derive(Clone, Debug)]
82#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
83pub struct ResourceErrorIdent {
84    r#type: Cow<'static, str>,
85    label: String,
86}
87
88impl fmt::Display for ResourceErrorIdent {
89    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
90        write!(f, "{} with '{}' label", self.r#type, self.label)
91    }
92}
93
94pub trait ParentDevice: Labeled {
95    fn device(&self) -> &Arc<Device>;
96
97    fn is_equal(self: &Arc<Self>, other: &Arc<Self>) -> bool {
98        Arc::ptr_eq(self, other)
99    }
100
101    fn same_device_as<O: ParentDevice>(&self, other: &O) -> Result<(), DeviceError> {
102        if Arc::ptr_eq(self.device(), other.device()) {
103            Ok(())
104        } else {
105            Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch {
106                res: self.error_ident(),
107                res_device: self.device().error_ident(),
108                target: Some(other.error_ident()),
109                target_device: other.device().error_ident(),
110            })))
111        }
112    }
113
114    fn same_device(&self, device: &Device) -> Result<(), DeviceError> {
115        if core::ptr::eq(&**self.device(), device) {
116            Ok(())
117        } else {
118            Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch {
119                res: self.error_ident(),
120                res_device: self.device().error_ident(),
121                target: None,
122                target_device: device.error_ident(),
123            })))
124        }
125    }
126}
127
128#[macro_export]
129macro_rules! impl_parent_device {
130    ($ty:ident) => {
131        impl $crate::resource::ParentDevice for $ty {
132            fn device(&self) -> &Arc<Device> {
133                &self.device
134            }
135        }
136    };
137}
138
139/// Allow access to the hal resource as guarded by the `SnatchGuard`.
140pub trait RawResourceAccess: ParentDevice {
141    type DynResource: hal::DynResource + ?Sized;
142
143    /// Get access to the raw resource if it is not destroyed.
144    ///
145    /// Returns `None` if the resource has been destroyed. This method
146    /// does not allocate in either case.
147    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource>;
148
149    /// Get access to the raw resource if it is not destroyed.
150    ///
151    /// Returns a full error if the resource has been destroyed. This
152    /// method allocates a label in the error case.
153    fn try_raw<'a>(
154        &'a self,
155        guard: &'a SnatchGuard,
156    ) -> Result<&'a Self::DynResource, DestroyedResourceError> {
157        self.raw(guard)
158            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
159    }
160}
161
162pub trait ResourceType {
163    const TYPE: &'static str;
164}
165
166#[macro_export]
167macro_rules! impl_resource_type {
168    ($ty:ident) => {
169        impl $crate::resource::ResourceType for $ty {
170            const TYPE: &'static str = stringify!($ty);
171        }
172    };
173}
174
175pub trait Labeled: ResourceType {
176    /// Returns a string identifying this resource for logging and errors.
177    ///
178    /// It may be a user-provided string or it may be a placeholder from wgpu.
179    ///
180    /// It is non-empty unless the user-provided string was empty.
181    fn label(&self) -> &str;
182
183    fn error_ident(&self) -> ResourceErrorIdent {
184        ResourceErrorIdent {
185            r#type: Cow::Borrowed(Self::TYPE),
186            label: self.label().to_owned(),
187        }
188    }
189}
190
191#[macro_export]
192macro_rules! impl_labeled {
193    ($ty:ident) => {
194        impl $crate::resource::Labeled for $ty {
195            fn label(&self) -> &str {
196                &self.label
197            }
198        }
199    };
200}
201
202pub(crate) trait Trackable {
203    fn tracker_index(&self) -> TrackerIndex;
204}
205
206#[macro_export]
207macro_rules! impl_trackable {
208    ($ty:ident) => {
209        impl $crate::resource::Trackable for $ty {
210            fn tracker_index(&self) -> $crate::track::TrackerIndex {
211                self.tracking_data.tracker_index()
212            }
213        }
214    };
215}
216
217#[derive(Debug)]
218pub(crate) enum BufferMapState {
219    /// Mapped at creation.
220    Init { staging_buffer: StagingBuffer },
221    /// Waiting for GPU to be done before mapping
222    Waiting(BufferPendingMapping),
223    /// Mapped
224    Active {
225        mapping: hal::BufferMapping,
226        range: hal::MemoryRange,
227        host: HostMap,
228    },
229    /// Not mapped
230    Idle,
231}
232
233#[cfg(send_sync)]
234unsafe impl Send for BufferMapState {}
235#[cfg(send_sync)]
236unsafe impl Sync for BufferMapState {}
237
238#[cfg(send_sync)]
239pub type BufferMapCallback = Box<dyn FnOnce(BufferAccessResult) + Send + 'static>;
240#[cfg(not(send_sync))]
241pub type BufferMapCallback = Box<dyn FnOnce(BufferAccessResult) + 'static>;
242
243pub struct BufferMapOperation {
244    pub host: HostMap,
245    pub callback: Option<BufferMapCallback>,
246}
247
248impl fmt::Debug for BufferMapOperation {
249    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
250        f.debug_struct("BufferMapOperation")
251            .field("host", &self.host)
252            .field("callback", &self.callback.as_ref().map(|_| "?"))
253            .finish()
254    }
255}
256
257#[derive(Clone, Debug, Error)]
258#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
259#[non_exhaustive]
260pub enum BufferAccessError {
261    #[error(transparent)]
262    Device(#[from] DeviceError),
263    #[error("Buffer map failed")]
264    Failed,
265    #[error(transparent)]
266    DestroyedResource(#[from] DestroyedResourceError),
267    #[error("Buffer is already mapped")]
268    AlreadyMapped,
269    #[error("Buffer map is pending")]
270    MapAlreadyPending,
271    #[error(transparent)]
272    MissingBufferUsage(#[from] MissingBufferUsageError),
273    #[error("Buffer is not mapped")]
274    NotMapped,
275    #[error(
276        "Buffer map range must start aligned to `MAP_ALIGNMENT` and end to `COPY_BUFFER_ALIGNMENT`"
277    )]
278    UnalignedRange,
279    #[error("Buffer offset invalid: offset {offset} must be multiple of 8")]
280    UnalignedOffset { offset: wgt::BufferAddress },
281    #[error("Buffer range size invalid: range_size {range_size} must be multiple of 4")]
282    UnalignedRangeSize { range_size: wgt::BufferAddress },
283    #[error("Buffer access out of bounds: index {index} would underrun the buffer (limit: {min})")]
284    OutOfBoundsUnderrun {
285        index: wgt::BufferAddress,
286        min: wgt::BufferAddress,
287    },
288    #[error(
289        "Buffer access out of bounds: last index {index} would overrun the buffer (limit: {max})"
290    )]
291    OutOfBoundsOverrun {
292        index: wgt::BufferAddress,
293        max: wgt::BufferAddress,
294    },
295    #[error("Buffer map range start {start} is greater than end {end}")]
296    NegativeRange {
297        start: wgt::BufferAddress,
298        end: wgt::BufferAddress,
299    },
300    #[error("Buffer map aborted")]
301    MapAborted,
302    #[error(transparent)]
303    InvalidResource(#[from] InvalidResourceError),
304}
305
306impl WebGpuError for BufferAccessError {
307    fn webgpu_error_type(&self) -> ErrorType {
308        let e: &dyn WebGpuError = match self {
309            Self::Device(e) => e,
310            Self::InvalidResource(e) => e,
311            Self::DestroyedResource(e) => e,
312
313            Self::Failed
314            | Self::AlreadyMapped
315            | Self::MapAlreadyPending
316            | Self::MissingBufferUsage(_)
317            | Self::NotMapped
318            | Self::UnalignedRange
319            | Self::UnalignedOffset { .. }
320            | Self::UnalignedRangeSize { .. }
321            | Self::OutOfBoundsUnderrun { .. }
322            | Self::OutOfBoundsOverrun { .. }
323            | Self::NegativeRange { .. }
324            | Self::MapAborted => return ErrorType::Validation,
325        };
326        e.webgpu_error_type()
327    }
328}
329
330#[derive(Clone, Debug, Error)]
331#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
332#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")]
333pub struct MissingBufferUsageError {
334    pub(crate) res: ResourceErrorIdent,
335    pub(crate) actual: wgt::BufferUsages,
336    pub(crate) expected: wgt::BufferUsages,
337}
338
339impl WebGpuError for MissingBufferUsageError {
340    fn webgpu_error_type(&self) -> ErrorType {
341        ErrorType::Validation
342    }
343}
344
345#[derive(Clone, Debug, Error)]
346#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")]
347pub struct MissingTextureUsageError {
348    pub(crate) res: ResourceErrorIdent,
349    pub(crate) actual: wgt::TextureUsages,
350    pub(crate) expected: wgt::TextureUsages,
351}
352
353impl WebGpuError for MissingTextureUsageError {
354    fn webgpu_error_type(&self) -> ErrorType {
355        ErrorType::Validation
356    }
357}
358
359#[derive(Clone, Debug, Error)]
360#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
361#[error("{0} has been destroyed")]
362pub struct DestroyedResourceError(pub ResourceErrorIdent);
363
364impl WebGpuError for DestroyedResourceError {
365    fn webgpu_error_type(&self) -> ErrorType {
366        ErrorType::Validation
367    }
368}
369
370#[derive(Clone, Debug, Error)]
371#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
372#[error("{0} is invalid")]
373pub struct InvalidResourceError(pub ResourceErrorIdent);
374
375impl WebGpuError for InvalidResourceError {
376    fn webgpu_error_type(&self) -> ErrorType {
377        ErrorType::Validation
378    }
379}
380
381pub enum Fallible<T: ParentDevice> {
382    Valid(Arc<T>),
383    Invalid(Arc<String>),
384}
385
386impl<T: ParentDevice> Fallible<T> {
387    pub fn get(self) -> Result<Arc<T>, InvalidResourceError> {
388        match self {
389            Fallible::Valid(v) => Ok(v),
390            Fallible::Invalid(label) => Err(InvalidResourceError(ResourceErrorIdent {
391                r#type: Cow::Borrowed(T::TYPE),
392                label: (*label).clone(),
393            })),
394        }
395    }
396}
397
398impl<T: ParentDevice> Clone for Fallible<T> {
399    fn clone(&self) -> Self {
400        match self {
401            Self::Valid(v) => Self::Valid(v.clone()),
402            Self::Invalid(l) => Self::Invalid(l.clone()),
403        }
404    }
405}
406
407impl<T: ParentDevice> ResourceType for Fallible<T> {
408    const TYPE: &'static str = T::TYPE;
409}
410
411impl<T: ParentDevice + crate::storage::StorageItem> crate::storage::StorageItem for Fallible<T> {
412    type Marker = T::Marker;
413}
414
415pub type BufferAccessResult = Result<(), BufferAccessError>;
416
417#[derive(Debug)]
418pub(crate) struct BufferPendingMapping {
419    pub(crate) range: Range<wgt::BufferAddress>,
420    pub(crate) op: BufferMapOperation,
421    // hold the parent alive while the mapping is active
422    pub(crate) _parent_buffer: Arc<Buffer>,
423}
424
425pub type BufferDescriptor<'a> = wgt::BufferDescriptor<Label<'a>>;
426
427#[derive(Debug)]
428pub struct Buffer {
429    pub(crate) raw: Snatchable<Box<dyn hal::DynBuffer>>,
430    pub(crate) device: Arc<Device>,
431    pub(crate) usage: wgt::BufferUsages,
432    pub(crate) size: wgt::BufferAddress,
433    pub(crate) initialization_status: RwLock<BufferInitTracker>,
434    /// The `label` from the descriptor used to create the resource.
435    pub(crate) label: String,
436    pub(crate) tracking_data: TrackingData,
437    pub(crate) map_state: Mutex<BufferMapState>,
438    pub(crate) bind_groups: Mutex<WeakVec<BindGroup>>,
439    pub(crate) timestamp_normalization_bind_group: Snatchable<TimestampNormalizationBindGroup>,
440    pub(crate) indirect_validation_bind_groups: Snatchable<crate::indirect_validation::BindGroups>,
441}
442
443impl Drop for Buffer {
444    fn drop(&mut self) {
445        if let Some(raw) = self.timestamp_normalization_bind_group.take() {
446            raw.dispose(self.device.raw());
447        }
448
449        if let Some(raw) = self.indirect_validation_bind_groups.take() {
450            raw.dispose(self.device.raw());
451        }
452
453        if let Some(raw) = self.raw.take() {
454            resource_log!("Destroy raw {}", self.error_ident());
455            unsafe {
456                self.device.raw().destroy_buffer(raw);
457            }
458        }
459    }
460}
461
462impl RawResourceAccess for Buffer {
463    type DynResource = dyn hal::DynBuffer;
464
465    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
466        self.raw.get(guard).map(|b| b.as_ref())
467    }
468}
469
470impl Buffer {
471    pub(crate) fn check_destroyed(
472        &self,
473        guard: &SnatchGuard,
474    ) -> Result<(), DestroyedResourceError> {
475        self.raw
476            .get(guard)
477            .map(|_| ())
478            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
479    }
480
481    /// Checks that the given buffer usage contains the required buffer usage,
482    /// returns an error otherwise.
483    pub(crate) fn check_usage(
484        &self,
485        expected: wgt::BufferUsages,
486    ) -> Result<(), MissingBufferUsageError> {
487        if self.usage.contains(expected) {
488            Ok(())
489        } else {
490            Err(MissingBufferUsageError {
491                res: self.error_ident(),
492                actual: self.usage,
493                expected,
494            })
495        }
496    }
497
498    /// Resolve the size of a binding for buffer with `offset` and `size`.
499    ///
500    /// If `size` is `None`, then the remainder of the buffer starting from
501    /// `offset` is used.
502    ///
503    /// If the binding would overflow the buffer, then an error is returned.
504    ///
505    /// Zero-size bindings are permitted here for historical reasons. Although
506    /// zero-size bindings are permitted by WebGPU, they are not permitted by
507    /// some backends. See [`Buffer::binding`] and
508    /// [#3170](https://github.com/gfx-rs/wgpu/issues/3170).
509    pub fn resolve_binding_size(
510        &self,
511        offset: wgt::BufferAddress,
512        binding_size: Option<wgt::BufferSize>,
513    ) -> Result<u64, BindingError> {
514        let buffer_size = self.size;
515
516        match binding_size {
517            Some(binding_size) => match offset.checked_add(binding_size.get()) {
518                Some(end) if end <= buffer_size => Ok(binding_size.get()),
519                _ => Err(BindingError::BindingRangeTooLarge {
520                    buffer: self.error_ident(),
521                    offset,
522                    binding_size: binding_size.get(),
523                    buffer_size,
524                }),
525            },
526            None => {
527                buffer_size
528                    .checked_sub(offset)
529                    .ok_or_else(|| BindingError::BindingOffsetTooLarge {
530                        buffer: self.error_ident(),
531                        offset,
532                        buffer_size,
533                    })
534            }
535        }
536    }
537
538    /// Create a new [`hal::BufferBinding`] for the buffer with `offset` and
539    /// `binding_size`.
540    ///
541    /// If `binding_size` is `None`, then the remainder of the buffer starting
542    /// from `offset` is used.
543    ///
544    /// If the binding would overflow the buffer, then an error is returned.
545    ///
546    /// A zero-size binding at the end of the buffer is permitted here for historical reasons. Although
547    /// zero-size bindings are permitted by WebGPU, they are not permitted by
548    /// some backends. The zero-size binding need to be quashed or remapped to a
549    /// non-zero size, either universally in wgpu-core, or in specific backends
550    /// that do not support them. See
551    /// [#3170](https://github.com/gfx-rs/wgpu/issues/3170).
552    ///
553    /// Although it seems like it would be simpler and safer to use the resolved
554    /// size in the returned [`hal::BufferBinding`], doing this (and removing
555    /// redundant logic in backends to resolve the implicit size) was observed
556    /// to cause problems in certain CTS tests, so an implicit size
557    /// specification is preserved in the output.
558    pub fn binding<'a>(
559        &'a self,
560        offset: wgt::BufferAddress,
561        binding_size: Option<wgt::BufferSize>,
562        snatch_guard: &'a SnatchGuard,
563    ) -> Result<(hal::BufferBinding<'a, dyn hal::DynBuffer>, u64), BindingError> {
564        let buf_raw = self.try_raw(snatch_guard)?;
565        let resolved_size = self.resolve_binding_size(offset, binding_size)?;
566        // SAFETY: The offset and size passed to hal::BufferBinding::new_unchecked must
567        // define a binding contained within the buffer.
568        Ok((
569            hal::BufferBinding::new_unchecked(buf_raw, offset, binding_size),
570            resolved_size,
571        ))
572    }
573
574    /// Returns the mapping callback in case of error so that the callback can be fired outside
575    /// of the locks that are held in this function.
576    pub(crate) fn map_async(
577        self: &Arc<Self>,
578        offset: wgt::BufferAddress,
579        size: Option<wgt::BufferAddress>,
580        op: BufferMapOperation,
581    ) -> Result<SubmissionIndex, (BufferMapOperation, BufferAccessError)> {
582        let range_size = if let Some(size) = size {
583            size
584        } else {
585            self.size.saturating_sub(offset)
586        };
587
588        if offset % wgt::MAP_ALIGNMENT != 0 {
589            return Err((op, BufferAccessError::UnalignedOffset { offset }));
590        }
591        if range_size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
592            return Err((op, BufferAccessError::UnalignedRangeSize { range_size }));
593        }
594
595        let range = offset..(offset + range_size);
596
597        if range.start % wgt::MAP_ALIGNMENT != 0 || range.end % wgt::COPY_BUFFER_ALIGNMENT != 0 {
598            return Err((op, BufferAccessError::UnalignedRange));
599        }
600
601        let (pub_usage, internal_use) = match op.host {
602            HostMap::Read => (wgt::BufferUsages::MAP_READ, wgt::BufferUses::MAP_READ),
603            HostMap::Write => (wgt::BufferUsages::MAP_WRITE, wgt::BufferUses::MAP_WRITE),
604        };
605
606        if let Err(e) = self.check_usage(pub_usage) {
607            return Err((op, e.into()));
608        }
609
610        if range.start > range.end {
611            return Err((
612                op,
613                BufferAccessError::NegativeRange {
614                    start: range.start,
615                    end: range.end,
616                },
617            ));
618        }
619        if range.end > self.size {
620            return Err((
621                op,
622                BufferAccessError::OutOfBoundsOverrun {
623                    index: range.end,
624                    max: self.size,
625                },
626            ));
627        }
628
629        let device = &self.device;
630        if let Err(e) = device.check_is_valid() {
631            return Err((op, e.into()));
632        }
633
634        {
635            let snatch_guard = device.snatchable_lock.read();
636            if let Err(e) = self.check_destroyed(&snatch_guard) {
637                return Err((op, e.into()));
638            }
639        }
640
641        {
642            let map_state = &mut *self.map_state.lock();
643            *map_state = match *map_state {
644                BufferMapState::Init { .. } | BufferMapState::Active { .. } => {
645                    return Err((op, BufferAccessError::AlreadyMapped));
646                }
647                BufferMapState::Waiting(_) => {
648                    return Err((op, BufferAccessError::MapAlreadyPending));
649                }
650                BufferMapState::Idle => BufferMapState::Waiting(BufferPendingMapping {
651                    range,
652                    op,
653                    _parent_buffer: self.clone(),
654                }),
655            };
656        }
657
658        // TODO: we are ignoring the transition here, I think we need to add a barrier
659        // at the end of the submission
660        device
661            .trackers
662            .lock()
663            .buffers
664            .set_single(self, internal_use);
665
666        let submit_index = if let Some(queue) = device.get_queue() {
667            queue.lock_life().map(self).unwrap_or(0) // '0' means no wait is necessary
668        } else {
669            // We can safely unwrap below since we just set the `map_state` to `BufferMapState::Waiting`.
670            let (mut operation, status) = self.map(&device.snatchable_lock.read()).unwrap();
671            if let Some(callback) = operation.callback.take() {
672                callback(status);
673            }
674            0
675        };
676
677        Ok(submit_index)
678    }
679
680    /// This function returns [`None`] only if [`Self::map_state`] is not [`BufferMapState::Waiting`].
681    #[must_use]
682    pub(crate) fn map(&self, snatch_guard: &SnatchGuard) -> Option<BufferMapPendingClosure> {
683        // This _cannot_ be inlined into the match. If it is, the lock will be held
684        // open through the whole match, resulting in a deadlock when we try to re-lock
685        // the buffer back to active.
686        let mapping = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
687        let pending_mapping = match mapping {
688            BufferMapState::Waiting(pending_mapping) => pending_mapping,
689            // Mapping cancelled
690            BufferMapState::Idle => return None,
691            // Mapping queued at least twice by map -> unmap -> map
692            // and was already successfully mapped below
693            BufferMapState::Active { .. } => {
694                *self.map_state.lock() = mapping;
695                return None;
696            }
697            _ => panic!("No pending mapping."),
698        };
699        let status = if pending_mapping.range.start != pending_mapping.range.end {
700            let host = pending_mapping.op.host;
701            let size = pending_mapping.range.end - pending_mapping.range.start;
702            match crate::device::map_buffer(
703                self,
704                pending_mapping.range.start,
705                size,
706                host,
707                snatch_guard,
708            ) {
709                Ok(mapping) => {
710                    *self.map_state.lock() = BufferMapState::Active {
711                        mapping,
712                        range: pending_mapping.range.clone(),
713                        host,
714                    };
715                    Ok(())
716                }
717                Err(e) => Err(e),
718            }
719        } else {
720            *self.map_state.lock() = BufferMapState::Active {
721                mapping: hal::BufferMapping {
722                    ptr: NonNull::dangling(),
723                    is_coherent: true,
724                },
725                range: pending_mapping.range,
726                host: pending_mapping.op.host,
727            };
728            Ok(())
729        };
730        Some((pending_mapping.op, status))
731    }
732
733    // Note: This must not be called while holding a lock.
734    pub(crate) fn unmap(
735        self: &Arc<Self>,
736        #[cfg(feature = "trace")] buffer_id: crate::id::BufferId,
737    ) -> Result<(), BufferAccessError> {
738        if let Some((mut operation, status)) = self.unmap_inner(
739            #[cfg(feature = "trace")]
740            buffer_id,
741        )? {
742            if let Some(callback) = operation.callback.take() {
743                callback(status);
744            }
745        }
746
747        Ok(())
748    }
749
750    fn unmap_inner(
751        self: &Arc<Self>,
752        #[cfg(feature = "trace")] buffer_id: crate::id::BufferId,
753    ) -> Result<Option<BufferMapPendingClosure>, BufferAccessError> {
754        let device = &self.device;
755        let snatch_guard = device.snatchable_lock.read();
756        let raw_buf = self.try_raw(&snatch_guard)?;
757        match mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle) {
758            BufferMapState::Init { staging_buffer } => {
759                #[cfg(feature = "trace")]
760                if let Some(ref mut trace) = *device.trace.lock() {
761                    let data = trace.make_binary("bin", staging_buffer.get_data());
762                    trace.add(trace::Action::WriteBuffer {
763                        id: buffer_id,
764                        data,
765                        range: 0..self.size,
766                        queued: true,
767                    });
768                }
769
770                let staging_buffer = staging_buffer.flush();
771
772                if let Some(queue) = device.get_queue() {
773                    let region = wgt::BufferSize::new(self.size).map(|size| hal::BufferCopy {
774                        src_offset: 0,
775                        dst_offset: 0,
776                        size,
777                    });
778                    let transition_src = hal::BufferBarrier {
779                        buffer: staging_buffer.raw(),
780                        usage: hal::StateTransition {
781                            from: wgt::BufferUses::MAP_WRITE,
782                            to: wgt::BufferUses::COPY_SRC,
783                        },
784                    };
785                    let transition_dst = hal::BufferBarrier::<dyn hal::DynBuffer> {
786                        buffer: raw_buf,
787                        usage: hal::StateTransition {
788                            from: wgt::BufferUses::empty(),
789                            to: wgt::BufferUses::COPY_DST,
790                        },
791                    };
792                    let mut pending_writes = queue.pending_writes.lock();
793                    let encoder = pending_writes.activate();
794                    unsafe {
795                        encoder.transition_buffers(&[transition_src, transition_dst]);
796                        if self.size > 0 {
797                            encoder.copy_buffer_to_buffer(
798                                staging_buffer.raw(),
799                                raw_buf,
800                                region.as_slice(),
801                            );
802                        }
803                    }
804                    pending_writes.consume(staging_buffer);
805                    pending_writes.insert_buffer(self);
806                }
807            }
808            BufferMapState::Idle => {
809                return Err(BufferAccessError::NotMapped);
810            }
811            BufferMapState::Waiting(pending) => {
812                return Ok(Some((pending.op, Err(BufferAccessError::MapAborted))));
813            }
814            BufferMapState::Active {
815                mapping,
816                range,
817                host,
818            } => {
819                #[allow(clippy::collapsible_if)]
820                if host == HostMap::Write {
821                    #[cfg(feature = "trace")]
822                    if let Some(ref mut trace) = *device.trace.lock() {
823                        let size = range.end - range.start;
824                        let data = trace.make_binary("bin", unsafe {
825                            core::slice::from_raw_parts(mapping.ptr.as_ptr(), size as usize)
826                        });
827                        trace.add(trace::Action::WriteBuffer {
828                            id: buffer_id,
829                            data,
830                            range: range.clone(),
831                            queued: false,
832                        });
833                    }
834                    if !mapping.is_coherent {
835                        unsafe { device.raw().flush_mapped_ranges(raw_buf, &[range]) };
836                    }
837                }
838                unsafe { device.raw().unmap_buffer(raw_buf) };
839            }
840        }
841        Ok(None)
842    }
843
844    pub(crate) fn destroy(self: &Arc<Self>) {
845        let device = &self.device;
846
847        let temp = {
848            let mut snatch_guard = device.snatchable_lock.write();
849
850            let raw = match self.raw.snatch(&mut snatch_guard) {
851                Some(raw) => raw,
852                None => {
853                    // Per spec, it is valid to call `destroy` multiple times.
854                    return;
855                }
856            };
857
858            let timestamp_normalization_bind_group = self
859                .timestamp_normalization_bind_group
860                .snatch(&mut snatch_guard);
861
862            let indirect_validation_bind_groups = self
863                .indirect_validation_bind_groups
864                .snatch(&mut snatch_guard);
865
866            drop(snatch_guard);
867
868            let bind_groups = {
869                let mut guard = self.bind_groups.lock();
870                mem::take(&mut *guard)
871            };
872
873            queue::TempResource::DestroyedBuffer(DestroyedBuffer {
874                raw: ManuallyDrop::new(raw),
875                device: Arc::clone(&self.device),
876                label: self.label().to_owned(),
877                bind_groups,
878                timestamp_normalization_bind_group,
879                indirect_validation_bind_groups,
880            })
881        };
882
883        if let Some(queue) = device.get_queue() {
884            let mut pending_writes = queue.pending_writes.lock();
885            if pending_writes.contains_buffer(self) {
886                pending_writes.consume_temp(temp);
887            } else {
888                let mut life_lock = queue.lock_life();
889                let last_submit_index = life_lock.get_buffer_latest_submission_index(self);
890                if let Some(last_submit_index) = last_submit_index {
891                    life_lock.schedule_resource_destruction(temp, last_submit_index);
892                }
893            }
894        }
895    }
896}
897
898#[derive(Clone, Debug, Error)]
899#[non_exhaustive]
900pub enum CreateBufferError {
901    #[error(transparent)]
902    Device(#[from] DeviceError),
903    #[error("Failed to map buffer while creating: {0}")]
904    AccessError(#[from] BufferAccessError),
905    #[error("Buffers that are mapped at creation have to be aligned to `COPY_BUFFER_ALIGNMENT`")]
906    UnalignedSize,
907    #[error("Invalid usage flags {0:?}")]
908    InvalidUsage(wgt::BufferUsages),
909    #[error("`MAP` usage can only be combined with the opposite `COPY`, requested {0:?}")]
910    UsageMismatch(wgt::BufferUsages),
911    #[error("Buffer size {requested} is greater than the maximum buffer size ({maximum})")]
912    MaxBufferSize { requested: u64, maximum: u64 },
913    #[error(transparent)]
914    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
915    #[error(transparent)]
916    MissingFeatures(#[from] MissingFeatures),
917    #[error("Failed to create bind group for indirect buffer validation: {0}")]
918    IndirectValidationBindGroup(DeviceError),
919}
920
921crate::impl_resource_type!(Buffer);
922crate::impl_labeled!(Buffer);
923crate::impl_parent_device!(Buffer);
924crate::impl_storage_item!(Buffer);
925crate::impl_trackable!(Buffer);
926
927impl WebGpuError for CreateBufferError {
928    fn webgpu_error_type(&self) -> ErrorType {
929        let e: &dyn WebGpuError = match self {
930            Self::Device(e) => e,
931            Self::AccessError(e) => e,
932            Self::MissingDownlevelFlags(e) => e,
933            Self::IndirectValidationBindGroup(e) => e,
934            Self::MissingFeatures(e) => e,
935
936            Self::UnalignedSize
937            | Self::InvalidUsage(_)
938            | Self::UsageMismatch(_)
939            | Self::MaxBufferSize { .. } => return ErrorType::Validation,
940        };
941        e.webgpu_error_type()
942    }
943}
944
945/// A buffer that has been marked as destroyed and is staged for actual deletion soon.
946#[derive(Debug)]
947pub struct DestroyedBuffer {
948    raw: ManuallyDrop<Box<dyn hal::DynBuffer>>,
949    device: Arc<Device>,
950    label: String,
951    bind_groups: WeakVec<BindGroup>,
952    timestamp_normalization_bind_group: Option<TimestampNormalizationBindGroup>,
953    indirect_validation_bind_groups: Option<crate::indirect_validation::BindGroups>,
954}
955
956impl DestroyedBuffer {
957    pub fn label(&self) -> &dyn fmt::Debug {
958        &self.label
959    }
960}
961
962impl Drop for DestroyedBuffer {
963    fn drop(&mut self) {
964        let mut deferred = self.device.deferred_destroy.lock();
965        deferred.push(DeferredDestroy::BindGroups(mem::take(
966            &mut self.bind_groups,
967        )));
968        drop(deferred);
969
970        if let Some(raw) = self.timestamp_normalization_bind_group.take() {
971            raw.dispose(self.device.raw());
972        }
973
974        if let Some(raw) = self.indirect_validation_bind_groups.take() {
975            raw.dispose(self.device.raw());
976        }
977
978        resource_log!("Destroy raw Buffer (destroyed) {:?}", self.label());
979        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
980        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
981        unsafe {
982            hal::DynDevice::destroy_buffer(self.device.raw(), raw);
983        }
984    }
985}
986
987#[cfg(send_sync)]
988unsafe impl Send for StagingBuffer {}
989#[cfg(send_sync)]
990unsafe impl Sync for StagingBuffer {}
991
992/// A temporary buffer, consumed by the command that uses it.
993///
994/// A [`StagingBuffer`] is designed for one-shot uploads of data to the GPU. It
995/// is always created mapped, and the command that uses it destroys the buffer
996/// when it is done.
997///
998/// [`StagingBuffer`]s can be created with [`queue_create_staging_buffer`] and
999/// used with [`queue_write_staging_buffer`]. They are also used internally by
1000/// operations like [`queue_write_texture`] that need to upload data to the GPU,
1001/// but that don't belong to any particular wgpu command buffer.
1002///
1003/// Used `StagingBuffer`s are accumulated in [`Device::pending_writes`], to be
1004/// freed once their associated operation's queue submission has finished
1005/// execution.
1006///
1007/// [`queue_create_staging_buffer`]: crate::global::Global::queue_create_staging_buffer
1008/// [`queue_write_staging_buffer`]: crate::global::Global::queue_write_staging_buffer
1009/// [`queue_write_texture`]: crate::global::Global::queue_write_texture
1010/// [`Device::pending_writes`]: crate::device::Device
1011#[derive(Debug)]
1012pub struct StagingBuffer {
1013    raw: Box<dyn hal::DynBuffer>,
1014    device: Arc<Device>,
1015    pub(crate) size: wgt::BufferSize,
1016    is_coherent: bool,
1017    ptr: NonNull<u8>,
1018}
1019
1020impl StagingBuffer {
1021    pub(crate) fn new(device: &Arc<Device>, size: wgt::BufferSize) -> Result<Self, DeviceError> {
1022        profiling::scope!("StagingBuffer::new");
1023        let stage_desc = hal::BufferDescriptor {
1024            label: hal_label(Some("(wgpu internal) Staging"), device.instance_flags),
1025            size: size.get(),
1026            usage: wgt::BufferUses::MAP_WRITE | wgt::BufferUses::COPY_SRC,
1027            memory_flags: hal::MemoryFlags::TRANSIENT,
1028        };
1029
1030        let raw = unsafe { device.raw().create_buffer(&stage_desc) }
1031            .map_err(|e| device.handle_hal_error(e))?;
1032        let mapping = unsafe { device.raw().map_buffer(raw.as_ref(), 0..size.get()) }
1033            .map_err(|e| device.handle_hal_error(e))?;
1034
1035        let staging_buffer = StagingBuffer {
1036            raw,
1037            device: device.clone(),
1038            size,
1039            is_coherent: mapping.is_coherent,
1040            ptr: mapping.ptr,
1041        };
1042
1043        Ok(staging_buffer)
1044    }
1045
1046    /// SAFETY: You must not call any functions of `self`
1047    /// until you stopped using the returned pointer.
1048    pub(crate) unsafe fn ptr(&self) -> NonNull<u8> {
1049        self.ptr
1050    }
1051
1052    #[cfg(feature = "trace")]
1053    pub(crate) fn get_data(&self) -> &[u8] {
1054        unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.size.get() as usize) }
1055    }
1056
1057    pub(crate) fn write_zeros(&mut self) {
1058        unsafe { core::ptr::write_bytes(self.ptr.as_ptr(), 0, self.size.get() as usize) };
1059    }
1060
1061    pub(crate) fn write(&mut self, data: &[u8]) {
1062        assert!(data.len() >= self.size.get() as usize);
1063        // SAFETY: With the assert above, all of `copy_nonoverlapping`'s
1064        // requirements are satisfied.
1065        unsafe {
1066            core::ptr::copy_nonoverlapping(
1067                data.as_ptr(),
1068                self.ptr.as_ptr(),
1069                self.size.get() as usize,
1070            );
1071        }
1072    }
1073
1074    /// SAFETY: The offsets and size must be in-bounds.
1075    pub(crate) unsafe fn write_with_offset(
1076        &mut self,
1077        data: &[u8],
1078        src_offset: isize,
1079        dst_offset: isize,
1080        size: usize,
1081    ) {
1082        unsafe {
1083            debug_assert!(
1084                (src_offset + size as isize) as usize <= data.len(),
1085                "src_offset + size must be in-bounds: src_offset = {}, size = {}, data.len() = {}",
1086                src_offset,
1087                size,
1088                data.len()
1089            );
1090            core::ptr::copy_nonoverlapping(
1091                data.as_ptr().offset(src_offset),
1092                self.ptr.as_ptr().offset(dst_offset),
1093                size,
1094            );
1095        }
1096    }
1097
1098    pub(crate) fn flush(self) -> FlushedStagingBuffer {
1099        let device = self.device.raw();
1100        if !self.is_coherent {
1101            #[allow(clippy::single_range_in_vec_init)]
1102            unsafe {
1103                device.flush_mapped_ranges(self.raw.as_ref(), &[0..self.size.get()])
1104            };
1105        }
1106        unsafe { device.unmap_buffer(self.raw.as_ref()) };
1107
1108        let StagingBuffer {
1109            raw, device, size, ..
1110        } = self;
1111
1112        FlushedStagingBuffer {
1113            raw: ManuallyDrop::new(raw),
1114            device,
1115            size,
1116        }
1117    }
1118}
1119
1120crate::impl_resource_type!(StagingBuffer);
1121crate::impl_storage_item!(StagingBuffer);
1122
1123#[derive(Debug)]
1124pub struct FlushedStagingBuffer {
1125    raw: ManuallyDrop<Box<dyn hal::DynBuffer>>,
1126    device: Arc<Device>,
1127    pub(crate) size: wgt::BufferSize,
1128}
1129
1130impl FlushedStagingBuffer {
1131    pub(crate) fn raw(&self) -> &dyn hal::DynBuffer {
1132        self.raw.as_ref()
1133    }
1134}
1135
1136impl Drop for FlushedStagingBuffer {
1137    fn drop(&mut self) {
1138        resource_log!("Destroy raw StagingBuffer");
1139        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
1140        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1141        unsafe { self.device.raw().destroy_buffer(raw) };
1142    }
1143}
1144
1145pub type TextureDescriptor<'a> = wgt::TextureDescriptor<Label<'a>, Vec<wgt::TextureFormat>>;
1146
1147#[derive(Debug)]
1148pub(crate) enum TextureInner {
1149    Native {
1150        raw: Box<dyn hal::DynTexture>,
1151    },
1152    Surface {
1153        raw: Box<dyn hal::DynSurfaceTexture>,
1154    },
1155}
1156
1157impl TextureInner {
1158    pub(crate) fn raw(&self) -> &dyn hal::DynTexture {
1159        match self {
1160            Self::Native { raw } => raw.as_ref(),
1161            Self::Surface { raw, .. } => raw.as_ref().borrow(),
1162        }
1163    }
1164}
1165
1166#[derive(Debug)]
1167pub enum TextureClearMode {
1168    BufferCopy,
1169    // View for clear via RenderPass for every subsurface (mip/layer/slice)
1170    RenderPass {
1171        clear_views: SmallVec<[ManuallyDrop<Box<dyn hal::DynTextureView>>; 1]>,
1172        is_color: bool,
1173    },
1174    Surface {
1175        clear_view: ManuallyDrop<Box<dyn hal::DynTextureView>>,
1176    },
1177    // Texture can't be cleared, attempting to do so will cause panic.
1178    // (either because it is impossible for the type of texture or it is being destroyed)
1179    None,
1180}
1181
1182#[derive(Debug)]
1183pub struct Texture {
1184    pub(crate) inner: Snatchable<TextureInner>,
1185    pub(crate) device: Arc<Device>,
1186    pub(crate) desc: wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
1187    pub(crate) _hal_usage: wgt::TextureUses,
1188    pub(crate) format_features: wgt::TextureFormatFeatures,
1189    pub(crate) initialization_status: RwLock<TextureInitTracker>,
1190    pub(crate) full_range: TextureSelector,
1191    /// The `label` from the descriptor used to create the resource.
1192    pub(crate) label: String,
1193    pub(crate) tracking_data: TrackingData,
1194    pub(crate) clear_mode: RwLock<TextureClearMode>,
1195    pub(crate) views: Mutex<WeakVec<TextureView>>,
1196    pub(crate) bind_groups: Mutex<WeakVec<BindGroup>>,
1197}
1198
1199impl Texture {
1200    pub(crate) fn new(
1201        device: &Arc<Device>,
1202        inner: TextureInner,
1203        hal_usage: wgt::TextureUses,
1204        desc: &TextureDescriptor,
1205        format_features: wgt::TextureFormatFeatures,
1206        clear_mode: TextureClearMode,
1207        init: bool,
1208    ) -> Self {
1209        Texture {
1210            inner: Snatchable::new(inner),
1211            device: device.clone(),
1212            desc: desc.map_label(|_| ()),
1213            _hal_usage: hal_usage,
1214            format_features,
1215            initialization_status: RwLock::new(
1216                rank::TEXTURE_INITIALIZATION_STATUS,
1217                if init {
1218                    TextureInitTracker::new(desc.mip_level_count, desc.array_layer_count())
1219                } else {
1220                    TextureInitTracker::new(desc.mip_level_count, 0)
1221                },
1222            ),
1223            full_range: TextureSelector {
1224                mips: 0..desc.mip_level_count,
1225                layers: 0..desc.array_layer_count(),
1226            },
1227            label: desc.label.to_string(),
1228            tracking_data: TrackingData::new(device.tracker_indices.textures.clone()),
1229            clear_mode: RwLock::new(rank::TEXTURE_CLEAR_MODE, clear_mode),
1230            views: Mutex::new(rank::TEXTURE_VIEWS, WeakVec::new()),
1231            bind_groups: Mutex::new(rank::TEXTURE_BIND_GROUPS, WeakVec::new()),
1232        }
1233    }
1234
1235    /// Checks that the given texture usage contains the required texture usage,
1236    /// returns an error otherwise.
1237    pub(crate) fn check_usage(
1238        &self,
1239        expected: wgt::TextureUsages,
1240    ) -> Result<(), MissingTextureUsageError> {
1241        if self.desc.usage.contains(expected) {
1242            Ok(())
1243        } else {
1244            Err(MissingTextureUsageError {
1245                res: self.error_ident(),
1246                actual: self.desc.usage,
1247                expected,
1248            })
1249        }
1250    }
1251}
1252
1253impl Drop for Texture {
1254    fn drop(&mut self) {
1255        match *self.clear_mode.write() {
1256            TextureClearMode::Surface {
1257                ref mut clear_view, ..
1258            } => {
1259                // SAFETY: We are in the Drop impl and we don't use clear_view anymore after this point.
1260                let raw = unsafe { ManuallyDrop::take(clear_view) };
1261                unsafe {
1262                    self.device.raw().destroy_texture_view(raw);
1263                }
1264            }
1265            TextureClearMode::RenderPass {
1266                ref mut clear_views,
1267                ..
1268            } => {
1269                clear_views.iter_mut().for_each(|clear_view| {
1270                    // SAFETY: We are in the Drop impl and we don't use clear_view anymore after this point.
1271                    let raw = unsafe { ManuallyDrop::take(clear_view) };
1272                    unsafe {
1273                        self.device.raw().destroy_texture_view(raw);
1274                    }
1275                });
1276            }
1277            _ => {}
1278        };
1279
1280        if let Some(TextureInner::Native { raw }) = self.inner.take() {
1281            resource_log!("Destroy raw {}", self.error_ident());
1282            unsafe {
1283                self.device.raw().destroy_texture(raw);
1284            }
1285        }
1286    }
1287}
1288
1289impl RawResourceAccess for Texture {
1290    type DynResource = dyn hal::DynTexture;
1291
1292    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
1293        self.inner.get(guard).map(|t| t.raw())
1294    }
1295}
1296
1297impl Texture {
1298    pub(crate) fn try_inner<'a>(
1299        &'a self,
1300        guard: &'a SnatchGuard,
1301    ) -> Result<&'a TextureInner, DestroyedResourceError> {
1302        self.inner
1303            .get(guard)
1304            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
1305    }
1306
1307    pub(crate) fn check_destroyed(
1308        &self,
1309        guard: &SnatchGuard,
1310    ) -> Result<(), DestroyedResourceError> {
1311        self.inner
1312            .get(guard)
1313            .map(|_| ())
1314            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
1315    }
1316
1317    pub(crate) fn get_clear_view<'a>(
1318        clear_mode: &'a TextureClearMode,
1319        desc: &'a wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
1320        mip_level: u32,
1321        depth_or_layer: u32,
1322    ) -> &'a dyn hal::DynTextureView {
1323        match *clear_mode {
1324            TextureClearMode::BufferCopy => {
1325                panic!("Given texture is cleared with buffer copies, not render passes")
1326            }
1327            TextureClearMode::None => {
1328                panic!("Given texture can't be cleared")
1329            }
1330            TextureClearMode::Surface { ref clear_view, .. } => clear_view.as_ref(),
1331            TextureClearMode::RenderPass {
1332                ref clear_views, ..
1333            } => {
1334                let index = if desc.dimension == wgt::TextureDimension::D3 {
1335                    (0..mip_level).fold(0, |acc, mip| {
1336                        acc + (desc.size.depth_or_array_layers >> mip).max(1)
1337                    })
1338                } else {
1339                    mip_level * desc.size.depth_or_array_layers
1340                } + depth_or_layer;
1341                clear_views[index as usize].as_ref()
1342            }
1343        }
1344    }
1345
1346    pub(crate) fn destroy(self: &Arc<Self>) {
1347        let device = &self.device;
1348
1349        let temp = {
1350            let raw = match self.inner.snatch(&mut device.snatchable_lock.write()) {
1351                Some(TextureInner::Native { raw }) => raw,
1352                Some(TextureInner::Surface { .. }) => {
1353                    return;
1354                }
1355                None => {
1356                    // Per spec, it is valid to call `destroy` multiple times.
1357                    return;
1358                }
1359            };
1360
1361            let views = {
1362                let mut guard = self.views.lock();
1363                mem::take(&mut *guard)
1364            };
1365
1366            let bind_groups = {
1367                let mut guard = self.bind_groups.lock();
1368                mem::take(&mut *guard)
1369            };
1370
1371            queue::TempResource::DestroyedTexture(DestroyedTexture {
1372                raw: ManuallyDrop::new(raw),
1373                views,
1374                clear_mode: mem::replace(&mut *self.clear_mode.write(), TextureClearMode::None),
1375                bind_groups,
1376                device: Arc::clone(&self.device),
1377                label: self.label().to_owned(),
1378            })
1379        };
1380
1381        if let Some(queue) = device.get_queue() {
1382            let mut pending_writes = queue.pending_writes.lock();
1383            if pending_writes.contains_texture(self) {
1384                pending_writes.consume_temp(temp);
1385            } else {
1386                let mut life_lock = queue.lock_life();
1387                let last_submit_index = life_lock.get_texture_latest_submission_index(self);
1388                if let Some(last_submit_index) = last_submit_index {
1389                    life_lock.schedule_resource_destruction(temp, last_submit_index);
1390                }
1391            }
1392        }
1393    }
1394}
1395
1396/// A texture that has been marked as destroyed and is staged for actual deletion soon.
1397#[derive(Debug)]
1398pub struct DestroyedTexture {
1399    raw: ManuallyDrop<Box<dyn hal::DynTexture>>,
1400    views: WeakVec<TextureView>,
1401    clear_mode: TextureClearMode,
1402    bind_groups: WeakVec<BindGroup>,
1403    device: Arc<Device>,
1404    label: String,
1405}
1406
1407impl DestroyedTexture {
1408    pub fn label(&self) -> &dyn fmt::Debug {
1409        &self.label
1410    }
1411}
1412
1413impl Drop for DestroyedTexture {
1414    fn drop(&mut self) {
1415        let device = &self.device;
1416
1417        let mut deferred = device.deferred_destroy.lock();
1418        deferred.push(DeferredDestroy::TextureViews(mem::take(&mut self.views)));
1419        deferred.push(DeferredDestroy::BindGroups(mem::take(
1420            &mut self.bind_groups,
1421        )));
1422        drop(deferred);
1423
1424        match mem::replace(&mut self.clear_mode, TextureClearMode::None) {
1425            TextureClearMode::RenderPass { clear_views, .. } => {
1426                for clear_view in clear_views {
1427                    let raw = ManuallyDrop::into_inner(clear_view);
1428                    unsafe { self.device.raw().destroy_texture_view(raw) };
1429                }
1430            }
1431            TextureClearMode::Surface { clear_view } => {
1432                let raw = ManuallyDrop::into_inner(clear_view);
1433                unsafe { self.device.raw().destroy_texture_view(raw) };
1434            }
1435            _ => (),
1436        }
1437
1438        resource_log!("Destroy raw Texture (destroyed) {:?}", self.label());
1439        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
1440        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1441        unsafe {
1442            self.device.raw().destroy_texture(raw);
1443        }
1444    }
1445}
1446
1447#[derive(Clone, Copy, Debug)]
1448pub enum TextureErrorDimension {
1449    X,
1450    Y,
1451    Z,
1452}
1453
1454#[derive(Clone, Debug, Error)]
1455#[non_exhaustive]
1456pub enum TextureDimensionError {
1457    #[error("Dimension {0:?} is zero")]
1458    Zero(TextureErrorDimension),
1459    #[error("Dimension {dim:?} value {given} exceeds the limit of {limit}")]
1460    LimitExceeded {
1461        dim: TextureErrorDimension,
1462        given: u32,
1463        limit: u32,
1464    },
1465    #[error("Sample count {0} is invalid")]
1466    InvalidSampleCount(u32),
1467    #[error("Width {width} is not a multiple of {format:?}'s block width ({block_width})")]
1468    NotMultipleOfBlockWidth {
1469        width: u32,
1470        block_width: u32,
1471        format: wgt::TextureFormat,
1472    },
1473    #[error("Height {height} is not a multiple of {format:?}'s block height ({block_height})")]
1474    NotMultipleOfBlockHeight {
1475        height: u32,
1476        block_height: u32,
1477        format: wgt::TextureFormat,
1478    },
1479    #[error(
1480        "Width {width} is not a multiple of {format:?}'s width multiple requirement ({multiple})"
1481    )]
1482    WidthNotMultipleOf {
1483        width: u32,
1484        multiple: u32,
1485        format: wgt::TextureFormat,
1486    },
1487    #[error("Height {height} is not a multiple of {format:?}'s height multiple requirement ({multiple})")]
1488    HeightNotMultipleOf {
1489        height: u32,
1490        multiple: u32,
1491        format: wgt::TextureFormat,
1492    },
1493    #[error("Multisampled texture depth or array layers must be 1, got {0}")]
1494    MultisampledDepthOrArrayLayer(u32),
1495}
1496
1497impl WebGpuError for TextureDimensionError {
1498    fn webgpu_error_type(&self) -> ErrorType {
1499        ErrorType::Validation
1500    }
1501}
1502
1503#[derive(Clone, Debug, Error)]
1504#[non_exhaustive]
1505pub enum CreateTextureError {
1506    #[error(transparent)]
1507    Device(#[from] DeviceError),
1508    #[error(transparent)]
1509    CreateTextureView(#[from] CreateTextureViewError),
1510    #[error("Invalid usage flags {0:?}")]
1511    InvalidUsage(wgt::TextureUsages),
1512    #[error(transparent)]
1513    InvalidDimension(#[from] TextureDimensionError),
1514    #[error("Depth texture ({1:?}) can't be created as {0:?}")]
1515    InvalidDepthDimension(wgt::TextureDimension, wgt::TextureFormat),
1516    #[error("Compressed texture ({1:?}) can't be created as {0:?}")]
1517    InvalidCompressedDimension(wgt::TextureDimension, wgt::TextureFormat),
1518    #[error(
1519        "Texture descriptor mip level count {requested} is invalid, maximum allowed is {maximum}"
1520    )]
1521    InvalidMipLevelCount { requested: u32, maximum: u32 },
1522    #[error(
1523        "Texture usages {0:?} are not allowed on a texture of type {1:?}{downlevel_suffix}",
1524        downlevel_suffix = if *.2 { " due to downlevel restrictions" } else { "" }
1525    )]
1526    InvalidFormatUsages(wgt::TextureUsages, wgt::TextureFormat, bool),
1527    #[error("The view format {0:?} is not compatible with texture format {1:?}, only changing srgb-ness is allowed.")]
1528    InvalidViewFormat(wgt::TextureFormat, wgt::TextureFormat),
1529    #[error("Texture usages {0:?} are not allowed on a texture of dimensions {1:?}")]
1530    InvalidDimensionUsages(wgt::TextureUsages, wgt::TextureDimension),
1531    #[error("Texture usage STORAGE_BINDING is not allowed for multisampled textures")]
1532    InvalidMultisampledStorageBinding,
1533    #[error("Format {0:?} does not support multisampling")]
1534    InvalidMultisampledFormat(wgt::TextureFormat),
1535    #[error("Sample count {0} is not supported by format {1:?} on this device. The WebGPU spec guarantees {2:?} samples are supported by this format. With the TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES feature your device supports {3:?}.")]
1536    InvalidSampleCount(u32, wgt::TextureFormat, Vec<u32>, Vec<u32>),
1537    #[error("Multisampled textures must have RENDER_ATTACHMENT usage")]
1538    MultisampledNotRenderAttachment,
1539    #[error("Texture format {0:?} can't be used due to missing features")]
1540    MissingFeatures(wgt::TextureFormat, #[source] MissingFeatures),
1541    #[error(transparent)]
1542    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
1543}
1544
1545crate::impl_resource_type!(Texture);
1546crate::impl_labeled!(Texture);
1547crate::impl_parent_device!(Texture);
1548crate::impl_storage_item!(Texture);
1549crate::impl_trackable!(Texture);
1550
1551impl Borrow<TextureSelector> for Texture {
1552    fn borrow(&self) -> &TextureSelector {
1553        &self.full_range
1554    }
1555}
1556
1557impl WebGpuError for CreateTextureError {
1558    fn webgpu_error_type(&self) -> ErrorType {
1559        let e: &dyn WebGpuError = match self {
1560            Self::Device(e) => e,
1561            Self::CreateTextureView(e) => e,
1562            Self::InvalidDimension(e) => e,
1563            Self::MissingFeatures(_, e) => e,
1564            Self::MissingDownlevelFlags(e) => e,
1565
1566            Self::InvalidUsage(_)
1567            | Self::InvalidDepthDimension(_, _)
1568            | Self::InvalidCompressedDimension(_, _)
1569            | Self::InvalidMipLevelCount { .. }
1570            | Self::InvalidFormatUsages(_, _, _)
1571            | Self::InvalidViewFormat(_, _)
1572            | Self::InvalidDimensionUsages(_, _)
1573            | Self::InvalidMultisampledStorageBinding
1574            | Self::InvalidMultisampledFormat(_)
1575            | Self::InvalidSampleCount(..)
1576            | Self::MultisampledNotRenderAttachment => return ErrorType::Validation,
1577        };
1578        e.webgpu_error_type()
1579    }
1580}
1581
1582/// Describes a [`TextureView`].
1583#[derive(Clone, Debug, Default, Eq, PartialEq)]
1584#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
1585#[cfg_attr(feature = "serde", serde(default))]
1586pub struct TextureViewDescriptor<'a> {
1587    /// Debug label of the texture view.
1588    ///
1589    /// This will show up in graphics debuggers for easy identification.
1590    pub label: Label<'a>,
1591    /// Format of the texture view, or `None` for the same format as the texture
1592    /// itself.
1593    ///
1594    /// At this time, it must be the same the underlying format of the texture.
1595    pub format: Option<wgt::TextureFormat>,
1596    /// The dimension of the texture view.
1597    ///
1598    /// - For 1D textures, this must be `D1`.
1599    /// - For 2D textures it must be one of `D2`, `D2Array`, `Cube`, or `CubeArray`.
1600    /// - For 3D textures it must be `D3`.
1601    pub dimension: Option<wgt::TextureViewDimension>,
1602    /// The allowed usage(s) for the texture view. Must be a subset of the usage flags of the texture.
1603    /// If not provided, defaults to the full set of usage flags of the texture.
1604    pub usage: Option<wgt::TextureUsages>,
1605    /// Range within the texture that is accessible via this view.
1606    pub range: wgt::ImageSubresourceRange,
1607}
1608
1609#[derive(Debug)]
1610pub(crate) struct HalTextureViewDescriptor {
1611    pub texture_format: wgt::TextureFormat,
1612    pub format: wgt::TextureFormat,
1613    pub usage: wgt::TextureUsages,
1614    pub dimension: wgt::TextureViewDimension,
1615    pub range: wgt::ImageSubresourceRange,
1616}
1617
1618impl HalTextureViewDescriptor {
1619    pub fn aspects(&self) -> hal::FormatAspects {
1620        hal::FormatAspects::new(self.texture_format, self.range.aspect)
1621    }
1622}
1623
1624#[derive(Debug, Copy, Clone, Error)]
1625pub enum TextureViewNotRenderableReason {
1626    #[error("The texture this view references doesn't include the RENDER_ATTACHMENT usage. Provided usages: {0:?}")]
1627    Usage(wgt::TextureUsages),
1628    #[error("The dimension of this texture view is not 2D. View dimension: {0:?}")]
1629    Dimension(wgt::TextureViewDimension),
1630    #[error("This texture view has more than one mipmap level. View mipmap levels: {0:?}")]
1631    MipLevelCount(u32),
1632    #[error("This texture view has more than one array layer. View array layers: {0:?}")]
1633    ArrayLayerCount(u32),
1634    #[error(
1635        "The aspects of this texture view are a subset of the aspects in the original texture. Aspects: {0:?}"
1636    )]
1637    Aspects(hal::FormatAspects),
1638}
1639
1640#[derive(Debug)]
1641pub struct TextureView {
1642    pub(crate) raw: Snatchable<Box<dyn hal::DynTextureView>>,
1643    // if it's a surface texture - it's none
1644    pub(crate) parent: Arc<Texture>,
1645    pub(crate) device: Arc<Device>,
1646    pub(crate) desc: HalTextureViewDescriptor,
1647    pub(crate) format_features: wgt::TextureFormatFeatures,
1648    /// This is `Err` only if the texture view is not renderable
1649    pub(crate) render_extent: Result<wgt::Extent3d, TextureViewNotRenderableReason>,
1650    pub(crate) samples: u32,
1651    pub(crate) selector: TextureSelector,
1652    /// The `label` from the descriptor used to create the resource.
1653    pub(crate) label: String,
1654    pub(crate) tracking_data: TrackingData,
1655}
1656
1657impl Drop for TextureView {
1658    fn drop(&mut self) {
1659        if let Some(raw) = self.raw.take() {
1660            resource_log!("Destroy raw {}", self.error_ident());
1661            unsafe {
1662                self.device.raw().destroy_texture_view(raw);
1663            }
1664        }
1665    }
1666}
1667
1668impl RawResourceAccess for TextureView {
1669    type DynResource = dyn hal::DynTextureView;
1670
1671    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
1672        self.raw.get(guard).map(|it| it.as_ref())
1673    }
1674
1675    fn try_raw<'a>(
1676        &'a self,
1677        guard: &'a SnatchGuard,
1678    ) -> Result<&'a Self::DynResource, DestroyedResourceError> {
1679        self.parent.check_destroyed(guard)?;
1680
1681        self.raw(guard)
1682            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
1683    }
1684}
1685
1686impl TextureView {
1687    /// Checks that the given texture usage contains the required texture usage,
1688    /// returns an error otherwise.
1689    pub(crate) fn check_usage(
1690        &self,
1691        expected: wgt::TextureUsages,
1692    ) -> Result<(), MissingTextureUsageError> {
1693        if self.desc.usage.contains(expected) {
1694            Ok(())
1695        } else {
1696            Err(MissingTextureUsageError {
1697                res: self.error_ident(),
1698                actual: self.desc.usage,
1699                expected,
1700            })
1701        }
1702    }
1703}
1704
1705#[derive(Clone, Debug, Error)]
1706#[non_exhaustive]
1707pub enum CreateTextureViewError {
1708    #[error(transparent)]
1709    Device(#[from] DeviceError),
1710    #[error(transparent)]
1711    DestroyedResource(#[from] DestroyedResourceError),
1712    #[error("Invalid texture view dimension `{view:?}` with texture of dimension `{texture:?}`")]
1713    InvalidTextureViewDimension {
1714        view: wgt::TextureViewDimension,
1715        texture: wgt::TextureDimension,
1716    },
1717    #[error("Texture view format `{0:?}` is not renderable")]
1718    TextureViewFormatNotRenderable(wgt::TextureFormat),
1719    #[error("Texture view format `{0:?}` is not storage bindable")]
1720    TextureViewFormatNotStorage(wgt::TextureFormat),
1721    #[error("Invalid texture view usage `{view:?}` with texture of usage `{texture:?}`")]
1722    InvalidTextureViewUsage {
1723        view: wgt::TextureUsages,
1724        texture: wgt::TextureUsages,
1725    },
1726    #[error("Invalid texture view dimension `{0:?}` of a multisampled texture")]
1727    InvalidMultisampledTextureViewDimension(wgt::TextureViewDimension),
1728    #[error("Invalid texture depth `{depth}` for texture view of dimension `Cubemap`. Cubemap views must use images of size 6.")]
1729    InvalidCubemapTextureDepth { depth: u32 },
1730    #[error("Invalid texture depth `{depth}` for texture view of dimension `CubemapArray`. Cubemap views must use images with sizes which are a multiple of 6.")]
1731    InvalidCubemapArrayTextureDepth { depth: u32 },
1732    #[error("Source texture width and height must be equal for a texture view of dimension `Cube`/`CubeArray`")]
1733    InvalidCubeTextureViewSize,
1734    #[error("Mip level count is 0")]
1735    ZeroMipLevelCount,
1736    #[error("Array layer count is 0")]
1737    ZeroArrayLayerCount,
1738    #[error(
1739        "TextureView mip level count + base mip level {requested} must be <= Texture mip level count {total}"
1740    )]
1741    TooManyMipLevels { requested: u32, total: u32 },
1742    #[error("TextureView array layer count + base array layer {requested} must be <= Texture depth/array layer count {total}")]
1743    TooManyArrayLayers { requested: u32, total: u32 },
1744    #[error("Requested array layer count {requested} is not valid for the target view dimension {dim:?}")]
1745    InvalidArrayLayerCount {
1746        requested: u32,
1747        dim: wgt::TextureViewDimension,
1748    },
1749    #[error("Aspect {requested_aspect:?} is not in the source texture format {texture_format:?}")]
1750    InvalidAspect {
1751        texture_format: wgt::TextureFormat,
1752        requested_aspect: wgt::TextureAspect,
1753    },
1754    #[error("Unable to view texture {texture:?} as {view:?}")]
1755    FormatReinterpretation {
1756        texture: wgt::TextureFormat,
1757        view: wgt::TextureFormat,
1758    },
1759    #[error(transparent)]
1760    InvalidResource(#[from] InvalidResourceError),
1761    #[error(transparent)]
1762    MissingFeatures(#[from] MissingFeatures),
1763}
1764
1765impl WebGpuError for CreateTextureViewError {
1766    fn webgpu_error_type(&self) -> ErrorType {
1767        match self {
1768            Self::Device(e) => e.webgpu_error_type(),
1769
1770            Self::InvalidTextureViewDimension { .. }
1771            | Self::InvalidResource(_)
1772            | Self::InvalidMultisampledTextureViewDimension(_)
1773            | Self::InvalidCubemapTextureDepth { .. }
1774            | Self::InvalidCubemapArrayTextureDepth { .. }
1775            | Self::InvalidCubeTextureViewSize
1776            | Self::ZeroMipLevelCount
1777            | Self::ZeroArrayLayerCount
1778            | Self::TooManyMipLevels { .. }
1779            | Self::TooManyArrayLayers { .. }
1780            | Self::InvalidArrayLayerCount { .. }
1781            | Self::InvalidAspect { .. }
1782            | Self::FormatReinterpretation { .. }
1783            | Self::DestroyedResource(_)
1784            | Self::TextureViewFormatNotRenderable(_)
1785            | Self::TextureViewFormatNotStorage(_)
1786            | Self::InvalidTextureViewUsage { .. }
1787            | Self::MissingFeatures(_) => ErrorType::Validation,
1788        }
1789    }
1790}
1791
1792#[derive(Clone, Debug, Error)]
1793#[non_exhaustive]
1794pub enum TextureViewDestroyError {}
1795
1796crate::impl_resource_type!(TextureView);
1797crate::impl_labeled!(TextureView);
1798crate::impl_parent_device!(TextureView);
1799crate::impl_storage_item!(TextureView);
1800crate::impl_trackable!(TextureView);
1801
1802pub type ExternalTextureDescriptor<'a> = wgt::ExternalTextureDescriptor<Label<'a>>;
1803
1804#[derive(Debug)]
1805pub struct ExternalTexture {
1806    pub(crate) device: Arc<Device>,
1807    /// Between 1 and 3 (inclusive) planes of texture data.
1808    pub(crate) planes: arrayvec::ArrayVec<Arc<TextureView>, 3>,
1809    /// Buffer containing a [`crate::device::resource::ExternalTextureParams`]
1810    /// describing the external texture.
1811    pub(crate) params: Arc<Buffer>,
1812    /// The `label` from the descriptor used to create the resource.
1813    pub(crate) label: String,
1814    pub(crate) tracking_data: TrackingData,
1815}
1816
1817impl Drop for ExternalTexture {
1818    fn drop(&mut self) {
1819        resource_log!("Destroy raw {}", self.error_ident());
1820    }
1821}
1822
1823impl ExternalTexture {
1824    pub(crate) fn destroy(self: &Arc<Self>) {
1825        self.params.destroy();
1826    }
1827}
1828
1829#[derive(Clone, Debug, Error)]
1830#[non_exhaustive]
1831pub enum CreateExternalTextureError {
1832    #[error(transparent)]
1833    Device(#[from] DeviceError),
1834    #[error(transparent)]
1835    MissingFeatures(#[from] MissingFeatures),
1836    #[error(transparent)]
1837    InvalidResource(#[from] InvalidResourceError),
1838    #[error(transparent)]
1839    CreateBuffer(#[from] CreateBufferError),
1840    #[error(transparent)]
1841    QueueWrite(#[from] queue::QueueWriteError),
1842    #[error("External texture format {format:?} expects {expected} planes, but given {provided}")]
1843    IncorrectPlaneCount {
1844        format: wgt::ExternalTextureFormat,
1845        expected: usize,
1846        provided: usize,
1847    },
1848    #[error("External texture planes cannot be multisampled, but given view with samples = {0}")]
1849    InvalidPlaneMultisample(u32),
1850    #[error("External texture planes expect a filterable float sample type, but given view with format {format:?} (sample type {sample_type:?})")]
1851    InvalidPlaneSampleType {
1852        format: wgt::TextureFormat,
1853        sample_type: wgt::TextureSampleType,
1854    },
1855    #[error("External texture planes expect 2D dimension, but given view with dimension = {0:?}")]
1856    InvalidPlaneDimension(wgt::TextureViewDimension),
1857    #[error(transparent)]
1858    MissingTextureUsage(#[from] MissingTextureUsageError),
1859    #[error("External texture format {format:?} plane {plane} expects format with {expected} components but given view with format {provided:?} ({} components)",
1860        provided.components())]
1861    InvalidPlaneFormat {
1862        format: wgt::ExternalTextureFormat,
1863        plane: usize,
1864        expected: u8,
1865        provided: wgt::TextureFormat,
1866    },
1867}
1868
1869impl WebGpuError for CreateExternalTextureError {
1870    fn webgpu_error_type(&self) -> ErrorType {
1871        let e: &dyn WebGpuError = match self {
1872            CreateExternalTextureError::Device(e) => e,
1873            CreateExternalTextureError::MissingFeatures(e) => e,
1874            CreateExternalTextureError::InvalidResource(e) => e,
1875            CreateExternalTextureError::CreateBuffer(e) => e,
1876            CreateExternalTextureError::QueueWrite(e) => e,
1877            CreateExternalTextureError::MissingTextureUsage(e) => e,
1878            CreateExternalTextureError::IncorrectPlaneCount { .. }
1879            | CreateExternalTextureError::InvalidPlaneMultisample(_)
1880            | CreateExternalTextureError::InvalidPlaneSampleType { .. }
1881            | CreateExternalTextureError::InvalidPlaneDimension(_)
1882            | CreateExternalTextureError::InvalidPlaneFormat { .. } => {
1883                return ErrorType::Validation
1884            }
1885        };
1886        e.webgpu_error_type()
1887    }
1888}
1889
1890crate::impl_resource_type!(ExternalTexture);
1891crate::impl_labeled!(ExternalTexture);
1892crate::impl_parent_device!(ExternalTexture);
1893crate::impl_storage_item!(ExternalTexture);
1894crate::impl_trackable!(ExternalTexture);
1895
1896/// Describes a [`Sampler`]
1897#[derive(Clone, Debug, PartialEq)]
1898#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
1899pub struct SamplerDescriptor<'a> {
1900    /// Debug label of the sampler.
1901    ///
1902    /// This will show up in graphics debuggers for easy identification.
1903    pub label: Label<'a>,
1904    /// How to deal with out of bounds accesses in the u (i.e. x) direction
1905    pub address_modes: [wgt::AddressMode; 3],
1906    /// How to filter the texture when it needs to be magnified (made larger)
1907    pub mag_filter: wgt::FilterMode,
1908    /// How to filter the texture when it needs to be minified (made smaller)
1909    pub min_filter: wgt::FilterMode,
1910    /// How to filter between mip map levels
1911    pub mipmap_filter: wgt::FilterMode,
1912    /// Minimum level of detail (i.e. mip level) to use
1913    pub lod_min_clamp: f32,
1914    /// Maximum level of detail (i.e. mip level) to use
1915    pub lod_max_clamp: f32,
1916    /// If this is enabled, this is a comparison sampler using the given comparison function.
1917    pub compare: Option<wgt::CompareFunction>,
1918    /// Must be at least 1. If this is not 1, all filter modes must be linear.
1919    pub anisotropy_clamp: u16,
1920    /// Border color to use when address_mode is
1921    /// [`AddressMode::ClampToBorder`](wgt::AddressMode::ClampToBorder)
1922    pub border_color: Option<wgt::SamplerBorderColor>,
1923}
1924
1925#[derive(Debug)]
1926pub struct Sampler {
1927    pub(crate) raw: ManuallyDrop<Box<dyn hal::DynSampler>>,
1928    pub(crate) device: Arc<Device>,
1929    /// The `label` from the descriptor used to create the resource.
1930    pub(crate) label: String,
1931    pub(crate) tracking_data: TrackingData,
1932    /// `true` if this is a comparison sampler
1933    pub(crate) comparison: bool,
1934    /// `true` if this is a filtering sampler
1935    pub(crate) filtering: bool,
1936}
1937
1938impl Drop for Sampler {
1939    fn drop(&mut self) {
1940        resource_log!("Destroy raw {}", self.error_ident());
1941        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
1942        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1943        unsafe {
1944            self.device.raw().destroy_sampler(raw);
1945        }
1946    }
1947}
1948
1949impl Sampler {
1950    pub(crate) fn raw(&self) -> &dyn hal::DynSampler {
1951        self.raw.as_ref()
1952    }
1953}
1954
1955#[derive(Copy, Clone)]
1956pub enum SamplerFilterErrorType {
1957    MagFilter,
1958    MinFilter,
1959    MipmapFilter,
1960}
1961
1962impl fmt::Debug for SamplerFilterErrorType {
1963    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1964        match *self {
1965            SamplerFilterErrorType::MagFilter => write!(f, "magFilter"),
1966            SamplerFilterErrorType::MinFilter => write!(f, "minFilter"),
1967            SamplerFilterErrorType::MipmapFilter => write!(f, "mipmapFilter"),
1968        }
1969    }
1970}
1971
1972#[derive(Clone, Debug, Error)]
1973#[non_exhaustive]
1974pub enum CreateSamplerError {
1975    #[error(transparent)]
1976    Device(#[from] DeviceError),
1977    #[error("Invalid lodMinClamp: {0}. Must be greater or equal to 0.0")]
1978    InvalidLodMinClamp(f32),
1979    #[error("Invalid lodMaxClamp: {lod_max_clamp}. Must be greater or equal to lodMinClamp (which is {lod_min_clamp}).")]
1980    InvalidLodMaxClamp {
1981        lod_min_clamp: f32,
1982        lod_max_clamp: f32,
1983    },
1984    #[error("Invalid anisotropic clamp: {0}. Must be at least 1.")]
1985    InvalidAnisotropy(u16),
1986    #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")]
1987    InvalidFilterModeWithAnisotropy {
1988        filter_type: SamplerFilterErrorType,
1989        filter_mode: wgt::FilterMode,
1990        anisotropic_clamp: u16,
1991    },
1992    #[error(transparent)]
1993    MissingFeatures(#[from] MissingFeatures),
1994}
1995
1996crate::impl_resource_type!(Sampler);
1997crate::impl_labeled!(Sampler);
1998crate::impl_parent_device!(Sampler);
1999crate::impl_storage_item!(Sampler);
2000crate::impl_trackable!(Sampler);
2001
2002impl WebGpuError for CreateSamplerError {
2003    fn webgpu_error_type(&self) -> ErrorType {
2004        let e: &dyn WebGpuError = match self {
2005            Self::Device(e) => e,
2006            Self::MissingFeatures(e) => e,
2007
2008            Self::InvalidLodMinClamp(_)
2009            | Self::InvalidLodMaxClamp { .. }
2010            | Self::InvalidAnisotropy(_)
2011            | Self::InvalidFilterModeWithAnisotropy { .. } => return ErrorType::Validation,
2012        };
2013        e.webgpu_error_type()
2014    }
2015}
2016
2017#[derive(Clone, Debug, Error)]
2018#[non_exhaustive]
2019pub enum CreateQuerySetError {
2020    #[error(transparent)]
2021    Device(#[from] DeviceError),
2022    #[error("QuerySets cannot be made with zero queries")]
2023    ZeroCount,
2024    #[error("{count} is too many queries for a single QuerySet. QuerySets cannot be made more than {maximum} queries.")]
2025    TooManyQueries { count: u32, maximum: u32 },
2026    #[error(transparent)]
2027    MissingFeatures(#[from] MissingFeatures),
2028}
2029
2030impl WebGpuError for CreateQuerySetError {
2031    fn webgpu_error_type(&self) -> ErrorType {
2032        let e: &dyn WebGpuError = match self {
2033            Self::Device(e) => e,
2034            Self::MissingFeatures(e) => e,
2035
2036            Self::TooManyQueries { .. } | Self::ZeroCount => return ErrorType::Validation,
2037        };
2038        e.webgpu_error_type()
2039    }
2040}
2041
2042pub type QuerySetDescriptor<'a> = wgt::QuerySetDescriptor<Label<'a>>;
2043
2044#[derive(Debug)]
2045pub struct QuerySet {
2046    pub(crate) raw: ManuallyDrop<Box<dyn hal::DynQuerySet>>,
2047    pub(crate) device: Arc<Device>,
2048    /// The `label` from the descriptor used to create the resource.
2049    pub(crate) label: String,
2050    pub(crate) tracking_data: TrackingData,
2051    pub(crate) desc: wgt::QuerySetDescriptor<()>,
2052}
2053
2054impl Drop for QuerySet {
2055    fn drop(&mut self) {
2056        resource_log!("Destroy raw {}", self.error_ident());
2057        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
2058        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
2059        unsafe {
2060            self.device.raw().destroy_query_set(raw);
2061        }
2062    }
2063}
2064
2065crate::impl_resource_type!(QuerySet);
2066crate::impl_labeled!(QuerySet);
2067crate::impl_parent_device!(QuerySet);
2068crate::impl_storage_item!(QuerySet);
2069crate::impl_trackable!(QuerySet);
2070
2071impl QuerySet {
2072    pub(crate) fn raw(&self) -> &dyn hal::DynQuerySet {
2073        self.raw.as_ref()
2074    }
2075}
2076
2077pub type BlasDescriptor<'a> = wgt::CreateBlasDescriptor<Label<'a>>;
2078pub type TlasDescriptor<'a> = wgt::CreateTlasDescriptor<Label<'a>>;
2079
2080pub type BlasPrepareCompactResult = Result<(), BlasPrepareCompactError>;
2081
2082#[cfg(send_sync)]
2083pub type BlasCompactCallback = Box<dyn FnOnce(BlasPrepareCompactResult) + Send + 'static>;
2084#[cfg(not(send_sync))]
2085pub type BlasCompactCallback = Box<dyn FnOnce(BlasPrepareCompactResult) + 'static>;
2086
2087pub(crate) struct BlasPendingCompact {
2088    pub(crate) op: Option<BlasCompactCallback>,
2089    // hold the parent alive while the mapping is active
2090    pub(crate) _parent_blas: Arc<Blas>,
2091}
2092
2093impl fmt::Debug for BlasPendingCompact {
2094    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2095        f.debug_struct("BlasPendingCompact")
2096            .field("op", &())
2097            .field("_parent_blas", &self._parent_blas)
2098            .finish()
2099    }
2100}
2101
2102#[derive(Debug)]
2103pub(crate) enum BlasCompactState {
2104    /// Created from a compact operation.
2105    Compacted,
2106    /// Waiting for GPU to be done before mapping to get compacted size
2107    Waiting(BlasPendingCompact),
2108    /// Ready to be compacted
2109    Ready { size: wgt::BufferAddress },
2110    /// Ready to prepare to compact.
2111    Idle,
2112}
2113
2114#[cfg(send_sync)]
2115unsafe impl Send for BlasCompactState {}
2116#[cfg(send_sync)]
2117unsafe impl Sync for BlasCompactState {}
2118
2119#[derive(Debug)]
2120pub struct Blas {
2121    pub(crate) raw: Snatchable<Box<dyn hal::DynAccelerationStructure>>,
2122    pub(crate) device: Arc<Device>,
2123    pub(crate) size_info: hal::AccelerationStructureBuildSizes,
2124    pub(crate) sizes: wgt::BlasGeometrySizeDescriptors,
2125    pub(crate) flags: wgt::AccelerationStructureFlags,
2126    pub(crate) update_mode: wgt::AccelerationStructureUpdateMode,
2127    pub(crate) built_index: RwLock<Option<NonZeroU64>>,
2128    pub(crate) handle: u64,
2129    /// The `label` from the descriptor used to create the resource.
2130    pub(crate) label: String,
2131    pub(crate) tracking_data: TrackingData,
2132    pub(crate) compaction_buffer: Option<ManuallyDrop<Box<dyn hal::DynBuffer>>>,
2133    pub(crate) compacted_state: Mutex<BlasCompactState>,
2134}
2135
2136impl Drop for Blas {
2137    fn drop(&mut self) {
2138        resource_log!("Destroy raw {}", self.error_ident());
2139        // SAFETY: We are in the Drop impl, and we don't use self.raw or self.compaction_buffer anymore after this point.
2140        if let Some(raw) = self.raw.take() {
2141            unsafe {
2142                self.device.raw().destroy_acceleration_structure(raw);
2143            }
2144        }
2145        if let Some(mut raw) = self.compaction_buffer.take() {
2146            unsafe {
2147                self.device
2148                    .raw()
2149                    .destroy_buffer(ManuallyDrop::take(&mut raw))
2150            }
2151        }
2152    }
2153}
2154
2155impl RawResourceAccess for Blas {
2156    type DynResource = dyn hal::DynAccelerationStructure;
2157
2158    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2159        self.raw.get(guard).map(|it| it.as_ref())
2160    }
2161}
2162
2163impl Blas {
2164    pub(crate) fn prepare_compact_async(
2165        self: &Arc<Self>,
2166        op: Option<BlasCompactCallback>,
2167    ) -> Result<SubmissionIndex, (Option<BlasCompactCallback>, BlasPrepareCompactError)> {
2168        let device = &self.device;
2169        if let Err(e) = device.check_is_valid() {
2170            return Err((op, e.into()));
2171        }
2172
2173        if self.built_index.read().is_none() {
2174            return Err((op, BlasPrepareCompactError::NotBuilt));
2175        }
2176
2177        if !self
2178            .flags
2179            .contains(wgt::AccelerationStructureFlags::ALLOW_COMPACTION)
2180        {
2181            return Err((op, BlasPrepareCompactError::CompactionUnsupported));
2182        }
2183
2184        let mut state = self.compacted_state.lock();
2185        *state = match *state {
2186            BlasCompactState::Compacted => {
2187                return Err((op, BlasPrepareCompactError::DoubleCompaction))
2188            }
2189            BlasCompactState::Waiting(_) => {
2190                return Err((op, BlasPrepareCompactError::CompactionPreparingAlready))
2191            }
2192            BlasCompactState::Ready { .. } => {
2193                return Err((op, BlasPrepareCompactError::CompactionPreparingAlready))
2194            }
2195            BlasCompactState::Idle => BlasCompactState::Waiting(BlasPendingCompact {
2196                op,
2197                _parent_blas: self.clone(),
2198            }),
2199        };
2200
2201        let submit_index = if let Some(queue) = device.get_queue() {
2202            queue.lock_life().prepare_compact(self).unwrap_or(0) // '0' means no wait is necessary
2203        } else {
2204            // We can safely unwrap below since we just set the `compacted_state` to `BlasCompactState::Waiting`.
2205            let (mut callback, status) = self.read_back_compact_size().unwrap();
2206            if let Some(callback) = callback.take() {
2207                callback(status);
2208            }
2209            0
2210        };
2211
2212        Ok(submit_index)
2213    }
2214
2215    /// This function returns [`None`] only if [`Self::compacted_state`] is not [`BlasCompactState::Waiting`].
2216    #[must_use]
2217    pub(crate) fn read_back_compact_size(&self) -> Option<BlasCompactReadyPendingClosure> {
2218        let mut state = self.compacted_state.lock();
2219        let pending_compact = match mem::replace(&mut *state, BlasCompactState::Idle) {
2220            BlasCompactState::Waiting(pending_mapping) => pending_mapping,
2221            // Compaction cancelled e.g. by rebuild
2222            BlasCompactState::Idle => return None,
2223            BlasCompactState::Ready { .. } => {
2224                unreachable!("This should be validated out by `prepare_for_compaction`")
2225            }
2226            _ => panic!("No pending mapping."),
2227        };
2228        let status = {
2229            let compaction_buffer = self.compaction_buffer.as_ref().unwrap().as_ref();
2230            unsafe {
2231                let map_res = self.device.raw().map_buffer(
2232                    compaction_buffer,
2233                    0..size_of::<wgpu_types::BufferAddress>() as wgt::BufferAddress,
2234                );
2235                match map_res {
2236                    Ok(mapping) => {
2237                        if !mapping.is_coherent {
2238                            // Clippy complains about this because it might not be intended, but
2239                            // this is intentional.
2240                            #[expect(clippy::single_range_in_vec_init)]
2241                            self.device.raw().flush_mapped_ranges(
2242                                compaction_buffer,
2243                                &[0..size_of::<wgpu_types::BufferAddress>() as wgt::BufferAddress],
2244                            );
2245                        }
2246                        let size = core::ptr::read_unaligned(
2247                            mapping.ptr.as_ptr().cast::<wgt::BufferAddress>(),
2248                        );
2249                        self.device.raw().unmap_buffer(compaction_buffer);
2250                        if self.size_info.acceleration_structure_size != 0 {
2251                            debug_assert_ne!(size, 0);
2252                        }
2253                        *state = BlasCompactState::Ready { size };
2254                        Ok(())
2255                    }
2256                    Err(err) => Err(BlasPrepareCompactError::from(DeviceError::from_hal(err))),
2257                }
2258            }
2259        };
2260        Some((pending_compact.op, status))
2261    }
2262}
2263
2264crate::impl_resource_type!(Blas);
2265crate::impl_labeled!(Blas);
2266crate::impl_parent_device!(Blas);
2267crate::impl_storage_item!(Blas);
2268crate::impl_trackable!(Blas);
2269
2270#[derive(Debug)]
2271pub struct Tlas {
2272    pub(crate) raw: Snatchable<Box<dyn hal::DynAccelerationStructure>>,
2273    pub(crate) device: Arc<Device>,
2274    pub(crate) size_info: hal::AccelerationStructureBuildSizes,
2275    pub(crate) max_instance_count: u32,
2276    pub(crate) flags: wgt::AccelerationStructureFlags,
2277    pub(crate) update_mode: wgt::AccelerationStructureUpdateMode,
2278    pub(crate) built_index: RwLock<Option<NonZeroU64>>,
2279    pub(crate) dependencies: RwLock<Vec<Arc<Blas>>>,
2280    pub(crate) instance_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
2281    /// The `label` from the descriptor used to create the resource.
2282    pub(crate) label: String,
2283    pub(crate) tracking_data: TrackingData,
2284}
2285
2286impl Drop for Tlas {
2287    fn drop(&mut self) {
2288        unsafe {
2289            resource_log!("Destroy raw {}", self.error_ident());
2290            if let Some(structure) = self.raw.take() {
2291                self.device.raw().destroy_acceleration_structure(structure);
2292            }
2293            let buffer = ManuallyDrop::take(&mut self.instance_buffer);
2294            self.device.raw().destroy_buffer(buffer);
2295        }
2296    }
2297}
2298
2299impl RawResourceAccess for Tlas {
2300    type DynResource = dyn hal::DynAccelerationStructure;
2301
2302    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2303        self.raw.get(guard).map(|raw| raw.as_ref())
2304    }
2305}
2306
2307crate::impl_resource_type!(Tlas);
2308crate::impl_labeled!(Tlas);
2309crate::impl_parent_device!(Tlas);
2310crate::impl_storage_item!(Tlas);
2311crate::impl_trackable!(Tlas);