wgpu_core/device/
mod.rs

1use alloc::{boxed::Box, string::String, vec::Vec};
2use core::{fmt, num::NonZeroU32};
3
4use crate::{
5    binding_model,
6    ray_tracing::BlasCompactReadyPendingClosure,
7    resource::{
8        Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation, Labeled,
9        RawResourceAccess, ResourceErrorIdent,
10    },
11    snatch::SnatchGuard,
12    Label, DOWNLEVEL_ERROR_MESSAGE,
13};
14
15use arrayvec::ArrayVec;
16use smallvec::SmallVec;
17use thiserror::Error;
18use wgt::{
19    error::{ErrorType, WebGpuError},
20    BufferAddress, DeviceLostReason, MapMode, TextureFormat,
21};
22
23pub(crate) mod bgl;
24mod life;
25pub mod queue;
26pub mod ray_tracing;
27pub mod resource;
28pub(crate) mod surface_config;
29#[cfg(any(feature = "trace", feature = "replay"))]
30pub mod trace;
31pub use {life::WaitIdleError, resource::Device};
32
33pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES;
34// Should be large enough for the largest possible texture row. This
35// value is enough for a 16k texture with float4 format.
36pub(crate) const ZERO_BUFFER_SIZE: BufferAddress = 512 << 10;
37
38pub(crate) const ENTRYPOINT_FAILURE_ERROR: &str = "The given EntryPoint is Invalid";
39
40pub type DeviceDescriptor<'a> = wgt::DeviceDescriptor<Label<'a>>;
41pub type QueueDescriptor<'a> = wgt::QueueDescriptor<Label<'a>>;
42
43#[derive(Clone, Debug, Hash, PartialEq)]
44#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
45pub(crate) struct AttachmentData<T> {
46    pub colors: ArrayVec<Option<T>, { hal::MAX_COLOR_ATTACHMENTS }>,
47    pub resolves: ArrayVec<T, { hal::MAX_COLOR_ATTACHMENTS }>,
48    pub depth_stencil: Option<T>,
49}
50impl<T: PartialEq> Eq for AttachmentData<T> {}
51
52#[derive(Clone, Debug, Hash, PartialEq)]
53#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
54pub(crate) struct RenderPassContext {
55    pub attachments: AttachmentData<TextureFormat>,
56    pub sample_count: u32,
57    pub multiview_mask: Option<NonZeroU32>,
58}
59
60impl Default for RenderPassContext {
61    fn default() -> Self {
62        Self {
63            attachments: AttachmentData {
64                colors: ArrayVec::new(),
65                resolves: ArrayVec::new(),
66                depth_stencil: None,
67            },
68            sample_count: Default::default(),
69            multiview_mask: Default::default(),
70        }
71    }
72}
73
74#[derive(Clone, Debug, Error)]
75#[non_exhaustive]
76pub enum RenderPassCompatibilityError {
77    #[error(
78        "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {res} uses attachments with formats {actual:?}",
79    )]
80    IncompatibleColorAttachment {
81        indices: Vec<usize>,
82        expected: Vec<Option<TextureFormat>>,
83        actual: Vec<Option<TextureFormat>>,
84        res: ResourceErrorIdent,
85    },
86    #[error(
87        "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {res} uses an attachment with format {actual:?}",
88    )]
89    IncompatibleDepthStencilAttachment {
90        expected: Option<TextureFormat>,
91        actual: Option<TextureFormat>,
92        res: ResourceErrorIdent,
93    },
94    #[error(
95        "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {res} uses attachments with format {actual:?}",
96    )]
97    IncompatibleSampleCount {
98        expected: u32,
99        actual: u32,
100        res: ResourceErrorIdent,
101    },
102    #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {res} uses setting {actual:?}")]
103    IncompatibleMultiview {
104        expected: Option<NonZeroU32>,
105        actual: Option<NonZeroU32>,
106        res: ResourceErrorIdent,
107    },
108}
109
110impl WebGpuError for RenderPassCompatibilityError {
111    fn webgpu_error_type(&self) -> ErrorType {
112        ErrorType::Validation
113    }
114}
115
116impl RenderPassContext {
117    // Assumes the renderpass only contains one subpass
118    pub(crate) fn check_compatible<T: Labeled>(
119        &self,
120        other: &Self,
121        res: &T,
122    ) -> Result<(), RenderPassCompatibilityError> {
123        if self.attachments.colors != other.attachments.colors {
124            let indices = self
125                .attachments
126                .colors
127                .iter()
128                .zip(&other.attachments.colors)
129                .enumerate()
130                .filter_map(|(idx, (left, right))| (left != right).then_some(idx))
131                .collect();
132            return Err(RenderPassCompatibilityError::IncompatibleColorAttachment {
133                indices,
134                expected: self.attachments.colors.iter().cloned().collect(),
135                actual: other.attachments.colors.iter().cloned().collect(),
136                res: res.error_ident(),
137            });
138        }
139        if self.attachments.depth_stencil != other.attachments.depth_stencil {
140            return Err(
141                RenderPassCompatibilityError::IncompatibleDepthStencilAttachment {
142                    expected: self.attachments.depth_stencil,
143                    actual: other.attachments.depth_stencil,
144                    res: res.error_ident(),
145                },
146            );
147        }
148        if self.sample_count != other.sample_count {
149            return Err(RenderPassCompatibilityError::IncompatibleSampleCount {
150                expected: self.sample_count,
151                actual: other.sample_count,
152                res: res.error_ident(),
153            });
154        }
155        if self.multiview_mask != other.multiview_mask {
156            return Err(RenderPassCompatibilityError::IncompatibleMultiview {
157                expected: self.multiview_mask,
158                actual: other.multiview_mask,
159                res: res.error_ident(),
160            });
161        }
162        Ok(())
163    }
164}
165
166pub type BufferMapPendingClosure = (BufferMapOperation, BufferAccessResult);
167
168#[must_use]
169#[derive(Default)]
170pub struct UserClosures {
171    pub mappings: Vec<BufferMapPendingClosure>,
172    pub blas_compact_ready: Vec<BlasCompactReadyPendingClosure>,
173    pub submissions: SmallVec<[queue::SubmittedWorkDoneClosure; 1]>,
174    pub device_lost_invocations: SmallVec<[DeviceLostInvocation; 1]>,
175}
176
177impl UserClosures {
178    pub(crate) fn extend(&mut self, other: Self) {
179        self.mappings.extend(other.mappings);
180        self.blas_compact_ready.extend(other.blas_compact_ready);
181        self.submissions.extend(other.submissions);
182        self.device_lost_invocations
183            .extend(other.device_lost_invocations);
184    }
185
186    pub(crate) fn fire(self) {
187        // Note: this logic is specifically moved out of `handle_mapping()` in order to
188        // have nothing locked by the time we execute users callback code.
189
190        // Mappings _must_ be fired before submissions, as the spec requires all mapping callbacks that are registered before
191        // a on_submitted_work_done callback to be fired before the on_submitted_work_done callback.
192        for (mut operation, status) in self.mappings {
193            if let Some(callback) = operation.callback.take() {
194                callback(status);
195            }
196        }
197        for (mut operation, status) in self.blas_compact_ready {
198            if let Some(callback) = operation.take() {
199                callback(status);
200            }
201        }
202        for closure in self.submissions {
203            closure();
204        }
205        for invocation in self.device_lost_invocations {
206            (invocation.closure)(invocation.reason, invocation.message);
207        }
208    }
209}
210
211#[cfg(send_sync)]
212pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + Send + 'static>;
213#[cfg(not(send_sync))]
214pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + 'static>;
215
216pub struct DeviceLostInvocation {
217    closure: DeviceLostClosure,
218    reason: DeviceLostReason,
219    message: String,
220}
221
222pub(crate) fn map_buffer(
223    buffer: &Buffer,
224    offset: BufferAddress,
225    size: BufferAddress,
226    kind: MapMode,
227    snatch_guard: &SnatchGuard,
228) -> Result<hal::BufferMapping, BufferAccessError> {
229    let raw_device = buffer.device.raw();
230    let raw_buffer = buffer.try_raw(snatch_guard)?;
231    let mapping = unsafe {
232        raw_device
233            .map_buffer(raw_buffer, offset..offset + size)
234            .map_err(|e| buffer.device.handle_hal_error(e))?
235    };
236
237    if !mapping.is_coherent && kind == MapMode::Read {
238        #[allow(clippy::single_range_in_vec_init)]
239        unsafe {
240            raw_device.invalidate_mapped_ranges(raw_buffer, &[offset..offset + size]);
241        }
242    }
243
244    assert_eq!(offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
245    assert_eq!(size % wgt::COPY_BUFFER_ALIGNMENT, 0);
246    // Zero out uninitialized parts of the mapping. (Spec dictates all resources
247    // behave as if they were initialized with zero)
248    //
249    // If this is a read mapping, ideally we would use a `clear_buffer` command
250    // before reading the data from GPU (i.e. `invalidate_range`). However, this
251    // would require us to kick off and wait for a command buffer or piggy back
252    // on an existing one (the later is likely the only worthwhile option). As
253    // reading uninitialized memory isn't a particular important path to
254    // support, we instead just initialize the memory here and make sure it is
255    // GPU visible, so this happens at max only once for every buffer region.
256    //
257    // If this is a write mapping zeroing out the memory here is the only
258    // reasonable way as all data is pushed to GPU anyways.
259
260    let mapped = unsafe { core::slice::from_raw_parts_mut(mapping.ptr.as_ptr(), size as usize) };
261
262    // We can't call flush_mapped_ranges in this case, so we can't drain the uninitialized ranges either
263    if !mapping.is_coherent
264        && kind == MapMode::Read
265        && !buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
266    {
267        for uninitialized in buffer
268            .initialization_status
269            .write()
270            .uninitialized(offset..(size + offset))
271        {
272            // The mapping's pointer is already offset, however we track the
273            // uninitialized range relative to the buffer's start.
274            let fill_range =
275                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
276            mapped[fill_range].fill(0);
277        }
278    } else {
279        for uninitialized in buffer
280            .initialization_status
281            .write()
282            .drain(offset..(size + offset))
283        {
284            // The mapping's pointer is already offset, however we track the
285            // uninitialized range relative to the buffer's start.
286            let fill_range =
287                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
288            mapped[fill_range].fill(0);
289
290            // NOTE: This is only possible when MAPPABLE_PRIMARY_BUFFERS is enabled.
291            if !mapping.is_coherent
292                && kind == MapMode::Read
293                && buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
294            {
295                unsafe { raw_device.flush_mapped_ranges(raw_buffer, &[uninitialized]) };
296            }
297        }
298    }
299
300    Ok(mapping)
301}
302
303#[derive(Clone, Debug)]
304#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
305pub struct DeviceMismatch {
306    pub(super) res: ResourceErrorIdent,
307    pub(super) res_device: ResourceErrorIdent,
308    pub(super) target: Option<ResourceErrorIdent>,
309    pub(super) target_device: ResourceErrorIdent,
310}
311
312impl fmt::Display for DeviceMismatch {
313    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
314        write!(
315            f,
316            "{} of {} doesn't match {}",
317            self.res_device, self.res, self.target_device
318        )?;
319        if let Some(target) = self.target.as_ref() {
320            write!(f, " of {target}")?;
321        }
322        Ok(())
323    }
324}
325
326impl core::error::Error for DeviceMismatch {}
327
328impl WebGpuError for DeviceMismatch {
329    fn webgpu_error_type(&self) -> ErrorType {
330        ErrorType::Validation
331    }
332}
333
334#[derive(Clone, Debug, Error)]
335#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
336#[non_exhaustive]
337pub enum DeviceError {
338    #[error("Parent device is lost")]
339    Lost,
340    #[error("Not enough memory left.")]
341    OutOfMemory,
342    #[error(transparent)]
343    DeviceMismatch(#[from] Box<DeviceMismatch>),
344}
345
346impl WebGpuError for DeviceError {
347    fn webgpu_error_type(&self) -> ErrorType {
348        match self {
349            Self::DeviceMismatch(e) => e.webgpu_error_type(),
350            Self::Lost => ErrorType::DeviceLost,
351            Self::OutOfMemory => ErrorType::OutOfMemory,
352        }
353    }
354}
355
356impl DeviceError {
357    /// Only use this function in contexts where there is no `Device`.
358    ///
359    /// Use [`Device::handle_hal_error`] otherwise.
360    pub fn from_hal(error: hal::DeviceError) -> Self {
361        match error {
362            hal::DeviceError::Lost => Self::Lost,
363            hal::DeviceError::OutOfMemory => Self::OutOfMemory,
364            hal::DeviceError::Unexpected => Self::Lost,
365        }
366    }
367}
368
369#[derive(Clone, Debug, Error)]
370#[error("Features {0:?} are required but not enabled on the device")]
371pub struct MissingFeatures(pub wgt::Features);
372
373impl WebGpuError for MissingFeatures {
374    fn webgpu_error_type(&self) -> ErrorType {
375        ErrorType::Validation
376    }
377}
378
379#[derive(Clone, Debug, Error)]
380#[error(
381    "Downlevel flags {0:?} are required but not supported on the device.\n{DOWNLEVEL_ERROR_MESSAGE}",
382)]
383pub struct MissingDownlevelFlags(pub wgt::DownlevelFlags);
384
385impl WebGpuError for MissingDownlevelFlags {
386    fn webgpu_error_type(&self) -> ErrorType {
387        ErrorType::Validation
388    }
389}
390
391pub use wgpu_naga_bridge::create_validator;
392pub use wgpu_naga_bridge::features_to_naga_capabilities;
wgpu_core/device/mod.rs

wgpu_core/device/
mod.rs
