wgpu_core/device/
mod.rs

1use alloc::{boxed::Box, string::String, vec::Vec};
2use core::{fmt, num::NonZeroU32};
3
4use crate::{
5    binding_model,
6    ray_tracing::BlasCompactReadyPendingClosure,
7    resource::{
8        Buffer, BufferAccessError, BufferAccessResult, BufferMapCallback, Labeled,
9        RawResourceAccess, ResourceErrorIdent,
10    },
11    snatch::SnatchGuard,
12    Label, DOWNLEVEL_ERROR_MESSAGE,
13};
14
15use arrayvec::ArrayVec;
16use smallvec::SmallVec;
17use thiserror::Error;
18use wgt::{
19    error::{ErrorType, WebGpuError},
20    BufferAddress, DeviceLostReason, MapMode, TextureFormat,
21};
22
23pub(crate) mod bgl;
24mod life;
25pub mod queue;
26pub mod ray_tracing;
27pub mod resource;
28mod stage_compiler;
29pub(crate) mod surface_config;
30#[cfg(any(feature = "trace", feature = "replay"))]
31pub mod trace;
32pub use {life::WaitIdleError, resource::Device};
33
34pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES;
35// Should be large enough for the largest possible texture row. This
36// value is enough for a 16k texture with float4 format.
37pub(crate) const ZERO_BUFFER_SIZE: BufferAddress = 512 << 10;
38
39pub(crate) const ENTRYPOINT_FAILURE_ERROR: &str = "The given EntryPoint is Invalid";
40
41pub type DeviceDescriptor<'a> = wgt::DeviceDescriptor<Label<'a>>;
42pub type QueueDescriptor<'a> = wgt::QueueDescriptor<Label<'a>>;
43
44#[derive(Clone, Debug, Hash, PartialEq)]
45#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
46pub(crate) struct AttachmentData<T> {
47    pub colors: ArrayVec<Option<T>, { hal::MAX_COLOR_ATTACHMENTS }>,
48    pub resolves: ArrayVec<T, { hal::MAX_COLOR_ATTACHMENTS }>,
49    pub depth_stencil: Option<T>,
50}
51impl<T: PartialEq> Eq for AttachmentData<T> {}
52
53#[derive(Clone, Debug, Hash, PartialEq)]
54#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
55pub(crate) struct RenderPassContext {
56    pub attachments: AttachmentData<TextureFormat>,
57    pub sample_count: u32,
58    pub multiview_mask: Option<NonZeroU32>,
59}
60
61impl Default for RenderPassContext {
62    fn default() -> Self {
63        Self {
64            attachments: AttachmentData {
65                colors: ArrayVec::new(),
66                resolves: ArrayVec::new(),
67                depth_stencil: None,
68            },
69            sample_count: Default::default(),
70            multiview_mask: Default::default(),
71        }
72    }
73}
74
75#[derive(Clone, Debug, Error)]
76#[non_exhaustive]
77pub enum RenderPassCompatibilityError {
78    #[error(
79        "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {res} uses attachments with formats {actual:?}",
80    )]
81    IncompatibleColorAttachment {
82        indices: Vec<usize>,
83        expected: Vec<Option<TextureFormat>>,
84        actual: Vec<Option<TextureFormat>>,
85        res: ResourceErrorIdent,
86    },
87    #[error(
88        "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {res} uses an attachment with format {actual:?}",
89    )]
90    IncompatibleDepthStencilAttachment {
91        expected: Option<TextureFormat>,
92        actual: Option<TextureFormat>,
93        res: ResourceErrorIdent,
94    },
95    #[error(
96        "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {res} uses attachments with format {actual:?}",
97    )]
98    IncompatibleSampleCount {
99        expected: u32,
100        actual: u32,
101        res: ResourceErrorIdent,
102    },
103    #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {res} uses setting {actual:?}")]
104    IncompatibleMultiview {
105        expected: Option<NonZeroU32>,
106        actual: Option<NonZeroU32>,
107        res: ResourceErrorIdent,
108    },
109}
110
111impl WebGpuError for RenderPassCompatibilityError {
112    fn webgpu_error_type(&self) -> ErrorType {
113        ErrorType::Validation
114    }
115}
116
117impl RenderPassContext {
118    // Assumes the renderpass only contains one subpass
119    pub(crate) fn check_compatible<T: Labeled>(
120        &self,
121        other: &Self,
122        res: &T,
123    ) -> Result<(), RenderPassCompatibilityError> {
124        if self.attachments.colors != other.attachments.colors {
125            let indices = self
126                .attachments
127                .colors
128                .iter()
129                .zip(&other.attachments.colors)
130                .enumerate()
131                .filter_map(|(idx, (left, right))| (left != right).then_some(idx))
132                .collect();
133            return Err(RenderPassCompatibilityError::IncompatibleColorAttachment {
134                indices,
135                expected: self.attachments.colors.iter().cloned().collect(),
136                actual: other.attachments.colors.iter().cloned().collect(),
137                res: res.error_ident(),
138            });
139        }
140        if self.attachments.depth_stencil != other.attachments.depth_stencil {
141            return Err(
142                RenderPassCompatibilityError::IncompatibleDepthStencilAttachment {
143                    expected: self.attachments.depth_stencil,
144                    actual: other.attachments.depth_stencil,
145                    res: res.error_ident(),
146                },
147            );
148        }
149        if self.sample_count != other.sample_count {
150            return Err(RenderPassCompatibilityError::IncompatibleSampleCount {
151                expected: self.sample_count,
152                actual: other.sample_count,
153                res: res.error_ident(),
154            });
155        }
156        if self.multiview_mask != other.multiview_mask {
157            return Err(RenderPassCompatibilityError::IncompatibleMultiview {
158                expected: self.multiview_mask,
159                actual: other.multiview_mask,
160                res: res.error_ident(),
161            });
162        }
163        Ok(())
164    }
165}
166
167pub type BufferMapPendingClosure = (Option<BufferMapCallback>, BufferAccessResult);
168
169#[must_use]
170#[derive(Default)]
171pub struct UserClosures {
172    pub mappings: Vec<BufferMapPendingClosure>,
173    pub blas_compact_ready: Vec<BlasCompactReadyPendingClosure>,
174    pub submissions: SmallVec<[queue::SubmittedWorkDoneClosure; 1]>,
175    pub device_lost_invocations: SmallVec<[DeviceLostInvocation; 1]>,
176}
177
178impl UserClosures {
179    pub(crate) fn extend(&mut self, other: Self) {
180        self.mappings.extend(other.mappings);
181        self.blas_compact_ready.extend(other.blas_compact_ready);
182        self.submissions.extend(other.submissions);
183        self.device_lost_invocations
184            .extend(other.device_lost_invocations);
185    }
186
187    pub(crate) fn fire(self) {
188        // Note: this logic is specifically moved out of `handle_mapping()` in order to
189        // have nothing locked by the time we execute users callback code.
190
191        // Mappings _must_ be fired before submissions, as the spec requires all mapping callbacks that are registered before
192        // a on_submitted_work_done callback to be fired before the on_submitted_work_done callback.
193        for (mut callback, status) in self.mappings {
194            if let Some(callback) = callback.take() {
195                callback(status);
196            }
197        }
198        for (mut operation, status) in self.blas_compact_ready {
199            if let Some(callback) = operation.take() {
200                callback(status);
201            }
202        }
203        for closure in self.submissions {
204            closure();
205        }
206        for invocation in self.device_lost_invocations {
207            (invocation.closure)(invocation.reason, invocation.message);
208        }
209    }
210}
211
212#[cfg(send_sync)]
213pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + Send + 'static>;
214#[cfg(not(send_sync))]
215pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + 'static>;
216
217pub struct DeviceLostInvocation {
218    closure: DeviceLostClosure,
219    reason: DeviceLostReason,
220    message: String,
221}
222
223pub(crate) fn map_buffer(
224    buffer: &Buffer,
225    offset: BufferAddress,
226    size: BufferAddress,
227    kind: MapMode,
228    snatch_guard: &SnatchGuard,
229) -> Result<hal::BufferMapping, BufferAccessError> {
230    let raw_device = buffer.device.raw();
231    let raw_buffer = buffer.try_raw(snatch_guard)?;
232    let mapping = unsafe {
233        raw_device
234            .map_buffer(raw_buffer, offset..offset + size)
235            .map_err(|e| buffer.device.handle_hal_error(e))?
236    };
237
238    if !mapping.is_coherent && kind == MapMode::Read {
239        #[allow(clippy::single_range_in_vec_init)]
240        unsafe {
241            raw_device.invalidate_mapped_ranges(raw_buffer, &[offset..offset + size]);
242        }
243    }
244
245    assert_eq!(offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
246    assert_eq!(size % wgt::COPY_BUFFER_ALIGNMENT, 0);
247    // Zero out uninitialized parts of the mapping. (Spec dictates all resources
248    // behave as if they were initialized with zero)
249    //
250    // If this is a read mapping, ideally we would use a `clear_buffer` command
251    // before reading the data from GPU (i.e. `invalidate_range`). However, this
252    // would require us to kick off and wait for a command buffer or piggy back
253    // on an existing one (the later is likely the only worthwhile option). As
254    // reading uninitialized memory isn't a particular important path to
255    // support, we instead just initialize the memory here and make sure it is
256    // GPU visible, so this happens at max only once for every buffer region.
257    //
258    // If this is a write mapping zeroing out the memory here is the only
259    // reasonable way as all data is pushed to GPU anyways.
260
261    let mapped = unsafe { core::slice::from_raw_parts_mut(mapping.ptr.as_ptr(), size as usize) };
262
263    // We can't call flush_mapped_ranges in this case, so we can't drain the uninitialized ranges either
264    if !mapping.is_coherent
265        && kind == MapMode::Read
266        && !buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
267    {
268        for uninitialized in buffer
269            .initialization_status
270            .write()
271            .uninitialized(offset..(size + offset))
272        {
273            // The mapping's pointer is already offset, however we track the
274            // uninitialized range relative to the buffer's start.
275            let fill_range =
276                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
277            mapped[fill_range].fill(0);
278        }
279    } else {
280        for uninitialized in buffer
281            .initialization_status
282            .write()
283            .drain(offset..(size + offset))
284        {
285            // The mapping's pointer is already offset, however we track the
286            // uninitialized range relative to the buffer's start.
287            let fill_range =
288                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
289            mapped[fill_range].fill(0);
290
291            // NOTE: This is only possible when MAPPABLE_PRIMARY_BUFFERS is enabled.
292            if !mapping.is_coherent
293                && kind == MapMode::Read
294                && buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
295            {
296                unsafe { raw_device.flush_mapped_ranges(raw_buffer, &[uninitialized]) };
297            }
298        }
299    }
300
301    Ok(mapping)
302}
303
304#[derive(Clone, Debug)]
305#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
306pub struct DeviceMismatch {
307    pub(super) res: ResourceErrorIdent,
308    pub(super) res_device: ResourceErrorIdent,
309    pub(super) target: Option<ResourceErrorIdent>,
310    pub(super) target_device: ResourceErrorIdent,
311}
312
313impl fmt::Display for DeviceMismatch {
314    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
315        write!(
316            f,
317            "{} of {} doesn't match {}",
318            self.res_device, self.res, self.target_device
319        )?;
320        if let Some(target) = self.target.as_ref() {
321            write!(f, " of {target}")?;
322        }
323        Ok(())
324    }
325}
326
327impl core::error::Error for DeviceMismatch {}
328
329impl WebGpuError for DeviceMismatch {
330    fn webgpu_error_type(&self) -> ErrorType {
331        ErrorType::Validation
332    }
333}
334
335#[derive(Clone, Debug, Error)]
336#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
337#[non_exhaustive]
338pub enum DeviceError {
339    #[error("Parent device is lost")]
340    Lost,
341    #[error("Not enough memory left.")]
342    OutOfMemory,
343    #[error(transparent)]
344    DeviceMismatch(#[from] Box<DeviceMismatch>),
345}
346
347impl WebGpuError for DeviceError {
348    fn webgpu_error_type(&self) -> ErrorType {
349        match self {
350            Self::DeviceMismatch(e) => e.webgpu_error_type(),
351            Self::Lost => ErrorType::DeviceLost,
352            Self::OutOfMemory => ErrorType::OutOfMemory,
353        }
354    }
355}
356
357impl DeviceError {
358    /// Only use this function in contexts where there is no `Device`.
359    ///
360    /// Use [`Device::handle_hal_error`] otherwise.
361    pub fn from_hal(error: hal::DeviceError) -> Self {
362        match error {
363            hal::DeviceError::Lost => Self::Lost,
364            hal::DeviceError::OutOfMemory => Self::OutOfMemory,
365            hal::DeviceError::Unexpected => Self::Lost,
366        }
367    }
368}
369
370#[derive(Clone, Debug, Error)]
371#[error("Features {0:?} are required but not enabled on the device")]
372pub struct MissingFeatures(pub wgt::Features);
373
374impl WebGpuError for MissingFeatures {
375    fn webgpu_error_type(&self) -> ErrorType {
376        ErrorType::Validation
377    }
378}
379
380#[derive(Clone, Debug, Error)]
381#[error(
382    "Downlevel flags {0:?} are required but not supported on the device.\n{DOWNLEVEL_ERROR_MESSAGE}",
383)]
384pub struct MissingDownlevelFlags(pub wgt::DownlevelFlags);
385
386impl WebGpuError for MissingDownlevelFlags {
387    fn webgpu_error_type(&self) -> ErrorType {
388        ErrorType::Validation
389    }
390}
391
392pub use wgpu_naga_bridge::create_validator;
393pub use wgpu_naga_bridge::features_to_naga_capabilities;
wgpu_core/device/mod.rs

wgpu_core/device/
mod.rs
