wgpu_core/device/
mod.rs

1use alloc::{boxed::Box, string::String, vec::Vec};
2use core::{fmt, num::NonZeroU32};
3
4use crate::{
5    binding_model,
6    ray_tracing::BlasCompactReadyPendingClosure,
7    resource::{
8        Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation, Labeled,
9        RawResourceAccess, ResourceErrorIdent,
10    },
11    snatch::SnatchGuard,
12    Label, DOWNLEVEL_ERROR_MESSAGE,
13};
14
15use arrayvec::ArrayVec;
16use smallvec::SmallVec;
17use thiserror::Error;
18use wgt::{
19    error::{ErrorType, WebGpuError},
20    BufferAddress, DeviceLostReason, TextureFormat,
21};
22
23pub(crate) mod bgl;
24mod life;
25pub mod queue;
26pub mod ray_tracing;
27pub mod resource;
28pub(crate) mod surface_config;
29#[cfg(any(feature = "trace", feature = "replay"))]
30pub mod trace;
31pub use {life::WaitIdleError, resource::Device};
32
33pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES;
34// Should be large enough for the largest possible texture row. This
35// value is enough for a 16k texture with float4 format.
36pub(crate) const ZERO_BUFFER_SIZE: BufferAddress = 512 << 10;
37
38pub(crate) const ENTRYPOINT_FAILURE_ERROR: &str = "The given EntryPoint is Invalid";
39
40pub type DeviceDescriptor<'a> = wgt::DeviceDescriptor<Label<'a>>;
41pub type QueueDescriptor<'a> = wgt::QueueDescriptor<Label<'a>>;
42
43#[repr(C)]
44#[derive(Clone, Copy, Debug, Eq, PartialEq)]
45#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
46pub enum HostMap {
47    Read,
48    Write,
49}
50
51#[derive(Clone, Debug, Hash, PartialEq)]
52#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
53pub(crate) struct AttachmentData<T> {
54    pub colors: ArrayVec<Option<T>, { hal::MAX_COLOR_ATTACHMENTS }>,
55    pub resolves: ArrayVec<T, { hal::MAX_COLOR_ATTACHMENTS }>,
56    pub depth_stencil: Option<T>,
57}
58impl<T: PartialEq> Eq for AttachmentData<T> {}
59
60#[derive(Clone, Debug, Hash, PartialEq)]
61#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
62pub(crate) struct RenderPassContext {
63    pub attachments: AttachmentData<TextureFormat>,
64    pub sample_count: u32,
65    pub multiview_mask: Option<NonZeroU32>,
66}
67
68impl Default for RenderPassContext {
69    fn default() -> Self {
70        Self {
71            attachments: AttachmentData {
72                colors: ArrayVec::new(),
73                resolves: ArrayVec::new(),
74                depth_stencil: None,
75            },
76            sample_count: Default::default(),
77            multiview_mask: Default::default(),
78        }
79    }
80}
81
82#[derive(Clone, Debug, Error)]
83#[non_exhaustive]
84pub enum RenderPassCompatibilityError {
85    #[error(
86        "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {res} uses attachments with formats {actual:?}",
87    )]
88    IncompatibleColorAttachment {
89        indices: Vec<usize>,
90        expected: Vec<Option<TextureFormat>>,
91        actual: Vec<Option<TextureFormat>>,
92        res: ResourceErrorIdent,
93    },
94    #[error(
95        "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {res} uses an attachment with format {actual:?}",
96    )]
97    IncompatibleDepthStencilAttachment {
98        expected: Option<TextureFormat>,
99        actual: Option<TextureFormat>,
100        res: ResourceErrorIdent,
101    },
102    #[error(
103        "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {res} uses attachments with format {actual:?}",
104    )]
105    IncompatibleSampleCount {
106        expected: u32,
107        actual: u32,
108        res: ResourceErrorIdent,
109    },
110    #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {res} uses setting {actual:?}")]
111    IncompatibleMultiview {
112        expected: Option<NonZeroU32>,
113        actual: Option<NonZeroU32>,
114        res: ResourceErrorIdent,
115    },
116}
117
118impl WebGpuError for RenderPassCompatibilityError {
119    fn webgpu_error_type(&self) -> ErrorType {
120        ErrorType::Validation
121    }
122}
123
124impl RenderPassContext {
125    // Assumes the renderpass only contains one subpass
126    pub(crate) fn check_compatible<T: Labeled>(
127        &self,
128        other: &Self,
129        res: &T,
130    ) -> Result<(), RenderPassCompatibilityError> {
131        if self.attachments.colors != other.attachments.colors {
132            let indices = self
133                .attachments
134                .colors
135                .iter()
136                .zip(&other.attachments.colors)
137                .enumerate()
138                .filter_map(|(idx, (left, right))| (left != right).then_some(idx))
139                .collect();
140            return Err(RenderPassCompatibilityError::IncompatibleColorAttachment {
141                indices,
142                expected: self.attachments.colors.iter().cloned().collect(),
143                actual: other.attachments.colors.iter().cloned().collect(),
144                res: res.error_ident(),
145            });
146        }
147        if self.attachments.depth_stencil != other.attachments.depth_stencil {
148            return Err(
149                RenderPassCompatibilityError::IncompatibleDepthStencilAttachment {
150                    expected: self.attachments.depth_stencil,
151                    actual: other.attachments.depth_stencil,
152                    res: res.error_ident(),
153                },
154            );
155        }
156        if self.sample_count != other.sample_count {
157            return Err(RenderPassCompatibilityError::IncompatibleSampleCount {
158                expected: self.sample_count,
159                actual: other.sample_count,
160                res: res.error_ident(),
161            });
162        }
163        if self.multiview_mask != other.multiview_mask {
164            return Err(RenderPassCompatibilityError::IncompatibleMultiview {
165                expected: self.multiview_mask,
166                actual: other.multiview_mask,
167                res: res.error_ident(),
168            });
169        }
170        Ok(())
171    }
172}
173
174pub type BufferMapPendingClosure = (BufferMapOperation, BufferAccessResult);
175
176#[must_use]
177#[derive(Default)]
178pub struct UserClosures {
179    pub mappings: Vec<BufferMapPendingClosure>,
180    pub blas_compact_ready: Vec<BlasCompactReadyPendingClosure>,
181    pub submissions: SmallVec<[queue::SubmittedWorkDoneClosure; 1]>,
182    pub device_lost_invocations: SmallVec<[DeviceLostInvocation; 1]>,
183}
184
185impl UserClosures {
186    pub(crate) fn extend(&mut self, other: Self) {
187        self.mappings.extend(other.mappings);
188        self.blas_compact_ready.extend(other.blas_compact_ready);
189        self.submissions.extend(other.submissions);
190        self.device_lost_invocations
191            .extend(other.device_lost_invocations);
192    }
193
194    pub(crate) fn fire(self) {
195        // Note: this logic is specifically moved out of `handle_mapping()` in order to
196        // have nothing locked by the time we execute users callback code.
197
198        // Mappings _must_ be fired before submissions, as the spec requires all mapping callbacks that are registered before
199        // a on_submitted_work_done callback to be fired before the on_submitted_work_done callback.
200        for (mut operation, status) in self.mappings {
201            if let Some(callback) = operation.callback.take() {
202                callback(status);
203            }
204        }
205        for (mut operation, status) in self.blas_compact_ready {
206            if let Some(callback) = operation.take() {
207                callback(status);
208            }
209        }
210        for closure in self.submissions {
211            closure();
212        }
213        for invocation in self.device_lost_invocations {
214            (invocation.closure)(invocation.reason, invocation.message);
215        }
216    }
217}
218
219#[cfg(send_sync)]
220pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + Send + 'static>;
221#[cfg(not(send_sync))]
222pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + 'static>;
223
224pub struct DeviceLostInvocation {
225    closure: DeviceLostClosure,
226    reason: DeviceLostReason,
227    message: String,
228}
229
230pub(crate) fn map_buffer(
231    buffer: &Buffer,
232    offset: BufferAddress,
233    size: BufferAddress,
234    kind: HostMap,
235    snatch_guard: &SnatchGuard,
236) -> Result<hal::BufferMapping, BufferAccessError> {
237    let raw_device = buffer.device.raw();
238    let raw_buffer = buffer.try_raw(snatch_guard)?;
239    let mapping = unsafe {
240        raw_device
241            .map_buffer(raw_buffer, offset..offset + size)
242            .map_err(|e| buffer.device.handle_hal_error(e))?
243    };
244
245    if !mapping.is_coherent && kind == HostMap::Read {
246        #[allow(clippy::single_range_in_vec_init)]
247        unsafe {
248            raw_device.invalidate_mapped_ranges(raw_buffer, &[offset..offset + size]);
249        }
250    }
251
252    assert_eq!(offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
253    assert_eq!(size % wgt::COPY_BUFFER_ALIGNMENT, 0);
254    // Zero out uninitialized parts of the mapping. (Spec dictates all resources
255    // behave as if they were initialized with zero)
256    //
257    // If this is a read mapping, ideally we would use a `clear_buffer` command
258    // before reading the data from GPU (i.e. `invalidate_range`). However, this
259    // would require us to kick off and wait for a command buffer or piggy back
260    // on an existing one (the later is likely the only worthwhile option). As
261    // reading uninitialized memory isn't a particular important path to
262    // support, we instead just initialize the memory here and make sure it is
263    // GPU visible, so this happens at max only once for every buffer region.
264    //
265    // If this is a write mapping zeroing out the memory here is the only
266    // reasonable way as all data is pushed to GPU anyways.
267
268    let mapped = unsafe { core::slice::from_raw_parts_mut(mapping.ptr.as_ptr(), size as usize) };
269
270    // We can't call flush_mapped_ranges in this case, so we can't drain the uninitialized ranges either
271    if !mapping.is_coherent
272        && kind == HostMap::Read
273        && !buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
274    {
275        for uninitialized in buffer
276            .initialization_status
277            .write()
278            .uninitialized(offset..(size + offset))
279        {
280            // The mapping's pointer is already offset, however we track the
281            // uninitialized range relative to the buffer's start.
282            let fill_range =
283                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
284            mapped[fill_range].fill(0);
285        }
286    } else {
287        for uninitialized in buffer
288            .initialization_status
289            .write()
290            .drain(offset..(size + offset))
291        {
292            // The mapping's pointer is already offset, however we track the
293            // uninitialized range relative to the buffer's start.
294            let fill_range =
295                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
296            mapped[fill_range].fill(0);
297
298            // NOTE: This is only possible when MAPPABLE_PRIMARY_BUFFERS is enabled.
299            if !mapping.is_coherent
300                && kind == HostMap::Read
301                && buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
302            {
303                unsafe { raw_device.flush_mapped_ranges(raw_buffer, &[uninitialized]) };
304            }
305        }
306    }
307
308    Ok(mapping)
309}
310
311#[derive(Clone, Debug)]
312#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
313pub struct DeviceMismatch {
314    pub(super) res: ResourceErrorIdent,
315    pub(super) res_device: ResourceErrorIdent,
316    pub(super) target: Option<ResourceErrorIdent>,
317    pub(super) target_device: ResourceErrorIdent,
318}
319
320impl fmt::Display for DeviceMismatch {
321    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
322        write!(
323            f,
324            "{} of {} doesn't match {}",
325            self.res_device, self.res, self.target_device
326        )?;
327        if let Some(target) = self.target.as_ref() {
328            write!(f, " of {target}")?;
329        }
330        Ok(())
331    }
332}
333
334impl core::error::Error for DeviceMismatch {}
335
336impl WebGpuError for DeviceMismatch {
337    fn webgpu_error_type(&self) -> ErrorType {
338        ErrorType::Validation
339    }
340}
341
342#[derive(Clone, Debug, Error)]
343#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
344#[non_exhaustive]
345pub enum DeviceError {
346    #[error("Parent device is lost")]
347    Lost,
348    #[error("Not enough memory left.")]
349    OutOfMemory,
350    #[error(transparent)]
351    DeviceMismatch(#[from] Box<DeviceMismatch>),
352}
353
354impl WebGpuError for DeviceError {
355    fn webgpu_error_type(&self) -> ErrorType {
356        match self {
357            Self::DeviceMismatch(e) => e.webgpu_error_type(),
358            Self::Lost => ErrorType::DeviceLost,
359            Self::OutOfMemory => ErrorType::OutOfMemory,
360        }
361    }
362}
363
364impl DeviceError {
365    /// Only use this function in contexts where there is no `Device`.
366    ///
367    /// Use [`Device::handle_hal_error`] otherwise.
368    pub fn from_hal(error: hal::DeviceError) -> Self {
369        match error {
370            hal::DeviceError::Lost => Self::Lost,
371            hal::DeviceError::OutOfMemory => Self::OutOfMemory,
372            hal::DeviceError::Unexpected => Self::Lost,
373        }
374    }
375}
376
377#[derive(Clone, Debug, Error)]
378#[error("Features {0:?} are required but not enabled on the device")]
379pub struct MissingFeatures(pub wgt::Features);
380
381impl WebGpuError for MissingFeatures {
382    fn webgpu_error_type(&self) -> ErrorType {
383        ErrorType::Validation
384    }
385}
386
387#[derive(Clone, Debug, Error)]
388#[error(
389    "Downlevel flags {0:?} are required but not supported on the device.\n{DOWNLEVEL_ERROR_MESSAGE}",
390)]
391pub struct MissingDownlevelFlags(pub wgt::DownlevelFlags);
392
393impl WebGpuError for MissingDownlevelFlags {
394    fn webgpu_error_type(&self) -> ErrorType {
395        ErrorType::Validation
396    }
397}
398
399pub use wgpu_naga_bridge::create_validator;
400pub use wgpu_naga_bridge::features_to_naga_capabilities;
wgpu_core/device/mod.rs

wgpu_core/device/
mod.rs
