wgpu_core/device/
mod.rs

1use alloc::{boxed::Box, string::String, vec::Vec};
2use core::{fmt, num::NonZeroU32};
3
4use crate::{
5    binding_model,
6    ray_tracing::BlasCompactReadyPendingClosure,
7    resource::{
8        Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation, Labeled,
9        RawResourceAccess, ResourceErrorIdent,
10    },
11    snatch::SnatchGuard,
12    Label, DOWNLEVEL_ERROR_MESSAGE,
13};
14
15use arrayvec::ArrayVec;
16use smallvec::SmallVec;
17use thiserror::Error;
18use wgt::{
19    error::{ErrorType, WebGpuError},
20    BufferAddress, DeviceLostReason, TextureFormat,
21};
22
23pub(crate) mod bgl;
24pub mod global;
25mod life;
26pub mod queue;
27pub mod ray_tracing;
28pub mod resource;
29#[cfg(any(feature = "trace", feature = "replay"))]
30pub mod trace;
31pub(crate) use resource::{FenceReadGuard, FenceWriteGuard};
32pub use {life::WaitIdleError, resource::Device};
33
34pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES;
35// Should be large enough for the largest possible texture row. This
36// value is enough for a 16k texture with float4 format.
37pub(crate) const ZERO_BUFFER_SIZE: BufferAddress = 512 << 10;
38
39pub(crate) const ENTRYPOINT_FAILURE_ERROR: &str = "The given EntryPoint is Invalid";
40
41pub type DeviceDescriptor<'a> = wgt::DeviceDescriptor<Label<'a>>;
42
43#[repr(C)]
44#[derive(Clone, Copy, Debug, Eq, PartialEq)]
45#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
46pub enum HostMap {
47    Read,
48    Write,
49}
50
51#[derive(Clone, Debug, Hash, PartialEq)]
52#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
53pub(crate) struct AttachmentData<T> {
54    pub colors: ArrayVec<Option<T>, { hal::MAX_COLOR_ATTACHMENTS }>,
55    pub resolves: ArrayVec<T, { hal::MAX_COLOR_ATTACHMENTS }>,
56    pub depth_stencil: Option<T>,
57}
58impl<T: PartialEq> Eq for AttachmentData<T> {}
59
60#[derive(Clone, Debug, Hash, PartialEq)]
61#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
62pub(crate) struct RenderPassContext {
63    pub attachments: AttachmentData<TextureFormat>,
64    pub sample_count: u32,
65    pub multiview_mask: Option<NonZeroU32>,
66}
67#[derive(Clone, Debug, Error)]
68#[non_exhaustive]
69pub enum RenderPassCompatibilityError {
70    #[error(
71        "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {res} uses attachments with formats {actual:?}",
72    )]
73    IncompatibleColorAttachment {
74        indices: Vec<usize>,
75        expected: Vec<Option<TextureFormat>>,
76        actual: Vec<Option<TextureFormat>>,
77        res: ResourceErrorIdent,
78    },
79    #[error(
80        "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {res} uses an attachment with format {actual:?}",
81    )]
82    IncompatibleDepthStencilAttachment {
83        expected: Option<TextureFormat>,
84        actual: Option<TextureFormat>,
85        res: ResourceErrorIdent,
86    },
87    #[error(
88        "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {res} uses attachments with format {actual:?}",
89    )]
90    IncompatibleSampleCount {
91        expected: u32,
92        actual: u32,
93        res: ResourceErrorIdent,
94    },
95    #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {res} uses setting {actual:?}")]
96    IncompatibleMultiview {
97        expected: Option<NonZeroU32>,
98        actual: Option<NonZeroU32>,
99        res: ResourceErrorIdent,
100    },
101}
102
103impl WebGpuError for RenderPassCompatibilityError {
104    fn webgpu_error_type(&self) -> ErrorType {
105        ErrorType::Validation
106    }
107}
108
109impl RenderPassContext {
110    // Assumes the renderpass only contains one subpass
111    pub(crate) fn check_compatible<T: Labeled>(
112        &self,
113        other: &Self,
114        res: &T,
115    ) -> Result<(), RenderPassCompatibilityError> {
116        if self.attachments.colors != other.attachments.colors {
117            let indices = self
118                .attachments
119                .colors
120                .iter()
121                .zip(&other.attachments.colors)
122                .enumerate()
123                .filter_map(|(idx, (left, right))| (left != right).then_some(idx))
124                .collect();
125            return Err(RenderPassCompatibilityError::IncompatibleColorAttachment {
126                indices,
127                expected: self.attachments.colors.iter().cloned().collect(),
128                actual: other.attachments.colors.iter().cloned().collect(),
129                res: res.error_ident(),
130            });
131        }
132        if self.attachments.depth_stencil != other.attachments.depth_stencil {
133            return Err(
134                RenderPassCompatibilityError::IncompatibleDepthStencilAttachment {
135                    expected: self.attachments.depth_stencil,
136                    actual: other.attachments.depth_stencil,
137                    res: res.error_ident(),
138                },
139            );
140        }
141        if self.sample_count != other.sample_count {
142            return Err(RenderPassCompatibilityError::IncompatibleSampleCount {
143                expected: self.sample_count,
144                actual: other.sample_count,
145                res: res.error_ident(),
146            });
147        }
148        if self.multiview_mask != other.multiview_mask {
149            return Err(RenderPassCompatibilityError::IncompatibleMultiview {
150                expected: self.multiview_mask,
151                actual: other.multiview_mask,
152                res: res.error_ident(),
153            });
154        }
155        Ok(())
156    }
157}
158
159pub type BufferMapPendingClosure = (BufferMapOperation, BufferAccessResult);
160
161#[derive(Default)]
162pub struct UserClosures {
163    pub mappings: Vec<BufferMapPendingClosure>,
164    pub blas_compact_ready: Vec<BlasCompactReadyPendingClosure>,
165    pub submissions: SmallVec<[queue::SubmittedWorkDoneClosure; 1]>,
166    pub device_lost_invocations: SmallVec<[DeviceLostInvocation; 1]>,
167}
168
169impl UserClosures {
170    fn extend(&mut self, other: Self) {
171        self.mappings.extend(other.mappings);
172        self.blas_compact_ready.extend(other.blas_compact_ready);
173        self.submissions.extend(other.submissions);
174        self.device_lost_invocations
175            .extend(other.device_lost_invocations);
176    }
177
178    fn fire(self) {
179        // Note: this logic is specifically moved out of `handle_mapping()` in order to
180        // have nothing locked by the time we execute users callback code.
181
182        // Mappings _must_ be fired before submissions, as the spec requires all mapping callbacks that are registered before
183        // a on_submitted_work_done callback to be fired before the on_submitted_work_done callback.
184        for (mut operation, status) in self.mappings {
185            if let Some(callback) = operation.callback.take() {
186                callback(status);
187            }
188        }
189        for (mut operation, status) in self.blas_compact_ready {
190            if let Some(callback) = operation.take() {
191                callback(status);
192            }
193        }
194        for closure in self.submissions {
195            closure();
196        }
197        for invocation in self.device_lost_invocations {
198            (invocation.closure)(invocation.reason, invocation.message);
199        }
200    }
201}
202
203#[cfg(send_sync)]
204pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + Send + 'static>;
205#[cfg(not(send_sync))]
206pub type DeviceLostClosure = Box<dyn FnOnce(DeviceLostReason, String) + 'static>;
207
208pub struct DeviceLostInvocation {
209    closure: DeviceLostClosure,
210    reason: DeviceLostReason,
211    message: String,
212}
213
214pub(crate) fn map_buffer(
215    buffer: &Buffer,
216    offset: BufferAddress,
217    size: BufferAddress,
218    kind: HostMap,
219    snatch_guard: &SnatchGuard,
220) -> Result<hal::BufferMapping, BufferAccessError> {
221    let raw_device = buffer.device.raw();
222    let raw_buffer = buffer.try_raw(snatch_guard)?;
223    let mapping = unsafe {
224        raw_device
225            .map_buffer(raw_buffer, offset..offset + size)
226            .map_err(|e| buffer.device.handle_hal_error(e))?
227    };
228
229    if !mapping.is_coherent && kind == HostMap::Read {
230        #[allow(clippy::single_range_in_vec_init)]
231        unsafe {
232            raw_device.invalidate_mapped_ranges(raw_buffer, &[offset..offset + size]);
233        }
234    }
235
236    assert_eq!(offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
237    assert_eq!(size % wgt::COPY_BUFFER_ALIGNMENT, 0);
238    // Zero out uninitialized parts of the mapping. (Spec dictates all resources
239    // behave as if they were initialized with zero)
240    //
241    // If this is a read mapping, ideally we would use a `clear_buffer` command
242    // before reading the data from GPU (i.e. `invalidate_range`). However, this
243    // would require us to kick off and wait for a command buffer or piggy back
244    // on an existing one (the later is likely the only worthwhile option). As
245    // reading uninitialized memory isn't a particular important path to
246    // support, we instead just initialize the memory here and make sure it is
247    // GPU visible, so this happens at max only once for every buffer region.
248    //
249    // If this is a write mapping zeroing out the memory here is the only
250    // reasonable way as all data is pushed to GPU anyways.
251
252    let mapped = unsafe { core::slice::from_raw_parts_mut(mapping.ptr.as_ptr(), size as usize) };
253
254    // We can't call flush_mapped_ranges in this case, so we can't drain the uninitialized ranges either
255    if !mapping.is_coherent
256        && kind == HostMap::Read
257        && !buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
258    {
259        for uninitialized in buffer
260            .initialization_status
261            .write()
262            .uninitialized(offset..(size + offset))
263        {
264            // The mapping's pointer is already offset, however we track the
265            // uninitialized range relative to the buffer's start.
266            let fill_range =
267                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
268            mapped[fill_range].fill(0);
269        }
270    } else {
271        for uninitialized in buffer
272            .initialization_status
273            .write()
274            .drain(offset..(size + offset))
275        {
276            // The mapping's pointer is already offset, however we track the
277            // uninitialized range relative to the buffer's start.
278            let fill_range =
279                (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize;
280            mapped[fill_range].fill(0);
281
282            // NOTE: This is only possible when MAPPABLE_PRIMARY_BUFFERS is enabled.
283            if !mapping.is_coherent
284                && kind == HostMap::Read
285                && buffer.usage.contains(wgt::BufferUsages::MAP_WRITE)
286            {
287                unsafe { raw_device.flush_mapped_ranges(raw_buffer, &[uninitialized]) };
288            }
289        }
290    }
291
292    Ok(mapping)
293}
294
295#[derive(Clone, Debug)]
296#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
297pub struct DeviceMismatch {
298    pub(super) res: ResourceErrorIdent,
299    pub(super) res_device: ResourceErrorIdent,
300    pub(super) target: Option<ResourceErrorIdent>,
301    pub(super) target_device: ResourceErrorIdent,
302}
303
304impl fmt::Display for DeviceMismatch {
305    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
306        write!(
307            f,
308            "{} of {} doesn't match {}",
309            self.res_device, self.res, self.target_device
310        )?;
311        if let Some(target) = self.target.as_ref() {
312            write!(f, " of {target}")?;
313        }
314        Ok(())
315    }
316}
317
318impl core::error::Error for DeviceMismatch {}
319
320impl WebGpuError for DeviceMismatch {
321    fn webgpu_error_type(&self) -> ErrorType {
322        ErrorType::Validation
323    }
324}
325
326#[derive(Clone, Debug, Error)]
327#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
328#[non_exhaustive]
329pub enum DeviceError {
330    #[error("Parent device is lost")]
331    Lost,
332    #[error("Not enough memory left.")]
333    OutOfMemory,
334    #[error(transparent)]
335    DeviceMismatch(#[from] Box<DeviceMismatch>),
336}
337
338impl WebGpuError for DeviceError {
339    fn webgpu_error_type(&self) -> ErrorType {
340        match self {
341            Self::DeviceMismatch(e) => e.webgpu_error_type(),
342            Self::Lost => ErrorType::DeviceLost,
343            Self::OutOfMemory => ErrorType::OutOfMemory,
344        }
345    }
346}
347
348impl DeviceError {
349    /// Only use this function in contexts where there is no `Device`.
350    ///
351    /// Use [`Device::handle_hal_error`] otherwise.
352    pub fn from_hal(error: hal::DeviceError) -> Self {
353        match error {
354            hal::DeviceError::Lost => Self::Lost,
355            hal::DeviceError::OutOfMemory => Self::OutOfMemory,
356            hal::DeviceError::Unexpected => Self::Lost,
357        }
358    }
359}
360
361#[derive(Clone, Debug, Error)]
362#[error("Features {0:?} are required but not enabled on the device")]
363pub struct MissingFeatures(pub wgt::Features);
364
365impl WebGpuError for MissingFeatures {
366    fn webgpu_error_type(&self) -> ErrorType {
367        ErrorType::Validation
368    }
369}
370
371#[derive(Clone, Debug, Error)]
372#[error(
373    "Downlevel flags {0:?} are required but not supported on the device.\n{DOWNLEVEL_ERROR_MESSAGE}",
374)]
375pub struct MissingDownlevelFlags(pub wgt::DownlevelFlags);
376
377impl WebGpuError for MissingDownlevelFlags {
378    fn webgpu_error_type(&self) -> ErrorType {
379        ErrorType::Validation
380    }
381}
382
383pub use wgpu_naga_bridge::create_validator;
384pub use wgpu_naga_bridge::features_to_naga_capabilities;
wgpu_core/device/mod.rs

wgpu_core/device/
mod.rs
