Skip to main content

wgpu_core/device/
queue.rs

1#[cfg(feature = "trace")]
2use alloc::string::ToString as _;
3use alloc::{boxed::Box, string::String, sync::Arc, vec, vec::Vec};
4use core::{
5    mem::{self, ManuallyDrop},
6    num::NonZeroU64,
7    ptr::NonNull,
8    sync::atomic::Ordering,
9};
10use smallvec::SmallVec;
11use thiserror::Error;
12use wgt::{
13    error::{ErrorType, WebGpuError},
14    AccelerationStructureFlags,
15};
16
17use super::{life::LifetimeTracker, Device};
18#[cfg(feature = "trace")]
19use crate::device::trace::{Action, IntoTrace};
20use crate::{
21    api_log,
22    command::{
23        validate_linear_texture_data, validate_texture_buffer_copy,
24        validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
25        CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopyRange, CopySide,
26        TransferError,
27    },
28    device::{DeviceError, QueueDescriptor, WaitIdleError},
29    get_lowest_common_denom, hal_label,
30    init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
31    lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
32    ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
33    ray_tracing_pipeline,
34    resource::{
35        Blas, BlasCompactState, BlasDescriptor, BlasState, Buffer, BufferAccessError,
36        BufferMapState, DestroyedBuffer, DestroyedQuerySet, DestroyedResourceError,
37        DestroyedTexture, FlushedStagingBuffer, InvalidOrDestroyedResourceError,
38        InvalidResourceError, Labeled, ParentDevice, ResourceErrorIdent, ResourceState,
39        StagingBuffer, Texture, TextureInner, Trackable, TrackingData,
40    },
41    resource_log,
42    scratch::ScratchBuffer,
43    snatch::{SnatchGuard, Snatchable},
44    track::{self, Tracker, TrackerIndex},
45    FastHashMap, LabelHelpers, SubmissionIndex,
46};
47use crate::{device::resource::CommandIndices, resource::RawResourceAccess};
48
49pub struct Queue {
50    raw: Box<dyn hal::DynQueue>,
51    pub(crate) pending_writes: Mutex<PendingWrites>,
52    life_tracker: Mutex<LifetimeTracker>,
53    label: String,
54    // The device needs to be dropped last (`Device.zero_buffer` might be referenced by the encoder in pending writes).
55    pub(crate) device: Arc<Device>,
56}
57
58impl Queue {
59    pub(crate) fn new(
60        device: Arc<Device>,
61        raw: Box<dyn hal::DynQueue>,
62        desc: QueueDescriptor,
63        instance_flags: wgt::InstanceFlags,
64    ) -> Result<Self, DeviceError> {
65        let pending_encoder = device
66            .command_allocator
67            .acquire_encoder(device.raw(), raw.as_ref())
68            .map_err(DeviceError::from_hal);
69
70        let pending_encoder = match pending_encoder {
71            Ok(pending_encoder) => pending_encoder,
72            Err(e) => {
73                return Err(e);
74            }
75        };
76
77        let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);
78
79        let zero_buffer = device.zero_buffer.as_ref();
80        pending_writes.activate();
81        unsafe {
82            pending_writes
83                .command_encoder
84                .transition_buffers(&[hal::BufferBarrier {
85                    buffer: zero_buffer,
86                    usage: hal::StateTransition {
87                        from: wgt::BufferUses::empty(),
88                        to: wgt::BufferUses::COPY_DST,
89                    },
90                }]);
91            pending_writes
92                .command_encoder
93                .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
94            pending_writes
95                .command_encoder
96                .transition_buffers(&[hal::BufferBarrier {
97                    buffer: zero_buffer,
98                    usage: hal::StateTransition {
99                        from: wgt::BufferUses::COPY_DST,
100                        to: wgt::BufferUses::COPY_SRC,
101                    },
102                }]);
103        }
104
105        Ok(Queue {
106            raw,
107            device,
108            label: desc.label.to_string(),
109            pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
110            life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
111        })
112    }
113
114    pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
115        self.raw.as_ref()
116    }
117
118    #[track_caller]
119    pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
120        self.life_tracker.lock()
121    }
122
123    /// Ensure the surface texture is in the PRESENT state, clearing it if it was never rendered to.
124    /// Submits any necessary work to the GPU before the HAL present call.
125    ///
126    /// See <https://github.com/gfx-rs/wgpu/issues/6748>
127    pub(crate) fn prepare_surface_texture_for_present(
128        &self,
129        texture: &Arc<Texture>,
130    ) -> Result<(), DeviceError> {
131        let snatch_guard = self.device.snatchable_lock.read();
132        let submission = self.allocate_submission(snatch_guard)?;
133        let device = &self.device;
134
135        // If the texture is uninitialized it needs to be cleared before presenting
136        let needs_clear = {
137            let status = texture.initialization_status.read();
138            status
139                .mips
140                .first()
141                .is_some_and(|mip| mip.check(0..1).is_some())
142        };
143
144        let mut pending_writes = self.pending_writes.lock();
145
146        if needs_clear {
147            // After encoding the clear operation, we must not return without
148            // adding the texture to `pending_writes`.
149            let encoder = pending_writes.activate();
150            let mut trackers = device.trackers.lock();
151            crate::command::clear_texture(
152                texture,
153                TextureInitRange {
154                    mip_range: 0..1,
155                    layer_range: 0..1,
156                },
157                None,
158                encoder,
159                &mut trackers.textures,
160                &device.alignments,
161                device.zero_buffer.as_ref(),
162                &submission.snatch_guard,
163                device.instance_flags,
164            )
165            .map_err(|e| match e {
166                ClearError::Device(e) => e,
167                _ => DeviceError::Lost,
168            })?;
169            texture.initialization_status.write().mips[0].drain(0..1);
170        }
171
172        // Transition the texture to PRESENT in the device tracker.
173        // If it's already in PRESENT, this produces no barriers and we can skip the submission.
174        //
175        // This has to be after any clear_texture call because clear_texture modifies the tracker state internally.
176        // Computing transitions afterward ensures they reflect the actual current state.
177        let pending = {
178            let mut trackers = device.trackers.lock();
179            let pending: Vec<track::PendingTransition<wgt::TextureUses>> = trackers
180                .textures
181                .set_single(
182                    texture,
183                    texture.full_range.clone(),
184                    wgt::TextureUses::PRESENT,
185                )
186                .collect();
187            pending
188        };
189
190        if pending.is_empty() {
191            // This assert checks that we don't return here if we encoded a
192            // clear operation for the texture, which would be a problem since
193            // we haven't done anything yet to ensure it stays alive. If we
194            // cleared the texture, then we must have produced a barrier to put
195            // it in PRESENT state, so `pending` will not be empty.
196            debug_assert!(!needs_clear);
197            return Ok(());
198        }
199
200        // Emit the transition barriers to PRESENT.
201        {
202            let raw_texture = texture
203                .raw(&submission.snatch_guard)
204                .ok_or(DeviceError::Lost)?;
205            let barriers: Vec<hal::TextureBarrier<'_, dyn hal::DynTexture>> = pending
206                .into_iter()
207                .map(|pt| pt.into_hal(raw_texture))
208                .collect();
209
210            let encoder = pending_writes.activate();
211            // SAFETY:
212            // - The encoder is in the recording state after `activate()`
213            // - The texture is kept alive by adding it to `PendingWrites` below
214            unsafe {
215                encoder.transition_textures(&barriers);
216            }
217        }
218
219        // Add the texture to `PendingWrites`. This will cause `submit()` to:
220        // - Flush any pending writes to the texture.
221        // - Include the texture in `surface_textures` for the submission.
222        // - Keep the texture alive so the texture and its clear_view aren't
223        //   destroyed before the GPU finishes the `clear_texture` operation
224        //   encoded above.
225        pending_writes.insert_texture(texture);
226
227        submission.submit(pending_writes)?;
228
229        Ok(())
230    }
231
232    /// Maintains the queue's list of finished command buffers.
233    ///
234    /// Command buffers submitted before `submission_index` was submitted
235    /// stop being tracked and callbacks which are waiting for them are
236    /// returned. Also returned is whether the queue is empty. This may
237    /// be stale unless new submissions are prevented by locking
238    /// [`Device::command_indices`]
239    pub(crate) fn maintain(
240        &self,
241        submission_index: u64,
242        snatch_guard: &SnatchGuard,
243    ) -> (
244        SmallVec<[SubmittedWorkDoneClosure; 1]>,
245        Vec<super::BufferMapPendingClosure>,
246        Vec<BlasCompactReadyPendingClosure>,
247        bool,
248    ) {
249        let mut life_tracker = self.lock_life();
250        let submission_closures = life_tracker.triage_submissions(submission_index);
251
252        let mapping_closures = life_tracker.handle_mapping(snatch_guard);
253        let blas_closures = life_tracker.handle_compact_read_back();
254
255        let queue_empty = life_tracker.queue_empty();
256
257        (
258            submission_closures,
259            mapping_closures,
260            blas_closures,
261            queue_empty,
262        )
263    }
264}
265
266crate::impl_resource_type!(Queue);
267crate::impl_labeled!(Queue);
268crate::impl_parent_device!(Queue);
269crate::impl_storage_item!(Queue);
270
271impl Drop for Queue {
272    #[allow(trivial_casts)]
273    fn drop(&mut self) {
274        profiling::scope!("Queue::drop");
275        api_log!("Queue::drop {:?}", self as *const _);
276        resource_log!("Drop {}", self.error_ident());
277
278        // On Vulkan, pending presents are not tracked by fences.
279        // wait_for_idle covers both fence-tracked submissions and pending presents.
280        let result = {
281            // Synchronizes with surfaces being configured on other threads.
282            let _command_indices = self.device.command_indices.write();
283            unsafe { self.raw.wait_for_idle() }
284        };
285        match result {
286            Ok(()) => {}
287            Err(hal::DeviceError::Lost) => {
288                self.device.handle_hal_error(hal::DeviceError::Lost);
289            }
290            Err(e) => {
291                panic!("Unexpected error while waiting for queue idle on drop: {e:?}");
292            }
293        }
294
295        let last_successful_submission_index = self
296            .device
297            .last_successful_submission_index
298            .load(Ordering::Acquire);
299
300        let snatch_guard = self.device.snatchable_lock.read();
301        let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
302            self.maintain(last_successful_submission_index, &snatch_guard);
303        drop(snatch_guard);
304
305        assert!(queue_empty);
306
307        let closures = crate::device::UserClosures {
308            mappings: mapping_closures,
309            blas_compact_ready: blas_compact_ready_closures,
310            submissions: submission_closures,
311            device_lost_invocations: SmallVec::new(),
312        };
313
314        closures.fire();
315    }
316}
317
318#[cfg(send_sync)]
319pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
320#[cfg(not(send_sync))]
321pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
322
323/// A texture or buffer to be freed soon.
324///
325/// This is just a tagged raw texture or buffer, generally about to be added to
326/// some other more specific container like:
327///
328/// - `PendingWrites::temp_resources`: resources used by queue writes and
329///   unmaps, waiting to be folded in with the next queue submission
330///
331/// - `ActiveSubmission::temp_resources`: temporary resources used by a queue
332///   submission, to be freed when it completes
333#[derive(Debug)]
334pub enum TempResource {
335    StagingBuffer(FlushedStagingBuffer),
336    ScratchBuffer(ScratchBuffer),
337    DestroyedBuffer(DestroyedBuffer),
338    DestroyedTexture(DestroyedTexture),
339    DestroyedQuerySet(DestroyedQuerySet),
340}
341
342/// A series of raw [`CommandBuffer`]s that have been submitted to a
343/// queue, and the [`wgpu_hal::CommandEncoder`] that built them.
344///
345/// [`CommandBuffer`]: hal::Api::CommandBuffer
346/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
347pub(crate) struct EncoderInFlight {
348    inner: crate::command::InnerCommandEncoder,
349    pub(crate) trackers: Tracker,
350    pub(crate) temp_resources: Vec<TempResource>,
351    /// We only need to keep these resources alive.
352    _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
353
354    /// These are the buffers that have been tracked by `PendingWrites`.
355    pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
356    /// These are the textures that have been tracked by `PendingWrites`.
357    pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
358    /// These are the BLASes that have been tracked by `PendingWrites`.
359    pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
360    /// These are the shader binding data buffers that have been tracked by `PendingWrites`.
361    /// Only need to keep these live while the writes wait.
362    _pending_shader_binding_data: Vec<Arc<ray_tracing_pipeline::ShaderBindingData>>,
363}
364
365/// A private command encoder for writes made directly on the device
366/// or queue.
367///
368/// Operations like `buffer_unmap`, `queue_write_buffer`, and
369/// `queue_write_texture` need to copy data to the GPU. At the hal
370/// level, this must be done by encoding and submitting commands, but
371/// these operations are not associated with any specific wgpu command
372/// buffer.
373///
374/// Instead, `Device::pending_writes` owns one of these values, which
375/// has its own hal command encoder and resource lists. The commands
376/// accumulated here are automatically submitted to the queue at the
377/// sooner of:
378///
379/// 1. The user's next wgpu command buffer submission. (Pending writes
380///    are inserted ahead of the user's commands.)
381/// 2. The next `mapAsync` request for a buffer that has pending
382///    writes.
383///
384/// Important:
385/// When locking pending_writes be sure that tracker is not locked
386/// and try to lock trackers for the minimum timespan possible
387///
388/// All uses of [`StagingBuffer`]s end up here.
389#[derive(Debug)]
390pub(crate) struct PendingWrites {
391    // The command encoder needs to be destroyed before any other resource in pending writes.
392    pub command_encoder: Box<dyn hal::DynCommandEncoder>,
393
394    /// True if `command_encoder` is in the "recording" state, as
395    /// described in the docs for the [`wgpu_hal::CommandEncoder`]
396    /// trait.
397    ///
398    /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
399    pub is_recording: bool,
400
401    temp_resources: Vec<TempResource>,
402    dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
403    dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
404    copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
405    /// Buffers of shader binding data that have been written to when creating their
406    /// ray tracing pipelines.
407    written_shader_binding_data: Vec<Arc<ray_tracing_pipeline::ShaderBindingData>>,
408    instance_flags: wgt::InstanceFlags,
409
410    /// Staging buffers that are the source of a copy recorded in
411    /// `command_encoder`, and so need a `MAP_WRITE` -> `COPY_SRC` transition.
412    ///
413    /// See [`hoisted_buffer_barriers`] for why these are not recorded
414    /// immediately. Moved to `temp_resources` in [`pre_submit`].
415    ///
416    /// [`hoisted_buffer_barriers`]: PendingWrites::hoisted_buffer_barriers
417    /// [`pre_submit`]: PendingWrites::pre_submit
418    hoisted_staging_buffers: Vec<FlushedStagingBuffer>,
419
420    /// Buffer transitions that must happen before the copies recorded in
421    /// `command_encoder`.
422    ///
423    /// Recording a barrier immediately before each copy means that a frame
424    /// which calls `Queue::write_buffer` a few dozen times ends up with many
425    /// back-to-back barrier/copy pairs, when a single barrier covering every
426    /// buffer would have done just as well.
427    ///
428    /// Instead we accumulate the transitions here and record them all at once,
429    /// into a command buffer that [`pre_submit`] places in front of
430    /// `command_encoder`'s.
431    ///
432    /// Only a buffer's *first* transition in a batch may be hoisted like this.
433    /// A buffer written more than once per batch gets a second, `COPY_DST` ->
434    /// `COPY_DST` transition from the tracker (unless the adapter reports
435    /// `COPY_DST` as an ordered usage), and that barrier exists precisely to
436    /// separate the two copies, so it has to stay where it was recorded. See
437    /// [`hoist_buffer_barrier`].
438    ///
439    /// [`pre_submit`]: PendingWrites::pre_submit
440    /// [`hoist_buffer_barrier`]: PendingWrites::hoist_buffer_barrier
441    hoisted_buffer_barriers: Vec<(Arc<Buffer>, hal::StateTransition<wgt::BufferUses>)>,
442}
443
444impl PendingWrites {
445    pub fn new(
446        command_encoder: Box<dyn hal::DynCommandEncoder>,
447        instance_flags: wgt::InstanceFlags,
448    ) -> Self {
449        Self {
450            command_encoder,
451            is_recording: false,
452            temp_resources: Vec::new(),
453            dst_buffers: FastHashMap::default(),
454            dst_textures: FastHashMap::default(),
455            copied_blas_s: FastHashMap::default(),
456            instance_flags,
457            hoisted_staging_buffers: Vec::new(),
458            hoisted_buffer_barriers: Vec::new(),
459            written_shader_binding_data: Vec::new(),
460        }
461    }
462
463    pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
464        self.dst_buffers
465            .insert(buffer.tracker_index(), buffer.clone());
466    }
467
468    pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
469        self.dst_textures
470            .insert(texture.tracker_index(), texture.clone());
471    }
472
473    pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
474        self.copied_blas_s
475            .insert(blas.tracker_index(), blas.clone());
476    }
477
478    pub fn use_shader_binding_data(&mut self, sbd: &Arc<ray_tracing_pipeline::ShaderBindingData>) {
479        self.written_shader_binding_data.push(sbd.clone())
480    }
481
482    pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
483        self.dst_buffers.contains_key(&buffer.tracker_index())
484    }
485
486    pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
487        self.dst_textures.contains_key(&texture.tracker_index())
488    }
489
490    pub fn consume_temp(&mut self, resource: TempResource) {
491        self.temp_resources.push(resource);
492    }
493
494    /// Take ownership of a staging buffer that is the source of a copy
495    /// recorded into this batch.
496    ///
497    /// This also queues up the staging buffer's `MAP_WRITE` -> `COPY_SRC`
498    /// transition, so callers must not record one themselves.
499    pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
500        self.hoisted_staging_buffers.push(buffer);
501    }
502
503    /// Queue up a buffer transition to be recorded ahead of this batch's
504    /// copies.
505    ///
506    /// Returns the transition instead if it cannot be hoisted, in which case
507    /// the caller must record it immediately, before its own copy. This happens
508    /// when this batch has already copied into `buffer`: the transition is then
509    /// a `COPY_DST` -> `COPY_DST` write-after-write barrier separating the
510    /// earlier copy from the caller's, as hoisting it to the front of the
511    /// batch would leave the two copies unsynchronized.
512    ///
513    /// See [`hoisted_buffer_barriers`].
514    ///
515    /// [`hoisted_buffer_barriers`]: PendingWrites::hoisted_buffer_barriers
516    #[must_use]
517    fn hoist_buffer_barrier(
518        &mut self,
519        buffer: &Arc<Buffer>,
520        transition: Option<track::PendingTransition<wgt::BufferUses>>,
521    ) -> Option<track::PendingTransition<wgt::BufferUses>> {
522        let transition = transition?;
523
524        // `dst_buffers` is only populated once a copy has been recorded, so
525        // true here means an earlier copy in this batch already wrote `buffer`.
526        if self.dst_buffers.contains_key(&buffer.tracker_index()) {
527            return Some(transition);
528        }
529
530        self.hoisted_buffer_barriers
531            .push((buffer.clone(), transition.usage));
532        None
533    }
534
535    pub fn clear_buffer(
536        &mut self,
537        device: &Arc<Device>,
538        buffer: &Arc<Buffer>,
539        range: core::ops::Range<wgt::BufferAddress>,
540        snatch_guard: &SnatchGuard,
541    ) -> Result<(), QueueWriteError> {
542        let transition = {
543            let mut trackers = device.trackers.lock();
544            trackers
545                .buffers
546                .set_single(buffer, wgt::BufferUses::COPY_DST)
547        };
548        let inline_transition = self.hoist_buffer_barrier(buffer, transition);
549
550        let dst_raw = buffer.try_raw(snatch_guard)?;
551
552        let encoder = self.activate();
553        unsafe {
554            if let Some(transition) = inline_transition {
555                encoder.transition_buffers(&[transition.into_hal(buffer, snatch_guard)]);
556            }
557            encoder.clear_buffer(dst_raw, range.clone());
558        }
559
560        self.insert_buffer(buffer);
561
562        // Ensure the overwritten bytes are marked as initialized so
563        // they don't need to be nulled prior to mapping or binding.
564        buffer.initialization_status.write().drain(range);
565
566        Ok(())
567    }
568
569    /// Record every transition this batch of writes needs as a single barrier,
570    /// into a command buffer placed ahead of the one holding the batch's
571    /// copies.
572    ///
573    /// Returns the staging buffers whose transitions were recorded. The caller
574    /// is responsible for keeping them alive until the submission completes.
575    fn record_hoisted_barriers(
576        &mut self,
577        encoder: &mut EncoderInFlight,
578        snatch_guard: &SnatchGuard,
579    ) -> Result<Vec<FlushedStagingBuffer>, DeviceError> {
580        let staging_buffers = mem::take(&mut self.hoisted_staging_buffers);
581        let buffer_barriers = mem::take(&mut self.hoisted_buffer_barriers);
582
583        let barriers = staging_buffers
584            .iter()
585            .map(|staging_buffer| hal::BufferBarrier {
586                buffer: staging_buffer.raw(),
587                usage: hal::StateTransition {
588                    from: wgt::BufferUses::MAP_WRITE,
589                    to: wgt::BufferUses::COPY_SRC,
590                },
591            })
592            .chain(buffer_barriers.iter().filter_map(|(buffer, usage)| {
593                // The buffer may have been destroyed since the copy was
594                // recorded, in which case there is nothing to sync.
595                Some(hal::BufferBarrier {
596                    buffer: buffer.raw(snatch_guard)?,
597                    usage: usage.clone(),
598                })
599            }))
600            .collect::<Vec<_>>();
601
602        if !barriers.is_empty() {
603            encoder
604                .inner
605                .open_pass(Some("(wgpu internal) PendingWrites Transit"))?;
606            unsafe { encoder.inner.raw.transition_buffers(&barriers) };
607            encoder.inner.close_and_push_front()?;
608        }
609
610        Ok(staging_buffers)
611    }
612
613    fn pre_submit(
614        &mut self,
615        command_allocator: &CommandAllocator,
616        device: &Arc<Device>,
617        queue: &Queue,
618        snatch_guard: &SnatchGuard,
619    ) -> Result<Option<EncoderInFlight>, DeviceError> {
620        if self.is_recording {
621            let pending_buffers = mem::take(&mut self.dst_buffers);
622            let pending_textures = mem::take(&mut self.dst_textures);
623            let pending_blas_s = mem::take(&mut self.copied_blas_s);
624            let pending_shader_binding_data = mem::take(&mut self.written_shader_binding_data);
625
626            let cmd_buf = unsafe { self.command_encoder.end_encoding() }
627                .map_err(|e| device.handle_hal_error(e))?;
628            self.is_recording = false;
629
630            let new_encoder = command_allocator
631                .acquire_encoder(device.raw(), queue.raw())
632                .map_err(|e| device.handle_hal_error(e))?;
633
634            let mut encoder = EncoderInFlight {
635                inner: crate::command::InnerCommandEncoder {
636                    raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
637                    list: vec![cmd_buf],
638                    device: device.clone(),
639                    is_open: false,
640                    api: crate::command::EncodingApi::InternalUse,
641                    label: "(wgpu internal) PendingWrites command encoder".into(),
642                },
643                trackers: Tracker::new(device.ordered_buffer_usages, device.ordered_texture_usages),
644                temp_resources: mem::take(&mut self.temp_resources),
645                _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
646                    device.clone(),
647                ),
648                pending_buffers,
649                pending_textures,
650                pending_blas_s,
651                _pending_shader_binding_data: pending_shader_binding_data,
652            };
653
654            let staging_buffers = self.record_hoisted_barriers(&mut encoder, snatch_guard)?;
655            encoder
656                .temp_resources
657                .extend(staging_buffers.into_iter().map(TempResource::StagingBuffer));
658
659            Ok(Some(encoder))
660        } else {
661            self.dst_buffers.clear();
662            self.dst_textures.clear();
663            self.copied_blas_s.clear();
664            self.written_shader_binding_data.clear();
665            Ok(None)
666        }
667    }
668
669    pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
670        if !self.is_recording {
671            unsafe {
672                self.command_encoder
673                    .begin_encoding(hal_label(
674                        Some("(wgpu internal) PendingWrites"),
675                        self.instance_flags,
676                    ))
677                    .unwrap();
678            }
679            self.is_recording = true;
680        }
681        self.command_encoder.as_mut()
682    }
683}
684
685impl Drop for PendingWrites {
686    fn drop(&mut self) {
687        unsafe {
688            if self.is_recording {
689                self.command_encoder.discard_encoding();
690            }
691        }
692    }
693}
694
695#[derive(Clone, Debug, Error)]
696#[non_exhaustive]
697pub enum QueueWriteError {
698    #[error(transparent)]
699    Queue(#[from] DeviceError),
700    #[error(transparent)]
701    Transfer(#[from] TransferError),
702    #[error(transparent)]
703    MemoryInitFailure(#[from] ClearError),
704    #[error(transparent)]
705    DestroyedResource(#[from] DestroyedResourceError),
706    #[error(transparent)]
707    InvalidResource(#[from] InvalidResourceError),
708}
709
710impl From<InvalidOrDestroyedResourceError> for QueueWriteError {
711    fn from(e: InvalidOrDestroyedResourceError) -> Self {
712        match e {
713            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
714            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
715        }
716    }
717}
718
719impl WebGpuError for QueueWriteError {
720    fn webgpu_error_type(&self) -> ErrorType {
721        match self {
722            Self::Queue(e) => e.webgpu_error_type(),
723            Self::Transfer(e) => e.webgpu_error_type(),
724            Self::MemoryInitFailure(e) => e.webgpu_error_type(),
725            Self::DestroyedResource(e) => e.webgpu_error_type(),
726            Self::InvalidResource(e) => e.webgpu_error_type(),
727        }
728    }
729}
730
731#[derive(Clone, Debug, Error)]
732#[non_exhaustive]
733pub enum QueueSubmitError {
734    #[error(transparent)]
735    Queue(#[from] DeviceError),
736    #[error(transparent)]
737    DestroyedResource(#[from] DestroyedResourceError),
738    #[error("{0} is still mapped")]
739    BufferStillMapped(ResourceErrorIdent),
740    #[error(transparent)]
741    InvalidResource(#[from] InvalidResourceError),
742    #[error(transparent)]
743    CommandEncoder(#[from] CommandEncoderError),
744    #[error(transparent)]
745    ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
746}
747
748impl From<InvalidOrDestroyedResourceError> for QueueSubmitError {
749    fn from(e: InvalidOrDestroyedResourceError) -> Self {
750        match e {
751            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
752            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
753        }
754    }
755}
756
757impl WebGpuError for QueueSubmitError {
758    fn webgpu_error_type(&self) -> ErrorType {
759        match self {
760            Self::Queue(e) => e.webgpu_error_type(),
761            Self::CommandEncoder(e) => e.webgpu_error_type(),
762            Self::ValidateAsActionsError(e) => e.webgpu_error_type(),
763            Self::InvalidResource(e) => e.webgpu_error_type(),
764            Self::DestroyedResource(_) | Self::BufferStillMapped(_) => ErrorType::Validation,
765        }
766    }
767}
768
769/// A command submission in the process of being assembled.
770///
771/// Within `wgpu_core`, enqueuing commands for execution on the GPU is a
772/// three-step process:
773///
774/// 1) Call [`Queue::allocate_submission`] to acquire the necessary locks,
775///    assign a submission index, wrap them all up as a [`PendingSubmission`],
776///    and return it.
777///
778/// 2) Add the command buffers to be submitted to [`executions`], and note any
779///    surface textures they reference in [`surface_textures`]. Contribute to
780///    [`Queue::pending_writes`] as necessary.
781///
782/// 3) Acquire the pending writes lock. This may be done at any point between
783///    the return from [`Queue::allocate_submission`] and the call to
784///    [`submit`]. Typically it should be done as late as is possible given
785///    any necessary pending writes activity.
786///
787/// 4) Call the `PendingSubmission`'s [`submit`] method (which is a convenience
788///    wrapper around [`Queue::submit_pending_submission`]). Pass the pending
789///    writes mutex guard to [`submit`].
790///
791/// It is also acceptable to drop the `PendingSubmission` without submitting;
792/// this frees its locks in the appropriate order. This may be necessary when
793/// those locks are required to access the state that determines whether a
794/// submission is needed at all.
795///
796/// This split allows the various places in `wgpu_core` that need to submit
797/// commands to the GPU to share the common initial code for locking and final
798/// code for actually submitting the commands to `wgpu_hal`:
799///
800/// - [`Queue::submit`] just submits user-constructed [`CommandBuffer`]s.
801///
802/// - [`Queue::prepare_surface_texture_for_present`] examines the surface
803///   texture being presented, and submits deferred initialization commands and
804///   barriers to get it ready.
805///
806/// - [`Queue::flush_writes_for_buffer`] and [`Queue::flush_pending_writes`]
807///   simply submit the operations already staged in [`Queue::pending_writes`].
808///
809/// Returned from [`Queue::allocate_submission`] and consumed by [`submit`].
810/// These are internal APIs used in `Queue::submit` and other places within
811/// `wgpu-core` that need to submit work.
812///
813/// [`submit`]: PendingSubmission::submit
814/// [`executions`]: PendingSubmission::executions
815/// [`surface_textures`]: PendingSubmission::surface_textures
816pub(crate) struct PendingSubmission<'a> {
817    queue: &'a Queue,
818
819    // These lock guards must appear in this struct in the order given.
820    //
821    // The instrumented locks in [`lock::ranked`] require that locks be acquired
822    // and released in a stack-like order. Since `rank::DEVICE_COMMAND_INDICES`
823    // follows `rank::DEVICE_SNATCHABLE_LOCK`, the lock on
824    // `Device::command_indices` must be released before the lock on
825    // `Device::snatchable_lock`. Rust drops struct members from first to last,
826    // so this ordering of fields ensures the order we want.
827    /// A guard for the lock on `Device::command_indices`.
828    command_index_guard: RwLockWriteGuard<'a, CommandIndices>,
829
830    /// A guard for the lock on `Device::snatchable_lock`.
831    snatch_guard: SnatchGuard<'a>,
832
833    /// Command buffers to be submitted, along with trackers for the resources
834    /// they use.
835    pub executions: Vec<EncoderInFlight>,
836
837    /// Surface textures referenced by command buffers in this submission.
838    ///
839    /// These need to be passed to [`wgpu_hal::Queue::submit`], which
840    /// requires that the list contains no duplicates, so we store them in a
841    /// `HashMap` keyed by `SurfaceTexture` address.
842    surface_textures: FastHashMap<*const Texture, Arc<Texture>>,
843
844    /// The index this submission has been assigned.
845    pub index: SubmissionIndex,
846}
847
848pub(crate) struct SubmissionResult<'a> {
849    pub snatch_guard: SnatchGuard<'a>,
850}
851
852impl<'a> PendingSubmission<'a> {
853    fn submit(
854        self,
855        pending_writes: MutexGuard<'a, PendingWrites>,
856    ) -> Result<SubmissionResult<'a>, DeviceError> {
857        self.queue.submit_pending_submission(pending_writes, self)
858    }
859}
860
861//TODO: move out common parts of write_xxx.
862
863impl Queue {
864    pub(crate) fn write_buffer_inner(
865        &self,
866        buffer: Arc<Buffer>,
867        buffer_offset: wgt::BufferAddress,
868        data: &[u8],
869    ) -> Result<(), QueueWriteError> {
870        profiling::scope!("Queue::write_buffer");
871        api_log!("Queue::write_buffer");
872
873        #[cfg(feature = "trace")]
874        if let Some(ref mut trace) = *self.device.trace.lock() {
875            use crate::device::trace::DataKind;
876            let size = data.len() as u64;
877            let data = trace.make_binary(DataKind::Bin, data);
878            trace.add(Action::WriteBuffer {
879                id: buffer.to_trace(),
880                data,
881                offset: buffer_offset,
882                size,
883                queued: true,
884            });
885        }
886
887        buffer.check_is_valid()?;
888        self.device.check_is_valid()?;
889
890        let data_size = data.len() as wgt::BufferAddress;
891
892        self.same_device_as(buffer.as_ref())?;
893
894        let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
895            data_size
896        } else {
897            // even though a zero-length write is a no-op and no copy operation will occur,
898            // we must still validate the copy operation. This ensures that invalid
899            // API calls—like writing to a mapped buffer or out-of-bounds offsets—are
900            // caught consistently, even if no data is actually moved.
901            self.validate_write_buffer_impl(buffer.as_ref(), buffer_offset, 0)?;
902
903            log::trace!("Ignoring write_buffer of size 0");
904            return Ok(());
905        };
906
907        // Platform validation requires that the staging buffer always be
908        // freed, even if an error occurs. All paths from here must call
909        // `device.pending_writes.consume`.
910        let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
911
912        let staging_buffer = {
913            profiling::scope!("copy");
914            staging_buffer.write_exact(data);
915            staging_buffer.flush()
916        };
917
918        let snatch_guard = self.device.snatchable_lock.read();
919        let mut pending_writes = self.pending_writes.lock();
920
921        let result = self.write_staging_buffer_impl(
922            &snatch_guard,
923            &mut pending_writes,
924            &staging_buffer,
925            buffer,
926            buffer_offset,
927        );
928
929        pending_writes.consume(staging_buffer);
930
931        result
932    }
933
934    pub fn write_buffer(
935        &self,
936        buffer: Arc<Buffer>,
937        buffer_offset: wgt::BufferAddress,
938        data: &[u8],
939    ) {
940        if let Err(error) = self.write_buffer_inner(buffer, buffer_offset, data) {
941            self.device
942                .handle_error(error, Some(self.label()), "Queue::write_buffer");
943        }
944    }
945
946    pub fn create_staging_buffer(
947        &self,
948        buffer_size: wgt::BufferSize,
949    ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
950        profiling::scope!("Queue::create_staging_buffer");
951        resource_log!("Queue::create_staging_buffer");
952
953        self.device.check_is_valid()?;
954
955        let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
956        let ptr = unsafe { staging_buffer.ptr() };
957
958        Ok((staging_buffer, ptr))
959    }
960
961    pub fn write_staging_buffer(
962        &self,
963        buffer: Arc<Buffer>,
964        buffer_offset: wgt::BufferAddress,
965        staging_buffer: StagingBuffer,
966    ) -> Result<(), QueueWriteError> {
967        profiling::scope!("Queue::write_staging_buffer");
968
969        buffer.check_is_valid()?;
970        self.device.check_is_valid()?;
971
972        // At this point, we have taken ownership of the staging_buffer from the
973        // user. Platform validation requires that the staging buffer always
974        // be freed, even if an error occurs. All paths from here must call
975        // `device.pending_writes.consume`.
976        let staging_buffer = staging_buffer.flush();
977
978        let snatch_guard = self.device.snatchable_lock.read();
979        let mut pending_writes = self.pending_writes.lock();
980
981        let result = self.write_staging_buffer_impl(
982            &snatch_guard,
983            &mut pending_writes,
984            &staging_buffer,
985            buffer,
986            buffer_offset,
987        );
988
989        drop(snatch_guard);
990
991        pending_writes.consume(staging_buffer);
992
993        drop(pending_writes);
994
995        result
996    }
997
998    pub fn validate_write_buffer(
999        &self,
1000        buffer: Arc<Buffer>,
1001        buffer_offset: u64,
1002        buffer_size: wgt::BufferSize,
1003    ) -> Result<(), QueueWriteError> {
1004        profiling::scope!("Queue::validate_write_buffer");
1005
1006        self.device.check_is_valid()?;
1007        buffer.check_is_valid()?;
1008
1009        self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size.into())?;
1010
1011        Ok(())
1012    }
1013
1014    fn validate_write_buffer_impl(
1015        &self,
1016        buffer: &Buffer,
1017        buffer_offset: u64,
1018        buffer_size: u64,
1019    ) -> Result<(), TransferError> {
1020        if !matches!(&*buffer.map_state.read(), BufferMapState::Idle) {
1021            return Err(TransferError::BufferNotAvailable);
1022        }
1023        buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
1024        if !buffer_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1025            return Err(TransferError::UnalignedCopySize(buffer_size));
1026        }
1027        if !buffer_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1028            return Err(TransferError::UnalignedBufferOffset(buffer_offset));
1029        }
1030
1031        if buffer_offset > buffer.size {
1032            return Err(TransferError::BufferStartOffsetOverrun {
1033                start_offset: buffer_offset,
1034                buffer_size: buffer.size,
1035                side: CopySide::Destination,
1036            });
1037        }
1038        if buffer_size > buffer.size - buffer_offset {
1039            return Err(TransferError::BufferEndOffsetOverrun {
1040                start_offset: buffer_offset,
1041                size: buffer_size,
1042                buffer_size: buffer.size,
1043                side: CopySide::Destination,
1044            });
1045        }
1046
1047        Ok(())
1048    }
1049
1050    fn write_staging_buffer_impl(
1051        &self,
1052        snatch_guard: &SnatchGuard,
1053        pending_writes: &mut PendingWrites,
1054        staging_buffer: &FlushedStagingBuffer,
1055        buffer: Arc<Buffer>,
1056        buffer_offset: u64,
1057    ) -> Result<(), QueueWriteError> {
1058        self.device.check_is_valid()?;
1059
1060        let transition = {
1061            let mut trackers = self.device.trackers.lock();
1062            trackers
1063                .buffers
1064                .set_single(&buffer, wgt::BufferUses::COPY_DST)
1065        };
1066
1067        let dst_raw = buffer.try_raw(snatch_guard)?;
1068
1069        self.same_device_as(buffer.as_ref())?;
1070
1071        self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size.into())?;
1072
1073        let region = hal::BufferCopy {
1074            src_offset: 0,
1075            dst_offset: buffer_offset,
1076            size: staging_buffer.size,
1077        };
1078        // The staging buffer's own `MAP_WRITE` -> `COPY_SRC` transition is
1079        // hoisted by `PendingWrites::consume`, which our caller invokes.
1080        let inline_transition = pending_writes.hoist_buffer_barrier(&buffer, transition);
1081        let encoder = pending_writes.activate();
1082        unsafe {
1083            if let Some(transition) = inline_transition {
1084                encoder.transition_buffers(&[transition.into_hal(&buffer, snatch_guard)]);
1085            }
1086            encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
1087        }
1088
1089        pending_writes.insert_buffer(&buffer);
1090
1091        // Ensure the overwritten bytes are marked as initialized so
1092        // they don't need to be nulled prior to mapping or binding.
1093        {
1094            buffer
1095                .initialization_status
1096                .write()
1097                .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
1098        }
1099
1100        Ok(())
1101    }
1102
1103    pub fn write_texture_inner(
1104        &self,
1105        destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
1106        data: &[u8],
1107        data_layout: &wgt::TexelCopyBufferLayout,
1108        size: &wgt::Extent3d,
1109    ) -> Result<(), QueueWriteError> {
1110        profiling::scope!("Queue::write_texture");
1111        api_log!("Queue::write_texture");
1112
1113        #[cfg(feature = "trace")]
1114        if let Some(ref mut trace) = *self.device.trace.lock() {
1115            use crate::device::trace::DataKind;
1116            let data = trace.make_binary(DataKind::Bin, data);
1117            trace.add(Action::WriteTexture {
1118                to: destination.to_trace(),
1119                data,
1120                layout: *data_layout,
1121                size: *size,
1122            });
1123        }
1124
1125        self.device.check_is_valid()?;
1126
1127        let dst = destination.texture;
1128        let destination = wgt::TexelCopyTextureInfo {
1129            texture: (),
1130            mip_level: destination.mip_level,
1131            origin: destination.origin,
1132            aspect: destination.aspect,
1133        };
1134
1135        self.same_device_as(dst.as_ref())?;
1136
1137        dst.check_valid()?;
1138
1139        dst.check_usage(wgt::TextureUsages::COPY_DST)
1140            .map_err(TransferError::MissingTextureUsage)?;
1141
1142        // Note: Doing the copy range validation early is important because ensures that the
1143        // dimensions are not going to cause overflow in other parts of the validation.
1144        let CopyRange {
1145            hal_copy_extent,
1146            hal_base: dst_base,
1147            array_layer_count,
1148            selector,
1149        } = validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
1150
1151        validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;
1152
1153        validate_texture_buffer_copy(
1154            &destination,
1155            dst_base.aspect,
1156            &dst.desc,
1157            data_layout,
1158            false, // alignment not required for buffer offset or bytes per row
1159            self.device.features,
1160        )?;
1161
1162        // Note: `_source_bytes_per_array_layer` is ignored since we
1163        // have a staging copy, and it can have a different value.
1164        let (required_bytes_in_copy, _source_bytes_per_array_layer, _) =
1165            validate_linear_texture_data(
1166                data_layout,
1167                dst.desc.format,
1168                destination.aspect,
1169                data.len() as wgt::BufferAddress,
1170                CopySide::Source,
1171                size,
1172            )?;
1173
1174        if dst.desc.format.is_depth_stencil_format() {
1175            self.device
1176                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1177                .map_err(TransferError::from)?;
1178        }
1179
1180        let snatch_guard = self.device.snatchable_lock.read();
1181
1182        let dst_raw = dst.try_inner(&snatch_guard)?.raw();
1183
1184        // This must happen after parameter validation (so that errors are reported
1185        // as required by the spec), but before any side effects.
1186        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
1187            log::trace!("Ignoring write_texture of size 0");
1188            return Ok(());
1189        }
1190
1191        let mut pending_writes = self.pending_writes.lock();
1192        let encoder = pending_writes.activate();
1193
1194        // If the copy does not fully cover the layers, we need to initialize to
1195        // zero *first* as we don't keep track of partial texture layer inits.
1196        //
1197        // Strictly speaking we only need to clear the areas of a layer
1198        // untouched, but this would get increasingly messy.
1199        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1200            // volume textures don't have a layer range as array volumes aren't supported
1201            0..1
1202        } else {
1203            destination.origin.z..destination.origin.z + size.depth_or_array_layers
1204        };
1205        let layer_ranges_to_clear = {
1206            let mut dst_initialization_status = dst.initialization_status.write();
1207            if dst_initialization_status.mips[destination.mip_level as usize]
1208                .check(init_layer_range.clone())
1209                .is_some()
1210            {
1211                if has_copy_partial_init_tracker_coverage(size, &destination, &dst.desc) {
1212                    dst_initialization_status.mips[destination.mip_level as usize]
1213                        .drain(init_layer_range)
1214                        .collect::<Vec<core::ops::Range<u32>>>()
1215                } else {
1216                    dst_initialization_status.mips[destination.mip_level as usize]
1217                        .drain(init_layer_range);
1218                    vec![]
1219                }
1220            } else {
1221                vec![]
1222            }
1223        };
1224        if !layer_ranges_to_clear.is_empty() {
1225            let mut trackers = self.device.trackers.lock();
1226            for layer_range in layer_ranges_to_clear {
1227                crate::command::clear_texture(
1228                    &dst,
1229                    TextureInitRange {
1230                        mip_range: destination.mip_level..(destination.mip_level + 1),
1231                        layer_range,
1232                    },
1233                    None,
1234                    encoder,
1235                    &mut trackers.textures,
1236                    &self.device.alignments,
1237                    self.device.zero_buffer.as_ref(),
1238                    &snatch_guard,
1239                    self.device.instance_flags,
1240                )
1241                .map_err(QueueWriteError::from)?;
1242            }
1243        }
1244
1245        let (block_width, block_height) = dst.desc.format.block_dimensions();
1246        let width_in_blocks = size.width / block_width;
1247        let height_in_blocks = size.height / block_height;
1248
1249        let block_size = dst
1250            .desc
1251            .format
1252            .block_copy_size(Some(destination.aspect))
1253            .unwrap();
1254        let bytes_in_last_row = width_in_blocks * block_size;
1255
1256        let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
1257        let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
1258
1259        let bytes_per_row_alignment = get_lowest_common_denom(
1260            self.device.alignments.buffer_copy_pitch.get() as u32,
1261            block_size,
1262        );
1263        assert!(u32::MAX - bytes_in_last_row >= bytes_per_row_alignment);
1264        let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
1265
1266        // Platform validation requires that the staging buffer always be
1267        // freed, even if an error occurs. All paths from here must call
1268        // `device.pending_writes.consume`.
1269        let staging_buffer = if stage_bytes_per_row == bytes_per_row {
1270            profiling::scope!("copy aligned");
1271            // Fast path if the data is already being aligned optimally.
1272            let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
1273            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1274            staging_buffer
1275                .write_exact(&data[data_layout.offset as usize..][..stage_size.get() as usize]);
1276            staging_buffer
1277        } else {
1278            profiling::scope!("copy chunked");
1279            // Copy row by row into the optimal alignment.
1280            let block_rows_in_copy = u64::from(size.depth_or_array_layers - 1)
1281                * u64::from(rows_per_image)
1282                + u64::from(height_in_blocks);
1283            // The copy size was validated against the source buffer, however,
1284            // `stage_bytes_per_row` can differ, so let's be paranoid.
1285            let stage_size = u64::from(stage_bytes_per_row)
1286                .checked_mul(block_rows_in_copy)
1287                .and_then(wgt::BufferSize::new)
1288                .unwrap();
1289            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1290            for layer in 0..u64::from(size.depth_or_array_layers) {
1291                let rows_offset = layer * u64::from(rows_per_image);
1292                for row in rows_offset..rows_offset + u64::from(height_in_blocks) {
1293                    let src_offset = data_layout.offset + row * u64::from(bytes_per_row);
1294                    let dst_offset = row * u64::from(stage_bytes_per_row);
1295                    unsafe {
1296                        staging_buffer.write_with_offset(
1297                            data,
1298                            src_offset as isize,
1299                            dst_offset as isize,
1300                            bytes_in_last_row as usize,
1301                        )
1302                    }
1303                }
1304            }
1305            staging_buffer
1306        };
1307
1308        let staging_buffer = staging_buffer.flush();
1309
1310        let regions = (0..array_layer_count)
1311            .map(|array_layer_offset| {
1312                let mut texture_base = dst_base.clone();
1313                texture_base.array_layer += array_layer_offset;
1314                hal::BufferTextureCopy {
1315                    buffer_layout: wgt::TexelCopyBufferLayout {
1316                        offset: array_layer_offset as u64
1317                            * rows_per_image as u64
1318                            * stage_bytes_per_row as u64,
1319                        bytes_per_row: Some(stage_bytes_per_row),
1320                        rows_per_image: Some(rows_per_image),
1321                    },
1322                    texture_base,
1323                    size: hal_copy_extent,
1324                }
1325            })
1326            .collect::<Vec<_>>();
1327
1328        {
1329            let mut trackers = self.device.trackers.lock();
1330            let transition =
1331                trackers
1332                    .textures
1333                    .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1334            let texture_barriers = transition
1335                .map(|pending| pending.into_hal(dst_raw))
1336                .collect::<Vec<_>>();
1337
1338            // The staging buffer's `MAP_WRITE` -> `COPY_SRC` transition is
1339            // hoisted to the front of the batch by `PendingWrites::consume`,
1340            // below.
1341            unsafe {
1342                encoder.transition_textures(&texture_barriers);
1343                encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, &regions);
1344            }
1345        }
1346
1347        pending_writes.consume(staging_buffer);
1348        pending_writes.insert_texture(&dst);
1349
1350        Ok(())
1351    }
1352
1353    pub fn write_texture(
1354        &self,
1355        destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
1356        data: &[u8],
1357        data_layout: &wgt::TexelCopyBufferLayout,
1358        size: &wgt::Extent3d,
1359    ) {
1360        if let Err(error) = self.write_texture_inner(destination, data, data_layout, size) {
1361            self.device
1362                .handle_error(error, Some(self.label()), "Queue::write_texture");
1363        }
1364    }
1365
1366    #[cfg(webgl)]
1367    pub fn copy_external_image_to_texture(
1368        &self,
1369        source: &wgt::CopyExternalImageSourceInfo,
1370        destination: wgt::CopyExternalImageDestInfo<Arc<Texture>>,
1371        size: wgt::Extent3d,
1372    ) -> Result<(), QueueWriteError> {
1373        use crate::conv;
1374
1375        profiling::scope!("Queue::copy_external_image_to_texture");
1376
1377        self.device.check_is_valid()?;
1378
1379        let mut needs_flag = false;
1380        // `OffscreenCanvas` needs no downlevel flag: WebGL2's `texSubImage2D`
1381        // accepts it as a `TexImageSource` and the gles backend uploads it the
1382        // same way as `HTMLCanvasElement`.
1383        needs_flag |= source.origin != wgt::Origin2d::ZERO;
1384        needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
1385        #[allow(clippy::bool_comparison)]
1386        if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
1387            needs_flag |= source.flip_y != false;
1388            needs_flag |= destination.premultiplied_alpha != false;
1389        }
1390
1391        if needs_flag {
1392            self.device
1393                .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
1394                .map_err(TransferError::from)?;
1395        }
1396
1397        let src_width = source.source.width();
1398        let src_height = source.source.height();
1399
1400        let dst = destination.texture;
1401        let premultiplied_alpha = destination.premultiplied_alpha;
1402        let destination = wgt::TexelCopyTextureInfo {
1403            texture: (),
1404            mip_level: destination.mip_level,
1405            origin: destination.origin,
1406            aspect: destination.aspect,
1407        };
1408
1409        dst.check_valid()?;
1410
1411        if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
1412            return Err(
1413                TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
1414            );
1415        }
1416        if dst.desc.dimension != wgt::TextureDimension::D2 {
1417            return Err(TransferError::InvalidDimensionExternal.into());
1418        }
1419        dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
1420            .map_err(TransferError::MissingTextureUsage)?;
1421        if dst.desc.sample_count != 1 {
1422            return Err(TransferError::InvalidSampleCount {
1423                sample_count: dst.desc.sample_count,
1424            }
1425            .into());
1426        }
1427
1428        if source.origin.x > src_width || src_width - source.origin.x < size.width {
1429            return Err(TransferError::TextureOverrun {
1430                start_offset: source.origin.x,
1431                end_offset: source.origin.x.saturating_add(size.width),
1432                texture_size: src_width,
1433                dimension: crate::resource::TextureErrorDimension::X,
1434                side: CopySide::Source,
1435                plane: None,
1436            }
1437            .into());
1438        }
1439        if source.origin.y > src_height || src_height - source.origin.y < size.height {
1440            return Err(TransferError::TextureOverrun {
1441                start_offset: source.origin.y,
1442                end_offset: source.origin.y.saturating_add(size.height),
1443                texture_size: src_height,
1444                dimension: crate::resource::TextureErrorDimension::Y,
1445                side: CopySide::Source,
1446                plane: None,
1447            }
1448            .into());
1449        }
1450        if size.depth_or_array_layers != 1 {
1451            return Err(TransferError::TextureOverrun {
1452                start_offset: 0,
1453                end_offset: size.depth_or_array_layers,
1454                texture_size: 1,
1455                dimension: crate::resource::TextureErrorDimension::Z,
1456                side: CopySide::Source,
1457                plane: None,
1458            }
1459            .into());
1460        }
1461
1462        // Note: Doing the copy range validation early is important because ensures that the
1463        // dimensions are not going to cause overflow in other parts of the validation.
1464        let CopyRange {
1465            hal_copy_extent,
1466            hal_base: dst_base,
1467            array_layer_count: _,
1468            selector,
1469        } = validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
1470
1471        // Ensured by is_valid_external_image_copy_dst_texture_format.
1472        assert!(dst_base.aspect == hal::FormatAspects::COLOR);
1473
1474        let snatch_guard = self.device.snatchable_lock.read();
1475
1476        let dst_raw = dst.try_raw(&snatch_guard)?;
1477
1478        // This must happen after parameter validation (so that errors are reported
1479        // as required by the spec), but before any side effects.
1480        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
1481            log::trace!("Ignoring copy_external_image_to_texture of size 0");
1482            return Ok(());
1483        }
1484
1485        let mut pending_writes = self.pending_writes.lock();
1486        let encoder = pending_writes.activate();
1487
1488        // If the copy does not fully cover the layers, we need to initialize to
1489        // zero *first* as we don't keep track of partial texture layer inits.
1490        //
1491        // Strictly speaking we only need to clear the areas of a layer
1492        // untouched, but this would get increasingly messy.
1493        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1494            // volume textures don't have a layer range as array volumes aren't supported
1495            0..1
1496        } else {
1497            destination.origin.z..destination.origin.z + size.depth_or_array_layers
1498        };
1499        let layer_ranges_to_clear = {
1500            let mut dst_initialization_status = dst.initialization_status.write();
1501            if dst_initialization_status.mips[destination.mip_level as usize]
1502                .check(init_layer_range.clone())
1503                .is_some()
1504            {
1505                if has_copy_partial_init_tracker_coverage(&size, &destination, &dst.desc) {
1506                    dst_initialization_status.mips[destination.mip_level as usize]
1507                        .drain(init_layer_range)
1508                        .collect::<Vec<core::ops::Range<u32>>>()
1509                } else {
1510                    dst_initialization_status.mips[destination.mip_level as usize]
1511                        .drain(init_layer_range);
1512                    vec![]
1513                }
1514            } else {
1515                vec![]
1516            }
1517        };
1518        if !layer_ranges_to_clear.is_empty() {
1519            let mut trackers = self.device.trackers.lock();
1520            for layer_range in layer_ranges_to_clear {
1521                crate::command::clear_texture(
1522                    &dst,
1523                    TextureInitRange {
1524                        mip_range: destination.mip_level..(destination.mip_level + 1),
1525                        layer_range,
1526                    },
1527                    None,
1528                    encoder,
1529                    &mut trackers.textures,
1530                    &self.device.alignments,
1531                    self.device.zero_buffer.as_ref(),
1532                    &snatch_guard,
1533                    self.device.instance_flags,
1534                )
1535                .map_err(QueueWriteError::from)?;
1536            }
1537        }
1538
1539        let regions = hal::TextureCopy {
1540            src_base: hal::TextureCopyBase {
1541                mip_level: 0,
1542                array_layer: 0,
1543                origin: source.origin.to_3d(0),
1544                aspect: hal::FormatAspects::COLOR,
1545            },
1546            dst_base,
1547            size: hal_copy_extent,
1548        };
1549
1550        let mut trackers = self.device.trackers.lock();
1551        let transitions = trackers
1552            .textures
1553            .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1554
1555        // `copy_external_image_to_texture` is exclusive to the WebGL backend.
1556        // Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend.
1557        let encoder_webgl = encoder
1558            .as_any_mut()
1559            .downcast_mut::<hal::gles::CommandEncoder>()
1560            .unwrap();
1561        let dst_raw_webgl = dst_raw
1562            .as_any()
1563            .downcast_ref::<hal::gles::Texture>()
1564            .unwrap();
1565        let transitions_webgl = transitions.map(|pending| {
1566            let dyn_transition = pending.into_hal(dst_raw);
1567            hal::TextureBarrier {
1568                texture: dst_raw_webgl,
1569                range: dyn_transition.range,
1570                usage: dyn_transition.usage,
1571                queue_family_ownership_transfer: None,
1572            }
1573        });
1574
1575        use hal::CommandEncoder as _;
1576        unsafe {
1577            encoder_webgl.transition_textures(transitions_webgl);
1578            encoder_webgl.copy_external_image_to_texture(
1579                source,
1580                dst_raw_webgl,
1581                premultiplied_alpha,
1582                core::iter::once(regions),
1583            );
1584        }
1585
1586        pending_writes.insert_texture(&dst);
1587
1588        Ok(())
1589    }
1590
1591    /// Flush `PendingWrites` if it contains a write to `buffer`.
1592    pub fn flush_writes_for_buffer(
1593        &self,
1594        buffer: &Arc<Buffer>,
1595        snatch_guard: SnatchGuard,
1596    ) -> Result<(), BufferAccessError> {
1597        let submission = self.allocate_submission(snatch_guard)?;
1598
1599        let pending_writes = self.pending_writes.lock();
1600        if !pending_writes.contains_buffer(buffer) {
1601            return Ok(());
1602        }
1603
1604        submission.submit(pending_writes)?;
1605
1606        Ok(())
1607    }
1608
1609    fn flush_pending_writes(&self) -> Result<Option<SubmissionIndex>, DeviceError> {
1610        let snatch_guard = self.device.snatchable_lock.read();
1611        let submission = self.allocate_submission(snatch_guard)?;
1612        let submit_index = submission.index;
1613        let pending_writes = self.pending_writes.lock();
1614        if pending_writes.is_recording {
1615            submission.submit(pending_writes)?;
1616            Ok(Some(submit_index))
1617        } else {
1618            Ok(None)
1619        }
1620    }
1621
1622    #[cfg(feature = "trace")]
1623    fn trace_submission(
1624        &self,
1625        submit_index: SubmissionIndex,
1626        commands: Vec<crate::command::Command<crate::command::PointerReferences>>,
1627    ) {
1628        if let Some(ref mut trace) = *self.device.trace.lock() {
1629            trace.add(Action::Submit(submit_index, commands));
1630        }
1631    }
1632
1633    #[cfg(feature = "trace")]
1634    fn trace_failed_submission(
1635        &self,
1636        submit_index: SubmissionIndex,
1637        commands: Option<Vec<crate::command::Command<crate::command::PointerReferences>>>,
1638        error: String,
1639    ) {
1640        if let Some(ref mut trace) = *self.device.trace.lock() {
1641            trace.add(Action::FailedCommands {
1642                commands,
1643                failed_at_submit: Some(submit_index),
1644                error,
1645            });
1646        }
1647    }
1648
1649    fn submit_inner(
1650        &self,
1651        command_buffers: &[Arc<CommandBuffer>],
1652    ) -> Result<SubmissionIndex, Box<(Option<SubmissionIndex>, QueueSubmitError)>> {
1653        profiling::scope!("Queue::submit");
1654        api_log!("Queue::submit");
1655
1656        let snatch_guard = self.device.snatchable_lock.read();
1657        let mut submission = self
1658            .allocate_submission(snatch_guard)
1659            .map_err(|e| (None, e.into()))?;
1660        let submit_index = submission.index;
1661
1662        // If we encounter an error after we have started updating global state and before
1663        // successful submission, we must lose the device to avoid continuing with
1664        // potentially inaccurate resource state.
1665        let mut lose_device_on_error = false;
1666        let mut successfully_submitted = false;
1667
1668        let res = 'error: {
1669            let mut used_surface_textures = track::TextureUsageScope::default();
1670            let mut baked_command_buffers = Vec::with_capacity(command_buffers.len());
1671
1672            if !command_buffers.is_empty() {
1673                profiling::scope!("prepare");
1674
1675                let mut first_error = None;
1676
1677                // We are required to invalidate all command buffers in both the success and
1678                // failure paths, so we `continue` after errors, and disallow `?`.
1679                #[deny(clippy::question_mark_used)]
1680                for command_buffer in command_buffers {
1681                    profiling::scope!("process command buffer");
1682
1683                    // we reset the used surface textures every time we use
1684                    // it, so make sure to set_size on it.
1685                    used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1686
1687                    // Anything other than our own submission work in the remainder of this
1688                    // function that attempts to use the WebGPU command buffer after this
1689                    // point, will find it vacant (invalid), and produce an error.
1690                    #[cfg_attr(not(feature = "trace"), expect(unused_mut))]
1691                    let mut cmd_buf_data = command_buffer.take_finished();
1692
1693                    if first_error.is_some() {
1694                        continue;
1695                    }
1696
1697                    #[cfg(feature = "trace")]
1698                    let trace_commands = cmd_buf_data
1699                        .as_mut()
1700                        .ok()
1701                        .and_then(|data| mem::take(&mut data.trace_commands));
1702
1703                    let mut baked = match cmd_buf_data {
1704                        Ok(cmd_buf_data) => {
1705                            let res = validate_command_buffer(
1706                                command_buffer,
1707                                self,
1708                                &cmd_buf_data,
1709                                &submission.snatch_guard,
1710                                &mut submission.surface_textures,
1711                                &mut used_surface_textures,
1712                                &mut submission.command_index_guard,
1713                            );
1714                            if let Err(err) = res {
1715                                #[cfg(feature = "trace")]
1716                                self.trace_failed_submission(
1717                                    submit_index,
1718                                    trace_commands,
1719                                    err.to_string(),
1720                                );
1721                                first_error.get_or_insert(err);
1722                                continue;
1723                            }
1724
1725                            #[cfg(feature = "trace")]
1726                            if let Some(commands) = trace_commands {
1727                                self.trace_submission(submit_index, commands);
1728                            }
1729
1730                            cmd_buf_data
1731                                .set_acceleration_structure_dependencies(&submission.snatch_guard);
1732                            cmd_buf_data.into_baked_commands()
1733                        }
1734                        Err(err) => {
1735                            #[cfg(feature = "trace")]
1736                            self.trace_failed_submission(
1737                                submit_index,
1738                                trace_commands,
1739                                err.to_string(),
1740                            );
1741                            first_error.get_or_insert(err.into());
1742                            continue;
1743                        }
1744                    };
1745
1746                    if let Err(e) = baked
1747                        .process_deferred_query_set_resolves(&self.device, &submission.snatch_guard)
1748                    {
1749                        break 'error Err(e.into());
1750                    }
1751
1752                    baked_command_buffers.push(baked);
1753                }
1754
1755                if let Some(first_error) = first_error {
1756                    break 'error Err(first_error);
1757                }
1758
1759                // At this point we have validated all the command buffers, and we start
1760                // making updates to global state. If we fail between here and successful
1761                // submission, we must lose the device, or else we could leave that global
1762                // state inaccurate or inconsistent.
1763                //
1764                // At time of writing, there were two kinds of errors that can occur in
1765                // this stage:
1766                //  - `hal` command encoding errors. These produce device loss in
1767                //    `handle_hal_error`, independent of what we do here.
1768                //  - Errors from texture initialization. The error cases that
1769                //    can actually occur should also be encoder errors, but we map
1770                //    everything to device loss, just in case.
1771                lose_device_on_error = true;
1772
1773                // Note: locking the trackers has to be done after the storages
1774                let mut trackers = self.device.trackers.lock();
1775
1776                for mut baked in baked_command_buffers {
1777                    profiling::scope!("process baked commands");
1778
1779                    // execute resource transitions
1780                    if let Err(e) = baked.encoder.open_pass(hal_label(
1781                        Some("(wgpu internal) Transit"),
1782                        self.device.instance_flags,
1783                    )) {
1784                        break 'error Err(e.into());
1785                    }
1786
1787                    baked.initialize_buffer_memory(&mut trackers, &submission.snatch_guard);
1788
1789                    let depth_slice_discards = match baked.initialize_texture_memory(
1790                        &mut trackers,
1791                        &self.device,
1792                        &submission.snatch_guard,
1793                    ) {
1794                        Ok(discards) => discards,
1795                        Err(e) => {
1796                            break 'error Err(QueueSubmitError::CommandEncoder(
1797                                CommandEncoderError::Clear(e),
1798                            ));
1799                        }
1800                    };
1801
1802                    //Note: stateless trackers are not merged:
1803                    // device already knows these resources exist.
1804                    CommandEncoder::insert_barriers_from_device_tracker(
1805                        baked.encoder.raw.as_mut(),
1806                        &mut trackers,
1807                        &baked.trackers,
1808                        &submission.snatch_guard,
1809                    );
1810
1811                    if let Err(e) = baked.encoder.close_and_push_front() {
1812                        break 'error Err(e.into());
1813                    }
1814
1815                    if !depth_slice_discards.is_empty() || !used_surface_textures.is_empty() {
1816                        if let Err(e) = baked.encoder.open_pass(hal_label(
1817                            Some("(wgpu internal) Finalize"),
1818                            self.device.instance_flags,
1819                        )) {
1820                            break 'error Err(e.into());
1821                        }
1822
1823                        if let Err(e) = baked.initialize_discarded_depth_slices(
1824                            depth_slice_discards,
1825                            &mut trackers,
1826                            &self.device,
1827                            &submission.snatch_guard,
1828                        ) {
1829                            break 'error Err(QueueSubmitError::CommandEncoder(
1830                                CommandEncoderError::Clear(e),
1831                            ));
1832                        }
1833
1834                        // Transition surface textures into `Present` state.
1835                        // Note: we could technically do it after all of the command buffers,
1836                        // but here we have a command encoder by hand, so it's easier to use it.
1837                        let texture_barriers = trackers
1838                            .textures
1839                            .set_from_usage_scope_and_drain_transitions(
1840                                &used_surface_textures,
1841                                &submission.snatch_guard,
1842                            )
1843                            .collect::<Vec<_>>();
1844                        unsafe {
1845                            baked.encoder.raw.transition_textures(&texture_barriers);
1846                        };
1847                        if let Err(e) = baked.encoder.close() {
1848                            break 'error Err(e.into());
1849                        }
1850                        used_surface_textures = track::TextureUsageScope::default();
1851                    }
1852
1853                    // done
1854                    submission.executions.push(EncoderInFlight {
1855                        inner: baked.encoder,
1856                        trackers: baked.trackers,
1857                        temp_resources: baked.temp_resources,
1858                        _indirect_draw_validation_resources: baked
1859                            .indirect_draw_validation_resources,
1860                        pending_buffers: FastHashMap::default(),
1861                        pending_textures: FastHashMap::default(),
1862                        pending_blas_s: FastHashMap::default(),
1863                        _pending_shader_binding_data: Vec::new(),
1864                    });
1865                }
1866            }
1867
1868            let pending_writes = self.pending_writes.lock();
1869
1870            let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) {
1871                Ok(result) => result,
1872                Err(e) => break 'error Err(e.into()),
1873            };
1874            successfully_submitted = true;
1875
1876            profiling::scope!("cleanup");
1877
1878            // Failing in `Device::maintain` won't put resource state at risk, but returning
1879            // an error from a successful submission could be confusing, do we really want
1880            // to do that?
1881            lose_device_on_error = false;
1882
1883            // This will schedule destruction of all resources that are no longer needed
1884            // by the user but used in the command stream, among other things.
1885            // `device.maintain` consumes and will release the snatch guard.
1886            let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard);
1887            match result {
1888                Ok(status) => {
1889                    debug_assert!(matches!(
1890                        status,
1891                        wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1892                    ));
1893                }
1894                Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1895                Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1896                    unreachable!("Cannot get WrongSubmissionIndex from Poll")
1897                }
1898                Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1899            };
1900
1901            Ok(closures)
1902        }; // 'error
1903
1904        let callbacks = match res {
1905            Ok(ok) => ok,
1906            Err(e) => {
1907                if lose_device_on_error {
1908                    self.device.lose("submission failed");
1909                }
1910                return Err(Box::new((
1911                    successfully_submitted.then_some(submit_index),
1912                    e,
1913                )));
1914            }
1915        };
1916
1917        // the closures should execute with nothing locked!
1918        callbacks.fire();
1919
1920        self.device.lose_if_oom();
1921
1922        api_log!("Queue::submit returned submit index {submit_index}");
1923
1924        Ok(submit_index)
1925    }
1926
1927    pub fn submit(&self, command_buffers: &[Arc<CommandBuffer>]) -> Option<SubmissionIndex> {
1928        match self.submit_inner(command_buffers) {
1929            Ok(submit_index) => Some(submit_index),
1930            Err(e) => {
1931                let (submit_index, e) = *e;
1932                self.device
1933                    .handle_error(e, Some(self.label()), "Queue::submit");
1934                submit_index
1935            }
1936        }
1937    }
1938
1939    /// Allocate a submission index and prepare for a submission.
1940    ///
1941    /// This is an internal API used in [`Queue::submit`] and other places within
1942    /// `wgpu-core` that need to submit work.
1943    ///
1944    /// Returns the index and a [`PendingSubmission`].
1945    ///
1946    /// The caller passes in the already-acquired [`SnatchGuard`]. This function acquires
1947    /// the command index lock.
1948    ///
1949    /// The caller should update [`PendingSubmission::executions`] with details of the
1950    /// submission.
1951    ///
1952    /// To finalize and submit the submission, call [`PendingSubmission::submit`] (which is
1953    /// a convenience wrapper around [`Queue::submit_pending_submission`]).
1954    ///
1955    /// After calling this function and before submitting, the caller must acquire the
1956    /// pending writes lock, and pass it to `submit`.
1957    ///
1958    /// It is also acceptable to drop the `PendingSubmission` without submitting. This may
1959    /// be necessary when locks are required to access the state that determines whether a
1960    /// submission is needed.
1961    fn allocate_submission<'a>(
1962        &'a self,
1963        snatch_guard: SnatchGuard<'a>,
1964    ) -> Result<PendingSubmission<'a>, DeviceError> {
1965        let mut command_index_guard = self.device.command_indices.write();
1966        command_index_guard.active_submission_index += 1;
1967        let index = command_index_guard.active_submission_index;
1968
1969        self.device.check_is_valid()?;
1970
1971        let submission = PendingSubmission {
1972            queue: self,
1973            command_index_guard,
1974            snatch_guard,
1975            executions: Vec::new(),
1976            surface_textures: FastHashMap::default(),
1977            index,
1978        };
1979
1980        Ok(submission)
1981    }
1982
1983    /// Finalize and submit a [`PendingSubmission`] that was returned by
1984    /// [`Queue::allocate_submission`].
1985    ///
1986    /// This is an internal API used in `Queue::submit` and other places within
1987    /// `wgpu-core` that need to submit work. See [`Queue::allocate_submission`]
1988    /// for more details.
1989    ///
1990    /// This function:
1991    ///
1992    /// - Performs a HAL submission of the pending writes command
1993    ///   encoder and any other command encoders that were added to the
1994    ///   [`PendingSubmission`].
1995    /// - Advances `last_successful_submission_index` and registers the
1996    ///   submission with the lifetime tracker.
1997    /// - Returns a [`SubmissionResult`], which contains the snatch guard.
1998    fn submit_pending_submission<'a>(
1999        &self,
2000        mut pending_writes: MutexGuard<'_, PendingWrites>,
2001        prepared: PendingSubmission<'a>,
2002    ) -> Result<SubmissionResult<'a>, DeviceError> {
2003        let PendingSubmission {
2004            queue: _,
2005            snatch_guard,
2006            command_index_guard,
2007            mut executions,
2008            mut surface_textures,
2009            index: submit_index,
2010        } = prepared;
2011
2012        let mut used_surface_textures = track::TextureUsageScope::default();
2013        used_surface_textures.set_size(self.device.tracker_indices.textures.size());
2014        for texture in pending_writes.dst_textures.values() {
2015            match texture.try_inner(&snatch_guard) {
2016                Ok(TextureInner::Native { .. }) => {}
2017                Ok(TextureInner::Surface { .. }) => {
2018                    // Compare the Arcs by pointer as Textures don't implement Eq
2019                    surface_textures.insert(Arc::as_ptr(texture), texture.clone());
2020
2021                    unsafe {
2022                        used_surface_textures
2023                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
2024                            .unwrap()
2025                    };
2026                }
2027                // The texture must not have been destroyed when its usage here was
2028                // encoded. If it was destroyed after that, then it was transferred
2029                // to `pending_writes.temp_resources` at the time of destruction, so
2030                // we are still okay to use it.
2031                Err(InvalidOrDestroyedResourceError::DestroyedResource(_)) => {}
2032                Err(InvalidOrDestroyedResourceError::InvalidResource(_)) => {
2033                    unreachable!()
2034                }
2035            }
2036        }
2037
2038        if !used_surface_textures.is_empty() {
2039            let mut trackers = self.device.trackers.lock();
2040
2041            let texture_barriers = trackers
2042                .textures
2043                .set_from_usage_scope_and_drain_transitions(&used_surface_textures, &snatch_guard)
2044                .collect::<Vec<_>>();
2045            unsafe {
2046                pending_writes
2047                    .command_encoder
2048                    .transition_textures(&texture_barriers);
2049            };
2050        }
2051
2052        match pending_writes.pre_submit(
2053            &self.device.command_allocator,
2054            &self.device,
2055            self,
2056            &snatch_guard,
2057        ) {
2058            Ok(Some(pending_execution)) => {
2059                executions.insert(0, pending_execution);
2060            }
2061            Ok(None) => {}
2062            Err(e) => return Err(e),
2063        }
2064        let hal_command_buffers = executions
2065            .iter()
2066            .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
2067            .collect::<Vec<_>>();
2068
2069        {
2070            let mut submit_surface_textures =
2071                SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(surface_textures.len());
2072            for texture in surface_textures.values() {
2073                let raw = match texture.try_inner(&snatch_guard).ok() {
2074                    Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
2075                    _ => unreachable!(),
2076                };
2077                submit_surface_textures.push(raw);
2078            }
2079
2080            unsafe {
2081                self.raw().submit(
2082                    &hal_command_buffers,
2083                    &submit_surface_textures,
2084                    (self.device.fence.as_ref(), submit_index),
2085                )
2086            }
2087            .map_err(|e| self.device.handle_hal_error(e))?;
2088
2089            // Submissions must have strictly increasing indices, so we must hold the
2090            // command index guard until we have submitted, to prevent another submission
2091            // from claiming the next index and reaching `submit` before we do.
2092            drop(pending_writes);
2093
2094            // Advance the successful submission index.
2095            self.device
2096                .last_successful_submission_index
2097                .fetch_max(submit_index, Ordering::SeqCst);
2098        }
2099
2100        // this will register the new submission to the life time tracker
2101        self.lock_life().track_submission(submit_index, executions);
2102
2103        // `device.maintain` relies on being able to prevent new submissions by
2104        // using `command_index_guard` while also checking whether there are
2105        // no tracked submissions to guarantee no new submissions will happen
2106        // after a device is lost. This requires `command_index_guard` to be
2107        // held over `self.lock_life()`
2108        drop(command_index_guard);
2109
2110        Ok(SubmissionResult { snatch_guard })
2111    }
2112
2113    pub(crate) fn get_raw_timestamp_period(&self) -> f32 {
2114        unsafe { self.raw().get_timestamp_period() }
2115    }
2116
2117    pub fn get_timestamp_period(&self) -> f32 {
2118        if self.device.timestamp_normalizer.get().unwrap().enabled() {
2119            return 1.0;
2120        }
2121
2122        self.get_raw_timestamp_period()
2123    }
2124
2125    /// `closure` is guaranteed to be called.
2126    pub fn on_submitted_work_done(
2127        &self,
2128        closure: SubmittedWorkDoneClosure,
2129    ) -> Option<SubmissionIndex> {
2130        api_log!("Queue::on_submitted_work_done");
2131
2132        // A `DeviceError` means we're losing the device anyways, so we can ignore it here
2133        // (mostly to avoid a breaking change to the `on_submitted_work_done` signature
2134        // for an error case that it is unlikely the caller will be able to handle).
2135        let _: Result<_, DeviceError> = self.flush_pending_writes();
2136
2137        self.lock_life().add_work_done_closure(closure)
2138    }
2139
2140    #[allow(trivial_casts)]
2141    pub fn compact_blas(&self, blas: &Arc<Blas>) -> (Arc<Blas>, Option<CompactBlasError>) {
2142        api_log!(
2143            "Queue::compact_blas {:?}, {:?}",
2144            self as *const _,
2145            Arc::as_ptr(blas)
2146        );
2147
2148        let (blas, error) = match self.compact_blas_inner(blas) {
2149            Ok(blas) => (blas, None),
2150            Err(err) => {
2151                let new_label = blas.label.clone() + " (compacted)";
2152                (
2153                    Blas::invalid(
2154                        self.device.clone(),
2155                        &BlasDescriptor {
2156                            label: Some(new_label.into()),
2157                            flags: blas.flags,
2158                            update_mode: blas.update_mode,
2159                        },
2160                    ),
2161                    Some(err),
2162                )
2163            }
2164        };
2165
2166        // TODO: Tracing
2167
2168        (blas, error)
2169    }
2170
2171    pub(crate) fn compact_blas_inner(
2172        &self,
2173        blas: &Arc<Blas>,
2174    ) -> Result<Arc<Blas>, CompactBlasError> {
2175        profiling::scope!("Queue::compact_blas");
2176        api_log!("Queue::compact_blas");
2177
2178        let new_label = blas.label.clone() + " (compacted)";
2179
2180        self.device.check_is_valid()?;
2181
2182        self.device.require_acceleration_structures()?;
2183
2184        blas.check_is_valid()?;
2185        self.same_device_as(blas.as_ref())?;
2186
2187        let device = blas.device.clone();
2188
2189        let snatch_guard = device.snatchable_lock.read();
2190
2191        let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
2192            return Err(CompactBlasError::BlasNotReady);
2193        };
2194
2195        let mut size_info = blas.size_info;
2196        size_info.acceleration_structure_size = size;
2197
2198        let mut command_indices_lock = device.command_indices.write();
2199        let mut pending_writes = self.pending_writes.lock();
2200        let cmd_buf_raw = pending_writes.activate();
2201
2202        let raw = unsafe {
2203            device
2204                .raw()
2205                .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
2206                    label: hal_label(Some(&new_label), device.instance_flags),
2207                    size: size_info.acceleration_structure_size,
2208                    format: hal::AccelerationStructureFormat::BottomLevel,
2209                    allow_compaction: false,
2210                })
2211        }
2212        .map_err(DeviceError::from_hal)?;
2213
2214        let src_raw = blas.try_raw(&snatch_guard)?;
2215
2216        unsafe {
2217            cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
2218                src_raw,
2219                raw.as_ref(),
2220                wgt::AccelerationStructureCopy::Compact,
2221            )
2222        };
2223
2224        let handle = unsafe {
2225            device
2226                .raw()
2227                .get_acceleration_structure_device_address(raw.as_ref())
2228        };
2229
2230        command_indices_lock.next_acceleration_structure_build_command_index += 1;
2231        let built_index =
2232            NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
2233                .unwrap();
2234
2235        let new_blas = Arc::new(Blas {
2236            state: ResourceState::Valid(BlasState {
2237                raw: Snatchable::new(raw),
2238            }),
2239            device: device.clone(),
2240            size_info,
2241            sizes: blas.sizes.clone(),
2242            flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
2243            update_mode: blas.update_mode,
2244            // Bypass the submit checks which update this because we don't submit this normally.
2245            built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
2246            handle,
2247            label: new_label,
2248            tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
2249            compaction_buffer: None,
2250            compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
2251        });
2252
2253        pending_writes.insert_blas(blas);
2254        pending_writes.insert_blas(&new_blas);
2255
2256        // We should have no more errors after this because we have marked the command encoder as successful.
2257        let old_blas_size = blas.size_info.acceleration_structure_size;
2258        let new_blas_size = new_blas.size_info.acceleration_structure_size;
2259
2260        api_log!("CommandEncoder::compact_blas {:?} (size: {old_blas_size}) -> {:?} (size: {new_blas_size})", Arc::as_ptr(blas), Arc::as_ptr(&new_blas));
2261
2262        Ok(new_blas)
2263    }
2264}
2265
2266fn validate_command_buffer(
2267    command_buffer: &CommandBuffer,
2268    queue: &Queue,
2269    cmd_buf_data: &crate::command::CommandBufferMutable,
2270    snatch_guard: &SnatchGuard,
2271    surface_textures: &mut FastHashMap<*const Texture, Arc<Texture>>,
2272    used_surface_textures: &mut track::TextureUsageScope,
2273    command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
2274) -> Result<(), QueueSubmitError> {
2275    command_buffer.same_device_as(queue)?;
2276
2277    {
2278        profiling::scope!("check resource state");
2279
2280        {
2281            profiling::scope!("buffers");
2282            for buffer in cmd_buf_data.trackers.buffers.used_resources() {
2283                buffer.check_destroyed(snatch_guard)?;
2284
2285                match *buffer.map_state.read() {
2286                    BufferMapState::Idle => (),
2287                    _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
2288                }
2289            }
2290        }
2291        {
2292            profiling::scope!("textures");
2293            for texture in cmd_buf_data.trackers.textures.used_resources() {
2294                let should_extend = match texture.try_inner(snatch_guard)? {
2295                    TextureInner::Native { .. } => false,
2296                    TextureInner::Surface { .. } => {
2297                        // Compare the Arcs by pointer as Textures don't implement Eq.
2298                        surface_textures.insert(Arc::as_ptr(texture), texture.clone());
2299
2300                        true
2301                    }
2302                };
2303                if should_extend {
2304                    unsafe {
2305                        used_surface_textures
2306                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
2307                            .unwrap();
2308                    };
2309                }
2310            }
2311        }
2312        {
2313            profiling::scope!("query sets");
2314            for query_set in cmd_buf_data.trackers.query_sets.used_resources() {
2315                query_set.try_raw(snatch_guard)?;
2316            }
2317        }
2318        // WebGPU requires that we check every bind group referenced during
2319        // encoding, even ones that may have been replaced before being used.
2320        // TODO(<https://github.com/gfx-rs/wgpu/issues/8510>): Optimize this.
2321        {
2322            profiling::scope!("bind groups");
2323            for bind_group in &cmd_buf_data.trackers.bind_groups {
2324                // This checks the bind group and all resources it references.
2325                bind_group.try_raw(snatch_guard)?;
2326            }
2327        }
2328
2329        if let Err(e) =
2330            cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
2331        {
2332            return Err(e.into());
2333        }
2334    }
2335    Ok(())
2336}