wgpu_core/device/

queue.rs

#[cfg(feature = "trace")]
use alloc::string::ToString as _;
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
use core::{
    iter,
    mem::{self, ManuallyDrop},
    num::NonZeroU64,
    ptr::NonNull,
    sync::atomic::Ordering,
};
use smallvec::SmallVec;
use thiserror::Error;
use wgt::{
    error::{ErrorType, WebGpuError},
    AccelerationStructureFlags,
};

use super::{life::LifetimeTracker, Device};
#[cfg(feature = "trace")]
use crate::device::trace::{Action, IntoTrace};
use crate::{
    api_log,
    command::{
        extract_texture_selector, validate_linear_texture_data, validate_texture_buffer_copy,
        validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
        CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide,
        TransferError,
    },
    device::{DeviceError, WaitIdleError},
    get_lowest_common_denom,
    global::Global,
    hal_label,
    id::{self, BlasId, QueueId},
    init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
    lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
    ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
    resource::{
        Blas, BlasCompactState, BlasDescriptor, BlasState, Buffer, BufferAccessError,
        BufferMapState, DestroyedBuffer, DestroyedQuerySet, DestroyedResourceError,
        DestroyedTexture, FlushedStagingBuffer, InvalidOrDestroyedResourceError,
        InvalidResourceError, Labeled, ParentDevice, ResourceErrorIdent, ResourceState,
        StagingBuffer, Texture, TextureInner, Trackable, TrackingData,
    },
    resource_log,
    scratch::ScratchBuffer,
    snatch::{SnatchGuard, Snatchable},
    track::{self, Tracker, TrackerIndex},
    FastHashMap, SubmissionIndex,
};
use crate::{device::resource::CommandIndices, resource::RawResourceAccess};

pub struct Queue {
    raw: Box<dyn hal::DynQueue>,
    pub(crate) pending_writes: Mutex<PendingWrites>,
    life_tracker: Mutex<LifetimeTracker>,
    // The device needs to be dropped last (`Device.zero_buffer` might be referenced by the encoder in pending writes).
    pub(crate) device: Arc<Device>,
}

impl Queue {
    pub(crate) fn new(
        device: Arc<Device>,
        raw: Box<dyn hal::DynQueue>,
        instance_flags: wgt::InstanceFlags,
    ) -> Result<Self, DeviceError> {
        let pending_encoder = device
            .command_allocator
            .acquire_encoder(device.raw(), raw.as_ref())
            .map_err(DeviceError::from_hal);

        let pending_encoder = match pending_encoder {
            Ok(pending_encoder) => pending_encoder,
            Err(e) => {
                return Err(e);
            }
        };

        let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);

        let zero_buffer = device.zero_buffer.as_ref();
        pending_writes.activate();
        unsafe {
            pending_writes
                .command_encoder
                .transition_buffers(&[hal::BufferBarrier {
                    buffer: zero_buffer,
                    usage: hal::StateTransition {
                        from: wgt::BufferUses::empty(),
                        to: wgt::BufferUses::COPY_DST,
                    },
                }]);
            pending_writes
                .command_encoder
                .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
            pending_writes
                .command_encoder
                .transition_buffers(&[hal::BufferBarrier {
                    buffer: zero_buffer,
                    usage: hal::StateTransition {
                        from: wgt::BufferUses::COPY_DST,
                        to: wgt::BufferUses::COPY_SRC,
                    },
                }]);
        }

        Ok(Queue {
            raw,
            device,
            pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
            life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
        })
    }

    pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
        self.raw.as_ref()
    }

    #[track_caller]
    pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
        self.life_tracker.lock()
    }

    /// Ensure the surface texture is in the PRESENT state, clearing it if it was never rendered to.
    /// Submits any necessary work to the GPU before the HAL present call.
    ///
    /// See <https://github.com/gfx-rs/wgpu/issues/6748>
    pub(crate) fn prepare_surface_texture_for_present(
        &self,
        texture: &Arc<Texture>,
    ) -> Result<(), DeviceError> {
        let snatch_guard = self.device.snatchable_lock.read();
        let submission = self
            .allocate_submission(snatch_guard)
            .map_err(|(_index, e)| e)?;
        let device = &self.device;

        // If the texture is uninitialized it needs to be cleared before presenting
        let needs_clear = {
            let status = texture.initialization_status.read();
            status
                .mips
                .first()
                .is_some_and(|mip| mip.check(0..1).is_some())
        };

        let mut pending_writes = self.pending_writes.lock();

        if needs_clear {
            // After encoding the clear operation, we must not return without
            // adding the texture to `pending_writes`.
            let encoder = pending_writes.activate();
            let mut trackers = device.trackers.lock();
            crate::command::clear_texture(
                texture,
                TextureInitRange {
                    mip_range: 0..1,
                    layer_range: 0..1,
                },
                encoder,
                &mut trackers.textures,
                &device.alignments,
                device.zero_buffer.as_ref(),
                &submission.snatch_guard,
                device.instance_flags,
            )
            .map_err(|e| match e {
                ClearError::Device(e) => e,
                _ => DeviceError::Lost,
            })?;
            texture.initialization_status.write().mips[0].drain(0..1);
        }

        // Transition the texture to PRESENT in the device tracker.
        // If it's already in PRESENT, this produces no barriers and we can skip the submission.
        //
        // This has to be after any clear_texture call because clear_texture modifies the tracker state internally.
        // Computing transitions afterward ensures they reflect the actual current state.
        let pending = {
            let mut trackers = device.trackers.lock();
            let pending: Vec<track::PendingTransition<wgt::TextureUses>> = trackers
                .textures
                .set_single(
                    texture,
                    texture.full_range.clone(),
                    wgt::TextureUses::PRESENT,
                )
                .collect();
            pending
        };

        if pending.is_empty() {
            // This assert checks that we don't return here if we encoded a
            // clear operation for the texture, which would be a problem since
            // we haven't done anything yet to ensure it stays alive. If we
            // cleared the texture, then we must have produced a barrier to put
            // it in PRESENT state, so `pending` will not be empty.
            debug_assert!(!needs_clear);
            return Ok(());
        }

        // Emit the transition barriers to PRESENT.
        {
            let raw_texture = texture
                .raw(&submission.snatch_guard)
                .ok_or(DeviceError::Lost)?;
            let barriers: Vec<hal::TextureBarrier<'_, dyn hal::DynTexture>> = pending
                .into_iter()
                .map(|pt| pt.into_hal(raw_texture))
                .collect();

            let encoder = pending_writes.activate();
            // SAFETY:
            // - The encoder is in the recording state after `activate()`
            // - The texture is kept alive by adding it to `PendingWrites` below
            unsafe {
                encoder.transition_textures(&barriers);
            }
        }

        // Add the texture to `PendingWrites`. This will cause `submit()` to:
        // - Flush any pending writes to the texture.
        // - Include the texture in `surface_textures` for the submission.
        // - Keep the texture alive so the texture and its clear_view aren't
        //   destroyed before the GPU finishes the `clear_texture` operation
        //   encoded above.
        pending_writes.insert_texture(texture);

        submission.submit(pending_writes)?;

        Ok(())
    }

    /// Maintains the queue's list of finished command buffers.
    ///
    /// Command buffers submitted before `submission_index` was submitted
    /// stop being tracked and callbacks which are waiting for them are
    /// returned. Also returned is whether the queue is empty. This may
    /// be stale unless new submissions are prevented by locking
    /// [`Device::command_indices`]
    pub(crate) fn maintain(
        &self,
        submission_index: u64,
        snatch_guard: &SnatchGuard,
    ) -> (
        SmallVec<[SubmittedWorkDoneClosure; 1]>,
        Vec<super::BufferMapPendingClosure>,
        Vec<BlasCompactReadyPendingClosure>,
        bool,
    ) {
        let mut life_tracker = self.lock_life();
        let submission_closures = life_tracker.triage_submissions(submission_index);

        let mapping_closures = life_tracker.handle_mapping(snatch_guard);
        let blas_closures = life_tracker.handle_compact_read_back();

        let queue_empty = life_tracker.queue_empty();

        (
            submission_closures,
            mapping_closures,
            blas_closures,
            queue_empty,
        )
    }
}

crate::impl_resource_type!(Queue);
// TODO: https://github.com/gfx-rs/wgpu/issues/4014
impl Labeled for Queue {
    fn label(&self) -> &str {
        ""
    }
}
crate::impl_parent_device!(Queue);
crate::impl_storage_item!(Queue);

impl Drop for Queue {
    #[allow(trivial_casts)]
    fn drop(&mut self) {
        profiling::scope!("Queue::drop");
        api_log!("Queue::drop {:?}", self as *const _);
        resource_log!("Drop {}", self.error_ident());

        // On Vulkan, pending presents are not tracked by fences.
        // wait_for_idle covers both fence-tracked submissions and pending presents.
        match unsafe { self.raw.wait_for_idle() } {
            Ok(()) => {}
            Err(hal::DeviceError::Lost) => {
                self.device.handle_hal_error(hal::DeviceError::Lost);
            }
            Err(e) => {
                panic!("Unexpected error while waiting for queue idle on drop: {e:?}");
            }
        }

        let last_successful_submission_index = self
            .device
            .last_successful_submission_index
            .load(Ordering::Acquire);

        let snatch_guard = self.device.snatchable_lock.read();
        let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
            self.maintain(last_successful_submission_index, &snatch_guard);
        drop(snatch_guard);

        assert!(queue_empty);

        let closures = crate::device::UserClosures {
            mappings: mapping_closures,
            blas_compact_ready: blas_compact_ready_closures,
            submissions: submission_closures,
            device_lost_invocations: SmallVec::new(),
        };

        closures.fire();
    }
}

#[cfg(send_sync)]
pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
#[cfg(not(send_sync))]
pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;

/// A texture or buffer to be freed soon.
///
/// This is just a tagged raw texture or buffer, generally about to be added to
/// some other more specific container like:
///
/// - `PendingWrites::temp_resources`: resources used by queue writes and
///   unmaps, waiting to be folded in with the next queue submission
///
/// - `ActiveSubmission::temp_resources`: temporary resources used by a queue
///   submission, to be freed when it completes
#[derive(Debug)]
pub enum TempResource {
    StagingBuffer(FlushedStagingBuffer),
    ScratchBuffer(ScratchBuffer),
    DestroyedBuffer(DestroyedBuffer),
    DestroyedTexture(DestroyedTexture),
    DestroyedQuerySet(DestroyedQuerySet),
}

/// A series of raw [`CommandBuffer`]s that have been submitted to a
/// queue, and the [`wgpu_hal::CommandEncoder`] that built them.
///
/// [`CommandBuffer`]: hal::Api::CommandBuffer
/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
pub(crate) struct EncoderInFlight {
    inner: crate::command::InnerCommandEncoder,
    pub(crate) trackers: Tracker,
    pub(crate) temp_resources: Vec<TempResource>,
    /// We only need to keep these resources alive.
    _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,

    /// These are the buffers that have been tracked by `PendingWrites`.
    pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
    /// These are the textures that have been tracked by `PendingWrites`.
    pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
    /// These are the BLASes that have been tracked by `PendingWrites`.
    pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
}

/// A private command encoder for writes made directly on the device
/// or queue.
///
/// Operations like `buffer_unmap`, `queue_write_buffer`, and
/// `queue_write_texture` need to copy data to the GPU. At the hal
/// level, this must be done by encoding and submitting commands, but
/// these operations are not associated with any specific wgpu command
/// buffer.
///
/// Instead, `Device::pending_writes` owns one of these values, which
/// has its own hal command encoder and resource lists. The commands
/// accumulated here are automatically submitted to the queue at the
/// sooner of:
///
/// 1. The user's next wgpu command buffer submission. (Pending writes
///    are inserted ahead of the user's commands.)
/// 2. The next `mapAsync` request for a buffer that has pending
///    writes.
///
/// Important:
/// When locking pending_writes be sure that tracker is not locked
/// and try to lock trackers for the minimum timespan possible
///
/// All uses of [`StagingBuffer`]s end up here.
#[derive(Debug)]
pub(crate) struct PendingWrites {
    // The command encoder needs to be destroyed before any other resource in pending writes.
    pub command_encoder: Box<dyn hal::DynCommandEncoder>,

    /// True if `command_encoder` is in the "recording" state, as
    /// described in the docs for the [`wgpu_hal::CommandEncoder`]
    /// trait.
    ///
    /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
    pub is_recording: bool,

    temp_resources: Vec<TempResource>,
    dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
    dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
    copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
    instance_flags: wgt::InstanceFlags,
}

impl PendingWrites {
    pub fn new(
        command_encoder: Box<dyn hal::DynCommandEncoder>,
        instance_flags: wgt::InstanceFlags,
    ) -> Self {
        Self {
            command_encoder,
            is_recording: false,
            temp_resources: Vec::new(),
            dst_buffers: FastHashMap::default(),
            dst_textures: FastHashMap::default(),
            copied_blas_s: FastHashMap::default(),
            instance_flags,
        }
    }

    pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
        self.dst_buffers
            .insert(buffer.tracker_index(), buffer.clone());
    }

    pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
        self.dst_textures
            .insert(texture.tracker_index(), texture.clone());
    }

    pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
        self.copied_blas_s
            .insert(blas.tracker_index(), blas.clone());
    }

    pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
        self.dst_buffers.contains_key(&buffer.tracker_index())
    }

    pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
        self.dst_textures.contains_key(&texture.tracker_index())
    }

    pub fn consume_temp(&mut self, resource: TempResource) {
        self.temp_resources.push(resource);
    }

    pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
        self.temp_resources
            .push(TempResource::StagingBuffer(buffer));
    }

    pub fn clear_buffer(
        &mut self,
        device: &Arc<Device>,
        buffer: &Arc<Buffer>,
        range: core::ops::Range<wgt::BufferAddress>,
        snatch_guard: &SnatchGuard,
    ) -> Result<(), QueueWriteError> {
        let barriers = {
            let mut trackers = device.trackers.lock();
            trackers
                .buffers
                .set_single(buffer, wgt::BufferUses::COPY_DST)
                .map(|pending| pending.into_hal(buffer, snatch_guard))
        };

        let dst_raw = buffer.try_raw(snatch_guard)?;

        let encoder = self.activate();
        unsafe {
            encoder.transition_buffers(barriers.as_slice());
            encoder.clear_buffer(dst_raw, range.clone());
        }

        self.insert_buffer(buffer);

        // Ensure the overwritten bytes are marked as initialized so
        // they don't need to be nulled prior to mapping or binding.
        buffer.initialization_status.write().drain(range);

        Ok(())
    }

    fn pre_submit(
        &mut self,
        command_allocator: &CommandAllocator,
        device: &Arc<Device>,
        queue: &Queue,
    ) -> Result<Option<EncoderInFlight>, DeviceError> {
        if self.is_recording {
            let pending_buffers = mem::take(&mut self.dst_buffers);
            let pending_textures = mem::take(&mut self.dst_textures);
            let pending_blas_s = mem::take(&mut self.copied_blas_s);

            let cmd_buf = unsafe { self.command_encoder.end_encoding() }
                .map_err(|e| device.handle_hal_error(e))?;
            self.is_recording = false;

            let new_encoder = command_allocator
                .acquire_encoder(device.raw(), queue.raw())
                .map_err(|e| device.handle_hal_error(e))?;

            let encoder = EncoderInFlight {
                inner: crate::command::InnerCommandEncoder {
                    raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
                    list: vec![cmd_buf],
                    device: device.clone(),
                    is_open: false,
                    api: crate::command::EncodingApi::InternalUse,
                    label: "(wgpu internal) PendingWrites command encoder".into(),
                },
                trackers: Tracker::new(device.ordered_buffer_usages, device.ordered_texture_usages),
                temp_resources: mem::take(&mut self.temp_resources),
                _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
                    device.clone(),
                ),
                pending_buffers,
                pending_textures,
                pending_blas_s,
            };
            Ok(Some(encoder))
        } else {
            self.dst_buffers.clear();
            self.dst_textures.clear();
            self.copied_blas_s.clear();
            Ok(None)
        }
    }

    pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
        if !self.is_recording {
            unsafe {
                self.command_encoder
                    .begin_encoding(hal_label(
                        Some("(wgpu internal) PendingWrites"),
                        self.instance_flags,
                    ))
                    .unwrap();
            }
            self.is_recording = true;
        }
        self.command_encoder.as_mut()
    }
}

impl Drop for PendingWrites {
    fn drop(&mut self) {
        unsafe {
            if self.is_recording {
                self.command_encoder.discard_encoding();
            }
        }
    }
}

#[derive(Clone, Debug, Error)]
#[non_exhaustive]
pub enum QueueWriteError {
    #[error(transparent)]
    Queue(#[from] DeviceError),
    #[error(transparent)]
    Transfer(#[from] TransferError),
    #[error(transparent)]
    MemoryInitFailure(#[from] ClearError),
    #[error(transparent)]
    DestroyedResource(#[from] DestroyedResourceError),
    #[error(transparent)]
    InvalidResource(#[from] InvalidResourceError),
}

impl From<InvalidOrDestroyedResourceError> for QueueWriteError {
    fn from(e: InvalidOrDestroyedResourceError) -> Self {
        match e {
            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
        }
    }
}

impl WebGpuError for QueueWriteError {
    fn webgpu_error_type(&self) -> ErrorType {
        match self {
            Self::Queue(e) => e.webgpu_error_type(),
            Self::Transfer(e) => e.webgpu_error_type(),
            Self::MemoryInitFailure(e) => e.webgpu_error_type(),
            Self::DestroyedResource(e) => e.webgpu_error_type(),
            Self::InvalidResource(e) => e.webgpu_error_type(),
        }
    }
}

#[derive(Clone, Debug, Error)]
#[non_exhaustive]
pub enum QueueSubmitError {
    #[error(transparent)]
    Queue(#[from] DeviceError),
    #[error(transparent)]
    DestroyedResource(#[from] DestroyedResourceError),
    #[error("{0} is still mapped")]
    BufferStillMapped(ResourceErrorIdent),
    #[error(transparent)]
    InvalidResource(#[from] InvalidResourceError),
    #[error(transparent)]
    CommandEncoder(#[from] CommandEncoderError),
    #[error(transparent)]
    ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
}

impl From<InvalidOrDestroyedResourceError> for QueueSubmitError {
    fn from(e: InvalidOrDestroyedResourceError) -> Self {
        match e {
            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
        }
    }
}

impl WebGpuError for QueueSubmitError {
    fn webgpu_error_type(&self) -> ErrorType {
        match self {
            Self::Queue(e) => e.webgpu_error_type(),
            Self::CommandEncoder(e) => e.webgpu_error_type(),
            Self::ValidateAsActionsError(e) => e.webgpu_error_type(),
            Self::InvalidResource(e) => e.webgpu_error_type(),
            Self::DestroyedResource(_) | Self::BufferStillMapped(_) => ErrorType::Validation,
        }
    }
}

/// A partially-assembled submission.
///
/// Returned from [`Queue::allocate_submission`] and consumed by [`submit`].
/// These are internal APIs used in `Queue::submit` and other places within
/// `wgpu-core` that need to submit work.
///
/// [`submit`]: `PendingSubmission::submit`
pub(crate) struct PendingSubmission<'a> {
    queue: &'a Queue,
    snatch_guard: SnatchGuard<'a>,
    command_index_guard: RwLockWriteGuard<'a, CommandIndices>,
    // Command buffers to be executed, along with trackers for the resources they use.
    pub executions: Vec<EncoderInFlight>,
    // Surface textures referenced by command buffers in this submission. These need to be
    // passed to the HAL `submit` call. Deduplicated using a hashmap to avoid vulkan
    // deadlocking from the same surface texture being submitted multiple times.
    surface_textures: FastHashMap<*const Texture, Arc<Texture>>,
    pub index: SubmissionIndex,
}

pub(crate) struct SubmissionResult<'a> {
    pub snatch_guard: SnatchGuard<'a>,
}

impl<'a> PendingSubmission<'a> {
    fn submit(
        self,
        pending_writes: MutexGuard<'a, PendingWrites>,
    ) -> Result<SubmissionResult<'a>, DeviceError> {
        self.queue.submit_pending_submission(pending_writes, self)
    }
}

//TODO: move out common parts of write_xxx.

impl Queue {
    pub fn write_buffer(
        &self,
        buffer: Arc<Buffer>,
        buffer_offset: wgt::BufferAddress,
        data: &[u8],
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::write_buffer");
        api_log!("Queue::write_buffer");

        #[cfg(feature = "trace")]
        if let Some(ref mut trace) = *self.device.trace.lock() {
            use crate::device::trace::DataKind;
            let size = data.len() as u64;
            let data = trace.make_binary(DataKind::Bin, data);
            trace.add(Action::WriteBuffer {
                id: buffer.to_trace(),
                data,
                offset: buffer_offset,
                size,
                queued: true,
            });
        }

        buffer.check_is_valid()?;
        self.device.check_is_valid()?;

        let data_size = data.len() as wgt::BufferAddress;

        self.same_device_as(buffer.as_ref())?;

        let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
            data_size
        } else {
            // even though a zero-length write is a no-op and no copy operation will occur,
            // we must still validate the copy operation. This ensures that invalid
            // API calls—like writing to a mapped buffer or out-of-bounds offsets—are
            // caught consistently, even if no data is actually moved.
            self.validate_write_buffer_impl(buffer.as_ref(), buffer_offset, 0)?;

            log::trace!("Ignoring write_buffer of size 0");
            return Ok(());
        };

        // Platform validation requires that the staging buffer always be
        // freed, even if an error occurs. All paths from here must call
        // `device.pending_writes.consume`.
        let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;

        let staging_buffer = {
            profiling::scope!("copy");
            staging_buffer.write(data);
            staging_buffer.flush()
        };

        let snatch_guard = self.device.snatchable_lock.read();
        let mut pending_writes = self.pending_writes.lock();

        let result = self.write_staging_buffer_impl(
            &snatch_guard,
            &mut pending_writes,
            &staging_buffer,
            buffer,
            buffer_offset,
        );

        drop(snatch_guard);

        pending_writes.consume(staging_buffer);

        drop(pending_writes);

        result
    }

    pub fn create_staging_buffer(
        &self,
        buffer_size: wgt::BufferSize,
    ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
        profiling::scope!("Queue::create_staging_buffer");
        resource_log!("Queue::create_staging_buffer");

        self.device.check_is_valid()?;

        let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
        let ptr = unsafe { staging_buffer.ptr() };

        Ok((staging_buffer, ptr))
    }

    pub fn write_staging_buffer(
        &self,
        buffer: Arc<Buffer>,
        buffer_offset: wgt::BufferAddress,
        staging_buffer: StagingBuffer,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::write_staging_buffer");

        buffer.check_is_valid()?;
        self.device.check_is_valid()?;

        // At this point, we have taken ownership of the staging_buffer from the
        // user. Platform validation requires that the staging buffer always
        // be freed, even if an error occurs. All paths from here must call
        // `device.pending_writes.consume`.
        let staging_buffer = staging_buffer.flush();

        let snatch_guard = self.device.snatchable_lock.read();
        let mut pending_writes = self.pending_writes.lock();

        let result = self.write_staging_buffer_impl(
            &snatch_guard,
            &mut pending_writes,
            &staging_buffer,
            buffer,
            buffer_offset,
        );

        drop(snatch_guard);

        pending_writes.consume(staging_buffer);

        drop(pending_writes);

        result
    }

    pub fn validate_write_buffer(
        &self,
        buffer: Arc<Buffer>,
        buffer_offset: u64,
        buffer_size: wgt::BufferSize,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::validate_write_buffer");

        self.device.check_is_valid()?;
        buffer.check_is_valid()?;

        self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size.into())?;

        Ok(())
    }

    fn validate_write_buffer_impl(
        &self,
        buffer: &Buffer,
        buffer_offset: u64,
        buffer_size: u64,
    ) -> Result<(), TransferError> {
        if !matches!(&*buffer.map_state.lock(), BufferMapState::Idle) {
            return Err(TransferError::BufferNotAvailable);
        }
        buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
        if !buffer_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
            return Err(TransferError::UnalignedCopySize(buffer_size));
        }
        if !buffer_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
            return Err(TransferError::UnalignedBufferOffset(buffer_offset));
        }

        if buffer_offset > buffer.size {
            return Err(TransferError::BufferStartOffsetOverrun {
                start_offset: buffer_offset,
                buffer_size: buffer.size,
                side: CopySide::Destination,
            });
        }
        if buffer_size > buffer.size - buffer_offset {
            return Err(TransferError::BufferEndOffsetOverrun {
                start_offset: buffer_offset,
                size: buffer_size,
                buffer_size: buffer.size,
                side: CopySide::Destination,
            });
        }

        Ok(())
    }

    fn write_staging_buffer_impl(
        &self,
        snatch_guard: &SnatchGuard,
        pending_writes: &mut PendingWrites,
        staging_buffer: &FlushedStagingBuffer,
        buffer: Arc<Buffer>,
        buffer_offset: u64,
    ) -> Result<(), QueueWriteError> {
        self.device.check_is_valid()?;

        let transition = {
            let mut trackers = self.device.trackers.lock();
            trackers
                .buffers
                .set_single(&buffer, wgt::BufferUses::COPY_DST)
        };

        let dst_raw = buffer.try_raw(snatch_guard)?;

        self.same_device_as(buffer.as_ref())?;

        self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size.into())?;

        let region = hal::BufferCopy {
            src_offset: 0,
            dst_offset: buffer_offset,
            size: staging_buffer.size,
        };
        let barriers = iter::once(hal::BufferBarrier {
            buffer: staging_buffer.raw(),
            usage: hal::StateTransition {
                from: wgt::BufferUses::MAP_WRITE,
                to: wgt::BufferUses::COPY_SRC,
            },
        })
        .chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard)))
        .collect::<Vec<_>>();
        let encoder = pending_writes.activate();
        unsafe {
            encoder.transition_buffers(&barriers);
            encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
        }

        pending_writes.insert_buffer(&buffer);

        // Ensure the overwritten bytes are marked as initialized so
        // they don't need to be nulled prior to mapping or binding.
        {
            buffer
                .initialization_status
                .write()
                .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
        }

        Ok(())
    }

    pub fn write_texture(
        &self,
        destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
        data: &[u8],
        data_layout: &wgt::TexelCopyBufferLayout,
        size: &wgt::Extent3d,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::write_texture");
        api_log!("Queue::write_texture");

        self.device.check_is_valid()?;

        let dst = destination.texture;
        let destination = wgt::TexelCopyTextureInfo {
            texture: (),
            mip_level: destination.mip_level,
            origin: destination.origin,
            aspect: destination.aspect,
        };

        self.same_device_as(dst.as_ref())?;

        dst.check_valid()?;

        dst.check_usage(wgt::TextureUsages::COPY_DST)
            .map_err(TransferError::MissingTextureUsage)?;

        // Note: Doing the copy range validation early is important because ensures that the
        // dimensions are not going to cause overflow in other parts of the validation.
        let (hal_copy_size, array_layer_count) =
            validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;

        let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?;

        validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;

        validate_texture_buffer_copy(
            &destination,
            dst_base.aspect,
            &dst.desc,
            data_layout,
            false, // alignment not required for buffer offset or bytes per row
        )?;

        // Note: `_source_bytes_per_array_layer` is ignored since we
        // have a staging copy, and it can have a different value.
        let (required_bytes_in_copy, _source_bytes_per_array_layer, _) =
            validate_linear_texture_data(
                data_layout,
                dst.desc.format,
                destination.aspect,
                data.len() as wgt::BufferAddress,
                CopySide::Source,
                size,
            )?;

        if dst.desc.format.is_depth_stencil_format() {
            self.device
                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
                .map_err(TransferError::from)?;
        }

        let snatch_guard = self.device.snatchable_lock.read();

        let dst_raw = dst.try_inner(&snatch_guard)?.raw();

        // This must happen after parameter validation (so that errors are reported
        // as required by the spec), but before any side effects.
        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
            log::trace!("Ignoring write_texture of size 0");
            return Ok(());
        }

        let mut pending_writes = self.pending_writes.lock();
        let encoder = pending_writes.activate();

        // If the copy does not fully cover the layers, we need to initialize to
        // zero *first* as we don't keep track of partial texture layer inits.
        //
        // Strictly speaking we only need to clear the areas of a layer
        // untouched, but this would get increasingly messy.
        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
            // volume textures don't have a layer range as array volumes aren't supported
            0..1
        } else {
            destination.origin.z..destination.origin.z + size.depth_or_array_layers
        };
        let mut dst_initialization_status = dst.initialization_status.write();
        if dst_initialization_status.mips[destination.mip_level as usize]
            .check(init_layer_range.clone())
            .is_some()
        {
            if has_copy_partial_init_tracker_coverage(size, &destination, &dst.desc) {
                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range)
                    .collect::<Vec<core::ops::Range<u32>>>()
                {
                    let mut trackers = self.device.trackers.lock();
                    crate::command::clear_texture(
                        &dst,
                        TextureInitRange {
                            mip_range: destination.mip_level..(destination.mip_level + 1),
                            layer_range,
                        },
                        encoder,
                        &mut trackers.textures,
                        &self.device.alignments,
                        self.device.zero_buffer.as_ref(),
                        &snatch_guard,
                        self.device.instance_flags,
                    )
                    .map_err(QueueWriteError::from)?;
                }
            } else {
                dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range);
            }
        }

        let (block_width, block_height) = dst.desc.format.block_dimensions();
        let width_in_blocks = size.width / block_width;
        let height_in_blocks = size.height / block_height;

        let block_size = dst
            .desc
            .format
            .block_copy_size(Some(destination.aspect))
            .unwrap();
        let bytes_in_last_row = width_in_blocks * block_size;

        let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
        let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);

        let bytes_per_row_alignment = get_lowest_common_denom(
            self.device.alignments.buffer_copy_pitch.get() as u32,
            block_size,
        );
        assert!(u32::MAX - bytes_in_last_row >= bytes_per_row_alignment);
        let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);

        // Platform validation requires that the staging buffer always be
        // freed, even if an error occurs. All paths from here must call
        // `device.pending_writes.consume`.
        let staging_buffer = if stage_bytes_per_row == bytes_per_row {
            profiling::scope!("copy aligned");
            // Fast path if the data is already being aligned optimally.
            let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
            staging_buffer.write(&data[data_layout.offset as usize..]);
            staging_buffer
        } else {
            profiling::scope!("copy chunked");
            // Copy row by row into the optimal alignment.
            let block_rows_in_copy = u64::from(size.depth_or_array_layers - 1)
                * u64::from(rows_per_image)
                + u64::from(height_in_blocks);
            // The copy size was validated against the source buffer, however,
            // `stage_bytes_per_row` can differ, so let's be paranoid.
            let stage_size = u64::from(stage_bytes_per_row)
                .checked_mul(block_rows_in_copy)
                .and_then(wgt::BufferSize::new)
                .unwrap();
            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
            for layer in 0..u64::from(size.depth_or_array_layers) {
                let rows_offset = layer * u64::from(rows_per_image);
                for row in rows_offset..rows_offset + u64::from(height_in_blocks) {
                    let src_offset = data_layout.offset + row * u64::from(bytes_per_row);
                    let dst_offset = row * u64::from(stage_bytes_per_row);
                    unsafe {
                        staging_buffer.write_with_offset(
                            data,
                            src_offset as isize,
                            dst_offset as isize,
                            bytes_in_last_row as usize,
                        )
                    }
                }
            }
            staging_buffer
        };

        let staging_buffer = staging_buffer.flush();

        let regions = (0..array_layer_count)
            .map(|array_layer_offset| {
                let mut texture_base = dst_base.clone();
                texture_base.array_layer += array_layer_offset;
                hal::BufferTextureCopy {
                    buffer_layout: wgt::TexelCopyBufferLayout {
                        offset: array_layer_offset as u64
                            * rows_per_image as u64
                            * stage_bytes_per_row as u64,
                        bytes_per_row: Some(stage_bytes_per_row),
                        rows_per_image: Some(rows_per_image),
                    },
                    texture_base,
                    size: hal_copy_size,
                }
            })
            .collect::<Vec<_>>();

        {
            let buffer_barrier = hal::BufferBarrier {
                buffer: staging_buffer.raw(),
                usage: hal::StateTransition {
                    from: wgt::BufferUses::MAP_WRITE,
                    to: wgt::BufferUses::COPY_SRC,
                },
            };

            let mut trackers = self.device.trackers.lock();
            let transition =
                trackers
                    .textures
                    .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
            let texture_barriers = transition
                .map(|pending| pending.into_hal(dst_raw))
                .collect::<Vec<_>>();

            unsafe {
                encoder.transition_textures(&texture_barriers);
                encoder.transition_buffers(&[buffer_barrier]);
                encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, &regions);
            }
        }

        pending_writes.consume(staging_buffer);
        pending_writes.insert_texture(&dst);

        Ok(())
    }

    #[cfg(webgl)]
    pub fn copy_external_image_to_texture(
        &self,
        source: &wgt::CopyExternalImageSourceInfo,
        destination: wgt::CopyExternalImageDestInfo<Arc<Texture>>,
        size: wgt::Extent3d,
    ) -> Result<(), QueueWriteError> {
        use crate::conv;

        profiling::scope!("Queue::copy_external_image_to_texture");

        self.device.check_is_valid()?;

        let mut needs_flag = false;
        needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
        needs_flag |= source.origin != wgt::Origin2d::ZERO;
        needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
        #[allow(clippy::bool_comparison)]
        if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
            needs_flag |= source.flip_y != false;
            needs_flag |= destination.premultiplied_alpha != false;
        }

        if needs_flag {
            self.device
                .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
                .map_err(TransferError::from)?;
        }

        let src_width = source.source.width();
        let src_height = source.source.height();

        let dst = destination.texture;
        let premultiplied_alpha = destination.premultiplied_alpha;
        let destination = wgt::TexelCopyTextureInfo {
            texture: (),
            mip_level: destination.mip_level,
            origin: destination.origin,
            aspect: destination.aspect,
        };

        dst.check_valid()?;

        if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
            return Err(
                TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
            );
        }
        if dst.desc.dimension != wgt::TextureDimension::D2 {
            return Err(TransferError::InvalidDimensionExternal.into());
        }
        dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
            .map_err(TransferError::MissingTextureUsage)?;
        if dst.desc.sample_count != 1 {
            return Err(TransferError::InvalidSampleCount {
                sample_count: dst.desc.sample_count,
            }
            .into());
        }

        if source.origin.x > src_width || src_width - source.origin.x < size.width {
            return Err(TransferError::TextureOverrun {
                start_offset: source.origin.x,
                end_offset: source.origin.x.saturating_add(size.width),
                texture_size: src_width,
                dimension: crate::resource::TextureErrorDimension::X,
                side: CopySide::Source,
            }
            .into());
        }
        if source.origin.y > src_height || src_height - source.origin.y < size.height {
            return Err(TransferError::TextureOverrun {
                start_offset: source.origin.y,
                end_offset: source.origin.y.saturating_add(size.height),
                texture_size: src_height,
                dimension: crate::resource::TextureErrorDimension::Y,
                side: CopySide::Source,
            }
            .into());
        }
        if size.depth_or_array_layers != 1 {
            return Err(TransferError::TextureOverrun {
                start_offset: 0,
                end_offset: size.depth_or_array_layers,
                texture_size: 1,
                dimension: crate::resource::TextureErrorDimension::Z,
                side: CopySide::Source,
            }
            .into());
        }

        // Note: Doing the copy range validation early is important because ensures that the
        // dimensions are not going to cause overflow in other parts of the validation.
        let (hal_copy_size, _) =
            validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;

        let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?;

        // This must happen after parameter validation (so that errors are reported
        // as required by the spec), but before any side effects.
        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
            log::trace!("Ignoring copy_external_image_to_texture of size 0");
            return Ok(());
        }

        let mut pending_writes = self.pending_writes.lock();
        let encoder = pending_writes.activate();

        // If the copy does not fully cover the layers, we need to initialize to
        // zero *first* as we don't keep track of partial texture layer inits.
        //
        // Strictly speaking we only need to clear the areas of a layer
        // untouched, but this would get increasingly messy.
        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
            // volume textures don't have a layer range as array volumes aren't supported
            0..1
        } else {
            destination.origin.z..destination.origin.z + size.depth_or_array_layers
        };
        let mut dst_initialization_status = dst.initialization_status.write();
        if dst_initialization_status.mips[destination.mip_level as usize]
            .check(init_layer_range.clone())
            .is_some()
        {
            if has_copy_partial_init_tracker_coverage(&size, &destination, &dst.desc) {
                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range)
                    .collect::<Vec<core::ops::Range<u32>>>()
                {
                    let mut trackers = self.device.trackers.lock();
                    crate::command::clear_texture(
                        &dst,
                        TextureInitRange {
                            mip_range: destination.mip_level..(destination.mip_level + 1),
                            layer_range,
                        },
                        encoder,
                        &mut trackers.textures,
                        &self.device.alignments,
                        self.device.zero_buffer.as_ref(),
                        &self.device.snatchable_lock.read(),
                        self.device.instance_flags,
                    )
                    .map_err(QueueWriteError::from)?;
                }
            } else {
                dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range);
            }
        }

        let snatch_guard = self.device.snatchable_lock.read();
        let dst_raw = dst.try_raw(&snatch_guard)?;

        let regions = hal::TextureCopy {
            src_base: hal::TextureCopyBase {
                mip_level: 0,
                array_layer: 0,
                origin: source.origin.to_3d(0),
                aspect: hal::FormatAspects::COLOR,
            },
            dst_base,
            size: hal_copy_size,
        };

        let mut trackers = self.device.trackers.lock();
        let transitions = trackers
            .textures
            .set_single(&dst, selector, wgt::TextureUses::COPY_DST);

        // `copy_external_image_to_texture` is exclusive to the WebGL backend.
        // Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend.
        let encoder_webgl = encoder
            .as_any_mut()
            .downcast_mut::<hal::gles::CommandEncoder>()
            .unwrap();
        let dst_raw_webgl = dst_raw
            .as_any()
            .downcast_ref::<hal::gles::Texture>()
            .unwrap();
        let transitions_webgl = transitions.map(|pending| {
            let dyn_transition = pending.into_hal(dst_raw);
            hal::TextureBarrier {
                texture: dst_raw_webgl,
                range: dyn_transition.range,
                usage: dyn_transition.usage,
            }
        });

        use hal::CommandEncoder as _;
        unsafe {
            encoder_webgl.transition_textures(transitions_webgl);
            encoder_webgl.copy_external_image_to_texture(
                source,
                dst_raw_webgl,
                premultiplied_alpha,
                iter::once(regions),
            );
        }

        pending_writes.insert_texture(&dst);

        Ok(())
    }

    /// Flush `PendingWrites` if it contains a write to `buffer`.
    pub fn flush_writes_for_buffer(
        &self,
        buffer: &Arc<Buffer>,
        snatch_guard: SnatchGuard,
    ) -> Result<(), BufferAccessError> {
        let submission = self
            .allocate_submission(snatch_guard)
            .map_err(|(_index, e)| e)?;

        let pending_writes = self.pending_writes.lock();
        if !pending_writes.contains_buffer(buffer) {
            return Ok(());
        }

        submission.submit(pending_writes)?;

        Ok(())
    }

    fn flush_pending_writes(&self) -> Result<Option<SubmissionIndex>, DeviceError> {
        let snatch_guard = self.device.snatchable_lock.read();
        let submission = self
            .allocate_submission(snatch_guard)
            .map_err(|(_index, e)| e)?;
        let submit_index = submission.index;
        let pending_writes = self.pending_writes.lock();
        if pending_writes.is_recording {
            submission.submit(pending_writes)?;
            Ok(Some(submit_index))
        } else {
            Ok(None)
        }
    }

    #[cfg(feature = "trace")]
    fn trace_submission(
        &self,
        submit_index: SubmissionIndex,
        commands: Vec<crate::command::Command<crate::command::PointerReferences>>,
    ) {
        if let Some(ref mut trace) = *self.device.trace.lock() {
            trace.add(Action::Submit(submit_index, commands));
        }
    }

    #[cfg(feature = "trace")]
    fn trace_failed_submission(
        &self,
        submit_index: SubmissionIndex,
        commands: Option<Vec<crate::command::Command<crate::command::PointerReferences>>>,
        error: alloc::string::String,
    ) {
        if let Some(ref mut trace) = *self.device.trace.lock() {
            trace.add(Action::FailedCommands {
                commands,
                failed_at_submit: Some(submit_index),
                error,
            });
        }
    }

    pub fn submit(
        &self,
        command_buffers: &[Arc<CommandBuffer>],
    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
        profiling::scope!("Queue::submit");
        api_log!("Queue::submit");

        let snatch_guard = self.device.snatchable_lock.read();
        let mut submission = self
            .allocate_submission(snatch_guard)
            .map_err(|(index, e)| (index, e.into()))?;
        let submit_index = submission.index;

        let res = 'error: {
            let mut used_surface_textures = track::TextureUsageScope::default();

            {
                if !command_buffers.is_empty() {
                    profiling::scope!("prepare");

                    let mut first_error = None;

                    //TODO: if multiple command buffers are submitted, we can re-use the last
                    // native command buffer of the previous chain instead of always creating
                    // a temporary one, since the chains are not finished.

                    // finish all the command buffers first
                    for command_buffer in command_buffers {
                        profiling::scope!("process command buffer");

                        // we reset the used surface textures every time we use
                        // it, so make sure to set_size on it.
                        used_surface_textures.set_size(self.device.tracker_indices.textures.size());

                        // Note that we are required to invalidate all command buffers in both the success and failure paths.
                        // This is why we `continue` and don't early return via `?`.
                        #[allow(unused_mut)]
                        let mut cmd_buf_data = command_buffer.take_finished();

                        if first_error.is_some() {
                            continue;
                        }

                        #[cfg(feature = "trace")]
                        let trace_commands = cmd_buf_data
                            .as_mut()
                            .ok()
                            .and_then(|data| mem::take(&mut data.trace_commands));

                        let mut baked = match cmd_buf_data {
                            Ok(cmd_buf_data) => {
                                let res = validate_command_buffer(
                                    command_buffer,
                                    self,
                                    &cmd_buf_data,
                                    &submission.snatch_guard,
                                    &mut submission.surface_textures,
                                    &mut used_surface_textures,
                                    &mut submission.command_index_guard,
                                );
                                if let Err(err) = res {
                                    #[cfg(feature = "trace")]
                                    self.trace_failed_submission(
                                        submit_index,
                                        trace_commands,
                                        err.to_string(),
                                    );
                                    first_error.get_or_insert(err);
                                    continue;
                                }

                                #[cfg(feature = "trace")]
                                if let Some(commands) = trace_commands {
                                    self.trace_submission(submit_index, commands);
                                }

                                cmd_buf_data.set_acceleration_structure_dependencies(
                                    &submission.snatch_guard,
                                );
                                cmd_buf_data.into_baked_commands()
                            }
                            Err(err) => {
                                #[cfg(feature = "trace")]
                                self.trace_failed_submission(
                                    submit_index,
                                    trace_commands,
                                    err.to_string(),
                                );
                                first_error.get_or_insert(err.into());
                                continue;
                            }
                        };

                        if let Err(e) = baked.process_deferred_query_set_resolves(
                            &self.device,
                            &submission.snatch_guard,
                        ) {
                            break 'error Err(e.into());
                        }

                        // execute resource transitions
                        if let Err(e) = baked.encoder.open_pass(hal_label(
                            Some("(wgpu internal) Transit"),
                            self.device.instance_flags,
                        )) {
                            break 'error Err(e.into());
                        }

                        //Note: locking the trackers has to be done after the storages
                        let mut trackers = self.device.trackers.lock();
                        if let Err(e) =
                            baked.initialize_buffer_memory(&mut trackers, &submission.snatch_guard)
                        {
                            break 'error Err(e.into());
                        }
                        if let Err(e) = baked.initialize_texture_memory(
                            &mut trackers,
                            &self.device,
                            &submission.snatch_guard,
                        ) {
                            break 'error Err(e.into());
                        }

                        //Note: stateless trackers are not merged:
                        // device already knows these resources exist.
                        CommandEncoder::insert_barriers_from_device_tracker(
                            baked.encoder.raw.as_mut(),
                            &mut trackers,
                            &baked.trackers,
                            &submission.snatch_guard,
                        );

                        if let Err(e) = baked.encoder.close_and_push_front() {
                            break 'error Err(e.into());
                        }

                        // Transition surface textures into `Present` state.
                        // Note: we could technically do it after all of the command buffers,
                        // but here we have a command encoder by hand, so it's easier to use it.
                        if !used_surface_textures.is_empty() {
                            if let Err(e) = baked.encoder.open_pass(hal_label(
                                Some("(wgpu internal) Present"),
                                self.device.instance_flags,
                            )) {
                                break 'error Err(e.into());
                            }
                            let texture_barriers = trackers
                                .textures
                                .set_from_usage_scope_and_drain_transitions(
                                    &used_surface_textures,
                                    &submission.snatch_guard,
                                )
                                .collect::<Vec<_>>();
                            unsafe {
                                baked.encoder.raw.transition_textures(&texture_barriers);
                            };
                            if let Err(e) = baked.encoder.close() {
                                break 'error Err(e.into());
                            }
                            used_surface_textures = track::TextureUsageScope::default();
                        }

                        // done
                        submission.executions.push(EncoderInFlight {
                            inner: baked.encoder,
                            trackers: baked.trackers,
                            temp_resources: baked.temp_resources,
                            _indirect_draw_validation_resources: baked
                                .indirect_draw_validation_resources,
                            pending_buffers: FastHashMap::default(),
                            pending_textures: FastHashMap::default(),
                            pending_blas_s: FastHashMap::default(),
                        });
                    }

                    if let Some(first_error) = first_error {
                        break 'error Err(first_error);
                    }
                }
            }

            let pending_writes = self.pending_writes.lock();

            let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) {
                Ok(result) => result,
                Err(e) => break 'error Err(e.into()),
            };

            profiling::scope!("cleanup");

            // This will schedule destruction of all resources that are no longer needed
            // by the user but used in the command stream, among other things.
            // `device.maintain` consumes and will release the snatch guard.
            let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard);
            match result {
                Ok(status) => {
                    debug_assert!(matches!(
                        status,
                        wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
                    ));
                }
                Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
                Err(WaitIdleError::WrongSubmissionIndex(..)) => {
                    unreachable!("Cannot get WrongSubmissionIndex from Poll")
                }
                Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
            };

            Ok(closures)
        };

        let callbacks = match res {
            Ok(ok) => ok,
            Err(e) => return Err((submit_index, e)),
        };

        // the closures should execute with nothing locked!
        callbacks.fire();

        self.device.lose_if_oom();

        api_log!("Queue::submit returned submit index {submit_index}");

        Ok(submit_index)
    }

    /// Allocate a submission index and prepare for a submission.
    ///
    /// This is an internal API used in [`Queue::submit`] and other places within
    /// `wgpu-core` that need to submit work.
    ///
    /// Returns the index and a [`PendingSubmission`].
    ///
    /// The caller passes in the already-acquired [`SnatchGuard`]. This function acquires
    /// the fence lock and the command index lock.
    ///
    /// The caller should update [`PendingSubmission::executions`] with details of the
    /// submission.
    ///
    /// To finalize and submit the submission, call [`PendingSubmission::submit`] (which is
    /// a convenience wrapper around [`Queue::submit_pending_submission`]).
    ///
    /// After calling this function and before submitting, the caller must acquire the
    /// pending writes lock, and pass it to `submit`.
    ///
    /// It is also acceptable to drop the `PendingSubmission` without submitting. This may
    /// be necessary when locks are required to access the state that determines whether a
    /// submission is needed.
    fn allocate_submission<'a>(
        &'a self,
        snatch_guard: SnatchGuard<'a>,
    ) -> Result<PendingSubmission<'a>, (SubmissionIndex, DeviceError)> {
        let mut command_index_guard = self.device.command_indices.write();
        command_index_guard.active_submission_index += 1;
        let index = command_index_guard.active_submission_index;

        if let Err(e) = self.device.check_is_valid() {
            return Err((index, e));
        }

        let submission = PendingSubmission {
            queue: self,
            snatch_guard,
            command_index_guard,
            executions: Vec::new(),
            surface_textures: FastHashMap::default(),
            index,
        };

        Ok(submission)
    }

    /// Finalize and submit a [`PendingSubmission`] that was returned by
    /// [`Queue::allocate_submission`].
    ///
    /// This is an internal API used in `Queue::submit` and other places within
    /// `wgpu-core` that need to submit work. See [`Queue::allocate_submission`]
    /// for more details.
    ///
    /// This function:
    ///
    /// - Performs a HAL submission of the pending writes command
    ///   encoder and any other command encoders that were added to the
    ///   [`PendingSubmission`].
    /// - Advances `last_successful_submission_index` and registers the
    ///   submission with the lifetime tracker.
    /// - Returns a [`SubmissionResult`], which contains the snatch guard.
    fn submit_pending_submission<'a>(
        &self,
        mut pending_writes: MutexGuard<'_, PendingWrites>,
        prepared: PendingSubmission<'a>,
    ) -> Result<SubmissionResult<'a>, DeviceError> {
        let PendingSubmission {
            queue: _,
            snatch_guard,
            command_index_guard,
            mut executions,
            mut surface_textures,
            index: submit_index,
        } = prepared;

        let mut used_surface_textures = track::TextureUsageScope::default();
        used_surface_textures.set_size(self.device.tracker_indices.textures.size());
        for texture in pending_writes.dst_textures.values() {
            match texture.try_inner(&snatch_guard) {
                Ok(TextureInner::Native { .. }) => {}
                Ok(TextureInner::Surface { .. }) => {
                    // Compare the Arcs by pointer as Textures don't implement Eq
                    surface_textures.insert(Arc::as_ptr(texture), texture.clone());

                    unsafe {
                        used_surface_textures
                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
                            .unwrap()
                    };
                }
                // The texture must not have been destroyed when its usage here was
                // encoded. If it was destroyed after that, then it was transferred
                // to `pending_writes.temp_resources` at the time of destruction, so
                // we are still okay to use it.
                Err(InvalidOrDestroyedResourceError::DestroyedResource(_)) => {}
                Err(InvalidOrDestroyedResourceError::InvalidResource(_)) => {
                    unreachable!()
                }
            }
        }

        if !used_surface_textures.is_empty() {
            let mut trackers = self.device.trackers.lock();

            let texture_barriers = trackers
                .textures
                .set_from_usage_scope_and_drain_transitions(&used_surface_textures, &snatch_guard)
                .collect::<Vec<_>>();
            unsafe {
                pending_writes
                    .command_encoder
                    .transition_textures(&texture_barriers);
            };
        }

        match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) {
            Ok(Some(pending_execution)) => {
                executions.insert(0, pending_execution);
            }
            Ok(None) => {}
            Err(e) => return Err(e),
        }
        let hal_command_buffers = executions
            .iter()
            .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
            .collect::<Vec<_>>();

        {
            let mut submit_surface_textures =
                SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(surface_textures.len());
            for texture in surface_textures.values() {
                let raw = match texture.try_inner(&snatch_guard).ok() {
                    Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
                    _ => unreachable!(),
                };
                submit_surface_textures.push(raw);
            }

            unsafe {
                self.raw().submit(
                    &hal_command_buffers,
                    &submit_surface_textures,
                    (self.device.fence.as_ref(), submit_index),
                )
            }
            .map_err(|e| self.device.handle_hal_error(e))?;

            // Submissions must have strictly increasing indices, so we must hold the
            // command index guard until we have submitted, to prevent another submission
            // from claiming the next index and reaching `submit` before we do.
            drop(pending_writes);

            // Advance the successful submission index.
            self.device
                .last_successful_submission_index
                .fetch_max(submit_index, Ordering::SeqCst);
        }

        // this will register the new submission to the life time tracker
        self.lock_life().track_submission(submit_index, executions);

        // `device.maintain` relies on being able to prevent new submissions by
        // using `command_index_guard` while also checking whether there are
        // no tracked submissions to guarantee no new submissions will happen
        // after a device is lost. This requires `command_index_guard` to be
        // held over `self.lock_life()`
        drop(command_index_guard);

        Ok(SubmissionResult { snatch_guard })
    }

    pub fn get_timestamp_period(&self) -> f32 {
        unsafe { self.raw().get_timestamp_period() }
    }

    /// `closure` is guaranteed to be called.
    pub fn on_submitted_work_done(
        &self,
        closure: SubmittedWorkDoneClosure,
    ) -> Option<SubmissionIndex> {
        api_log!("Queue::on_submitted_work_done");

        // A `DeviceError` means we're losing the device anyways, so we can ignore it here
        // (mostly to avoid a breaking change to the `on_submitted_work_done` signature
        // for an error case that it is unlikely the caller will be able to handle).
        let _: Result<_, DeviceError> = self.flush_pending_writes();

        self.lock_life().add_work_done_closure(closure)
    }

    #[allow(trivial_casts)]
    pub fn compact_blas(&self, blas: &Arc<Blas>) -> (Arc<Blas>, Option<CompactBlasError>) {
        api_log!(
            "Queue::compact_blas {:?}, {:?}",
            self as *const _,
            Arc::as_ptr(blas)
        );

        let (blas, error) = match self.compact_blas_inner(blas) {
            Ok(blas) => (blas, None),
            Err(err) => {
                let new_label = blas.label.clone() + " (compacted)";
                (
                    Blas::invalid(
                        self.device.clone(),
                        &BlasDescriptor {
                            label: Some(new_label.into()),
                            flags: blas.flags,
                            update_mode: blas.update_mode,
                        },
                    ),
                    Some(err),
                )
            }
        };

        // TODO: Tracing

        (blas, error)
    }

    pub(crate) fn compact_blas_inner(
        &self,
        blas: &Arc<Blas>,
    ) -> Result<Arc<Blas>, CompactBlasError> {
        profiling::scope!("Queue::compact_blas");
        api_log!("Queue::compact_blas");

        let new_label = blas.label.clone() + " (compacted)";

        self.device.check_is_valid()?;

        self.device
            .require_features(wgpu_types::Features::EXPERIMENTAL_RAY_QUERY)?;

        blas.check_is_valid()?;
        self.same_device_as(blas.as_ref())?;

        let device = blas.device.clone();

        let snatch_guard = device.snatchable_lock.read();

        let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
            return Err(CompactBlasError::BlasNotReady);
        };

        let mut size_info = blas.size_info;
        size_info.acceleration_structure_size = size;

        let mut pending_writes = self.pending_writes.lock();
        let cmd_buf_raw = pending_writes.activate();

        let raw = unsafe {
            device
                .raw()
                .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
                    label: hal_label(Some(&new_label), device.instance_flags),
                    size: size_info.acceleration_structure_size,
                    format: hal::AccelerationStructureFormat::BottomLevel,
                    allow_compaction: false,
                })
        }
        .map_err(DeviceError::from_hal)?;

        let src_raw = blas.try_raw(&snatch_guard)?;

        unsafe {
            cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
                src_raw,
                raw.as_ref(),
                wgt::AccelerationStructureCopy::Compact,
            )
        };

        let handle = unsafe {
            device
                .raw()
                .get_acceleration_structure_device_address(raw.as_ref())
        };

        drop(snatch_guard);

        let mut command_indices_lock = device.command_indices.write();
        command_indices_lock.next_acceleration_structure_build_command_index += 1;
        let built_index =
            NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
                .unwrap();

        let new_blas = Arc::new(Blas {
            state: ResourceState::Valid(BlasState {
                raw: Snatchable::new(raw),
            }),
            device: device.clone(),
            size_info,
            sizes: blas.sizes.clone(),
            flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
            update_mode: blas.update_mode,
            // Bypass the submit checks which update this because we don't submit this normally.
            built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
            handle,
            label: new_label,
            tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
            compaction_buffer: None,
            compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
        });

        pending_writes.insert_blas(blas);
        pending_writes.insert_blas(&new_blas);

        // We should have no more errors after this because we have marked the command encoder as successful.
        let old_blas_size = blas.size_info.acceleration_structure_size;
        let new_blas_size = new_blas.size_info.acceleration_structure_size;

        api_log!("CommandEncoder::compact_blas {:?} (size: {old_blas_size}) -> {:?} (size: {new_blas_size})", Arc::as_ptr(blas), Arc::as_ptr(&new_blas));

        Ok(new_blas)
    }
}

impl Global {
    pub fn queue_write_buffer(
        &self,
        queue_id: QueueId,
        buffer_id: id::BufferId,
        buffer_offset: wgt::BufferAddress,
        data: &[u8],
    ) -> Result<(), QueueWriteError> {
        let queue = self.hub.queues.get(queue_id);
        let buffer = self.hub.buffers.get(buffer_id);

        queue.write_buffer(buffer, buffer_offset, data)
    }

    pub fn queue_create_staging_buffer(
        &self,
        queue_id: QueueId,
        buffer_size: wgt::BufferSize,
        id_in: Option<id::StagingBufferId>,
    ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
        let queue = self.hub.queues.get(queue_id);
        let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?;

        let fid = self.hub.staging_buffers.prepare(id_in);
        let id = fid.assign(staging_buffer);

        Ok((id, ptr))
    }

    pub fn queue_write_staging_buffer(
        &self,
        queue_id: QueueId,
        buffer_id: id::BufferId,
        buffer_offset: wgt::BufferAddress,
        staging_buffer_id: id::StagingBufferId,
    ) -> Result<(), QueueWriteError> {
        let queue = self.hub.queues.get(queue_id);
        let buffer = self.hub.buffers.get(buffer_id);
        let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id);
        queue.write_staging_buffer(buffer, buffer_offset, staging_buffer)
    }

    pub fn queue_validate_write_buffer(
        &self,
        queue_id: QueueId,
        buffer_id: id::BufferId,
        buffer_offset: u64,
        buffer_size: wgt::BufferSize,
    ) -> Result<(), QueueWriteError> {
        let queue = self.hub.queues.get(queue_id);
        let buffer = self.hub.buffers.get(buffer_id);
        queue.validate_write_buffer(buffer, buffer_offset, buffer_size)
    }

    pub fn queue_write_texture(
        &self,
        queue_id: QueueId,
        destination: &wgt::TexelCopyTextureInfo<id::TextureId>,
        data: &[u8],
        data_layout: &wgt::TexelCopyBufferLayout,
        size: &wgt::Extent3d,
    ) -> Result<(), QueueWriteError> {
        let queue = self.hub.queues.get(queue_id);
        let texture = self.hub.textures.get(destination.texture);
        let destination = wgt::TexelCopyTextureInfo {
            texture,
            mip_level: destination.mip_level,
            origin: destination.origin,
            aspect: destination.aspect,
        };

        #[cfg(feature = "trace")]
        if let Some(ref mut trace) = *queue.device.trace.lock() {
            use crate::device::trace::DataKind;
            let data = trace.make_binary(DataKind::Bin, data);
            trace.add(Action::WriteTexture {
                to: destination.to_trace(),
                data,
                layout: *data_layout,
                size: *size,
            });
        }

        queue.write_texture(destination, data, data_layout, size)
    }

    #[cfg(webgl)]
    pub fn queue_copy_external_image_to_texture(
        &self,
        queue_id: QueueId,
        source: &wgt::CopyExternalImageSourceInfo,
        destination: crate::command::CopyExternalImageDestInfo,
        size: wgt::Extent3d,
    ) -> Result<(), QueueWriteError> {
        let queue = self.hub.queues.get(queue_id);
        let destination = wgt::CopyExternalImageDestInfo {
            texture: self.hub.textures.get(destination.texture),
            mip_level: destination.mip_level,
            origin: destination.origin,
            aspect: destination.aspect,
            color_space: destination.color_space,
            premultiplied_alpha: destination.premultiplied_alpha,
        };
        queue.copy_external_image_to_texture(source, destination, size)
    }

    pub fn queue_submit(
        &self,
        queue_id: QueueId,
        command_buffer_ids: &[id::CommandBufferId],
    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
        let queue = self.hub.queues.get(queue_id);
        let command_buffer_guard = self.hub.command_buffers.read();
        let command_buffers = command_buffer_ids
            .iter()
            .map(|id| command_buffer_guard.get(*id))
            .collect::<Vec<_>>();
        drop(command_buffer_guard);
        queue.submit(&command_buffers)
    }

    pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
        let queue = self.hub.queues.get(queue_id);

        if queue.device.timestamp_normalizer.get().unwrap().enabled() {
            return 1.0;
        }

        queue.get_timestamp_period()
    }

    pub fn queue_on_submitted_work_done(
        &self,
        queue_id: QueueId,
        closure: SubmittedWorkDoneClosure,
    ) -> SubmissionIndex {
        api_log!("Queue::on_submitted_work_done {queue_id:?}");

        let queue = self.hub.queues.get(queue_id);
        let result = queue.on_submitted_work_done(closure);
        result.unwrap_or(0) // '0' means no wait is necessary
    }

    pub fn queue_compact_blas(
        &self,
        queue_id: QueueId,
        blas_id: BlasId,
        id_in: Option<BlasId>,
    ) -> (BlasId, Option<u64>, Option<CompactBlasError>) {
        let fid = self.hub.blas_s.prepare(id_in);

        let queue = self.hub.queues.get(queue_id);
        let blas = self.hub.blas_s.get(blas_id);

        let (blas, error) = queue.compact_blas(&blas);

        let handle = blas.handle();
        let id = fid.assign(blas);

        (id, handle, error)
    }
}

fn validate_command_buffer(
    command_buffer: &CommandBuffer,
    queue: &Queue,
    cmd_buf_data: &crate::command::CommandBufferMutable,
    snatch_guard: &SnatchGuard,
    surface_textures: &mut FastHashMap<*const Texture, Arc<Texture>>,
    used_surface_textures: &mut track::TextureUsageScope,
    command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
) -> Result<(), QueueSubmitError> {
    command_buffer.same_device_as(queue)?;

    {
        profiling::scope!("check resource state");

        {
            profiling::scope!("buffers");
            for buffer in cmd_buf_data.trackers.buffers.used_resources() {
                buffer.check_destroyed(snatch_guard)?;

                match *buffer.map_state.lock() {
                    BufferMapState::Idle => (),
                    _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
                }
            }
        }
        {
            profiling::scope!("textures");
            for texture in cmd_buf_data.trackers.textures.used_resources() {
                let should_extend = match texture.try_inner(snatch_guard)? {
                    TextureInner::Native { .. } => false,
                    TextureInner::Surface { .. } => {
                        // Compare the Arcs by pointer as Textures don't implement Eq.
                        surface_textures.insert(Arc::as_ptr(texture), texture.clone());

                        true
                    }
                };
                if should_extend {
                    unsafe {
                        used_surface_textures
                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
                            .unwrap();
                    };
                }
            }
        }
        {
            profiling::scope!("query sets");
            for query_set in cmd_buf_data.trackers.query_sets.used_resources() {
                query_set.try_raw(snatch_guard)?;
            }
        }
        // WebGPU requires that we check every bind group referenced during
        // encoding, even ones that may have been replaced before being used.
        // TODO(<https://github.com/gfx-rs/wgpu/issues/8510>): Optimize this.
        {
            profiling::scope!("bind groups");
            for bind_group in &cmd_buf_data.trackers.bind_groups {
                // This checks the bind group and all resources it references.
                bind_group.try_raw(snatch_guard)?;
            }
        }

        if let Err(e) =
            cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
        {
            return Err(e.into());
        }
    }
    Ok(())
}