wgpu_core/device/
queue.rs

1use alloc::{boxed::Box, string::ToString, sync::Arc, vec, vec::Vec};
2use core::{
3    iter,
4    mem::{self, ManuallyDrop},
5    num::NonZeroU64,
6    ptr::NonNull,
7    sync::atomic::Ordering,
8};
9use smallvec::SmallVec;
10use thiserror::Error;
11use wgt::{
12    error::{ErrorType, WebGpuError},
13    AccelerationStructureFlags,
14};
15
16use super::{life::LifetimeTracker, Device};
17#[cfg(feature = "trace")]
18use crate::device::trace::Action;
19use crate::{
20    api_log,
21    command::{
22        extract_texture_selector, validate_linear_texture_data, validate_texture_buffer_copy,
23        validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
24        CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide,
25        TexelCopyTextureInfo, TransferError,
26    },
27    device::{DeviceError, WaitIdleError},
28    get_lowest_common_denom,
29    global::Global,
30    hal_label,
31    id::{self, BlasId, QueueId},
32    init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
33    lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
34    ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
35    resource::{
36        Blas, BlasCompactState, Buffer, BufferAccessError, BufferMapState, DestroyedBuffer,
37        DestroyedResourceError, DestroyedTexture, Fallible, FlushedStagingBuffer,
38        InvalidResourceError, Labeled, ParentDevice, ResourceErrorIdent, StagingBuffer, Texture,
39        TextureInner, Trackable, TrackingData,
40    },
41    resource_log,
42    scratch::ScratchBuffer,
43    snatch::{SnatchGuard, Snatchable},
44    track::{self, Tracker, TrackerIndex},
45    FastHashMap, SubmissionIndex,
46};
47use crate::{device::resource::CommandIndices, resource::RawResourceAccess};
48
49pub struct Queue {
50    raw: Box<dyn hal::DynQueue>,
51    pub(crate) pending_writes: Mutex<PendingWrites>,
52    life_tracker: Mutex<LifetimeTracker>,
53    // The device needs to be dropped last (`Device.zero_buffer` might be referenced by the encoder in pending writes).
54    pub(crate) device: Arc<Device>,
55}
56
57impl Queue {
58    pub(crate) fn new(
59        device: Arc<Device>,
60        raw: Box<dyn hal::DynQueue>,
61        instance_flags: wgt::InstanceFlags,
62    ) -> Result<Self, DeviceError> {
63        let pending_encoder = device
64            .command_allocator
65            .acquire_encoder(device.raw(), raw.as_ref())
66            .map_err(DeviceError::from_hal);
67
68        let pending_encoder = match pending_encoder {
69            Ok(pending_encoder) => pending_encoder,
70            Err(e) => {
71                return Err(e);
72            }
73        };
74
75        let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);
76
77        let zero_buffer = device.zero_buffer.as_ref();
78        pending_writes.activate();
79        unsafe {
80            pending_writes
81                .command_encoder
82                .transition_buffers(&[hal::BufferBarrier {
83                    buffer: zero_buffer,
84                    usage: hal::StateTransition {
85                        from: wgt::BufferUses::empty(),
86                        to: wgt::BufferUses::COPY_DST,
87                    },
88                }]);
89            pending_writes
90                .command_encoder
91                .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
92            pending_writes
93                .command_encoder
94                .transition_buffers(&[hal::BufferBarrier {
95                    buffer: zero_buffer,
96                    usage: hal::StateTransition {
97                        from: wgt::BufferUses::COPY_DST,
98                        to: wgt::BufferUses::COPY_SRC,
99                    },
100                }]);
101        }
102
103        Ok(Queue {
104            raw,
105            device,
106            pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
107            life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
108        })
109    }
110
111    pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
112        self.raw.as_ref()
113    }
114
115    #[track_caller]
116    pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
117        self.life_tracker.lock()
118    }
119
120    pub(crate) fn maintain(
121        &self,
122        submission_index: u64,
123        snatch_guard: &SnatchGuard,
124    ) -> (
125        SmallVec<[SubmittedWorkDoneClosure; 1]>,
126        Vec<super::BufferMapPendingClosure>,
127        Vec<BlasCompactReadyPendingClosure>,
128        bool,
129    ) {
130        let mut life_tracker = self.lock_life();
131        let submission_closures = life_tracker.triage_submissions(submission_index);
132
133        let mapping_closures = life_tracker.handle_mapping(snatch_guard);
134        let blas_closures = life_tracker.handle_compact_read_back();
135
136        let queue_empty = life_tracker.queue_empty();
137
138        (
139            submission_closures,
140            mapping_closures,
141            blas_closures,
142            queue_empty,
143        )
144    }
145}
146
147crate::impl_resource_type!(Queue);
148// TODO: https://github.com/gfx-rs/wgpu/issues/4014
149impl Labeled for Queue {
150    fn label(&self) -> &str {
151        ""
152    }
153}
154crate::impl_parent_device!(Queue);
155crate::impl_storage_item!(Queue);
156
157impl Drop for Queue {
158    fn drop(&mut self) {
159        resource_log!("Drop {}", self.error_ident());
160
161        let last_successful_submission_index = self
162            .device
163            .last_successful_submission_index
164            .load(Ordering::Acquire);
165
166        let fence = self.device.fence.read();
167
168        // Try waiting on the last submission using the following sequence of timeouts
169        let timeouts_in_ms = [100, 200, 400, 800, 1600, 3200];
170
171        for (i, timeout_ms) in timeouts_in_ms.into_iter().enumerate() {
172            let is_last_iter = i == timeouts_in_ms.len() - 1;
173
174            api_log!(
175                "Waiting on last submission. try: {}/{}. timeout: {}ms",
176                i + 1,
177                timeouts_in_ms.len(),
178                timeout_ms
179            );
180
181            let wait_res = unsafe {
182                self.device.raw().wait(
183                    fence.as_ref(),
184                    last_successful_submission_index,
185                    #[cfg(not(target_arch = "wasm32"))]
186                    timeout_ms,
187                    #[cfg(target_arch = "wasm32")]
188                    0, // WebKit and Chromium don't support a non-0 timeout
189                )
190            };
191            // Note: If we don't panic below we are in UB land (destroying resources while they are still in use by the GPU).
192            match wait_res {
193                Ok(true) => break,
194                Ok(false) => {
195                    // It's fine that we timed out on WebGL; GL objects can be deleted early as they
196                    // will be kept around by the driver if GPU work hasn't finished.
197                    // Moreover, the way we emulate read mappings on WebGL allows us to execute map_buffer earlier than on other
198                    // backends since getBufferSubData is synchronous with respect to the other previously enqueued GL commands.
199                    // Relying on this behavior breaks the clean abstraction wgpu-hal tries to maintain and
200                    // we should find ways to improve this. See https://github.com/gfx-rs/wgpu/issues/6538.
201                    #[cfg(target_arch = "wasm32")]
202                    {
203                        break;
204                    }
205                    #[cfg(not(target_arch = "wasm32"))]
206                    {
207                        if is_last_iter {
208                            panic!(
209                                "We timed out while waiting on the last successful submission to complete!"
210                            );
211                        }
212                    }
213                }
214                Err(e) => match e {
215                    hal::DeviceError::OutOfMemory => {
216                        if is_last_iter {
217                            panic!(
218                                "We ran into an OOM error while waiting on the last successful submission to complete!"
219                            );
220                        }
221                    }
222                    hal::DeviceError::Lost => {
223                        self.device.handle_hal_error(e); // will lose the device
224                        break;
225                    }
226                    hal::DeviceError::Unexpected => {
227                        panic!(
228                            "We ran into an unexpected error while waiting on the last successful submission to complete!"
229                        );
230                    }
231                },
232            }
233        }
234        drop(fence);
235
236        let snatch_guard = self.device.snatchable_lock.read();
237        let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
238            self.maintain(last_successful_submission_index, &snatch_guard);
239        drop(snatch_guard);
240
241        assert!(queue_empty);
242
243        let closures = crate::device::UserClosures {
244            mappings: mapping_closures,
245            blas_compact_ready: blas_compact_ready_closures,
246            submissions: submission_closures,
247            device_lost_invocations: SmallVec::new(),
248        };
249
250        closures.fire();
251    }
252}
253
254#[cfg(send_sync)]
255pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
256#[cfg(not(send_sync))]
257pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
258
259/// A texture or buffer to be freed soon.
260///
261/// This is just a tagged raw texture or buffer, generally about to be added to
262/// some other more specific container like:
263///
264/// - `PendingWrites::temp_resources`: resources used by queue writes and
265///   unmaps, waiting to be folded in with the next queue submission
266///
267/// - `ActiveSubmission::temp_resources`: temporary resources used by a queue
268///   submission, to be freed when it completes
269#[derive(Debug)]
270pub enum TempResource {
271    StagingBuffer(FlushedStagingBuffer),
272    ScratchBuffer(ScratchBuffer),
273    DestroyedBuffer(DestroyedBuffer),
274    DestroyedTexture(DestroyedTexture),
275}
276
277/// A series of raw [`CommandBuffer`]s that have been submitted to a
278/// queue, and the [`wgpu_hal::CommandEncoder`] that built them.
279///
280/// [`CommandBuffer`]: hal::Api::CommandBuffer
281/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
282pub(crate) struct EncoderInFlight {
283    inner: crate::command::InnerCommandEncoder,
284    pub(crate) trackers: Tracker,
285    pub(crate) temp_resources: Vec<TempResource>,
286    /// We only need to keep these resources alive.
287    _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
288
289    /// These are the buffers that have been tracked by `PendingWrites`.
290    pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
291    /// These are the textures that have been tracked by `PendingWrites`.
292    pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
293    /// These are the BLASes that have been tracked by `PendingWrites`.
294    pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
295}
296
297/// A private command encoder for writes made directly on the device
298/// or queue.
299///
300/// Operations like `buffer_unmap`, `queue_write_buffer`, and
301/// `queue_write_texture` need to copy data to the GPU. At the hal
302/// level, this must be done by encoding and submitting commands, but
303/// these operations are not associated with any specific wgpu command
304/// buffer.
305///
306/// Instead, `Device::pending_writes` owns one of these values, which
307/// has its own hal command encoder and resource lists. The commands
308/// accumulated here are automatically submitted to the queue the next
309/// time the user submits a wgpu command buffer, ahead of the user's
310/// commands.
311///
312/// Important:
313/// When locking pending_writes be sure that tracker is not locked
314/// and try to lock trackers for the minimum timespan possible
315///
316/// All uses of [`StagingBuffer`]s end up here.
317#[derive(Debug)]
318pub(crate) struct PendingWrites {
319    // The command encoder needs to be destroyed before any other resource in pending writes.
320    pub command_encoder: Box<dyn hal::DynCommandEncoder>,
321
322    /// True if `command_encoder` is in the "recording" state, as
323    /// described in the docs for the [`wgpu_hal::CommandEncoder`]
324    /// trait.
325    ///
326    /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
327    pub is_recording: bool,
328
329    temp_resources: Vec<TempResource>,
330    dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
331    dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
332    copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
333    instance_flags: wgt::InstanceFlags,
334}
335
336impl PendingWrites {
337    pub fn new(
338        command_encoder: Box<dyn hal::DynCommandEncoder>,
339        instance_flags: wgt::InstanceFlags,
340    ) -> Self {
341        Self {
342            command_encoder,
343            is_recording: false,
344            temp_resources: Vec::new(),
345            dst_buffers: FastHashMap::default(),
346            dst_textures: FastHashMap::default(),
347            copied_blas_s: FastHashMap::default(),
348            instance_flags,
349        }
350    }
351
352    pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
353        self.dst_buffers
354            .insert(buffer.tracker_index(), buffer.clone());
355    }
356
357    pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
358        self.dst_textures
359            .insert(texture.tracker_index(), texture.clone());
360    }
361
362    pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
363        self.copied_blas_s
364            .insert(blas.tracker_index(), blas.clone());
365    }
366
367    pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
368        self.dst_buffers.contains_key(&buffer.tracker_index())
369    }
370
371    pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
372        self.dst_textures.contains_key(&texture.tracker_index())
373    }
374
375    pub fn consume_temp(&mut self, resource: TempResource) {
376        self.temp_resources.push(resource);
377    }
378
379    pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
380        self.temp_resources
381            .push(TempResource::StagingBuffer(buffer));
382    }
383
384    fn pre_submit(
385        &mut self,
386        command_allocator: &CommandAllocator,
387        device: &Arc<Device>,
388        queue: &Queue,
389    ) -> Result<Option<EncoderInFlight>, DeviceError> {
390        if self.is_recording {
391            let pending_buffers = mem::take(&mut self.dst_buffers);
392            let pending_textures = mem::take(&mut self.dst_textures);
393            let pending_blas_s = mem::take(&mut self.copied_blas_s);
394
395            let cmd_buf = unsafe { self.command_encoder.end_encoding() }
396                .map_err(|e| device.handle_hal_error(e))?;
397            self.is_recording = false;
398
399            let new_encoder = command_allocator
400                .acquire_encoder(device.raw(), queue.raw())
401                .map_err(|e| device.handle_hal_error(e))?;
402
403            let encoder = EncoderInFlight {
404                inner: crate::command::InnerCommandEncoder {
405                    raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
406                    list: vec![cmd_buf],
407                    device: device.clone(),
408                    is_open: false,
409                    label: "(wgpu internal) PendingWrites command encoder".into(),
410                },
411                trackers: Tracker::new(),
412                temp_resources: mem::take(&mut self.temp_resources),
413                _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
414                    device.clone(),
415                ),
416                pending_buffers,
417                pending_textures,
418                pending_blas_s,
419            };
420            Ok(Some(encoder))
421        } else {
422            self.dst_buffers.clear();
423            self.dst_textures.clear();
424            self.copied_blas_s.clear();
425            Ok(None)
426        }
427    }
428
429    pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
430        if !self.is_recording {
431            unsafe {
432                self.command_encoder
433                    .begin_encoding(hal_label(
434                        Some("(wgpu internal) PendingWrites"),
435                        self.instance_flags,
436                    ))
437                    .unwrap();
438            }
439            self.is_recording = true;
440        }
441        self.command_encoder.as_mut()
442    }
443}
444
445impl Drop for PendingWrites {
446    fn drop(&mut self) {
447        unsafe {
448            if self.is_recording {
449                self.command_encoder.discard_encoding();
450            }
451        }
452    }
453}
454
455#[derive(Clone, Debug, Error)]
456#[non_exhaustive]
457pub enum QueueWriteError {
458    #[error(transparent)]
459    Queue(#[from] DeviceError),
460    #[error(transparent)]
461    Transfer(#[from] TransferError),
462    #[error(transparent)]
463    MemoryInitFailure(#[from] ClearError),
464    #[error(transparent)]
465    DestroyedResource(#[from] DestroyedResourceError),
466    #[error(transparent)]
467    InvalidResource(#[from] InvalidResourceError),
468}
469
470impl WebGpuError for QueueWriteError {
471    fn webgpu_error_type(&self) -> ErrorType {
472        let e: &dyn WebGpuError = match self {
473            Self::Queue(e) => e,
474            Self::Transfer(e) => e,
475            Self::MemoryInitFailure(e) => e,
476            Self::DestroyedResource(e) => e,
477            Self::InvalidResource(e) => e,
478        };
479        e.webgpu_error_type()
480    }
481}
482
483#[derive(Clone, Debug, Error)]
484#[non_exhaustive]
485pub enum QueueSubmitError {
486    #[error(transparent)]
487    Queue(#[from] DeviceError),
488    #[error(transparent)]
489    DestroyedResource(#[from] DestroyedResourceError),
490    #[error(transparent)]
491    Unmap(#[from] BufferAccessError),
492    #[error("{0} is still mapped")]
493    BufferStillMapped(ResourceErrorIdent),
494    #[error(transparent)]
495    InvalidResource(#[from] InvalidResourceError),
496    #[error(transparent)]
497    CommandEncoder(#[from] CommandEncoderError),
498    #[error(transparent)]
499    ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
500}
501
502impl WebGpuError for QueueSubmitError {
503    fn webgpu_error_type(&self) -> ErrorType {
504        let e: &dyn WebGpuError = match self {
505            Self::Queue(e) => e,
506            Self::Unmap(e) => e,
507            Self::CommandEncoder(e) => e,
508            Self::ValidateAsActionsError(e) => e,
509            Self::InvalidResource(e) => e,
510            Self::DestroyedResource(_) | Self::BufferStillMapped(_) => {
511                return ErrorType::Validation
512            }
513        };
514        e.webgpu_error_type()
515    }
516}
517
518//TODO: move out common parts of write_xxx.
519
520impl Queue {
521    pub fn write_buffer(
522        &self,
523        buffer: Fallible<Buffer>,
524        buffer_offset: wgt::BufferAddress,
525        data: &[u8],
526    ) -> Result<(), QueueWriteError> {
527        profiling::scope!("Queue::write_buffer");
528        api_log!("Queue::write_buffer");
529
530        self.device.check_is_valid()?;
531
532        let buffer = buffer.get()?;
533
534        let data_size = data.len() as wgt::BufferAddress;
535
536        self.same_device_as(buffer.as_ref())?;
537
538        let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
539            data_size
540        } else {
541            log::trace!("Ignoring write_buffer of size 0");
542            return Ok(());
543        };
544
545        // Platform validation requires that the staging buffer always be
546        // freed, even if an error occurs. All paths from here must call
547        // `device.pending_writes.consume`.
548        let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
549
550        let staging_buffer = {
551            profiling::scope!("copy");
552            staging_buffer.write(data);
553            staging_buffer.flush()
554        };
555
556        let snatch_guard = self.device.snatchable_lock.read();
557        let mut pending_writes = self.pending_writes.lock();
558
559        let result = self.write_staging_buffer_impl(
560            &snatch_guard,
561            &mut pending_writes,
562            &staging_buffer,
563            buffer,
564            buffer_offset,
565        );
566
567        drop(snatch_guard);
568
569        pending_writes.consume(staging_buffer);
570
571        drop(pending_writes);
572
573        result
574    }
575
576    pub fn create_staging_buffer(
577        &self,
578        buffer_size: wgt::BufferSize,
579    ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
580        profiling::scope!("Queue::create_staging_buffer");
581        resource_log!("Queue::create_staging_buffer");
582
583        self.device.check_is_valid()?;
584
585        let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
586        let ptr = unsafe { staging_buffer.ptr() };
587
588        Ok((staging_buffer, ptr))
589    }
590
591    pub fn write_staging_buffer(
592        &self,
593        buffer: Fallible<Buffer>,
594        buffer_offset: wgt::BufferAddress,
595        staging_buffer: StagingBuffer,
596    ) -> Result<(), QueueWriteError> {
597        profiling::scope!("Queue::write_staging_buffer");
598
599        self.device.check_is_valid()?;
600
601        let buffer = buffer.get()?;
602
603        // At this point, we have taken ownership of the staging_buffer from the
604        // user. Platform validation requires that the staging buffer always
605        // be freed, even if an error occurs. All paths from here must call
606        // `device.pending_writes.consume`.
607        let staging_buffer = staging_buffer.flush();
608
609        let snatch_guard = self.device.snatchable_lock.read();
610        let mut pending_writes = self.pending_writes.lock();
611
612        let result = self.write_staging_buffer_impl(
613            &snatch_guard,
614            &mut pending_writes,
615            &staging_buffer,
616            buffer,
617            buffer_offset,
618        );
619
620        drop(snatch_guard);
621
622        pending_writes.consume(staging_buffer);
623
624        drop(pending_writes);
625
626        result
627    }
628
629    pub fn validate_write_buffer(
630        &self,
631        buffer: Fallible<Buffer>,
632        buffer_offset: u64,
633        buffer_size: wgt::BufferSize,
634    ) -> Result<(), QueueWriteError> {
635        profiling::scope!("Queue::validate_write_buffer");
636
637        self.device.check_is_valid()?;
638
639        let buffer = buffer.get()?;
640
641        self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size)?;
642
643        Ok(())
644    }
645
646    fn validate_write_buffer_impl(
647        &self,
648        buffer: &Buffer,
649        buffer_offset: u64,
650        buffer_size: wgt::BufferSize,
651    ) -> Result<(), TransferError> {
652        buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
653        if buffer_size.get() % wgt::COPY_BUFFER_ALIGNMENT != 0 {
654            return Err(TransferError::UnalignedCopySize(buffer_size.get()));
655        }
656        if buffer_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
657            return Err(TransferError::UnalignedBufferOffset(buffer_offset));
658        }
659        if buffer_offset + buffer_size.get() > buffer.size {
660            return Err(TransferError::BufferOverrun {
661                start_offset: buffer_offset,
662                end_offset: buffer_offset + buffer_size.get(),
663                buffer_size: buffer.size,
664                side: CopySide::Destination,
665            });
666        }
667
668        Ok(())
669    }
670
671    fn write_staging_buffer_impl(
672        &self,
673        snatch_guard: &SnatchGuard,
674        pending_writes: &mut PendingWrites,
675        staging_buffer: &FlushedStagingBuffer,
676        buffer: Arc<Buffer>,
677        buffer_offset: u64,
678    ) -> Result<(), QueueWriteError> {
679        self.device.check_is_valid()?;
680
681        let transition = {
682            let mut trackers = self.device.trackers.lock();
683            trackers
684                .buffers
685                .set_single(&buffer, wgt::BufferUses::COPY_DST)
686        };
687
688        let dst_raw = buffer.try_raw(snatch_guard)?;
689
690        self.same_device_as(buffer.as_ref())?;
691
692        self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size)?;
693
694        let region = hal::BufferCopy {
695            src_offset: 0,
696            dst_offset: buffer_offset,
697            size: staging_buffer.size,
698        };
699        let barriers = iter::once(hal::BufferBarrier {
700            buffer: staging_buffer.raw(),
701            usage: hal::StateTransition {
702                from: wgt::BufferUses::MAP_WRITE,
703                to: wgt::BufferUses::COPY_SRC,
704            },
705        })
706        .chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard)))
707        .collect::<Vec<_>>();
708        let encoder = pending_writes.activate();
709        unsafe {
710            encoder.transition_buffers(&barriers);
711            encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
712        }
713
714        pending_writes.insert_buffer(&buffer);
715
716        // Ensure the overwritten bytes are marked as initialized so
717        // they don't need to be nulled prior to mapping or binding.
718        {
719            buffer
720                .initialization_status
721                .write()
722                .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
723        }
724
725        Ok(())
726    }
727
728    pub fn write_texture(
729        &self,
730        destination: wgt::TexelCopyTextureInfo<Fallible<Texture>>,
731        data: &[u8],
732        data_layout: &wgt::TexelCopyBufferLayout,
733        size: &wgt::Extent3d,
734    ) -> Result<(), QueueWriteError> {
735        profiling::scope!("Queue::write_texture");
736        api_log!("Queue::write_texture");
737
738        self.device.check_is_valid()?;
739
740        let dst = destination.texture.get()?;
741        let destination = wgt::TexelCopyTextureInfo {
742            texture: (),
743            mip_level: destination.mip_level,
744            origin: destination.origin,
745            aspect: destination.aspect,
746        };
747
748        self.same_device_as(dst.as_ref())?;
749
750        dst.check_usage(wgt::TextureUsages::COPY_DST)
751            .map_err(TransferError::MissingTextureUsage)?;
752
753        // Note: Doing the copy range validation early is important because ensures that the
754        // dimensions are not going to cause overflow in other parts of the validation.
755        let (hal_copy_size, array_layer_count) =
756            validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
757
758        let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?;
759
760        validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;
761
762        validate_texture_buffer_copy(
763            &destination,
764            dst_base.aspect,
765            &dst.desc,
766            data_layout.offset,
767            false, // alignment not required for buffer offset
768        )?;
769
770        // Note: `_source_bytes_per_array_layer` is ignored since we
771        // have a staging copy, and it can have a different value.
772        let (required_bytes_in_copy, _source_bytes_per_array_layer) = validate_linear_texture_data(
773            data_layout,
774            dst.desc.format,
775            destination.aspect,
776            data.len() as wgt::BufferAddress,
777            CopySide::Source,
778            size,
779            false,
780        )?;
781
782        if dst.desc.format.is_depth_stencil_format() {
783            self.device
784                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
785                .map_err(TransferError::from)?;
786        }
787
788        let snatch_guard = self.device.snatchable_lock.read();
789
790        let dst_raw = dst.try_raw(&snatch_guard)?;
791
792        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
793            log::trace!("Ignoring write_texture of size 0");
794            return Ok(());
795        }
796
797        let mut pending_writes = self.pending_writes.lock();
798        let encoder = pending_writes.activate();
799
800        // If the copy does not fully cover the layers, we need to initialize to
801        // zero *first* as we don't keep track of partial texture layer inits.
802        //
803        // Strictly speaking we only need to clear the areas of a layer
804        // untouched, but this would get increasingly messy.
805        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
806            // volume textures don't have a layer range as array volumes aren't supported
807            0..1
808        } else {
809            destination.origin.z..destination.origin.z + size.depth_or_array_layers
810        };
811        let mut dst_initialization_status = dst.initialization_status.write();
812        if dst_initialization_status.mips[destination.mip_level as usize]
813            .check(init_layer_range.clone())
814            .is_some()
815        {
816            if has_copy_partial_init_tracker_coverage(size, destination.mip_level, &dst.desc) {
817                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
818                    .drain(init_layer_range)
819                    .collect::<Vec<core::ops::Range<u32>>>()
820                {
821                    let mut trackers = self.device.trackers.lock();
822                    crate::command::clear_texture(
823                        &dst,
824                        TextureInitRange {
825                            mip_range: destination.mip_level..(destination.mip_level + 1),
826                            layer_range,
827                        },
828                        encoder,
829                        &mut trackers.textures,
830                        &self.device.alignments,
831                        self.device.zero_buffer.as_ref(),
832                        &snatch_guard,
833                        self.device.instance_flags,
834                    )
835                    .map_err(QueueWriteError::from)?;
836                }
837            } else {
838                dst_initialization_status.mips[destination.mip_level as usize]
839                    .drain(init_layer_range);
840            }
841        }
842
843        let (block_width, block_height) = dst.desc.format.block_dimensions();
844        let width_in_blocks = size.width / block_width;
845        let height_in_blocks = size.height / block_height;
846
847        let block_size = dst
848            .desc
849            .format
850            .block_copy_size(Some(destination.aspect))
851            .unwrap();
852        let bytes_in_last_row = width_in_blocks * block_size;
853
854        let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
855        let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
856
857        let bytes_per_row_alignment = get_lowest_common_denom(
858            self.device.alignments.buffer_copy_pitch.get() as u32,
859            block_size,
860        );
861        let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
862
863        // Platform validation requires that the staging buffer always be
864        // freed, even if an error occurs. All paths from here must call
865        // `device.pending_writes.consume`.
866        let staging_buffer = if stage_bytes_per_row == bytes_per_row {
867            profiling::scope!("copy aligned");
868            // Fast path if the data is already being aligned optimally.
869            let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
870            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
871            staging_buffer.write(&data[data_layout.offset as usize..]);
872            staging_buffer
873        } else {
874            profiling::scope!("copy chunked");
875            // Copy row by row into the optimal alignment.
876            let block_rows_in_copy =
877                (size.depth_or_array_layers - 1) * rows_per_image + height_in_blocks;
878            let stage_size =
879                wgt::BufferSize::new(stage_bytes_per_row as u64 * block_rows_in_copy as u64)
880                    .unwrap();
881            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
882            for layer in 0..size.depth_or_array_layers {
883                let rows_offset = layer * rows_per_image;
884                for row in rows_offset..rows_offset + height_in_blocks {
885                    let src_offset = data_layout.offset as u32 + row * bytes_per_row;
886                    let dst_offset = row * stage_bytes_per_row;
887                    unsafe {
888                        staging_buffer.write_with_offset(
889                            data,
890                            src_offset as isize,
891                            dst_offset as isize,
892                            bytes_in_last_row as usize,
893                        )
894                    }
895                }
896            }
897            staging_buffer
898        };
899
900        let staging_buffer = staging_buffer.flush();
901
902        let regions = (0..array_layer_count)
903            .map(|array_layer_offset| {
904                let mut texture_base = dst_base.clone();
905                texture_base.array_layer += array_layer_offset;
906                hal::BufferTextureCopy {
907                    buffer_layout: wgt::TexelCopyBufferLayout {
908                        offset: array_layer_offset as u64
909                            * rows_per_image as u64
910                            * stage_bytes_per_row as u64,
911                        bytes_per_row: Some(stage_bytes_per_row),
912                        rows_per_image: Some(rows_per_image),
913                    },
914                    texture_base,
915                    size: hal_copy_size,
916                }
917            })
918            .collect::<Vec<_>>();
919
920        {
921            let buffer_barrier = hal::BufferBarrier {
922                buffer: staging_buffer.raw(),
923                usage: hal::StateTransition {
924                    from: wgt::BufferUses::MAP_WRITE,
925                    to: wgt::BufferUses::COPY_SRC,
926                },
927            };
928
929            let mut trackers = self.device.trackers.lock();
930            let transition =
931                trackers
932                    .textures
933                    .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
934            let texture_barriers = transition
935                .map(|pending| pending.into_hal(dst_raw))
936                .collect::<Vec<_>>();
937
938            unsafe {
939                encoder.transition_textures(&texture_barriers);
940                encoder.transition_buffers(&[buffer_barrier]);
941                encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, &regions);
942            }
943        }
944
945        pending_writes.consume(staging_buffer);
946        pending_writes.insert_texture(&dst);
947
948        Ok(())
949    }
950
951    #[cfg(webgl)]
952    pub fn copy_external_image_to_texture(
953        &self,
954        source: &wgt::CopyExternalImageSourceInfo,
955        destination: wgt::CopyExternalImageDestInfo<Fallible<Texture>>,
956        size: wgt::Extent3d,
957    ) -> Result<(), QueueWriteError> {
958        use crate::conv;
959
960        profiling::scope!("Queue::copy_external_image_to_texture");
961
962        self.device.check_is_valid()?;
963
964        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
965            log::trace!("Ignoring write_texture of size 0");
966            return Ok(());
967        }
968
969        let mut needs_flag = false;
970        needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
971        needs_flag |= source.origin != wgt::Origin2d::ZERO;
972        needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
973        #[allow(clippy::bool_comparison)]
974        if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
975            needs_flag |= source.flip_y != false;
976            needs_flag |= destination.premultiplied_alpha != false;
977        }
978
979        if needs_flag {
980            self.device
981                .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
982                .map_err(TransferError::from)?;
983        }
984
985        let src_width = source.source.width();
986        let src_height = source.source.height();
987
988        let dst = destination.texture.get()?;
989        let premultiplied_alpha = destination.premultiplied_alpha;
990        let destination = wgt::TexelCopyTextureInfo {
991            texture: (),
992            mip_level: destination.mip_level,
993            origin: destination.origin,
994            aspect: destination.aspect,
995        };
996
997        if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
998            return Err(
999                TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
1000            );
1001        }
1002        if dst.desc.dimension != wgt::TextureDimension::D2 {
1003            return Err(TransferError::InvalidDimensionExternal.into());
1004        }
1005        dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
1006            .map_err(TransferError::MissingTextureUsage)?;
1007        if dst.desc.sample_count != 1 {
1008            return Err(TransferError::InvalidSampleCount {
1009                sample_count: dst.desc.sample_count,
1010            }
1011            .into());
1012        }
1013
1014        if source.origin.x + size.width > src_width {
1015            return Err(TransferError::TextureOverrun {
1016                start_offset: source.origin.x,
1017                end_offset: source.origin.x + size.width,
1018                texture_size: src_width,
1019                dimension: crate::resource::TextureErrorDimension::X,
1020                side: CopySide::Source,
1021            }
1022            .into());
1023        }
1024        if source.origin.y + size.height > src_height {
1025            return Err(TransferError::TextureOverrun {
1026                start_offset: source.origin.y,
1027                end_offset: source.origin.y + size.height,
1028                texture_size: src_height,
1029                dimension: crate::resource::TextureErrorDimension::Y,
1030                side: CopySide::Source,
1031            }
1032            .into());
1033        }
1034        if size.depth_or_array_layers != 1 {
1035            return Err(TransferError::TextureOverrun {
1036                start_offset: 0,
1037                end_offset: size.depth_or_array_layers,
1038                texture_size: 1,
1039                dimension: crate::resource::TextureErrorDimension::Z,
1040                side: CopySide::Source,
1041            }
1042            .into());
1043        }
1044
1045        // Note: Doing the copy range validation early is important because ensures that the
1046        // dimensions are not going to cause overflow in other parts of the validation.
1047        let (hal_copy_size, _) =
1048            validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
1049
1050        let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?;
1051
1052        let mut pending_writes = self.pending_writes.lock();
1053        let encoder = pending_writes.activate();
1054
1055        // If the copy does not fully cover the layers, we need to initialize to
1056        // zero *first* as we don't keep track of partial texture layer inits.
1057        //
1058        // Strictly speaking we only need to clear the areas of a layer
1059        // untouched, but this would get increasingly messy.
1060        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1061            // volume textures don't have a layer range as array volumes aren't supported
1062            0..1
1063        } else {
1064            destination.origin.z..destination.origin.z + size.depth_or_array_layers
1065        };
1066        let mut dst_initialization_status = dst.initialization_status.write();
1067        if dst_initialization_status.mips[destination.mip_level as usize]
1068            .check(init_layer_range.clone())
1069            .is_some()
1070        {
1071            if has_copy_partial_init_tracker_coverage(&size, destination.mip_level, &dst.desc) {
1072                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
1073                    .drain(init_layer_range)
1074                    .collect::<Vec<core::ops::Range<u32>>>()
1075                {
1076                    let mut trackers = self.device.trackers.lock();
1077                    crate::command::clear_texture(
1078                        &dst,
1079                        TextureInitRange {
1080                            mip_range: destination.mip_level..(destination.mip_level + 1),
1081                            layer_range,
1082                        },
1083                        encoder,
1084                        &mut trackers.textures,
1085                        &self.device.alignments,
1086                        self.device.zero_buffer.as_ref(),
1087                        &self.device.snatchable_lock.read(),
1088                        self.device.instance_flags,
1089                    )
1090                    .map_err(QueueWriteError::from)?;
1091                }
1092            } else {
1093                dst_initialization_status.mips[destination.mip_level as usize]
1094                    .drain(init_layer_range);
1095            }
1096        }
1097
1098        let snatch_guard = self.device.snatchable_lock.read();
1099        let dst_raw = dst.try_raw(&snatch_guard)?;
1100
1101        let regions = hal::TextureCopy {
1102            src_base: hal::TextureCopyBase {
1103                mip_level: 0,
1104                array_layer: 0,
1105                origin: source.origin.to_3d(0),
1106                aspect: hal::FormatAspects::COLOR,
1107            },
1108            dst_base,
1109            size: hal_copy_size,
1110        };
1111
1112        let mut trackers = self.device.trackers.lock();
1113        let transitions = trackers
1114            .textures
1115            .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1116
1117        // `copy_external_image_to_texture` is exclusive to the WebGL backend.
1118        // Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend.
1119        let encoder_webgl = encoder
1120            .as_any_mut()
1121            .downcast_mut::<hal::gles::CommandEncoder>()
1122            .unwrap();
1123        let dst_raw_webgl = dst_raw
1124            .as_any()
1125            .downcast_ref::<hal::gles::Texture>()
1126            .unwrap();
1127        let transitions_webgl = transitions.map(|pending| {
1128            let dyn_transition = pending.into_hal(dst_raw);
1129            hal::TextureBarrier {
1130                texture: dst_raw_webgl,
1131                range: dyn_transition.range,
1132                usage: dyn_transition.usage,
1133            }
1134        });
1135
1136        use hal::CommandEncoder as _;
1137        unsafe {
1138            encoder_webgl.transition_textures(transitions_webgl);
1139            encoder_webgl.copy_external_image_to_texture(
1140                source,
1141                dst_raw_webgl,
1142                premultiplied_alpha,
1143                iter::once(regions),
1144            );
1145        }
1146
1147        Ok(())
1148    }
1149
1150    pub fn submit(
1151        &self,
1152        command_buffers: &[Arc<CommandBuffer>],
1153    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1154        profiling::scope!("Queue::submit");
1155        api_log!("Queue::submit");
1156
1157        let submit_index;
1158
1159        let res = 'error: {
1160            let snatch_guard = self.device.snatchable_lock.read();
1161
1162            // Fence lock must be acquired after the snatch lock everywhere to avoid deadlocks.
1163            let mut fence = self.device.fence.write();
1164
1165            let mut command_index_guard = self.device.command_indices.write();
1166            command_index_guard.active_submission_index += 1;
1167            submit_index = command_index_guard.active_submission_index;
1168
1169            if let Err(e) = self.device.check_is_valid() {
1170                break 'error Err(e.into());
1171            }
1172
1173            let mut active_executions = Vec::new();
1174
1175            let mut used_surface_textures = track::TextureUsageScope::default();
1176
1177            // Use a hashmap here to deduplicate the surface textures that are used in the command buffers.
1178            // This avoids vulkan deadlocking from the same surface texture being submitted multiple times.
1179            let mut submit_surface_textures_owned = FastHashMap::default();
1180
1181            {
1182                if !command_buffers.is_empty() {
1183                    profiling::scope!("prepare");
1184
1185                    let mut first_error = None;
1186
1187                    //TODO: if multiple command buffers are submitted, we can re-use the last
1188                    // native command buffer of the previous chain instead of always creating
1189                    // a temporary one, since the chains are not finished.
1190
1191                    // finish all the command buffers first
1192                    for command_buffer in command_buffers {
1193                        profiling::scope!("process command buffer");
1194
1195                        // we reset the used surface textures every time we use
1196                        // it, so make sure to set_size on it.
1197                        used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1198
1199                        // Note that we are required to invalidate all command buffers in both the success and failure paths.
1200                        // This is why we `continue` and don't early return via `?`.
1201                        #[allow(unused_mut)]
1202                        let mut cmd_buf_data = command_buffer.take_finished();
1203
1204                        #[cfg(feature = "trace")]
1205                        if let Some(ref mut trace) = *self.device.trace.lock() {
1206                            if let Ok(ref mut cmd_buf_data) = cmd_buf_data {
1207                                trace.add(Action::Submit(
1208                                    submit_index,
1209                                    cmd_buf_data.commands.take().unwrap(),
1210                                ));
1211                            }
1212                        }
1213
1214                        if first_error.is_some() {
1215                            continue;
1216                        }
1217
1218                        let mut baked = match cmd_buf_data {
1219                            Ok(cmd_buf_data) => {
1220                                let res = validate_command_buffer(
1221                                    command_buffer,
1222                                    self,
1223                                    &cmd_buf_data,
1224                                    &snatch_guard,
1225                                    &mut submit_surface_textures_owned,
1226                                    &mut used_surface_textures,
1227                                    &mut command_index_guard,
1228                                );
1229                                if let Err(err) = res {
1230                                    first_error.get_or_insert(err);
1231                                    continue;
1232                                }
1233                                cmd_buf_data.into_baked_commands()
1234                            }
1235                            Err(err) => {
1236                                first_error.get_or_insert(err.into());
1237                                continue;
1238                            }
1239                        };
1240
1241                        // execute resource transitions
1242                        if let Err(e) = baked.encoder.open_pass(hal_label(
1243                            Some("(wgpu internal) Transit"),
1244                            self.device.instance_flags,
1245                        )) {
1246                            break 'error Err(e.into());
1247                        }
1248
1249                        //Note: locking the trackers has to be done after the storages
1250                        let mut trackers = self.device.trackers.lock();
1251                        if let Err(e) = baked.initialize_buffer_memory(&mut trackers, &snatch_guard)
1252                        {
1253                            break 'error Err(e.into());
1254                        }
1255                        if let Err(e) = baked.initialize_texture_memory(
1256                            &mut trackers,
1257                            &self.device,
1258                            &snatch_guard,
1259                        ) {
1260                            break 'error Err(e.into());
1261                        }
1262
1263                        //Note: stateless trackers are not merged:
1264                        // device already knows these resources exist.
1265                        CommandEncoder::insert_barriers_from_device_tracker(
1266                            baked.encoder.raw.as_mut(),
1267                            &mut trackers,
1268                            &baked.trackers,
1269                            &snatch_guard,
1270                        );
1271
1272                        if let Err(e) = baked.encoder.close_and_push_front() {
1273                            break 'error Err(e.into());
1274                        }
1275
1276                        // Transition surface textures into `Present` state.
1277                        // Note: we could technically do it after all of the command buffers,
1278                        // but here we have a command encoder by hand, so it's easier to use it.
1279                        if !used_surface_textures.is_empty() {
1280                            if let Err(e) = baked.encoder.open_pass(hal_label(
1281                                Some("(wgpu internal) Present"),
1282                                self.device.instance_flags,
1283                            )) {
1284                                break 'error Err(e.into());
1285                            }
1286                            let texture_barriers = trackers
1287                                .textures
1288                                .set_from_usage_scope_and_drain_transitions(
1289                                    &used_surface_textures,
1290                                    &snatch_guard,
1291                                )
1292                                .collect::<Vec<_>>();
1293                            unsafe {
1294                                baked.encoder.raw.transition_textures(&texture_barriers);
1295                            };
1296                            if let Err(e) = baked.encoder.close() {
1297                                break 'error Err(e.into());
1298                            }
1299                            used_surface_textures = track::TextureUsageScope::default();
1300                        }
1301
1302                        // done
1303                        active_executions.push(EncoderInFlight {
1304                            inner: baked.encoder,
1305                            trackers: baked.trackers,
1306                            temp_resources: baked.temp_resources,
1307                            _indirect_draw_validation_resources: baked
1308                                .indirect_draw_validation_resources,
1309                            pending_buffers: FastHashMap::default(),
1310                            pending_textures: FastHashMap::default(),
1311                            pending_blas_s: FastHashMap::default(),
1312                        });
1313                    }
1314
1315                    if let Some(first_error) = first_error {
1316                        break 'error Err(first_error);
1317                    }
1318                }
1319            }
1320
1321            let mut pending_writes = self.pending_writes.lock();
1322
1323            {
1324                used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1325                for texture in pending_writes.dst_textures.values() {
1326                    match texture.try_inner(&snatch_guard) {
1327                        Ok(TextureInner::Native { .. }) => {}
1328                        Ok(TextureInner::Surface { .. }) => {
1329                            // Compare the Arcs by pointer as Textures don't implement Eq
1330                            submit_surface_textures_owned
1331                                .insert(Arc::as_ptr(texture), texture.clone());
1332
1333                            unsafe {
1334                                used_surface_textures
1335                                    .merge_single(texture, None, wgt::TextureUses::PRESENT)
1336                                    .unwrap()
1337                            };
1338                        }
1339                        // The texture must not have been destroyed when its usage here was
1340                        // encoded. If it was destroyed after that, then it was transferred
1341                        // to `pending_writes.temp_resources` at the time of destruction, so
1342                        // we are still okay to use it.
1343                        Err(DestroyedResourceError(_)) => {}
1344                    }
1345                }
1346
1347                if !used_surface_textures.is_empty() {
1348                    let mut trackers = self.device.trackers.lock();
1349
1350                    let texture_barriers = trackers
1351                        .textures
1352                        .set_from_usage_scope_and_drain_transitions(
1353                            &used_surface_textures,
1354                            &snatch_guard,
1355                        )
1356                        .collect::<Vec<_>>();
1357                    unsafe {
1358                        pending_writes
1359                            .command_encoder
1360                            .transition_textures(&texture_barriers);
1361                    };
1362                }
1363            }
1364
1365            match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) {
1366                Ok(Some(pending_execution)) => {
1367                    active_executions.insert(0, pending_execution);
1368                }
1369                Ok(None) => {}
1370                Err(e) => break 'error Err(e.into()),
1371            }
1372            let hal_command_buffers = active_executions
1373                .iter()
1374                .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
1375                .collect::<Vec<_>>();
1376
1377            {
1378                let mut submit_surface_textures =
1379                    SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(
1380                        submit_surface_textures_owned.len(),
1381                    );
1382
1383                for texture in submit_surface_textures_owned.values() {
1384                    let raw = match texture.inner.get(&snatch_guard) {
1385                        Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
1386                        _ => unreachable!(),
1387                    };
1388                    submit_surface_textures.push(raw);
1389                }
1390
1391                if let Err(e) = unsafe {
1392                    self.raw().submit(
1393                        &hal_command_buffers,
1394                        &submit_surface_textures,
1395                        (fence.as_mut(), submit_index),
1396                    )
1397                }
1398                .map_err(|e| self.device.handle_hal_error(e))
1399                {
1400                    break 'error Err(e.into());
1401                }
1402
1403                drop(command_index_guard);
1404
1405                // Advance the successful submission index.
1406                self.device
1407                    .last_successful_submission_index
1408                    .fetch_max(submit_index, Ordering::SeqCst);
1409            }
1410
1411            profiling::scope!("cleanup");
1412
1413            // this will register the new submission to the life time tracker
1414            self.lock_life()
1415                .track_submission(submit_index, active_executions);
1416            drop(pending_writes);
1417
1418            // This will schedule destruction of all resources that are no longer needed
1419            // by the user but used in the command stream, among other things.
1420            let fence_guard = RwLockWriteGuard::downgrade(fence);
1421            let (closures, result) =
1422                self.device
1423                    .maintain(fence_guard, wgt::PollType::Poll, snatch_guard);
1424            match result {
1425                Ok(status) => {
1426                    debug_assert!(matches!(
1427                        status,
1428                        wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1429                    ));
1430                }
1431                Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1432                Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1433                    unreachable!("Cannot get WrongSubmissionIndex from Poll")
1434                }
1435                Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1436            };
1437
1438            Ok(closures)
1439        };
1440
1441        let callbacks = match res {
1442            Ok(ok) => ok,
1443            Err(e) => return Err((submit_index, e)),
1444        };
1445
1446        // the closures should execute with nothing locked!
1447        callbacks.fire();
1448
1449        self.device.lose_if_oom();
1450
1451        api_log!("Queue::submit returned submit index {submit_index}");
1452
1453        Ok(submit_index)
1454    }
1455
1456    pub fn get_timestamp_period(&self) -> f32 {
1457        unsafe { self.raw().get_timestamp_period() }
1458    }
1459
1460    /// `closure` is guaranteed to be called.
1461    pub fn on_submitted_work_done(
1462        &self,
1463        closure: SubmittedWorkDoneClosure,
1464    ) -> Option<SubmissionIndex> {
1465        api_log!("Queue::on_submitted_work_done");
1466        //TODO: flush pending writes
1467        self.lock_life().add_work_done_closure(closure)
1468    }
1469
1470    pub fn compact_blas(&self, blas: &Arc<Blas>) -> Result<Arc<Blas>, CompactBlasError> {
1471        profiling::scope!("Queue::compact_blas");
1472        api_log!("Queue::compact_blas");
1473
1474        let new_label = blas.label.clone() + " (compacted)";
1475
1476        self.device.check_is_valid()?;
1477        self.same_device_as(blas.as_ref())?;
1478
1479        let device = blas.device.clone();
1480
1481        let snatch_guard = device.snatchable_lock.read();
1482
1483        let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
1484            return Err(CompactBlasError::BlasNotReady);
1485        };
1486
1487        let mut size_info = blas.size_info;
1488        size_info.acceleration_structure_size = size;
1489
1490        let mut pending_writes = self.pending_writes.lock();
1491        let cmd_buf_raw = pending_writes.activate();
1492
1493        let raw = unsafe {
1494            device
1495                .raw()
1496                .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
1497                    label: hal_label(Some(&new_label), device.instance_flags),
1498                    size: size_info.acceleration_structure_size,
1499                    format: hal::AccelerationStructureFormat::BottomLevel,
1500                    allow_compaction: false,
1501                })
1502        }
1503        .map_err(DeviceError::from_hal)?;
1504
1505        let src_raw = blas.try_raw(&snatch_guard)?;
1506
1507        unsafe {
1508            cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
1509                src_raw,
1510                raw.as_ref(),
1511                wgt::AccelerationStructureCopy::Compact,
1512            )
1513        };
1514
1515        let handle = unsafe {
1516            device
1517                .raw()
1518                .get_acceleration_structure_device_address(raw.as_ref())
1519        };
1520
1521        drop(snatch_guard);
1522
1523        let mut command_indices_lock = device.command_indices.write();
1524        command_indices_lock.next_acceleration_structure_build_command_index += 1;
1525        let built_index =
1526            NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
1527                .unwrap();
1528
1529        let new_blas = Arc::new(Blas {
1530            raw: Snatchable::new(raw),
1531            device: device.clone(),
1532            size_info,
1533            sizes: blas.sizes.clone(),
1534            flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
1535            update_mode: blas.update_mode,
1536            // Bypass the submit checks which update this because we don't submit this normally.
1537            built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
1538            handle,
1539            label: new_label,
1540            tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
1541            compaction_buffer: None,
1542            compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
1543        });
1544
1545        pending_writes.insert_blas(blas);
1546        pending_writes.insert_blas(&new_blas);
1547
1548        Ok(new_blas)
1549    }
1550}
1551
1552impl Global {
1553    pub fn queue_write_buffer(
1554        &self,
1555        queue_id: QueueId,
1556        buffer_id: id::BufferId,
1557        buffer_offset: wgt::BufferAddress,
1558        data: &[u8],
1559    ) -> Result<(), QueueWriteError> {
1560        let queue = self.hub.queues.get(queue_id);
1561
1562        #[cfg(feature = "trace")]
1563        if let Some(ref mut trace) = *queue.device.trace.lock() {
1564            let data_path = trace.make_binary("bin", data);
1565            trace.add(Action::WriteBuffer {
1566                id: buffer_id,
1567                data: data_path,
1568                range: buffer_offset..buffer_offset + data.len() as u64,
1569                queued: true,
1570            });
1571        }
1572
1573        let buffer = self.hub.buffers.get(buffer_id);
1574        queue.write_buffer(buffer, buffer_offset, data)
1575    }
1576
1577    pub fn queue_create_staging_buffer(
1578        &self,
1579        queue_id: QueueId,
1580        buffer_size: wgt::BufferSize,
1581        id_in: Option<id::StagingBufferId>,
1582    ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
1583        let queue = self.hub.queues.get(queue_id);
1584        let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?;
1585
1586        let fid = self.hub.staging_buffers.prepare(id_in);
1587        let id = fid.assign(staging_buffer);
1588
1589        Ok((id, ptr))
1590    }
1591
1592    pub fn queue_write_staging_buffer(
1593        &self,
1594        queue_id: QueueId,
1595        buffer_id: id::BufferId,
1596        buffer_offset: wgt::BufferAddress,
1597        staging_buffer_id: id::StagingBufferId,
1598    ) -> Result<(), QueueWriteError> {
1599        let queue = self.hub.queues.get(queue_id);
1600        let buffer = self.hub.buffers.get(buffer_id);
1601        let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id);
1602        queue.write_staging_buffer(buffer, buffer_offset, staging_buffer)
1603    }
1604
1605    pub fn queue_validate_write_buffer(
1606        &self,
1607        queue_id: QueueId,
1608        buffer_id: id::BufferId,
1609        buffer_offset: u64,
1610        buffer_size: wgt::BufferSize,
1611    ) -> Result<(), QueueWriteError> {
1612        let queue = self.hub.queues.get(queue_id);
1613        let buffer = self.hub.buffers.get(buffer_id);
1614        queue.validate_write_buffer(buffer, buffer_offset, buffer_size)
1615    }
1616
1617    pub fn queue_write_texture(
1618        &self,
1619        queue_id: QueueId,
1620        destination: &TexelCopyTextureInfo,
1621        data: &[u8],
1622        data_layout: &wgt::TexelCopyBufferLayout,
1623        size: &wgt::Extent3d,
1624    ) -> Result<(), QueueWriteError> {
1625        let queue = self.hub.queues.get(queue_id);
1626
1627        #[cfg(feature = "trace")]
1628        if let Some(ref mut trace) = *queue.device.trace.lock() {
1629            let data_path = trace.make_binary("bin", data);
1630            trace.add(Action::WriteTexture {
1631                to: *destination,
1632                data: data_path,
1633                layout: *data_layout,
1634                size: *size,
1635            });
1636        }
1637
1638        let destination = wgt::TexelCopyTextureInfo {
1639            texture: self.hub.textures.get(destination.texture),
1640            mip_level: destination.mip_level,
1641            origin: destination.origin,
1642            aspect: destination.aspect,
1643        };
1644        queue.write_texture(destination, data, data_layout, size)
1645    }
1646
1647    #[cfg(webgl)]
1648    pub fn queue_copy_external_image_to_texture(
1649        &self,
1650        queue_id: QueueId,
1651        source: &wgt::CopyExternalImageSourceInfo,
1652        destination: crate::command::CopyExternalImageDestInfo,
1653        size: wgt::Extent3d,
1654    ) -> Result<(), QueueWriteError> {
1655        let queue = self.hub.queues.get(queue_id);
1656        let destination = wgt::CopyExternalImageDestInfo {
1657            texture: self.hub.textures.get(destination.texture),
1658            mip_level: destination.mip_level,
1659            origin: destination.origin,
1660            aspect: destination.aspect,
1661            color_space: destination.color_space,
1662            premultiplied_alpha: destination.premultiplied_alpha,
1663        };
1664        queue.copy_external_image_to_texture(source, destination, size)
1665    }
1666
1667    pub fn queue_submit(
1668        &self,
1669        queue_id: QueueId,
1670        command_buffer_ids: &[id::CommandBufferId],
1671    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1672        let queue = self.hub.queues.get(queue_id);
1673        let command_buffer_guard = self.hub.command_buffers.read();
1674        let command_buffers = command_buffer_ids
1675            .iter()
1676            .map(|id| command_buffer_guard.get(*id))
1677            .collect::<Vec<_>>();
1678        drop(command_buffer_guard);
1679        queue.submit(&command_buffers)
1680    }
1681
1682    pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
1683        let queue = self.hub.queues.get(queue_id);
1684
1685        if queue.device.timestamp_normalizer.get().unwrap().enabled() {
1686            return 1.0;
1687        }
1688
1689        queue.get_timestamp_period()
1690    }
1691
1692    pub fn queue_on_submitted_work_done(
1693        &self,
1694        queue_id: QueueId,
1695        closure: SubmittedWorkDoneClosure,
1696    ) -> SubmissionIndex {
1697        api_log!("Queue::on_submitted_work_done {queue_id:?}");
1698
1699        //TODO: flush pending writes
1700        let queue = self.hub.queues.get(queue_id);
1701        let result = queue.on_submitted_work_done(closure);
1702        result.unwrap_or(0) // '0' means no wait is necessary
1703    }
1704
1705    pub fn queue_compact_blas(
1706        &self,
1707        queue_id: QueueId,
1708        blas_id: BlasId,
1709        id_in: Option<BlasId>,
1710    ) -> (BlasId, Option<u64>, Option<CompactBlasError>) {
1711        api_log!("Queue::compact_blas {queue_id:?}, {blas_id:?}");
1712
1713        let fid = self.hub.blas_s.prepare(id_in);
1714
1715        let queue = self.hub.queues.get(queue_id);
1716        let blas = self.hub.blas_s.get(blas_id);
1717        let device = &queue.device;
1718
1719        // TODO: Tracing
1720
1721        let error = 'error: {
1722            match device.require_features(wgpu_types::Features::EXPERIMENTAL_RAY_QUERY) {
1723                Ok(_) => {}
1724                Err(err) => break 'error err.into(),
1725            }
1726
1727            let blas = match blas.get() {
1728                Ok(blas) => blas,
1729                Err(err) => break 'error err.into(),
1730            };
1731
1732            let new_blas = match queue.compact_blas(&blas) {
1733                Ok(blas) => blas,
1734                Err(err) => break 'error err,
1735            };
1736
1737            // We should have no more errors after this because we have marked the command encoder as successful.
1738            let old_blas_size = blas.size_info.acceleration_structure_size;
1739            let new_blas_size = new_blas.size_info.acceleration_structure_size;
1740            let handle = new_blas.handle;
1741
1742            let id = fid.assign(Fallible::Valid(new_blas));
1743
1744            api_log!("CommandEncoder::compact_blas {blas_id:?} (size: {old_blas_size}) -> {id:?} (size: {new_blas_size})");
1745
1746            return (id, Some(handle), None);
1747        };
1748
1749        let id = fid.assign(Fallible::Invalid(Arc::new(error.to_string())));
1750
1751        (id, None, Some(error))
1752    }
1753}
1754
1755fn validate_command_buffer(
1756    command_buffer: &CommandBuffer,
1757    queue: &Queue,
1758    cmd_buf_data: &crate::command::CommandBufferMutable,
1759    snatch_guard: &SnatchGuard,
1760    submit_surface_textures_owned: &mut FastHashMap<*const Texture, Arc<Texture>>,
1761    used_surface_textures: &mut track::TextureUsageScope,
1762    command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
1763) -> Result<(), QueueSubmitError> {
1764    command_buffer.same_device_as(queue)?;
1765
1766    {
1767        profiling::scope!("check resource state");
1768
1769        {
1770            profiling::scope!("buffers");
1771            for buffer in cmd_buf_data.trackers.buffers.used_resources() {
1772                buffer.check_destroyed(snatch_guard)?;
1773
1774                match *buffer.map_state.lock() {
1775                    BufferMapState::Idle => (),
1776                    _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
1777                }
1778            }
1779        }
1780        {
1781            profiling::scope!("textures");
1782            for texture in cmd_buf_data.trackers.textures.used_resources() {
1783                let should_extend = match texture.try_inner(snatch_guard)? {
1784                    TextureInner::Native { .. } => false,
1785                    TextureInner::Surface { .. } => {
1786                        // Compare the Arcs by pointer as Textures don't implement Eq.
1787                        submit_surface_textures_owned.insert(Arc::as_ptr(texture), texture.clone());
1788
1789                        true
1790                    }
1791                };
1792                if should_extend {
1793                    unsafe {
1794                        used_surface_textures
1795                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
1796                            .unwrap();
1797                    };
1798                }
1799            }
1800        }
1801
1802        if let Err(e) =
1803            cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
1804        {
1805            return Err(e.into());
1806        }
1807    }
1808    Ok(())
1809}