1use alloc::{boxed::Box, string::ToString, sync::Arc, vec, vec::Vec};
2use core::{
3 iter,
4 mem::{self, ManuallyDrop},
5 num::NonZeroU64,
6 ptr::NonNull,
7 sync::atomic::Ordering,
8};
9use smallvec::SmallVec;
10use thiserror::Error;
11use wgt::{
12 error::{ErrorType, WebGpuError},
13 AccelerationStructureFlags,
14};
15
16use super::{life::LifetimeTracker, Device};
17#[cfg(feature = "trace")]
18use crate::device::trace::Action;
19use crate::{
20 api_log,
21 command::{
22 extract_texture_selector, validate_linear_texture_data, validate_texture_buffer_copy,
23 validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
24 CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide,
25 TexelCopyTextureInfo, TransferError,
26 },
27 device::{DeviceError, WaitIdleError},
28 get_lowest_common_denom,
29 global::Global,
30 hal_label,
31 id::{self, BlasId, QueueId},
32 init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
33 lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
34 ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
35 resource::{
36 Blas, BlasCompactState, Buffer, BufferAccessError, BufferMapState, DestroyedBuffer,
37 DestroyedResourceError, DestroyedTexture, Fallible, FlushedStagingBuffer,
38 InvalidResourceError, Labeled, ParentDevice, ResourceErrorIdent, StagingBuffer, Texture,
39 TextureInner, Trackable, TrackingData,
40 },
41 resource_log,
42 scratch::ScratchBuffer,
43 snatch::{SnatchGuard, Snatchable},
44 track::{self, Tracker, TrackerIndex},
45 FastHashMap, SubmissionIndex,
46};
47use crate::{device::resource::CommandIndices, resource::RawResourceAccess};
48
49pub struct Queue {
50 raw: Box<dyn hal::DynQueue>,
51 pub(crate) pending_writes: Mutex<PendingWrites>,
52 life_tracker: Mutex<LifetimeTracker>,
53 pub(crate) device: Arc<Device>,
55}
56
57impl Queue {
58 pub(crate) fn new(
59 device: Arc<Device>,
60 raw: Box<dyn hal::DynQueue>,
61 instance_flags: wgt::InstanceFlags,
62 ) -> Result<Self, DeviceError> {
63 let pending_encoder = device
64 .command_allocator
65 .acquire_encoder(device.raw(), raw.as_ref())
66 .map_err(DeviceError::from_hal);
67
68 let pending_encoder = match pending_encoder {
69 Ok(pending_encoder) => pending_encoder,
70 Err(e) => {
71 return Err(e);
72 }
73 };
74
75 let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);
76
77 let zero_buffer = device.zero_buffer.as_ref();
78 pending_writes.activate();
79 unsafe {
80 pending_writes
81 .command_encoder
82 .transition_buffers(&[hal::BufferBarrier {
83 buffer: zero_buffer,
84 usage: hal::StateTransition {
85 from: wgt::BufferUses::empty(),
86 to: wgt::BufferUses::COPY_DST,
87 },
88 }]);
89 pending_writes
90 .command_encoder
91 .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
92 pending_writes
93 .command_encoder
94 .transition_buffers(&[hal::BufferBarrier {
95 buffer: zero_buffer,
96 usage: hal::StateTransition {
97 from: wgt::BufferUses::COPY_DST,
98 to: wgt::BufferUses::COPY_SRC,
99 },
100 }]);
101 }
102
103 Ok(Queue {
104 raw,
105 device,
106 pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
107 life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
108 })
109 }
110
111 pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
112 self.raw.as_ref()
113 }
114
115 #[track_caller]
116 pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
117 self.life_tracker.lock()
118 }
119
120 pub(crate) fn maintain(
121 &self,
122 submission_index: u64,
123 snatch_guard: &SnatchGuard,
124 ) -> (
125 SmallVec<[SubmittedWorkDoneClosure; 1]>,
126 Vec<super::BufferMapPendingClosure>,
127 Vec<BlasCompactReadyPendingClosure>,
128 bool,
129 ) {
130 let mut life_tracker = self.lock_life();
131 let submission_closures = life_tracker.triage_submissions(submission_index);
132
133 let mapping_closures = life_tracker.handle_mapping(snatch_guard);
134 let blas_closures = life_tracker.handle_compact_read_back();
135
136 let queue_empty = life_tracker.queue_empty();
137
138 (
139 submission_closures,
140 mapping_closures,
141 blas_closures,
142 queue_empty,
143 )
144 }
145}
146
147crate::impl_resource_type!(Queue);
148impl Labeled for Queue {
150 fn label(&self) -> &str {
151 ""
152 }
153}
154crate::impl_parent_device!(Queue);
155crate::impl_storage_item!(Queue);
156
157impl Drop for Queue {
158 fn drop(&mut self) {
159 resource_log!("Drop {}", self.error_ident());
160
161 let last_successful_submission_index = self
162 .device
163 .last_successful_submission_index
164 .load(Ordering::Acquire);
165
166 let fence = self.device.fence.read();
167
168 let timeouts_in_ms = [100, 200, 400, 800, 1600, 3200];
170
171 for (i, timeout_ms) in timeouts_in_ms.into_iter().enumerate() {
172 let is_last_iter = i == timeouts_in_ms.len() - 1;
173
174 api_log!(
175 "Waiting on last submission. try: {}/{}. timeout: {}ms",
176 i + 1,
177 timeouts_in_ms.len(),
178 timeout_ms
179 );
180
181 let wait_res = unsafe {
182 self.device.raw().wait(
183 fence.as_ref(),
184 last_successful_submission_index,
185 #[cfg(not(target_arch = "wasm32"))]
186 timeout_ms,
187 #[cfg(target_arch = "wasm32")]
188 0, )
190 };
191 match wait_res {
193 Ok(true) => break,
194 Ok(false) => {
195 #[cfg(target_arch = "wasm32")]
202 {
203 break;
204 }
205 #[cfg(not(target_arch = "wasm32"))]
206 {
207 if is_last_iter {
208 panic!(
209 "We timed out while waiting on the last successful submission to complete!"
210 );
211 }
212 }
213 }
214 Err(e) => match e {
215 hal::DeviceError::OutOfMemory => {
216 if is_last_iter {
217 panic!(
218 "We ran into an OOM error while waiting on the last successful submission to complete!"
219 );
220 }
221 }
222 hal::DeviceError::Lost => {
223 self.device.handle_hal_error(e); break;
225 }
226 hal::DeviceError::Unexpected => {
227 panic!(
228 "We ran into an unexpected error while waiting on the last successful submission to complete!"
229 );
230 }
231 },
232 }
233 }
234 drop(fence);
235
236 let snatch_guard = self.device.snatchable_lock.read();
237 let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
238 self.maintain(last_successful_submission_index, &snatch_guard);
239 drop(snatch_guard);
240
241 assert!(queue_empty);
242
243 let closures = crate::device::UserClosures {
244 mappings: mapping_closures,
245 blas_compact_ready: blas_compact_ready_closures,
246 submissions: submission_closures,
247 device_lost_invocations: SmallVec::new(),
248 };
249
250 closures.fire();
251 }
252}
253
254#[cfg(send_sync)]
255pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
256#[cfg(not(send_sync))]
257pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
258
259#[derive(Debug)]
270pub enum TempResource {
271 StagingBuffer(FlushedStagingBuffer),
272 ScratchBuffer(ScratchBuffer),
273 DestroyedBuffer(DestroyedBuffer),
274 DestroyedTexture(DestroyedTexture),
275}
276
277pub(crate) struct EncoderInFlight {
283 inner: crate::command::InnerCommandEncoder,
284 pub(crate) trackers: Tracker,
285 pub(crate) temp_resources: Vec<TempResource>,
286 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
288
289 pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
291 pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
293 pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
295}
296
297#[derive(Debug)]
318pub(crate) struct PendingWrites {
319 pub command_encoder: Box<dyn hal::DynCommandEncoder>,
321
322 pub is_recording: bool,
328
329 temp_resources: Vec<TempResource>,
330 dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
331 dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
332 copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
333 instance_flags: wgt::InstanceFlags,
334}
335
336impl PendingWrites {
337 pub fn new(
338 command_encoder: Box<dyn hal::DynCommandEncoder>,
339 instance_flags: wgt::InstanceFlags,
340 ) -> Self {
341 Self {
342 command_encoder,
343 is_recording: false,
344 temp_resources: Vec::new(),
345 dst_buffers: FastHashMap::default(),
346 dst_textures: FastHashMap::default(),
347 copied_blas_s: FastHashMap::default(),
348 instance_flags,
349 }
350 }
351
352 pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
353 self.dst_buffers
354 .insert(buffer.tracker_index(), buffer.clone());
355 }
356
357 pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
358 self.dst_textures
359 .insert(texture.tracker_index(), texture.clone());
360 }
361
362 pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
363 self.copied_blas_s
364 .insert(blas.tracker_index(), blas.clone());
365 }
366
367 pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
368 self.dst_buffers.contains_key(&buffer.tracker_index())
369 }
370
371 pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
372 self.dst_textures.contains_key(&texture.tracker_index())
373 }
374
375 pub fn consume_temp(&mut self, resource: TempResource) {
376 self.temp_resources.push(resource);
377 }
378
379 pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
380 self.temp_resources
381 .push(TempResource::StagingBuffer(buffer));
382 }
383
384 fn pre_submit(
385 &mut self,
386 command_allocator: &CommandAllocator,
387 device: &Arc<Device>,
388 queue: &Queue,
389 ) -> Result<Option<EncoderInFlight>, DeviceError> {
390 if self.is_recording {
391 let pending_buffers = mem::take(&mut self.dst_buffers);
392 let pending_textures = mem::take(&mut self.dst_textures);
393 let pending_blas_s = mem::take(&mut self.copied_blas_s);
394
395 let cmd_buf = unsafe { self.command_encoder.end_encoding() }
396 .map_err(|e| device.handle_hal_error(e))?;
397 self.is_recording = false;
398
399 let new_encoder = command_allocator
400 .acquire_encoder(device.raw(), queue.raw())
401 .map_err(|e| device.handle_hal_error(e))?;
402
403 let encoder = EncoderInFlight {
404 inner: crate::command::InnerCommandEncoder {
405 raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
406 list: vec![cmd_buf],
407 device: device.clone(),
408 is_open: false,
409 label: "(wgpu internal) PendingWrites command encoder".into(),
410 },
411 trackers: Tracker::new(),
412 temp_resources: mem::take(&mut self.temp_resources),
413 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
414 device.clone(),
415 ),
416 pending_buffers,
417 pending_textures,
418 pending_blas_s,
419 };
420 Ok(Some(encoder))
421 } else {
422 self.dst_buffers.clear();
423 self.dst_textures.clear();
424 self.copied_blas_s.clear();
425 Ok(None)
426 }
427 }
428
429 pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
430 if !self.is_recording {
431 unsafe {
432 self.command_encoder
433 .begin_encoding(hal_label(
434 Some("(wgpu internal) PendingWrites"),
435 self.instance_flags,
436 ))
437 .unwrap();
438 }
439 self.is_recording = true;
440 }
441 self.command_encoder.as_mut()
442 }
443}
444
445impl Drop for PendingWrites {
446 fn drop(&mut self) {
447 unsafe {
448 if self.is_recording {
449 self.command_encoder.discard_encoding();
450 }
451 }
452 }
453}
454
455#[derive(Clone, Debug, Error)]
456#[non_exhaustive]
457pub enum QueueWriteError {
458 #[error(transparent)]
459 Queue(#[from] DeviceError),
460 #[error(transparent)]
461 Transfer(#[from] TransferError),
462 #[error(transparent)]
463 MemoryInitFailure(#[from] ClearError),
464 #[error(transparent)]
465 DestroyedResource(#[from] DestroyedResourceError),
466 #[error(transparent)]
467 InvalidResource(#[from] InvalidResourceError),
468}
469
470impl WebGpuError for QueueWriteError {
471 fn webgpu_error_type(&self) -> ErrorType {
472 let e: &dyn WebGpuError = match self {
473 Self::Queue(e) => e,
474 Self::Transfer(e) => e,
475 Self::MemoryInitFailure(e) => e,
476 Self::DestroyedResource(e) => e,
477 Self::InvalidResource(e) => e,
478 };
479 e.webgpu_error_type()
480 }
481}
482
483#[derive(Clone, Debug, Error)]
484#[non_exhaustive]
485pub enum QueueSubmitError {
486 #[error(transparent)]
487 Queue(#[from] DeviceError),
488 #[error(transparent)]
489 DestroyedResource(#[from] DestroyedResourceError),
490 #[error(transparent)]
491 Unmap(#[from] BufferAccessError),
492 #[error("{0} is still mapped")]
493 BufferStillMapped(ResourceErrorIdent),
494 #[error(transparent)]
495 InvalidResource(#[from] InvalidResourceError),
496 #[error(transparent)]
497 CommandEncoder(#[from] CommandEncoderError),
498 #[error(transparent)]
499 ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
500}
501
502impl WebGpuError for QueueSubmitError {
503 fn webgpu_error_type(&self) -> ErrorType {
504 let e: &dyn WebGpuError = match self {
505 Self::Queue(e) => e,
506 Self::Unmap(e) => e,
507 Self::CommandEncoder(e) => e,
508 Self::ValidateAsActionsError(e) => e,
509 Self::InvalidResource(e) => e,
510 Self::DestroyedResource(_) | Self::BufferStillMapped(_) => {
511 return ErrorType::Validation
512 }
513 };
514 e.webgpu_error_type()
515 }
516}
517
518impl Queue {
521 pub fn write_buffer(
522 &self,
523 buffer: Fallible<Buffer>,
524 buffer_offset: wgt::BufferAddress,
525 data: &[u8],
526 ) -> Result<(), QueueWriteError> {
527 profiling::scope!("Queue::write_buffer");
528 api_log!("Queue::write_buffer");
529
530 self.device.check_is_valid()?;
531
532 let buffer = buffer.get()?;
533
534 let data_size = data.len() as wgt::BufferAddress;
535
536 self.same_device_as(buffer.as_ref())?;
537
538 let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
539 data_size
540 } else {
541 log::trace!("Ignoring write_buffer of size 0");
542 return Ok(());
543 };
544
545 let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
549
550 let staging_buffer = {
551 profiling::scope!("copy");
552 staging_buffer.write(data);
553 staging_buffer.flush()
554 };
555
556 let snatch_guard = self.device.snatchable_lock.read();
557 let mut pending_writes = self.pending_writes.lock();
558
559 let result = self.write_staging_buffer_impl(
560 &snatch_guard,
561 &mut pending_writes,
562 &staging_buffer,
563 buffer,
564 buffer_offset,
565 );
566
567 drop(snatch_guard);
568
569 pending_writes.consume(staging_buffer);
570
571 drop(pending_writes);
572
573 result
574 }
575
576 pub fn create_staging_buffer(
577 &self,
578 buffer_size: wgt::BufferSize,
579 ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
580 profiling::scope!("Queue::create_staging_buffer");
581 resource_log!("Queue::create_staging_buffer");
582
583 self.device.check_is_valid()?;
584
585 let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
586 let ptr = unsafe { staging_buffer.ptr() };
587
588 Ok((staging_buffer, ptr))
589 }
590
591 pub fn write_staging_buffer(
592 &self,
593 buffer: Fallible<Buffer>,
594 buffer_offset: wgt::BufferAddress,
595 staging_buffer: StagingBuffer,
596 ) -> Result<(), QueueWriteError> {
597 profiling::scope!("Queue::write_staging_buffer");
598
599 self.device.check_is_valid()?;
600
601 let buffer = buffer.get()?;
602
603 let staging_buffer = staging_buffer.flush();
608
609 let snatch_guard = self.device.snatchable_lock.read();
610 let mut pending_writes = self.pending_writes.lock();
611
612 let result = self.write_staging_buffer_impl(
613 &snatch_guard,
614 &mut pending_writes,
615 &staging_buffer,
616 buffer,
617 buffer_offset,
618 );
619
620 drop(snatch_guard);
621
622 pending_writes.consume(staging_buffer);
623
624 drop(pending_writes);
625
626 result
627 }
628
629 pub fn validate_write_buffer(
630 &self,
631 buffer: Fallible<Buffer>,
632 buffer_offset: u64,
633 buffer_size: wgt::BufferSize,
634 ) -> Result<(), QueueWriteError> {
635 profiling::scope!("Queue::validate_write_buffer");
636
637 self.device.check_is_valid()?;
638
639 let buffer = buffer.get()?;
640
641 self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size)?;
642
643 Ok(())
644 }
645
646 fn validate_write_buffer_impl(
647 &self,
648 buffer: &Buffer,
649 buffer_offset: u64,
650 buffer_size: wgt::BufferSize,
651 ) -> Result<(), TransferError> {
652 buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
653 if buffer_size.get() % wgt::COPY_BUFFER_ALIGNMENT != 0 {
654 return Err(TransferError::UnalignedCopySize(buffer_size.get()));
655 }
656 if buffer_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
657 return Err(TransferError::UnalignedBufferOffset(buffer_offset));
658 }
659 if buffer_offset + buffer_size.get() > buffer.size {
660 return Err(TransferError::BufferOverrun {
661 start_offset: buffer_offset,
662 end_offset: buffer_offset + buffer_size.get(),
663 buffer_size: buffer.size,
664 side: CopySide::Destination,
665 });
666 }
667
668 Ok(())
669 }
670
671 fn write_staging_buffer_impl(
672 &self,
673 snatch_guard: &SnatchGuard,
674 pending_writes: &mut PendingWrites,
675 staging_buffer: &FlushedStagingBuffer,
676 buffer: Arc<Buffer>,
677 buffer_offset: u64,
678 ) -> Result<(), QueueWriteError> {
679 self.device.check_is_valid()?;
680
681 let transition = {
682 let mut trackers = self.device.trackers.lock();
683 trackers
684 .buffers
685 .set_single(&buffer, wgt::BufferUses::COPY_DST)
686 };
687
688 let dst_raw = buffer.try_raw(snatch_guard)?;
689
690 self.same_device_as(buffer.as_ref())?;
691
692 self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size)?;
693
694 let region = hal::BufferCopy {
695 src_offset: 0,
696 dst_offset: buffer_offset,
697 size: staging_buffer.size,
698 };
699 let barriers = iter::once(hal::BufferBarrier {
700 buffer: staging_buffer.raw(),
701 usage: hal::StateTransition {
702 from: wgt::BufferUses::MAP_WRITE,
703 to: wgt::BufferUses::COPY_SRC,
704 },
705 })
706 .chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard)))
707 .collect::<Vec<_>>();
708 let encoder = pending_writes.activate();
709 unsafe {
710 encoder.transition_buffers(&barriers);
711 encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
712 }
713
714 pending_writes.insert_buffer(&buffer);
715
716 {
719 buffer
720 .initialization_status
721 .write()
722 .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
723 }
724
725 Ok(())
726 }
727
728 pub fn write_texture(
729 &self,
730 destination: wgt::TexelCopyTextureInfo<Fallible<Texture>>,
731 data: &[u8],
732 data_layout: &wgt::TexelCopyBufferLayout,
733 size: &wgt::Extent3d,
734 ) -> Result<(), QueueWriteError> {
735 profiling::scope!("Queue::write_texture");
736 api_log!("Queue::write_texture");
737
738 self.device.check_is_valid()?;
739
740 let dst = destination.texture.get()?;
741 let destination = wgt::TexelCopyTextureInfo {
742 texture: (),
743 mip_level: destination.mip_level,
744 origin: destination.origin,
745 aspect: destination.aspect,
746 };
747
748 self.same_device_as(dst.as_ref())?;
749
750 dst.check_usage(wgt::TextureUsages::COPY_DST)
751 .map_err(TransferError::MissingTextureUsage)?;
752
753 let (hal_copy_size, array_layer_count) =
756 validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
757
758 let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?;
759
760 validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;
761
762 validate_texture_buffer_copy(
763 &destination,
764 dst_base.aspect,
765 &dst.desc,
766 data_layout.offset,
767 false, )?;
769
770 let (required_bytes_in_copy, _source_bytes_per_array_layer) = validate_linear_texture_data(
773 data_layout,
774 dst.desc.format,
775 destination.aspect,
776 data.len() as wgt::BufferAddress,
777 CopySide::Source,
778 size,
779 false,
780 )?;
781
782 if dst.desc.format.is_depth_stencil_format() {
783 self.device
784 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
785 .map_err(TransferError::from)?;
786 }
787
788 let snatch_guard = self.device.snatchable_lock.read();
789
790 let dst_raw = dst.try_raw(&snatch_guard)?;
791
792 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
793 log::trace!("Ignoring write_texture of size 0");
794 return Ok(());
795 }
796
797 let mut pending_writes = self.pending_writes.lock();
798 let encoder = pending_writes.activate();
799
800 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
806 0..1
808 } else {
809 destination.origin.z..destination.origin.z + size.depth_or_array_layers
810 };
811 let mut dst_initialization_status = dst.initialization_status.write();
812 if dst_initialization_status.mips[destination.mip_level as usize]
813 .check(init_layer_range.clone())
814 .is_some()
815 {
816 if has_copy_partial_init_tracker_coverage(size, destination.mip_level, &dst.desc) {
817 for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
818 .drain(init_layer_range)
819 .collect::<Vec<core::ops::Range<u32>>>()
820 {
821 let mut trackers = self.device.trackers.lock();
822 crate::command::clear_texture(
823 &dst,
824 TextureInitRange {
825 mip_range: destination.mip_level..(destination.mip_level + 1),
826 layer_range,
827 },
828 encoder,
829 &mut trackers.textures,
830 &self.device.alignments,
831 self.device.zero_buffer.as_ref(),
832 &snatch_guard,
833 self.device.instance_flags,
834 )
835 .map_err(QueueWriteError::from)?;
836 }
837 } else {
838 dst_initialization_status.mips[destination.mip_level as usize]
839 .drain(init_layer_range);
840 }
841 }
842
843 let (block_width, block_height) = dst.desc.format.block_dimensions();
844 let width_in_blocks = size.width / block_width;
845 let height_in_blocks = size.height / block_height;
846
847 let block_size = dst
848 .desc
849 .format
850 .block_copy_size(Some(destination.aspect))
851 .unwrap();
852 let bytes_in_last_row = width_in_blocks * block_size;
853
854 let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
855 let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
856
857 let bytes_per_row_alignment = get_lowest_common_denom(
858 self.device.alignments.buffer_copy_pitch.get() as u32,
859 block_size,
860 );
861 let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
862
863 let staging_buffer = if stage_bytes_per_row == bytes_per_row {
867 profiling::scope!("copy aligned");
868 let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
870 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
871 staging_buffer.write(&data[data_layout.offset as usize..]);
872 staging_buffer
873 } else {
874 profiling::scope!("copy chunked");
875 let block_rows_in_copy =
877 (size.depth_or_array_layers - 1) * rows_per_image + height_in_blocks;
878 let stage_size =
879 wgt::BufferSize::new(stage_bytes_per_row as u64 * block_rows_in_copy as u64)
880 .unwrap();
881 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
882 for layer in 0..size.depth_or_array_layers {
883 let rows_offset = layer * rows_per_image;
884 for row in rows_offset..rows_offset + height_in_blocks {
885 let src_offset = data_layout.offset as u32 + row * bytes_per_row;
886 let dst_offset = row * stage_bytes_per_row;
887 unsafe {
888 staging_buffer.write_with_offset(
889 data,
890 src_offset as isize,
891 dst_offset as isize,
892 bytes_in_last_row as usize,
893 )
894 }
895 }
896 }
897 staging_buffer
898 };
899
900 let staging_buffer = staging_buffer.flush();
901
902 let regions = (0..array_layer_count)
903 .map(|array_layer_offset| {
904 let mut texture_base = dst_base.clone();
905 texture_base.array_layer += array_layer_offset;
906 hal::BufferTextureCopy {
907 buffer_layout: wgt::TexelCopyBufferLayout {
908 offset: array_layer_offset as u64
909 * rows_per_image as u64
910 * stage_bytes_per_row as u64,
911 bytes_per_row: Some(stage_bytes_per_row),
912 rows_per_image: Some(rows_per_image),
913 },
914 texture_base,
915 size: hal_copy_size,
916 }
917 })
918 .collect::<Vec<_>>();
919
920 {
921 let buffer_barrier = hal::BufferBarrier {
922 buffer: staging_buffer.raw(),
923 usage: hal::StateTransition {
924 from: wgt::BufferUses::MAP_WRITE,
925 to: wgt::BufferUses::COPY_SRC,
926 },
927 };
928
929 let mut trackers = self.device.trackers.lock();
930 let transition =
931 trackers
932 .textures
933 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
934 let texture_barriers = transition
935 .map(|pending| pending.into_hal(dst_raw))
936 .collect::<Vec<_>>();
937
938 unsafe {
939 encoder.transition_textures(&texture_barriers);
940 encoder.transition_buffers(&[buffer_barrier]);
941 encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, ®ions);
942 }
943 }
944
945 pending_writes.consume(staging_buffer);
946 pending_writes.insert_texture(&dst);
947
948 Ok(())
949 }
950
951 #[cfg(webgl)]
952 pub fn copy_external_image_to_texture(
953 &self,
954 source: &wgt::CopyExternalImageSourceInfo,
955 destination: wgt::CopyExternalImageDestInfo<Fallible<Texture>>,
956 size: wgt::Extent3d,
957 ) -> Result<(), QueueWriteError> {
958 use crate::conv;
959
960 profiling::scope!("Queue::copy_external_image_to_texture");
961
962 self.device.check_is_valid()?;
963
964 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
965 log::trace!("Ignoring write_texture of size 0");
966 return Ok(());
967 }
968
969 let mut needs_flag = false;
970 needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
971 needs_flag |= source.origin != wgt::Origin2d::ZERO;
972 needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
973 #[allow(clippy::bool_comparison)]
974 if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
975 needs_flag |= source.flip_y != false;
976 needs_flag |= destination.premultiplied_alpha != false;
977 }
978
979 if needs_flag {
980 self.device
981 .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
982 .map_err(TransferError::from)?;
983 }
984
985 let src_width = source.source.width();
986 let src_height = source.source.height();
987
988 let dst = destination.texture.get()?;
989 let premultiplied_alpha = destination.premultiplied_alpha;
990 let destination = wgt::TexelCopyTextureInfo {
991 texture: (),
992 mip_level: destination.mip_level,
993 origin: destination.origin,
994 aspect: destination.aspect,
995 };
996
997 if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
998 return Err(
999 TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
1000 );
1001 }
1002 if dst.desc.dimension != wgt::TextureDimension::D2 {
1003 return Err(TransferError::InvalidDimensionExternal.into());
1004 }
1005 dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
1006 .map_err(TransferError::MissingTextureUsage)?;
1007 if dst.desc.sample_count != 1 {
1008 return Err(TransferError::InvalidSampleCount {
1009 sample_count: dst.desc.sample_count,
1010 }
1011 .into());
1012 }
1013
1014 if source.origin.x + size.width > src_width {
1015 return Err(TransferError::TextureOverrun {
1016 start_offset: source.origin.x,
1017 end_offset: source.origin.x + size.width,
1018 texture_size: src_width,
1019 dimension: crate::resource::TextureErrorDimension::X,
1020 side: CopySide::Source,
1021 }
1022 .into());
1023 }
1024 if source.origin.y + size.height > src_height {
1025 return Err(TransferError::TextureOverrun {
1026 start_offset: source.origin.y,
1027 end_offset: source.origin.y + size.height,
1028 texture_size: src_height,
1029 dimension: crate::resource::TextureErrorDimension::Y,
1030 side: CopySide::Source,
1031 }
1032 .into());
1033 }
1034 if size.depth_or_array_layers != 1 {
1035 return Err(TransferError::TextureOverrun {
1036 start_offset: 0,
1037 end_offset: size.depth_or_array_layers,
1038 texture_size: 1,
1039 dimension: crate::resource::TextureErrorDimension::Z,
1040 side: CopySide::Source,
1041 }
1042 .into());
1043 }
1044
1045 let (hal_copy_size, _) =
1048 validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
1049
1050 let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?;
1051
1052 let mut pending_writes = self.pending_writes.lock();
1053 let encoder = pending_writes.activate();
1054
1055 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1061 0..1
1063 } else {
1064 destination.origin.z..destination.origin.z + size.depth_or_array_layers
1065 };
1066 let mut dst_initialization_status = dst.initialization_status.write();
1067 if dst_initialization_status.mips[destination.mip_level as usize]
1068 .check(init_layer_range.clone())
1069 .is_some()
1070 {
1071 if has_copy_partial_init_tracker_coverage(&size, destination.mip_level, &dst.desc) {
1072 for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
1073 .drain(init_layer_range)
1074 .collect::<Vec<core::ops::Range<u32>>>()
1075 {
1076 let mut trackers = self.device.trackers.lock();
1077 crate::command::clear_texture(
1078 &dst,
1079 TextureInitRange {
1080 mip_range: destination.mip_level..(destination.mip_level + 1),
1081 layer_range,
1082 },
1083 encoder,
1084 &mut trackers.textures,
1085 &self.device.alignments,
1086 self.device.zero_buffer.as_ref(),
1087 &self.device.snatchable_lock.read(),
1088 self.device.instance_flags,
1089 )
1090 .map_err(QueueWriteError::from)?;
1091 }
1092 } else {
1093 dst_initialization_status.mips[destination.mip_level as usize]
1094 .drain(init_layer_range);
1095 }
1096 }
1097
1098 let snatch_guard = self.device.snatchable_lock.read();
1099 let dst_raw = dst.try_raw(&snatch_guard)?;
1100
1101 let regions = hal::TextureCopy {
1102 src_base: hal::TextureCopyBase {
1103 mip_level: 0,
1104 array_layer: 0,
1105 origin: source.origin.to_3d(0),
1106 aspect: hal::FormatAspects::COLOR,
1107 },
1108 dst_base,
1109 size: hal_copy_size,
1110 };
1111
1112 let mut trackers = self.device.trackers.lock();
1113 let transitions = trackers
1114 .textures
1115 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1116
1117 let encoder_webgl = encoder
1120 .as_any_mut()
1121 .downcast_mut::<hal::gles::CommandEncoder>()
1122 .unwrap();
1123 let dst_raw_webgl = dst_raw
1124 .as_any()
1125 .downcast_ref::<hal::gles::Texture>()
1126 .unwrap();
1127 let transitions_webgl = transitions.map(|pending| {
1128 let dyn_transition = pending.into_hal(dst_raw);
1129 hal::TextureBarrier {
1130 texture: dst_raw_webgl,
1131 range: dyn_transition.range,
1132 usage: dyn_transition.usage,
1133 }
1134 });
1135
1136 use hal::CommandEncoder as _;
1137 unsafe {
1138 encoder_webgl.transition_textures(transitions_webgl);
1139 encoder_webgl.copy_external_image_to_texture(
1140 source,
1141 dst_raw_webgl,
1142 premultiplied_alpha,
1143 iter::once(regions),
1144 );
1145 }
1146
1147 Ok(())
1148 }
1149
1150 pub fn submit(
1151 &self,
1152 command_buffers: &[Arc<CommandBuffer>],
1153 ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1154 profiling::scope!("Queue::submit");
1155 api_log!("Queue::submit");
1156
1157 let submit_index;
1158
1159 let res = 'error: {
1160 let snatch_guard = self.device.snatchable_lock.read();
1161
1162 let mut fence = self.device.fence.write();
1164
1165 let mut command_index_guard = self.device.command_indices.write();
1166 command_index_guard.active_submission_index += 1;
1167 submit_index = command_index_guard.active_submission_index;
1168
1169 if let Err(e) = self.device.check_is_valid() {
1170 break 'error Err(e.into());
1171 }
1172
1173 let mut active_executions = Vec::new();
1174
1175 let mut used_surface_textures = track::TextureUsageScope::default();
1176
1177 let mut submit_surface_textures_owned = FastHashMap::default();
1180
1181 {
1182 if !command_buffers.is_empty() {
1183 profiling::scope!("prepare");
1184
1185 let mut first_error = None;
1186
1187 for command_buffer in command_buffers {
1193 profiling::scope!("process command buffer");
1194
1195 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1198
1199 #[allow(unused_mut)]
1202 let mut cmd_buf_data = command_buffer.take_finished();
1203
1204 #[cfg(feature = "trace")]
1205 if let Some(ref mut trace) = *self.device.trace.lock() {
1206 if let Ok(ref mut cmd_buf_data) = cmd_buf_data {
1207 trace.add(Action::Submit(
1208 submit_index,
1209 cmd_buf_data.commands.take().unwrap(),
1210 ));
1211 }
1212 }
1213
1214 if first_error.is_some() {
1215 continue;
1216 }
1217
1218 let mut baked = match cmd_buf_data {
1219 Ok(cmd_buf_data) => {
1220 let res = validate_command_buffer(
1221 command_buffer,
1222 self,
1223 &cmd_buf_data,
1224 &snatch_guard,
1225 &mut submit_surface_textures_owned,
1226 &mut used_surface_textures,
1227 &mut command_index_guard,
1228 );
1229 if let Err(err) = res {
1230 first_error.get_or_insert(err);
1231 continue;
1232 }
1233 cmd_buf_data.into_baked_commands()
1234 }
1235 Err(err) => {
1236 first_error.get_or_insert(err.into());
1237 continue;
1238 }
1239 };
1240
1241 if let Err(e) = baked.encoder.open_pass(hal_label(
1243 Some("(wgpu internal) Transit"),
1244 self.device.instance_flags,
1245 )) {
1246 break 'error Err(e.into());
1247 }
1248
1249 let mut trackers = self.device.trackers.lock();
1251 if let Err(e) = baked.initialize_buffer_memory(&mut trackers, &snatch_guard)
1252 {
1253 break 'error Err(e.into());
1254 }
1255 if let Err(e) = baked.initialize_texture_memory(
1256 &mut trackers,
1257 &self.device,
1258 &snatch_guard,
1259 ) {
1260 break 'error Err(e.into());
1261 }
1262
1263 CommandEncoder::insert_barriers_from_device_tracker(
1266 baked.encoder.raw.as_mut(),
1267 &mut trackers,
1268 &baked.trackers,
1269 &snatch_guard,
1270 );
1271
1272 if let Err(e) = baked.encoder.close_and_push_front() {
1273 break 'error Err(e.into());
1274 }
1275
1276 if !used_surface_textures.is_empty() {
1280 if let Err(e) = baked.encoder.open_pass(hal_label(
1281 Some("(wgpu internal) Present"),
1282 self.device.instance_flags,
1283 )) {
1284 break 'error Err(e.into());
1285 }
1286 let texture_barriers = trackers
1287 .textures
1288 .set_from_usage_scope_and_drain_transitions(
1289 &used_surface_textures,
1290 &snatch_guard,
1291 )
1292 .collect::<Vec<_>>();
1293 unsafe {
1294 baked.encoder.raw.transition_textures(&texture_barriers);
1295 };
1296 if let Err(e) = baked.encoder.close() {
1297 break 'error Err(e.into());
1298 }
1299 used_surface_textures = track::TextureUsageScope::default();
1300 }
1301
1302 active_executions.push(EncoderInFlight {
1304 inner: baked.encoder,
1305 trackers: baked.trackers,
1306 temp_resources: baked.temp_resources,
1307 _indirect_draw_validation_resources: baked
1308 .indirect_draw_validation_resources,
1309 pending_buffers: FastHashMap::default(),
1310 pending_textures: FastHashMap::default(),
1311 pending_blas_s: FastHashMap::default(),
1312 });
1313 }
1314
1315 if let Some(first_error) = first_error {
1316 break 'error Err(first_error);
1317 }
1318 }
1319 }
1320
1321 let mut pending_writes = self.pending_writes.lock();
1322
1323 {
1324 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1325 for texture in pending_writes.dst_textures.values() {
1326 match texture.try_inner(&snatch_guard) {
1327 Ok(TextureInner::Native { .. }) => {}
1328 Ok(TextureInner::Surface { .. }) => {
1329 submit_surface_textures_owned
1331 .insert(Arc::as_ptr(texture), texture.clone());
1332
1333 unsafe {
1334 used_surface_textures
1335 .merge_single(texture, None, wgt::TextureUses::PRESENT)
1336 .unwrap()
1337 };
1338 }
1339 Err(DestroyedResourceError(_)) => {}
1344 }
1345 }
1346
1347 if !used_surface_textures.is_empty() {
1348 let mut trackers = self.device.trackers.lock();
1349
1350 let texture_barriers = trackers
1351 .textures
1352 .set_from_usage_scope_and_drain_transitions(
1353 &used_surface_textures,
1354 &snatch_guard,
1355 )
1356 .collect::<Vec<_>>();
1357 unsafe {
1358 pending_writes
1359 .command_encoder
1360 .transition_textures(&texture_barriers);
1361 };
1362 }
1363 }
1364
1365 match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) {
1366 Ok(Some(pending_execution)) => {
1367 active_executions.insert(0, pending_execution);
1368 }
1369 Ok(None) => {}
1370 Err(e) => break 'error Err(e.into()),
1371 }
1372 let hal_command_buffers = active_executions
1373 .iter()
1374 .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
1375 .collect::<Vec<_>>();
1376
1377 {
1378 let mut submit_surface_textures =
1379 SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(
1380 submit_surface_textures_owned.len(),
1381 );
1382
1383 for texture in submit_surface_textures_owned.values() {
1384 let raw = match texture.inner.get(&snatch_guard) {
1385 Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
1386 _ => unreachable!(),
1387 };
1388 submit_surface_textures.push(raw);
1389 }
1390
1391 if let Err(e) = unsafe {
1392 self.raw().submit(
1393 &hal_command_buffers,
1394 &submit_surface_textures,
1395 (fence.as_mut(), submit_index),
1396 )
1397 }
1398 .map_err(|e| self.device.handle_hal_error(e))
1399 {
1400 break 'error Err(e.into());
1401 }
1402
1403 drop(command_index_guard);
1404
1405 self.device
1407 .last_successful_submission_index
1408 .fetch_max(submit_index, Ordering::SeqCst);
1409 }
1410
1411 profiling::scope!("cleanup");
1412
1413 self.lock_life()
1415 .track_submission(submit_index, active_executions);
1416 drop(pending_writes);
1417
1418 let fence_guard = RwLockWriteGuard::downgrade(fence);
1421 let (closures, result) =
1422 self.device
1423 .maintain(fence_guard, wgt::PollType::Poll, snatch_guard);
1424 match result {
1425 Ok(status) => {
1426 debug_assert!(matches!(
1427 status,
1428 wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1429 ));
1430 }
1431 Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1432 Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1433 unreachable!("Cannot get WrongSubmissionIndex from Poll")
1434 }
1435 Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1436 };
1437
1438 Ok(closures)
1439 };
1440
1441 let callbacks = match res {
1442 Ok(ok) => ok,
1443 Err(e) => return Err((submit_index, e)),
1444 };
1445
1446 callbacks.fire();
1448
1449 self.device.lose_if_oom();
1450
1451 api_log!("Queue::submit returned submit index {submit_index}");
1452
1453 Ok(submit_index)
1454 }
1455
1456 pub fn get_timestamp_period(&self) -> f32 {
1457 unsafe { self.raw().get_timestamp_period() }
1458 }
1459
1460 pub fn on_submitted_work_done(
1462 &self,
1463 closure: SubmittedWorkDoneClosure,
1464 ) -> Option<SubmissionIndex> {
1465 api_log!("Queue::on_submitted_work_done");
1466 self.lock_life().add_work_done_closure(closure)
1468 }
1469
1470 pub fn compact_blas(&self, blas: &Arc<Blas>) -> Result<Arc<Blas>, CompactBlasError> {
1471 profiling::scope!("Queue::compact_blas");
1472 api_log!("Queue::compact_blas");
1473
1474 let new_label = blas.label.clone() + " (compacted)";
1475
1476 self.device.check_is_valid()?;
1477 self.same_device_as(blas.as_ref())?;
1478
1479 let device = blas.device.clone();
1480
1481 let snatch_guard = device.snatchable_lock.read();
1482
1483 let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
1484 return Err(CompactBlasError::BlasNotReady);
1485 };
1486
1487 let mut size_info = blas.size_info;
1488 size_info.acceleration_structure_size = size;
1489
1490 let mut pending_writes = self.pending_writes.lock();
1491 let cmd_buf_raw = pending_writes.activate();
1492
1493 let raw = unsafe {
1494 device
1495 .raw()
1496 .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
1497 label: hal_label(Some(&new_label), device.instance_flags),
1498 size: size_info.acceleration_structure_size,
1499 format: hal::AccelerationStructureFormat::BottomLevel,
1500 allow_compaction: false,
1501 })
1502 }
1503 .map_err(DeviceError::from_hal)?;
1504
1505 let src_raw = blas.try_raw(&snatch_guard)?;
1506
1507 unsafe {
1508 cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
1509 src_raw,
1510 raw.as_ref(),
1511 wgt::AccelerationStructureCopy::Compact,
1512 )
1513 };
1514
1515 let handle = unsafe {
1516 device
1517 .raw()
1518 .get_acceleration_structure_device_address(raw.as_ref())
1519 };
1520
1521 drop(snatch_guard);
1522
1523 let mut command_indices_lock = device.command_indices.write();
1524 command_indices_lock.next_acceleration_structure_build_command_index += 1;
1525 let built_index =
1526 NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
1527 .unwrap();
1528
1529 let new_blas = Arc::new(Blas {
1530 raw: Snatchable::new(raw),
1531 device: device.clone(),
1532 size_info,
1533 sizes: blas.sizes.clone(),
1534 flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
1535 update_mode: blas.update_mode,
1536 built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
1538 handle,
1539 label: new_label,
1540 tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
1541 compaction_buffer: None,
1542 compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
1543 });
1544
1545 pending_writes.insert_blas(blas);
1546 pending_writes.insert_blas(&new_blas);
1547
1548 Ok(new_blas)
1549 }
1550}
1551
1552impl Global {
1553 pub fn queue_write_buffer(
1554 &self,
1555 queue_id: QueueId,
1556 buffer_id: id::BufferId,
1557 buffer_offset: wgt::BufferAddress,
1558 data: &[u8],
1559 ) -> Result<(), QueueWriteError> {
1560 let queue = self.hub.queues.get(queue_id);
1561
1562 #[cfg(feature = "trace")]
1563 if let Some(ref mut trace) = *queue.device.trace.lock() {
1564 let data_path = trace.make_binary("bin", data);
1565 trace.add(Action::WriteBuffer {
1566 id: buffer_id,
1567 data: data_path,
1568 range: buffer_offset..buffer_offset + data.len() as u64,
1569 queued: true,
1570 });
1571 }
1572
1573 let buffer = self.hub.buffers.get(buffer_id);
1574 queue.write_buffer(buffer, buffer_offset, data)
1575 }
1576
1577 pub fn queue_create_staging_buffer(
1578 &self,
1579 queue_id: QueueId,
1580 buffer_size: wgt::BufferSize,
1581 id_in: Option<id::StagingBufferId>,
1582 ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
1583 let queue = self.hub.queues.get(queue_id);
1584 let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?;
1585
1586 let fid = self.hub.staging_buffers.prepare(id_in);
1587 let id = fid.assign(staging_buffer);
1588
1589 Ok((id, ptr))
1590 }
1591
1592 pub fn queue_write_staging_buffer(
1593 &self,
1594 queue_id: QueueId,
1595 buffer_id: id::BufferId,
1596 buffer_offset: wgt::BufferAddress,
1597 staging_buffer_id: id::StagingBufferId,
1598 ) -> Result<(), QueueWriteError> {
1599 let queue = self.hub.queues.get(queue_id);
1600 let buffer = self.hub.buffers.get(buffer_id);
1601 let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id);
1602 queue.write_staging_buffer(buffer, buffer_offset, staging_buffer)
1603 }
1604
1605 pub fn queue_validate_write_buffer(
1606 &self,
1607 queue_id: QueueId,
1608 buffer_id: id::BufferId,
1609 buffer_offset: u64,
1610 buffer_size: wgt::BufferSize,
1611 ) -> Result<(), QueueWriteError> {
1612 let queue = self.hub.queues.get(queue_id);
1613 let buffer = self.hub.buffers.get(buffer_id);
1614 queue.validate_write_buffer(buffer, buffer_offset, buffer_size)
1615 }
1616
1617 pub fn queue_write_texture(
1618 &self,
1619 queue_id: QueueId,
1620 destination: &TexelCopyTextureInfo,
1621 data: &[u8],
1622 data_layout: &wgt::TexelCopyBufferLayout,
1623 size: &wgt::Extent3d,
1624 ) -> Result<(), QueueWriteError> {
1625 let queue = self.hub.queues.get(queue_id);
1626
1627 #[cfg(feature = "trace")]
1628 if let Some(ref mut trace) = *queue.device.trace.lock() {
1629 let data_path = trace.make_binary("bin", data);
1630 trace.add(Action::WriteTexture {
1631 to: *destination,
1632 data: data_path,
1633 layout: *data_layout,
1634 size: *size,
1635 });
1636 }
1637
1638 let destination = wgt::TexelCopyTextureInfo {
1639 texture: self.hub.textures.get(destination.texture),
1640 mip_level: destination.mip_level,
1641 origin: destination.origin,
1642 aspect: destination.aspect,
1643 };
1644 queue.write_texture(destination, data, data_layout, size)
1645 }
1646
1647 #[cfg(webgl)]
1648 pub fn queue_copy_external_image_to_texture(
1649 &self,
1650 queue_id: QueueId,
1651 source: &wgt::CopyExternalImageSourceInfo,
1652 destination: crate::command::CopyExternalImageDestInfo,
1653 size: wgt::Extent3d,
1654 ) -> Result<(), QueueWriteError> {
1655 let queue = self.hub.queues.get(queue_id);
1656 let destination = wgt::CopyExternalImageDestInfo {
1657 texture: self.hub.textures.get(destination.texture),
1658 mip_level: destination.mip_level,
1659 origin: destination.origin,
1660 aspect: destination.aspect,
1661 color_space: destination.color_space,
1662 premultiplied_alpha: destination.premultiplied_alpha,
1663 };
1664 queue.copy_external_image_to_texture(source, destination, size)
1665 }
1666
1667 pub fn queue_submit(
1668 &self,
1669 queue_id: QueueId,
1670 command_buffer_ids: &[id::CommandBufferId],
1671 ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1672 let queue = self.hub.queues.get(queue_id);
1673 let command_buffer_guard = self.hub.command_buffers.read();
1674 let command_buffers = command_buffer_ids
1675 .iter()
1676 .map(|id| command_buffer_guard.get(*id))
1677 .collect::<Vec<_>>();
1678 drop(command_buffer_guard);
1679 queue.submit(&command_buffers)
1680 }
1681
1682 pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
1683 let queue = self.hub.queues.get(queue_id);
1684
1685 if queue.device.timestamp_normalizer.get().unwrap().enabled() {
1686 return 1.0;
1687 }
1688
1689 queue.get_timestamp_period()
1690 }
1691
1692 pub fn queue_on_submitted_work_done(
1693 &self,
1694 queue_id: QueueId,
1695 closure: SubmittedWorkDoneClosure,
1696 ) -> SubmissionIndex {
1697 api_log!("Queue::on_submitted_work_done {queue_id:?}");
1698
1699 let queue = self.hub.queues.get(queue_id);
1701 let result = queue.on_submitted_work_done(closure);
1702 result.unwrap_or(0) }
1704
1705 pub fn queue_compact_blas(
1706 &self,
1707 queue_id: QueueId,
1708 blas_id: BlasId,
1709 id_in: Option<BlasId>,
1710 ) -> (BlasId, Option<u64>, Option<CompactBlasError>) {
1711 api_log!("Queue::compact_blas {queue_id:?}, {blas_id:?}");
1712
1713 let fid = self.hub.blas_s.prepare(id_in);
1714
1715 let queue = self.hub.queues.get(queue_id);
1716 let blas = self.hub.blas_s.get(blas_id);
1717 let device = &queue.device;
1718
1719 let error = 'error: {
1722 match device.require_features(wgpu_types::Features::EXPERIMENTAL_RAY_QUERY) {
1723 Ok(_) => {}
1724 Err(err) => break 'error err.into(),
1725 }
1726
1727 let blas = match blas.get() {
1728 Ok(blas) => blas,
1729 Err(err) => break 'error err.into(),
1730 };
1731
1732 let new_blas = match queue.compact_blas(&blas) {
1733 Ok(blas) => blas,
1734 Err(err) => break 'error err,
1735 };
1736
1737 let old_blas_size = blas.size_info.acceleration_structure_size;
1739 let new_blas_size = new_blas.size_info.acceleration_structure_size;
1740 let handle = new_blas.handle;
1741
1742 let id = fid.assign(Fallible::Valid(new_blas));
1743
1744 api_log!("CommandEncoder::compact_blas {blas_id:?} (size: {old_blas_size}) -> {id:?} (size: {new_blas_size})");
1745
1746 return (id, Some(handle), None);
1747 };
1748
1749 let id = fid.assign(Fallible::Invalid(Arc::new(error.to_string())));
1750
1751 (id, None, Some(error))
1752 }
1753}
1754
1755fn validate_command_buffer(
1756 command_buffer: &CommandBuffer,
1757 queue: &Queue,
1758 cmd_buf_data: &crate::command::CommandBufferMutable,
1759 snatch_guard: &SnatchGuard,
1760 submit_surface_textures_owned: &mut FastHashMap<*const Texture, Arc<Texture>>,
1761 used_surface_textures: &mut track::TextureUsageScope,
1762 command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
1763) -> Result<(), QueueSubmitError> {
1764 command_buffer.same_device_as(queue)?;
1765
1766 {
1767 profiling::scope!("check resource state");
1768
1769 {
1770 profiling::scope!("buffers");
1771 for buffer in cmd_buf_data.trackers.buffers.used_resources() {
1772 buffer.check_destroyed(snatch_guard)?;
1773
1774 match *buffer.map_state.lock() {
1775 BufferMapState::Idle => (),
1776 _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
1777 }
1778 }
1779 }
1780 {
1781 profiling::scope!("textures");
1782 for texture in cmd_buf_data.trackers.textures.used_resources() {
1783 let should_extend = match texture.try_inner(snatch_guard)? {
1784 TextureInner::Native { .. } => false,
1785 TextureInner::Surface { .. } => {
1786 submit_surface_textures_owned.insert(Arc::as_ptr(texture), texture.clone());
1788
1789 true
1790 }
1791 };
1792 if should_extend {
1793 unsafe {
1794 used_surface_textures
1795 .merge_single(texture, None, wgt::TextureUses::PRESENT)
1796 .unwrap();
1797 };
1798 }
1799 }
1800 }
1801
1802 if let Err(e) =
1803 cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
1804 {
1805 return Err(e.into());
1806 }
1807 }
1808 Ok(())
1809}