1#[cfg(feature = "trace")]
2use alloc::string::ToString as _;
3use alloc::{boxed::Box, string::String, sync::Arc, vec, vec::Vec};
4use core::{
5 mem::{self, ManuallyDrop},
6 num::NonZeroU64,
7 ptr::NonNull,
8 sync::atomic::Ordering,
9};
10use smallvec::SmallVec;
11use thiserror::Error;
12use wgt::{
13 error::{ErrorType, WebGpuError},
14 AccelerationStructureFlags,
15};
16
17use super::{life::LifetimeTracker, Device};
18#[cfg(feature = "trace")]
19use crate::device::trace::{Action, IntoTrace};
20use crate::{
21 api_log,
22 command::{
23 validate_linear_texture_data, validate_texture_buffer_copy,
24 validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
25 CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopyRange, CopySide,
26 TransferError,
27 },
28 device::{DeviceError, QueueDescriptor, WaitIdleError},
29 get_lowest_common_denom, hal_label,
30 init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
31 lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
32 ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
33 ray_tracing_pipeline,
34 resource::{
35 Blas, BlasCompactState, BlasDescriptor, BlasState, Buffer, BufferAccessError,
36 BufferMapState, DestroyedBuffer, DestroyedQuerySet, DestroyedResourceError,
37 DestroyedTexture, FlushedStagingBuffer, InvalidOrDestroyedResourceError,
38 InvalidResourceError, Labeled, ParentDevice, ResourceErrorIdent, ResourceState,
39 StagingBuffer, Texture, TextureInner, Trackable, TrackingData,
40 },
41 resource_log,
42 scratch::ScratchBuffer,
43 snatch::{SnatchGuard, Snatchable},
44 track::{self, Tracker, TrackerIndex},
45 FastHashMap, LabelHelpers, SubmissionIndex,
46};
47use crate::{device::resource::CommandIndices, resource::RawResourceAccess};
48
49pub struct Queue {
50 raw: Box<dyn hal::DynQueue>,
51 pub(crate) pending_writes: Mutex<PendingWrites>,
52 life_tracker: Mutex<LifetimeTracker>,
53 label: String,
54 pub(crate) device: Arc<Device>,
56}
57
58impl Queue {
59 pub(crate) fn new(
60 device: Arc<Device>,
61 raw: Box<dyn hal::DynQueue>,
62 desc: QueueDescriptor,
63 instance_flags: wgt::InstanceFlags,
64 ) -> Result<Self, DeviceError> {
65 let pending_encoder = device
66 .command_allocator
67 .acquire_encoder(device.raw(), raw.as_ref())
68 .map_err(DeviceError::from_hal);
69
70 let pending_encoder = match pending_encoder {
71 Ok(pending_encoder) => pending_encoder,
72 Err(e) => {
73 return Err(e);
74 }
75 };
76
77 let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);
78
79 let zero_buffer = device.zero_buffer.as_ref();
80 pending_writes.activate();
81 unsafe {
82 pending_writes
83 .command_encoder
84 .transition_buffers(&[hal::BufferBarrier {
85 buffer: zero_buffer,
86 usage: hal::StateTransition {
87 from: wgt::BufferUses::empty(),
88 to: wgt::BufferUses::COPY_DST,
89 },
90 }]);
91 pending_writes
92 .command_encoder
93 .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
94 pending_writes
95 .command_encoder
96 .transition_buffers(&[hal::BufferBarrier {
97 buffer: zero_buffer,
98 usage: hal::StateTransition {
99 from: wgt::BufferUses::COPY_DST,
100 to: wgt::BufferUses::COPY_SRC,
101 },
102 }]);
103 }
104
105 Ok(Queue {
106 raw,
107 device,
108 label: desc.label.to_string(),
109 pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
110 life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
111 })
112 }
113
114 pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
115 self.raw.as_ref()
116 }
117
118 #[track_caller]
119 pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
120 self.life_tracker.lock()
121 }
122
123 pub(crate) fn prepare_surface_texture_for_present(
128 &self,
129 texture: &Arc<Texture>,
130 ) -> Result<(), DeviceError> {
131 let snatch_guard = self.device.snatchable_lock.read();
132 let submission = self.allocate_submission(snatch_guard)?;
133 let device = &self.device;
134
135 let needs_clear = {
137 let status = texture.initialization_status.read();
138 status
139 .mips
140 .first()
141 .is_some_and(|mip| mip.check(0..1).is_some())
142 };
143
144 let mut pending_writes = self.pending_writes.lock();
145
146 if needs_clear {
147 let encoder = pending_writes.activate();
150 let mut trackers = device.trackers.lock();
151 crate::command::clear_texture(
152 texture,
153 TextureInitRange {
154 mip_range: 0..1,
155 layer_range: 0..1,
156 },
157 None,
158 encoder,
159 &mut trackers.textures,
160 &device.alignments,
161 device.zero_buffer.as_ref(),
162 &submission.snatch_guard,
163 device.instance_flags,
164 )
165 .map_err(|e| match e {
166 ClearError::Device(e) => e,
167 _ => DeviceError::Lost,
168 })?;
169 texture.initialization_status.write().mips[0].drain(0..1);
170 }
171
172 let pending = {
178 let mut trackers = device.trackers.lock();
179 let pending: Vec<track::PendingTransition<wgt::TextureUses>> = trackers
180 .textures
181 .set_single(
182 texture,
183 texture.full_range.clone(),
184 wgt::TextureUses::PRESENT,
185 )
186 .collect();
187 pending
188 };
189
190 if pending.is_empty() {
191 debug_assert!(!needs_clear);
197 return Ok(());
198 }
199
200 {
202 let raw_texture = texture
203 .raw(&submission.snatch_guard)
204 .ok_or(DeviceError::Lost)?;
205 let barriers: Vec<hal::TextureBarrier<'_, dyn hal::DynTexture>> = pending
206 .into_iter()
207 .map(|pt| pt.into_hal(raw_texture))
208 .collect();
209
210 let encoder = pending_writes.activate();
211 unsafe {
215 encoder.transition_textures(&barriers);
216 }
217 }
218
219 pending_writes.insert_texture(texture);
226
227 submission.submit(pending_writes)?;
228
229 Ok(())
230 }
231
232 pub(crate) fn maintain(
240 &self,
241 submission_index: u64,
242 snatch_guard: &SnatchGuard,
243 ) -> (
244 SmallVec<[SubmittedWorkDoneClosure; 1]>,
245 Vec<super::BufferMapPendingClosure>,
246 Vec<BlasCompactReadyPendingClosure>,
247 bool,
248 ) {
249 let mut life_tracker = self.lock_life();
250 let submission_closures = life_tracker.triage_submissions(submission_index);
251
252 let mapping_closures = life_tracker.handle_mapping(snatch_guard);
253 let blas_closures = life_tracker.handle_compact_read_back();
254
255 let queue_empty = life_tracker.queue_empty();
256
257 (
258 submission_closures,
259 mapping_closures,
260 blas_closures,
261 queue_empty,
262 )
263 }
264}
265
266crate::impl_resource_type!(Queue);
267crate::impl_labeled!(Queue);
268crate::impl_parent_device!(Queue);
269crate::impl_storage_item!(Queue);
270
271impl Drop for Queue {
272 #[allow(trivial_casts)]
273 fn drop(&mut self) {
274 profiling::scope!("Queue::drop");
275 api_log!("Queue::drop {:?}", self as *const _);
276 resource_log!("Drop {}", self.error_ident());
277
278 let result = {
281 let _command_indices = self.device.command_indices.write();
283 unsafe { self.raw.wait_for_idle() }
284 };
285 match result {
286 Ok(()) => {}
287 Err(hal::DeviceError::Lost) => {
288 self.device.handle_hal_error(hal::DeviceError::Lost);
289 }
290 Err(e) => {
291 panic!("Unexpected error while waiting for queue idle on drop: {e:?}");
292 }
293 }
294
295 let last_successful_submission_index = self
296 .device
297 .last_successful_submission_index
298 .load(Ordering::Acquire);
299
300 let snatch_guard = self.device.snatchable_lock.read();
301 let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
302 self.maintain(last_successful_submission_index, &snatch_guard);
303 drop(snatch_guard);
304
305 assert!(queue_empty);
306
307 let closures = crate::device::UserClosures {
308 mappings: mapping_closures,
309 blas_compact_ready: blas_compact_ready_closures,
310 submissions: submission_closures,
311 device_lost_invocations: SmallVec::new(),
312 };
313
314 closures.fire();
315 }
316}
317
318#[cfg(send_sync)]
319pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
320#[cfg(not(send_sync))]
321pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
322
323#[derive(Debug)]
334pub enum TempResource {
335 StagingBuffer(FlushedStagingBuffer),
336 ScratchBuffer(ScratchBuffer),
337 DestroyedBuffer(DestroyedBuffer),
338 DestroyedTexture(DestroyedTexture),
339 DestroyedQuerySet(DestroyedQuerySet),
340}
341
342pub(crate) struct EncoderInFlight {
348 inner: crate::command::InnerCommandEncoder,
349 pub(crate) trackers: Tracker,
350 pub(crate) temp_resources: Vec<TempResource>,
351 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
353
354 pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
356 pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
358 pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
360 _pending_shader_binding_data: Vec<Arc<ray_tracing_pipeline::ShaderBindingData>>,
363}
364
365#[derive(Debug)]
390pub(crate) struct PendingWrites {
391 pub command_encoder: Box<dyn hal::DynCommandEncoder>,
393
394 pub is_recording: bool,
400
401 temp_resources: Vec<TempResource>,
402 dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
403 dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
404 copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
405 written_shader_binding_data: Vec<Arc<ray_tracing_pipeline::ShaderBindingData>>,
408 instance_flags: wgt::InstanceFlags,
409
410 hoisted_staging_buffers: Vec<FlushedStagingBuffer>,
419
420 hoisted_buffer_barriers: Vec<(Arc<Buffer>, hal::StateTransition<wgt::BufferUses>)>,
442}
443
444impl PendingWrites {
445 pub fn new(
446 command_encoder: Box<dyn hal::DynCommandEncoder>,
447 instance_flags: wgt::InstanceFlags,
448 ) -> Self {
449 Self {
450 command_encoder,
451 is_recording: false,
452 temp_resources: Vec::new(),
453 dst_buffers: FastHashMap::default(),
454 dst_textures: FastHashMap::default(),
455 copied_blas_s: FastHashMap::default(),
456 instance_flags,
457 hoisted_staging_buffers: Vec::new(),
458 hoisted_buffer_barriers: Vec::new(),
459 written_shader_binding_data: Vec::new(),
460 }
461 }
462
463 pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
464 self.dst_buffers
465 .insert(buffer.tracker_index(), buffer.clone());
466 }
467
468 pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
469 self.dst_textures
470 .insert(texture.tracker_index(), texture.clone());
471 }
472
473 pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
474 self.copied_blas_s
475 .insert(blas.tracker_index(), blas.clone());
476 }
477
478 pub fn use_shader_binding_data(&mut self, sbd: &Arc<ray_tracing_pipeline::ShaderBindingData>) {
479 self.written_shader_binding_data.push(sbd.clone())
480 }
481
482 pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
483 self.dst_buffers.contains_key(&buffer.tracker_index())
484 }
485
486 pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
487 self.dst_textures.contains_key(&texture.tracker_index())
488 }
489
490 pub fn consume_temp(&mut self, resource: TempResource) {
491 self.temp_resources.push(resource);
492 }
493
494 pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
500 self.hoisted_staging_buffers.push(buffer);
501 }
502
503 #[must_use]
517 fn hoist_buffer_barrier(
518 &mut self,
519 buffer: &Arc<Buffer>,
520 transition: Option<track::PendingTransition<wgt::BufferUses>>,
521 ) -> Option<track::PendingTransition<wgt::BufferUses>> {
522 let transition = transition?;
523
524 if self.dst_buffers.contains_key(&buffer.tracker_index()) {
527 return Some(transition);
528 }
529
530 self.hoisted_buffer_barriers
531 .push((buffer.clone(), transition.usage));
532 None
533 }
534
535 pub fn clear_buffer(
536 &mut self,
537 device: &Arc<Device>,
538 buffer: &Arc<Buffer>,
539 range: core::ops::Range<wgt::BufferAddress>,
540 snatch_guard: &SnatchGuard,
541 ) -> Result<(), QueueWriteError> {
542 let transition = {
543 let mut trackers = device.trackers.lock();
544 trackers
545 .buffers
546 .set_single(buffer, wgt::BufferUses::COPY_DST)
547 };
548 let inline_transition = self.hoist_buffer_barrier(buffer, transition);
549
550 let dst_raw = buffer.try_raw(snatch_guard)?;
551
552 let encoder = self.activate();
553 unsafe {
554 if let Some(transition) = inline_transition {
555 encoder.transition_buffers(&[transition.into_hal(buffer, snatch_guard)]);
556 }
557 encoder.clear_buffer(dst_raw, range.clone());
558 }
559
560 self.insert_buffer(buffer);
561
562 buffer.initialization_status.write().drain(range);
565
566 Ok(())
567 }
568
569 fn record_hoisted_barriers(
576 &mut self,
577 encoder: &mut EncoderInFlight,
578 snatch_guard: &SnatchGuard,
579 ) -> Result<Vec<FlushedStagingBuffer>, DeviceError> {
580 let staging_buffers = mem::take(&mut self.hoisted_staging_buffers);
581 let buffer_barriers = mem::take(&mut self.hoisted_buffer_barriers);
582
583 let barriers = staging_buffers
584 .iter()
585 .map(|staging_buffer| hal::BufferBarrier {
586 buffer: staging_buffer.raw(),
587 usage: hal::StateTransition {
588 from: wgt::BufferUses::MAP_WRITE,
589 to: wgt::BufferUses::COPY_SRC,
590 },
591 })
592 .chain(buffer_barriers.iter().filter_map(|(buffer, usage)| {
593 Some(hal::BufferBarrier {
596 buffer: buffer.raw(snatch_guard)?,
597 usage: usage.clone(),
598 })
599 }))
600 .collect::<Vec<_>>();
601
602 if !barriers.is_empty() {
603 encoder
604 .inner
605 .open_pass(Some("(wgpu internal) PendingWrites Transit"))?;
606 unsafe { encoder.inner.raw.transition_buffers(&barriers) };
607 encoder.inner.close_and_push_front()?;
608 }
609
610 Ok(staging_buffers)
611 }
612
613 fn pre_submit(
614 &mut self,
615 command_allocator: &CommandAllocator,
616 device: &Arc<Device>,
617 queue: &Queue,
618 snatch_guard: &SnatchGuard,
619 ) -> Result<Option<EncoderInFlight>, DeviceError> {
620 if self.is_recording {
621 let pending_buffers = mem::take(&mut self.dst_buffers);
622 let pending_textures = mem::take(&mut self.dst_textures);
623 let pending_blas_s = mem::take(&mut self.copied_blas_s);
624 let pending_shader_binding_data = mem::take(&mut self.written_shader_binding_data);
625
626 let cmd_buf = unsafe { self.command_encoder.end_encoding() }
627 .map_err(|e| device.handle_hal_error(e))?;
628 self.is_recording = false;
629
630 let new_encoder = command_allocator
631 .acquire_encoder(device.raw(), queue.raw())
632 .map_err(|e| device.handle_hal_error(e))?;
633
634 let mut encoder = EncoderInFlight {
635 inner: crate::command::InnerCommandEncoder {
636 raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
637 list: vec![cmd_buf],
638 device: device.clone(),
639 is_open: false,
640 api: crate::command::EncodingApi::InternalUse,
641 label: "(wgpu internal) PendingWrites command encoder".into(),
642 },
643 trackers: Tracker::new(device.ordered_buffer_usages, device.ordered_texture_usages),
644 temp_resources: mem::take(&mut self.temp_resources),
645 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
646 device.clone(),
647 ),
648 pending_buffers,
649 pending_textures,
650 pending_blas_s,
651 _pending_shader_binding_data: pending_shader_binding_data,
652 };
653
654 let staging_buffers = self.record_hoisted_barriers(&mut encoder, snatch_guard)?;
655 encoder
656 .temp_resources
657 .extend(staging_buffers.into_iter().map(TempResource::StagingBuffer));
658
659 Ok(Some(encoder))
660 } else {
661 self.dst_buffers.clear();
662 self.dst_textures.clear();
663 self.copied_blas_s.clear();
664 self.written_shader_binding_data.clear();
665 Ok(None)
666 }
667 }
668
669 pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
670 if !self.is_recording {
671 unsafe {
672 self.command_encoder
673 .begin_encoding(hal_label(
674 Some("(wgpu internal) PendingWrites"),
675 self.instance_flags,
676 ))
677 .unwrap();
678 }
679 self.is_recording = true;
680 }
681 self.command_encoder.as_mut()
682 }
683}
684
685impl Drop for PendingWrites {
686 fn drop(&mut self) {
687 unsafe {
688 if self.is_recording {
689 self.command_encoder.discard_encoding();
690 }
691 }
692 }
693}
694
695#[derive(Clone, Debug, Error)]
696#[non_exhaustive]
697pub enum QueueWriteError {
698 #[error(transparent)]
699 Queue(#[from] DeviceError),
700 #[error(transparent)]
701 Transfer(#[from] TransferError),
702 #[error(transparent)]
703 MemoryInitFailure(#[from] ClearError),
704 #[error(transparent)]
705 DestroyedResource(#[from] DestroyedResourceError),
706 #[error(transparent)]
707 InvalidResource(#[from] InvalidResourceError),
708}
709
710impl From<InvalidOrDestroyedResourceError> for QueueWriteError {
711 fn from(e: InvalidOrDestroyedResourceError) -> Self {
712 match e {
713 InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
714 InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
715 }
716 }
717}
718
719impl WebGpuError for QueueWriteError {
720 fn webgpu_error_type(&self) -> ErrorType {
721 match self {
722 Self::Queue(e) => e.webgpu_error_type(),
723 Self::Transfer(e) => e.webgpu_error_type(),
724 Self::MemoryInitFailure(e) => e.webgpu_error_type(),
725 Self::DestroyedResource(e) => e.webgpu_error_type(),
726 Self::InvalidResource(e) => e.webgpu_error_type(),
727 }
728 }
729}
730
731#[derive(Clone, Debug, Error)]
732#[non_exhaustive]
733pub enum QueueSubmitError {
734 #[error(transparent)]
735 Queue(#[from] DeviceError),
736 #[error(transparent)]
737 DestroyedResource(#[from] DestroyedResourceError),
738 #[error("{0} is still mapped")]
739 BufferStillMapped(ResourceErrorIdent),
740 #[error(transparent)]
741 InvalidResource(#[from] InvalidResourceError),
742 #[error(transparent)]
743 CommandEncoder(#[from] CommandEncoderError),
744 #[error(transparent)]
745 ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
746}
747
748impl From<InvalidOrDestroyedResourceError> for QueueSubmitError {
749 fn from(e: InvalidOrDestroyedResourceError) -> Self {
750 match e {
751 InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
752 InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
753 }
754 }
755}
756
757impl WebGpuError for QueueSubmitError {
758 fn webgpu_error_type(&self) -> ErrorType {
759 match self {
760 Self::Queue(e) => e.webgpu_error_type(),
761 Self::CommandEncoder(e) => e.webgpu_error_type(),
762 Self::ValidateAsActionsError(e) => e.webgpu_error_type(),
763 Self::InvalidResource(e) => e.webgpu_error_type(),
764 Self::DestroyedResource(_) | Self::BufferStillMapped(_) => ErrorType::Validation,
765 }
766 }
767}
768
769pub(crate) struct PendingSubmission<'a> {
817 queue: &'a Queue,
818
819 command_index_guard: RwLockWriteGuard<'a, CommandIndices>,
829
830 snatch_guard: SnatchGuard<'a>,
832
833 pub executions: Vec<EncoderInFlight>,
836
837 surface_textures: FastHashMap<*const Texture, Arc<Texture>>,
843
844 pub index: SubmissionIndex,
846}
847
848pub(crate) struct SubmissionResult<'a> {
849 pub snatch_guard: SnatchGuard<'a>,
850}
851
852impl<'a> PendingSubmission<'a> {
853 fn submit(
854 self,
855 pending_writes: MutexGuard<'a, PendingWrites>,
856 ) -> Result<SubmissionResult<'a>, DeviceError> {
857 self.queue.submit_pending_submission(pending_writes, self)
858 }
859}
860
861impl Queue {
864 pub(crate) fn write_buffer_inner(
865 &self,
866 buffer: Arc<Buffer>,
867 buffer_offset: wgt::BufferAddress,
868 data: &[u8],
869 ) -> Result<(), QueueWriteError> {
870 profiling::scope!("Queue::write_buffer");
871 api_log!("Queue::write_buffer");
872
873 #[cfg(feature = "trace")]
874 if let Some(ref mut trace) = *self.device.trace.lock() {
875 use crate::device::trace::DataKind;
876 let size = data.len() as u64;
877 let data = trace.make_binary(DataKind::Bin, data);
878 trace.add(Action::WriteBuffer {
879 id: buffer.to_trace(),
880 data,
881 offset: buffer_offset,
882 size,
883 queued: true,
884 });
885 }
886
887 buffer.check_is_valid()?;
888 self.device.check_is_valid()?;
889
890 let data_size = data.len() as wgt::BufferAddress;
891
892 self.same_device_as(buffer.as_ref())?;
893
894 let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
895 data_size
896 } else {
897 self.validate_write_buffer_impl(buffer.as_ref(), buffer_offset, 0)?;
902
903 log::trace!("Ignoring write_buffer of size 0");
904 return Ok(());
905 };
906
907 let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
911
912 let staging_buffer = {
913 profiling::scope!("copy");
914 staging_buffer.write_exact(data);
915 staging_buffer.flush()
916 };
917
918 let snatch_guard = self.device.snatchable_lock.read();
919 let mut pending_writes = self.pending_writes.lock();
920
921 let result = self.write_staging_buffer_impl(
922 &snatch_guard,
923 &mut pending_writes,
924 &staging_buffer,
925 buffer,
926 buffer_offset,
927 );
928
929 pending_writes.consume(staging_buffer);
930
931 result
932 }
933
934 pub fn write_buffer(
935 &self,
936 buffer: Arc<Buffer>,
937 buffer_offset: wgt::BufferAddress,
938 data: &[u8],
939 ) {
940 if let Err(error) = self.write_buffer_inner(buffer, buffer_offset, data) {
941 self.device
942 .handle_error(error, Some(self.label()), "Queue::write_buffer");
943 }
944 }
945
946 pub fn create_staging_buffer(
947 &self,
948 buffer_size: wgt::BufferSize,
949 ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
950 profiling::scope!("Queue::create_staging_buffer");
951 resource_log!("Queue::create_staging_buffer");
952
953 self.device.check_is_valid()?;
954
955 let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
956 let ptr = unsafe { staging_buffer.ptr() };
957
958 Ok((staging_buffer, ptr))
959 }
960
961 pub fn write_staging_buffer(
962 &self,
963 buffer: Arc<Buffer>,
964 buffer_offset: wgt::BufferAddress,
965 staging_buffer: StagingBuffer,
966 ) -> Result<(), QueueWriteError> {
967 profiling::scope!("Queue::write_staging_buffer");
968
969 buffer.check_is_valid()?;
970 self.device.check_is_valid()?;
971
972 let staging_buffer = staging_buffer.flush();
977
978 let snatch_guard = self.device.snatchable_lock.read();
979 let mut pending_writes = self.pending_writes.lock();
980
981 let result = self.write_staging_buffer_impl(
982 &snatch_guard,
983 &mut pending_writes,
984 &staging_buffer,
985 buffer,
986 buffer_offset,
987 );
988
989 drop(snatch_guard);
990
991 pending_writes.consume(staging_buffer);
992
993 drop(pending_writes);
994
995 result
996 }
997
998 pub fn validate_write_buffer(
999 &self,
1000 buffer: Arc<Buffer>,
1001 buffer_offset: u64,
1002 buffer_size: wgt::BufferSize,
1003 ) -> Result<(), QueueWriteError> {
1004 profiling::scope!("Queue::validate_write_buffer");
1005
1006 self.device.check_is_valid()?;
1007 buffer.check_is_valid()?;
1008
1009 self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size.into())?;
1010
1011 Ok(())
1012 }
1013
1014 fn validate_write_buffer_impl(
1015 &self,
1016 buffer: &Buffer,
1017 buffer_offset: u64,
1018 buffer_size: u64,
1019 ) -> Result<(), TransferError> {
1020 if !matches!(&*buffer.map_state.read(), BufferMapState::Idle) {
1021 return Err(TransferError::BufferNotAvailable);
1022 }
1023 buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
1024 if !buffer_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1025 return Err(TransferError::UnalignedCopySize(buffer_size));
1026 }
1027 if !buffer_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1028 return Err(TransferError::UnalignedBufferOffset(buffer_offset));
1029 }
1030
1031 if buffer_offset > buffer.size {
1032 return Err(TransferError::BufferStartOffsetOverrun {
1033 start_offset: buffer_offset,
1034 buffer_size: buffer.size,
1035 side: CopySide::Destination,
1036 });
1037 }
1038 if buffer_size > buffer.size - buffer_offset {
1039 return Err(TransferError::BufferEndOffsetOverrun {
1040 start_offset: buffer_offset,
1041 size: buffer_size,
1042 buffer_size: buffer.size,
1043 side: CopySide::Destination,
1044 });
1045 }
1046
1047 Ok(())
1048 }
1049
1050 fn write_staging_buffer_impl(
1051 &self,
1052 snatch_guard: &SnatchGuard,
1053 pending_writes: &mut PendingWrites,
1054 staging_buffer: &FlushedStagingBuffer,
1055 buffer: Arc<Buffer>,
1056 buffer_offset: u64,
1057 ) -> Result<(), QueueWriteError> {
1058 self.device.check_is_valid()?;
1059
1060 let transition = {
1061 let mut trackers = self.device.trackers.lock();
1062 trackers
1063 .buffers
1064 .set_single(&buffer, wgt::BufferUses::COPY_DST)
1065 };
1066
1067 let dst_raw = buffer.try_raw(snatch_guard)?;
1068
1069 self.same_device_as(buffer.as_ref())?;
1070
1071 self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size.into())?;
1072
1073 let region = hal::BufferCopy {
1074 src_offset: 0,
1075 dst_offset: buffer_offset,
1076 size: staging_buffer.size,
1077 };
1078 let inline_transition = pending_writes.hoist_buffer_barrier(&buffer, transition);
1081 let encoder = pending_writes.activate();
1082 unsafe {
1083 if let Some(transition) = inline_transition {
1084 encoder.transition_buffers(&[transition.into_hal(&buffer, snatch_guard)]);
1085 }
1086 encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
1087 }
1088
1089 pending_writes.insert_buffer(&buffer);
1090
1091 {
1094 buffer
1095 .initialization_status
1096 .write()
1097 .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
1098 }
1099
1100 Ok(())
1101 }
1102
1103 pub fn write_texture_inner(
1104 &self,
1105 destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
1106 data: &[u8],
1107 data_layout: &wgt::TexelCopyBufferLayout,
1108 size: &wgt::Extent3d,
1109 ) -> Result<(), QueueWriteError> {
1110 profiling::scope!("Queue::write_texture");
1111 api_log!("Queue::write_texture");
1112
1113 #[cfg(feature = "trace")]
1114 if let Some(ref mut trace) = *self.device.trace.lock() {
1115 use crate::device::trace::DataKind;
1116 let data = trace.make_binary(DataKind::Bin, data);
1117 trace.add(Action::WriteTexture {
1118 to: destination.to_trace(),
1119 data,
1120 layout: *data_layout,
1121 size: *size,
1122 });
1123 }
1124
1125 self.device.check_is_valid()?;
1126
1127 let dst = destination.texture;
1128 let destination = wgt::TexelCopyTextureInfo {
1129 texture: (),
1130 mip_level: destination.mip_level,
1131 origin: destination.origin,
1132 aspect: destination.aspect,
1133 };
1134
1135 self.same_device_as(dst.as_ref())?;
1136
1137 dst.check_valid()?;
1138
1139 dst.check_usage(wgt::TextureUsages::COPY_DST)
1140 .map_err(TransferError::MissingTextureUsage)?;
1141
1142 let CopyRange {
1145 hal_copy_extent,
1146 hal_base: dst_base,
1147 array_layer_count,
1148 selector,
1149 } = validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
1150
1151 validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;
1152
1153 validate_texture_buffer_copy(
1154 &destination,
1155 dst_base.aspect,
1156 &dst.desc,
1157 data_layout,
1158 false, self.device.features,
1160 )?;
1161
1162 let (required_bytes_in_copy, _source_bytes_per_array_layer, _) =
1165 validate_linear_texture_data(
1166 data_layout,
1167 dst.desc.format,
1168 destination.aspect,
1169 data.len() as wgt::BufferAddress,
1170 CopySide::Source,
1171 size,
1172 )?;
1173
1174 if dst.desc.format.is_depth_stencil_format() {
1175 self.device
1176 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1177 .map_err(TransferError::from)?;
1178 }
1179
1180 let snatch_guard = self.device.snatchable_lock.read();
1181
1182 let dst_raw = dst.try_inner(&snatch_guard)?.raw();
1183
1184 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
1187 log::trace!("Ignoring write_texture of size 0");
1188 return Ok(());
1189 }
1190
1191 let mut pending_writes = self.pending_writes.lock();
1192 let encoder = pending_writes.activate();
1193
1194 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1200 0..1
1202 } else {
1203 destination.origin.z..destination.origin.z + size.depth_or_array_layers
1204 };
1205 let layer_ranges_to_clear = {
1206 let mut dst_initialization_status = dst.initialization_status.write();
1207 if dst_initialization_status.mips[destination.mip_level as usize]
1208 .check(init_layer_range.clone())
1209 .is_some()
1210 {
1211 if has_copy_partial_init_tracker_coverage(size, &destination, &dst.desc) {
1212 dst_initialization_status.mips[destination.mip_level as usize]
1213 .drain(init_layer_range)
1214 .collect::<Vec<core::ops::Range<u32>>>()
1215 } else {
1216 dst_initialization_status.mips[destination.mip_level as usize]
1217 .drain(init_layer_range);
1218 vec![]
1219 }
1220 } else {
1221 vec![]
1222 }
1223 };
1224 if !layer_ranges_to_clear.is_empty() {
1225 let mut trackers = self.device.trackers.lock();
1226 for layer_range in layer_ranges_to_clear {
1227 crate::command::clear_texture(
1228 &dst,
1229 TextureInitRange {
1230 mip_range: destination.mip_level..(destination.mip_level + 1),
1231 layer_range,
1232 },
1233 None,
1234 encoder,
1235 &mut trackers.textures,
1236 &self.device.alignments,
1237 self.device.zero_buffer.as_ref(),
1238 &snatch_guard,
1239 self.device.instance_flags,
1240 )
1241 .map_err(QueueWriteError::from)?;
1242 }
1243 }
1244
1245 let (block_width, block_height) = dst.desc.format.block_dimensions();
1246 let width_in_blocks = size.width / block_width;
1247 let height_in_blocks = size.height / block_height;
1248
1249 let block_size = dst
1250 .desc
1251 .format
1252 .block_copy_size(Some(destination.aspect))
1253 .unwrap();
1254 let bytes_in_last_row = width_in_blocks * block_size;
1255
1256 let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
1257 let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
1258
1259 let bytes_per_row_alignment = get_lowest_common_denom(
1260 self.device.alignments.buffer_copy_pitch.get() as u32,
1261 block_size,
1262 );
1263 assert!(u32::MAX - bytes_in_last_row >= bytes_per_row_alignment);
1264 let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
1265
1266 let staging_buffer = if stage_bytes_per_row == bytes_per_row {
1270 profiling::scope!("copy aligned");
1271 let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
1273 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1274 staging_buffer
1275 .write_exact(&data[data_layout.offset as usize..][..stage_size.get() as usize]);
1276 staging_buffer
1277 } else {
1278 profiling::scope!("copy chunked");
1279 let block_rows_in_copy = u64::from(size.depth_or_array_layers - 1)
1281 * u64::from(rows_per_image)
1282 + u64::from(height_in_blocks);
1283 let stage_size = u64::from(stage_bytes_per_row)
1286 .checked_mul(block_rows_in_copy)
1287 .and_then(wgt::BufferSize::new)
1288 .unwrap();
1289 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1290 for layer in 0..u64::from(size.depth_or_array_layers) {
1291 let rows_offset = layer * u64::from(rows_per_image);
1292 for row in rows_offset..rows_offset + u64::from(height_in_blocks) {
1293 let src_offset = data_layout.offset + row * u64::from(bytes_per_row);
1294 let dst_offset = row * u64::from(stage_bytes_per_row);
1295 unsafe {
1296 staging_buffer.write_with_offset(
1297 data,
1298 src_offset as isize,
1299 dst_offset as isize,
1300 bytes_in_last_row as usize,
1301 )
1302 }
1303 }
1304 }
1305 staging_buffer
1306 };
1307
1308 let staging_buffer = staging_buffer.flush();
1309
1310 let regions = (0..array_layer_count)
1311 .map(|array_layer_offset| {
1312 let mut texture_base = dst_base.clone();
1313 texture_base.array_layer += array_layer_offset;
1314 hal::BufferTextureCopy {
1315 buffer_layout: wgt::TexelCopyBufferLayout {
1316 offset: array_layer_offset as u64
1317 * rows_per_image as u64
1318 * stage_bytes_per_row as u64,
1319 bytes_per_row: Some(stage_bytes_per_row),
1320 rows_per_image: Some(rows_per_image),
1321 },
1322 texture_base,
1323 size: hal_copy_extent,
1324 }
1325 })
1326 .collect::<Vec<_>>();
1327
1328 {
1329 let mut trackers = self.device.trackers.lock();
1330 let transition =
1331 trackers
1332 .textures
1333 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1334 let texture_barriers = transition
1335 .map(|pending| pending.into_hal(dst_raw))
1336 .collect::<Vec<_>>();
1337
1338 unsafe {
1342 encoder.transition_textures(&texture_barriers);
1343 encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, ®ions);
1344 }
1345 }
1346
1347 pending_writes.consume(staging_buffer);
1348 pending_writes.insert_texture(&dst);
1349
1350 Ok(())
1351 }
1352
1353 pub fn write_texture(
1354 &self,
1355 destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
1356 data: &[u8],
1357 data_layout: &wgt::TexelCopyBufferLayout,
1358 size: &wgt::Extent3d,
1359 ) {
1360 if let Err(error) = self.write_texture_inner(destination, data, data_layout, size) {
1361 self.device
1362 .handle_error(error, Some(self.label()), "Queue::write_texture");
1363 }
1364 }
1365
1366 #[cfg(webgl)]
1367 pub fn copy_external_image_to_texture(
1368 &self,
1369 source: &wgt::CopyExternalImageSourceInfo,
1370 destination: wgt::CopyExternalImageDestInfo<Arc<Texture>>,
1371 size: wgt::Extent3d,
1372 ) -> Result<(), QueueWriteError> {
1373 use crate::conv;
1374
1375 profiling::scope!("Queue::copy_external_image_to_texture");
1376
1377 self.device.check_is_valid()?;
1378
1379 let mut needs_flag = false;
1380 needs_flag |= source.origin != wgt::Origin2d::ZERO;
1384 needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
1385 #[allow(clippy::bool_comparison)]
1386 if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
1387 needs_flag |= source.flip_y != false;
1388 needs_flag |= destination.premultiplied_alpha != false;
1389 }
1390
1391 if needs_flag {
1392 self.device
1393 .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
1394 .map_err(TransferError::from)?;
1395 }
1396
1397 let src_width = source.source.width();
1398 let src_height = source.source.height();
1399
1400 let dst = destination.texture;
1401 let premultiplied_alpha = destination.premultiplied_alpha;
1402 let destination = wgt::TexelCopyTextureInfo {
1403 texture: (),
1404 mip_level: destination.mip_level,
1405 origin: destination.origin,
1406 aspect: destination.aspect,
1407 };
1408
1409 dst.check_valid()?;
1410
1411 if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
1412 return Err(
1413 TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
1414 );
1415 }
1416 if dst.desc.dimension != wgt::TextureDimension::D2 {
1417 return Err(TransferError::InvalidDimensionExternal.into());
1418 }
1419 dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
1420 .map_err(TransferError::MissingTextureUsage)?;
1421 if dst.desc.sample_count != 1 {
1422 return Err(TransferError::InvalidSampleCount {
1423 sample_count: dst.desc.sample_count,
1424 }
1425 .into());
1426 }
1427
1428 if source.origin.x > src_width || src_width - source.origin.x < size.width {
1429 return Err(TransferError::TextureOverrun {
1430 start_offset: source.origin.x,
1431 end_offset: source.origin.x.saturating_add(size.width),
1432 texture_size: src_width,
1433 dimension: crate::resource::TextureErrorDimension::X,
1434 side: CopySide::Source,
1435 plane: None,
1436 }
1437 .into());
1438 }
1439 if source.origin.y > src_height || src_height - source.origin.y < size.height {
1440 return Err(TransferError::TextureOverrun {
1441 start_offset: source.origin.y,
1442 end_offset: source.origin.y.saturating_add(size.height),
1443 texture_size: src_height,
1444 dimension: crate::resource::TextureErrorDimension::Y,
1445 side: CopySide::Source,
1446 plane: None,
1447 }
1448 .into());
1449 }
1450 if size.depth_or_array_layers != 1 {
1451 return Err(TransferError::TextureOverrun {
1452 start_offset: 0,
1453 end_offset: size.depth_or_array_layers,
1454 texture_size: 1,
1455 dimension: crate::resource::TextureErrorDimension::Z,
1456 side: CopySide::Source,
1457 plane: None,
1458 }
1459 .into());
1460 }
1461
1462 let CopyRange {
1465 hal_copy_extent,
1466 hal_base: dst_base,
1467 array_layer_count: _,
1468 selector,
1469 } = validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
1470
1471 assert!(dst_base.aspect == hal::FormatAspects::COLOR);
1473
1474 let snatch_guard = self.device.snatchable_lock.read();
1475
1476 let dst_raw = dst.try_raw(&snatch_guard)?;
1477
1478 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
1481 log::trace!("Ignoring copy_external_image_to_texture of size 0");
1482 return Ok(());
1483 }
1484
1485 let mut pending_writes = self.pending_writes.lock();
1486 let encoder = pending_writes.activate();
1487
1488 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1494 0..1
1496 } else {
1497 destination.origin.z..destination.origin.z + size.depth_or_array_layers
1498 };
1499 let layer_ranges_to_clear = {
1500 let mut dst_initialization_status = dst.initialization_status.write();
1501 if dst_initialization_status.mips[destination.mip_level as usize]
1502 .check(init_layer_range.clone())
1503 .is_some()
1504 {
1505 if has_copy_partial_init_tracker_coverage(&size, &destination, &dst.desc) {
1506 dst_initialization_status.mips[destination.mip_level as usize]
1507 .drain(init_layer_range)
1508 .collect::<Vec<core::ops::Range<u32>>>()
1509 } else {
1510 dst_initialization_status.mips[destination.mip_level as usize]
1511 .drain(init_layer_range);
1512 vec![]
1513 }
1514 } else {
1515 vec![]
1516 }
1517 };
1518 if !layer_ranges_to_clear.is_empty() {
1519 let mut trackers = self.device.trackers.lock();
1520 for layer_range in layer_ranges_to_clear {
1521 crate::command::clear_texture(
1522 &dst,
1523 TextureInitRange {
1524 mip_range: destination.mip_level..(destination.mip_level + 1),
1525 layer_range,
1526 },
1527 None,
1528 encoder,
1529 &mut trackers.textures,
1530 &self.device.alignments,
1531 self.device.zero_buffer.as_ref(),
1532 &snatch_guard,
1533 self.device.instance_flags,
1534 )
1535 .map_err(QueueWriteError::from)?;
1536 }
1537 }
1538
1539 let regions = hal::TextureCopy {
1540 src_base: hal::TextureCopyBase {
1541 mip_level: 0,
1542 array_layer: 0,
1543 origin: source.origin.to_3d(0),
1544 aspect: hal::FormatAspects::COLOR,
1545 },
1546 dst_base,
1547 size: hal_copy_extent,
1548 };
1549
1550 let mut trackers = self.device.trackers.lock();
1551 let transitions = trackers
1552 .textures
1553 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1554
1555 let encoder_webgl = encoder
1558 .as_any_mut()
1559 .downcast_mut::<hal::gles::CommandEncoder>()
1560 .unwrap();
1561 let dst_raw_webgl = dst_raw
1562 .as_any()
1563 .downcast_ref::<hal::gles::Texture>()
1564 .unwrap();
1565 let transitions_webgl = transitions.map(|pending| {
1566 let dyn_transition = pending.into_hal(dst_raw);
1567 hal::TextureBarrier {
1568 texture: dst_raw_webgl,
1569 range: dyn_transition.range,
1570 usage: dyn_transition.usage,
1571 queue_family_ownership_transfer: None,
1572 }
1573 });
1574
1575 use hal::CommandEncoder as _;
1576 unsafe {
1577 encoder_webgl.transition_textures(transitions_webgl);
1578 encoder_webgl.copy_external_image_to_texture(
1579 source,
1580 dst_raw_webgl,
1581 premultiplied_alpha,
1582 core::iter::once(regions),
1583 );
1584 }
1585
1586 pending_writes.insert_texture(&dst);
1587
1588 Ok(())
1589 }
1590
1591 pub fn flush_writes_for_buffer(
1593 &self,
1594 buffer: &Arc<Buffer>,
1595 snatch_guard: SnatchGuard,
1596 ) -> Result<(), BufferAccessError> {
1597 let submission = self.allocate_submission(snatch_guard)?;
1598
1599 let pending_writes = self.pending_writes.lock();
1600 if !pending_writes.contains_buffer(buffer) {
1601 return Ok(());
1602 }
1603
1604 submission.submit(pending_writes)?;
1605
1606 Ok(())
1607 }
1608
1609 fn flush_pending_writes(&self) -> Result<Option<SubmissionIndex>, DeviceError> {
1610 let snatch_guard = self.device.snatchable_lock.read();
1611 let submission = self.allocate_submission(snatch_guard)?;
1612 let submit_index = submission.index;
1613 let pending_writes = self.pending_writes.lock();
1614 if pending_writes.is_recording {
1615 submission.submit(pending_writes)?;
1616 Ok(Some(submit_index))
1617 } else {
1618 Ok(None)
1619 }
1620 }
1621
1622 #[cfg(feature = "trace")]
1623 fn trace_submission(
1624 &self,
1625 submit_index: SubmissionIndex,
1626 commands: Vec<crate::command::Command<crate::command::PointerReferences>>,
1627 ) {
1628 if let Some(ref mut trace) = *self.device.trace.lock() {
1629 trace.add(Action::Submit(submit_index, commands));
1630 }
1631 }
1632
1633 #[cfg(feature = "trace")]
1634 fn trace_failed_submission(
1635 &self,
1636 submit_index: SubmissionIndex,
1637 commands: Option<Vec<crate::command::Command<crate::command::PointerReferences>>>,
1638 error: String,
1639 ) {
1640 if let Some(ref mut trace) = *self.device.trace.lock() {
1641 trace.add(Action::FailedCommands {
1642 commands,
1643 failed_at_submit: Some(submit_index),
1644 error,
1645 });
1646 }
1647 }
1648
1649 fn submit_inner(
1650 &self,
1651 command_buffers: &[Arc<CommandBuffer>],
1652 ) -> Result<SubmissionIndex, Box<(Option<SubmissionIndex>, QueueSubmitError)>> {
1653 profiling::scope!("Queue::submit");
1654 api_log!("Queue::submit");
1655
1656 let snatch_guard = self.device.snatchable_lock.read();
1657 let mut submission = self
1658 .allocate_submission(snatch_guard)
1659 .map_err(|e| (None, e.into()))?;
1660 let submit_index = submission.index;
1661
1662 let mut lose_device_on_error = false;
1666 let mut successfully_submitted = false;
1667
1668 let res = 'error: {
1669 let mut used_surface_textures = track::TextureUsageScope::default();
1670 let mut baked_command_buffers = Vec::with_capacity(command_buffers.len());
1671
1672 if !command_buffers.is_empty() {
1673 profiling::scope!("prepare");
1674
1675 let mut first_error = None;
1676
1677 #[deny(clippy::question_mark_used)]
1680 for command_buffer in command_buffers {
1681 profiling::scope!("process command buffer");
1682
1683 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1686
1687 #[cfg_attr(not(feature = "trace"), expect(unused_mut))]
1691 let mut cmd_buf_data = command_buffer.take_finished();
1692
1693 if first_error.is_some() {
1694 continue;
1695 }
1696
1697 #[cfg(feature = "trace")]
1698 let trace_commands = cmd_buf_data
1699 .as_mut()
1700 .ok()
1701 .and_then(|data| mem::take(&mut data.trace_commands));
1702
1703 let mut baked = match cmd_buf_data {
1704 Ok(cmd_buf_data) => {
1705 let res = validate_command_buffer(
1706 command_buffer,
1707 self,
1708 &cmd_buf_data,
1709 &submission.snatch_guard,
1710 &mut submission.surface_textures,
1711 &mut used_surface_textures,
1712 &mut submission.command_index_guard,
1713 );
1714 if let Err(err) = res {
1715 #[cfg(feature = "trace")]
1716 self.trace_failed_submission(
1717 submit_index,
1718 trace_commands,
1719 err.to_string(),
1720 );
1721 first_error.get_or_insert(err);
1722 continue;
1723 }
1724
1725 #[cfg(feature = "trace")]
1726 if let Some(commands) = trace_commands {
1727 self.trace_submission(submit_index, commands);
1728 }
1729
1730 cmd_buf_data
1731 .set_acceleration_structure_dependencies(&submission.snatch_guard);
1732 cmd_buf_data.into_baked_commands()
1733 }
1734 Err(err) => {
1735 #[cfg(feature = "trace")]
1736 self.trace_failed_submission(
1737 submit_index,
1738 trace_commands,
1739 err.to_string(),
1740 );
1741 first_error.get_or_insert(err.into());
1742 continue;
1743 }
1744 };
1745
1746 if let Err(e) = baked
1747 .process_deferred_query_set_resolves(&self.device, &submission.snatch_guard)
1748 {
1749 break 'error Err(e.into());
1750 }
1751
1752 baked_command_buffers.push(baked);
1753 }
1754
1755 if let Some(first_error) = first_error {
1756 break 'error Err(first_error);
1757 }
1758
1759 lose_device_on_error = true;
1772
1773 let mut trackers = self.device.trackers.lock();
1775
1776 for mut baked in baked_command_buffers {
1777 profiling::scope!("process baked commands");
1778
1779 if let Err(e) = baked.encoder.open_pass(hal_label(
1781 Some("(wgpu internal) Transit"),
1782 self.device.instance_flags,
1783 )) {
1784 break 'error Err(e.into());
1785 }
1786
1787 baked.initialize_buffer_memory(&mut trackers, &submission.snatch_guard);
1788
1789 let depth_slice_discards = match baked.initialize_texture_memory(
1790 &mut trackers,
1791 &self.device,
1792 &submission.snatch_guard,
1793 ) {
1794 Ok(discards) => discards,
1795 Err(e) => {
1796 break 'error Err(QueueSubmitError::CommandEncoder(
1797 CommandEncoderError::Clear(e),
1798 ));
1799 }
1800 };
1801
1802 CommandEncoder::insert_barriers_from_device_tracker(
1805 baked.encoder.raw.as_mut(),
1806 &mut trackers,
1807 &baked.trackers,
1808 &submission.snatch_guard,
1809 );
1810
1811 if let Err(e) = baked.encoder.close_and_push_front() {
1812 break 'error Err(e.into());
1813 }
1814
1815 if !depth_slice_discards.is_empty() || !used_surface_textures.is_empty() {
1816 if let Err(e) = baked.encoder.open_pass(hal_label(
1817 Some("(wgpu internal) Finalize"),
1818 self.device.instance_flags,
1819 )) {
1820 break 'error Err(e.into());
1821 }
1822
1823 if let Err(e) = baked.initialize_discarded_depth_slices(
1824 depth_slice_discards,
1825 &mut trackers,
1826 &self.device,
1827 &submission.snatch_guard,
1828 ) {
1829 break 'error Err(QueueSubmitError::CommandEncoder(
1830 CommandEncoderError::Clear(e),
1831 ));
1832 }
1833
1834 let texture_barriers = trackers
1838 .textures
1839 .set_from_usage_scope_and_drain_transitions(
1840 &used_surface_textures,
1841 &submission.snatch_guard,
1842 )
1843 .collect::<Vec<_>>();
1844 unsafe {
1845 baked.encoder.raw.transition_textures(&texture_barriers);
1846 };
1847 if let Err(e) = baked.encoder.close() {
1848 break 'error Err(e.into());
1849 }
1850 used_surface_textures = track::TextureUsageScope::default();
1851 }
1852
1853 submission.executions.push(EncoderInFlight {
1855 inner: baked.encoder,
1856 trackers: baked.trackers,
1857 temp_resources: baked.temp_resources,
1858 _indirect_draw_validation_resources: baked
1859 .indirect_draw_validation_resources,
1860 pending_buffers: FastHashMap::default(),
1861 pending_textures: FastHashMap::default(),
1862 pending_blas_s: FastHashMap::default(),
1863 _pending_shader_binding_data: Vec::new(),
1864 });
1865 }
1866 }
1867
1868 let pending_writes = self.pending_writes.lock();
1869
1870 let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) {
1871 Ok(result) => result,
1872 Err(e) => break 'error Err(e.into()),
1873 };
1874 successfully_submitted = true;
1875
1876 profiling::scope!("cleanup");
1877
1878 lose_device_on_error = false;
1882
1883 let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard);
1887 match result {
1888 Ok(status) => {
1889 debug_assert!(matches!(
1890 status,
1891 wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1892 ));
1893 }
1894 Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1895 Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1896 unreachable!("Cannot get WrongSubmissionIndex from Poll")
1897 }
1898 Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1899 };
1900
1901 Ok(closures)
1902 }; let callbacks = match res {
1905 Ok(ok) => ok,
1906 Err(e) => {
1907 if lose_device_on_error {
1908 self.device.lose("submission failed");
1909 }
1910 return Err(Box::new((
1911 successfully_submitted.then_some(submit_index),
1912 e,
1913 )));
1914 }
1915 };
1916
1917 callbacks.fire();
1919
1920 self.device.lose_if_oom();
1921
1922 api_log!("Queue::submit returned submit index {submit_index}");
1923
1924 Ok(submit_index)
1925 }
1926
1927 pub fn submit(&self, command_buffers: &[Arc<CommandBuffer>]) -> Option<SubmissionIndex> {
1928 match self.submit_inner(command_buffers) {
1929 Ok(submit_index) => Some(submit_index),
1930 Err(e) => {
1931 let (submit_index, e) = *e;
1932 self.device
1933 .handle_error(e, Some(self.label()), "Queue::submit");
1934 submit_index
1935 }
1936 }
1937 }
1938
1939 fn allocate_submission<'a>(
1962 &'a self,
1963 snatch_guard: SnatchGuard<'a>,
1964 ) -> Result<PendingSubmission<'a>, DeviceError> {
1965 let mut command_index_guard = self.device.command_indices.write();
1966 command_index_guard.active_submission_index += 1;
1967 let index = command_index_guard.active_submission_index;
1968
1969 self.device.check_is_valid()?;
1970
1971 let submission = PendingSubmission {
1972 queue: self,
1973 command_index_guard,
1974 snatch_guard,
1975 executions: Vec::new(),
1976 surface_textures: FastHashMap::default(),
1977 index,
1978 };
1979
1980 Ok(submission)
1981 }
1982
1983 fn submit_pending_submission<'a>(
1999 &self,
2000 mut pending_writes: MutexGuard<'_, PendingWrites>,
2001 prepared: PendingSubmission<'a>,
2002 ) -> Result<SubmissionResult<'a>, DeviceError> {
2003 let PendingSubmission {
2004 queue: _,
2005 snatch_guard,
2006 command_index_guard,
2007 mut executions,
2008 mut surface_textures,
2009 index: submit_index,
2010 } = prepared;
2011
2012 let mut used_surface_textures = track::TextureUsageScope::default();
2013 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
2014 for texture in pending_writes.dst_textures.values() {
2015 match texture.try_inner(&snatch_guard) {
2016 Ok(TextureInner::Native { .. }) => {}
2017 Ok(TextureInner::Surface { .. }) => {
2018 surface_textures.insert(Arc::as_ptr(texture), texture.clone());
2020
2021 unsafe {
2022 used_surface_textures
2023 .merge_single(texture, None, wgt::TextureUses::PRESENT)
2024 .unwrap()
2025 };
2026 }
2027 Err(InvalidOrDestroyedResourceError::DestroyedResource(_)) => {}
2032 Err(InvalidOrDestroyedResourceError::InvalidResource(_)) => {
2033 unreachable!()
2034 }
2035 }
2036 }
2037
2038 if !used_surface_textures.is_empty() {
2039 let mut trackers = self.device.trackers.lock();
2040
2041 let texture_barriers = trackers
2042 .textures
2043 .set_from_usage_scope_and_drain_transitions(&used_surface_textures, &snatch_guard)
2044 .collect::<Vec<_>>();
2045 unsafe {
2046 pending_writes
2047 .command_encoder
2048 .transition_textures(&texture_barriers);
2049 };
2050 }
2051
2052 match pending_writes.pre_submit(
2053 &self.device.command_allocator,
2054 &self.device,
2055 self,
2056 &snatch_guard,
2057 ) {
2058 Ok(Some(pending_execution)) => {
2059 executions.insert(0, pending_execution);
2060 }
2061 Ok(None) => {}
2062 Err(e) => return Err(e),
2063 }
2064 let hal_command_buffers = executions
2065 .iter()
2066 .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
2067 .collect::<Vec<_>>();
2068
2069 {
2070 let mut submit_surface_textures =
2071 SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(surface_textures.len());
2072 for texture in surface_textures.values() {
2073 let raw = match texture.try_inner(&snatch_guard).ok() {
2074 Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
2075 _ => unreachable!(),
2076 };
2077 submit_surface_textures.push(raw);
2078 }
2079
2080 unsafe {
2081 self.raw().submit(
2082 &hal_command_buffers,
2083 &submit_surface_textures,
2084 (self.device.fence.as_ref(), submit_index),
2085 )
2086 }
2087 .map_err(|e| self.device.handle_hal_error(e))?;
2088
2089 drop(pending_writes);
2093
2094 self.device
2096 .last_successful_submission_index
2097 .fetch_max(submit_index, Ordering::SeqCst);
2098 }
2099
2100 self.lock_life().track_submission(submit_index, executions);
2102
2103 drop(command_index_guard);
2109
2110 Ok(SubmissionResult { snatch_guard })
2111 }
2112
2113 pub(crate) fn get_raw_timestamp_period(&self) -> f32 {
2114 unsafe { self.raw().get_timestamp_period() }
2115 }
2116
2117 pub fn get_timestamp_period(&self) -> f32 {
2118 if self.device.timestamp_normalizer.get().unwrap().enabled() {
2119 return 1.0;
2120 }
2121
2122 self.get_raw_timestamp_period()
2123 }
2124
2125 pub fn on_submitted_work_done(
2127 &self,
2128 closure: SubmittedWorkDoneClosure,
2129 ) -> Option<SubmissionIndex> {
2130 api_log!("Queue::on_submitted_work_done");
2131
2132 let _: Result<_, DeviceError> = self.flush_pending_writes();
2136
2137 self.lock_life().add_work_done_closure(closure)
2138 }
2139
2140 #[allow(trivial_casts)]
2141 pub fn compact_blas(&self, blas: &Arc<Blas>) -> (Arc<Blas>, Option<CompactBlasError>) {
2142 api_log!(
2143 "Queue::compact_blas {:?}, {:?}",
2144 self as *const _,
2145 Arc::as_ptr(blas)
2146 );
2147
2148 let (blas, error) = match self.compact_blas_inner(blas) {
2149 Ok(blas) => (blas, None),
2150 Err(err) => {
2151 let new_label = blas.label.clone() + " (compacted)";
2152 (
2153 Blas::invalid(
2154 self.device.clone(),
2155 &BlasDescriptor {
2156 label: Some(new_label.into()),
2157 flags: blas.flags,
2158 update_mode: blas.update_mode,
2159 },
2160 ),
2161 Some(err),
2162 )
2163 }
2164 };
2165
2166 (blas, error)
2169 }
2170
2171 pub(crate) fn compact_blas_inner(
2172 &self,
2173 blas: &Arc<Blas>,
2174 ) -> Result<Arc<Blas>, CompactBlasError> {
2175 profiling::scope!("Queue::compact_blas");
2176 api_log!("Queue::compact_blas");
2177
2178 let new_label = blas.label.clone() + " (compacted)";
2179
2180 self.device.check_is_valid()?;
2181
2182 self.device.require_acceleration_structures()?;
2183
2184 blas.check_is_valid()?;
2185 self.same_device_as(blas.as_ref())?;
2186
2187 let device = blas.device.clone();
2188
2189 let snatch_guard = device.snatchable_lock.read();
2190
2191 let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
2192 return Err(CompactBlasError::BlasNotReady);
2193 };
2194
2195 let mut size_info = blas.size_info;
2196 size_info.acceleration_structure_size = size;
2197
2198 let mut command_indices_lock = device.command_indices.write();
2199 let mut pending_writes = self.pending_writes.lock();
2200 let cmd_buf_raw = pending_writes.activate();
2201
2202 let raw = unsafe {
2203 device
2204 .raw()
2205 .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
2206 label: hal_label(Some(&new_label), device.instance_flags),
2207 size: size_info.acceleration_structure_size,
2208 format: hal::AccelerationStructureFormat::BottomLevel,
2209 allow_compaction: false,
2210 })
2211 }
2212 .map_err(DeviceError::from_hal)?;
2213
2214 let src_raw = blas.try_raw(&snatch_guard)?;
2215
2216 unsafe {
2217 cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
2218 src_raw,
2219 raw.as_ref(),
2220 wgt::AccelerationStructureCopy::Compact,
2221 )
2222 };
2223
2224 let handle = unsafe {
2225 device
2226 .raw()
2227 .get_acceleration_structure_device_address(raw.as_ref())
2228 };
2229
2230 command_indices_lock.next_acceleration_structure_build_command_index += 1;
2231 let built_index =
2232 NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
2233 .unwrap();
2234
2235 let new_blas = Arc::new(Blas {
2236 state: ResourceState::Valid(BlasState {
2237 raw: Snatchable::new(raw),
2238 }),
2239 device: device.clone(),
2240 size_info,
2241 sizes: blas.sizes.clone(),
2242 flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
2243 update_mode: blas.update_mode,
2244 built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
2246 handle,
2247 label: new_label,
2248 tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
2249 compaction_buffer: None,
2250 compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
2251 });
2252
2253 pending_writes.insert_blas(blas);
2254 pending_writes.insert_blas(&new_blas);
2255
2256 let old_blas_size = blas.size_info.acceleration_structure_size;
2258 let new_blas_size = new_blas.size_info.acceleration_structure_size;
2259
2260 api_log!("CommandEncoder::compact_blas {:?} (size: {old_blas_size}) -> {:?} (size: {new_blas_size})", Arc::as_ptr(blas), Arc::as_ptr(&new_blas));
2261
2262 Ok(new_blas)
2263 }
2264}
2265
2266fn validate_command_buffer(
2267 command_buffer: &CommandBuffer,
2268 queue: &Queue,
2269 cmd_buf_data: &crate::command::CommandBufferMutable,
2270 snatch_guard: &SnatchGuard,
2271 surface_textures: &mut FastHashMap<*const Texture, Arc<Texture>>,
2272 used_surface_textures: &mut track::TextureUsageScope,
2273 command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
2274) -> Result<(), QueueSubmitError> {
2275 command_buffer.same_device_as(queue)?;
2276
2277 {
2278 profiling::scope!("check resource state");
2279
2280 {
2281 profiling::scope!("buffers");
2282 for buffer in cmd_buf_data.trackers.buffers.used_resources() {
2283 buffer.check_destroyed(snatch_guard)?;
2284
2285 match *buffer.map_state.read() {
2286 BufferMapState::Idle => (),
2287 _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
2288 }
2289 }
2290 }
2291 {
2292 profiling::scope!("textures");
2293 for texture in cmd_buf_data.trackers.textures.used_resources() {
2294 let should_extend = match texture.try_inner(snatch_guard)? {
2295 TextureInner::Native { .. } => false,
2296 TextureInner::Surface { .. } => {
2297 surface_textures.insert(Arc::as_ptr(texture), texture.clone());
2299
2300 true
2301 }
2302 };
2303 if should_extend {
2304 unsafe {
2305 used_surface_textures
2306 .merge_single(texture, None, wgt::TextureUses::PRESENT)
2307 .unwrap();
2308 };
2309 }
2310 }
2311 }
2312 {
2313 profiling::scope!("query sets");
2314 for query_set in cmd_buf_data.trackers.query_sets.used_resources() {
2315 query_set.try_raw(snatch_guard)?;
2316 }
2317 }
2318 {
2322 profiling::scope!("bind groups");
2323 for bind_group in &cmd_buf_data.trackers.bind_groups {
2324 bind_group.try_raw(snatch_guard)?;
2326 }
2327 }
2328
2329 if let Err(e) =
2330 cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
2331 {
2332 return Err(e.into());
2333 }
2334 }
2335 Ok(())
2336}