1use alloc::{format, string::String, sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6 error::{ErrorType, WebGpuError},
7 BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8 TextureUsages,
9};
10
11#[cfg(feature = "trace")]
12use crate::device::trace::Command as TraceCommand;
13use crate::{
14 api_log,
15 command::{clear_texture, CommandEncoderError, EncoderStateError},
16 device::{Device, MissingDownlevelFlags},
17 global::Global,
18 id::{BufferId, CommandEncoderId, TextureId},
19 init_tracker::{
20 has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
21 TextureInitTrackerAction,
22 },
23 resource::{
24 MissingBufferUsageError, MissingTextureUsageError, ParentDevice, RawResourceAccess,
25 Texture, TextureErrorDimension,
26 },
27 snatch::SnatchGuard,
28};
29
30use super::{ClearError, CommandBufferMutable};
31
32pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo<BufferId>;
33pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<TextureId>;
34pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo<TextureId>;
35
36#[derive(Clone, Copy, Debug)]
37pub enum CopySide {
38 Source,
39 Destination,
40}
41
42#[derive(Clone, Debug, Error)]
44#[non_exhaustive]
45pub enum TransferError {
46 #[error("Source and destination cannot be the same buffer")]
47 SameSourceDestinationBuffer,
48 #[error(transparent)]
49 MissingBufferUsage(#[from] MissingBufferUsageError),
50 #[error(transparent)]
51 MissingTextureUsage(#[from] MissingTextureUsageError),
52 #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
53 BufferOverrun {
54 start_offset: BufferAddress,
55 end_offset: BufferAddress,
56 buffer_size: BufferAddress,
57 side: CopySide,
58 },
59 #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
60 TextureOverrun {
61 start_offset: u32,
62 end_offset: u32,
63 texture_size: u32,
64 dimension: TextureErrorDimension,
65 side: CopySide,
66 },
67 #[error("Partial copy of {start_offset}..{end_offset} on {dimension:?} dimension with size {texture_size} \
68 is not supported for the {side:?} texture format {format:?} with {sample_count} samples")]
69 UnsupportedPartialTransfer {
70 format: wgt::TextureFormat,
71 sample_count: u32,
72 start_offset: u32,
73 end_offset: u32,
74 texture_size: u32,
75 dimension: TextureErrorDimension,
76 side: CopySide,
77 },
78 #[error(
79 "Copying{} layers {}..{} to{} layers {}..{} of the same texture is not allowed",
80 if *src_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {src_aspects:?}") },
81 src_origin_z,
82 src_origin_z + array_layer_count,
83 if *dst_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {dst_aspects:?}") },
84 dst_origin_z,
85 dst_origin_z + array_layer_count,
86 )]
87 InvalidCopyWithinSameTexture {
88 src_aspects: wgt::TextureAspect,
89 dst_aspects: wgt::TextureAspect,
90 src_origin_z: u32,
91 dst_origin_z: u32,
92 array_layer_count: u32,
93 },
94 #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
95 InvalidTextureAspect {
96 format: wgt::TextureFormat,
97 aspect: wgt::TextureAspect,
98 },
99 #[error("Unable to select texture mip level {level} out of {total}")]
100 InvalidTextureMipLevel { level: u32, total: u32 },
101 #[error("Texture dimension must be 2D when copying from an external texture")]
102 InvalidDimensionExternal,
103 #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
104 UnalignedBufferOffset(BufferAddress),
105 #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
106 UnalignedCopySize(BufferAddress),
107 #[error("Copy width is not a multiple of block width")]
108 UnalignedCopyWidth,
109 #[error("Copy height is not a multiple of block height")]
110 UnalignedCopyHeight,
111 #[error("Copy origin's x component is not a multiple of block width")]
112 UnalignedCopyOriginX,
113 #[error("Copy origin's y component is not a multiple of block height")]
114 UnalignedCopyOriginY,
115 #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
116 UnalignedBytesPerRow,
117 #[error("Number of bytes per row needs to be specified since more than one row is copied")]
118 UnspecifiedBytesPerRow,
119 #[error("Number of rows per image needs to be specified since more than one image is copied")]
120 UnspecifiedRowsPerImage,
121 #[error("Number of bytes per row is less than the number of bytes in a complete row")]
122 InvalidBytesPerRow,
123 #[error("Number of rows per image is invalid")]
124 InvalidRowsPerImage,
125 #[error("Overflow while computing the size of the copy")]
126 SizeOverflow,
127 #[error("Copy source aspects must refer to all aspects of the source texture format")]
128 CopySrcMissingAspects,
129 #[error(
130 "Copy destination aspects must refer to all aspects of the destination texture format"
131 )]
132 CopyDstMissingAspects,
133 #[error("Copy aspect must refer to a single aspect of texture format")]
134 CopyAspectNotOne,
135 #[error("Copying from textures with format {0:?} is forbidden")]
136 CopyFromForbiddenTextureFormat(wgt::TextureFormat),
137 #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
138 CopyFromForbiddenTextureFormatAspect {
139 format: wgt::TextureFormat,
140 aspect: wgt::TextureAspect,
141 },
142 #[error("Copying to textures with format {0:?} is forbidden")]
143 CopyToForbiddenTextureFormat(wgt::TextureFormat),
144 #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
145 CopyToForbiddenTextureFormatAspect {
146 format: wgt::TextureFormat,
147 aspect: wgt::TextureAspect,
148 },
149 #[error(
150 "Copying to textures with format {0:?} is forbidden when copying from external texture"
151 )]
152 ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
153 #[error(
154 "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
155 )]
156 TextureFormatsNotCopyCompatible {
157 src_format: wgt::TextureFormat,
158 dst_format: wgt::TextureFormat,
159 },
160 #[error(transparent)]
161 MemoryInitFailure(#[from] ClearError),
162 #[error("Cannot encode this copy because of a missing downelevel flag")]
163 MissingDownlevelFlags(#[from] MissingDownlevelFlags),
164 #[error("Source texture sample count must be 1, got {sample_count}")]
165 InvalidSampleCount { sample_count: u32 },
166 #[error(
167 "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
168 )]
169 SampleCountNotEqual {
170 src_sample_count: u32,
171 dst_sample_count: u32,
172 },
173 #[error("Requested mip level {requested} does no exist (count: {count})")]
174 InvalidMipLevel { requested: u32, count: u32 },
175}
176
177impl WebGpuError for TransferError {
178 fn webgpu_error_type(&self) -> ErrorType {
179 let e: &dyn WebGpuError = match self {
180 Self::MissingBufferUsage(e) => e,
181 Self::MissingTextureUsage(e) => e,
182 Self::MemoryInitFailure(e) => e,
183
184 Self::BufferOverrun { .. }
185 | Self::TextureOverrun { .. }
186 | Self::UnsupportedPartialTransfer { .. }
187 | Self::InvalidCopyWithinSameTexture { .. }
188 | Self::InvalidTextureAspect { .. }
189 | Self::InvalidTextureMipLevel { .. }
190 | Self::InvalidDimensionExternal
191 | Self::UnalignedBufferOffset(..)
192 | Self::UnalignedCopySize(..)
193 | Self::UnalignedCopyWidth
194 | Self::UnalignedCopyHeight
195 | Self::UnalignedCopyOriginX
196 | Self::UnalignedCopyOriginY
197 | Self::UnalignedBytesPerRow
198 | Self::UnspecifiedBytesPerRow
199 | Self::UnspecifiedRowsPerImage
200 | Self::InvalidBytesPerRow
201 | Self::InvalidRowsPerImage
202 | Self::SizeOverflow
203 | Self::CopySrcMissingAspects
204 | Self::CopyDstMissingAspects
205 | Self::CopyAspectNotOne
206 | Self::CopyFromForbiddenTextureFormat(..)
207 | Self::CopyFromForbiddenTextureFormatAspect { .. }
208 | Self::CopyToForbiddenTextureFormat(..)
209 | Self::CopyToForbiddenTextureFormatAspect { .. }
210 | Self::ExternalCopyToForbiddenTextureFormat(..)
211 | Self::TextureFormatsNotCopyCompatible { .. }
212 | Self::MissingDownlevelFlags(..)
213 | Self::InvalidSampleCount { .. }
214 | Self::SampleCountNotEqual { .. }
215 | Self::InvalidMipLevel { .. }
216 | Self::SameSourceDestinationBuffer => return ErrorType::Validation,
217 };
218 e.webgpu_error_type()
219 }
220}
221
222impl From<BufferTextureCopyInfoError> for TransferError {
223 fn from(value: BufferTextureCopyInfoError) -> Self {
224 match value {
225 BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
226 BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
227 BufferTextureCopyInfoError::ImageStrideOverflow
228 | BufferTextureCopyInfoError::ImageBytesOverflow(_)
229 | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
230 }
231 }
232}
233
234pub(crate) fn extract_texture_selector<T>(
235 copy_texture: &wgt::TexelCopyTextureInfo<T>,
236 copy_size: &Extent3d,
237 texture: &Texture,
238) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
239 let format = texture.desc.format;
240 let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
241 if copy_aspect.is_empty() {
242 return Err(TransferError::InvalidTextureAspect {
243 format,
244 aspect: copy_texture.aspect,
245 });
246 }
247
248 let (layers, origin_z) = match texture.desc.dimension {
249 wgt::TextureDimension::D1 => (0..1, 0),
250 wgt::TextureDimension::D2 => (
251 copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
252 0,
253 ),
254 wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
255 };
256 let base = hal::TextureCopyBase {
257 origin: wgt::Origin3d {
258 x: copy_texture.origin.x,
259 y: copy_texture.origin.y,
260 z: origin_z,
261 },
262 array_layer: layers.start,
264 mip_level: copy_texture.mip_level,
265 aspect: copy_aspect,
266 };
267 let selector = TextureSelector {
268 mips: copy_texture.mip_level..copy_texture.mip_level + 1,
269 layers,
270 };
271
272 Ok((selector, base))
273}
274
275pub(crate) fn validate_linear_texture_data(
285 layout: &wgt::TexelCopyBufferLayout,
286 format: wgt::TextureFormat,
287 aspect: wgt::TextureAspect,
288 buffer_size: BufferAddress,
289 buffer_side: CopySide,
290 copy_size: &Extent3d,
291 need_copy_aligned_rows: bool,
292) -> Result<(BufferAddress, BufferAddress), TransferError> {
293 let wgt::BufferTextureCopyInfo {
294 copy_width,
295 copy_height,
296 depth_or_array_layers,
297
298 offset,
299
300 block_size_bytes,
301 block_width_texels,
302 block_height_texels,
303
304 width_blocks: _,
305 height_blocks,
306
307 row_bytes_dense: _,
308 row_stride_bytes,
309
310 image_stride_rows: _,
311 image_stride_bytes,
312
313 image_rows_dense: _,
314 image_bytes_dense: _,
315
316 bytes_in_copy,
317 } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
318
319 if copy_width % block_width_texels != 0 {
320 return Err(TransferError::UnalignedCopyWidth);
321 }
322 if copy_height % block_height_texels != 0 {
323 return Err(TransferError::UnalignedCopyHeight);
324 }
325
326 let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
327 let requires_multiple_images = depth_or_array_layers > 1;
328
329 if layout.bytes_per_row.is_none() && requires_multiple_rows {
334 return Err(TransferError::UnspecifiedBytesPerRow);
335 }
336
337 if layout.rows_per_image.is_none() && requires_multiple_images {
338 return Err(TransferError::UnspecifiedRowsPerImage);
339 };
340
341 if need_copy_aligned_rows {
342 let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
343
344 let mut offset_alignment = block_size_bytes;
345 if format.is_depth_stencil_format() {
346 offset_alignment = 4
347 }
348 if offset % offset_alignment != 0 {
349 return Err(TransferError::UnalignedBufferOffset(offset));
350 }
351
352 if requires_multiple_rows && row_stride_bytes % bytes_per_row_alignment != 0 {
355 return Err(TransferError::UnalignedBytesPerRow);
356 }
357 }
358
359 if bytes_in_copy > buffer_size || offset > buffer_size - bytes_in_copy {
361 return Err(TransferError::BufferOverrun {
362 start_offset: offset,
363 end_offset: offset.wrapping_add(bytes_in_copy),
364 buffer_size,
365 side: buffer_side,
366 });
367 }
368
369 Ok((bytes_in_copy, image_stride_bytes))
370}
371
372pub(crate) fn validate_texture_copy_src_format(
381 format: wgt::TextureFormat,
382 aspect: wgt::TextureAspect,
383) -> Result<(), TransferError> {
384 use wgt::TextureAspect as Ta;
385 use wgt::TextureFormat as Tf;
386 match (format, aspect) {
387 (Tf::Depth24Plus, _) => Err(TransferError::CopyFromForbiddenTextureFormat(format)),
388 (Tf::Depth24PlusStencil8, Ta::DepthOnly) => {
389 Err(TransferError::CopyFromForbiddenTextureFormatAspect { format, aspect })
390 }
391 _ => Ok(()),
392 }
393}
394
395pub(crate) fn validate_texture_copy_dst_format(
404 format: wgt::TextureFormat,
405 aspect: wgt::TextureAspect,
406) -> Result<(), TransferError> {
407 use wgt::TextureAspect as Ta;
408 use wgt::TextureFormat as Tf;
409 match (format, aspect) {
410 (Tf::Depth24Plus | Tf::Depth32Float, _) => {
411 Err(TransferError::CopyToForbiddenTextureFormat(format))
412 }
413 (Tf::Depth24PlusStencil8 | Tf::Depth32FloatStencil8, Ta::DepthOnly) => {
414 Err(TransferError::CopyToForbiddenTextureFormatAspect { format, aspect })
415 }
416 _ => Ok(()),
417 }
418}
419
420pub(crate) fn validate_texture_buffer_copy<T>(
439 texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
440 aspect: hal::FormatAspects,
441 desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
442 offset: BufferAddress,
443 aligned: bool,
444) -> Result<(), TransferError> {
445 if desc.sample_count != 1 {
446 return Err(TransferError::InvalidSampleCount {
447 sample_count: desc.sample_count,
448 });
449 }
450
451 if !aspect.is_one() {
452 return Err(TransferError::CopyAspectNotOne);
453 }
454
455 let offset_alignment = if desc.format.is_depth_stencil_format() {
456 4
457 } else {
458 desc.format
463 .block_copy_size(Some(texture_copy_view.aspect))
464 .expect("non-copyable formats should have been rejected previously")
465 };
466
467 if aligned && offset % u64::from(offset_alignment) != 0 {
468 return Err(TransferError::UnalignedBufferOffset(offset));
469 }
470
471 Ok(())
472}
473
474pub(crate) fn validate_texture_copy_range<T>(
485 texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
486 desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
487 texture_side: CopySide,
488 copy_size: &Extent3d,
489) -> Result<(hal::CopyExtent, u32), TransferError> {
490 let (block_width, block_height) = desc.format.block_dimensions();
491
492 let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
493 TransferError::InvalidTextureMipLevel {
494 level: texture_copy_view.mip_level,
495 total: desc.mip_level_count,
496 },
497 )?;
498 let extent = extent_virtual.physical_size(desc.format);
500
501 let requires_exact_size = desc.format.is_depth_stencil_format() || desc.sample_count > 1;
504
505 let check_dimension = |dimension: TextureErrorDimension,
508 start_offset: u32,
509 size: u32,
510 texture_size: u32,
511 requires_exact_size: bool|
512 -> Result<(), TransferError> {
513 if requires_exact_size && (start_offset != 0 || size != texture_size) {
514 Err(TransferError::UnsupportedPartialTransfer {
515 format: desc.format,
516 sample_count: desc.sample_count,
517 start_offset,
518 end_offset: start_offset.wrapping_add(size),
519 texture_size,
520 dimension,
521 side: texture_side,
522 })
523 } else if start_offset > texture_size || texture_size - start_offset < size {
526 Err(TransferError::TextureOverrun {
527 start_offset,
528 end_offset: start_offset.wrapping_add(size),
529 texture_size,
530 dimension,
531 side: texture_side,
532 })
533 } else {
534 Ok(())
535 }
536 };
537
538 check_dimension(
539 TextureErrorDimension::X,
540 texture_copy_view.origin.x,
541 copy_size.width,
542 extent.width,
543 requires_exact_size,
544 )?;
545 check_dimension(
546 TextureErrorDimension::Y,
547 texture_copy_view.origin.y,
548 copy_size.height,
549 extent.height,
550 requires_exact_size,
551 )?;
552 check_dimension(
553 TextureErrorDimension::Z,
554 texture_copy_view.origin.z,
555 copy_size.depth_or_array_layers,
556 extent.depth_or_array_layers,
557 false, )?;
559
560 if texture_copy_view.origin.x % block_width != 0 {
561 return Err(TransferError::UnalignedCopyOriginX);
562 }
563 if texture_copy_view.origin.y % block_height != 0 {
564 return Err(TransferError::UnalignedCopyOriginY);
565 }
566 if copy_size.width % block_width != 0 {
567 return Err(TransferError::UnalignedCopyWidth);
568 }
569 if copy_size.height % block_height != 0 {
570 return Err(TransferError::UnalignedCopyHeight);
571 }
572
573 let (depth, array_layer_count) = match desc.dimension {
574 wgt::TextureDimension::D1 => (1, 1),
575 wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
576 wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
577 };
578
579 let copy_extent = hal::CopyExtent {
580 width: copy_size.width,
581 height: copy_size.height,
582 depth,
583 };
584 Ok((copy_extent, array_layer_count))
585}
586
587pub(crate) fn validate_copy_within_same_texture<T>(
598 src: &wgt::TexelCopyTextureInfo<T>,
599 dst: &wgt::TexelCopyTextureInfo<T>,
600 format: wgt::TextureFormat,
601 array_layer_count: u32,
602) -> Result<(), TransferError> {
603 let src_aspects = hal::FormatAspects::new(format, src.aspect);
604 let dst_aspects = hal::FormatAspects::new(format, dst.aspect);
605 if (src_aspects & dst_aspects).is_empty() {
606 return Ok(());
608 }
609
610 if src.origin.z >= dst.origin.z + array_layer_count
611 || dst.origin.z >= src.origin.z + array_layer_count
612 {
613 return Ok(());
615 }
616
617 if src.mip_level != dst.mip_level {
618 return Ok(());
620 }
621
622 Err(TransferError::InvalidCopyWithinSameTexture {
623 src_aspects: src.aspect,
624 dst_aspects: dst.aspect,
625 src_origin_z: src.origin.z,
626 dst_origin_z: dst.origin.z,
627 array_layer_count,
628 })
629}
630
631fn handle_texture_init(
632 init_kind: MemoryInitKind,
633 cmd_buf_data: &mut CommandBufferMutable,
634 device: &Device,
635 copy_texture: &TexelCopyTextureInfo,
636 copy_size: &Extent3d,
637 texture: &Arc<Texture>,
638 snatch_guard: &SnatchGuard<'_>,
639) -> Result<(), ClearError> {
640 let init_action = TextureInitTrackerAction {
641 texture: texture.clone(),
642 range: TextureInitRange {
643 mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
644 layer_range: copy_texture.origin.z
645 ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
646 },
647 kind: init_kind,
648 };
649
650 let immediate_inits = cmd_buf_data
652 .texture_memory_actions
653 .register_init_action(&{ init_action });
654
655 if !immediate_inits.is_empty() {
657 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
658 for init in immediate_inits {
659 clear_texture(
660 &init.texture,
661 TextureInitRange {
662 mip_range: init.mip_level..(init.mip_level + 1),
663 layer_range: init.layer..(init.layer + 1),
664 },
665 cmd_buf_raw,
666 &mut cmd_buf_data.trackers.textures,
667 &device.alignments,
668 device.zero_buffer.as_ref(),
669 snatch_guard,
670 device.instance_flags,
671 )?;
672 }
673 }
674
675 Ok(())
676}
677
678fn handle_src_texture_init(
683 cmd_buf_data: &mut CommandBufferMutable,
684 device: &Device,
685 source: &TexelCopyTextureInfo,
686 copy_size: &Extent3d,
687 texture: &Arc<Texture>,
688 snatch_guard: &SnatchGuard<'_>,
689) -> Result<(), TransferError> {
690 handle_texture_init(
691 MemoryInitKind::NeedsInitializedMemory,
692 cmd_buf_data,
693 device,
694 source,
695 copy_size,
696 texture,
697 snatch_guard,
698 )?;
699 Ok(())
700}
701
702fn handle_dst_texture_init(
707 cmd_buf_data: &mut CommandBufferMutable,
708 device: &Device,
709 destination: &TexelCopyTextureInfo,
710 copy_size: &Extent3d,
711 texture: &Arc<Texture>,
712 snatch_guard: &SnatchGuard<'_>,
713) -> Result<(), TransferError> {
714 let dst_init_kind = if has_copy_partial_init_tracker_coverage(
719 copy_size,
720 destination.mip_level,
721 &texture.desc,
722 ) {
723 MemoryInitKind::NeedsInitializedMemory
724 } else {
725 MemoryInitKind::ImplicitlyInitialized
726 };
727
728 handle_texture_init(
729 dst_init_kind,
730 cmd_buf_data,
731 device,
732 destination,
733 copy_size,
734 texture,
735 snatch_guard,
736 )?;
737 Ok(())
738}
739
740impl Global {
741 pub fn command_encoder_copy_buffer_to_buffer(
742 &self,
743 command_encoder_id: CommandEncoderId,
744 source: BufferId,
745 source_offset: BufferAddress,
746 destination: BufferId,
747 destination_offset: BufferAddress,
748 size: Option<BufferAddress>,
749 ) -> Result<(), EncoderStateError> {
750 profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
751 api_log!(
752 "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
753 );
754
755 let hub = &self.hub;
756
757 let cmd_enc = hub.command_encoders.get(command_encoder_id);
758 let mut cmd_buf_data = cmd_enc.data.lock();
759 cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
760 let device = &cmd_enc.device;
761 device.check_is_valid()?;
762
763 if source == destination {
764 return Err(TransferError::SameSourceDestinationBuffer.into());
765 }
766
767 #[cfg(feature = "trace")]
768 if let Some(ref mut list) = cmd_buf_data.commands {
769 list.push(TraceCommand::CopyBufferToBuffer {
770 src: source,
771 src_offset: source_offset,
772 dst: destination,
773 dst_offset: destination_offset,
774 size,
775 });
776 }
777
778 let snatch_guard = device.snatchable_lock.read();
779
780 let src_buffer = hub.buffers.get(source).get()?;
781
782 src_buffer.same_device_as(cmd_enc.as_ref())?;
783
784 let src_pending = cmd_buf_data
785 .trackers
786 .buffers
787 .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
788
789 let src_raw = src_buffer.try_raw(&snatch_guard)?;
790 src_buffer
791 .check_usage(BufferUsages::COPY_SRC)
792 .map_err(TransferError::MissingBufferUsage)?;
793 let src_barrier =
795 src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
796
797 let dst_buffer = hub.buffers.get(destination).get()?;
798
799 dst_buffer.same_device_as(cmd_enc.as_ref())?;
800
801 let dst_pending = cmd_buf_data
802 .trackers
803 .buffers
804 .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
805
806 let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
807 dst_buffer
808 .check_usage(BufferUsages::COPY_DST)
809 .map_err(TransferError::MissingBufferUsage)?;
810 let dst_barrier =
811 dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
812
813 let (size, source_end_offset) = match size {
814 Some(size) => (size, source_offset + size),
815 None => (src_buffer.size - source_offset, src_buffer.size),
816 };
817
818 if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
819 return Err(TransferError::UnalignedCopySize(size).into());
820 }
821 if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
822 return Err(TransferError::UnalignedBufferOffset(source_offset).into());
823 }
824 if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
825 return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
826 }
827 if !device
828 .downlevel
829 .flags
830 .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
831 && (src_buffer.usage.contains(BufferUsages::INDEX)
832 || dst_buffer.usage.contains(BufferUsages::INDEX))
833 {
834 let forbidden_usages = BufferUsages::VERTEX
835 | BufferUsages::UNIFORM
836 | BufferUsages::INDIRECT
837 | BufferUsages::STORAGE;
838 if src_buffer.usage.intersects(forbidden_usages)
839 || dst_buffer.usage.intersects(forbidden_usages)
840 {
841 return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
842 wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
843 ))
844 .into());
845 }
846 }
847
848 let destination_end_offset = destination_offset + size;
849 if source_end_offset > src_buffer.size {
850 return Err(TransferError::BufferOverrun {
851 start_offset: source_offset,
852 end_offset: source_end_offset,
853 buffer_size: src_buffer.size,
854 side: CopySide::Source,
855 }
856 .into());
857 }
858 if destination_end_offset > dst_buffer.size {
859 return Err(TransferError::BufferOverrun {
860 start_offset: destination_offset,
861 end_offset: destination_end_offset,
862 buffer_size: dst_buffer.size,
863 side: CopySide::Destination,
864 }
865 .into());
866 }
867
868 if size == 0 {
869 log::trace!("Ignoring copy_buffer_to_buffer of size 0");
870 return Ok(());
871 }
872
873 cmd_buf_data.buffer_memory_init_actions.extend(
875 dst_buffer.initialization_status.read().create_action(
876 &dst_buffer,
877 destination_offset..(destination_offset + size),
878 MemoryInitKind::ImplicitlyInitialized,
879 ),
880 );
881 cmd_buf_data.buffer_memory_init_actions.extend(
882 src_buffer.initialization_status.read().create_action(
883 &src_buffer,
884 source_offset..(source_offset + size),
885 MemoryInitKind::NeedsInitializedMemory,
886 ),
887 );
888
889 let region = hal::BufferCopy {
890 src_offset: source_offset,
891 dst_offset: destination_offset,
892 size: wgt::BufferSize::new(size).unwrap(),
893 };
894 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
895 let barriers = src_barrier
896 .into_iter()
897 .chain(dst_barrier)
898 .collect::<Vec<_>>();
899 unsafe {
900 cmd_buf_raw.transition_buffers(&barriers);
901 cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
902 }
903
904 Ok(())
905 })
906 }
907
908 pub fn command_encoder_copy_buffer_to_texture(
909 &self,
910 command_encoder_id: CommandEncoderId,
911 source: &TexelCopyBufferInfo,
912 destination: &TexelCopyTextureInfo,
913 copy_size: &Extent3d,
914 ) -> Result<(), EncoderStateError> {
915 profiling::scope!("CommandEncoder::copy_buffer_to_texture");
916 api_log!(
917 "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
918 source.buffer,
919 destination.texture
920 );
921
922 let hub = &self.hub;
923
924 let cmd_enc = hub.command_encoders.get(command_encoder_id);
925 let mut cmd_buf_data = cmd_enc.data.lock();
926 cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
927 let device = &cmd_enc.device;
928 device.check_is_valid()?;
929
930 #[cfg(feature = "trace")]
931 if let Some(ref mut list) = cmd_buf_data.commands {
932 list.push(TraceCommand::CopyBufferToTexture {
933 src: *source,
934 dst: *destination,
935 size: *copy_size,
936 });
937 }
938
939 let dst_texture = hub.textures.get(destination.texture).get()?;
940 let src_buffer = hub.buffers.get(source.buffer).get()?;
941
942 dst_texture.same_device_as(cmd_enc.as_ref())?;
943 src_buffer.same_device_as(cmd_enc.as_ref())?;
944
945 let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
946 destination,
947 &dst_texture.desc,
948 CopySide::Destination,
949 copy_size,
950 )?;
951
952 let (dst_range, dst_base) =
953 extract_texture_selector(destination, copy_size, &dst_texture)?;
954
955 let snatch_guard = device.snatchable_lock.read();
956
957 let src_raw = src_buffer.try_raw(&snatch_guard)?;
958 let dst_raw = dst_texture.try_raw(&snatch_guard)?;
959
960 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
961 {
962 log::trace!("Ignoring copy_buffer_to_texture of size 0");
963 return Ok(());
964 }
965
966 handle_dst_texture_init(
970 cmd_buf_data,
971 device,
972 destination,
973 copy_size,
974 &dst_texture,
975 &snatch_guard,
976 )?;
977
978 let src_pending = cmd_buf_data
979 .trackers
980 .buffers
981 .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
982
983 src_buffer
984 .check_usage(BufferUsages::COPY_SRC)
985 .map_err(TransferError::MissingBufferUsage)?;
986 let src_barrier =
987 src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
988
989 let dst_pending = cmd_buf_data.trackers.textures.set_single(
990 &dst_texture,
991 dst_range,
992 wgt::TextureUses::COPY_DST,
993 );
994 dst_texture
995 .check_usage(TextureUsages::COPY_DST)
996 .map_err(TransferError::MissingTextureUsage)?;
997 let dst_barrier = dst_pending
998 .map(|pending| pending.into_hal(dst_raw))
999 .collect::<Vec<_>>();
1000
1001 validate_texture_copy_dst_format(dst_texture.desc.format, destination.aspect)?;
1002
1003 validate_texture_buffer_copy(
1004 destination,
1005 dst_base.aspect,
1006 &dst_texture.desc,
1007 source.layout.offset,
1008 true, )?;
1010
1011 let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
1012 validate_linear_texture_data(
1013 &source.layout,
1014 dst_texture.desc.format,
1015 destination.aspect,
1016 src_buffer.size,
1017 CopySide::Source,
1018 copy_size,
1019 true,
1020 )?;
1021
1022 if dst_texture.desc.format.is_depth_stencil_format() {
1023 device
1024 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1025 .map_err(TransferError::from)?;
1026 }
1027
1028 cmd_buf_data.buffer_memory_init_actions.extend(
1029 src_buffer.initialization_status.read().create_action(
1030 &src_buffer,
1031 source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
1032 MemoryInitKind::NeedsInitializedMemory,
1033 ),
1034 );
1035
1036 let regions = (0..array_layer_count)
1037 .map(|rel_array_layer| {
1038 let mut texture_base = dst_base.clone();
1039 texture_base.array_layer += rel_array_layer;
1040 let mut buffer_layout = source.layout;
1041 buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1042 hal::BufferTextureCopy {
1043 buffer_layout,
1044 texture_base,
1045 size: hal_copy_size,
1046 }
1047 })
1048 .collect::<Vec<_>>();
1049
1050 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1051 unsafe {
1052 cmd_buf_raw.transition_textures(&dst_barrier);
1053 cmd_buf_raw.transition_buffers(src_barrier.as_slice());
1054 cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, ®ions);
1055 }
1056
1057 Ok(())
1058 })
1059 }
1060
1061 pub fn command_encoder_copy_texture_to_buffer(
1062 &self,
1063 command_encoder_id: CommandEncoderId,
1064 source: &TexelCopyTextureInfo,
1065 destination: &TexelCopyBufferInfo,
1066 copy_size: &Extent3d,
1067 ) -> Result<(), EncoderStateError> {
1068 profiling::scope!("CommandEncoder::copy_texture_to_buffer");
1069 api_log!(
1070 "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
1071 source.texture,
1072 destination.buffer
1073 );
1074
1075 let hub = &self.hub;
1076
1077 let cmd_enc = hub.command_encoders.get(command_encoder_id);
1078 let mut cmd_buf_data = cmd_enc.data.lock();
1079 cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1080 let device = &cmd_enc.device;
1081 device.check_is_valid()?;
1082
1083 #[cfg(feature = "trace")]
1084 if let Some(list) = cmd_buf_data.commands.as_mut() {
1085 list.push(TraceCommand::CopyTextureToBuffer {
1086 src: *source,
1087 dst: *destination,
1088 size: *copy_size,
1089 });
1090 }
1091
1092 let src_texture = hub.textures.get(source.texture).get()?;
1093 let dst_buffer = hub.buffers.get(destination.buffer).get()?;
1094
1095 src_texture.same_device_as(cmd_enc.as_ref())?;
1096 dst_buffer.same_device_as(cmd_enc.as_ref())?;
1097
1098 let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
1099 source,
1100 &src_texture.desc,
1101 CopySide::Source,
1102 copy_size,
1103 )?;
1104
1105 let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1106
1107 let snatch_guard = device.snatchable_lock.read();
1108
1109 let src_raw = src_texture.try_raw(&snatch_guard)?;
1110 src_texture
1111 .check_usage(TextureUsages::COPY_SRC)
1112 .map_err(TransferError::MissingTextureUsage)?;
1113
1114 if source.mip_level >= src_texture.desc.mip_level_count {
1115 return Err(TransferError::InvalidMipLevel {
1116 requested: source.mip_level,
1117 count: src_texture.desc.mip_level_count,
1118 }
1119 .into());
1120 }
1121
1122 validate_texture_copy_src_format(src_texture.desc.format, source.aspect)?;
1123
1124 validate_texture_buffer_copy(
1125 source,
1126 src_base.aspect,
1127 &src_texture.desc,
1128 destination.layout.offset,
1129 true, )?;
1131
1132 let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
1133 validate_linear_texture_data(
1134 &destination.layout,
1135 src_texture.desc.format,
1136 source.aspect,
1137 dst_buffer.size,
1138 CopySide::Destination,
1139 copy_size,
1140 true,
1141 )?;
1142
1143 if src_texture.desc.format.is_depth_stencil_format() {
1144 device
1145 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1146 .map_err(TransferError::from)?;
1147 }
1148
1149 let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
1150 dst_buffer
1151 .check_usage(BufferUsages::COPY_DST)
1152 .map_err(TransferError::MissingBufferUsage)?;
1153
1154 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1155 {
1156 log::trace!("Ignoring copy_texture_to_buffer of size 0");
1157 return Ok(());
1158 }
1159
1160 handle_src_texture_init(
1164 cmd_buf_data,
1165 device,
1166 source,
1167 copy_size,
1168 &src_texture,
1169 &snatch_guard,
1170 )?;
1171
1172 let src_pending = cmd_buf_data.trackers.textures.set_single(
1173 &src_texture,
1174 src_range,
1175 wgt::TextureUses::COPY_SRC,
1176 );
1177 let src_barrier = src_pending
1178 .map(|pending| pending.into_hal(src_raw))
1179 .collect::<Vec<_>>();
1180
1181 let dst_pending = cmd_buf_data
1182 .trackers
1183 .buffers
1184 .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
1185
1186 let dst_barrier =
1187 dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
1188
1189 cmd_buf_data.buffer_memory_init_actions.extend(
1190 dst_buffer.initialization_status.read().create_action(
1191 &dst_buffer,
1192 destination.layout.offset
1193 ..(destination.layout.offset + required_buffer_bytes_in_copy),
1194 MemoryInitKind::ImplicitlyInitialized,
1195 ),
1196 );
1197
1198 let regions = (0..array_layer_count)
1199 .map(|rel_array_layer| {
1200 let mut texture_base = src_base.clone();
1201 texture_base.array_layer += rel_array_layer;
1202 let mut buffer_layout = destination.layout;
1203 buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1204 hal::BufferTextureCopy {
1205 buffer_layout,
1206 texture_base,
1207 size: hal_copy_size,
1208 }
1209 })
1210 .collect::<Vec<_>>();
1211 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1212 unsafe {
1213 cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1214 cmd_buf_raw.transition_textures(&src_barrier);
1215 cmd_buf_raw.copy_texture_to_buffer(
1216 src_raw,
1217 wgt::TextureUses::COPY_SRC,
1218 dst_raw,
1219 ®ions,
1220 );
1221 }
1222
1223 Ok(())
1224 })
1225 }
1226
1227 pub fn command_encoder_copy_texture_to_texture(
1228 &self,
1229 command_encoder_id: CommandEncoderId,
1230 source: &TexelCopyTextureInfo,
1231 destination: &TexelCopyTextureInfo,
1232 copy_size: &Extent3d,
1233 ) -> Result<(), EncoderStateError> {
1234 profiling::scope!("CommandEncoder::copy_texture_to_texture");
1235 api_log!(
1236 "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1237 source.texture,
1238 destination.texture
1239 );
1240
1241 let hub = &self.hub;
1242
1243 let cmd_enc = hub.command_encoders.get(command_encoder_id);
1244 let mut cmd_buf_data = cmd_enc.data.lock();
1245 cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1246 let device = &cmd_enc.device;
1247 device.check_is_valid()?;
1248
1249 let snatch_guard = device.snatchable_lock.read();
1250
1251 #[cfg(feature = "trace")]
1252 if let Some(ref mut list) = cmd_buf_data.commands {
1253 list.push(TraceCommand::CopyTextureToTexture {
1254 src: *source,
1255 dst: *destination,
1256 size: *copy_size,
1257 });
1258 }
1259
1260 let src_texture = hub.textures.get(source.texture).get()?;
1261 let dst_texture = hub.textures.get(destination.texture).get()?;
1262
1263 src_texture.same_device_as(cmd_enc.as_ref())?;
1264 dst_texture.same_device_as(cmd_enc.as_ref())?;
1265
1266 if src_texture.desc.format.remove_srgb_suffix()
1269 != dst_texture.desc.format.remove_srgb_suffix()
1270 {
1271 return Err(TransferError::TextureFormatsNotCopyCompatible {
1272 src_format: src_texture.desc.format,
1273 dst_format: dst_texture.desc.format,
1274 }
1275 .into());
1276 }
1277
1278 let (src_copy_size, array_layer_count) = validate_texture_copy_range(
1279 source,
1280 &src_texture.desc,
1281 CopySide::Source,
1282 copy_size,
1283 )?;
1284 let (dst_copy_size, _) = validate_texture_copy_range(
1285 destination,
1286 &dst_texture.desc,
1287 CopySide::Destination,
1288 copy_size,
1289 )?;
1290
1291 if Arc::as_ptr(&src_texture) == Arc::as_ptr(&dst_texture) {
1292 validate_copy_within_same_texture(
1293 source,
1294 destination,
1295 src_texture.desc.format,
1296 array_layer_count,
1297 )?;
1298 }
1299
1300 let (src_range, src_tex_base) =
1301 extract_texture_selector(source, copy_size, &src_texture)?;
1302 let (dst_range, dst_tex_base) =
1303 extract_texture_selector(destination, copy_size, &dst_texture)?;
1304 let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1305 let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1306 if src_tex_base.aspect != src_texture_aspects {
1307 return Err(TransferError::CopySrcMissingAspects.into());
1308 }
1309 if dst_tex_base.aspect != dst_texture_aspects {
1310 return Err(TransferError::CopyDstMissingAspects.into());
1311 }
1312
1313 if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1314 return Err(TransferError::SampleCountNotEqual {
1315 src_sample_count: src_texture.desc.sample_count,
1316 dst_sample_count: dst_texture.desc.sample_count,
1317 }
1318 .into());
1319 }
1320
1321 handle_src_texture_init(
1325 cmd_buf_data,
1326 device,
1327 source,
1328 copy_size,
1329 &src_texture,
1330 &snatch_guard,
1331 )?;
1332 handle_dst_texture_init(
1333 cmd_buf_data,
1334 device,
1335 destination,
1336 copy_size,
1337 &dst_texture,
1338 &snatch_guard,
1339 )?;
1340
1341 let src_raw = src_texture.try_raw(&snatch_guard)?;
1342 src_texture
1343 .check_usage(TextureUsages::COPY_SRC)
1344 .map_err(TransferError::MissingTextureUsage)?;
1345 let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1346 dst_texture
1347 .check_usage(TextureUsages::COPY_DST)
1348 .map_err(TransferError::MissingTextureUsage)?;
1349
1350 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1351 {
1352 log::trace!("Ignoring copy_texture_to_texture of size 0");
1353 return Ok(());
1354 }
1355
1356 let src_pending = cmd_buf_data.trackers.textures.set_single(
1357 &src_texture,
1358 src_range,
1359 wgt::TextureUses::COPY_SRC,
1360 );
1361
1362 let mut barriers: ArrayVec<_, 2> = src_pending
1365 .map(|pending| pending.into_hal(src_raw))
1366 .collect();
1367
1368 let dst_pending = cmd_buf_data.trackers.textures.set_single(
1369 &dst_texture,
1370 dst_range,
1371 wgt::TextureUses::COPY_DST,
1372 );
1373 barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1374
1375 let hal_copy_size = hal::CopyExtent {
1376 width: src_copy_size.width.min(dst_copy_size.width),
1377 height: src_copy_size.height.min(dst_copy_size.height),
1378 depth: src_copy_size.depth.min(dst_copy_size.depth),
1379 };
1380
1381 let regions = (0..array_layer_count).map(|rel_array_layer| {
1382 let mut src_base = src_tex_base.clone();
1383 let mut dst_base = dst_tex_base.clone();
1384 src_base.array_layer += rel_array_layer;
1385 dst_base.array_layer += rel_array_layer;
1386 hal::TextureCopy {
1387 src_base,
1388 dst_base,
1389 size: hal_copy_size,
1390 }
1391 });
1392
1393 let regions = if dst_tex_base.aspect == hal::FormatAspects::DEPTH_STENCIL {
1394 regions
1395 .flat_map(|region| {
1396 let (mut depth, mut stencil) = (region.clone(), region);
1397 depth.src_base.aspect = hal::FormatAspects::DEPTH;
1398 depth.dst_base.aspect = hal::FormatAspects::DEPTH;
1399 stencil.src_base.aspect = hal::FormatAspects::STENCIL;
1400 stencil.dst_base.aspect = hal::FormatAspects::STENCIL;
1401 [depth, stencil]
1402 })
1403 .collect::<Vec<_>>()
1404 } else {
1405 regions.collect::<Vec<_>>()
1406 };
1407 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1408 unsafe {
1409 cmd_buf_raw.transition_textures(&barriers);
1410 cmd_buf_raw.copy_texture_to_texture(
1411 src_raw,
1412 wgt::TextureUses::COPY_SRC,
1413 dst_raw,
1414 ®ions,
1415 );
1416 }
1417
1418 Ok(())
1419 })
1420 }
1421}