1use alloc::{format, string::String, sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6 error::{ErrorType, WebGpuError},
7 BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8 TextureUsages,
9};
10
11use crate::{
12 api_log,
13 command::{
14 clear_texture, encoder::EncodingState, ArcCommand, CommandEncoderError, EncoderStateError,
15 },
16 device::MissingDownlevelFlags,
17 init_tracker::{
18 has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
19 TextureInitTrackerAction,
20 },
21 resource::{
22 Buffer, Labeled, MissingBufferUsageError, MissingTextureUsageError, ParentDevice,
23 RawResourceAccess, Texture, TextureErrorDimension,
24 },
25};
26
27use super::ClearError;
28
29type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<Arc<Texture>>;
30
31#[derive(Clone, Copy, Debug, Eq, PartialEq)]
32pub enum CopySide {
33 Source,
34 Destination,
35}
36
37#[derive(Clone, Debug, Error)]
39#[non_exhaustive]
40pub enum TransferError {
41 #[error("Source and destination cannot be the same buffer")]
42 SameSourceDestinationBuffer,
43 #[error(transparent)]
44 MissingBufferUsage(#[from] MissingBufferUsageError),
45 #[error(transparent)]
46 MissingTextureUsage(#[from] MissingTextureUsageError),
47 #[error(
48 "Copy at offset {start_offset} bytes would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}"
49 )]
50 BufferStartOffsetOverrun {
51 start_offset: BufferAddress,
52 buffer_size: BufferAddress,
53 side: CopySide,
54 },
55 #[error(
56 "Copy at offset {start_offset} for {size} bytes would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}"
57 )]
58 BufferEndOffsetOverrun {
59 start_offset: BufferAddress,
60 size: BufferAddress,
61 buffer_size: BufferAddress,
62 side: CopySide,
63 },
64 #[error(
65 "Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the \
66 bounds{plane} of the {side:?} texture of {dimension:?} size {texture_size}",
67 plane = plane.map(|plane| format!(" of plane {plane}")).unwrap_or("".into()),
68 )]
69 TextureOverrun {
70 start_offset: u32,
71 end_offset: u32,
72 texture_size: u32,
73 dimension: TextureErrorDimension,
74 side: CopySide,
75 plane: Option<u32>,
76 },
77 #[error("Partial copy of {start_offset}..{end_offset} on {dimension:?} dimension with size {texture_size} \
78 is not supported for the {side:?} texture format {format:?} with {sample_count} samples")]
79 UnsupportedPartialTransfer {
80 format: wgt::TextureFormat,
81 sample_count: u32,
82 start_offset: u32,
83 end_offset: u32,
84 texture_size: u32,
85 dimension: TextureErrorDimension,
86 side: CopySide,
87 },
88 #[error(
89 "Copying{} layers {}..{} to{} layers {}..{} of the same texture is not allowed",
90 if *src_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {src_aspects:?}") },
91 src_origin_z,
92 src_origin_z + array_layer_count,
93 if *dst_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {dst_aspects:?}") },
94 dst_origin_z,
95 dst_origin_z + array_layer_count,
96 )]
97 InvalidCopyWithinSameTexture {
98 src_aspects: wgt::TextureAspect,
99 dst_aspects: wgt::TextureAspect,
100 src_origin_z: u32,
101 dst_origin_z: u32,
102 array_layer_count: u32,
103 },
104 #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
105 InvalidTextureAspect {
106 format: wgt::TextureFormat,
107 aspect: wgt::TextureAspect,
108 },
109 #[error("Unable to select texture mip level {level} out of {total}")]
110 InvalidTextureMipLevel { level: u32, total: u32 },
111 #[error("Texture dimension must be 2D when copying from an external texture")]
112 InvalidDimensionExternal,
113 #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
114 UnalignedBufferOffset(BufferAddress),
115 #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
116 UnalignedCopySize(BufferAddress),
117 #[error("Copy width is not a multiple of the required alignment")]
118 UnalignedCopyWidth,
119 #[error("Copy height is not a multiple of the required alignment")]
120 UnalignedCopyHeight,
121 #[error("Copy origin's x component is not a multiple of the required alignment")]
122 UnalignedCopyOriginX,
123 #[error("Copy origin's y component is not a multiple of the required alignment")]
124 UnalignedCopyOriginY,
125 #[error("Bytes per row ({bytes_per_row}) is not a multiple of {alignment}")]
126 UnalignedBytesPerRow { bytes_per_row: u32, alignment: u32 },
127 #[error("Number of bytes per row needs to be specified since more than one row is copied")]
128 UnspecifiedBytesPerRow,
129 #[error("Number of rows per image needs to be specified since more than one image is copied")]
130 UnspecifiedRowsPerImage,
131 #[error("Number of bytes per row is less than the number of bytes in a complete row")]
132 InvalidBytesPerRow,
133 #[error("Number of rows per image is invalid")]
134 InvalidRowsPerImage,
135 #[error("Overflow while computing the size of the copy")]
136 SizeOverflow,
137 #[error("Copy source aspects must refer to all aspects of the source texture format")]
138 CopySrcMissingAspects,
139 #[error(
140 "Copy destination aspects must refer to all aspects of the destination texture format"
141 )]
142 CopyDstMissingAspects,
143 #[error("Copy aspect must refer to a single aspect of texture format")]
144 CopyAspectNotOne,
145 #[error("Copying from textures with format {0:?} is forbidden")]
146 CopyFromForbiddenTextureFormat(wgt::TextureFormat),
147 #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
148 CopyFromForbiddenTextureFormatAspect {
149 format: wgt::TextureFormat,
150 aspect: wgt::TextureAspect,
151 },
152 #[error("Copying to textures with format {0:?} is forbidden")]
153 CopyToForbiddenTextureFormat(wgt::TextureFormat),
154 #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
155 CopyToForbiddenTextureFormatAspect {
156 format: wgt::TextureFormat,
157 aspect: wgt::TextureAspect,
158 },
159 #[error(
160 "Copying to textures with format {0:?} is forbidden when copying from external texture"
161 )]
162 ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
163 #[error(
166 "Source format ({src_format:?}{src_plane_info}) and destination format ({dst_format:?}{dst_plane_info}) \
167 are not copy-compatible (they may only differ in srgb-ness)",
168 src_plane_info = src_multi_planar.map(|(format, plane)| format!(", in plane {plane} of {format:?}")).unwrap_or("".into()),
169 dst_plane_info = dst_multi_planar.map(|(format, plane)| format!(", in plane {plane} of {format:?}")).unwrap_or("".into()),
170 )]
171 TextureFormatsNotCopyCompatible {
172 src_format: wgt::TextureFormat,
173 src_multi_planar: Option<(wgt::TextureFormat, u32)>,
174 dst_format: wgt::TextureFormat,
175 dst_multi_planar: Option<(wgt::TextureFormat, u32)>,
176 },
177 #[error(transparent)]
178 MemoryInitFailure(#[from] ClearError),
179 #[error("Cannot encode this copy because of a missing downelevel flag")]
180 MissingDownlevelFlags(#[from] MissingDownlevelFlags),
181 #[error("Source texture sample count must be 1, got {sample_count}")]
182 InvalidSampleCount { sample_count: u32 },
183 #[error(
184 "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
185 )]
186 SampleCountNotEqual {
187 src_sample_count: u32,
188 dst_sample_count: u32,
189 },
190 #[error("Requested mip level {requested} does not exist (count: {count})")]
191 InvalidMipLevel { requested: u32, count: u32 },
192 #[error("Buffer is expected to be unmapped, but was not")]
193 BufferNotAvailable,
194}
195
196impl WebGpuError for TransferError {
197 fn webgpu_error_type(&self) -> ErrorType {
198 match self {
199 Self::MissingBufferUsage(e) => e.webgpu_error_type(),
200 Self::MissingTextureUsage(e) => e.webgpu_error_type(),
201 Self::MemoryInitFailure(e) => e.webgpu_error_type(),
202
203 Self::BufferEndOffsetOverrun { .. }
204 | Self::TextureOverrun { .. }
205 | Self::BufferStartOffsetOverrun { .. }
206 | Self::UnsupportedPartialTransfer { .. }
207 | Self::InvalidCopyWithinSameTexture { .. }
208 | Self::InvalidTextureAspect { .. }
209 | Self::InvalidTextureMipLevel { .. }
210 | Self::InvalidDimensionExternal
211 | Self::UnalignedBufferOffset(..)
212 | Self::UnalignedCopySize(..)
213 | Self::UnalignedCopyWidth
214 | Self::UnalignedCopyHeight
215 | Self::UnalignedCopyOriginX
216 | Self::UnalignedCopyOriginY
217 | Self::UnalignedBytesPerRow { .. }
218 | Self::UnspecifiedBytesPerRow
219 | Self::UnspecifiedRowsPerImage
220 | Self::InvalidBytesPerRow
221 | Self::InvalidRowsPerImage
222 | Self::SizeOverflow
223 | Self::CopySrcMissingAspects
224 | Self::CopyDstMissingAspects
225 | Self::CopyAspectNotOne
226 | Self::CopyFromForbiddenTextureFormat(..)
227 | Self::CopyFromForbiddenTextureFormatAspect { .. }
228 | Self::CopyToForbiddenTextureFormat(..)
229 | Self::CopyToForbiddenTextureFormatAspect { .. }
230 | Self::ExternalCopyToForbiddenTextureFormat(..)
231 | Self::TextureFormatsNotCopyCompatible { .. }
232 | Self::MissingDownlevelFlags(..)
233 | Self::InvalidSampleCount { .. }
234 | Self::SampleCountNotEqual { .. }
235 | Self::InvalidMipLevel { .. }
236 | Self::SameSourceDestinationBuffer
237 | Self::BufferNotAvailable => ErrorType::Validation,
238 }
239 }
240}
241
242impl From<BufferTextureCopyInfoError> for TransferError {
243 fn from(value: BufferTextureCopyInfoError) -> Self {
244 match value {
245 BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
246 BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
247 BufferTextureCopyInfoError::ImageStrideOverflow
248 | BufferTextureCopyInfoError::ImageBytesOverflow(_)
249 | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
250 }
251 }
252}
253
254pub(crate) fn validate_linear_texture_data(
266 layout: &wgt::TexelCopyBufferLayout,
267 format: wgt::TextureFormat,
268 aspect: wgt::TextureAspect,
269 buffer_size: BufferAddress,
270 buffer_side: CopySide,
271 copy_size: &Extent3d,
272) -> Result<(BufferAddress, BufferAddress, bool), TransferError> {
273 let wgt::BufferTextureCopyInfo {
274 copy_width,
275 copy_height,
276 depth_or_array_layers,
277
278 offset,
279
280 block_size_bytes: _,
281 block_width_texels,
282 block_height_texels,
283
284 width_blocks: _,
285 height_blocks,
286
287 row_bytes_dense,
288 row_stride_bytes,
289
290 image_stride_rows: _,
291 image_stride_bytes,
292
293 image_rows_dense: _,
294 image_bytes_dense,
295
296 bytes_in_copy,
297 } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
298
299 if !copy_width.is_multiple_of(block_width_texels) {
300 return Err(TransferError::UnalignedCopyWidth);
301 }
302 if !copy_height.is_multiple_of(block_height_texels) {
303 return Err(TransferError::UnalignedCopyHeight);
304 }
305
306 let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
307 let requires_multiple_images = depth_or_array_layers > 1;
308
309 if layout.bytes_per_row.is_none() && requires_multiple_rows {
314 return Err(TransferError::UnspecifiedBytesPerRow);
315 }
316
317 if layout.rows_per_image.is_none() && requires_multiple_images {
318 return Err(TransferError::UnspecifiedRowsPerImage);
319 };
320
321 if offset > buffer_size {
322 return Err(TransferError::BufferStartOffsetOverrun {
323 start_offset: offset,
324 buffer_size,
325 side: buffer_side,
326 });
327 }
328 if bytes_in_copy > buffer_size - offset {
330 return Err(TransferError::BufferEndOffsetOverrun {
331 start_offset: offset,
332 size: bytes_in_copy,
333 buffer_size,
334 side: buffer_side,
335 });
336 }
337
338 let is_contiguous = (row_stride_bytes == row_bytes_dense || !requires_multiple_rows)
339 && (image_stride_bytes == image_bytes_dense || !requires_multiple_images);
340
341 Ok((bytes_in_copy, image_stride_bytes, is_contiguous))
342}
343
344pub(crate) fn validate_texture_copy_src_format(
355 format: wgt::TextureFormat,
356 aspect: wgt::TextureAspect,
357) -> Result<(), TransferError> {
358 use wgt::TextureAspect as Ta;
359 use wgt::TextureFormat as Tf;
360 match (format, aspect) {
361 (Tf::Depth24Plus, _) => Err(TransferError::CopyFromForbiddenTextureFormat(format)),
362 (Tf::Depth24PlusStencil8, Ta::DepthOnly) => {
363 Err(TransferError::CopyFromForbiddenTextureFormatAspect { format, aspect })
364 }
365 _ => Ok(()),
366 }
367}
368
369pub(crate) fn validate_texture_copy_dst_format(
380 format: wgt::TextureFormat,
381 aspect: wgt::TextureAspect,
382) -> Result<(), TransferError> {
383 use wgt::TextureAspect as Ta;
384 use wgt::TextureFormat as Tf;
385 match (format, aspect) {
386 (Tf::Depth24Plus | Tf::Depth32Float, _) => {
387 Err(TransferError::CopyToForbiddenTextureFormat(format))
388 }
389 (Tf::Depth24PlusStencil8 | Tf::Depth32FloatStencil8, Ta::DepthOnly) => {
390 Err(TransferError::CopyToForbiddenTextureFormatAspect { format, aspect })
391 }
392 _ => Ok(()),
393 }
394}
395
396pub(crate) fn validate_texture_buffer_copy<T>(
425 texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
426 aspect: hal::FormatAspects,
427 desc: &wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
428 layout: &wgt::TexelCopyBufferLayout,
429 aligned: bool,
430 features: wgt::Features,
431) -> Result<(), TransferError> {
432 if desc.sample_count != 1 {
433 return Err(TransferError::InvalidSampleCount {
434 sample_count: desc.sample_count,
435 });
436 }
437
438 if !aspect.is_one() {
439 return Err(TransferError::CopyAspectNotOne);
440 }
441
442 let block_copy_size = || {
443 desc.format
444 .block_copy_size(Some(texture_copy_view.aspect))
445 .expect("non-copyable formats should have been rejected previously")
446 };
447 let offset_alignment = if desc.format.is_depth_stencil_format() {
448 4
449 } else {
450 block_copy_size()
451 };
452
453 if aligned && !layout.offset.is_multiple_of(u64::from(offset_alignment)) {
454 return Err(TransferError::UnalignedBufferOffset(layout.offset));
455 }
456
457 if let Some(bytes_per_row) = layout.bytes_per_row {
458 let alignment = if features.contains(wgt::Features::UNRESTRICTED_BYTES_PER_ROW) {
459 block_copy_size()
460 } else {
461 wgt::COPY_BYTES_PER_ROW_ALIGNMENT
462 };
463 if aligned && !bytes_per_row.is_multiple_of(alignment) {
464 return Err(TransferError::UnalignedBytesPerRow {
465 bytes_per_row,
466 alignment,
467 });
468 }
469 }
470
471 Ok(())
472}
473
474pub(crate) struct CopyRange {
475 pub hal_copy_extent: hal::CopyExtent,
476 pub hal_base: hal::TextureCopyBase,
477 pub array_layer_count: u32,
478 pub selector: TextureSelector,
479}
480
481pub(crate) fn validate_texture_copy_range<T>(
492 texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
493 desc: &wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
494 texture_side: CopySide,
495 copy_size: &Extent3d,
496) -> Result<CopyRange, TransferError> {
497 let hal_aspects = hal::FormatAspects::new(desc.format, texture_copy_view.aspect);
498 if hal_aspects.is_empty() {
499 return Err(TransferError::InvalidTextureAspect {
500 format: desc.format,
501 aspect: texture_copy_view.aspect,
502 });
503 }
504
505 let is_multi_planar_all =
506 desc.format.is_multi_planar_format() && texture_copy_view.aspect == wgt::TextureAspect::All;
507
508 let (block_width, block_height) = if is_multi_planar_all {
509 desc.format.size_multiple_requirement()
513 } else {
514 desc.format.block_dimensions()
515 };
516
517 let mip_level = texture_copy_view.mip_level;
518 let plane = texture_copy_view.aspect.to_plane();
519 if mip_level >= desc.mip_level_count {
520 return Err(TransferError::InvalidTextureMipLevel {
521 level: mip_level,
522 total: desc.mip_level_count,
523 });
524 }
525 let extent_virtual = desc.plane_size(mip_level, plane.or(is_multi_planar_all.then_some(0)));
530 let extent = extent_virtual.physical_size(desc.format);
532
533 let requires_exact_size = desc.format.is_depth_stencil_format() || desc.sample_count > 1;
536
537 let check_dimension = |dimension: TextureErrorDimension,
540 start_offset: u32,
541 size: u32,
542 texture_size: u32,
543 requires_exact_size: bool|
544 -> Result<(), TransferError> {
545 if requires_exact_size && (start_offset != 0 || size != texture_size) {
546 Err(TransferError::UnsupportedPartialTransfer {
547 format: desc.format,
548 sample_count: desc.sample_count,
549 start_offset,
550 end_offset: start_offset.wrapping_add(size),
551 texture_size,
552 dimension,
553 side: texture_side,
554 })
555 } else if start_offset > texture_size || texture_size - start_offset < size {
558 Err(TransferError::TextureOverrun {
559 start_offset,
560 end_offset: start_offset.wrapping_add(size),
561 texture_size,
562 dimension,
563 side: texture_side,
564 plane,
565 })
566 } else {
567 Ok(())
568 }
569 };
570
571 check_dimension(
572 TextureErrorDimension::X,
573 texture_copy_view.origin.x,
574 copy_size.width,
575 extent.width,
576 requires_exact_size,
577 )?;
578 check_dimension(
579 TextureErrorDimension::Y,
580 texture_copy_view.origin.y,
581 copy_size.height,
582 extent.height,
583 requires_exact_size,
584 )?;
585 check_dimension(
586 TextureErrorDimension::Z,
587 texture_copy_view.origin.z,
588 copy_size.depth_or_array_layers,
589 extent.depth_or_array_layers,
590 false, )?;
592
593 if !texture_copy_view.origin.x.is_multiple_of(block_width) {
594 return Err(TransferError::UnalignedCopyOriginX);
595 }
596 if !texture_copy_view.origin.y.is_multiple_of(block_height) {
597 return Err(TransferError::UnalignedCopyOriginY);
598 }
599 if !copy_size.width.is_multiple_of(block_width) {
600 return Err(TransferError::UnalignedCopyWidth);
601 }
602 if !copy_size.height.is_multiple_of(block_height) {
603 return Err(TransferError::UnalignedCopyHeight);
604 }
605
606 let (depth, array_layer_count) = match desc.dimension {
607 wgt::TextureDimension::D1 => (1, 1),
608 wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
609 wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
610 };
611
612 let hal_copy_extent = hal::CopyExtent {
613 width: copy_size.width,
614 height: copy_size.height,
615 depth,
616 };
617
618 let (layers, origin_z) = match desc.dimension {
621 wgt::TextureDimension::D1 => (0..1, 0),
622 wgt::TextureDimension::D2 => (
623 texture_copy_view.origin.z
624 ..texture_copy_view.origin.z + copy_size.depth_or_array_layers,
625 0,
626 ),
627 wgt::TextureDimension::D3 => (0..1, texture_copy_view.origin.z),
628 };
629 let hal_base = hal::TextureCopyBase {
630 origin: wgt::Origin3d {
631 x: texture_copy_view.origin.x,
632 y: texture_copy_view.origin.y,
633 z: origin_z,
634 },
635 array_layer: layers.start,
637 mip_level: texture_copy_view.mip_level,
638 aspect: hal_aspects,
639 };
640 let selector = TextureSelector {
641 mips: texture_copy_view.mip_level..texture_copy_view.mip_level + 1,
642 layers,
643 };
644
645 Ok(CopyRange {
646 hal_copy_extent,
647 hal_base,
648 array_layer_count,
649 selector,
650 })
651}
652
653pub(crate) fn validate_copy_within_same_texture<T>(
664 src: &wgt::TexelCopyTextureInfo<T>,
665 dst: &wgt::TexelCopyTextureInfo<T>,
666 format: wgt::TextureFormat,
667 array_layer_count: u32,
668) -> Result<(), TransferError> {
669 let src_aspects = hal::FormatAspects::new(format, src.aspect);
670 let dst_aspects = hal::FormatAspects::new(format, dst.aspect);
671 if (src_aspects & dst_aspects).is_empty() {
672 return Ok(());
674 }
675
676 if src.origin.z >= dst.origin.z + array_layer_count
677 || dst.origin.z >= src.origin.z + array_layer_count
678 {
679 return Ok(());
681 }
682
683 if src.mip_level != dst.mip_level {
684 return Ok(());
686 }
687
688 Err(TransferError::InvalidCopyWithinSameTexture {
689 src_aspects: src.aspect,
690 dst_aspects: dst.aspect,
691 src_origin_z: src.origin.z,
692 dst_origin_z: dst.origin.z,
693 array_layer_count,
694 })
695}
696
697fn handle_texture_init(
698 state: &mut EncodingState,
699 init_kind: MemoryInitKind,
700 copy_texture: &TexelCopyTextureInfo,
701 copy_size: &Extent3d,
702 texture: &Arc<Texture>,
703) -> Result<(), ClearError> {
704 let init_layer_range = if texture.desc.dimension == wgt::TextureDimension::D3 {
705 0..1
707 } else {
708 copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers
709 };
710 let init_action = TextureInitTrackerAction {
711 texture: texture.clone(),
712 range: TextureInitRange {
713 mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
714 layer_range: init_layer_range,
715 },
716 kind: init_kind,
717 };
718
719 let accessed_depth_slices = (texture.desc.dimension == wgt::TextureDimension::D3)
728 .then(|| copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers);
729 let immediate_inits = state
730 .texture_memory_actions
731 .register_init_action(&{ init_action }, accessed_depth_slices);
732
733 if !immediate_inits.is_empty() {
735 for init in immediate_inits {
736 let index = init.layer_or_depth_slice;
737 let (layer_range, depth_slice) = if texture.desc.dimension == wgt::TextureDimension::D3
738 {
739 (0..1, Some(index))
740 } else {
741 (index..(index + 1), None)
742 };
743 clear_texture(
744 &init.texture,
745 TextureInitRange {
746 mip_range: init.mip_level..(init.mip_level + 1),
747 layer_range,
748 },
749 depth_slice,
750 state.raw_encoder,
751 &mut state.tracker.textures,
752 &state.device.alignments,
753 state.device.zero_buffer.as_ref(),
754 state.snatch_guard,
755 state.device.instance_flags,
756 )?;
757 }
758 }
759
760 Ok(())
761}
762
763fn handle_src_texture_init(
768 state: &mut EncodingState,
769 source: &TexelCopyTextureInfo,
770 copy_size: &Extent3d,
771 texture: &Arc<Texture>,
772) -> Result<(), TransferError> {
773 handle_texture_init(
774 state,
775 MemoryInitKind::NeedsInitializedMemory,
776 source,
777 copy_size,
778 texture,
779 )?;
780 Ok(())
781}
782
783fn handle_dst_texture_init(
788 state: &mut EncodingState,
789 destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
790 copy_size: &Extent3d,
791 texture: &Arc<Texture>,
792) -> Result<(), TransferError> {
793 let dst_init_kind =
798 if has_copy_partial_init_tracker_coverage(copy_size, destination, &texture.desc) {
799 MemoryInitKind::NeedsInitializedMemory
800 } else {
801 MemoryInitKind::ImplicitlyInitialized
802 };
803
804 handle_texture_init(state, dst_init_kind, destination, copy_size, texture)?;
805 Ok(())
806}
807
808fn handle_buffer_init(
813 state: &mut EncodingState,
814 info: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
815 direction: CopySide,
816 required_buffer_bytes_in_copy: BufferAddress,
817 is_contiguous: bool,
818) {
819 const ALIGN_SIZE: BufferAddress = wgt::COPY_BUFFER_ALIGNMENT;
820 const ALIGN_MASK: BufferAddress = wgt::COPY_BUFFER_ALIGNMENT - 1;
821
822 let buffer = &info.buffer;
823 let start = info.layout.offset;
824 let end = info.layout.offset + required_buffer_bytes_in_copy;
825 if !is_contiguous || direction == CopySide::Source {
826 let aligned_start = start & !ALIGN_MASK;
837 let aligned_end = (end + ALIGN_MASK) & !ALIGN_MASK;
838 state
839 .buffer_memory_init_actions
840 .extend(buffer.initialization_status.read().create_action(
841 buffer,
842 aligned_start..aligned_end,
843 MemoryInitKind::NeedsInitializedMemory,
844 ));
845 } else {
846 let aligned_start = (start + ALIGN_MASK) & !ALIGN_MASK;
856 let aligned_end = end & !ALIGN_MASK;
857 if aligned_start != start {
858 state.buffer_memory_init_actions.extend(
859 buffer.initialization_status.read().create_action(
860 buffer,
861 aligned_start - ALIGN_SIZE..aligned_start,
862 MemoryInitKind::NeedsInitializedMemory,
863 ),
864 );
865 }
866 if aligned_start != aligned_end {
867 state.buffer_memory_init_actions.extend(
868 buffer.initialization_status.read().create_action(
869 buffer,
870 aligned_start..aligned_end,
871 MemoryInitKind::ImplicitlyInitialized,
872 ),
873 );
874 }
875 if aligned_end != end {
876 state.buffer_memory_init_actions.extend(
882 buffer.initialization_status.read().create_action(
883 buffer,
884 aligned_end..aligned_end + ALIGN_SIZE,
885 MemoryInitKind::NeedsInitializedMemory,
886 ),
887 );
888 }
889 }
890}
891
892impl super::CommandEncoder {
893 fn copy_buffer_to_buffer_inner(
894 self: &Arc<Self>,
895 source: Arc<Buffer>,
896 source_offset: BufferAddress,
897 destination: Arc<Buffer>,
898 destination_offset: BufferAddress,
899 size: Option<BufferAddress>,
900 ) -> Result<(), EncoderStateError> {
901 profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
902 api_log!(
903 "CommandEncoder::copy_buffer_to_buffer {:?} -> {:?} {size:?}bytes",
904 Arc::as_ptr(&source),
905 Arc::as_ptr(&destination)
906 );
907
908 let mut cmd_buf_data = self.data.lock();
909
910 cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
911 source.check_is_valid()?;
912 destination.check_is_valid()?;
913 Ok(ArcCommand::CopyBufferToBuffer {
914 src: source,
915 src_offset: source_offset,
916 dst: destination,
917 dst_offset: destination_offset,
918 size,
919 })
920 })
921 }
922
923 pub fn copy_buffer_to_buffer(
924 self: &Arc<Self>,
925 source: Arc<Buffer>,
926 source_offset: BufferAddress,
927 destination: Arc<Buffer>,
928 destination_offset: BufferAddress,
929 size: Option<BufferAddress>,
930 ) {
931 if let Err(err) = self.copy_buffer_to_buffer_inner(
932 source,
933 source_offset,
934 destination,
935 destination_offset,
936 size,
937 ) {
938 self.device.handle_error(
939 err,
940 Some(self.label()),
941 "CommandEncoder::copy_buffer_to_buffer",
942 );
943 }
944 }
945
946 fn copy_buffer_to_texture_inner(
947 self: &Arc<Self>,
948 source: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
949 destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
950 copy_size: &Extent3d,
951 ) -> Result<(), EncoderStateError> {
952 profiling::scope!("CommandEncoder::copy_buffer_to_texture");
953 api_log!(
954 "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
955 Arc::as_ptr(&source.buffer),
956 Arc::as_ptr(&destination.texture)
957 );
958
959 let mut cmd_buf_data = self.data.lock();
960
961 cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
962 let texture = destination.texture.clone();
963 texture.check_valid()?;
964 let source_buffer = source.buffer.clone();
965 source_buffer.check_is_valid()?;
966 Ok(ArcCommand::CopyBufferToTexture {
967 src: wgt::TexelCopyBufferInfo::<Arc<Buffer>> {
968 buffer: source_buffer,
969 layout: source.layout,
970 },
971 dst: wgt::TexelCopyTextureInfo::<Arc<Texture>> {
972 texture,
973 mip_level: destination.mip_level,
974 origin: destination.origin,
975 aspect: destination.aspect,
976 },
977 size: *copy_size,
978 })
979 })
980 }
981
982 pub fn copy_buffer_to_texture(
983 self: &Arc<Self>,
984 source: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
985 destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
986 copy_size: &Extent3d,
987 ) {
988 if let Err(err) = self.copy_buffer_to_texture_inner(source, destination, copy_size) {
989 self.device.handle_error(
990 err,
991 Some(self.label()),
992 "CommandEncoder::copy_buffer_to_texture",
993 );
994 }
995 }
996
997 fn copy_texture_to_buffer_inner(
998 self: &Arc<Self>,
999 source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1000 destination: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1001 copy_size: &Extent3d,
1002 ) -> Result<(), EncoderStateError> {
1003 profiling::scope!("CommandEncoder::copy_texture_to_buffer");
1004 api_log!(
1005 "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
1006 Arc::as_ptr(&source.texture),
1007 Arc::as_ptr(&destination.buffer)
1008 );
1009
1010 let mut cmd_buf_data = self.data.lock();
1011
1012 cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
1013 let texture = source.texture.clone();
1014 texture.check_valid()?;
1015 let destination_buffer = destination.buffer.clone();
1016 destination_buffer.check_is_valid()?;
1017 Ok(ArcCommand::CopyTextureToBuffer {
1018 src: wgt::TexelCopyTextureInfo::<Arc<Texture>> {
1019 texture,
1020 mip_level: source.mip_level,
1021 origin: source.origin,
1022 aspect: source.aspect,
1023 },
1024 dst: wgt::TexelCopyBufferInfo::<Arc<Buffer>> {
1025 buffer: destination_buffer,
1026 layout: destination.layout,
1027 },
1028 size: *copy_size,
1029 })
1030 })
1031 }
1032
1033 pub fn copy_texture_to_buffer(
1034 self: &Arc<Self>,
1035 source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1036 destination: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1037 copy_size: &Extent3d,
1038 ) {
1039 if let Err(err) = self.copy_texture_to_buffer_inner(source, destination, copy_size) {
1040 self.device.handle_error(
1041 err,
1042 Some(self.label()),
1043 "CommandEncoder::copy_texture_to_buffer",
1044 );
1045 }
1046 }
1047
1048 fn copy_texture_to_texture_inner(
1049 self: &Arc<Self>,
1050 source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1051 destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1052 copy_size: &Extent3d,
1053 ) -> Result<(), EncoderStateError> {
1054 profiling::scope!("CommandEncoder::copy_texture_to_texture");
1055 api_log!(
1056 "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1057 Arc::as_ptr(&source.texture),
1058 Arc::as_ptr(&destination.texture)
1059 );
1060
1061 let mut cmd_buf_data = self.data.lock();
1062
1063 cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
1064 let src_texture = source.texture.clone();
1065 let dst_texture = destination.texture.clone();
1066 src_texture.check_valid()?;
1067 dst_texture.check_valid()?;
1068 Ok(ArcCommand::CopyTextureToTexture {
1069 src: wgt::TexelCopyTextureInfo {
1070 texture: src_texture,
1071 mip_level: source.mip_level,
1072 origin: source.origin,
1073 aspect: source.aspect,
1074 },
1075 dst: wgt::TexelCopyTextureInfo {
1076 texture: dst_texture,
1077 mip_level: destination.mip_level,
1078 origin: destination.origin,
1079 aspect: destination.aspect,
1080 },
1081 size: *copy_size,
1082 })
1083 })
1084 }
1085
1086 pub fn copy_texture_to_texture(
1087 self: &Arc<Self>,
1088 source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1089 destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1090 copy_size: &Extent3d,
1091 ) {
1092 if let Err(err) = self.copy_texture_to_texture_inner(source, destination, copy_size) {
1093 self.device.handle_error(
1094 err,
1095 Some(self.label()),
1096 "CommandEncoder::copy_texture_to_texture",
1097 );
1098 }
1099 }
1100}
1101
1102pub(super) fn copy_buffer_to_buffer(
1103 state: &mut EncodingState,
1104 src_buffer: &Arc<Buffer>,
1105 source_offset: BufferAddress,
1106 dst_buffer: &Arc<Buffer>,
1107 destination_offset: BufferAddress,
1108 size: Option<BufferAddress>,
1109) -> Result<(), CommandEncoderError> {
1110 if src_buffer.is_equal(dst_buffer) {
1111 return Err(TransferError::SameSourceDestinationBuffer.into());
1112 }
1113
1114 src_buffer.same_device(state.device)?;
1115
1116 let src_pending = state
1117 .tracker
1118 .buffers
1119 .set_single(src_buffer, wgt::BufferUses::COPY_SRC);
1120
1121 let src_raw = src_buffer.try_raw(state.snatch_guard)?;
1122 src_buffer
1123 .check_usage(BufferUsages::COPY_SRC)
1124 .map_err(TransferError::MissingBufferUsage)?;
1125 let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer, state.snatch_guard));
1127
1128 dst_buffer.same_device(state.device)?;
1129
1130 let dst_pending = state
1131 .tracker
1132 .buffers
1133 .set_single(dst_buffer, wgt::BufferUses::COPY_DST);
1134
1135 let dst_raw = dst_buffer.try_raw(state.snatch_guard)?;
1136 dst_buffer
1137 .check_usage(BufferUsages::COPY_DST)
1138 .map_err(TransferError::MissingBufferUsage)?;
1139 let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer, state.snatch_guard));
1140
1141 if source_offset > src_buffer.size {
1142 return Err(TransferError::BufferStartOffsetOverrun {
1143 start_offset: source_offset,
1144 buffer_size: src_buffer.size,
1145 side: CopySide::Source,
1146 }
1147 .into());
1148 }
1149 let size = size.unwrap_or_else(|| {
1150 src_buffer.size - source_offset
1152 });
1153
1154 if !size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1155 return Err(TransferError::UnalignedCopySize(size).into());
1156 }
1157 if !source_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1158 return Err(TransferError::UnalignedBufferOffset(source_offset).into());
1159 }
1160 if !destination_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1161 return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
1162 }
1163 if !state
1164 .device
1165 .downlevel
1166 .flags
1167 .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
1168 && (src_buffer.usage.contains(BufferUsages::INDEX)
1169 || dst_buffer.usage.contains(BufferUsages::INDEX))
1170 {
1171 let forbidden_usages = BufferUsages::VERTEX
1172 | BufferUsages::UNIFORM
1173 | BufferUsages::INDIRECT
1174 | BufferUsages::STORAGE;
1175 if src_buffer.usage.intersects(forbidden_usages)
1176 || dst_buffer.usage.intersects(forbidden_usages)
1177 {
1178 return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
1179 wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
1180 ))
1181 .into());
1182 }
1183 }
1184
1185 if size > src_buffer.size - source_offset {
1186 return Err(TransferError::BufferEndOffsetOverrun {
1187 start_offset: source_offset,
1188 size,
1189 buffer_size: src_buffer.size,
1190 side: CopySide::Source,
1191 }
1192 .into());
1193 }
1194 let source_end_offset = source_offset + size;
1196
1197 if destination_offset > dst_buffer.size {
1198 return Err(TransferError::BufferStartOffsetOverrun {
1199 start_offset: destination_offset,
1200 buffer_size: dst_buffer.size,
1201 side: CopySide::Destination,
1202 }
1203 .into());
1204 }
1205 if size > dst_buffer.size - destination_offset {
1207 return Err(TransferError::BufferEndOffsetOverrun {
1208 start_offset: destination_offset,
1209 size,
1210 buffer_size: dst_buffer.size,
1211 side: CopySide::Destination,
1212 }
1213 .into());
1214 }
1215 let destination_end_offset = destination_offset + size;
1217
1218 if size == 0 {
1221 log::trace!("Ignoring copy_buffer_to_buffer of size 0");
1222 return Ok(());
1223 }
1224
1225 state
1227 .buffer_memory_init_actions
1228 .extend(dst_buffer.initialization_status.read().create_action(
1229 dst_buffer,
1230 destination_offset..destination_end_offset,
1231 MemoryInitKind::ImplicitlyInitialized,
1232 ));
1233 state
1234 .buffer_memory_init_actions
1235 .extend(src_buffer.initialization_status.read().create_action(
1236 src_buffer,
1237 source_offset..source_end_offset,
1238 MemoryInitKind::NeedsInitializedMemory,
1239 ));
1240
1241 let region = hal::BufferCopy {
1242 src_offset: source_offset,
1243 dst_offset: destination_offset,
1244 size: wgt::BufferSize::new(size).unwrap(),
1245 };
1246 let barriers = src_barrier
1247 .into_iter()
1248 .chain(dst_barrier)
1249 .collect::<Vec<_>>();
1250 unsafe {
1251 state.raw_encoder.transition_buffers(&barriers);
1252 state
1253 .raw_encoder
1254 .copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
1255 }
1256
1257 Ok(())
1258}
1259
1260pub(super) fn copy_buffer_to_texture(
1261 state: &mut EncodingState,
1262 source: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1263 destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1264 copy_size: &Extent3d,
1265) -> Result<(), CommandEncoderError> {
1266 let dst_texture = &destination.texture;
1267 let src_buffer = &source.buffer;
1268
1269 dst_texture.same_device(state.device)?;
1270 src_buffer.same_device(state.device)?;
1271
1272 let CopyRange {
1273 hal_copy_extent,
1274 hal_base: dst_base,
1275 array_layer_count,
1276 selector: dst_range,
1277 } = validate_texture_copy_range(
1278 destination,
1279 &dst_texture.desc,
1280 CopySide::Destination,
1281 copy_size,
1282 )?;
1283
1284 let src_raw = src_buffer.try_raw(state.snatch_guard)?;
1285 src_buffer
1286 .check_usage(BufferUsages::COPY_SRC)
1287 .map_err(TransferError::MissingBufferUsage)?;
1288
1289 let dst_raw = dst_texture.try_inner(state.snatch_guard)?.raw();
1290 dst_texture
1291 .check_usage(TextureUsages::COPY_DST)
1292 .map_err(TransferError::MissingTextureUsage)?;
1293
1294 validate_texture_copy_dst_format(dst_texture.desc.format, destination.aspect)?;
1295
1296 validate_texture_buffer_copy(
1297 destination,
1298 dst_base.aspect,
1299 &dst_texture.desc,
1300 &source.layout,
1301 true, state.device.features,
1303 )?;
1304
1305 let (required_buffer_bytes_in_copy, bytes_per_array_layer, is_contiguous) =
1306 validate_linear_texture_data(
1307 &source.layout,
1308 dst_texture.desc.format,
1309 destination.aspect,
1310 src_buffer.size,
1311 CopySide::Source,
1312 copy_size,
1313 )?;
1314
1315 if dst_texture.desc.format.is_depth_stencil_format() {
1316 state
1317 .device
1318 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1319 .map_err(TransferError::from)?;
1320 }
1321
1322 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1325 log::trace!("Ignoring copy_buffer_to_texture of size 0");
1326 return Ok(());
1327 }
1328
1329 handle_dst_texture_init(state, destination, copy_size, dst_texture)?;
1333
1334 let src_pending = state
1335 .tracker
1336 .buffers
1337 .set_single(src_buffer, wgt::BufferUses::COPY_SRC);
1338 let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer, state.snatch_guard));
1339
1340 let dst_pending =
1341 state
1342 .tracker
1343 .textures
1344 .set_single(dst_texture, dst_range, wgt::TextureUses::COPY_DST);
1345 let dst_barrier = dst_pending
1346 .map(|pending| pending.into_hal(dst_raw))
1347 .collect::<Vec<_>>();
1348
1349 handle_buffer_init(
1350 state,
1351 source,
1352 CopySide::Source,
1353 required_buffer_bytes_in_copy,
1354 is_contiguous,
1355 );
1356
1357 let regions = (0..array_layer_count)
1358 .map(|rel_array_layer| {
1359 let mut texture_base = dst_base.clone();
1360 texture_base.array_layer += rel_array_layer;
1361 let mut buffer_layout = source.layout;
1362 buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1363 hal::BufferTextureCopy {
1364 buffer_layout,
1365 texture_base,
1366 size: hal_copy_extent,
1367 }
1368 })
1369 .collect::<Vec<_>>();
1370
1371 unsafe {
1372 state.raw_encoder.transition_textures(&dst_barrier);
1373 state.raw_encoder.transition_buffers(src_barrier.as_slice());
1374 state
1375 .raw_encoder
1376 .copy_buffer_to_texture(src_raw, dst_raw, ®ions);
1377 }
1378
1379 Ok(())
1380}
1381
1382pub(super) fn copy_texture_to_buffer(
1383 state: &mut EncodingState,
1384 source: &TexelCopyTextureInfo,
1385 destination: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1386 copy_size: &Extent3d,
1387) -> Result<(), CommandEncoderError> {
1388 let src_texture = &source.texture;
1389 let dst_buffer = &destination.buffer;
1390
1391 src_texture.same_device(state.device)?;
1392 dst_buffer.same_device(state.device)?;
1393
1394 let CopyRange {
1395 hal_copy_extent,
1396 hal_base: src_base,
1397 array_layer_count,
1398 selector: src_range,
1399 } = validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1400
1401 let src_raw = src_texture.try_inner(state.snatch_guard)?.raw();
1402 src_texture
1403 .check_usage(TextureUsages::COPY_SRC)
1404 .map_err(TransferError::MissingTextureUsage)?;
1405
1406 if source.mip_level >= src_texture.desc.mip_level_count {
1407 return Err(TransferError::InvalidMipLevel {
1408 requested: source.mip_level,
1409 count: src_texture.desc.mip_level_count,
1410 }
1411 .into());
1412 }
1413
1414 validate_texture_copy_src_format(src_texture.desc.format, source.aspect)?;
1415
1416 validate_texture_buffer_copy(
1417 source,
1418 src_base.aspect,
1419 &src_texture.desc,
1420 &destination.layout,
1421 true, state.device.features,
1423 )?;
1424
1425 let (required_buffer_bytes_in_copy, bytes_per_array_layer, is_contiguous) =
1426 validate_linear_texture_data(
1427 &destination.layout,
1428 src_texture.desc.format,
1429 source.aspect,
1430 dst_buffer.size,
1431 CopySide::Destination,
1432 copy_size,
1433 )?;
1434
1435 if src_texture.desc.format.is_depth_stencil_format() {
1436 state
1437 .device
1438 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1439 .map_err(TransferError::from)?;
1440 }
1441
1442 let dst_raw = dst_buffer.try_raw(state.snatch_guard)?;
1443 dst_buffer
1444 .check_usage(BufferUsages::COPY_DST)
1445 .map_err(TransferError::MissingBufferUsage)?;
1446
1447 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1450 log::trace!("Ignoring copy_texture_to_buffer of size 0");
1451 return Ok(());
1452 }
1453
1454 handle_src_texture_init(state, source, copy_size, src_texture)?;
1458
1459 let src_pending =
1460 state
1461 .tracker
1462 .textures
1463 .set_single(src_texture, src_range, wgt::TextureUses::COPY_SRC);
1464 let src_barrier = src_pending
1465 .map(|pending| pending.into_hal(src_raw))
1466 .collect::<Vec<_>>();
1467
1468 let dst_pending = state
1469 .tracker
1470 .buffers
1471 .set_single(dst_buffer, wgt::BufferUses::COPY_DST);
1472
1473 let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer, state.snatch_guard));
1474
1475 handle_buffer_init(
1476 state,
1477 destination,
1478 CopySide::Destination,
1479 required_buffer_bytes_in_copy,
1480 is_contiguous,
1481 );
1482
1483 let regions = (0..array_layer_count)
1484 .map(|rel_array_layer| {
1485 let mut texture_base = src_base.clone();
1486 texture_base.array_layer += rel_array_layer;
1487 let mut buffer_layout = destination.layout;
1488 buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1489 hal::BufferTextureCopy {
1490 buffer_layout,
1491 texture_base,
1492 size: hal_copy_extent,
1493 }
1494 })
1495 .collect::<Vec<_>>();
1496 unsafe {
1497 state.raw_encoder.transition_buffers(dst_barrier.as_slice());
1498 state.raw_encoder.transition_textures(&src_barrier);
1499 state.raw_encoder.copy_texture_to_buffer(
1500 src_raw,
1501 wgt::TextureUses::COPY_SRC,
1502 dst_raw,
1503 ®ions,
1504 );
1505 }
1506
1507 Ok(())
1508}
1509
1510pub(super) fn copy_texture_to_texture(
1511 state: &mut EncodingState,
1512 source: &TexelCopyTextureInfo,
1513 destination: &TexelCopyTextureInfo,
1514 copy_size: &Extent3d,
1515) -> Result<(), CommandEncoderError> {
1516 let src_texture = &source.texture;
1517 let dst_texture = &destination.texture;
1518
1519 src_texture.same_device(state.device)?;
1520 dst_texture.same_device(state.device)?;
1521
1522 let CopyRange {
1523 hal_copy_extent: src_copy_size,
1524 hal_base: src_tex_base,
1525 array_layer_count,
1526 selector: src_range,
1527 } = validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1528 let CopyRange {
1529 hal_copy_extent: dst_copy_size,
1530 hal_base: dst_tex_base,
1531 array_layer_count: _,
1532 selector: dst_range,
1533 } = validate_texture_copy_range(
1534 destination,
1535 &dst_texture.desc,
1536 CopySide::Destination,
1537 copy_size,
1538 )?;
1539
1540 let copy_format =
1554 |format: wgt::TextureFormat, aspect: wgt::TextureAspect| match aspect.to_plane() {
1555 Some(_) => {
1556 let plane_fmt = format
1558 .aspect_specific_format(aspect)
1559 .expect("aspect was validated previously");
1560 (plane_fmt, plane_fmt)
1561 }
1562 None => {
1563 (format, format.remove_srgb_suffix())
1566 }
1567 };
1568 let (src_format, src_format_no_srgb) = copy_format(src_texture.desc.format, source.aspect);
1569 let (dst_format, dst_format_no_srgb) = copy_format(dst_texture.desc.format, destination.aspect);
1570 if src_format_no_srgb != dst_format_no_srgb {
1571 return Err(TransferError::TextureFormatsNotCopyCompatible {
1576 src_format,
1577 src_multi_planar: source
1578 .aspect
1579 .to_plane()
1580 .map(|plane| (src_texture.desc.format, plane)),
1581 dst_format,
1582 dst_multi_planar: destination
1583 .aspect
1584 .to_plane()
1585 .map(|plane| (dst_texture.desc.format, plane)),
1586 }
1587 .into());
1588 }
1589
1590 if Arc::as_ptr(src_texture) == Arc::as_ptr(dst_texture) {
1591 validate_copy_within_same_texture(
1592 source,
1593 destination,
1594 src_texture.desc.format,
1595 array_layer_count,
1596 )?;
1597 }
1598
1599 let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1600 let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1601 if src_texture.desc.format.is_depth_stencil_format()
1602 && src_tex_base.aspect != src_texture_aspects
1603 {
1604 return Err(TransferError::CopySrcMissingAspects.into());
1605 }
1606 if dst_texture.desc.format.is_depth_stencil_format()
1607 && dst_tex_base.aspect != dst_texture_aspects
1608 {
1609 return Err(TransferError::CopyDstMissingAspects.into());
1610 }
1611
1612 if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1613 return Err(TransferError::SampleCountNotEqual {
1614 src_sample_count: src_texture.desc.sample_count,
1615 dst_sample_count: dst_texture.desc.sample_count,
1616 }
1617 .into());
1618 }
1619
1620 let src_raw = src_texture.try_inner(state.snatch_guard)?.raw();
1621 src_texture
1622 .check_usage(TextureUsages::COPY_SRC)
1623 .map_err(TransferError::MissingTextureUsage)?;
1624 let dst_raw = dst_texture.try_inner(state.snatch_guard)?.raw();
1625 dst_texture
1626 .check_usage(TextureUsages::COPY_DST)
1627 .map_err(TransferError::MissingTextureUsage)?;
1628
1629 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1632 log::trace!("Ignoring copy_texture_to_texture of size 0");
1633 return Ok(());
1634 }
1635
1636 handle_src_texture_init(state, source, copy_size, src_texture)?;
1640 handle_dst_texture_init(state, destination, copy_size, dst_texture)?;
1641
1642 let src_pending =
1643 state
1644 .tracker
1645 .textures
1646 .set_single(src_texture, src_range, wgt::TextureUses::COPY_SRC);
1647
1648 let mut barriers: ArrayVec<_, 2> = src_pending
1651 .map(|pending| pending.into_hal(src_raw))
1652 .collect();
1653
1654 let dst_pending =
1655 state
1656 .tracker
1657 .textures
1658 .set_single(dst_texture, dst_range, wgt::TextureUses::COPY_DST);
1659 barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1660
1661 let hal_copy_size = hal::CopyExtent {
1662 width: src_copy_size.width.min(dst_copy_size.width),
1663 height: src_copy_size.height.min(dst_copy_size.height),
1664 depth: src_copy_size.depth.min(dst_copy_size.depth),
1665 };
1666
1667 let dst_format = dst_texture.desc.format;
1668
1669 let regions = (0..array_layer_count).map(|rel_array_layer| {
1670 let mut src_base = src_tex_base.clone();
1671 let mut dst_base = dst_tex_base.clone();
1672 src_base.array_layer += rel_array_layer;
1673 dst_base.array_layer += rel_array_layer;
1674 hal::TextureCopy {
1675 src_base,
1676 dst_base,
1677 size: hal_copy_size,
1678 }
1679 });
1680
1681 let regions = if dst_tex_base.aspect == hal::FormatAspects::DEPTH_STENCIL {
1682 regions
1683 .flat_map(|region| {
1684 let (mut depth, mut stencil) = (region.clone(), region);
1685 depth.src_base.aspect = hal::FormatAspects::DEPTH;
1686 depth.dst_base.aspect = hal::FormatAspects::DEPTH;
1687 stencil.src_base.aspect = hal::FormatAspects::STENCIL;
1688 stencil.dst_base.aspect = hal::FormatAspects::STENCIL;
1689 [depth, stencil]
1690 })
1691 .collect::<Vec<_>>()
1692 } else if let (Some(plane_count), wgt::TextureAspect::All) =
1693 (dst_format.planes(), destination.aspect)
1694 {
1695 regions
1696 .into_iter()
1697 .flat_map(|region| {
1698 (0..plane_count).map(move |plane| {
1699 let mut plane_region = region.clone();
1700
1701 let plane_aspect = wgt::TextureAspect::from_plane(plane)
1702 .expect("expected texture aspect to exist for the plane");
1703 let plane_aspect = hal::FormatAspects::new(dst_format, plane_aspect);
1704 plane_region.src_base.aspect = plane_aspect;
1705 plane_region.dst_base.aspect = plane_aspect;
1706
1707 let (w_subsampling, h_subsampling) =
1708 dst_format.subsampling_factors(Some(plane));
1709 plane_region.src_base.origin.x /= w_subsampling;
1710 plane_region.src_base.origin.y /= h_subsampling;
1711 plane_region.dst_base.origin.x /= w_subsampling;
1712 plane_region.dst_base.origin.y /= h_subsampling;
1713
1714 plane_region.size.width /= w_subsampling;
1715 plane_region.size.height /= h_subsampling;
1716
1717 plane_region
1718 })
1719 })
1720 .collect::<Vec<_>>()
1721 } else {
1722 regions.collect::<Vec<_>>()
1723 };
1724 unsafe {
1725 state.raw_encoder.transition_textures(&barriers);
1726 state.raw_encoder.copy_texture_to_texture(
1727 src_raw,
1728 wgt::TextureUses::COPY_SRC,
1729 dst_raw,
1730 ®ions,
1731 );
1732 }
1733
1734 Ok(())
1735}