wgpu_core/command/
transfer.rs

1use alloc::{format, string::String, sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6    error::{ErrorType, WebGpuError},
7    BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8    TextureUsages,
9};
10
11#[cfg(feature = "trace")]
12use crate::command::Command as TraceCommand;
13use crate::{
14    api_log,
15    command::{clear_texture, CommandEncoderError, EncoderStateError},
16    device::{Device, MissingDownlevelFlags},
17    global::Global,
18    id::{BufferId, CommandEncoderId},
19    init_tracker::{
20        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
21        TextureInitTrackerAction,
22    },
23    resource::{
24        Buffer, MissingBufferUsageError, MissingTextureUsageError, ParentDevice, RawResourceAccess,
25        Texture, TextureErrorDimension,
26    },
27    snatch::SnatchGuard,
28};
29
30use super::{ClearError, CommandBufferMutable};
31
32use super::TexelCopyBufferInfo;
33use super::TexelCopyTextureInfo;
34
35#[derive(Clone, Copy, Debug, Eq, PartialEq)]
36pub enum CopySide {
37    Source,
38    Destination,
39}
40
41/// Error encountered while attempting a data transfer.
42#[derive(Clone, Debug, Error)]
43#[non_exhaustive]
44pub enum TransferError {
45    #[error("Source and destination cannot be the same buffer")]
46    SameSourceDestinationBuffer,
47    #[error(transparent)]
48    MissingBufferUsage(#[from] MissingBufferUsageError),
49    #[error(transparent)]
50    MissingTextureUsage(#[from] MissingTextureUsageError),
51    #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
52    BufferOverrun {
53        start_offset: BufferAddress,
54        end_offset: BufferAddress,
55        buffer_size: BufferAddress,
56        side: CopySide,
57    },
58    #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
59    TextureOverrun {
60        start_offset: u32,
61        end_offset: u32,
62        texture_size: u32,
63        dimension: TextureErrorDimension,
64        side: CopySide,
65    },
66    #[error("Partial copy of {start_offset}..{end_offset} on {dimension:?} dimension with size {texture_size} \
67             is not supported for the {side:?} texture format {format:?} with {sample_count} samples")]
68    UnsupportedPartialTransfer {
69        format: wgt::TextureFormat,
70        sample_count: u32,
71        start_offset: u32,
72        end_offset: u32,
73        texture_size: u32,
74        dimension: TextureErrorDimension,
75        side: CopySide,
76    },
77    #[error(
78        "Copying{} layers {}..{} to{} layers {}..{} of the same texture is not allowed",
79        if *src_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {src_aspects:?}") },
80        src_origin_z,
81        src_origin_z + array_layer_count,
82        if *dst_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {dst_aspects:?}") },
83        dst_origin_z,
84        dst_origin_z + array_layer_count,
85    )]
86    InvalidCopyWithinSameTexture {
87        src_aspects: wgt::TextureAspect,
88        dst_aspects: wgt::TextureAspect,
89        src_origin_z: u32,
90        dst_origin_z: u32,
91        array_layer_count: u32,
92    },
93    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
94    InvalidTextureAspect {
95        format: wgt::TextureFormat,
96        aspect: wgt::TextureAspect,
97    },
98    #[error("Unable to select texture mip level {level} out of {total}")]
99    InvalidTextureMipLevel { level: u32, total: u32 },
100    #[error("Texture dimension must be 2D when copying from an external texture")]
101    InvalidDimensionExternal,
102    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
103    UnalignedBufferOffset(BufferAddress),
104    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
105    UnalignedCopySize(BufferAddress),
106    #[error("Copy width is not a multiple of block width")]
107    UnalignedCopyWidth,
108    #[error("Copy height is not a multiple of block height")]
109    UnalignedCopyHeight,
110    #[error("Copy origin's x component is not a multiple of block width")]
111    UnalignedCopyOriginX,
112    #[error("Copy origin's y component is not a multiple of block height")]
113    UnalignedCopyOriginY,
114    #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
115    UnalignedBytesPerRow,
116    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
117    UnspecifiedBytesPerRow,
118    #[error("Number of rows per image needs to be specified since more than one image is copied")]
119    UnspecifiedRowsPerImage,
120    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
121    InvalidBytesPerRow,
122    #[error("Number of rows per image is invalid")]
123    InvalidRowsPerImage,
124    #[error("Overflow while computing the size of the copy")]
125    SizeOverflow,
126    #[error("Copy source aspects must refer to all aspects of the source texture format")]
127    CopySrcMissingAspects,
128    #[error(
129        "Copy destination aspects must refer to all aspects of the destination texture format"
130    )]
131    CopyDstMissingAspects,
132    #[error("Copy aspect must refer to a single aspect of texture format")]
133    CopyAspectNotOne,
134    #[error("Copying from textures with format {0:?} is forbidden")]
135    CopyFromForbiddenTextureFormat(wgt::TextureFormat),
136    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
137    CopyFromForbiddenTextureFormatAspect {
138        format: wgt::TextureFormat,
139        aspect: wgt::TextureAspect,
140    },
141    #[error("Copying to textures with format {0:?} is forbidden")]
142    CopyToForbiddenTextureFormat(wgt::TextureFormat),
143    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
144    CopyToForbiddenTextureFormatAspect {
145        format: wgt::TextureFormat,
146        aspect: wgt::TextureAspect,
147    },
148    #[error(
149        "Copying to textures with format {0:?} is forbidden when copying from external texture"
150    )]
151    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
152    #[error(
153        "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
154    )]
155    TextureFormatsNotCopyCompatible {
156        src_format: wgt::TextureFormat,
157        dst_format: wgt::TextureFormat,
158    },
159    #[error(transparent)]
160    MemoryInitFailure(#[from] ClearError),
161    #[error("Cannot encode this copy because of a missing downelevel flag")]
162    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
163    #[error("Source texture sample count must be 1, got {sample_count}")]
164    InvalidSampleCount { sample_count: u32 },
165    #[error(
166        "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
167    )]
168    SampleCountNotEqual {
169        src_sample_count: u32,
170        dst_sample_count: u32,
171    },
172    #[error("Requested mip level {requested} does no exist (count: {count})")]
173    InvalidMipLevel { requested: u32, count: u32 },
174}
175
176impl WebGpuError for TransferError {
177    fn webgpu_error_type(&self) -> ErrorType {
178        let e: &dyn WebGpuError = match self {
179            Self::MissingBufferUsage(e) => e,
180            Self::MissingTextureUsage(e) => e,
181            Self::MemoryInitFailure(e) => e,
182
183            Self::BufferOverrun { .. }
184            | Self::TextureOverrun { .. }
185            | Self::UnsupportedPartialTransfer { .. }
186            | Self::InvalidCopyWithinSameTexture { .. }
187            | Self::InvalidTextureAspect { .. }
188            | Self::InvalidTextureMipLevel { .. }
189            | Self::InvalidDimensionExternal
190            | Self::UnalignedBufferOffset(..)
191            | Self::UnalignedCopySize(..)
192            | Self::UnalignedCopyWidth
193            | Self::UnalignedCopyHeight
194            | Self::UnalignedCopyOriginX
195            | Self::UnalignedCopyOriginY
196            | Self::UnalignedBytesPerRow
197            | Self::UnspecifiedBytesPerRow
198            | Self::UnspecifiedRowsPerImage
199            | Self::InvalidBytesPerRow
200            | Self::InvalidRowsPerImage
201            | Self::SizeOverflow
202            | Self::CopySrcMissingAspects
203            | Self::CopyDstMissingAspects
204            | Self::CopyAspectNotOne
205            | Self::CopyFromForbiddenTextureFormat(..)
206            | Self::CopyFromForbiddenTextureFormatAspect { .. }
207            | Self::CopyToForbiddenTextureFormat(..)
208            | Self::CopyToForbiddenTextureFormatAspect { .. }
209            | Self::ExternalCopyToForbiddenTextureFormat(..)
210            | Self::TextureFormatsNotCopyCompatible { .. }
211            | Self::MissingDownlevelFlags(..)
212            | Self::InvalidSampleCount { .. }
213            | Self::SampleCountNotEqual { .. }
214            | Self::InvalidMipLevel { .. }
215            | Self::SameSourceDestinationBuffer => return ErrorType::Validation,
216        };
217        e.webgpu_error_type()
218    }
219}
220
221impl From<BufferTextureCopyInfoError> for TransferError {
222    fn from(value: BufferTextureCopyInfoError) -> Self {
223        match value {
224            BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
225            BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
226            BufferTextureCopyInfoError::ImageStrideOverflow
227            | BufferTextureCopyInfoError::ImageBytesOverflow(_)
228            | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
229        }
230    }
231}
232
233pub(crate) fn extract_texture_selector<T>(
234    copy_texture: &wgt::TexelCopyTextureInfo<T>,
235    copy_size: &Extent3d,
236    texture: &Texture,
237) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
238    let format = texture.desc.format;
239    let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
240    if copy_aspect.is_empty() {
241        return Err(TransferError::InvalidTextureAspect {
242            format,
243            aspect: copy_texture.aspect,
244        });
245    }
246
247    let (layers, origin_z) = match texture.desc.dimension {
248        wgt::TextureDimension::D1 => (0..1, 0),
249        wgt::TextureDimension::D2 => (
250            copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
251            0,
252        ),
253        wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
254    };
255    let base = hal::TextureCopyBase {
256        origin: wgt::Origin3d {
257            x: copy_texture.origin.x,
258            y: copy_texture.origin.y,
259            z: origin_z,
260        },
261        // this value will be incremented per copied layer
262        array_layer: layers.start,
263        mip_level: copy_texture.mip_level,
264        aspect: copy_aspect,
265    };
266    let selector = TextureSelector {
267        mips: copy_texture.mip_level..copy_texture.mip_level + 1,
268        layers,
269    };
270
271    Ok((selector, base))
272}
273
274/// WebGPU's [validating linear texture data][vltd] algorithm.
275///
276/// Copied with some modifications from WebGPU standard.
277///
278/// If successful, returns a tuple `(bytes, stride, is_contiguous)`, where:
279/// - `bytes` is the number of buffer bytes required for this copy, and
280/// - `stride` number of bytes between array layers.
281/// - `is_contiguous` is true if the linear texture data does not have padding
282///   between rows or between images.
283///
284/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
285pub(crate) fn validate_linear_texture_data(
286    layout: &wgt::TexelCopyBufferLayout,
287    format: wgt::TextureFormat,
288    aspect: wgt::TextureAspect,
289    buffer_size: BufferAddress,
290    buffer_side: CopySide,
291    copy_size: &Extent3d,
292) -> Result<(BufferAddress, BufferAddress, bool), TransferError> {
293    let wgt::BufferTextureCopyInfo {
294        copy_width,
295        copy_height,
296        depth_or_array_layers,
297
298        offset,
299
300        block_size_bytes: _,
301        block_width_texels,
302        block_height_texels,
303
304        width_blocks: _,
305        height_blocks,
306
307        row_bytes_dense,
308        row_stride_bytes,
309
310        image_stride_rows: _,
311        image_stride_bytes,
312
313        image_rows_dense: _,
314        image_bytes_dense,
315
316        bytes_in_copy,
317    } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
318
319    if copy_width % block_width_texels != 0 {
320        return Err(TransferError::UnalignedCopyWidth);
321    }
322    if copy_height % block_height_texels != 0 {
323        return Err(TransferError::UnalignedCopyHeight);
324    }
325
326    let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
327    let requires_multiple_images = depth_or_array_layers > 1;
328
329    // `get_buffer_texture_copy_info()` already proceeded with defaults if these
330    // were not specified, and ensured that the values satisfy the minima if
331    // they were, but now we enforce the WebGPU requirement that they be
332    // specified any time they apply.
333    if layout.bytes_per_row.is_none() && requires_multiple_rows {
334        return Err(TransferError::UnspecifiedBytesPerRow);
335    }
336
337    if layout.rows_per_image.is_none() && requires_multiple_images {
338        return Err(TransferError::UnspecifiedRowsPerImage);
339    };
340
341    // Avoid underflow in the subtraction by checking bytes_in_copy against buffer_size first.
342    if bytes_in_copy > buffer_size || offset > buffer_size - bytes_in_copy {
343        return Err(TransferError::BufferOverrun {
344            start_offset: offset,
345            end_offset: offset.wrapping_add(bytes_in_copy),
346            buffer_size,
347            side: buffer_side,
348        });
349    }
350
351    let is_contiguous = (row_stride_bytes == row_bytes_dense || !requires_multiple_rows)
352        && (image_stride_bytes == image_bytes_dense || !requires_multiple_images);
353
354    Ok((bytes_in_copy, image_stride_bytes, is_contiguous))
355}
356
357/// Validate the source format of a texture copy.
358///
359/// This performs the check from WebGPU's [validating texture buffer copy][vtbc]
360/// algorithm that ensures that the format and aspect form a valid texel copy source
361/// as defined in the [depth-stencil formats][dsf].
362///
363/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
364/// [dsf]: https://gpuweb.github.io/gpuweb/#depth-formats
365pub(crate) fn validate_texture_copy_src_format(
366    format: wgt::TextureFormat,
367    aspect: wgt::TextureAspect,
368) -> Result<(), TransferError> {
369    use wgt::TextureAspect as Ta;
370    use wgt::TextureFormat as Tf;
371    match (format, aspect) {
372        (Tf::Depth24Plus, _) => Err(TransferError::CopyFromForbiddenTextureFormat(format)),
373        (Tf::Depth24PlusStencil8, Ta::DepthOnly) => {
374            Err(TransferError::CopyFromForbiddenTextureFormatAspect { format, aspect })
375        }
376        _ => Ok(()),
377    }
378}
379
380/// Validate the destination format of a texture copy.
381///
382/// This performs the check from WebGPU's [validating texture buffer copy][vtbc]
383/// algorithm that ensures that the format and aspect form a valid texel copy destination
384/// as defined in the [depth-stencil formats][dsf].
385///
386/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
387/// [dsf]: https://gpuweb.github.io/gpuweb/#depth-formats
388pub(crate) fn validate_texture_copy_dst_format(
389    format: wgt::TextureFormat,
390    aspect: wgt::TextureAspect,
391) -> Result<(), TransferError> {
392    use wgt::TextureAspect as Ta;
393    use wgt::TextureFormat as Tf;
394    match (format, aspect) {
395        (Tf::Depth24Plus | Tf::Depth32Float, _) => {
396            Err(TransferError::CopyToForbiddenTextureFormat(format))
397        }
398        (Tf::Depth24PlusStencil8 | Tf::Depth32FloatStencil8, Ta::DepthOnly) => {
399            Err(TransferError::CopyToForbiddenTextureFormatAspect { format, aspect })
400        }
401        _ => Ok(()),
402    }
403}
404
405/// Validation for texture/buffer copies.
406///
407/// This implements the following checks from WebGPU's [validating texture buffer copy][vtbc]
408/// algorithm:
409///  * The texture must not be multisampled.
410///  * The copy must be from/to a single aspect of the texture.
411///  * If `aligned` is true, the buffer offset must be aligned appropriately.
412///
413/// And implements the following check from WebGPU's [validating GPUTexelCopyBufferInfo][vtcbi]
414/// algorithm:
415///  * If `aligned` is true, `bytesPerRow` must be a multiple of 256.
416///
417/// Note that the `bytesPerRow` alignment check is enforced whenever
418/// `bytesPerRow` is specified, even if the transfer is not multiple rows and
419/// `bytesPerRow` could have been omitted.
420///
421/// The following steps in [validating texture buffer copy][vtbc] are implemented elsewhere:
422///  * Invocation of other validation algorithms.
423///  * The texture usage (COPY_DST / COPY_SRC) check.
424///  * The check for non-copyable depth/stencil formats. The caller must perform
425///    this check using `validate_texture_copy_src_format` / `validate_texture_copy_dst_format`
426///    before calling this function. This function will panic if
427///    [`wgt::TextureFormat::block_copy_size`] returns `None` due to a
428///    non-copyable format.
429///
430/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
431/// [vtcbi]: https://www.w3.org/TR/webgpu/#abstract-opdef-validating-gputexelcopybufferinfo
432pub(crate) fn validate_texture_buffer_copy<T>(
433    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
434    aspect: hal::FormatAspects,
435    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
436    layout: &wgt::TexelCopyBufferLayout,
437    aligned: bool,
438) -> Result<(), TransferError> {
439    if desc.sample_count != 1 {
440        return Err(TransferError::InvalidSampleCount {
441            sample_count: desc.sample_count,
442        });
443    }
444
445    if !aspect.is_one() {
446        return Err(TransferError::CopyAspectNotOne);
447    }
448
449    let offset_alignment = if desc.format.is_depth_stencil_format() {
450        4
451    } else {
452        // The case where `block_copy_size` returns `None` is currently
453        // unreachable both for the reason in the expect message, and also
454        // because the currently-defined non-copyable formats are depth/stencil
455        // formats so would take the `if` branch.
456        desc.format
457            .block_copy_size(Some(texture_copy_view.aspect))
458            .expect("non-copyable formats should have been rejected previously")
459    };
460
461    if aligned && layout.offset % u64::from(offset_alignment) != 0 {
462        return Err(TransferError::UnalignedBufferOffset(layout.offset));
463    }
464
465    if let Some(bytes_per_row) = layout.bytes_per_row {
466        if aligned && bytes_per_row % wgt::COPY_BYTES_PER_ROW_ALIGNMENT != 0 {
467            return Err(TransferError::UnalignedBytesPerRow);
468        }
469    }
470
471    Ok(())
472}
473
474/// Validate the extent and alignment of a texture copy.
475///
476/// Copied with minor modifications from WebGPU standard. This mostly follows
477/// the [validating GPUTexelCopyTextureInfo][vtcti] and [validating texture copy
478/// range][vtcr] algorithms.
479///
480/// Returns the HAL copy extent and the layer count.
481///
482/// [vtcti]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gputexelcopytextureinfo
483/// [vtcr]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-copy-range
484pub(crate) fn validate_texture_copy_range<T>(
485    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
486    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
487    texture_side: CopySide,
488    copy_size: &Extent3d,
489) -> Result<(hal::CopyExtent, u32), TransferError> {
490    let (block_width, block_height) = desc.format.block_dimensions();
491
492    let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
493        TransferError::InvalidTextureMipLevel {
494            level: texture_copy_view.mip_level,
495            total: desc.mip_level_count,
496        },
497    )?;
498    // physical size can be larger than the virtual
499    let extent = extent_virtual.physical_size(desc.format);
500
501    // Multisampled and depth-stencil formats do not support partial copies
502    // on x and y dimensions, but do support copying a subset of layers.
503    let requires_exact_size = desc.format.is_depth_stencil_format() || desc.sample_count > 1;
504
505    // Return `Ok` if a run `size` texels long starting at `start_offset` is
506    // valid for `texture_size`. Otherwise, return an appropriate a`Err`.
507    let check_dimension = |dimension: TextureErrorDimension,
508                           start_offset: u32,
509                           size: u32,
510                           texture_size: u32,
511                           requires_exact_size: bool|
512     -> Result<(), TransferError> {
513        if requires_exact_size && (start_offset != 0 || size != texture_size) {
514            Err(TransferError::UnsupportedPartialTransfer {
515                format: desc.format,
516                sample_count: desc.sample_count,
517                start_offset,
518                end_offset: start_offset.wrapping_add(size),
519                texture_size,
520                dimension,
521                side: texture_side,
522            })
523        // Avoid underflow in the subtraction by checking start_offset against
524        // texture_size first.
525        } else if start_offset > texture_size || texture_size - start_offset < size {
526            Err(TransferError::TextureOverrun {
527                start_offset,
528                end_offset: start_offset.wrapping_add(size),
529                texture_size,
530                dimension,
531                side: texture_side,
532            })
533        } else {
534            Ok(())
535        }
536    };
537
538    check_dimension(
539        TextureErrorDimension::X,
540        texture_copy_view.origin.x,
541        copy_size.width,
542        extent.width,
543        requires_exact_size,
544    )?;
545    check_dimension(
546        TextureErrorDimension::Y,
547        texture_copy_view.origin.y,
548        copy_size.height,
549        extent.height,
550        requires_exact_size,
551    )?;
552    check_dimension(
553        TextureErrorDimension::Z,
554        texture_copy_view.origin.z,
555        copy_size.depth_or_array_layers,
556        extent.depth_or_array_layers,
557        false, // partial copy always allowed on Z/layer dimension
558    )?;
559
560    if texture_copy_view.origin.x % block_width != 0 {
561        return Err(TransferError::UnalignedCopyOriginX);
562    }
563    if texture_copy_view.origin.y % block_height != 0 {
564        return Err(TransferError::UnalignedCopyOriginY);
565    }
566    if copy_size.width % block_width != 0 {
567        return Err(TransferError::UnalignedCopyWidth);
568    }
569    if copy_size.height % block_height != 0 {
570        return Err(TransferError::UnalignedCopyHeight);
571    }
572
573    let (depth, array_layer_count) = match desc.dimension {
574        wgt::TextureDimension::D1 => (1, 1),
575        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
576        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
577    };
578
579    let copy_extent = hal::CopyExtent {
580        width: copy_size.width,
581        height: copy_size.height,
582        depth,
583    };
584    Ok((copy_extent, array_layer_count))
585}
586
587/// Validate a copy within the same texture.
588///
589/// This implements the WebGPU requirement that the [sets of subresources for
590/// texture copy][srtc] of the source and destination be disjoint, i.e. that the
591/// source and destination do not overlap.
592///
593/// This function assumes that the copy ranges have already been validated with
594/// `validate_texture_copy_range`.
595///
596/// [srtc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-set-of-subresources-for-texture-copy
597pub(crate) fn validate_copy_within_same_texture<T>(
598    src: &wgt::TexelCopyTextureInfo<T>,
599    dst: &wgt::TexelCopyTextureInfo<T>,
600    format: wgt::TextureFormat,
601    array_layer_count: u32,
602) -> Result<(), TransferError> {
603    let src_aspects = hal::FormatAspects::new(format, src.aspect);
604    let dst_aspects = hal::FormatAspects::new(format, dst.aspect);
605    if (src_aspects & dst_aspects).is_empty() {
606        // Copying between different aspects (if it even makes sense), is okay.
607        return Ok(());
608    }
609
610    if src.origin.z >= dst.origin.z + array_layer_count
611        || dst.origin.z >= src.origin.z + array_layer_count
612    {
613        // Copying between non-overlapping layer ranges is okay.
614        return Ok(());
615    }
616
617    if src.mip_level != dst.mip_level {
618        // Copying between different mip levels is okay.
619        return Ok(());
620    }
621
622    Err(TransferError::InvalidCopyWithinSameTexture {
623        src_aspects: src.aspect,
624        dst_aspects: dst.aspect,
625        src_origin_z: src.origin.z,
626        dst_origin_z: dst.origin.z,
627        array_layer_count,
628    })
629}
630
631fn handle_texture_init(
632    init_kind: MemoryInitKind,
633    cmd_buf_data: &mut CommandBufferMutable,
634    device: &Device,
635    copy_texture: &TexelCopyTextureInfo,
636    copy_size: &Extent3d,
637    texture: &Arc<Texture>,
638    snatch_guard: &SnatchGuard<'_>,
639) -> Result<(), ClearError> {
640    let init_action = TextureInitTrackerAction {
641        texture: texture.clone(),
642        range: TextureInitRange {
643            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
644            layer_range: copy_texture.origin.z
645                ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
646        },
647        kind: init_kind,
648    };
649
650    // Register the init action.
651    let immediate_inits = cmd_buf_data
652        .texture_memory_actions
653        .register_init_action(&{ init_action });
654
655    // In rare cases we may need to insert an init operation immediately onto the command buffer.
656    if !immediate_inits.is_empty() {
657        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
658        for init in immediate_inits {
659            clear_texture(
660                &init.texture,
661                TextureInitRange {
662                    mip_range: init.mip_level..(init.mip_level + 1),
663                    layer_range: init.layer..(init.layer + 1),
664                },
665                cmd_buf_raw,
666                &mut cmd_buf_data.trackers.textures,
667                &device.alignments,
668                device.zero_buffer.as_ref(),
669                snatch_guard,
670                device.instance_flags,
671            )?;
672        }
673    }
674
675    Ok(())
676}
677
678/// Prepare a transfer's source texture.
679///
680/// Ensure the source texture of a transfer is in the right initialization
681/// state, and record the state for after the transfer operation.
682fn handle_src_texture_init(
683    cmd_buf_data: &mut CommandBufferMutable,
684    device: &Device,
685    source: &TexelCopyTextureInfo,
686    copy_size: &Extent3d,
687    texture: &Arc<Texture>,
688    snatch_guard: &SnatchGuard<'_>,
689) -> Result<(), TransferError> {
690    handle_texture_init(
691        MemoryInitKind::NeedsInitializedMemory,
692        cmd_buf_data,
693        device,
694        source,
695        copy_size,
696        texture,
697        snatch_guard,
698    )?;
699    Ok(())
700}
701
702/// Prepare a transfer's destination texture.
703///
704/// Ensure the destination texture of a transfer is in the right initialization
705/// state, and record the state for after the transfer operation.
706fn handle_dst_texture_init(
707    cmd_buf_data: &mut CommandBufferMutable,
708    device: &Device,
709    destination: &TexelCopyTextureInfo,
710    copy_size: &Extent3d,
711    texture: &Arc<Texture>,
712    snatch_guard: &SnatchGuard<'_>,
713) -> Result<(), TransferError> {
714    // Attention: If we don't write full texture subresources, we need to a full
715    // clear first since we don't track subrects. This means that in rare cases
716    // even a *destination* texture of a transfer may need an immediate texture
717    // init.
718    let dst_init_kind = if has_copy_partial_init_tracker_coverage(
719        copy_size,
720        destination.mip_level,
721        &texture.desc,
722    ) {
723        MemoryInitKind::NeedsInitializedMemory
724    } else {
725        MemoryInitKind::ImplicitlyInitialized
726    };
727
728    handle_texture_init(
729        dst_init_kind,
730        cmd_buf_data,
731        device,
732        destination,
733        copy_size,
734        texture,
735        snatch_guard,
736    )?;
737    Ok(())
738}
739
740/// Handle initialization tracking for a transfer's source or destination buffer.
741///
742/// Ensures that the transfer will not read from uninitialized memory, and updates
743/// the initialization state information to reflect the transfer.
744fn handle_buffer_init(
745    cmd_buf_data: &mut CommandBufferMutable,
746    info: &TexelCopyBufferInfo,
747    buffer: &Arc<Buffer>,
748    direction: CopySide,
749    required_buffer_bytes_in_copy: BufferAddress,
750    is_contiguous: bool,
751) {
752    const ALIGN_SIZE: BufferAddress = wgt::COPY_BUFFER_ALIGNMENT;
753    const ALIGN_MASK: BufferAddress = wgt::COPY_BUFFER_ALIGNMENT - 1;
754
755    let start = info.layout.offset;
756    let end = info.layout.offset + required_buffer_bytes_in_copy;
757    if !is_contiguous || direction == CopySide::Source {
758        // If the transfer will read the buffer, then the whole region needs to
759        // be initialized.
760        //
761        // If the transfer will not write a contiguous region of the buffer,
762        // then we need to make sure the padding areas are initialized. For now,
763        // initialize the whole region, although this could be improved to
764        // initialize only the necessary parts if doing so is likely to be
765        // faster than initializing the whole thing.
766        //
767        // Adjust the start/end outwards to 4B alignment.
768        let aligned_start = start & !ALIGN_MASK;
769        let aligned_end = (end + ALIGN_MASK) & !ALIGN_MASK;
770        cmd_buf_data.buffer_memory_init_actions.extend(
771            buffer.initialization_status.read().create_action(
772                buffer,
773                aligned_start..aligned_end,
774                MemoryInitKind::NeedsInitializedMemory,
775            ),
776        );
777    } else {
778        // If the transfer will write a contiguous region of the buffer, then we
779        // don't need to initialize that region.
780        //
781        // However, if the start and end are not 4B aligned, we need to make
782        // sure that we don't end up trying to initialize non-4B-aligned regions
783        // later.
784        //
785        // Adjust the start/end inwards to 4B alignment, we will handle the
786        // first/last pieces differently.
787        let aligned_start = (start + ALIGN_MASK) & !ALIGN_MASK;
788        let aligned_end = end & !ALIGN_MASK;
789        if aligned_start != start {
790            cmd_buf_data.buffer_memory_init_actions.extend(
791                buffer.initialization_status.read().create_action(
792                    buffer,
793                    aligned_start - ALIGN_SIZE..aligned_start,
794                    MemoryInitKind::NeedsInitializedMemory,
795                ),
796            );
797        }
798        if aligned_start != aligned_end {
799            cmd_buf_data.buffer_memory_init_actions.extend(
800                buffer.initialization_status.read().create_action(
801                    buffer,
802                    aligned_start..aligned_end,
803                    MemoryInitKind::ImplicitlyInitialized,
804                ),
805            );
806        }
807        if aligned_end != end {
808            // It is possible that `aligned_end + ALIGN_SIZE > dst_buffer.size`,
809            // because `dst_buffer.size` is the user-requested size, not the
810            // final size of the buffer. The final size of the buffer is not
811            // readily available, but was rounded up to COPY_BUFFER_ALIGNMENT,
812            // so no overrun is possible.
813            cmd_buf_data.buffer_memory_init_actions.extend(
814                buffer.initialization_status.read().create_action(
815                    buffer,
816                    aligned_end..aligned_end + ALIGN_SIZE,
817                    MemoryInitKind::NeedsInitializedMemory,
818                ),
819            );
820        }
821    }
822}
823
824impl Global {
825    pub fn command_encoder_copy_buffer_to_buffer(
826        &self,
827        command_encoder_id: CommandEncoderId,
828        source: BufferId,
829        source_offset: BufferAddress,
830        destination: BufferId,
831        destination_offset: BufferAddress,
832        size: Option<BufferAddress>,
833    ) -> Result<(), EncoderStateError> {
834        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
835        api_log!(
836            "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
837        );
838
839        let hub = &self.hub;
840
841        let cmd_enc = hub.command_encoders.get(command_encoder_id);
842        let mut cmd_buf_data = cmd_enc.data.lock();
843        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
844            copy_buffer_to_buffer(
845                cmd_buf_data,
846                hub,
847                &cmd_enc,
848                source,
849                source_offset,
850                destination,
851                destination_offset,
852                size,
853            )
854        })
855    }
856
857    pub fn command_encoder_copy_buffer_to_texture(
858        &self,
859        command_encoder_id: CommandEncoderId,
860        source: &TexelCopyBufferInfo,
861        destination: &TexelCopyTextureInfo,
862        copy_size: &Extent3d,
863    ) -> Result<(), EncoderStateError> {
864        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
865        api_log!(
866            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
867            source.buffer,
868            destination.texture
869        );
870
871        let hub = &self.hub;
872
873        let cmd_enc = hub.command_encoders.get(command_encoder_id);
874        let mut cmd_buf_data = cmd_enc.data.lock();
875        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
876            copy_buffer_to_texture(cmd_buf_data, hub, &cmd_enc, source, destination, copy_size)
877        })
878    }
879
880    pub fn command_encoder_copy_texture_to_buffer(
881        &self,
882        command_encoder_id: CommandEncoderId,
883        source: &TexelCopyTextureInfo,
884        destination: &TexelCopyBufferInfo,
885        copy_size: &Extent3d,
886    ) -> Result<(), EncoderStateError> {
887        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
888        api_log!(
889            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
890            source.texture,
891            destination.buffer
892        );
893
894        let hub = &self.hub;
895
896        let cmd_enc = hub.command_encoders.get(command_encoder_id);
897        let mut cmd_buf_data = cmd_enc.data.lock();
898        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
899            copy_texture_to_buffer(cmd_buf_data, hub, &cmd_enc, source, destination, copy_size)
900        })
901    }
902
903    pub fn command_encoder_copy_texture_to_texture(
904        &self,
905        command_encoder_id: CommandEncoderId,
906        source: &TexelCopyTextureInfo,
907        destination: &TexelCopyTextureInfo,
908        copy_size: &Extent3d,
909    ) -> Result<(), EncoderStateError> {
910        profiling::scope!("CommandEncoder::copy_texture_to_texture");
911        api_log!(
912            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
913            source.texture,
914            destination.texture
915        );
916
917        let hub = &self.hub;
918
919        let cmd_enc = hub.command_encoders.get(command_encoder_id);
920        let mut cmd_buf_data = cmd_enc.data.lock();
921        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
922            copy_texture_to_texture(cmd_buf_data, hub, &cmd_enc, source, destination, copy_size)
923        })
924    }
925}
926
927fn copy_buffer_to_buffer(
928    cmd_buf_data: &mut CommandBufferMutable,
929    hub: &crate::hub::Hub,
930    cmd_enc: &Arc<super::CommandEncoder>,
931    source: BufferId,
932    source_offset: BufferAddress,
933    destination: BufferId,
934    destination_offset: BufferAddress,
935    size: Option<BufferAddress>,
936) -> Result<(), CommandEncoderError> {
937    let device = &cmd_enc.device;
938    device.check_is_valid()?;
939
940    if source == destination {
941        return Err(TransferError::SameSourceDestinationBuffer.into());
942    }
943
944    #[cfg(feature = "trace")]
945    if let Some(ref mut list) = cmd_buf_data.trace_commands {
946        list.push(TraceCommand::CopyBufferToBuffer {
947            src: source,
948            src_offset: source_offset,
949            dst: destination,
950            dst_offset: destination_offset,
951            size,
952        });
953    }
954
955    let snatch_guard = device.snatchable_lock.read();
956
957    let src_buffer = hub.buffers.get(source).get()?;
958
959    src_buffer.same_device_as(cmd_enc.as_ref())?;
960
961    let src_pending = cmd_buf_data
962        .trackers
963        .buffers
964        .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
965
966    let src_raw = src_buffer.try_raw(&snatch_guard)?;
967    src_buffer
968        .check_usage(BufferUsages::COPY_SRC)
969        .map_err(TransferError::MissingBufferUsage)?;
970    // expecting only a single barrier
971    let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
972
973    let dst_buffer = hub.buffers.get(destination).get()?;
974
975    dst_buffer.same_device_as(cmd_enc.as_ref())?;
976
977    let dst_pending = cmd_buf_data
978        .trackers
979        .buffers
980        .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
981
982    let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
983    dst_buffer
984        .check_usage(BufferUsages::COPY_DST)
985        .map_err(TransferError::MissingBufferUsage)?;
986    let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
987
988    let (size, source_end_offset) = match size {
989        Some(size) => (size, source_offset + size),
990        None => (src_buffer.size - source_offset, src_buffer.size),
991    };
992
993    if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
994        return Err(TransferError::UnalignedCopySize(size).into());
995    }
996    if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
997        return Err(TransferError::UnalignedBufferOffset(source_offset).into());
998    }
999    if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
1000        return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
1001    }
1002    if !device
1003        .downlevel
1004        .flags
1005        .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
1006        && (src_buffer.usage.contains(BufferUsages::INDEX)
1007            || dst_buffer.usage.contains(BufferUsages::INDEX))
1008    {
1009        let forbidden_usages = BufferUsages::VERTEX
1010            | BufferUsages::UNIFORM
1011            | BufferUsages::INDIRECT
1012            | BufferUsages::STORAGE;
1013        if src_buffer.usage.intersects(forbidden_usages)
1014            || dst_buffer.usage.intersects(forbidden_usages)
1015        {
1016            return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
1017                wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
1018            ))
1019            .into());
1020        }
1021    }
1022
1023    let destination_end_offset = destination_offset + size;
1024    if source_end_offset > src_buffer.size {
1025        return Err(TransferError::BufferOverrun {
1026            start_offset: source_offset,
1027            end_offset: source_end_offset,
1028            buffer_size: src_buffer.size,
1029            side: CopySide::Source,
1030        }
1031        .into());
1032    }
1033    if destination_end_offset > dst_buffer.size {
1034        return Err(TransferError::BufferOverrun {
1035            start_offset: destination_offset,
1036            end_offset: destination_end_offset,
1037            buffer_size: dst_buffer.size,
1038            side: CopySide::Destination,
1039        }
1040        .into());
1041    }
1042
1043    if size == 0 {
1044        log::trace!("Ignoring copy_buffer_to_buffer of size 0");
1045        return Ok(());
1046    }
1047
1048    // Make sure source is initialized memory and mark dest as initialized.
1049    cmd_buf_data.buffer_memory_init_actions.extend(
1050        dst_buffer.initialization_status.read().create_action(
1051            &dst_buffer,
1052            destination_offset..(destination_offset + size),
1053            MemoryInitKind::ImplicitlyInitialized,
1054        ),
1055    );
1056    cmd_buf_data.buffer_memory_init_actions.extend(
1057        src_buffer.initialization_status.read().create_action(
1058            &src_buffer,
1059            source_offset..(source_offset + size),
1060            MemoryInitKind::NeedsInitializedMemory,
1061        ),
1062    );
1063
1064    let region = hal::BufferCopy {
1065        src_offset: source_offset,
1066        dst_offset: destination_offset,
1067        size: wgt::BufferSize::new(size).unwrap(),
1068    };
1069    let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1070    let barriers = src_barrier
1071        .into_iter()
1072        .chain(dst_barrier)
1073        .collect::<Vec<_>>();
1074    unsafe {
1075        cmd_buf_raw.transition_buffers(&barriers);
1076        cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
1077    }
1078
1079    Ok(())
1080}
1081
1082fn copy_buffer_to_texture(
1083    cmd_buf_data: &mut CommandBufferMutable,
1084    hub: &crate::hub::Hub,
1085    cmd_enc: &Arc<super::CommandEncoder>,
1086    source: &TexelCopyBufferInfo,
1087    destination: &TexelCopyTextureInfo,
1088    copy_size: &Extent3d,
1089) -> Result<(), CommandEncoderError> {
1090    let device = &cmd_enc.device;
1091    device.check_is_valid()?;
1092
1093    #[cfg(feature = "trace")]
1094    if let Some(ref mut list) = cmd_buf_data.trace_commands {
1095        list.push(TraceCommand::CopyBufferToTexture {
1096            src: *source,
1097            dst: *destination,
1098            size: *copy_size,
1099        });
1100    }
1101
1102    let dst_texture = hub.textures.get(destination.texture).get()?;
1103    let src_buffer = hub.buffers.get(source.buffer).get()?;
1104
1105    dst_texture.same_device_as(cmd_enc.as_ref())?;
1106    src_buffer.same_device_as(cmd_enc.as_ref())?;
1107
1108    let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
1109        destination,
1110        &dst_texture.desc,
1111        CopySide::Destination,
1112        copy_size,
1113    )?;
1114
1115    let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?;
1116
1117    let snatch_guard = device.snatchable_lock.read();
1118
1119    let src_raw = src_buffer.try_raw(&snatch_guard)?;
1120    let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1121
1122    if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1123        log::trace!("Ignoring copy_buffer_to_texture of size 0");
1124        return Ok(());
1125    }
1126
1127    // Handle texture init *before* dealing with barrier transitions so we
1128    // have an easier time inserting "immediate-inits" that may be required
1129    // by prior discards in rare cases.
1130    handle_dst_texture_init(
1131        cmd_buf_data,
1132        device,
1133        destination,
1134        copy_size,
1135        &dst_texture,
1136        &snatch_guard,
1137    )?;
1138
1139    let src_pending = cmd_buf_data
1140        .trackers
1141        .buffers
1142        .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
1143
1144    src_buffer
1145        .check_usage(BufferUsages::COPY_SRC)
1146        .map_err(TransferError::MissingBufferUsage)?;
1147    let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
1148
1149    let dst_pending = cmd_buf_data.trackers.textures.set_single(
1150        &dst_texture,
1151        dst_range,
1152        wgt::TextureUses::COPY_DST,
1153    );
1154    dst_texture
1155        .check_usage(TextureUsages::COPY_DST)
1156        .map_err(TransferError::MissingTextureUsage)?;
1157    let dst_barrier = dst_pending
1158        .map(|pending| pending.into_hal(dst_raw))
1159        .collect::<Vec<_>>();
1160
1161    validate_texture_copy_dst_format(dst_texture.desc.format, destination.aspect)?;
1162
1163    validate_texture_buffer_copy(
1164        destination,
1165        dst_base.aspect,
1166        &dst_texture.desc,
1167        &source.layout,
1168        true, // alignment required for buffer offset
1169    )?;
1170
1171    let (required_buffer_bytes_in_copy, bytes_per_array_layer, is_contiguous) =
1172        validate_linear_texture_data(
1173            &source.layout,
1174            dst_texture.desc.format,
1175            destination.aspect,
1176            src_buffer.size,
1177            CopySide::Source,
1178            copy_size,
1179        )?;
1180
1181    if dst_texture.desc.format.is_depth_stencil_format() {
1182        device
1183            .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1184            .map_err(TransferError::from)?;
1185    }
1186
1187    handle_buffer_init(
1188        cmd_buf_data,
1189        source,
1190        &src_buffer,
1191        CopySide::Source,
1192        required_buffer_bytes_in_copy,
1193        is_contiguous,
1194    );
1195
1196    let regions = (0..array_layer_count)
1197        .map(|rel_array_layer| {
1198            let mut texture_base = dst_base.clone();
1199            texture_base.array_layer += rel_array_layer;
1200            let mut buffer_layout = source.layout;
1201            buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1202            hal::BufferTextureCopy {
1203                buffer_layout,
1204                texture_base,
1205                size: hal_copy_size,
1206            }
1207        })
1208        .collect::<Vec<_>>();
1209
1210    let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1211    unsafe {
1212        cmd_buf_raw.transition_textures(&dst_barrier);
1213        cmd_buf_raw.transition_buffers(src_barrier.as_slice());
1214        cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, &regions);
1215    }
1216
1217    Ok(())
1218}
1219
1220fn copy_texture_to_buffer(
1221    cmd_buf_data: &mut CommandBufferMutable,
1222    hub: &crate::hub::Hub,
1223    cmd_enc: &Arc<super::CommandEncoder>,
1224    source: &TexelCopyTextureInfo,
1225    destination: &TexelCopyBufferInfo,
1226    copy_size: &Extent3d,
1227) -> Result<(), CommandEncoderError> {
1228    let device = &cmd_enc.device;
1229    device.check_is_valid()?;
1230
1231    #[cfg(feature = "trace")]
1232    if let Some(list) = cmd_buf_data.trace_commands.as_mut() {
1233        list.push(TraceCommand::CopyTextureToBuffer {
1234            src: *source,
1235            dst: *destination,
1236            size: *copy_size,
1237        });
1238    }
1239
1240    let src_texture = hub.textures.get(source.texture).get()?;
1241    let dst_buffer = hub.buffers.get(destination.buffer).get()?;
1242
1243    src_texture.same_device_as(cmd_enc.as_ref())?;
1244    dst_buffer.same_device_as(cmd_enc.as_ref())?;
1245
1246    let (hal_copy_size, array_layer_count) =
1247        validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1248
1249    let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1250
1251    let snatch_guard = device.snatchable_lock.read();
1252
1253    let src_raw = src_texture.try_raw(&snatch_guard)?;
1254    src_texture
1255        .check_usage(TextureUsages::COPY_SRC)
1256        .map_err(TransferError::MissingTextureUsage)?;
1257
1258    if source.mip_level >= src_texture.desc.mip_level_count {
1259        return Err(TransferError::InvalidMipLevel {
1260            requested: source.mip_level,
1261            count: src_texture.desc.mip_level_count,
1262        }
1263        .into());
1264    }
1265
1266    validate_texture_copy_src_format(src_texture.desc.format, source.aspect)?;
1267
1268    validate_texture_buffer_copy(
1269        source,
1270        src_base.aspect,
1271        &src_texture.desc,
1272        &destination.layout,
1273        true, // alignment required for buffer offset
1274    )?;
1275
1276    let (required_buffer_bytes_in_copy, bytes_per_array_layer, is_contiguous) =
1277        validate_linear_texture_data(
1278            &destination.layout,
1279            src_texture.desc.format,
1280            source.aspect,
1281            dst_buffer.size,
1282            CopySide::Destination,
1283            copy_size,
1284        )?;
1285
1286    if src_texture.desc.format.is_depth_stencil_format() {
1287        device
1288            .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1289            .map_err(TransferError::from)?;
1290    }
1291
1292    let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
1293    dst_buffer
1294        .check_usage(BufferUsages::COPY_DST)
1295        .map_err(TransferError::MissingBufferUsage)?;
1296
1297    if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1298        log::trace!("Ignoring copy_texture_to_buffer of size 0");
1299        return Ok(());
1300    }
1301
1302    // Handle texture init *before* dealing with barrier transitions so we
1303    // have an easier time inserting "immediate-inits" that may be required
1304    // by prior discards in rare cases.
1305    handle_src_texture_init(
1306        cmd_buf_data,
1307        device,
1308        source,
1309        copy_size,
1310        &src_texture,
1311        &snatch_guard,
1312    )?;
1313
1314    let src_pending = cmd_buf_data.trackers.textures.set_single(
1315        &src_texture,
1316        src_range,
1317        wgt::TextureUses::COPY_SRC,
1318    );
1319    let src_barrier = src_pending
1320        .map(|pending| pending.into_hal(src_raw))
1321        .collect::<Vec<_>>();
1322
1323    let dst_pending = cmd_buf_data
1324        .trackers
1325        .buffers
1326        .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
1327
1328    let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
1329
1330    handle_buffer_init(
1331        cmd_buf_data,
1332        destination,
1333        &dst_buffer,
1334        CopySide::Destination,
1335        required_buffer_bytes_in_copy,
1336        is_contiguous,
1337    );
1338
1339    let regions = (0..array_layer_count)
1340        .map(|rel_array_layer| {
1341            let mut texture_base = src_base.clone();
1342            texture_base.array_layer += rel_array_layer;
1343            let mut buffer_layout = destination.layout;
1344            buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1345            hal::BufferTextureCopy {
1346                buffer_layout,
1347                texture_base,
1348                size: hal_copy_size,
1349            }
1350        })
1351        .collect::<Vec<_>>();
1352    let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1353    unsafe {
1354        cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1355        cmd_buf_raw.transition_textures(&src_barrier);
1356        cmd_buf_raw.copy_texture_to_buffer(src_raw, wgt::TextureUses::COPY_SRC, dst_raw, &regions);
1357    }
1358
1359    Ok(())
1360}
1361
1362fn copy_texture_to_texture(
1363    cmd_buf_data: &mut CommandBufferMutable,
1364    hub: &crate::hub::Hub,
1365    cmd_enc: &Arc<super::CommandEncoder>,
1366    source: &TexelCopyTextureInfo,
1367    destination: &TexelCopyTextureInfo,
1368    copy_size: &Extent3d,
1369) -> Result<(), CommandEncoderError> {
1370    let device = &cmd_enc.device;
1371    device.check_is_valid()?;
1372
1373    let snatch_guard = device.snatchable_lock.read();
1374
1375    #[cfg(feature = "trace")]
1376    if let Some(ref mut list) = cmd_buf_data.trace_commands {
1377        list.push(TraceCommand::CopyTextureToTexture {
1378            src: *source,
1379            dst: *destination,
1380            size: *copy_size,
1381        });
1382    }
1383
1384    let src_texture = hub.textures.get(source.texture).get()?;
1385    let dst_texture = hub.textures.get(destination.texture).get()?;
1386
1387    src_texture.same_device_as(cmd_enc.as_ref())?;
1388    dst_texture.same_device_as(cmd_enc.as_ref())?;
1389
1390    // src and dst texture format must be copy-compatible
1391    // https://gpuweb.github.io/gpuweb/#copy-compatible
1392    if src_texture.desc.format.remove_srgb_suffix() != dst_texture.desc.format.remove_srgb_suffix()
1393    {
1394        return Err(TransferError::TextureFormatsNotCopyCompatible {
1395            src_format: src_texture.desc.format,
1396            dst_format: dst_texture.desc.format,
1397        }
1398        .into());
1399    }
1400
1401    let (src_copy_size, array_layer_count) =
1402        validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1403    let (dst_copy_size, _) = validate_texture_copy_range(
1404        destination,
1405        &dst_texture.desc,
1406        CopySide::Destination,
1407        copy_size,
1408    )?;
1409
1410    if Arc::as_ptr(&src_texture) == Arc::as_ptr(&dst_texture) {
1411        validate_copy_within_same_texture(
1412            source,
1413            destination,
1414            src_texture.desc.format,
1415            array_layer_count,
1416        )?;
1417    }
1418
1419    let (src_range, src_tex_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1420    let (dst_range, dst_tex_base) = extract_texture_selector(destination, copy_size, &dst_texture)?;
1421    let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1422    let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1423    if src_tex_base.aspect != src_texture_aspects {
1424        return Err(TransferError::CopySrcMissingAspects.into());
1425    }
1426    if dst_tex_base.aspect != dst_texture_aspects {
1427        return Err(TransferError::CopyDstMissingAspects.into());
1428    }
1429
1430    if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1431        return Err(TransferError::SampleCountNotEqual {
1432            src_sample_count: src_texture.desc.sample_count,
1433            dst_sample_count: dst_texture.desc.sample_count,
1434        }
1435        .into());
1436    }
1437
1438    // Handle texture init *before* dealing with barrier transitions so we
1439    // have an easier time inserting "immediate-inits" that may be required
1440    // by prior discards in rare cases.
1441    handle_src_texture_init(
1442        cmd_buf_data,
1443        device,
1444        source,
1445        copy_size,
1446        &src_texture,
1447        &snatch_guard,
1448    )?;
1449    handle_dst_texture_init(
1450        cmd_buf_data,
1451        device,
1452        destination,
1453        copy_size,
1454        &dst_texture,
1455        &snatch_guard,
1456    )?;
1457
1458    let src_raw = src_texture.try_raw(&snatch_guard)?;
1459    src_texture
1460        .check_usage(TextureUsages::COPY_SRC)
1461        .map_err(TransferError::MissingTextureUsage)?;
1462    let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1463    dst_texture
1464        .check_usage(TextureUsages::COPY_DST)
1465        .map_err(TransferError::MissingTextureUsage)?;
1466
1467    if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1468        log::trace!("Ignoring copy_texture_to_texture of size 0");
1469        return Ok(());
1470    }
1471
1472    let src_pending = cmd_buf_data.trackers.textures.set_single(
1473        &src_texture,
1474        src_range,
1475        wgt::TextureUses::COPY_SRC,
1476    );
1477
1478    //TODO: try to avoid this the collection. It's needed because both
1479    // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1480    let mut barriers: ArrayVec<_, 2> = src_pending
1481        .map(|pending| pending.into_hal(src_raw))
1482        .collect();
1483
1484    let dst_pending = cmd_buf_data.trackers.textures.set_single(
1485        &dst_texture,
1486        dst_range,
1487        wgt::TextureUses::COPY_DST,
1488    );
1489    barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1490
1491    let hal_copy_size = hal::CopyExtent {
1492        width: src_copy_size.width.min(dst_copy_size.width),
1493        height: src_copy_size.height.min(dst_copy_size.height),
1494        depth: src_copy_size.depth.min(dst_copy_size.depth),
1495    };
1496
1497    let regions = (0..array_layer_count).map(|rel_array_layer| {
1498        let mut src_base = src_tex_base.clone();
1499        let mut dst_base = dst_tex_base.clone();
1500        src_base.array_layer += rel_array_layer;
1501        dst_base.array_layer += rel_array_layer;
1502        hal::TextureCopy {
1503            src_base,
1504            dst_base,
1505            size: hal_copy_size,
1506        }
1507    });
1508
1509    let regions = if dst_tex_base.aspect == hal::FormatAspects::DEPTH_STENCIL {
1510        regions
1511            .flat_map(|region| {
1512                let (mut depth, mut stencil) = (region.clone(), region);
1513                depth.src_base.aspect = hal::FormatAspects::DEPTH;
1514                depth.dst_base.aspect = hal::FormatAspects::DEPTH;
1515                stencil.src_base.aspect = hal::FormatAspects::STENCIL;
1516                stencil.dst_base.aspect = hal::FormatAspects::STENCIL;
1517                [depth, stencil]
1518            })
1519            .collect::<Vec<_>>()
1520    } else {
1521        regions.collect::<Vec<_>>()
1522    };
1523    let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1524    unsafe {
1525        cmd_buf_raw.transition_textures(&barriers);
1526        cmd_buf_raw.copy_texture_to_texture(src_raw, wgt::TextureUses::COPY_SRC, dst_raw, &regions);
1527    }
1528
1529    Ok(())
1530}