wgpu_core/command/
transfer.rs

1use alloc::{format, string::String, sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6    error::{ErrorType, WebGpuError},
7    BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8    TextureUsages,
9};
10
11#[cfg(feature = "trace")]
12use crate::device::trace::Command as TraceCommand;
13use crate::{
14    api_log,
15    command::{clear_texture, CommandEncoderError, EncoderStateError},
16    device::{Device, MissingDownlevelFlags},
17    global::Global,
18    id::{BufferId, CommandEncoderId, TextureId},
19    init_tracker::{
20        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
21        TextureInitTrackerAction,
22    },
23    resource::{
24        MissingBufferUsageError, MissingTextureUsageError, ParentDevice, RawResourceAccess,
25        Texture, TextureErrorDimension,
26    },
27    snatch::SnatchGuard,
28};
29
30use super::{ClearError, CommandBufferMutable};
31
32pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo<BufferId>;
33pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<TextureId>;
34pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo<TextureId>;
35
36#[derive(Clone, Copy, Debug)]
37pub enum CopySide {
38    Source,
39    Destination,
40}
41
42/// Error encountered while attempting a data transfer.
43#[derive(Clone, Debug, Error)]
44#[non_exhaustive]
45pub enum TransferError {
46    #[error("Source and destination cannot be the same buffer")]
47    SameSourceDestinationBuffer,
48    #[error(transparent)]
49    MissingBufferUsage(#[from] MissingBufferUsageError),
50    #[error(transparent)]
51    MissingTextureUsage(#[from] MissingTextureUsageError),
52    #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
53    BufferOverrun {
54        start_offset: BufferAddress,
55        end_offset: BufferAddress,
56        buffer_size: BufferAddress,
57        side: CopySide,
58    },
59    #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
60    TextureOverrun {
61        start_offset: u32,
62        end_offset: u32,
63        texture_size: u32,
64        dimension: TextureErrorDimension,
65        side: CopySide,
66    },
67    #[error("Partial copy of {start_offset}..{end_offset} on {dimension:?} dimension with size {texture_size} \
68             is not supported for the {side:?} texture format {format:?} with {sample_count} samples")]
69    UnsupportedPartialTransfer {
70        format: wgt::TextureFormat,
71        sample_count: u32,
72        start_offset: u32,
73        end_offset: u32,
74        texture_size: u32,
75        dimension: TextureErrorDimension,
76        side: CopySide,
77    },
78    #[error(
79        "Copying{} layers {}..{} to{} layers {}..{} of the same texture is not allowed",
80        if *src_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {src_aspects:?}") },
81        src_origin_z,
82        src_origin_z + array_layer_count,
83        if *dst_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {dst_aspects:?}") },
84        dst_origin_z,
85        dst_origin_z + array_layer_count,
86    )]
87    InvalidCopyWithinSameTexture {
88        src_aspects: wgt::TextureAspect,
89        dst_aspects: wgt::TextureAspect,
90        src_origin_z: u32,
91        dst_origin_z: u32,
92        array_layer_count: u32,
93    },
94    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
95    InvalidTextureAspect {
96        format: wgt::TextureFormat,
97        aspect: wgt::TextureAspect,
98    },
99    #[error("Unable to select texture mip level {level} out of {total}")]
100    InvalidTextureMipLevel { level: u32, total: u32 },
101    #[error("Texture dimension must be 2D when copying from an external texture")]
102    InvalidDimensionExternal,
103    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
104    UnalignedBufferOffset(BufferAddress),
105    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
106    UnalignedCopySize(BufferAddress),
107    #[error("Copy width is not a multiple of block width")]
108    UnalignedCopyWidth,
109    #[error("Copy height is not a multiple of block height")]
110    UnalignedCopyHeight,
111    #[error("Copy origin's x component is not a multiple of block width")]
112    UnalignedCopyOriginX,
113    #[error("Copy origin's y component is not a multiple of block height")]
114    UnalignedCopyOriginY,
115    #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
116    UnalignedBytesPerRow,
117    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
118    UnspecifiedBytesPerRow,
119    #[error("Number of rows per image needs to be specified since more than one image is copied")]
120    UnspecifiedRowsPerImage,
121    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
122    InvalidBytesPerRow,
123    #[error("Number of rows per image is invalid")]
124    InvalidRowsPerImage,
125    #[error("Overflow while computing the size of the copy")]
126    SizeOverflow,
127    #[error("Copy source aspects must refer to all aspects of the source texture format")]
128    CopySrcMissingAspects,
129    #[error(
130        "Copy destination aspects must refer to all aspects of the destination texture format"
131    )]
132    CopyDstMissingAspects,
133    #[error("Copy aspect must refer to a single aspect of texture format")]
134    CopyAspectNotOne,
135    #[error("Copying from textures with format {0:?} is forbidden")]
136    CopyFromForbiddenTextureFormat(wgt::TextureFormat),
137    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
138    CopyFromForbiddenTextureFormatAspect {
139        format: wgt::TextureFormat,
140        aspect: wgt::TextureAspect,
141    },
142    #[error("Copying to textures with format {0:?} is forbidden")]
143    CopyToForbiddenTextureFormat(wgt::TextureFormat),
144    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
145    CopyToForbiddenTextureFormatAspect {
146        format: wgt::TextureFormat,
147        aspect: wgt::TextureAspect,
148    },
149    #[error(
150        "Copying to textures with format {0:?} is forbidden when copying from external texture"
151    )]
152    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
153    #[error(
154        "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
155    )]
156    TextureFormatsNotCopyCompatible {
157        src_format: wgt::TextureFormat,
158        dst_format: wgt::TextureFormat,
159    },
160    #[error(transparent)]
161    MemoryInitFailure(#[from] ClearError),
162    #[error("Cannot encode this copy because of a missing downelevel flag")]
163    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
164    #[error("Source texture sample count must be 1, got {sample_count}")]
165    InvalidSampleCount { sample_count: u32 },
166    #[error(
167        "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
168    )]
169    SampleCountNotEqual {
170        src_sample_count: u32,
171        dst_sample_count: u32,
172    },
173    #[error("Requested mip level {requested} does no exist (count: {count})")]
174    InvalidMipLevel { requested: u32, count: u32 },
175}
176
177impl WebGpuError for TransferError {
178    fn webgpu_error_type(&self) -> ErrorType {
179        let e: &dyn WebGpuError = match self {
180            Self::MissingBufferUsage(e) => e,
181            Self::MissingTextureUsage(e) => e,
182            Self::MemoryInitFailure(e) => e,
183
184            Self::BufferOverrun { .. }
185            | Self::TextureOverrun { .. }
186            | Self::UnsupportedPartialTransfer { .. }
187            | Self::InvalidCopyWithinSameTexture { .. }
188            | Self::InvalidTextureAspect { .. }
189            | Self::InvalidTextureMipLevel { .. }
190            | Self::InvalidDimensionExternal
191            | Self::UnalignedBufferOffset(..)
192            | Self::UnalignedCopySize(..)
193            | Self::UnalignedCopyWidth
194            | Self::UnalignedCopyHeight
195            | Self::UnalignedCopyOriginX
196            | Self::UnalignedCopyOriginY
197            | Self::UnalignedBytesPerRow
198            | Self::UnspecifiedBytesPerRow
199            | Self::UnspecifiedRowsPerImage
200            | Self::InvalidBytesPerRow
201            | Self::InvalidRowsPerImage
202            | Self::SizeOverflow
203            | Self::CopySrcMissingAspects
204            | Self::CopyDstMissingAspects
205            | Self::CopyAspectNotOne
206            | Self::CopyFromForbiddenTextureFormat(..)
207            | Self::CopyFromForbiddenTextureFormatAspect { .. }
208            | Self::CopyToForbiddenTextureFormat(..)
209            | Self::CopyToForbiddenTextureFormatAspect { .. }
210            | Self::ExternalCopyToForbiddenTextureFormat(..)
211            | Self::TextureFormatsNotCopyCompatible { .. }
212            | Self::MissingDownlevelFlags(..)
213            | Self::InvalidSampleCount { .. }
214            | Self::SampleCountNotEqual { .. }
215            | Self::InvalidMipLevel { .. }
216            | Self::SameSourceDestinationBuffer => return ErrorType::Validation,
217        };
218        e.webgpu_error_type()
219    }
220}
221
222impl From<BufferTextureCopyInfoError> for TransferError {
223    fn from(value: BufferTextureCopyInfoError) -> Self {
224        match value {
225            BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
226            BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
227            BufferTextureCopyInfoError::ImageStrideOverflow
228            | BufferTextureCopyInfoError::ImageBytesOverflow(_)
229            | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
230        }
231    }
232}
233
234pub(crate) fn extract_texture_selector<T>(
235    copy_texture: &wgt::TexelCopyTextureInfo<T>,
236    copy_size: &Extent3d,
237    texture: &Texture,
238) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
239    let format = texture.desc.format;
240    let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
241    if copy_aspect.is_empty() {
242        return Err(TransferError::InvalidTextureAspect {
243            format,
244            aspect: copy_texture.aspect,
245        });
246    }
247
248    let (layers, origin_z) = match texture.desc.dimension {
249        wgt::TextureDimension::D1 => (0..1, 0),
250        wgt::TextureDimension::D2 => (
251            copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
252            0,
253        ),
254        wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
255    };
256    let base = hal::TextureCopyBase {
257        origin: wgt::Origin3d {
258            x: copy_texture.origin.x,
259            y: copy_texture.origin.y,
260            z: origin_z,
261        },
262        // this value will be incremented per copied layer
263        array_layer: layers.start,
264        mip_level: copy_texture.mip_level,
265        aspect: copy_aspect,
266    };
267    let selector = TextureSelector {
268        mips: copy_texture.mip_level..copy_texture.mip_level + 1,
269        layers,
270    };
271
272    Ok((selector, base))
273}
274
275/// WebGPU's [validating linear texture data][vltd] algorithm.
276///
277/// Copied with some modifications from WebGPU standard.
278///
279/// If successful, returns a pair `(bytes, stride)`, where:
280/// - `bytes` is the number of buffer bytes required for this copy, and
281/// - `stride` number of bytes between array layers.
282///
283/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
284pub(crate) fn validate_linear_texture_data(
285    layout: &wgt::TexelCopyBufferLayout,
286    format: wgt::TextureFormat,
287    aspect: wgt::TextureAspect,
288    buffer_size: BufferAddress,
289    buffer_side: CopySide,
290    copy_size: &Extent3d,
291    need_copy_aligned_rows: bool,
292) -> Result<(BufferAddress, BufferAddress), TransferError> {
293    let wgt::BufferTextureCopyInfo {
294        copy_width,
295        copy_height,
296        depth_or_array_layers,
297
298        offset,
299
300        block_size_bytes,
301        block_width_texels,
302        block_height_texels,
303
304        width_blocks: _,
305        height_blocks,
306
307        row_bytes_dense: _,
308        row_stride_bytes,
309
310        image_stride_rows: _,
311        image_stride_bytes,
312
313        image_rows_dense: _,
314        image_bytes_dense: _,
315
316        bytes_in_copy,
317    } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
318
319    if copy_width % block_width_texels != 0 {
320        return Err(TransferError::UnalignedCopyWidth);
321    }
322    if copy_height % block_height_texels != 0 {
323        return Err(TransferError::UnalignedCopyHeight);
324    }
325
326    let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
327    let requires_multiple_images = depth_or_array_layers > 1;
328
329    // `get_buffer_texture_copy_info()` already proceeded with defaults if these
330    // were not specified, and ensured that the values satisfy the minima if
331    // they were, but now we enforce the WebGPU requirement that they be
332    // specified any time they apply.
333    if layout.bytes_per_row.is_none() && requires_multiple_rows {
334        return Err(TransferError::UnspecifiedBytesPerRow);
335    }
336
337    if layout.rows_per_image.is_none() && requires_multiple_images {
338        return Err(TransferError::UnspecifiedRowsPerImage);
339    };
340
341    if need_copy_aligned_rows {
342        let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
343
344        let mut offset_alignment = block_size_bytes;
345        if format.is_depth_stencil_format() {
346            offset_alignment = 4
347        }
348        if offset % offset_alignment != 0 {
349            return Err(TransferError::UnalignedBufferOffset(offset));
350        }
351
352        // The alignment of row_stride_bytes is only required if there are
353        // multiple rows
354        if requires_multiple_rows && row_stride_bytes % bytes_per_row_alignment != 0 {
355            return Err(TransferError::UnalignedBytesPerRow);
356        }
357    }
358
359    // Avoid underflow in the subtraction by checking bytes_in_copy against buffer_size first.
360    if bytes_in_copy > buffer_size || offset > buffer_size - bytes_in_copy {
361        return Err(TransferError::BufferOverrun {
362            start_offset: offset,
363            end_offset: offset.wrapping_add(bytes_in_copy),
364            buffer_size,
365            side: buffer_side,
366        });
367    }
368
369    Ok((bytes_in_copy, image_stride_bytes))
370}
371
372/// Validate the source format of a texture copy.
373///
374/// This performs the check from WebGPU's [validating texture buffer copy][vtbc]
375/// algorithm that ensures that the format and aspect form a valid texel copy source
376/// as defined in the [depth-stencil formats][dsf].
377///
378/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
379/// [dsf]: https://gpuweb.github.io/gpuweb/#depth-formats
380pub(crate) fn validate_texture_copy_src_format(
381    format: wgt::TextureFormat,
382    aspect: wgt::TextureAspect,
383) -> Result<(), TransferError> {
384    use wgt::TextureAspect as Ta;
385    use wgt::TextureFormat as Tf;
386    match (format, aspect) {
387        (Tf::Depth24Plus, _) => Err(TransferError::CopyFromForbiddenTextureFormat(format)),
388        (Tf::Depth24PlusStencil8, Ta::DepthOnly) => {
389            Err(TransferError::CopyFromForbiddenTextureFormatAspect { format, aspect })
390        }
391        _ => Ok(()),
392    }
393}
394
395/// Validate the destination format of a texture copy.
396///
397/// This performs the check from WebGPU's [validating texture buffer copy][vtbc]
398/// algorithm that ensures that the format and aspect form a valid texel copy destination
399/// as defined in the [depth-stencil formats][dsf].
400///
401/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
402/// [dsf]: https://gpuweb.github.io/gpuweb/#depth-formats
403pub(crate) fn validate_texture_copy_dst_format(
404    format: wgt::TextureFormat,
405    aspect: wgt::TextureAspect,
406) -> Result<(), TransferError> {
407    use wgt::TextureAspect as Ta;
408    use wgt::TextureFormat as Tf;
409    match (format, aspect) {
410        (Tf::Depth24Plus | Tf::Depth32Float, _) => {
411            Err(TransferError::CopyToForbiddenTextureFormat(format))
412        }
413        (Tf::Depth24PlusStencil8 | Tf::Depth32FloatStencil8, Ta::DepthOnly) => {
414            Err(TransferError::CopyToForbiddenTextureFormatAspect { format, aspect })
415        }
416        _ => Ok(()),
417    }
418}
419
420/// Validation for texture/buffer copies.
421///
422/// This implements the following checks from WebGPU's [validating texture buffer copy][vtbc]
423/// algorithm:
424///  * The texture must not be multisampled.
425///  * The copy must be from/to a single aspect of the texture.
426///  * If `aligned` is true, the buffer offset must be aligned appropriately.
427///
428/// The following steps in the algorithm are implemented elsewhere:
429///  * Invocation of other validation algorithms.
430///  * The texture usage (COPY_DST / COPY_SRC) check.
431///  * The check for non-copyable depth/stencil formats. The caller must perform
432///    this check using `validate_texture_copy_src_format` / `validate_texture_copy_dst_format`
433///    before calling this function. This function will panic if
434///    [`wgt::TextureFormat::block_copy_size`] returns `None` due to a
435///    non-copyable format.
436///
437/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
438pub(crate) fn validate_texture_buffer_copy<T>(
439    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
440    aspect: hal::FormatAspects,
441    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
442    offset: BufferAddress,
443    aligned: bool,
444) -> Result<(), TransferError> {
445    if desc.sample_count != 1 {
446        return Err(TransferError::InvalidSampleCount {
447            sample_count: desc.sample_count,
448        });
449    }
450
451    if !aspect.is_one() {
452        return Err(TransferError::CopyAspectNotOne);
453    }
454
455    let offset_alignment = if desc.format.is_depth_stencil_format() {
456        4
457    } else {
458        // The case where `block_copy_size` returns `None` is currently
459        // unreachable both for the reason in the expect message, and also
460        // because the currently-defined non-copyable formats are depth/stencil
461        // formats so would take the `if` branch.
462        desc.format
463            .block_copy_size(Some(texture_copy_view.aspect))
464            .expect("non-copyable formats should have been rejected previously")
465    };
466
467    if aligned && offset % u64::from(offset_alignment) != 0 {
468        return Err(TransferError::UnalignedBufferOffset(offset));
469    }
470
471    Ok(())
472}
473
474/// Validate the extent and alignment of a texture copy.
475///
476/// Copied with minor modifications from WebGPU standard. This mostly follows
477/// the [validating GPUTexelCopyTextureInfo][vtcti] and [validating texture copy
478/// range][vtcr] algorithms.
479///
480/// Returns the HAL copy extent and the layer count.
481///
482/// [vtcti]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gputexelcopytextureinfo
483/// [vtcr]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-copy-range
484pub(crate) fn validate_texture_copy_range<T>(
485    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
486    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
487    texture_side: CopySide,
488    copy_size: &Extent3d,
489) -> Result<(hal::CopyExtent, u32), TransferError> {
490    let (block_width, block_height) = desc.format.block_dimensions();
491
492    let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
493        TransferError::InvalidTextureMipLevel {
494            level: texture_copy_view.mip_level,
495            total: desc.mip_level_count,
496        },
497    )?;
498    // physical size can be larger than the virtual
499    let extent = extent_virtual.physical_size(desc.format);
500
501    // Multisampled and depth-stencil formats do not support partial copies
502    // on x and y dimensions, but do support copying a subset of layers.
503    let requires_exact_size = desc.format.is_depth_stencil_format() || desc.sample_count > 1;
504
505    // Return `Ok` if a run `size` texels long starting at `start_offset` is
506    // valid for `texture_size`. Otherwise, return an appropriate a`Err`.
507    let check_dimension = |dimension: TextureErrorDimension,
508                           start_offset: u32,
509                           size: u32,
510                           texture_size: u32,
511                           requires_exact_size: bool|
512     -> Result<(), TransferError> {
513        if requires_exact_size && (start_offset != 0 || size != texture_size) {
514            Err(TransferError::UnsupportedPartialTransfer {
515                format: desc.format,
516                sample_count: desc.sample_count,
517                start_offset,
518                end_offset: start_offset.wrapping_add(size),
519                texture_size,
520                dimension,
521                side: texture_side,
522            })
523        // Avoid underflow in the subtraction by checking start_offset against
524        // texture_size first.
525        } else if start_offset > texture_size || texture_size - start_offset < size {
526            Err(TransferError::TextureOverrun {
527                start_offset,
528                end_offset: start_offset.wrapping_add(size),
529                texture_size,
530                dimension,
531                side: texture_side,
532            })
533        } else {
534            Ok(())
535        }
536    };
537
538    check_dimension(
539        TextureErrorDimension::X,
540        texture_copy_view.origin.x,
541        copy_size.width,
542        extent.width,
543        requires_exact_size,
544    )?;
545    check_dimension(
546        TextureErrorDimension::Y,
547        texture_copy_view.origin.y,
548        copy_size.height,
549        extent.height,
550        requires_exact_size,
551    )?;
552    check_dimension(
553        TextureErrorDimension::Z,
554        texture_copy_view.origin.z,
555        copy_size.depth_or_array_layers,
556        extent.depth_or_array_layers,
557        false, // partial copy always allowed on Z/layer dimension
558    )?;
559
560    if texture_copy_view.origin.x % block_width != 0 {
561        return Err(TransferError::UnalignedCopyOriginX);
562    }
563    if texture_copy_view.origin.y % block_height != 0 {
564        return Err(TransferError::UnalignedCopyOriginY);
565    }
566    if copy_size.width % block_width != 0 {
567        return Err(TransferError::UnalignedCopyWidth);
568    }
569    if copy_size.height % block_height != 0 {
570        return Err(TransferError::UnalignedCopyHeight);
571    }
572
573    let (depth, array_layer_count) = match desc.dimension {
574        wgt::TextureDimension::D1 => (1, 1),
575        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
576        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
577    };
578
579    let copy_extent = hal::CopyExtent {
580        width: copy_size.width,
581        height: copy_size.height,
582        depth,
583    };
584    Ok((copy_extent, array_layer_count))
585}
586
587/// Validate a copy within the same texture.
588///
589/// This implements the WebGPU requirement that the [sets of subresources for
590/// texture copy][srtc] of the source and destination be disjoint, i.e. that the
591/// source and destination do not overlap.
592///
593/// This function assumes that the copy ranges have already been validated with
594/// `validate_texture_copy_range`.
595///
596/// [srtc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-set-of-subresources-for-texture-copy
597pub(crate) fn validate_copy_within_same_texture<T>(
598    src: &wgt::TexelCopyTextureInfo<T>,
599    dst: &wgt::TexelCopyTextureInfo<T>,
600    format: wgt::TextureFormat,
601    array_layer_count: u32,
602) -> Result<(), TransferError> {
603    let src_aspects = hal::FormatAspects::new(format, src.aspect);
604    let dst_aspects = hal::FormatAspects::new(format, dst.aspect);
605    if (src_aspects & dst_aspects).is_empty() {
606        // Copying between different aspects (if it even makes sense), is okay.
607        return Ok(());
608    }
609
610    if src.origin.z >= dst.origin.z + array_layer_count
611        || dst.origin.z >= src.origin.z + array_layer_count
612    {
613        // Copying between non-overlapping layer ranges is okay.
614        return Ok(());
615    }
616
617    if src.mip_level != dst.mip_level {
618        // Copying between different mip levels is okay.
619        return Ok(());
620    }
621
622    Err(TransferError::InvalidCopyWithinSameTexture {
623        src_aspects: src.aspect,
624        dst_aspects: dst.aspect,
625        src_origin_z: src.origin.z,
626        dst_origin_z: dst.origin.z,
627        array_layer_count,
628    })
629}
630
631fn handle_texture_init(
632    init_kind: MemoryInitKind,
633    cmd_buf_data: &mut CommandBufferMutable,
634    device: &Device,
635    copy_texture: &TexelCopyTextureInfo,
636    copy_size: &Extent3d,
637    texture: &Arc<Texture>,
638    snatch_guard: &SnatchGuard<'_>,
639) -> Result<(), ClearError> {
640    let init_action = TextureInitTrackerAction {
641        texture: texture.clone(),
642        range: TextureInitRange {
643            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
644            layer_range: copy_texture.origin.z
645                ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
646        },
647        kind: init_kind,
648    };
649
650    // Register the init action.
651    let immediate_inits = cmd_buf_data
652        .texture_memory_actions
653        .register_init_action(&{ init_action });
654
655    // In rare cases we may need to insert an init operation immediately onto the command buffer.
656    if !immediate_inits.is_empty() {
657        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
658        for init in immediate_inits {
659            clear_texture(
660                &init.texture,
661                TextureInitRange {
662                    mip_range: init.mip_level..(init.mip_level + 1),
663                    layer_range: init.layer..(init.layer + 1),
664                },
665                cmd_buf_raw,
666                &mut cmd_buf_data.trackers.textures,
667                &device.alignments,
668                device.zero_buffer.as_ref(),
669                snatch_guard,
670                device.instance_flags,
671            )?;
672        }
673    }
674
675    Ok(())
676}
677
678/// Prepare a transfer's source texture.
679///
680/// Ensure the source texture of a transfer is in the right initialization
681/// state, and record the state for after the transfer operation.
682fn handle_src_texture_init(
683    cmd_buf_data: &mut CommandBufferMutable,
684    device: &Device,
685    source: &TexelCopyTextureInfo,
686    copy_size: &Extent3d,
687    texture: &Arc<Texture>,
688    snatch_guard: &SnatchGuard<'_>,
689) -> Result<(), TransferError> {
690    handle_texture_init(
691        MemoryInitKind::NeedsInitializedMemory,
692        cmd_buf_data,
693        device,
694        source,
695        copy_size,
696        texture,
697        snatch_guard,
698    )?;
699    Ok(())
700}
701
702/// Prepare a transfer's destination texture.
703///
704/// Ensure the destination texture of a transfer is in the right initialization
705/// state, and record the state for after the transfer operation.
706fn handle_dst_texture_init(
707    cmd_buf_data: &mut CommandBufferMutable,
708    device: &Device,
709    destination: &TexelCopyTextureInfo,
710    copy_size: &Extent3d,
711    texture: &Arc<Texture>,
712    snatch_guard: &SnatchGuard<'_>,
713) -> Result<(), TransferError> {
714    // Attention: If we don't write full texture subresources, we need to a full
715    // clear first since we don't track subrects. This means that in rare cases
716    // even a *destination* texture of a transfer may need an immediate texture
717    // init.
718    let dst_init_kind = if has_copy_partial_init_tracker_coverage(
719        copy_size,
720        destination.mip_level,
721        &texture.desc,
722    ) {
723        MemoryInitKind::NeedsInitializedMemory
724    } else {
725        MemoryInitKind::ImplicitlyInitialized
726    };
727
728    handle_texture_init(
729        dst_init_kind,
730        cmd_buf_data,
731        device,
732        destination,
733        copy_size,
734        texture,
735        snatch_guard,
736    )?;
737    Ok(())
738}
739
740impl Global {
741    pub fn command_encoder_copy_buffer_to_buffer(
742        &self,
743        command_encoder_id: CommandEncoderId,
744        source: BufferId,
745        source_offset: BufferAddress,
746        destination: BufferId,
747        destination_offset: BufferAddress,
748        size: Option<BufferAddress>,
749    ) -> Result<(), EncoderStateError> {
750        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
751        api_log!(
752            "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
753        );
754
755        let hub = &self.hub;
756
757        let cmd_enc = hub.command_encoders.get(command_encoder_id);
758        let mut cmd_buf_data = cmd_enc.data.lock();
759        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
760            let device = &cmd_enc.device;
761            device.check_is_valid()?;
762
763            if source == destination {
764                return Err(TransferError::SameSourceDestinationBuffer.into());
765            }
766
767            #[cfg(feature = "trace")]
768            if let Some(ref mut list) = cmd_buf_data.commands {
769                list.push(TraceCommand::CopyBufferToBuffer {
770                    src: source,
771                    src_offset: source_offset,
772                    dst: destination,
773                    dst_offset: destination_offset,
774                    size,
775                });
776            }
777
778            let snatch_guard = device.snatchable_lock.read();
779
780            let src_buffer = hub.buffers.get(source).get()?;
781
782            src_buffer.same_device_as(cmd_enc.as_ref())?;
783
784            let src_pending = cmd_buf_data
785                .trackers
786                .buffers
787                .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
788
789            let src_raw = src_buffer.try_raw(&snatch_guard)?;
790            src_buffer
791                .check_usage(BufferUsages::COPY_SRC)
792                .map_err(TransferError::MissingBufferUsage)?;
793            // expecting only a single barrier
794            let src_barrier =
795                src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
796
797            let dst_buffer = hub.buffers.get(destination).get()?;
798
799            dst_buffer.same_device_as(cmd_enc.as_ref())?;
800
801            let dst_pending = cmd_buf_data
802                .trackers
803                .buffers
804                .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
805
806            let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
807            dst_buffer
808                .check_usage(BufferUsages::COPY_DST)
809                .map_err(TransferError::MissingBufferUsage)?;
810            let dst_barrier =
811                dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
812
813            let (size, source_end_offset) = match size {
814                Some(size) => (size, source_offset + size),
815                None => (src_buffer.size - source_offset, src_buffer.size),
816            };
817
818            if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
819                return Err(TransferError::UnalignedCopySize(size).into());
820            }
821            if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
822                return Err(TransferError::UnalignedBufferOffset(source_offset).into());
823            }
824            if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
825                return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
826            }
827            if !device
828                .downlevel
829                .flags
830                .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
831                && (src_buffer.usage.contains(BufferUsages::INDEX)
832                    || dst_buffer.usage.contains(BufferUsages::INDEX))
833            {
834                let forbidden_usages = BufferUsages::VERTEX
835                    | BufferUsages::UNIFORM
836                    | BufferUsages::INDIRECT
837                    | BufferUsages::STORAGE;
838                if src_buffer.usage.intersects(forbidden_usages)
839                    || dst_buffer.usage.intersects(forbidden_usages)
840                {
841                    return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
842                        wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
843                    ))
844                    .into());
845                }
846            }
847
848            let destination_end_offset = destination_offset + size;
849            if source_end_offset > src_buffer.size {
850                return Err(TransferError::BufferOverrun {
851                    start_offset: source_offset,
852                    end_offset: source_end_offset,
853                    buffer_size: src_buffer.size,
854                    side: CopySide::Source,
855                }
856                .into());
857            }
858            if destination_end_offset > dst_buffer.size {
859                return Err(TransferError::BufferOverrun {
860                    start_offset: destination_offset,
861                    end_offset: destination_end_offset,
862                    buffer_size: dst_buffer.size,
863                    side: CopySide::Destination,
864                }
865                .into());
866            }
867
868            if size == 0 {
869                log::trace!("Ignoring copy_buffer_to_buffer of size 0");
870                return Ok(());
871            }
872
873            // Make sure source is initialized memory and mark dest as initialized.
874            cmd_buf_data.buffer_memory_init_actions.extend(
875                dst_buffer.initialization_status.read().create_action(
876                    &dst_buffer,
877                    destination_offset..(destination_offset + size),
878                    MemoryInitKind::ImplicitlyInitialized,
879                ),
880            );
881            cmd_buf_data.buffer_memory_init_actions.extend(
882                src_buffer.initialization_status.read().create_action(
883                    &src_buffer,
884                    source_offset..(source_offset + size),
885                    MemoryInitKind::NeedsInitializedMemory,
886                ),
887            );
888
889            let region = hal::BufferCopy {
890                src_offset: source_offset,
891                dst_offset: destination_offset,
892                size: wgt::BufferSize::new(size).unwrap(),
893            };
894            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
895            let barriers = src_barrier
896                .into_iter()
897                .chain(dst_barrier)
898                .collect::<Vec<_>>();
899            unsafe {
900                cmd_buf_raw.transition_buffers(&barriers);
901                cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
902            }
903
904            Ok(())
905        })
906    }
907
908    pub fn command_encoder_copy_buffer_to_texture(
909        &self,
910        command_encoder_id: CommandEncoderId,
911        source: &TexelCopyBufferInfo,
912        destination: &TexelCopyTextureInfo,
913        copy_size: &Extent3d,
914    ) -> Result<(), EncoderStateError> {
915        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
916        api_log!(
917            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
918            source.buffer,
919            destination.texture
920        );
921
922        let hub = &self.hub;
923
924        let cmd_enc = hub.command_encoders.get(command_encoder_id);
925        let mut cmd_buf_data = cmd_enc.data.lock();
926        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
927            let device = &cmd_enc.device;
928            device.check_is_valid()?;
929
930            #[cfg(feature = "trace")]
931            if let Some(ref mut list) = cmd_buf_data.commands {
932                list.push(TraceCommand::CopyBufferToTexture {
933                    src: *source,
934                    dst: *destination,
935                    size: *copy_size,
936                });
937            }
938
939            let dst_texture = hub.textures.get(destination.texture).get()?;
940            let src_buffer = hub.buffers.get(source.buffer).get()?;
941
942            dst_texture.same_device_as(cmd_enc.as_ref())?;
943            src_buffer.same_device_as(cmd_enc.as_ref())?;
944
945            let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
946                destination,
947                &dst_texture.desc,
948                CopySide::Destination,
949                copy_size,
950            )?;
951
952            let (dst_range, dst_base) =
953                extract_texture_selector(destination, copy_size, &dst_texture)?;
954
955            let snatch_guard = device.snatchable_lock.read();
956
957            let src_raw = src_buffer.try_raw(&snatch_guard)?;
958            let dst_raw = dst_texture.try_raw(&snatch_guard)?;
959
960            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
961            {
962                log::trace!("Ignoring copy_buffer_to_texture of size 0");
963                return Ok(());
964            }
965
966            // Handle texture init *before* dealing with barrier transitions so we
967            // have an easier time inserting "immediate-inits" that may be required
968            // by prior discards in rare cases.
969            handle_dst_texture_init(
970                cmd_buf_data,
971                device,
972                destination,
973                copy_size,
974                &dst_texture,
975                &snatch_guard,
976            )?;
977
978            let src_pending = cmd_buf_data
979                .trackers
980                .buffers
981                .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
982
983            src_buffer
984                .check_usage(BufferUsages::COPY_SRC)
985                .map_err(TransferError::MissingBufferUsage)?;
986            let src_barrier =
987                src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
988
989            let dst_pending = cmd_buf_data.trackers.textures.set_single(
990                &dst_texture,
991                dst_range,
992                wgt::TextureUses::COPY_DST,
993            );
994            dst_texture
995                .check_usage(TextureUsages::COPY_DST)
996                .map_err(TransferError::MissingTextureUsage)?;
997            let dst_barrier = dst_pending
998                .map(|pending| pending.into_hal(dst_raw))
999                .collect::<Vec<_>>();
1000
1001            validate_texture_copy_dst_format(dst_texture.desc.format, destination.aspect)?;
1002
1003            validate_texture_buffer_copy(
1004                destination,
1005                dst_base.aspect,
1006                &dst_texture.desc,
1007                source.layout.offset,
1008                true, // alignment required for buffer offset
1009            )?;
1010
1011            let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
1012                validate_linear_texture_data(
1013                    &source.layout,
1014                    dst_texture.desc.format,
1015                    destination.aspect,
1016                    src_buffer.size,
1017                    CopySide::Source,
1018                    copy_size,
1019                    true,
1020                )?;
1021
1022            if dst_texture.desc.format.is_depth_stencil_format() {
1023                device
1024                    .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1025                    .map_err(TransferError::from)?;
1026            }
1027
1028            cmd_buf_data.buffer_memory_init_actions.extend(
1029                src_buffer.initialization_status.read().create_action(
1030                    &src_buffer,
1031                    source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
1032                    MemoryInitKind::NeedsInitializedMemory,
1033                ),
1034            );
1035
1036            let regions = (0..array_layer_count)
1037                .map(|rel_array_layer| {
1038                    let mut texture_base = dst_base.clone();
1039                    texture_base.array_layer += rel_array_layer;
1040                    let mut buffer_layout = source.layout;
1041                    buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1042                    hal::BufferTextureCopy {
1043                        buffer_layout,
1044                        texture_base,
1045                        size: hal_copy_size,
1046                    }
1047                })
1048                .collect::<Vec<_>>();
1049
1050            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1051            unsafe {
1052                cmd_buf_raw.transition_textures(&dst_barrier);
1053                cmd_buf_raw.transition_buffers(src_barrier.as_slice());
1054                cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, &regions);
1055            }
1056
1057            Ok(())
1058        })
1059    }
1060
1061    pub fn command_encoder_copy_texture_to_buffer(
1062        &self,
1063        command_encoder_id: CommandEncoderId,
1064        source: &TexelCopyTextureInfo,
1065        destination: &TexelCopyBufferInfo,
1066        copy_size: &Extent3d,
1067    ) -> Result<(), EncoderStateError> {
1068        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
1069        api_log!(
1070            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
1071            source.texture,
1072            destination.buffer
1073        );
1074
1075        let hub = &self.hub;
1076
1077        let cmd_enc = hub.command_encoders.get(command_encoder_id);
1078        let mut cmd_buf_data = cmd_enc.data.lock();
1079        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1080            let device = &cmd_enc.device;
1081            device.check_is_valid()?;
1082
1083            #[cfg(feature = "trace")]
1084            if let Some(list) = cmd_buf_data.commands.as_mut() {
1085                list.push(TraceCommand::CopyTextureToBuffer {
1086                    src: *source,
1087                    dst: *destination,
1088                    size: *copy_size,
1089                });
1090            }
1091
1092            let src_texture = hub.textures.get(source.texture).get()?;
1093            let dst_buffer = hub.buffers.get(destination.buffer).get()?;
1094
1095            src_texture.same_device_as(cmd_enc.as_ref())?;
1096            dst_buffer.same_device_as(cmd_enc.as_ref())?;
1097
1098            let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
1099                source,
1100                &src_texture.desc,
1101                CopySide::Source,
1102                copy_size,
1103            )?;
1104
1105            let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1106
1107            let snatch_guard = device.snatchable_lock.read();
1108
1109            let src_raw = src_texture.try_raw(&snatch_guard)?;
1110            src_texture
1111                .check_usage(TextureUsages::COPY_SRC)
1112                .map_err(TransferError::MissingTextureUsage)?;
1113
1114            if source.mip_level >= src_texture.desc.mip_level_count {
1115                return Err(TransferError::InvalidMipLevel {
1116                    requested: source.mip_level,
1117                    count: src_texture.desc.mip_level_count,
1118                }
1119                .into());
1120            }
1121
1122            validate_texture_copy_src_format(src_texture.desc.format, source.aspect)?;
1123
1124            validate_texture_buffer_copy(
1125                source,
1126                src_base.aspect,
1127                &src_texture.desc,
1128                destination.layout.offset,
1129                true, // alignment required for buffer offset
1130            )?;
1131
1132            let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
1133                validate_linear_texture_data(
1134                    &destination.layout,
1135                    src_texture.desc.format,
1136                    source.aspect,
1137                    dst_buffer.size,
1138                    CopySide::Destination,
1139                    copy_size,
1140                    true,
1141                )?;
1142
1143            if src_texture.desc.format.is_depth_stencil_format() {
1144                device
1145                    .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1146                    .map_err(TransferError::from)?;
1147            }
1148
1149            let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
1150            dst_buffer
1151                .check_usage(BufferUsages::COPY_DST)
1152                .map_err(TransferError::MissingBufferUsage)?;
1153
1154            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1155            {
1156                log::trace!("Ignoring copy_texture_to_buffer of size 0");
1157                return Ok(());
1158            }
1159
1160            // Handle texture init *before* dealing with barrier transitions so we
1161            // have an easier time inserting "immediate-inits" that may be required
1162            // by prior discards in rare cases.
1163            handle_src_texture_init(
1164                cmd_buf_data,
1165                device,
1166                source,
1167                copy_size,
1168                &src_texture,
1169                &snatch_guard,
1170            )?;
1171
1172            let src_pending = cmd_buf_data.trackers.textures.set_single(
1173                &src_texture,
1174                src_range,
1175                wgt::TextureUses::COPY_SRC,
1176            );
1177            let src_barrier = src_pending
1178                .map(|pending| pending.into_hal(src_raw))
1179                .collect::<Vec<_>>();
1180
1181            let dst_pending = cmd_buf_data
1182                .trackers
1183                .buffers
1184                .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
1185
1186            let dst_barrier =
1187                dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
1188
1189            cmd_buf_data.buffer_memory_init_actions.extend(
1190                dst_buffer.initialization_status.read().create_action(
1191                    &dst_buffer,
1192                    destination.layout.offset
1193                        ..(destination.layout.offset + required_buffer_bytes_in_copy),
1194                    MemoryInitKind::ImplicitlyInitialized,
1195                ),
1196            );
1197
1198            let regions = (0..array_layer_count)
1199                .map(|rel_array_layer| {
1200                    let mut texture_base = src_base.clone();
1201                    texture_base.array_layer += rel_array_layer;
1202                    let mut buffer_layout = destination.layout;
1203                    buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1204                    hal::BufferTextureCopy {
1205                        buffer_layout,
1206                        texture_base,
1207                        size: hal_copy_size,
1208                    }
1209                })
1210                .collect::<Vec<_>>();
1211            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1212            unsafe {
1213                cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1214                cmd_buf_raw.transition_textures(&src_barrier);
1215                cmd_buf_raw.copy_texture_to_buffer(
1216                    src_raw,
1217                    wgt::TextureUses::COPY_SRC,
1218                    dst_raw,
1219                    &regions,
1220                );
1221            }
1222
1223            Ok(())
1224        })
1225    }
1226
1227    pub fn command_encoder_copy_texture_to_texture(
1228        &self,
1229        command_encoder_id: CommandEncoderId,
1230        source: &TexelCopyTextureInfo,
1231        destination: &TexelCopyTextureInfo,
1232        copy_size: &Extent3d,
1233    ) -> Result<(), EncoderStateError> {
1234        profiling::scope!("CommandEncoder::copy_texture_to_texture");
1235        api_log!(
1236            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1237            source.texture,
1238            destination.texture
1239        );
1240
1241        let hub = &self.hub;
1242
1243        let cmd_enc = hub.command_encoders.get(command_encoder_id);
1244        let mut cmd_buf_data = cmd_enc.data.lock();
1245        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1246            let device = &cmd_enc.device;
1247            device.check_is_valid()?;
1248
1249            let snatch_guard = device.snatchable_lock.read();
1250
1251            #[cfg(feature = "trace")]
1252            if let Some(ref mut list) = cmd_buf_data.commands {
1253                list.push(TraceCommand::CopyTextureToTexture {
1254                    src: *source,
1255                    dst: *destination,
1256                    size: *copy_size,
1257                });
1258            }
1259
1260            let src_texture = hub.textures.get(source.texture).get()?;
1261            let dst_texture = hub.textures.get(destination.texture).get()?;
1262
1263            src_texture.same_device_as(cmd_enc.as_ref())?;
1264            dst_texture.same_device_as(cmd_enc.as_ref())?;
1265
1266            // src and dst texture format must be copy-compatible
1267            // https://gpuweb.github.io/gpuweb/#copy-compatible
1268            if src_texture.desc.format.remove_srgb_suffix()
1269                != dst_texture.desc.format.remove_srgb_suffix()
1270            {
1271                return Err(TransferError::TextureFormatsNotCopyCompatible {
1272                    src_format: src_texture.desc.format,
1273                    dst_format: dst_texture.desc.format,
1274                }
1275                .into());
1276            }
1277
1278            let (src_copy_size, array_layer_count) = validate_texture_copy_range(
1279                source,
1280                &src_texture.desc,
1281                CopySide::Source,
1282                copy_size,
1283            )?;
1284            let (dst_copy_size, _) = validate_texture_copy_range(
1285                destination,
1286                &dst_texture.desc,
1287                CopySide::Destination,
1288                copy_size,
1289            )?;
1290
1291            if Arc::as_ptr(&src_texture) == Arc::as_ptr(&dst_texture) {
1292                validate_copy_within_same_texture(
1293                    source,
1294                    destination,
1295                    src_texture.desc.format,
1296                    array_layer_count,
1297                )?;
1298            }
1299
1300            let (src_range, src_tex_base) =
1301                extract_texture_selector(source, copy_size, &src_texture)?;
1302            let (dst_range, dst_tex_base) =
1303                extract_texture_selector(destination, copy_size, &dst_texture)?;
1304            let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1305            let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1306            if src_tex_base.aspect != src_texture_aspects {
1307                return Err(TransferError::CopySrcMissingAspects.into());
1308            }
1309            if dst_tex_base.aspect != dst_texture_aspects {
1310                return Err(TransferError::CopyDstMissingAspects.into());
1311            }
1312
1313            if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1314                return Err(TransferError::SampleCountNotEqual {
1315                    src_sample_count: src_texture.desc.sample_count,
1316                    dst_sample_count: dst_texture.desc.sample_count,
1317                }
1318                .into());
1319            }
1320
1321            // Handle texture init *before* dealing with barrier transitions so we
1322            // have an easier time inserting "immediate-inits" that may be required
1323            // by prior discards in rare cases.
1324            handle_src_texture_init(
1325                cmd_buf_data,
1326                device,
1327                source,
1328                copy_size,
1329                &src_texture,
1330                &snatch_guard,
1331            )?;
1332            handle_dst_texture_init(
1333                cmd_buf_data,
1334                device,
1335                destination,
1336                copy_size,
1337                &dst_texture,
1338                &snatch_guard,
1339            )?;
1340
1341            let src_raw = src_texture.try_raw(&snatch_guard)?;
1342            src_texture
1343                .check_usage(TextureUsages::COPY_SRC)
1344                .map_err(TransferError::MissingTextureUsage)?;
1345            let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1346            dst_texture
1347                .check_usage(TextureUsages::COPY_DST)
1348                .map_err(TransferError::MissingTextureUsage)?;
1349
1350            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1351            {
1352                log::trace!("Ignoring copy_texture_to_texture of size 0");
1353                return Ok(());
1354            }
1355
1356            let src_pending = cmd_buf_data.trackers.textures.set_single(
1357                &src_texture,
1358                src_range,
1359                wgt::TextureUses::COPY_SRC,
1360            );
1361
1362            //TODO: try to avoid this the collection. It's needed because both
1363            // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1364            let mut barriers: ArrayVec<_, 2> = src_pending
1365                .map(|pending| pending.into_hal(src_raw))
1366                .collect();
1367
1368            let dst_pending = cmd_buf_data.trackers.textures.set_single(
1369                &dst_texture,
1370                dst_range,
1371                wgt::TextureUses::COPY_DST,
1372            );
1373            barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1374
1375            let hal_copy_size = hal::CopyExtent {
1376                width: src_copy_size.width.min(dst_copy_size.width),
1377                height: src_copy_size.height.min(dst_copy_size.height),
1378                depth: src_copy_size.depth.min(dst_copy_size.depth),
1379            };
1380
1381            let regions = (0..array_layer_count).map(|rel_array_layer| {
1382                let mut src_base = src_tex_base.clone();
1383                let mut dst_base = dst_tex_base.clone();
1384                src_base.array_layer += rel_array_layer;
1385                dst_base.array_layer += rel_array_layer;
1386                hal::TextureCopy {
1387                    src_base,
1388                    dst_base,
1389                    size: hal_copy_size,
1390                }
1391            });
1392
1393            let regions = if dst_tex_base.aspect == hal::FormatAspects::DEPTH_STENCIL {
1394                regions
1395                    .flat_map(|region| {
1396                        let (mut depth, mut stencil) = (region.clone(), region);
1397                        depth.src_base.aspect = hal::FormatAspects::DEPTH;
1398                        depth.dst_base.aspect = hal::FormatAspects::DEPTH;
1399                        stencil.src_base.aspect = hal::FormatAspects::STENCIL;
1400                        stencil.dst_base.aspect = hal::FormatAspects::STENCIL;
1401                        [depth, stencil]
1402                    })
1403                    .collect::<Vec<_>>()
1404            } else {
1405                regions.collect::<Vec<_>>()
1406            };
1407            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1408            unsafe {
1409                cmd_buf_raw.transition_textures(&barriers);
1410                cmd_buf_raw.copy_texture_to_texture(
1411                    src_raw,
1412                    wgt::TextureUses::COPY_SRC,
1413                    dst_raw,
1414                    &regions,
1415                );
1416            }
1417
1418            Ok(())
1419        })
1420    }
1421}
wgpu_core/command/transfer.rs

wgpu_core/command/
transfer.rs
