Skip to main content

wgpu_core/command/
transfer.rs

1use alloc::{format, string::String, sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6    error::{ErrorType, WebGpuError},
7    BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8    TextureUsages,
9};
10
11use crate::{
12    api_log,
13    command::{
14        clear_texture, encoder::EncodingState, ArcCommand, CommandEncoderError, EncoderStateError,
15    },
16    device::MissingDownlevelFlags,
17    init_tracker::{
18        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
19        TextureInitTrackerAction,
20    },
21    resource::{
22        Buffer, Labeled, MissingBufferUsageError, MissingTextureUsageError, ParentDevice,
23        RawResourceAccess, Texture, TextureErrorDimension,
24    },
25};
26
27use super::ClearError;
28
29type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<Arc<Texture>>;
30
31#[derive(Clone, Copy, Debug, Eq, PartialEq)]
32pub enum CopySide {
33    Source,
34    Destination,
35}
36
37/// Error encountered while attempting a data transfer.
38#[derive(Clone, Debug, Error)]
39#[non_exhaustive]
40pub enum TransferError {
41    #[error("Source and destination cannot be the same buffer")]
42    SameSourceDestinationBuffer,
43    #[error(transparent)]
44    MissingBufferUsage(#[from] MissingBufferUsageError),
45    #[error(transparent)]
46    MissingTextureUsage(#[from] MissingTextureUsageError),
47    #[error(
48        "Copy at offset {start_offset} bytes would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}"
49    )]
50    BufferStartOffsetOverrun {
51        start_offset: BufferAddress,
52        buffer_size: BufferAddress,
53        side: CopySide,
54    },
55    #[error(
56        "Copy at offset {start_offset} for {size} bytes would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}"
57    )]
58    BufferEndOffsetOverrun {
59        start_offset: BufferAddress,
60        size: BufferAddress,
61        buffer_size: BufferAddress,
62        side: CopySide,
63    },
64    #[error(
65        "Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the \
66        bounds{plane} of the {side:?} texture of {dimension:?} size {texture_size}",
67        plane = plane.map(|plane| format!(" of plane {plane}")).unwrap_or("".into()),
68    )]
69    TextureOverrun {
70        start_offset: u32,
71        end_offset: u32,
72        texture_size: u32,
73        dimension: TextureErrorDimension,
74        side: CopySide,
75        plane: Option<u32>,
76    },
77    #[error("Partial copy of {start_offset}..{end_offset} on {dimension:?} dimension with size {texture_size} \
78             is not supported for the {side:?} texture format {format:?} with {sample_count} samples")]
79    UnsupportedPartialTransfer {
80        format: wgt::TextureFormat,
81        sample_count: u32,
82        start_offset: u32,
83        end_offset: u32,
84        texture_size: u32,
85        dimension: TextureErrorDimension,
86        side: CopySide,
87    },
88    #[error(
89        "Copying{} layers {}..{} to{} layers {}..{} of the same texture is not allowed",
90        if *src_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {src_aspects:?}") },
91        src_origin_z,
92        src_origin_z + array_layer_count,
93        if *dst_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {dst_aspects:?}") },
94        dst_origin_z,
95        dst_origin_z + array_layer_count,
96    )]
97    InvalidCopyWithinSameTexture {
98        src_aspects: wgt::TextureAspect,
99        dst_aspects: wgt::TextureAspect,
100        src_origin_z: u32,
101        dst_origin_z: u32,
102        array_layer_count: u32,
103    },
104    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
105    InvalidTextureAspect {
106        format: wgt::TextureFormat,
107        aspect: wgt::TextureAspect,
108    },
109    #[error("Unable to select texture mip level {level} out of {total}")]
110    InvalidTextureMipLevel { level: u32, total: u32 },
111    #[error("Texture dimension must be 2D when copying from an external texture")]
112    InvalidDimensionExternal,
113    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
114    UnalignedBufferOffset(BufferAddress),
115    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
116    UnalignedCopySize(BufferAddress),
117    #[error("Copy width is not a multiple of the required alignment")]
118    UnalignedCopyWidth,
119    #[error("Copy height is not a multiple of the required alignment")]
120    UnalignedCopyHeight,
121    #[error("Copy origin's x component is not a multiple of the required alignment")]
122    UnalignedCopyOriginX,
123    #[error("Copy origin's y component is not a multiple of the required alignment")]
124    UnalignedCopyOriginY,
125    #[error("Bytes per row ({bytes_per_row}) is not a multiple of {alignment}")]
126    UnalignedBytesPerRow { bytes_per_row: u32, alignment: u32 },
127    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
128    UnspecifiedBytesPerRow,
129    #[error("Number of rows per image needs to be specified since more than one image is copied")]
130    UnspecifiedRowsPerImage,
131    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
132    InvalidBytesPerRow,
133    #[error("Number of rows per image is invalid")]
134    InvalidRowsPerImage,
135    #[error("Overflow while computing the size of the copy")]
136    SizeOverflow,
137    #[error("Copy source aspects must refer to all aspects of the source texture format")]
138    CopySrcMissingAspects,
139    #[error(
140        "Copy destination aspects must refer to all aspects of the destination texture format"
141    )]
142    CopyDstMissingAspects,
143    #[error("Copy aspect must refer to a single aspect of texture format")]
144    CopyAspectNotOne,
145    #[error("Copying from textures with format {0:?} is forbidden")]
146    CopyFromForbiddenTextureFormat(wgt::TextureFormat),
147    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
148    CopyFromForbiddenTextureFormatAspect {
149        format: wgt::TextureFormat,
150        aspect: wgt::TextureAspect,
151    },
152    #[error("Copying to textures with format {0:?} is forbidden")]
153    CopyToForbiddenTextureFormat(wgt::TextureFormat),
154    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
155    CopyToForbiddenTextureFormatAspect {
156        format: wgt::TextureFormat,
157        aspect: wgt::TextureAspect,
158    },
159    #[error(
160        "Copying to textures with format {0:?} is forbidden when copying from external texture"
161    )]
162    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
163    // Copies in or out of a plane of a multi-planar texture use the plane's format,
164    // e.g. `R8Unorm` or `Rg8Unorm`. Report this as `R8Unorm, in plane 0 of NV12`.
165    #[error(
166        "Source format ({src_format:?}{src_plane_info}) and destination format ({dst_format:?}{dst_plane_info}) \
167        are not copy-compatible (they may only differ in srgb-ness)",
168        src_plane_info = src_multi_planar.map(|(format, plane)| format!(", in plane {plane} of {format:?}")).unwrap_or("".into()),
169        dst_plane_info = dst_multi_planar.map(|(format, plane)| format!(", in plane {plane} of {format:?}")).unwrap_or("".into()),
170    )]
171    TextureFormatsNotCopyCompatible {
172        src_format: wgt::TextureFormat,
173        src_multi_planar: Option<(wgt::TextureFormat, u32)>,
174        dst_format: wgt::TextureFormat,
175        dst_multi_planar: Option<(wgt::TextureFormat, u32)>,
176    },
177    #[error(transparent)]
178    MemoryInitFailure(#[from] ClearError),
179    #[error("Cannot encode this copy because of a missing downelevel flag")]
180    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
181    #[error("Source texture sample count must be 1, got {sample_count}")]
182    InvalidSampleCount { sample_count: u32 },
183    #[error(
184        "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
185    )]
186    SampleCountNotEqual {
187        src_sample_count: u32,
188        dst_sample_count: u32,
189    },
190    #[error("Requested mip level {requested} does not exist (count: {count})")]
191    InvalidMipLevel { requested: u32, count: u32 },
192    #[error("Buffer is expected to be unmapped, but was not")]
193    BufferNotAvailable,
194}
195
196impl WebGpuError for TransferError {
197    fn webgpu_error_type(&self) -> ErrorType {
198        match self {
199            Self::MissingBufferUsage(e) => e.webgpu_error_type(),
200            Self::MissingTextureUsage(e) => e.webgpu_error_type(),
201            Self::MemoryInitFailure(e) => e.webgpu_error_type(),
202
203            Self::BufferEndOffsetOverrun { .. }
204            | Self::TextureOverrun { .. }
205            | Self::BufferStartOffsetOverrun { .. }
206            | Self::UnsupportedPartialTransfer { .. }
207            | Self::InvalidCopyWithinSameTexture { .. }
208            | Self::InvalidTextureAspect { .. }
209            | Self::InvalidTextureMipLevel { .. }
210            | Self::InvalidDimensionExternal
211            | Self::UnalignedBufferOffset(..)
212            | Self::UnalignedCopySize(..)
213            | Self::UnalignedCopyWidth
214            | Self::UnalignedCopyHeight
215            | Self::UnalignedCopyOriginX
216            | Self::UnalignedCopyOriginY
217            | Self::UnalignedBytesPerRow { .. }
218            | Self::UnspecifiedBytesPerRow
219            | Self::UnspecifiedRowsPerImage
220            | Self::InvalidBytesPerRow
221            | Self::InvalidRowsPerImage
222            | Self::SizeOverflow
223            | Self::CopySrcMissingAspects
224            | Self::CopyDstMissingAspects
225            | Self::CopyAspectNotOne
226            | Self::CopyFromForbiddenTextureFormat(..)
227            | Self::CopyFromForbiddenTextureFormatAspect { .. }
228            | Self::CopyToForbiddenTextureFormat(..)
229            | Self::CopyToForbiddenTextureFormatAspect { .. }
230            | Self::ExternalCopyToForbiddenTextureFormat(..)
231            | Self::TextureFormatsNotCopyCompatible { .. }
232            | Self::MissingDownlevelFlags(..)
233            | Self::InvalidSampleCount { .. }
234            | Self::SampleCountNotEqual { .. }
235            | Self::InvalidMipLevel { .. }
236            | Self::SameSourceDestinationBuffer
237            | Self::BufferNotAvailable => ErrorType::Validation,
238        }
239    }
240}
241
242impl From<BufferTextureCopyInfoError> for TransferError {
243    fn from(value: BufferTextureCopyInfoError) -> Self {
244        match value {
245            BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
246            BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
247            BufferTextureCopyInfoError::ImageStrideOverflow
248            | BufferTextureCopyInfoError::ImageBytesOverflow(_)
249            | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
250        }
251    }
252}
253
254/// WebGPU's [validating linear texture data][vltd] algorithm.
255///
256/// Copied with some modifications from WebGPU standard.
257///
258/// If successful, returns a tuple `(bytes, stride, is_contiguous)`, where:
259/// - `bytes` is the number of buffer bytes required for this copy, and
260/// - `stride` number of bytes between array layers.
261/// - `is_contiguous` is true if the linear texture data does not have padding
262///   between rows or between images.
263///
264/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
265pub(crate) fn validate_linear_texture_data(
266    layout: &wgt::TexelCopyBufferLayout,
267    format: wgt::TextureFormat,
268    aspect: wgt::TextureAspect,
269    buffer_size: BufferAddress,
270    buffer_side: CopySide,
271    copy_size: &Extent3d,
272) -> Result<(BufferAddress, BufferAddress, bool), TransferError> {
273    let wgt::BufferTextureCopyInfo {
274        copy_width,
275        copy_height,
276        depth_or_array_layers,
277
278        offset,
279
280        block_size_bytes: _,
281        block_width_texels,
282        block_height_texels,
283
284        width_blocks: _,
285        height_blocks,
286
287        row_bytes_dense,
288        row_stride_bytes,
289
290        image_stride_rows: _,
291        image_stride_bytes,
292
293        image_rows_dense: _,
294        image_bytes_dense,
295
296        bytes_in_copy,
297    } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
298
299    if !copy_width.is_multiple_of(block_width_texels) {
300        return Err(TransferError::UnalignedCopyWidth);
301    }
302    if !copy_height.is_multiple_of(block_height_texels) {
303        return Err(TransferError::UnalignedCopyHeight);
304    }
305
306    let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
307    let requires_multiple_images = depth_or_array_layers > 1;
308
309    // `get_buffer_texture_copy_info()` already proceeded with defaults if these
310    // were not specified, and ensured that the values satisfy the minima if
311    // they were, but now we enforce the WebGPU requirement that they be
312    // specified any time they apply.
313    if layout.bytes_per_row.is_none() && requires_multiple_rows {
314        return Err(TransferError::UnspecifiedBytesPerRow);
315    }
316
317    if layout.rows_per_image.is_none() && requires_multiple_images {
318        return Err(TransferError::UnspecifiedRowsPerImage);
319    };
320
321    if offset > buffer_size {
322        return Err(TransferError::BufferStartOffsetOverrun {
323            start_offset: offset,
324            buffer_size,
325            side: buffer_side,
326        });
327    }
328    // NOTE: Should never underflow because of our earlier check.
329    if bytes_in_copy > buffer_size - offset {
330        return Err(TransferError::BufferEndOffsetOverrun {
331            start_offset: offset,
332            size: bytes_in_copy,
333            buffer_size,
334            side: buffer_side,
335        });
336    }
337
338    let is_contiguous = (row_stride_bytes == row_bytes_dense || !requires_multiple_rows)
339        && (image_stride_bytes == image_bytes_dense || !requires_multiple_images);
340
341    Ok((bytes_in_copy, image_stride_bytes, is_contiguous))
342}
343
344/// Validate the source format of a texture-to-buffer copy.
345///
346/// This performs the check from WebGPU's [validating texture buffer copy][vtbc]
347/// algorithm that ensures that the format and aspect form a valid texel copy source
348/// as defined in the [depth-stencil formats][dsf]. This check differs depending
349/// on whether the copy is buffer-to-texture or texture-to-buffer. Other checks
350/// that do not differ by direction are implemented in [`validate_texture_buffer_copy`].
351///
352/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
353/// [dsf]: https://gpuweb.github.io/gpuweb/#depth-formats
354pub(crate) fn validate_texture_copy_src_format(
355    format: wgt::TextureFormat,
356    aspect: wgt::TextureAspect,
357) -> Result<(), TransferError> {
358    use wgt::TextureAspect as Ta;
359    use wgt::TextureFormat as Tf;
360    match (format, aspect) {
361        (Tf::Depth24Plus, _) => Err(TransferError::CopyFromForbiddenTextureFormat(format)),
362        (Tf::Depth24PlusStencil8, Ta::DepthOnly) => {
363            Err(TransferError::CopyFromForbiddenTextureFormatAspect { format, aspect })
364        }
365        _ => Ok(()),
366    }
367}
368
369/// Validate the destination format of a buffer-to-texture copy.
370///
371/// This performs the check from WebGPU's [validating texture buffer copy][vtbc]
372/// algorithm that ensures that the format and aspect form a valid texel copy destination
373/// as defined in the [depth-stencil formats][dsf]. This check differs depending
374/// on whether the copy is buffer-to-texture or texture-to-buffer. Other checks
375/// that do not differ by direction are implemented in [`validate_texture_buffer_copy`].
376///
377/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
378/// [dsf]: https://gpuweb.github.io/gpuweb/#depth-formats
379pub(crate) fn validate_texture_copy_dst_format(
380    format: wgt::TextureFormat,
381    aspect: wgt::TextureAspect,
382) -> Result<(), TransferError> {
383    use wgt::TextureAspect as Ta;
384    use wgt::TextureFormat as Tf;
385    match (format, aspect) {
386        (Tf::Depth24Plus | Tf::Depth32Float, _) => {
387            Err(TransferError::CopyToForbiddenTextureFormat(format))
388        }
389        (Tf::Depth24PlusStencil8 | Tf::Depth32FloatStencil8, Ta::DepthOnly) => {
390            Err(TransferError::CopyToForbiddenTextureFormatAspect { format, aspect })
391        }
392        _ => Ok(()),
393    }
394}
395
396/// Validation for texture/buffer copies.
397///
398/// This implements the following checks from WebGPU's [validating texture buffer copy][vtbc]
399/// algorithm:
400///  * The texture must not be multisampled.
401///  * The copy must be from/to a single aspect of the texture.
402///  * If `aligned` is true, the buffer offset must be aligned appropriately.
403///
404/// And implements the following check from WebGPU's [validating GPUTexelCopyBufferInfo][vtcbi]
405/// algorithm:
406///  * If `aligned` is true, `bytesPerRow` must be a multiple of 256, or only of the texel
407///    block copy size with [`wgt::Features::UNRESTRICTED_BYTES_PER_ROW`].
408///
409/// Note that the `bytesPerRow` alignment check is enforced whenever
410/// `bytesPerRow` is specified, even if the transfer is not multiple rows and
411/// `bytesPerRow` could have been omitted.
412///
413/// The following steps in [validating texture buffer copy][vtbc] are implemented elsewhere:
414///  * Invocation of other validation algorithms.
415///  * The texture usage (COPY_DST / COPY_SRC) check.
416///  * The check for non-copyable depth/stencil formats. The caller must perform
417///    this check using `validate_texture_copy_src_format` / `validate_texture_copy_dst_format`
418///    before calling this function. This function will panic if
419///    [`wgt::TextureFormat::block_copy_size`] returns `None` due to a
420///    non-copyable format.
421///
422/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
423/// [vtcbi]: https://www.w3.org/TR/webgpu/#abstract-opdef-validating-gputexelcopybufferinfo
424pub(crate) fn validate_texture_buffer_copy<T>(
425    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
426    aspect: hal::FormatAspects,
427    desc: &wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
428    layout: &wgt::TexelCopyBufferLayout,
429    aligned: bool,
430    features: wgt::Features,
431) -> Result<(), TransferError> {
432    if desc.sample_count != 1 {
433        return Err(TransferError::InvalidSampleCount {
434            sample_count: desc.sample_count,
435        });
436    }
437
438    if !aspect.is_one() {
439        return Err(TransferError::CopyAspectNotOne);
440    }
441
442    let block_copy_size = || {
443        desc.format
444            .block_copy_size(Some(texture_copy_view.aspect))
445            .expect("non-copyable formats should have been rejected previously")
446    };
447    let offset_alignment = if desc.format.is_depth_stencil_format() {
448        4
449    } else {
450        block_copy_size()
451    };
452
453    if aligned && !layout.offset.is_multiple_of(u64::from(offset_alignment)) {
454        return Err(TransferError::UnalignedBufferOffset(layout.offset));
455    }
456
457    if let Some(bytes_per_row) = layout.bytes_per_row {
458        let alignment = if features.contains(wgt::Features::UNRESTRICTED_BYTES_PER_ROW) {
459            block_copy_size()
460        } else {
461            wgt::COPY_BYTES_PER_ROW_ALIGNMENT
462        };
463        if aligned && !bytes_per_row.is_multiple_of(alignment) {
464            return Err(TransferError::UnalignedBytesPerRow {
465                bytes_per_row,
466                alignment,
467            });
468        }
469    }
470
471    Ok(())
472}
473
474pub(crate) struct CopyRange {
475    pub hal_copy_extent: hal::CopyExtent,
476    pub hal_base: hal::TextureCopyBase,
477    pub array_layer_count: u32,
478    pub selector: TextureSelector,
479}
480
481/// Validate the extent and alignment of a texture copy.
482///
483/// Mostly follows the [validating GPUTexelCopyTextureInfo][vtcti] and [validating texture
484/// copy range][vtcr] algorithms in the WebGPU standard, with some modifications for
485/// wgpu extensions.
486///
487/// Returns a validated [`CopyRange`].
488///
489/// [vtcti]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gputexelcopytextureinfo
490/// [vtcr]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-copy-range
491pub(crate) fn validate_texture_copy_range<T>(
492    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
493    desc: &wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
494    texture_side: CopySide,
495    copy_size: &Extent3d,
496) -> Result<CopyRange, TransferError> {
497    let hal_aspects = hal::FormatAspects::new(desc.format, texture_copy_view.aspect);
498    if hal_aspects.is_empty() {
499        return Err(TransferError::InvalidTextureAspect {
500            format: desc.format,
501            aspect: texture_copy_view.aspect,
502        });
503    }
504
505    let is_multi_planar_all =
506        desc.format.is_multi_planar_format() && texture_copy_view.aspect == wgt::TextureAspect::All;
507
508    let (block_width, block_height) = if is_multi_planar_all {
509        // An all-planes copy is specified in luma (plane 0) coordinates, but must be
510        // aligned with the chroma subsampling. Single-plane copies do not have this
511        // requirement.
512        desc.format.size_multiple_requirement()
513    } else {
514        desc.format.block_dimensions()
515    };
516
517    let mip_level = texture_copy_view.mip_level;
518    let plane = texture_copy_view.aspect.to_plane();
519    if mip_level >= desc.mip_level_count {
520        return Err(TransferError::InvalidTextureMipLevel {
521            level: mip_level,
522            total: desc.mip_level_count,
523        });
524    }
525    // For copies to or from a single plane of a multi-planar format, `copy_size` and
526    // `origin` are interpreted in *plane* texels, i.e. the valid copy range for a 2->1
527    // subsampled chroma plane is (W/2, H/2). For copies of all planes, they are
528    // interpreted in plane 0 texels.
529    let extent_virtual = desc.plane_size(mip_level, plane.or(is_multi_planar_all.then_some(0)));
530    // physical size can be larger than the virtual
531    let extent = extent_virtual.physical_size(desc.format);
532
533    // Multisampled and depth-stencil formats do not support partial copies
534    // on x and y dimensions, but do support copying a subset of layers.
535    let requires_exact_size = desc.format.is_depth_stencil_format() || desc.sample_count > 1;
536
537    // Return `Ok` if a run `size` texels long starting at `start_offset` is
538    // valid for `texture_size`. Otherwise, return an appropriate a`Err`.
539    let check_dimension = |dimension: TextureErrorDimension,
540                           start_offset: u32,
541                           size: u32,
542                           texture_size: u32,
543                           requires_exact_size: bool|
544     -> Result<(), TransferError> {
545        if requires_exact_size && (start_offset != 0 || size != texture_size) {
546            Err(TransferError::UnsupportedPartialTransfer {
547                format: desc.format,
548                sample_count: desc.sample_count,
549                start_offset,
550                end_offset: start_offset.wrapping_add(size),
551                texture_size,
552                dimension,
553                side: texture_side,
554            })
555        // Avoid underflow in the subtraction by checking start_offset against
556        // texture_size first.
557        } else if start_offset > texture_size || texture_size - start_offset < size {
558            Err(TransferError::TextureOverrun {
559                start_offset,
560                end_offset: start_offset.wrapping_add(size),
561                texture_size,
562                dimension,
563                side: texture_side,
564                plane,
565            })
566        } else {
567            Ok(())
568        }
569    };
570
571    check_dimension(
572        TextureErrorDimension::X,
573        texture_copy_view.origin.x,
574        copy_size.width,
575        extent.width,
576        requires_exact_size,
577    )?;
578    check_dimension(
579        TextureErrorDimension::Y,
580        texture_copy_view.origin.y,
581        copy_size.height,
582        extent.height,
583        requires_exact_size,
584    )?;
585    check_dimension(
586        TextureErrorDimension::Z,
587        texture_copy_view.origin.z,
588        copy_size.depth_or_array_layers,
589        extent.depth_or_array_layers,
590        false, // partial copy always allowed on Z/layer dimension
591    )?;
592
593    if !texture_copy_view.origin.x.is_multiple_of(block_width) {
594        return Err(TransferError::UnalignedCopyOriginX);
595    }
596    if !texture_copy_view.origin.y.is_multiple_of(block_height) {
597        return Err(TransferError::UnalignedCopyOriginY);
598    }
599    if !copy_size.width.is_multiple_of(block_width) {
600        return Err(TransferError::UnalignedCopyWidth);
601    }
602    if !copy_size.height.is_multiple_of(block_height) {
603        return Err(TransferError::UnalignedCopyHeight);
604    }
605
606    let (depth, array_layer_count) = match desc.dimension {
607        wgt::TextureDimension::D1 => (1, 1),
608        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
609        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
610    };
611
612    let hal_copy_extent = hal::CopyExtent {
613        width: copy_size.width,
614        height: copy_size.height,
615        depth,
616    };
617
618    // Prepare `hal::TextureCopyBase` and `hal::TextureSelector`.
619
620    let (layers, origin_z) = match desc.dimension {
621        wgt::TextureDimension::D1 => (0..1, 0),
622        wgt::TextureDimension::D2 => (
623            texture_copy_view.origin.z
624                ..texture_copy_view.origin.z + copy_size.depth_or_array_layers,
625            0,
626        ),
627        wgt::TextureDimension::D3 => (0..1, texture_copy_view.origin.z),
628    };
629    let hal_base = hal::TextureCopyBase {
630        origin: wgt::Origin3d {
631            x: texture_copy_view.origin.x,
632            y: texture_copy_view.origin.y,
633            z: origin_z,
634        },
635        // this value will be incremented per copied layer
636        array_layer: layers.start,
637        mip_level: texture_copy_view.mip_level,
638        aspect: hal_aspects,
639    };
640    let selector = TextureSelector {
641        mips: texture_copy_view.mip_level..texture_copy_view.mip_level + 1,
642        layers,
643    };
644
645    Ok(CopyRange {
646        hal_copy_extent,
647        hal_base,
648        array_layer_count,
649        selector,
650    })
651}
652
653/// Validate a copy within the same texture.
654///
655/// This implements the WebGPU requirement that the [sets of subresources for
656/// texture copy][srtc] of the source and destination be disjoint, i.e. that the
657/// source and destination do not overlap.
658///
659/// This function assumes that the copy ranges have already been validated with
660/// `validate_texture_copy_range`.
661///
662/// [srtc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-set-of-subresources-for-texture-copy
663pub(crate) fn validate_copy_within_same_texture<T>(
664    src: &wgt::TexelCopyTextureInfo<T>,
665    dst: &wgt::TexelCopyTextureInfo<T>,
666    format: wgt::TextureFormat,
667    array_layer_count: u32,
668) -> Result<(), TransferError> {
669    let src_aspects = hal::FormatAspects::new(format, src.aspect);
670    let dst_aspects = hal::FormatAspects::new(format, dst.aspect);
671    if (src_aspects & dst_aspects).is_empty() {
672        // Copying between different aspects (if it even makes sense), is okay.
673        return Ok(());
674    }
675
676    if src.origin.z >= dst.origin.z + array_layer_count
677        || dst.origin.z >= src.origin.z + array_layer_count
678    {
679        // Copying between non-overlapping layer ranges is okay.
680        return Ok(());
681    }
682
683    if src.mip_level != dst.mip_level {
684        // Copying between different mip levels is okay.
685        return Ok(());
686    }
687
688    Err(TransferError::InvalidCopyWithinSameTexture {
689        src_aspects: src.aspect,
690        dst_aspects: dst.aspect,
691        src_origin_z: src.origin.z,
692        dst_origin_z: dst.origin.z,
693        array_layer_count,
694    })
695}
696
697fn handle_texture_init(
698    state: &mut EncodingState,
699    init_kind: MemoryInitKind,
700    copy_texture: &TexelCopyTextureInfo,
701    copy_size: &Extent3d,
702    texture: &Arc<Texture>,
703) -> Result<(), ClearError> {
704    let init_layer_range = if texture.desc.dimension == wgt::TextureDimension::D3 {
705        // Init tracking only considers array layers, not depth/volume slices
706        0..1
707    } else {
708        copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers
709    };
710    let init_action = TextureInitTrackerAction {
711        texture: texture.clone(),
712        range: TextureInitRange {
713            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
714            layer_range: init_layer_range,
715        },
716        kind: init_kind,
717    };
718
719    // Record the initialization action. Simultaneously, collect a list of any ranges of the
720    // texture that were discarded within the current command buffer, for immediate
721    // initialization. (The analogous case for passes is in `fixup_discarded_surfaces`.)
722    //
723    // Depth slices are only relevant to deciding which pending discards (in a command
724    // buffer with prior render passes) have to be repaired ahead of this copy. Any
725    // initialization that gets generated covers the whole mip level, because that is the
726    // granularity of the init tracker.
727    let accessed_depth_slices = (texture.desc.dimension == wgt::TextureDimension::D3)
728        .then(|| copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers);
729    let immediate_inits = state
730        .texture_memory_actions
731        .register_init_action(&{ init_action }, accessed_depth_slices);
732
733    // In rare cases we may need to insert an init operation immediately onto the command buffer.
734    if !immediate_inits.is_empty() {
735        for init in immediate_inits {
736            let index = init.layer_or_depth_slice;
737            let (layer_range, depth_slice) = if texture.desc.dimension == wgt::TextureDimension::D3
738            {
739                (0..1, Some(index))
740            } else {
741                (index..(index + 1), None)
742            };
743            clear_texture(
744                &init.texture,
745                TextureInitRange {
746                    mip_range: init.mip_level..(init.mip_level + 1),
747                    layer_range,
748                },
749                depth_slice,
750                state.raw_encoder,
751                &mut state.tracker.textures,
752                &state.device.alignments,
753                state.device.zero_buffer.as_ref(),
754                state.snatch_guard,
755                state.device.instance_flags,
756            )?;
757        }
758    }
759
760    Ok(())
761}
762
763/// Prepare a transfer's source texture.
764///
765/// Ensure the source texture of a transfer is in the right initialization
766/// state, and record the state for after the transfer operation.
767fn handle_src_texture_init(
768    state: &mut EncodingState,
769    source: &TexelCopyTextureInfo,
770    copy_size: &Extent3d,
771    texture: &Arc<Texture>,
772) -> Result<(), TransferError> {
773    handle_texture_init(
774        state,
775        MemoryInitKind::NeedsInitializedMemory,
776        source,
777        copy_size,
778        texture,
779    )?;
780    Ok(())
781}
782
783/// Prepare a transfer's destination texture.
784///
785/// Ensure the destination texture of a transfer is in the right initialization
786/// state, and record the state for after the transfer operation.
787fn handle_dst_texture_init(
788    state: &mut EncodingState,
789    destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
790    copy_size: &Extent3d,
791    texture: &Arc<Texture>,
792) -> Result<(), TransferError> {
793    // Attention: If we don't write full texture subresources, we need to a full
794    // clear first since we don't track subrects. This means that in rare cases
795    // even a *destination* texture of a transfer may need an immediate texture
796    // init.
797    let dst_init_kind =
798        if has_copy_partial_init_tracker_coverage(copy_size, destination, &texture.desc) {
799            MemoryInitKind::NeedsInitializedMemory
800        } else {
801            MemoryInitKind::ImplicitlyInitialized
802        };
803
804    handle_texture_init(state, dst_init_kind, destination, copy_size, texture)?;
805    Ok(())
806}
807
808/// Handle initialization tracking for a transfer's source or destination buffer.
809///
810/// Ensures that the transfer will not read from uninitialized memory, and updates
811/// the initialization state information to reflect the transfer.
812fn handle_buffer_init(
813    state: &mut EncodingState,
814    info: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
815    direction: CopySide,
816    required_buffer_bytes_in_copy: BufferAddress,
817    is_contiguous: bool,
818) {
819    const ALIGN_SIZE: BufferAddress = wgt::COPY_BUFFER_ALIGNMENT;
820    const ALIGN_MASK: BufferAddress = wgt::COPY_BUFFER_ALIGNMENT - 1;
821
822    let buffer = &info.buffer;
823    let start = info.layout.offset;
824    let end = info.layout.offset + required_buffer_bytes_in_copy;
825    if !is_contiguous || direction == CopySide::Source {
826        // If the transfer will read the buffer, then the whole region needs to
827        // be initialized.
828        //
829        // If the transfer will not write a contiguous region of the buffer,
830        // then we need to make sure the padding areas are initialized. For now,
831        // initialize the whole region, although this could be improved to
832        // initialize only the necessary parts if doing so is likely to be
833        // faster than initializing the whole thing.
834        //
835        // Adjust the start/end outwards to 4B alignment.
836        let aligned_start = start & !ALIGN_MASK;
837        let aligned_end = (end + ALIGN_MASK) & !ALIGN_MASK;
838        state
839            .buffer_memory_init_actions
840            .extend(buffer.initialization_status.read().create_action(
841                buffer,
842                aligned_start..aligned_end,
843                MemoryInitKind::NeedsInitializedMemory,
844            ));
845    } else {
846        // If the transfer will write a contiguous region of the buffer, then we
847        // don't need to initialize that region.
848        //
849        // However, if the start and end are not 4B aligned, we need to make
850        // sure that we don't end up trying to initialize non-4B-aligned regions
851        // later.
852        //
853        // Adjust the start/end inwards to 4B alignment, we will handle the
854        // first/last pieces differently.
855        let aligned_start = (start + ALIGN_MASK) & !ALIGN_MASK;
856        let aligned_end = end & !ALIGN_MASK;
857        if aligned_start != start {
858            state.buffer_memory_init_actions.extend(
859                buffer.initialization_status.read().create_action(
860                    buffer,
861                    aligned_start - ALIGN_SIZE..aligned_start,
862                    MemoryInitKind::NeedsInitializedMemory,
863                ),
864            );
865        }
866        if aligned_start != aligned_end {
867            state.buffer_memory_init_actions.extend(
868                buffer.initialization_status.read().create_action(
869                    buffer,
870                    aligned_start..aligned_end,
871                    MemoryInitKind::ImplicitlyInitialized,
872                ),
873            );
874        }
875        if aligned_end != end {
876            // It is possible that `aligned_end + ALIGN_SIZE > dst_buffer.size`,
877            // because `dst_buffer.size` is the user-requested size, not the
878            // final size of the buffer. The final size of the buffer is not
879            // readily available, but was rounded up to COPY_BUFFER_ALIGNMENT,
880            // so no overrun is possible.
881            state.buffer_memory_init_actions.extend(
882                buffer.initialization_status.read().create_action(
883                    buffer,
884                    aligned_end..aligned_end + ALIGN_SIZE,
885                    MemoryInitKind::NeedsInitializedMemory,
886                ),
887            );
888        }
889    }
890}
891
892impl super::CommandEncoder {
893    fn copy_buffer_to_buffer_inner(
894        self: &Arc<Self>,
895        source: Arc<Buffer>,
896        source_offset: BufferAddress,
897        destination: Arc<Buffer>,
898        destination_offset: BufferAddress,
899        size: Option<BufferAddress>,
900    ) -> Result<(), EncoderStateError> {
901        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
902        api_log!(
903            "CommandEncoder::copy_buffer_to_buffer {:?} -> {:?} {size:?}bytes",
904            Arc::as_ptr(&source),
905            Arc::as_ptr(&destination)
906        );
907
908        let mut cmd_buf_data = self.data.lock();
909
910        cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
911            source.check_is_valid()?;
912            destination.check_is_valid()?;
913            Ok(ArcCommand::CopyBufferToBuffer {
914                src: source,
915                src_offset: source_offset,
916                dst: destination,
917                dst_offset: destination_offset,
918                size,
919            })
920        })
921    }
922
923    pub fn copy_buffer_to_buffer(
924        self: &Arc<Self>,
925        source: Arc<Buffer>,
926        source_offset: BufferAddress,
927        destination: Arc<Buffer>,
928        destination_offset: BufferAddress,
929        size: Option<BufferAddress>,
930    ) {
931        if let Err(err) = self.copy_buffer_to_buffer_inner(
932            source,
933            source_offset,
934            destination,
935            destination_offset,
936            size,
937        ) {
938            self.device.handle_error(
939                err,
940                Some(self.label()),
941                "CommandEncoder::copy_buffer_to_buffer",
942            );
943        }
944    }
945
946    fn copy_buffer_to_texture_inner(
947        self: &Arc<Self>,
948        source: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
949        destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
950        copy_size: &Extent3d,
951    ) -> Result<(), EncoderStateError> {
952        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
953        api_log!(
954            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
955            Arc::as_ptr(&source.buffer),
956            Arc::as_ptr(&destination.texture)
957        );
958
959        let mut cmd_buf_data = self.data.lock();
960
961        cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
962            let texture = destination.texture.clone();
963            texture.check_valid()?;
964            let source_buffer = source.buffer.clone();
965            source_buffer.check_is_valid()?;
966            Ok(ArcCommand::CopyBufferToTexture {
967                src: wgt::TexelCopyBufferInfo::<Arc<Buffer>> {
968                    buffer: source_buffer,
969                    layout: source.layout,
970                },
971                dst: wgt::TexelCopyTextureInfo::<Arc<Texture>> {
972                    texture,
973                    mip_level: destination.mip_level,
974                    origin: destination.origin,
975                    aspect: destination.aspect,
976                },
977                size: *copy_size,
978            })
979        })
980    }
981
982    pub fn copy_buffer_to_texture(
983        self: &Arc<Self>,
984        source: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
985        destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
986        copy_size: &Extent3d,
987    ) {
988        if let Err(err) = self.copy_buffer_to_texture_inner(source, destination, copy_size) {
989            self.device.handle_error(
990                err,
991                Some(self.label()),
992                "CommandEncoder::copy_buffer_to_texture",
993            );
994        }
995    }
996
997    fn copy_texture_to_buffer_inner(
998        self: &Arc<Self>,
999        source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1000        destination: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1001        copy_size: &Extent3d,
1002    ) -> Result<(), EncoderStateError> {
1003        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
1004        api_log!(
1005            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
1006            Arc::as_ptr(&source.texture),
1007            Arc::as_ptr(&destination.buffer)
1008        );
1009
1010        let mut cmd_buf_data = self.data.lock();
1011
1012        cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
1013            let texture = source.texture.clone();
1014            texture.check_valid()?;
1015            let destination_buffer = destination.buffer.clone();
1016            destination_buffer.check_is_valid()?;
1017            Ok(ArcCommand::CopyTextureToBuffer {
1018                src: wgt::TexelCopyTextureInfo::<Arc<Texture>> {
1019                    texture,
1020                    mip_level: source.mip_level,
1021                    origin: source.origin,
1022                    aspect: source.aspect,
1023                },
1024                dst: wgt::TexelCopyBufferInfo::<Arc<Buffer>> {
1025                    buffer: destination_buffer,
1026                    layout: destination.layout,
1027                },
1028                size: *copy_size,
1029            })
1030        })
1031    }
1032
1033    pub fn copy_texture_to_buffer(
1034        self: &Arc<Self>,
1035        source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1036        destination: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1037        copy_size: &Extent3d,
1038    ) {
1039        if let Err(err) = self.copy_texture_to_buffer_inner(source, destination, copy_size) {
1040            self.device.handle_error(
1041                err,
1042                Some(self.label()),
1043                "CommandEncoder::copy_texture_to_buffer",
1044            );
1045        }
1046    }
1047
1048    fn copy_texture_to_texture_inner(
1049        self: &Arc<Self>,
1050        source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1051        destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1052        copy_size: &Extent3d,
1053    ) -> Result<(), EncoderStateError> {
1054        profiling::scope!("CommandEncoder::copy_texture_to_texture");
1055        api_log!(
1056            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1057            Arc::as_ptr(&source.texture),
1058            Arc::as_ptr(&destination.texture)
1059        );
1060
1061        let mut cmd_buf_data = self.data.lock();
1062
1063        cmd_buf_data.push_with(|| -> Result<_, CommandEncoderError> {
1064            let src_texture = source.texture.clone();
1065            let dst_texture = destination.texture.clone();
1066            src_texture.check_valid()?;
1067            dst_texture.check_valid()?;
1068            Ok(ArcCommand::CopyTextureToTexture {
1069                src: wgt::TexelCopyTextureInfo {
1070                    texture: src_texture,
1071                    mip_level: source.mip_level,
1072                    origin: source.origin,
1073                    aspect: source.aspect,
1074                },
1075                dst: wgt::TexelCopyTextureInfo {
1076                    texture: dst_texture,
1077                    mip_level: destination.mip_level,
1078                    origin: destination.origin,
1079                    aspect: destination.aspect,
1080                },
1081                size: *copy_size,
1082            })
1083        })
1084    }
1085
1086    pub fn copy_texture_to_texture(
1087        self: &Arc<Self>,
1088        source: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1089        destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1090        copy_size: &Extent3d,
1091    ) {
1092        if let Err(err) = self.copy_texture_to_texture_inner(source, destination, copy_size) {
1093            self.device.handle_error(
1094                err,
1095                Some(self.label()),
1096                "CommandEncoder::copy_texture_to_texture",
1097            );
1098        }
1099    }
1100}
1101
1102pub(super) fn copy_buffer_to_buffer(
1103    state: &mut EncodingState,
1104    src_buffer: &Arc<Buffer>,
1105    source_offset: BufferAddress,
1106    dst_buffer: &Arc<Buffer>,
1107    destination_offset: BufferAddress,
1108    size: Option<BufferAddress>,
1109) -> Result<(), CommandEncoderError> {
1110    if src_buffer.is_equal(dst_buffer) {
1111        return Err(TransferError::SameSourceDestinationBuffer.into());
1112    }
1113
1114    src_buffer.same_device(state.device)?;
1115
1116    let src_pending = state
1117        .tracker
1118        .buffers
1119        .set_single(src_buffer, wgt::BufferUses::COPY_SRC);
1120
1121    let src_raw = src_buffer.try_raw(state.snatch_guard)?;
1122    src_buffer
1123        .check_usage(BufferUsages::COPY_SRC)
1124        .map_err(TransferError::MissingBufferUsage)?;
1125    // expecting only a single barrier
1126    let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer, state.snatch_guard));
1127
1128    dst_buffer.same_device(state.device)?;
1129
1130    let dst_pending = state
1131        .tracker
1132        .buffers
1133        .set_single(dst_buffer, wgt::BufferUses::COPY_DST);
1134
1135    let dst_raw = dst_buffer.try_raw(state.snatch_guard)?;
1136    dst_buffer
1137        .check_usage(BufferUsages::COPY_DST)
1138        .map_err(TransferError::MissingBufferUsage)?;
1139    let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer, state.snatch_guard));
1140
1141    if source_offset > src_buffer.size {
1142        return Err(TransferError::BufferStartOffsetOverrun {
1143            start_offset: source_offset,
1144            buffer_size: src_buffer.size,
1145            side: CopySide::Source,
1146        }
1147        .into());
1148    }
1149    let size = size.unwrap_or_else(|| {
1150        // NOTE: Should never underflow because of our earlier check.
1151        src_buffer.size - source_offset
1152    });
1153
1154    if !size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1155        return Err(TransferError::UnalignedCopySize(size).into());
1156    }
1157    if !source_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1158        return Err(TransferError::UnalignedBufferOffset(source_offset).into());
1159    }
1160    if !destination_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1161        return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
1162    }
1163    if !state
1164        .device
1165        .downlevel
1166        .flags
1167        .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
1168        && (src_buffer.usage.contains(BufferUsages::INDEX)
1169            || dst_buffer.usage.contains(BufferUsages::INDEX))
1170    {
1171        let forbidden_usages = BufferUsages::VERTEX
1172            | BufferUsages::UNIFORM
1173            | BufferUsages::INDIRECT
1174            | BufferUsages::STORAGE;
1175        if src_buffer.usage.intersects(forbidden_usages)
1176            || dst_buffer.usage.intersects(forbidden_usages)
1177        {
1178            return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
1179                wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
1180            ))
1181            .into());
1182        }
1183    }
1184
1185    if size > src_buffer.size - source_offset {
1186        return Err(TransferError::BufferEndOffsetOverrun {
1187            start_offset: source_offset,
1188            size,
1189            buffer_size: src_buffer.size,
1190            side: CopySide::Source,
1191        }
1192        .into());
1193    }
1194    // NOTE: Should never overflow because of our earlier check.
1195    let source_end_offset = source_offset + size;
1196
1197    if destination_offset > dst_buffer.size {
1198        return Err(TransferError::BufferStartOffsetOverrun {
1199            start_offset: destination_offset,
1200            buffer_size: dst_buffer.size,
1201            side: CopySide::Destination,
1202        }
1203        .into());
1204    }
1205    // NOTE: Should never underflow because of our earlier check.
1206    if size > dst_buffer.size - destination_offset {
1207        return Err(TransferError::BufferEndOffsetOverrun {
1208            start_offset: destination_offset,
1209            size,
1210            buffer_size: dst_buffer.size,
1211            side: CopySide::Destination,
1212        }
1213        .into());
1214    }
1215    // NOTE: Should never overflow because of our earlier check.
1216    let destination_end_offset = destination_offset + size;
1217
1218    // This must happen after parameter validation (so that errors are reported
1219    // as required by the spec), but before any side effects.
1220    if size == 0 {
1221        log::trace!("Ignoring copy_buffer_to_buffer of size 0");
1222        return Ok(());
1223    }
1224
1225    // Make sure source is initialized memory and mark dest as initialized.
1226    state
1227        .buffer_memory_init_actions
1228        .extend(dst_buffer.initialization_status.read().create_action(
1229            dst_buffer,
1230            destination_offset..destination_end_offset,
1231            MemoryInitKind::ImplicitlyInitialized,
1232        ));
1233    state
1234        .buffer_memory_init_actions
1235        .extend(src_buffer.initialization_status.read().create_action(
1236            src_buffer,
1237            source_offset..source_end_offset,
1238            MemoryInitKind::NeedsInitializedMemory,
1239        ));
1240
1241    let region = hal::BufferCopy {
1242        src_offset: source_offset,
1243        dst_offset: destination_offset,
1244        size: wgt::BufferSize::new(size).unwrap(),
1245    };
1246    let barriers = src_barrier
1247        .into_iter()
1248        .chain(dst_barrier)
1249        .collect::<Vec<_>>();
1250    unsafe {
1251        state.raw_encoder.transition_buffers(&barriers);
1252        state
1253            .raw_encoder
1254            .copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
1255    }
1256
1257    Ok(())
1258}
1259
1260pub(super) fn copy_buffer_to_texture(
1261    state: &mut EncodingState,
1262    source: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1263    destination: &wgt::TexelCopyTextureInfo<Arc<Texture>>,
1264    copy_size: &Extent3d,
1265) -> Result<(), CommandEncoderError> {
1266    let dst_texture = &destination.texture;
1267    let src_buffer = &source.buffer;
1268
1269    dst_texture.same_device(state.device)?;
1270    src_buffer.same_device(state.device)?;
1271
1272    let CopyRange {
1273        hal_copy_extent,
1274        hal_base: dst_base,
1275        array_layer_count,
1276        selector: dst_range,
1277    } = validate_texture_copy_range(
1278        destination,
1279        &dst_texture.desc,
1280        CopySide::Destination,
1281        copy_size,
1282    )?;
1283
1284    let src_raw = src_buffer.try_raw(state.snatch_guard)?;
1285    src_buffer
1286        .check_usage(BufferUsages::COPY_SRC)
1287        .map_err(TransferError::MissingBufferUsage)?;
1288
1289    let dst_raw = dst_texture.try_inner(state.snatch_guard)?.raw();
1290    dst_texture
1291        .check_usage(TextureUsages::COPY_DST)
1292        .map_err(TransferError::MissingTextureUsage)?;
1293
1294    validate_texture_copy_dst_format(dst_texture.desc.format, destination.aspect)?;
1295
1296    validate_texture_buffer_copy(
1297        destination,
1298        dst_base.aspect,
1299        &dst_texture.desc,
1300        &source.layout,
1301        true, // alignment required for buffer offset
1302        state.device.features,
1303    )?;
1304
1305    let (required_buffer_bytes_in_copy, bytes_per_array_layer, is_contiguous) =
1306        validate_linear_texture_data(
1307            &source.layout,
1308            dst_texture.desc.format,
1309            destination.aspect,
1310            src_buffer.size,
1311            CopySide::Source,
1312            copy_size,
1313        )?;
1314
1315    if dst_texture.desc.format.is_depth_stencil_format() {
1316        state
1317            .device
1318            .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1319            .map_err(TransferError::from)?;
1320    }
1321
1322    // This must happen after parameter validation (so that errors are reported
1323    // as required by the spec), but before any side effects.
1324    if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1325        log::trace!("Ignoring copy_buffer_to_texture of size 0");
1326        return Ok(());
1327    }
1328
1329    // Handle texture init *before* dealing with barrier transitions so we
1330    // have an easier time inserting "immediate-inits" that may be required
1331    // by prior discards in rare cases.
1332    handle_dst_texture_init(state, destination, copy_size, dst_texture)?;
1333
1334    let src_pending = state
1335        .tracker
1336        .buffers
1337        .set_single(src_buffer, wgt::BufferUses::COPY_SRC);
1338    let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer, state.snatch_guard));
1339
1340    let dst_pending =
1341        state
1342            .tracker
1343            .textures
1344            .set_single(dst_texture, dst_range, wgt::TextureUses::COPY_DST);
1345    let dst_barrier = dst_pending
1346        .map(|pending| pending.into_hal(dst_raw))
1347        .collect::<Vec<_>>();
1348
1349    handle_buffer_init(
1350        state,
1351        source,
1352        CopySide::Source,
1353        required_buffer_bytes_in_copy,
1354        is_contiguous,
1355    );
1356
1357    let regions = (0..array_layer_count)
1358        .map(|rel_array_layer| {
1359            let mut texture_base = dst_base.clone();
1360            texture_base.array_layer += rel_array_layer;
1361            let mut buffer_layout = source.layout;
1362            buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1363            hal::BufferTextureCopy {
1364                buffer_layout,
1365                texture_base,
1366                size: hal_copy_extent,
1367            }
1368        })
1369        .collect::<Vec<_>>();
1370
1371    unsafe {
1372        state.raw_encoder.transition_textures(&dst_barrier);
1373        state.raw_encoder.transition_buffers(src_barrier.as_slice());
1374        state
1375            .raw_encoder
1376            .copy_buffer_to_texture(src_raw, dst_raw, &regions);
1377    }
1378
1379    Ok(())
1380}
1381
1382pub(super) fn copy_texture_to_buffer(
1383    state: &mut EncodingState,
1384    source: &TexelCopyTextureInfo,
1385    destination: &wgt::TexelCopyBufferInfo<Arc<Buffer>>,
1386    copy_size: &Extent3d,
1387) -> Result<(), CommandEncoderError> {
1388    let src_texture = &source.texture;
1389    let dst_buffer = &destination.buffer;
1390
1391    src_texture.same_device(state.device)?;
1392    dst_buffer.same_device(state.device)?;
1393
1394    let CopyRange {
1395        hal_copy_extent,
1396        hal_base: src_base,
1397        array_layer_count,
1398        selector: src_range,
1399    } = validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1400
1401    let src_raw = src_texture.try_inner(state.snatch_guard)?.raw();
1402    src_texture
1403        .check_usage(TextureUsages::COPY_SRC)
1404        .map_err(TransferError::MissingTextureUsage)?;
1405
1406    if source.mip_level >= src_texture.desc.mip_level_count {
1407        return Err(TransferError::InvalidMipLevel {
1408            requested: source.mip_level,
1409            count: src_texture.desc.mip_level_count,
1410        }
1411        .into());
1412    }
1413
1414    validate_texture_copy_src_format(src_texture.desc.format, source.aspect)?;
1415
1416    validate_texture_buffer_copy(
1417        source,
1418        src_base.aspect,
1419        &src_texture.desc,
1420        &destination.layout,
1421        true, // alignment required for buffer offset
1422        state.device.features,
1423    )?;
1424
1425    let (required_buffer_bytes_in_copy, bytes_per_array_layer, is_contiguous) =
1426        validate_linear_texture_data(
1427            &destination.layout,
1428            src_texture.desc.format,
1429            source.aspect,
1430            dst_buffer.size,
1431            CopySide::Destination,
1432            copy_size,
1433        )?;
1434
1435    if src_texture.desc.format.is_depth_stencil_format() {
1436        state
1437            .device
1438            .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1439            .map_err(TransferError::from)?;
1440    }
1441
1442    let dst_raw = dst_buffer.try_raw(state.snatch_guard)?;
1443    dst_buffer
1444        .check_usage(BufferUsages::COPY_DST)
1445        .map_err(TransferError::MissingBufferUsage)?;
1446
1447    // This must happen after parameter validation (so that errors are reported
1448    // as required by the spec), but before any side effects.
1449    if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1450        log::trace!("Ignoring copy_texture_to_buffer of size 0");
1451        return Ok(());
1452    }
1453
1454    // Handle texture init *before* dealing with barrier transitions so we
1455    // have an easier time inserting "immediate-inits" that may be required
1456    // by prior discards in rare cases.
1457    handle_src_texture_init(state, source, copy_size, src_texture)?;
1458
1459    let src_pending =
1460        state
1461            .tracker
1462            .textures
1463            .set_single(src_texture, src_range, wgt::TextureUses::COPY_SRC);
1464    let src_barrier = src_pending
1465        .map(|pending| pending.into_hal(src_raw))
1466        .collect::<Vec<_>>();
1467
1468    let dst_pending = state
1469        .tracker
1470        .buffers
1471        .set_single(dst_buffer, wgt::BufferUses::COPY_DST);
1472
1473    let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer, state.snatch_guard));
1474
1475    handle_buffer_init(
1476        state,
1477        destination,
1478        CopySide::Destination,
1479        required_buffer_bytes_in_copy,
1480        is_contiguous,
1481    );
1482
1483    let regions = (0..array_layer_count)
1484        .map(|rel_array_layer| {
1485            let mut texture_base = src_base.clone();
1486            texture_base.array_layer += rel_array_layer;
1487            let mut buffer_layout = destination.layout;
1488            buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1489            hal::BufferTextureCopy {
1490                buffer_layout,
1491                texture_base,
1492                size: hal_copy_extent,
1493            }
1494        })
1495        .collect::<Vec<_>>();
1496    unsafe {
1497        state.raw_encoder.transition_buffers(dst_barrier.as_slice());
1498        state.raw_encoder.transition_textures(&src_barrier);
1499        state.raw_encoder.copy_texture_to_buffer(
1500            src_raw,
1501            wgt::TextureUses::COPY_SRC,
1502            dst_raw,
1503            &regions,
1504        );
1505    }
1506
1507    Ok(())
1508}
1509
1510pub(super) fn copy_texture_to_texture(
1511    state: &mut EncodingState,
1512    source: &TexelCopyTextureInfo,
1513    destination: &TexelCopyTextureInfo,
1514    copy_size: &Extent3d,
1515) -> Result<(), CommandEncoderError> {
1516    let src_texture = &source.texture;
1517    let dst_texture = &destination.texture;
1518
1519    src_texture.same_device(state.device)?;
1520    dst_texture.same_device(state.device)?;
1521
1522    let CopyRange {
1523        hal_copy_extent: src_copy_size,
1524        hal_base: src_tex_base,
1525        array_layer_count,
1526        selector: src_range,
1527    } = validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1528    let CopyRange {
1529        hal_copy_extent: dst_copy_size,
1530        hal_base: dst_tex_base,
1531        array_layer_count: _,
1532        selector: dst_range,
1533    } = validate_texture_copy_range(
1534        destination,
1535        &dst_texture.desc,
1536        CopySide::Destination,
1537        copy_size,
1538    )?;
1539
1540    // src and dst texture format must be [copy-compatible]. For multi-planar formats (which
1541    // are a wgpu extension), a copy is either between a single plane and a format matching
1542    // that plane, or between two `TextureAspect::All` copies of the same multi-planar
1543    // format.
1544    //
1545    // Multi-planar formats don't have sRGB variants, so we never have to both remove
1546    // an sRGB suffix and call `aspect_specific_format` to look up a plane format.
1547    //
1548    // `validate_texture_copy_range` rejected aspects that the format does not have, so
1549    // `aspect_specific_format` could return `None` here only for multi-planar textures with
1550    // `TextureAspect::All`, and in that case we avoid calling it.
1551    //
1552    // [copy-compatible]: https://gpuweb.github.io/gpuweb/#copy-compatible
1553    let copy_format =
1554        |format: wgt::TextureFormat, aspect: wgt::TextureAspect| match aspect.to_plane() {
1555            Some(_) => {
1556                // Single-plane copy to/from multi-planar texture.
1557                let plane_fmt = format
1558                    .aspect_specific_format(aspect)
1559                    .expect("aspect was validated previously");
1560                (plane_fmt, plane_fmt)
1561            }
1562            None => {
1563                // Copy between ordinary single-plane textures, or all planes of
1564                // multi-planar textures.
1565                (format, format.remove_srgb_suffix())
1566            }
1567        };
1568    let (src_format, src_format_no_srgb) = copy_format(src_texture.desc.format, source.aspect);
1569    let (dst_format, dst_format_no_srgb) = copy_format(dst_texture.desc.format, destination.aspect);
1570    if src_format_no_srgb != dst_format_no_srgb {
1571        // We report original source and destination formats, not the versions with an sRGB
1572        // suffix removed. For single-plane copies to/from multi-planar textures,
1573        // `{src,dst}_format` is the plane format, and `{src,dst}_multi_planar` is used
1574        // by the error variant to add annotations like `in plane 0 of NV12`.
1575        return Err(TransferError::TextureFormatsNotCopyCompatible {
1576            src_format,
1577            src_multi_planar: source
1578                .aspect
1579                .to_plane()
1580                .map(|plane| (src_texture.desc.format, plane)),
1581            dst_format,
1582            dst_multi_planar: destination
1583                .aspect
1584                .to_plane()
1585                .map(|plane| (dst_texture.desc.format, plane)),
1586        }
1587        .into());
1588    }
1589
1590    if Arc::as_ptr(src_texture) == Arc::as_ptr(dst_texture) {
1591        validate_copy_within_same_texture(
1592            source,
1593            destination,
1594            src_texture.desc.format,
1595            array_layer_count,
1596        )?;
1597    }
1598
1599    let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1600    let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1601    if src_texture.desc.format.is_depth_stencil_format()
1602        && src_tex_base.aspect != src_texture_aspects
1603    {
1604        return Err(TransferError::CopySrcMissingAspects.into());
1605    }
1606    if dst_texture.desc.format.is_depth_stencil_format()
1607        && dst_tex_base.aspect != dst_texture_aspects
1608    {
1609        return Err(TransferError::CopyDstMissingAspects.into());
1610    }
1611
1612    if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1613        return Err(TransferError::SampleCountNotEqual {
1614            src_sample_count: src_texture.desc.sample_count,
1615            dst_sample_count: dst_texture.desc.sample_count,
1616        }
1617        .into());
1618    }
1619
1620    let src_raw = src_texture.try_inner(state.snatch_guard)?.raw();
1621    src_texture
1622        .check_usage(TextureUsages::COPY_SRC)
1623        .map_err(TransferError::MissingTextureUsage)?;
1624    let dst_raw = dst_texture.try_inner(state.snatch_guard)?.raw();
1625    dst_texture
1626        .check_usage(TextureUsages::COPY_DST)
1627        .map_err(TransferError::MissingTextureUsage)?;
1628
1629    // This must happen after parameter validation (so that errors are reported
1630    // as required by the spec), but before any side effects.
1631    if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1632        log::trace!("Ignoring copy_texture_to_texture of size 0");
1633        return Ok(());
1634    }
1635
1636    // Handle texture init *before* dealing with barrier transitions so we
1637    // have an easier time inserting "immediate-inits" that may be required
1638    // by prior discards in rare cases.
1639    handle_src_texture_init(state, source, copy_size, src_texture)?;
1640    handle_dst_texture_init(state, destination, copy_size, dst_texture)?;
1641
1642    let src_pending =
1643        state
1644            .tracker
1645            .textures
1646            .set_single(src_texture, src_range, wgt::TextureUses::COPY_SRC);
1647
1648    //TODO: try to avoid this the collection. It's needed because both
1649    // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1650    let mut barriers: ArrayVec<_, 2> = src_pending
1651        .map(|pending| pending.into_hal(src_raw))
1652        .collect();
1653
1654    let dst_pending =
1655        state
1656            .tracker
1657            .textures
1658            .set_single(dst_texture, dst_range, wgt::TextureUses::COPY_DST);
1659    barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1660
1661    let hal_copy_size = hal::CopyExtent {
1662        width: src_copy_size.width.min(dst_copy_size.width),
1663        height: src_copy_size.height.min(dst_copy_size.height),
1664        depth: src_copy_size.depth.min(dst_copy_size.depth),
1665    };
1666
1667    let dst_format = dst_texture.desc.format;
1668
1669    let regions = (0..array_layer_count).map(|rel_array_layer| {
1670        let mut src_base = src_tex_base.clone();
1671        let mut dst_base = dst_tex_base.clone();
1672        src_base.array_layer += rel_array_layer;
1673        dst_base.array_layer += rel_array_layer;
1674        hal::TextureCopy {
1675            src_base,
1676            dst_base,
1677            size: hal_copy_size,
1678        }
1679    });
1680
1681    let regions = if dst_tex_base.aspect == hal::FormatAspects::DEPTH_STENCIL {
1682        regions
1683            .flat_map(|region| {
1684                let (mut depth, mut stencil) = (region.clone(), region);
1685                depth.src_base.aspect = hal::FormatAspects::DEPTH;
1686                depth.dst_base.aspect = hal::FormatAspects::DEPTH;
1687                stencil.src_base.aspect = hal::FormatAspects::STENCIL;
1688                stencil.dst_base.aspect = hal::FormatAspects::STENCIL;
1689                [depth, stencil]
1690            })
1691            .collect::<Vec<_>>()
1692    } else if let (Some(plane_count), wgt::TextureAspect::All) =
1693        (dst_format.planes(), destination.aspect)
1694    {
1695        regions
1696            .into_iter()
1697            .flat_map(|region| {
1698                (0..plane_count).map(move |plane| {
1699                    let mut plane_region = region.clone();
1700
1701                    let plane_aspect = wgt::TextureAspect::from_plane(plane)
1702                        .expect("expected texture aspect to exist for the plane");
1703                    let plane_aspect = hal::FormatAspects::new(dst_format, plane_aspect);
1704                    plane_region.src_base.aspect = plane_aspect;
1705                    plane_region.dst_base.aspect = plane_aspect;
1706
1707                    let (w_subsampling, h_subsampling) =
1708                        dst_format.subsampling_factors(Some(plane));
1709                    plane_region.src_base.origin.x /= w_subsampling;
1710                    plane_region.src_base.origin.y /= h_subsampling;
1711                    plane_region.dst_base.origin.x /= w_subsampling;
1712                    plane_region.dst_base.origin.y /= h_subsampling;
1713
1714                    plane_region.size.width /= w_subsampling;
1715                    plane_region.size.height /= h_subsampling;
1716
1717                    plane_region
1718                })
1719            })
1720            .collect::<Vec<_>>()
1721    } else {
1722        regions.collect::<Vec<_>>()
1723    };
1724    unsafe {
1725        state.raw_encoder.transition_textures(&barriers);
1726        state.raw_encoder.copy_texture_to_texture(
1727            src_raw,
1728            wgt::TextureUses::COPY_SRC,
1729            dst_raw,
1730            &regions,
1731        );
1732    }
1733
1734    Ok(())
1735}