wgpu_core/command/
transfer.rs

Help
1use alloc::{format, string::String, sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6    error::{ErrorType, WebGpuError},
7    BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8    TextureUsages,
9};
10
11#[cfg(feature = "trace")]
12use crate::device::trace::Command as TraceCommand;
13use crate::{
14    api_log,
15    command::{clear_texture, CommandEncoderError, EncoderStateError},
16    conv,
17    device::{Device, MissingDownlevelFlags},
18    global::Global,
19    id::{BufferId, CommandEncoderId, TextureId},
20    init_tracker::{
21        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
22        TextureInitTrackerAction,
23    },
24    resource::{
25        MissingBufferUsageError, MissingTextureUsageError, ParentDevice, RawResourceAccess,
26        Texture, TextureErrorDimension,
27    },
28    snatch::SnatchGuard,
29};
30
31use super::{ClearError, CommandBufferMutable};
32
33pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo<BufferId>;
34pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<TextureId>;
35pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo<TextureId>;
36
37#[derive(Clone, Copy, Debug)]
38pub enum CopySide {
39    Source,
40    Destination,
41}
42
43/// Error encountered while attempting a data transfer.
44#[derive(Clone, Debug, Error)]
45#[non_exhaustive]
46pub enum TransferError {
47    #[error("Source and destination cannot be the same buffer")]
48    SameSourceDestinationBuffer,
49    #[error(transparent)]
50    MissingBufferUsage(#[from] MissingBufferUsageError),
51    #[error(transparent)]
52    MissingTextureUsage(#[from] MissingTextureUsageError),
53    #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
54    BufferOverrun {
55        start_offset: BufferAddress,
56        end_offset: BufferAddress,
57        buffer_size: BufferAddress,
58        side: CopySide,
59    },
60    #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
61    TextureOverrun {
62        start_offset: u32,
63        end_offset: u32,
64        texture_size: u32,
65        dimension: TextureErrorDimension,
66        side: CopySide,
67    },
68    #[error("Partial copy of {start_offset}..{end_offset} on {dimension:?} dimension with size {texture_size} \
69             is not supported for the {side:?} texture format {format:?} with {sample_count} samples")]
70    UnsupportedPartialTransfer {
71        format: wgt::TextureFormat,
72        sample_count: u32,
73        start_offset: u32,
74        end_offset: u32,
75        texture_size: u32,
76        dimension: TextureErrorDimension,
77        side: CopySide,
78    },
79    #[error(
80        "Copying{} layers {}..{} to{} layers {}..{} of the same texture is not allowed",
81        if *src_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {src_aspects:?}") },
82        src_origin_z,
83        src_origin_z + array_layer_count,
84        if *dst_aspects == wgt::TextureAspect::All { String::new() } else { format!(" {dst_aspects:?}") },
85        dst_origin_z,
86        dst_origin_z + array_layer_count,
87    )]
88    InvalidCopyWithinSameTexture {
89        src_aspects: wgt::TextureAspect,
90        dst_aspects: wgt::TextureAspect,
91        src_origin_z: u32,
92        dst_origin_z: u32,
93        array_layer_count: u32,
94    },
95    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
96    InvalidTextureAspect {
97        format: wgt::TextureFormat,
98        aspect: wgt::TextureAspect,
99    },
100    #[error("Unable to select texture mip level {level} out of {total}")]
101    InvalidTextureMipLevel { level: u32, total: u32 },
102    #[error("Texture dimension must be 2D when copying from an external texture")]
103    InvalidDimensionExternal,
104    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
105    UnalignedBufferOffset(BufferAddress),
106    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
107    UnalignedCopySize(BufferAddress),
108    #[error("Copy width is not a multiple of block width")]
109    UnalignedCopyWidth,
110    #[error("Copy height is not a multiple of block height")]
111    UnalignedCopyHeight,
112    #[error("Copy origin's x component is not a multiple of block width")]
113    UnalignedCopyOriginX,
114    #[error("Copy origin's y component is not a multiple of block height")]
115    UnalignedCopyOriginY,
116    #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
117    UnalignedBytesPerRow,
118    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
119    UnspecifiedBytesPerRow,
120    #[error("Number of rows per image needs to be specified since more than one image is copied")]
121    UnspecifiedRowsPerImage,
122    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
123    InvalidBytesPerRow,
124    #[error("Number of rows per image is invalid")]
125    InvalidRowsPerImage,
126    #[error("Overflow while computing the size of the copy")]
127    SizeOverflow,
128    #[error("Copy source aspects must refer to all aspects of the source texture format")]
129    CopySrcMissingAspects,
130    #[error(
131        "Copy destination aspects must refer to all aspects of the destination texture format"
132    )]
133    CopyDstMissingAspects,
134    #[error("Copy aspect must refer to a single aspect of texture format")]
135    CopyAspectNotOne,
136    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
137    CopyFromForbiddenTextureFormat {
138        format: wgt::TextureFormat,
139        aspect: wgt::TextureAspect,
140    },
141    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
142    CopyToForbiddenTextureFormat {
143        format: wgt::TextureFormat,
144        aspect: wgt::TextureAspect,
145    },
146    #[error(
147        "Copying to textures with format {0:?} is forbidden when copying from external texture"
148    )]
149    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
150    #[error(
151        "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
152    )]
153    TextureFormatsNotCopyCompatible {
154        src_format: wgt::TextureFormat,
155        dst_format: wgt::TextureFormat,
156    },
157    #[error(transparent)]
158    MemoryInitFailure(#[from] ClearError),
159    #[error("Cannot encode this copy because of a missing downelevel flag")]
160    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
161    #[error("Source texture sample count must be 1, got {sample_count}")]
162    InvalidSampleCount { sample_count: u32 },
163    #[error(
164        "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
165    )]
166    SampleCountNotEqual {
167        src_sample_count: u32,
168        dst_sample_count: u32,
169    },
170    #[error("Requested mip level {requested} does no exist (count: {count})")]
171    InvalidMipLevel { requested: u32, count: u32 },
172}
173
174impl WebGpuError for TransferError {
175    fn webgpu_error_type(&self) -> ErrorType {
176        let e: &dyn WebGpuError = match self {
177            Self::MissingBufferUsage(e) => e,
178            Self::MissingTextureUsage(e) => e,
179            Self::MemoryInitFailure(e) => e,
180
181            Self::BufferOverrun { .. }
182            | Self::TextureOverrun { .. }
183            | Self::UnsupportedPartialTransfer { .. }
184            | Self::InvalidCopyWithinSameTexture { .. }
185            | Self::InvalidTextureAspect { .. }
186            | Self::InvalidTextureMipLevel { .. }
187            | Self::InvalidDimensionExternal
188            | Self::UnalignedBufferOffset(..)
189            | Self::UnalignedCopySize(..)
190            | Self::UnalignedCopyWidth
191            | Self::UnalignedCopyHeight
192            | Self::UnalignedCopyOriginX
193            | Self::UnalignedCopyOriginY
194            | Self::UnalignedBytesPerRow
195            | Self::UnspecifiedBytesPerRow
196            | Self::UnspecifiedRowsPerImage
197            | Self::InvalidBytesPerRow
198            | Self::InvalidRowsPerImage
199            | Self::SizeOverflow
200            | Self::CopySrcMissingAspects
201            | Self::CopyDstMissingAspects
202            | Self::CopyAspectNotOne
203            | Self::CopyFromForbiddenTextureFormat { .. }
204            | Self::CopyToForbiddenTextureFormat { .. }
205            | Self::ExternalCopyToForbiddenTextureFormat(..)
206            | Self::TextureFormatsNotCopyCompatible { .. }
207            | Self::MissingDownlevelFlags(..)
208            | Self::InvalidSampleCount { .. }
209            | Self::SampleCountNotEqual { .. }
210            | Self::InvalidMipLevel { .. }
211            | Self::SameSourceDestinationBuffer => return ErrorType::Validation,
212        };
213        e.webgpu_error_type()
214    }
215}
216
217impl From<BufferTextureCopyInfoError> for TransferError {
218    fn from(value: BufferTextureCopyInfoError) -> Self {
219        match value {
220            BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
221            BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
222            BufferTextureCopyInfoError::ImageStrideOverflow
223            | BufferTextureCopyInfoError::ImageBytesOverflow(_)
224            | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
225        }
226    }
227}
228
229pub(crate) fn extract_texture_selector<T>(
230    copy_texture: &wgt::TexelCopyTextureInfo<T>,
231    copy_size: &Extent3d,
232    texture: &Texture,
233) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
234    let format = texture.desc.format;
235    let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
236    if copy_aspect.is_empty() {
237        return Err(TransferError::InvalidTextureAspect {
238            format,
239            aspect: copy_texture.aspect,
240        });
241    }
242
243    let (layers, origin_z) = match texture.desc.dimension {
244        wgt::TextureDimension::D1 => (0..1, 0),
245        wgt::TextureDimension::D2 => (
246            copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
247            0,
248        ),
249        wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
250    };
251    let base = hal::TextureCopyBase {
252        origin: wgt::Origin3d {
253            x: copy_texture.origin.x,
254            y: copy_texture.origin.y,
255            z: origin_z,
256        },
257        // this value will be incremented per copied layer
258        array_layer: layers.start,
259        mip_level: copy_texture.mip_level,
260        aspect: copy_aspect,
261    };
262    let selector = TextureSelector {
263        mips: copy_texture.mip_level..copy_texture.mip_level + 1,
264        layers,
265    };
266
267    Ok((selector, base))
268}
269
270/// WebGPU's [validating linear texture data][vltd] algorithm.
271///
272/// Copied with some modifications from WebGPU standard.
273///
274/// If successful, returns a pair `(bytes, stride)`, where:
275/// - `bytes` is the number of buffer bytes required for this copy, and
276/// - `stride` number of bytes between array layers.
277///
278/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
279pub(crate) fn validate_linear_texture_data(
280    layout: &wgt::TexelCopyBufferLayout,
281    format: wgt::TextureFormat,
282    aspect: wgt::TextureAspect,
283    buffer_size: BufferAddress,
284    buffer_side: CopySide,
285    copy_size: &Extent3d,
286    need_copy_aligned_rows: bool,
287) -> Result<(BufferAddress, BufferAddress), TransferError> {
288    let wgt::BufferTextureCopyInfo {
289        copy_width,
290        copy_height,
291        depth_or_array_layers,
292
293        offset,
294
295        block_size_bytes,
296        block_width_texels,
297        block_height_texels,
298
299        width_blocks: _,
300        height_blocks,
301
302        row_bytes_dense: _,
303        row_stride_bytes,
304
305        image_stride_rows: _,
306        image_stride_bytes,
307
308        image_rows_dense: _,
309        image_bytes_dense: _,
310
311        bytes_in_copy,
312    } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
313
314    if copy_width % block_width_texels != 0 {
315        return Err(TransferError::UnalignedCopyWidth);
316    }
317    if copy_height % block_height_texels != 0 {
318        return Err(TransferError::UnalignedCopyHeight);
319    }
320
321    let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
322    let requires_multiple_images = depth_or_array_layers > 1;
323
324    // `get_buffer_texture_copy_info()` already proceeded with defaults if these
325    // were not specified, and ensured that the values satisfy the minima if
326    // they were, but now we enforce the WebGPU requirement that they be
327    // specified any time they apply.
328    if layout.bytes_per_row.is_none() && requires_multiple_rows {
329        return Err(TransferError::UnspecifiedBytesPerRow);
330    }
331
332    if layout.rows_per_image.is_none() && requires_multiple_images {
333        return Err(TransferError::UnspecifiedRowsPerImage);
334    };
335
336    if need_copy_aligned_rows {
337        let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
338
339        let mut offset_alignment = block_size_bytes;
340        if format.is_depth_stencil_format() {
341            offset_alignment = 4
342        }
343        if offset % offset_alignment != 0 {
344            return Err(TransferError::UnalignedBufferOffset(offset));
345        }
346
347        // The alignment of row_stride_bytes is only required if there are
348        // multiple rows
349        if requires_multiple_rows && row_stride_bytes % bytes_per_row_alignment != 0 {
350            return Err(TransferError::UnalignedBytesPerRow);
351        }
352    }
353
354    // Avoid underflow in the subtraction by checking bytes_in_copy against buffer_size first.
355    if bytes_in_copy > buffer_size || offset > buffer_size - bytes_in_copy {
356        return Err(TransferError::BufferOverrun {
357            start_offset: offset,
358            end_offset: offset.wrapping_add(bytes_in_copy),
359            buffer_size,
360            side: buffer_side,
361        });
362    }
363
364    Ok((bytes_in_copy, image_stride_bytes))
365}
366
367/// Validation for texture/buffer copies.
368///
369/// This implements the following checks from WebGPU's [validating texture buffer copy][vtbc]
370/// algorithm:
371///  * The texture must not be multisampled.
372///  * The copy must be from/to a single aspect of the texture.
373///  * If `aligned` is true, the buffer offset must be aligned appropriately.
374///
375/// The following steps in the algorithm are implemented elsewhere:
376///  * Invocation of other validation algorithms.
377///  * The texture usage (COPY_DST / COPY_SRC) check.
378///  * The check for non-copyable depth/stencil formats. The caller must perform
379///    this check using `is_valid_copy_src_format` / `is_valid_copy_dst_format`
380///    before calling this function. This function will panic if
381///    [`wgt::TextureFormat::block_copy_size`] returns `None` due to a
382///    non-copyable format.
383///
384/// [vtbc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-buffer-copy
385pub(crate) fn validate_texture_buffer_copy<T>(
386    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
387    aspect: hal::FormatAspects,
388    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
389    offset: BufferAddress,
390    aligned: bool,
391) -> Result<(), TransferError> {
392    if desc.sample_count != 1 {
393        return Err(TransferError::InvalidSampleCount {
394            sample_count: desc.sample_count,
395        });
396    }
397
398    if !aspect.is_one() {
399        return Err(TransferError::CopyAspectNotOne);
400    }
401
402    let offset_alignment = if desc.format.is_depth_stencil_format() {
403        4
404    } else {
405        // The case where `block_copy_size` returns `None` is currently
406        // unreachable both for the reason in the expect message, and also
407        // because the currently-defined non-copyable formats are depth/stencil
408        // formats so would take the `if` branch.
409        desc.format
410            .block_copy_size(Some(texture_copy_view.aspect))
411            .expect("non-copyable formats should have been rejected previously")
412    };
413
414    if aligned && offset % u64::from(offset_alignment) != 0 {
415        return Err(TransferError::UnalignedBufferOffset(offset));
416    }
417
418    Ok(())
419}
420
421/// Validate the extent and alignment of a texture copy.
422///
423/// Copied with minor modifications from WebGPU standard. This mostly follows
424/// the [validating GPUTexelCopyTextureInfo][vtcti] and [validating texture copy
425/// range][vtcr] algorithms.
426///
427/// Returns the HAL copy extent and the layer count.
428///
429/// [vtcti]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gputexelcopytextureinfo
430/// [vtcr]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-texture-copy-range
431pub(crate) fn validate_texture_copy_range<T>(
432    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
433    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
434    texture_side: CopySide,
435    copy_size: &Extent3d,
436) -> Result<(hal::CopyExtent, u32), TransferError> {
437    let (block_width, block_height) = desc.format.block_dimensions();
438
439    let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
440        TransferError::InvalidTextureMipLevel {
441            level: texture_copy_view.mip_level,
442            total: desc.mip_level_count,
443        },
444    )?;
445    // physical size can be larger than the virtual
446    let extent = extent_virtual.physical_size(desc.format);
447
448    // Multisampled and depth-stencil formats do not support partial copies
449    // on x and y dimensions, but do support copying a subset of layers.
450    let requires_exact_size = desc.format.is_depth_stencil_format() || desc.sample_count > 1;
451
452    // Return `Ok` if a run `size` texels long starting at `start_offset` is
453    // valid for `texture_size`. Otherwise, return an appropriate a`Err`.
454    let check_dimension = |dimension: TextureErrorDimension,
455                           start_offset: u32,
456                           size: u32,
457                           texture_size: u32,
458                           requires_exact_size: bool|
459     -> Result<(), TransferError> {
460        if requires_exact_size && (start_offset != 0 || size != texture_size) {
461            Err(TransferError::UnsupportedPartialTransfer {
462                format: desc.format,
463                sample_count: desc.sample_count,
464                start_offset,
465                end_offset: start_offset.wrapping_add(size),
466                texture_size,
467                dimension,
468                side: texture_side,
469            })
470        // Avoid underflow in the subtraction by checking start_offset against
471        // texture_size first.
472        } else if start_offset > texture_size || texture_size - start_offset < size {
473            Err(TransferError::TextureOverrun {
474                start_offset,
475                end_offset: start_offset.wrapping_add(size),
476                texture_size,
477                dimension,
478                side: texture_side,
479            })
480        } else {
481            Ok(())
482        }
483    };
484
485    check_dimension(
486        TextureErrorDimension::X,
487        texture_copy_view.origin.x,
488        copy_size.width,
489        extent.width,
490        requires_exact_size,
491    )?;
492    check_dimension(
493        TextureErrorDimension::Y,
494        texture_copy_view.origin.y,
495        copy_size.height,
496        extent.height,
497        requires_exact_size,
498    )?;
499    check_dimension(
500        TextureErrorDimension::Z,
501        texture_copy_view.origin.z,
502        copy_size.depth_or_array_layers,
503        extent.depth_or_array_layers,
504        false, // partial copy always allowed on Z/layer dimension
505    )?;
506
507    if texture_copy_view.origin.x % block_width != 0 {
508        return Err(TransferError::UnalignedCopyOriginX);
509    }
510    if texture_copy_view.origin.y % block_height != 0 {
511        return Err(TransferError::UnalignedCopyOriginY);
512    }
513    if copy_size.width % block_width != 0 {
514        return Err(TransferError::UnalignedCopyWidth);
515    }
516    if copy_size.height % block_height != 0 {
517        return Err(TransferError::UnalignedCopyHeight);
518    }
519
520    let (depth, array_layer_count) = match desc.dimension {
521        wgt::TextureDimension::D1 => (1, 1),
522        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
523        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
524    };
525
526    let copy_extent = hal::CopyExtent {
527        width: copy_size.width,
528        height: copy_size.height,
529        depth,
530    };
531    Ok((copy_extent, array_layer_count))
532}
533
534/// Validate a copy within the same texture.
535///
536/// This implements the WebGPU requirement that the [sets of subresources for
537/// texture copy][srtc] of the source and destination be disjoint, i.e. that the
538/// source and destination do not overlap.
539///
540/// This function assumes that the copy ranges have already been validated with
541/// `validate_texture_copy_range`.
542///
543/// [srtc]: https://gpuweb.github.io/gpuweb/#abstract-opdef-set-of-subresources-for-texture-copy
544pub(crate) fn validate_copy_within_same_texture<T>(
545    src: &wgt::TexelCopyTextureInfo<T>,
546    dst: &wgt::TexelCopyTextureInfo<T>,
547    format: wgt::TextureFormat,
548    array_layer_count: u32,
549) -> Result<(), TransferError> {
550    let src_aspects = hal::FormatAspects::new(format, src.aspect);
551    let dst_aspects = hal::FormatAspects::new(format, dst.aspect);
552    if (src_aspects & dst_aspects).is_empty() {
553        // Copying between different aspects (if it even makes sense), is okay.
554        return Ok(());
555    }
556
557    if src.origin.z >= dst.origin.z + array_layer_count
558        || dst.origin.z >= src.origin.z + array_layer_count
559    {
560        // Copying between non-overlapping layer ranges is okay.
561        return Ok(());
562    }
563
564    if src.mip_level != dst.mip_level {
565        // Copying between different mip levels is okay.
566        return Ok(());
567    }
568
569    Err(TransferError::InvalidCopyWithinSameTexture {
570        src_aspects: src.aspect,
571        dst_aspects: dst.aspect,
572        src_origin_z: src.origin.z,
573        dst_origin_z: dst.origin.z,
574        array_layer_count,
575    })
576}
577
578fn handle_texture_init(
579    init_kind: MemoryInitKind,
580    cmd_buf_data: &mut CommandBufferMutable,
581    device: &Device,
582    copy_texture: &TexelCopyTextureInfo,
583    copy_size: &Extent3d,
584    texture: &Arc<Texture>,
585    snatch_guard: &SnatchGuard<'_>,
586) -> Result<(), ClearError> {
587    let init_action = TextureInitTrackerAction {
588        texture: texture.clone(),
589        range: TextureInitRange {
590            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
591            layer_range: copy_texture.origin.z
592                ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
593        },
594        kind: init_kind,
595    };
596
597    // Register the init action.
598    let immediate_inits = cmd_buf_data
599        .texture_memory_actions
600        .register_init_action(&{ init_action });
601
602    // In rare cases we may need to insert an init operation immediately onto the command buffer.
603    if !immediate_inits.is_empty() {
604        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
605        for init in immediate_inits {
606            clear_texture(
607                &init.texture,
608                TextureInitRange {
609                    mip_range: init.mip_level..(init.mip_level + 1),
610                    layer_range: init.layer..(init.layer + 1),
611                },
612                cmd_buf_raw,
613                &mut cmd_buf_data.trackers.textures,
614                &device.alignments,
615                device.zero_buffer.as_ref(),
616                snatch_guard,
617                device.instance_flags,
618            )?;
619        }
620    }
621
622    Ok(())
623}
624
625/// Prepare a transfer's source texture.
626///
627/// Ensure the source texture of a transfer is in the right initialization
628/// state, and record the state for after the transfer operation.
629fn handle_src_texture_init(
630    cmd_buf_data: &mut CommandBufferMutable,
631    device: &Device,
632    source: &TexelCopyTextureInfo,
633    copy_size: &Extent3d,
634    texture: &Arc<Texture>,
635    snatch_guard: &SnatchGuard<'_>,
636) -> Result<(), TransferError> {
637    handle_texture_init(
638        MemoryInitKind::NeedsInitializedMemory,
639        cmd_buf_data,
640        device,
641        source,
642        copy_size,
643        texture,
644        snatch_guard,
645    )?;
646    Ok(())
647}
648
649/// Prepare a transfer's destination texture.
650///
651/// Ensure the destination texture of a transfer is in the right initialization
652/// state, and record the state for after the transfer operation.
653fn handle_dst_texture_init(
654    cmd_buf_data: &mut CommandBufferMutable,
655    device: &Device,
656    destination: &TexelCopyTextureInfo,
657    copy_size: &Extent3d,
658    texture: &Arc<Texture>,
659    snatch_guard: &SnatchGuard<'_>,
660) -> Result<(), TransferError> {
661    // Attention: If we don't write full texture subresources, we need to a full
662    // clear first since we don't track subrects. This means that in rare cases
663    // even a *destination* texture of a transfer may need an immediate texture
664    // init.
665    let dst_init_kind = if has_copy_partial_init_tracker_coverage(
666        copy_size,
667        destination.mip_level,
668        &texture.desc,
669    ) {
670        MemoryInitKind::NeedsInitializedMemory
671    } else {
672        MemoryInitKind::ImplicitlyInitialized
673    };
674
675    handle_texture_init(
676        dst_init_kind,
677        cmd_buf_data,
678        device,
679        destination,
680        copy_size,
681        texture,
682        snatch_guard,
683    )?;
684    Ok(())
685}
686
687impl Global {
688    pub fn command_encoder_copy_buffer_to_buffer(
689        &self,
690        command_encoder_id: CommandEncoderId,
691        source: BufferId,
692        source_offset: BufferAddress,
693        destination: BufferId,
694        destination_offset: BufferAddress,
695        size: Option<BufferAddress>,
696    ) -> Result<(), EncoderStateError> {
697        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
698        api_log!(
699            "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
700        );
701
702        let hub = &self.hub;
703
704        let cmd_enc = hub.command_encoders.get(command_encoder_id);
705        let mut cmd_buf_data = cmd_enc.data.lock();
706        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
707            let device = &cmd_enc.device;
708            device.check_is_valid()?;
709
710            if source == destination {
711                return Err(TransferError::SameSourceDestinationBuffer.into());
712            }
713
714            #[cfg(feature = "trace")]
715            if let Some(ref mut list) = cmd_buf_data.commands {
716                list.push(TraceCommand::CopyBufferToBuffer {
717                    src: source,
718                    src_offset: source_offset,
719                    dst: destination,
720                    dst_offset: destination_offset,
721                    size,
722                });
723            }
724
725            let snatch_guard = device.snatchable_lock.read();
726
727            let src_buffer = hub.buffers.get(source).get()?;
728
729            src_buffer.same_device_as(cmd_enc.as_ref())?;
730
731            let src_pending = cmd_buf_data
732                .trackers
733                .buffers
734                .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
735
736            let src_raw = src_buffer.try_raw(&snatch_guard)?;
737            src_buffer
738                .check_usage(BufferUsages::COPY_SRC)
739                .map_err(TransferError::MissingBufferUsage)?;
740            // expecting only a single barrier
741            let src_barrier =
742                src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
743
744            let dst_buffer = hub.buffers.get(destination).get()?;
745
746            dst_buffer.same_device_as(cmd_enc.as_ref())?;
747
748            let dst_pending = cmd_buf_data
749                .trackers
750                .buffers
751                .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
752
753            let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
754            dst_buffer
755                .check_usage(BufferUsages::COPY_DST)
756                .map_err(TransferError::MissingBufferUsage)?;
757            let dst_barrier =
758                dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
759
760            let (size, source_end_offset) = match size {
761                Some(size) => (size, source_offset + size),
762                None => (src_buffer.size - source_offset, src_buffer.size),
763            };
764
765            if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
766                return Err(TransferError::UnalignedCopySize(size).into());
767            }
768            if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
769                return Err(TransferError::UnalignedBufferOffset(source_offset).into());
770            }
771            if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
772                return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
773            }
774            if !device
775                .downlevel
776                .flags
777                .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
778                && (src_buffer.usage.contains(BufferUsages::INDEX)
779                    || dst_buffer.usage.contains(BufferUsages::INDEX))
780            {
781                let forbidden_usages = BufferUsages::VERTEX
782                    | BufferUsages::UNIFORM
783                    | BufferUsages::INDIRECT
784                    | BufferUsages::STORAGE;
785                if src_buffer.usage.intersects(forbidden_usages)
786                    || dst_buffer.usage.intersects(forbidden_usages)
787                {
788                    return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
789                        wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
790                    ))
791                    .into());
792                }
793            }
794
795            let destination_end_offset = destination_offset + size;
796            if source_end_offset > src_buffer.size {
797                return Err(TransferError::BufferOverrun {
798                    start_offset: source_offset,
799                    end_offset: source_end_offset,
800                    buffer_size: src_buffer.size,
801                    side: CopySide::Source,
802                }
803                .into());
804            }
805            if destination_end_offset > dst_buffer.size {
806                return Err(TransferError::BufferOverrun {
807                    start_offset: destination_offset,
808                    end_offset: destination_end_offset,
809                    buffer_size: dst_buffer.size,
810                    side: CopySide::Destination,
811                }
812                .into());
813            }
814
815            if size == 0 {
816                log::trace!("Ignoring copy_buffer_to_buffer of size 0");
817                return Ok(());
818            }
819
820            // Make sure source is initialized memory and mark dest as initialized.
821            cmd_buf_data.buffer_memory_init_actions.extend(
822                dst_buffer.initialization_status.read().create_action(
823                    &dst_buffer,
824                    destination_offset..(destination_offset + size),
825                    MemoryInitKind::ImplicitlyInitialized,
826                ),
827            );
828            cmd_buf_data.buffer_memory_init_actions.extend(
829                src_buffer.initialization_status.read().create_action(
830                    &src_buffer,
831                    source_offset..(source_offset + size),
832                    MemoryInitKind::NeedsInitializedMemory,
833                ),
834            );
835
836            let region = hal::BufferCopy {
837                src_offset: source_offset,
838                dst_offset: destination_offset,
839                size: wgt::BufferSize::new(size).unwrap(),
840            };
841            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
842            let barriers = src_barrier
843                .into_iter()
844                .chain(dst_barrier)
845                .collect::<Vec<_>>();
846            unsafe {
847                cmd_buf_raw.transition_buffers(&barriers);
848                cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
849            }
850
851            Ok(())
852        })
853    }
854
855    pub fn command_encoder_copy_buffer_to_texture(
856        &self,
857        command_encoder_id: CommandEncoderId,
858        source: &TexelCopyBufferInfo,
859        destination: &TexelCopyTextureInfo,
860        copy_size: &Extent3d,
861    ) -> Result<(), EncoderStateError> {
862        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
863        api_log!(
864            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
865            source.buffer,
866            destination.texture
867        );
868
869        let hub = &self.hub;
870
871        let cmd_enc = hub.command_encoders.get(command_encoder_id);
872        let mut cmd_buf_data = cmd_enc.data.lock();
873        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
874            let device = &cmd_enc.device;
875            device.check_is_valid()?;
876
877            #[cfg(feature = "trace")]
878            if let Some(ref mut list) = cmd_buf_data.commands {
879                list.push(TraceCommand::CopyBufferToTexture {
880                    src: *source,
881                    dst: *destination,
882                    size: *copy_size,
883                });
884            }
885
886            let dst_texture = hub.textures.get(destination.texture).get()?;
887            let src_buffer = hub.buffers.get(source.buffer).get()?;
888
889            dst_texture.same_device_as(cmd_enc.as_ref())?;
890            src_buffer.same_device_as(cmd_enc.as_ref())?;
891
892            let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
893                destination,
894                &dst_texture.desc,
895                CopySide::Destination,
896                copy_size,
897            )?;
898
899            let (dst_range, dst_base) =
900                extract_texture_selector(destination, copy_size, &dst_texture)?;
901
902            let snatch_guard = device.snatchable_lock.read();
903
904            let src_raw = src_buffer.try_raw(&snatch_guard)?;
905            let dst_raw = dst_texture.try_raw(&snatch_guard)?;
906
907            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
908            {
909                log::trace!("Ignoring copy_buffer_to_texture of size 0");
910                return Ok(());
911            }
912
913            // Handle texture init *before* dealing with barrier transitions so we
914            // have an easier time inserting "immediate-inits" that may be required
915            // by prior discards in rare cases.
916            handle_dst_texture_init(
917                cmd_buf_data,
918                device,
919                destination,
920                copy_size,
921                &dst_texture,
922                &snatch_guard,
923            )?;
924
925            let src_pending = cmd_buf_data
926                .trackers
927                .buffers
928                .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
929
930            src_buffer
931                .check_usage(BufferUsages::COPY_SRC)
932                .map_err(TransferError::MissingBufferUsage)?;
933            let src_barrier =
934                src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
935
936            let dst_pending = cmd_buf_data.trackers.textures.set_single(
937                &dst_texture,
938                dst_range,
939                wgt::TextureUses::COPY_DST,
940            );
941            dst_texture
942                .check_usage(TextureUsages::COPY_DST)
943                .map_err(TransferError::MissingTextureUsage)?;
944            let dst_barrier = dst_pending
945                .map(|pending| pending.into_hal(dst_raw))
946                .collect::<Vec<_>>();
947
948            if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect)
949            {
950                return Err(TransferError::CopyToForbiddenTextureFormat {
951                    format: dst_texture.desc.format,
952                    aspect: destination.aspect,
953                }
954                .into());
955            }
956
957            validate_texture_buffer_copy(
958                destination,
959                dst_base.aspect,
960                &dst_texture.desc,
961                source.layout.offset,
962                true, // alignment required for buffer offset
963            )?;
964
965            let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
966                validate_linear_texture_data(
967                    &source.layout,
968                    dst_texture.desc.format,
969                    destination.aspect,
970                    src_buffer.size,
971                    CopySide::Source,
972                    copy_size,
973                    true,
974                )?;
975
976            if dst_texture.desc.format.is_depth_stencil_format() {
977                device
978                    .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
979                    .map_err(TransferError::from)?;
980            }
981
982            cmd_buf_data.buffer_memory_init_actions.extend(
983                src_buffer.initialization_status.read().create_action(
984                    &src_buffer,
985                    source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
986                    MemoryInitKind::NeedsInitializedMemory,
987                ),
988            );
989
990            let regions = (0..array_layer_count)
991                .map(|rel_array_layer| {
992                    let mut texture_base = dst_base.clone();
993                    texture_base.array_layer += rel_array_layer;
994                    let mut buffer_layout = source.layout;
995                    buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
996                    hal::BufferTextureCopy {
997                        buffer_layout,
998                        texture_base,
999                        size: hal_copy_size,
1000                    }
1001                })
1002                .collect::<Vec<_>>();
1003
1004            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1005            unsafe {
1006                cmd_buf_raw.transition_textures(&dst_barrier);
1007                cmd_buf_raw.transition_buffers(src_barrier.as_slice());
1008                cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, &regions);
1009            }
1010
1011            Ok(())
1012        })
1013    }
1014
1015    pub fn command_encoder_copy_texture_to_buffer(
1016        &self,
1017        command_encoder_id: CommandEncoderId,
1018        source: &TexelCopyTextureInfo,
1019        destination: &TexelCopyBufferInfo,
1020        copy_size: &Extent3d,
1021    ) -> Result<(), EncoderStateError> {
1022        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
1023        api_log!(
1024            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
1025            source.texture,
1026            destination.buffer
1027        );
1028
1029        let hub = &self.hub;
1030
1031        let cmd_enc = hub.command_encoders.get(command_encoder_id);
1032        let mut cmd_buf_data = cmd_enc.data.lock();
1033        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1034            let device = &cmd_enc.device;
1035            device.check_is_valid()?;
1036
1037            #[cfg(feature = "trace")]
1038            if let Some(list) = cmd_buf_data.commands.as_mut() {
1039                list.push(TraceCommand::CopyTextureToBuffer {
1040                    src: *source,
1041                    dst: *destination,
1042                    size: *copy_size,
1043                });
1044            }
1045
1046            let src_texture = hub.textures.get(source.texture).get()?;
1047            let dst_buffer = hub.buffers.get(destination.buffer).get()?;
1048
1049            src_texture.same_device_as(cmd_enc.as_ref())?;
1050            dst_buffer.same_device_as(cmd_enc.as_ref())?;
1051
1052            let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
1053                source,
1054                &src_texture.desc,
1055                CopySide::Source,
1056                copy_size,
1057            )?;
1058
1059            let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1060
1061            let snatch_guard = device.snatchable_lock.read();
1062
1063            let src_raw = src_texture.try_raw(&snatch_guard)?;
1064            src_texture
1065                .check_usage(TextureUsages::COPY_SRC)
1066                .map_err(TransferError::MissingTextureUsage)?;
1067
1068            if source.mip_level >= src_texture.desc.mip_level_count {
1069                return Err(TransferError::InvalidMipLevel {
1070                    requested: source.mip_level,
1071                    count: src_texture.desc.mip_level_count,
1072                }
1073                .into());
1074            }
1075
1076            if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) {
1077                return Err(TransferError::CopyFromForbiddenTextureFormat {
1078                    format: src_texture.desc.format,
1079                    aspect: source.aspect,
1080                }
1081                .into());
1082            }
1083
1084            validate_texture_buffer_copy(
1085                source,
1086                src_base.aspect,
1087                &src_texture.desc,
1088                destination.layout.offset,
1089                true, // alignment required for buffer offset
1090            )?;
1091
1092            let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
1093                validate_linear_texture_data(
1094                    &destination.layout,
1095                    src_texture.desc.format,
1096                    source.aspect,
1097                    dst_buffer.size,
1098                    CopySide::Destination,
1099                    copy_size,
1100                    true,
1101                )?;
1102
1103            if src_texture.desc.format.is_depth_stencil_format() {
1104                device
1105                    .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1106                    .map_err(TransferError::from)?;
1107            }
1108
1109            let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
1110            dst_buffer
1111                .check_usage(BufferUsages::COPY_DST)
1112                .map_err(TransferError::MissingBufferUsage)?;
1113
1114            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1115            {
1116                log::trace!("Ignoring copy_texture_to_buffer of size 0");
1117                return Ok(());
1118            }
1119
1120            // Handle texture init *before* dealing with barrier transitions so we
1121            // have an easier time inserting "immediate-inits" that may be required
1122            // by prior discards in rare cases.
1123            handle_src_texture_init(
1124                cmd_buf_data,
1125                device,
1126                source,
1127                copy_size,
1128                &src_texture,
1129                &snatch_guard,
1130            )?;
1131
1132            let src_pending = cmd_buf_data.trackers.textures.set_single(
1133                &src_texture,
1134                src_range,
1135                wgt::TextureUses::COPY_SRC,
1136            );
1137            let src_barrier = src_pending
1138                .map(|pending| pending.into_hal(src_raw))
1139                .collect::<Vec<_>>();
1140
1141            let dst_pending = cmd_buf_data
1142                .trackers
1143                .buffers
1144                .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
1145
1146            let dst_barrier =
1147                dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
1148
1149            cmd_buf_data.buffer_memory_init_actions.extend(
1150                dst_buffer.initialization_status.read().create_action(
1151                    &dst_buffer,
1152                    destination.layout.offset
1153                        ..(destination.layout.offset + required_buffer_bytes_in_copy),
1154                    MemoryInitKind::ImplicitlyInitialized,
1155                ),
1156            );
1157
1158            let regions = (0..array_layer_count)
1159                .map(|rel_array_layer| {
1160                    let mut texture_base = src_base.clone();
1161                    texture_base.array_layer += rel_array_layer;
1162                    let mut buffer_layout = destination.layout;
1163                    buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1164                    hal::BufferTextureCopy {
1165                        buffer_layout,
1166                        texture_base,
1167                        size: hal_copy_size,
1168                    }
1169                })
1170                .collect::<Vec<_>>();
1171            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1172            unsafe {
1173                cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1174                cmd_buf_raw.transition_textures(&src_barrier);
1175                cmd_buf_raw.copy_texture_to_buffer(
1176                    src_raw,
1177                    wgt::TextureUses::COPY_SRC,
1178                    dst_raw,
1179                    &regions,
1180                );
1181            }
1182
1183            Ok(())
1184        })
1185    }
1186
1187    pub fn command_encoder_copy_texture_to_texture(
1188        &self,
1189        command_encoder_id: CommandEncoderId,
1190        source: &TexelCopyTextureInfo,
1191        destination: &TexelCopyTextureInfo,
1192        copy_size: &Extent3d,
1193    ) -> Result<(), EncoderStateError> {
1194        profiling::scope!("CommandEncoder::copy_texture_to_texture");
1195        api_log!(
1196            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1197            source.texture,
1198            destination.texture
1199        );
1200
1201        let hub = &self.hub;
1202
1203        let cmd_enc = hub.command_encoders.get(command_encoder_id);
1204        let mut cmd_buf_data = cmd_enc.data.lock();
1205        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1206            let device = &cmd_enc.device;
1207            device.check_is_valid()?;
1208
1209            let snatch_guard = device.snatchable_lock.read();
1210
1211            #[cfg(feature = "trace")]
1212            if let Some(ref mut list) = cmd_buf_data.commands {
1213                list.push(TraceCommand::CopyTextureToTexture {
1214                    src: *source,
1215                    dst: *destination,
1216                    size: *copy_size,
1217                });
1218            }
1219
1220            let src_texture = hub.textures.get(source.texture).get()?;
1221            let dst_texture = hub.textures.get(destination.texture).get()?;
1222
1223            src_texture.same_device_as(cmd_enc.as_ref())?;
1224            dst_texture.same_device_as(cmd_enc.as_ref())?;
1225
1226            // src and dst texture format must be copy-compatible
1227            // https://gpuweb.github.io/gpuweb/#copy-compatible
1228            if src_texture.desc.format.remove_srgb_suffix()
1229                != dst_texture.desc.format.remove_srgb_suffix()
1230            {
1231                return Err(TransferError::TextureFormatsNotCopyCompatible {
1232                    src_format: src_texture.desc.format,
1233                    dst_format: dst_texture.desc.format,
1234                }
1235                .into());
1236            }
1237
1238            let (src_copy_size, array_layer_count) = validate_texture_copy_range(
1239                source,
1240                &src_texture.desc,
1241                CopySide::Source,
1242                copy_size,
1243            )?;
1244            let (dst_copy_size, _) = validate_texture_copy_range(
1245                destination,
1246                &dst_texture.desc,
1247                CopySide::Destination,
1248                copy_size,
1249            )?;
1250
1251            if Arc::as_ptr(&src_texture) == Arc::as_ptr(&dst_texture) {
1252                validate_copy_within_same_texture(
1253                    source,
1254                    destination,
1255                    src_texture.desc.format,
1256                    array_layer_count,
1257                )?;
1258            }
1259
1260            let (src_range, src_tex_base) =
1261                extract_texture_selector(source, copy_size, &src_texture)?;
1262            let (dst_range, dst_tex_base) =
1263                extract_texture_selector(destination, copy_size, &dst_texture)?;
1264            let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1265            let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1266            if src_tex_base.aspect != src_texture_aspects {
1267                return Err(TransferError::CopySrcMissingAspects.into());
1268            }
1269            if dst_tex_base.aspect != dst_texture_aspects {
1270                return Err(TransferError::CopyDstMissingAspects.into());
1271            }
1272
1273            if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1274                return Err(TransferError::SampleCountNotEqual {
1275                    src_sample_count: src_texture.desc.sample_count,
1276                    dst_sample_count: dst_texture.desc.sample_count,
1277                }
1278                .into());
1279            }
1280
1281            // Handle texture init *before* dealing with barrier transitions so we
1282            // have an easier time inserting "immediate-inits" that may be required
1283            // by prior discards in rare cases.
1284            handle_src_texture_init(
1285                cmd_buf_data,
1286                device,
1287                source,
1288                copy_size,
1289                &src_texture,
1290                &snatch_guard,
1291            )?;
1292            handle_dst_texture_init(
1293                cmd_buf_data,
1294                device,
1295                destination,
1296                copy_size,
1297                &dst_texture,
1298                &snatch_guard,
1299            )?;
1300
1301            let src_raw = src_texture.try_raw(&snatch_guard)?;
1302            src_texture
1303                .check_usage(TextureUsages::COPY_SRC)
1304                .map_err(TransferError::MissingTextureUsage)?;
1305            let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1306            dst_texture
1307                .check_usage(TextureUsages::COPY_DST)
1308                .map_err(TransferError::MissingTextureUsage)?;
1309
1310            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1311            {
1312                log::trace!("Ignoring copy_texture_to_texture of size 0");
1313                return Ok(());
1314            }
1315
1316            let src_pending = cmd_buf_data.trackers.textures.set_single(
1317                &src_texture,
1318                src_range,
1319                wgt::TextureUses::COPY_SRC,
1320            );
1321
1322            //TODO: try to avoid this the collection. It's needed because both
1323            // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1324            let mut barriers: ArrayVec<_, 2> = src_pending
1325                .map(|pending| pending.into_hal(src_raw))
1326                .collect();
1327
1328            let dst_pending = cmd_buf_data.trackers.textures.set_single(
1329                &dst_texture,
1330                dst_range,
1331                wgt::TextureUses::COPY_DST,
1332            );
1333            barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1334
1335            let hal_copy_size = hal::CopyExtent {
1336                width: src_copy_size.width.min(dst_copy_size.width),
1337                height: src_copy_size.height.min(dst_copy_size.height),
1338                depth: src_copy_size.depth.min(dst_copy_size.depth),
1339            };
1340
1341            let regions = (0..array_layer_count).map(|rel_array_layer| {
1342                let mut src_base = src_tex_base.clone();
1343                let mut dst_base = dst_tex_base.clone();
1344                src_base.array_layer += rel_array_layer;
1345                dst_base.array_layer += rel_array_layer;
1346                hal::TextureCopy {
1347                    src_base,
1348                    dst_base,
1349                    size: hal_copy_size,
1350                }
1351            });
1352
1353            let regions = if dst_tex_base.aspect == hal::FormatAspects::DEPTH_STENCIL {
1354                regions
1355                    .flat_map(|region| {
1356                        let (mut depth, mut stencil) = (region.clone(), region);
1357                        depth.src_base.aspect = hal::FormatAspects::DEPTH;
1358                        depth.dst_base.aspect = hal::FormatAspects::DEPTH;
1359                        stencil.src_base.aspect = hal::FormatAspects::STENCIL;
1360                        stencil.dst_base.aspect = hal::FormatAspects::STENCIL;
1361                        [depth, stencil]
1362                    })
1363                    .collect::<Vec<_>>()
1364            } else {
1365                regions.collect::<Vec<_>>()
1366            };
1367            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1368            unsafe {
1369                cmd_buf_raw.transition_textures(&barriers);
1370                cmd_buf_raw.copy_texture_to_texture(
1371                    src_raw,
1372                    wgt::TextureUses::COPY_SRC,
1373                    dst_raw,
1374                    &regions,
1375                );
1376            }
1377
1378            Ok(())
1379        })
1380    }
1381}
wgpu_core/command/transfer.rs

wgpu_core/command/
transfer.rs
