wgpu/util/belt.rs
1use crate::{
2 util::align_to, Buffer, BufferAddress, BufferDescriptor, BufferSize, BufferSlice, BufferUsages,
3 BufferViewMut, CommandEncoder, Device, MapMode,
4};
5use alloc::sync::Arc;
6use alloc::vec::Vec;
7use concurrent_queue::ConcurrentQueue;
8use core::fmt;
9use wgt::Features;
10
11use crate::COPY_BUFFER_ALIGNMENT;
12
13#[cfg(send_sync)]
14static_assertions::assert_impl_all!(StagingBelt: Send, Sync);
15
16/// Efficiently performs many buffer writes by sharing and reusing temporary buffers.
17///
18/// Internally it uses a ring-buffer of staging buffers that are sub-allocated.
19/// Its advantage over [`Queue::write_buffer_with()`] is that the individual allocations
20/// are cheaper; `StagingBelt` is most useful when you are writing very many small pieces
21/// of data. It can be understood as a sort of arena allocator.
22///
23/// Using a staging belt is slightly complicated, and generally goes as follows:
24/// 1. Use [`StagingBelt::write_buffer()`] or [`StagingBelt::allocate()`] to allocate
25/// buffer slices, then write your data to them.
26/// 2. Call [`StagingBelt::finish()`].
27/// 3. Submit all command encoders that were used in step 1.
28/// 4. Call [`StagingBelt::recall()`].
29///
30/// Alternatively, steps 2 and 4 can be combined into a single call to
31/// [`StagingBelt::finish_and_recall_on_submit()`], which schedules the re-map
32/// automatically when the encoder is submitted, so no explicit `recall()` is needed.
33///
34/// [`Queue::write_buffer_with()`]: crate::Queue::write_buffer_with
35pub struct StagingBelt {
36 device: Device,
37 chunk_size: BufferAddress,
38 /// User-specified [`BufferUsages`] used to create the chunk buffers are created.
39 ///
40 /// [`new`](Self::new) guarantees that this always contains
41 /// [`MAP_WRITE`](BufferUsages::MAP_WRITE).
42 buffer_usages: BufferUsages,
43 /// Chunks into which we are accumulating data to be transferred.
44 active_chunks: Vec<Chunk>,
45 /// Chunks that have scheduled transfers already; they are unmapped and some
46 /// command encoder has one or more commands with them as source.
47 closed_chunks: Vec<Chunk>,
48 /// Chunks that are back from the GPU and ready to be mapped for write and put
49 /// into `active_chunks`.
50 free_chunks: Vec<Chunk>,
51 /// When closed chunks are mapped again, the map callback sends them here.
52 /// Free chunks are retrieved from this to be put on `self.free_chunks`.
53 queue: Arc<ConcurrentQueue<Chunk>>,
54 /// Whether the belt has grown since last recall.
55 grew_since_recall: bool,
56}
57
58impl StagingBelt {
59 /// Create a new staging belt.
60 ///
61 /// The `chunk_size` is the unit of internal buffer allocation; writes will be
62 /// sub-allocated within each chunk. Therefore, for optimal use of memory, the
63 /// chunk size should be:
64 ///
65 /// * larger than the largest single [`StagingBelt::write_buffer()`] operation;
66 /// * 1-4 times less than the total amount of data uploaded per submission
67 /// (per [`StagingBelt::finish()`]); and
68 /// * bigger is better, within these bounds.
69 ///
70 /// The buffers returned by this [`StagingBelt`] will be have the buffer usages
71 /// [`COPY_SRC | MAP_WRITE`](crate::BufferUsages)
72 pub fn new(device: Device, chunk_size: BufferAddress) -> Self {
73 Self::new_with_buffer_usages(device, chunk_size, BufferUsages::COPY_SRC)
74 }
75
76 /// Create a new staging belt.
77 ///
78 /// The `chunk_size` is the unit of internal buffer allocation; writes will be
79 /// sub-allocated within each chunk. Therefore, for optimal use of memory, the
80 /// chunk size should be:
81 ///
82 /// * larger than the largest single [`StagingBelt::write_buffer()`] operation;
83 /// * 1-4 times less than the total amount of data uploaded per submission
84 /// (per [`StagingBelt::finish()`]); and
85 /// * bigger is better, within these bounds.
86 ///
87 /// `buffer_usages` specifies the [`BufferUsages`] the staging buffers
88 /// will be created with. [`MAP_WRITE`](BufferUsages::MAP_WRITE) will be added
89 /// automatically. The method will panic if the combination of usages is not
90 /// supported. Because [`MAP_WRITE`](BufferUsages::MAP_WRITE) is implied, the allowed usages
91 /// depends on if [`Features::MAPPABLE_PRIMARY_BUFFERS`] is enabled.
92 /// - If enabled: any usage is valid.
93 /// - If disabled: only [`COPY_SRC`](BufferUsages::COPY_SRC) can be used.
94 #[track_caller]
95 pub fn new_with_buffer_usages(
96 device: Device,
97 chunk_size: BufferAddress,
98 mut buffer_usages: BufferUsages,
99 ) -> Self {
100 // make sure anything other than MAP_WRITE | COPY_SRC is only allowed with MAPPABLE_PRIMARY_BUFFERS.
101 let extra_usages =
102 buffer_usages.difference(BufferUsages::MAP_WRITE | BufferUsages::COPY_SRC);
103 if !extra_usages.is_empty()
104 && !device
105 .features()
106 .contains(Features::MAPPABLE_PRIMARY_BUFFERS)
107 {
108 panic!("Only BufferUsages::COPY_SRC may be used when Features::MAPPABLE_PRIMARY_BUFFERS is not enabled. Specified buffer usages: {buffer_usages:?}");
109 }
110 // always set MAP_WRITE
111 buffer_usages.insert(BufferUsages::MAP_WRITE);
112
113 StagingBelt {
114 device,
115 chunk_size,
116 buffer_usages,
117 active_chunks: Vec::new(),
118 closed_chunks: Vec::new(),
119 free_chunks: Vec::new(),
120 queue: Arc::new(ConcurrentQueue::unbounded()),
121 grew_since_recall: false,
122 }
123 }
124
125 /// Allocate a staging belt slice of `size` to be copied into the `target` buffer
126 /// at the specified offset.
127 ///
128 /// `offset` and `size` must be multiples of [`COPY_BUFFER_ALIGNMENT`]
129 /// (as is required by the underlying buffer operations).
130 ///
131 /// The upload will be placed into the provided command encoder. This encoder
132 /// must be submitted after [`StagingBelt::finish()`] is called and before
133 /// [`StagingBelt::recall()`] is called.
134 ///
135 /// If the `size` is greater than the size of any free internal buffer, a new buffer
136 /// will be allocated for it. Therefore, the `chunk_size` passed to [`StagingBelt::new()`]
137 /// should ideally be larger than every such size.
138 #[track_caller]
139 pub fn write_buffer(
140 &mut self,
141 encoder: &mut CommandEncoder,
142 target: &Buffer,
143 offset: BufferAddress,
144 size: BufferSize,
145 ) -> BufferViewMut {
146 // Asserting this explicitly gives a usefully more specific, and more prompt, error than
147 // leaving it to regular API validation.
148 // We check only `offset`, not `size`, because `self.allocate()` will check the size.
149 assert!(
150 offset.is_multiple_of(COPY_BUFFER_ALIGNMENT),
151 "StagingBelt::write_buffer() offset {offset} must be a multiple of `COPY_BUFFER_ALIGNMENT`"
152 );
153
154 let slice_of_belt = self.allocate(
155 size,
156 const { BufferSize::new(crate::COPY_BUFFER_ALIGNMENT).unwrap() },
157 );
158 encoder.copy_buffer_to_buffer(
159 slice_of_belt.buffer(),
160 slice_of_belt.offset(),
161 target,
162 offset,
163 size.get(),
164 );
165 slice_of_belt
166 .get_mapped_range_mut()
167 .expect("Failed to get mapped range for staging belt buffer")
168 }
169
170 /// Allocate a staging belt slice with the given `size` and `alignment` and return it.
171 ///
172 /// `size` must be a multiple of [`COPY_BUFFER_ALIGNMENT`]
173 /// (as is required by the underlying buffer operations).
174 ///
175 /// To use this slice, call [`BufferSlice::get_mapped_range_mut()`] and write your data into
176 /// that [`BufferViewMut`].
177 /// (The view must be dropped before [`StagingBelt::finish()`] is called.)
178 ///
179 /// You can then record your own GPU commands to perform with the slice,
180 /// such as copying it to a texture (whereas
181 /// [`StagingBelt::write_buffer()`] can only write to other buffers).
182 /// All commands involving this slice must be submitted after
183 /// [`StagingBelt::finish()`] is called and before [`StagingBelt::recall()`] is called.
184 ///
185 /// If the `size` is greater than the space available in any free internal buffer, a new buffer
186 /// will be allocated for it. Therefore, the `chunk_size` passed to [`StagingBelt::new()`]
187 /// should ideally be larger than every such size.
188 ///
189 /// The chosen slice will be positioned within the buffer at a multiple of `alignment`,
190 /// which may be used to meet alignment requirements for the operation you wish to perform
191 /// with the slice. This does not necessarily affect the alignment of the [`BufferViewMut`].
192 #[track_caller]
193 pub fn allocate(&mut self, size: BufferSize, alignment: BufferSize) -> BufferSlice<'_> {
194 assert!(
195 size.get().is_multiple_of(COPY_BUFFER_ALIGNMENT),
196 "StagingBelt allocation size {size} must be a multiple of `COPY_BUFFER_ALIGNMENT`"
197 );
198 assert!(
199 alignment.is_power_of_two(),
200 "alignment must be a power of two, not {alignment}"
201 );
202 // At minimum, we must have alignment sufficient to map the buffer.
203 let alignment = alignment.get().max(crate::MAP_ALIGNMENT);
204
205 let mut chunk = if let Some(index) = self
206 .active_chunks
207 .iter()
208 .position(|chunk| chunk.can_allocate(size, alignment))
209 {
210 self.active_chunks.swap_remove(index)
211 } else {
212 self.receive_chunks(); // ensure self.free_chunks is up to date
213
214 if let Some(index) = self
215 .free_chunks
216 .iter()
217 .position(|chunk| chunk.can_allocate(size, alignment))
218 {
219 self.free_chunks.swap_remove(index)
220 } else {
221 self.grew_since_recall = true;
222 Chunk {
223 buffer: self.device.create_buffer(&BufferDescriptor {
224 label: Some("(wgpu internal) StagingBelt staging buffer"),
225 size: self.chunk_size.max(size.get()),
226 usage: self.buffer_usages,
227 mapped_at_creation: true,
228 }),
229 offset: 0,
230 used_since_recall: false,
231 }
232 }
233 };
234
235 let allocation_offset = chunk.allocate(size, alignment);
236 chunk.used_since_recall = true;
237
238 self.active_chunks.push(chunk);
239 let chunk = self.active_chunks.last().unwrap();
240
241 chunk
242 .buffer
243 .slice(allocation_offset..allocation_offset + size.get())
244 }
245
246 /// Prepare currently mapped buffers for use in a submission.
247 ///
248 /// This must be called before the command encoder(s) provided to
249 /// [`StagingBelt::write_buffer()`] are submitted.
250 ///
251 /// At this point, all the partially used staging buffers are closed (cannot be used for
252 /// further writes) until after [`StagingBelt::recall()`] is called *and* the GPU is done
253 /// copying the data from them.
254 pub fn finish(&mut self) {
255 for chunk in self.active_chunks.drain(..) {
256 chunk.buffer.unmap();
257 self.closed_chunks.push(chunk);
258 }
259 }
260
261 /// Recall all of the closed buffers back to be reused.
262 ///
263 /// This must only be called after the command encoder(s) provided to
264 /// [`StagingBelt::write_buffer()`] are submitted. Additional calls are harmless.
265 /// Not calling this as soon as possible may result in increased buffer memory usage.
266 pub fn recall(&mut self) {
267 self.receive_chunks();
268
269 for chunk in self.closed_chunks.drain(..) {
270 let queue = self.queue.clone();
271 chunk
272 .buffer
273 .clone()
274 .slice(..)
275 .map_async(MapMode::Write, move |_| {
276 let _ = queue.push(chunk);
277 });
278 }
279 }
280
281 /// Convenience for [`StagingBelt::finish()`] followed by a deferred
282 /// [`StagingBelt::recall()`] that runs automatically when `encoder`'s command
283 /// buffer is submitted.
284 ///
285 /// After calling this method, the staging belt's internal buffers will be
286 /// re-mapped for write once the submission completes, without requiring an
287 /// explicit call to [`StagingBelt::recall()`].
288 ///
289 /// Like [`StagingBelt::recall()`], this method does not block.
290 ///
291 /// # Important
292 ///
293 /// `encoder` must be finished (via [`CommandEncoder::finish()`]) and the
294 /// resulting [`CommandBuffer`] must be submitted to the [`Queue`] **before**
295 /// the next call that needs free staging-belt chunks. If the encoder is
296 /// never submitted, the belt's closed chunks will not be returned and the
297 /// belt will allocate new buffers indefinitely.
298 ///
299 /// [`CommandBuffer`]: crate::CommandBuffer
300 /// [`Queue`]: crate::Queue
301 pub fn finish_and_recall_on_submit(&mut self, encoder: &CommandEncoder) {
302 self.finish();
303 self.receive_chunks();
304
305 for chunk in self.closed_chunks.drain(..) {
306 let queue = self.queue.clone();
307 let buffer = chunk.buffer.clone();
308 encoder.map_buffer_on_submit(&buffer, MapMode::Write, .., move |_| {
309 let _ = queue.push(chunk);
310 });
311 }
312 }
313
314 /// Move all chunks that the GPU is done with (and are now mapped again)
315 /// from `self.queue` to `self.free_chunks`.
316 fn receive_chunks(&mut self) {
317 while let Ok(mut chunk) = self.queue.pop() {
318 chunk.offset = 0;
319 self.free_chunks.push(chunk);
320 }
321 }
322
323 /// Discard all free chunks that have not been used since the last
324 /// [`StagingBelt::recall()`] back to the device and unmap their buffers.
325 pub fn discard_free_chunks(&mut self) {
326 self.receive_chunks();
327 self.free_chunks.retain_mut(|chunk| {
328 if !self.grew_since_recall || chunk.used_since_recall {
329 chunk.used_since_recall = false;
330 true
331 } else {
332 chunk.buffer.unmap();
333 false
334 }
335 });
336 self.grew_since_recall = false;
337 }
338}
339
340impl fmt::Debug for StagingBelt {
341 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
342 let Self {
343 device,
344 chunk_size,
345 buffer_usages,
346 active_chunks,
347 closed_chunks,
348 free_chunks,
349 queue: _,
350 grew_since_recall: _,
351 } = self;
352 f.debug_struct("StagingBelt")
353 .field("device", device)
354 .field("chunk_size", chunk_size)
355 .field("buffer_usages", buffer_usages)
356 .field("active_chunks", &active_chunks.len())
357 .field("closed_chunks", &closed_chunks.len())
358 .field("free_chunks", &free_chunks.len())
359 .finish_non_exhaustive()
360 }
361}
362
363struct Chunk {
364 buffer: Buffer,
365 offset: BufferAddress,
366 used_since_recall: bool,
367}
368
369impl Chunk {
370 fn can_allocate(&self, size: BufferSize, alignment: BufferAddress) -> bool {
371 let alloc_start = align_to(self.offset, alignment);
372 let alloc_end = alloc_start + size.get();
373
374 alloc_end <= self.buffer.size()
375 }
376
377 fn allocate(&mut self, size: BufferSize, alignment: BufferAddress) -> BufferAddress {
378 let alloc_start = align_to(self.offset, alignment);
379 let alloc_end = alloc_start + size.get();
380
381 assert!(alloc_end <= self.buffer.size());
382 self.offset = alloc_end;
383 alloc_start
384 }
385}