1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
use crate::{
device::{
queue::{EncoderInFlight, SubmittedWorkDoneClosure, TempResource},
DeviceError,
},
resource::{Buffer, Texture, Trackable},
snatch::SnatchGuard,
SubmissionIndex,
};
use smallvec::SmallVec;
use crate::resource::{Blas, Tlas};
use std::sync::Arc;
use thiserror::Error;
/// A command submitted to the GPU for execution.
///
/// ## Keeping resources alive while the GPU is using them
///
/// [`wgpu_hal`] requires that, when a command is submitted to a queue, all the
/// resources it uses must remain alive until it has finished executing.
///
/// [`wgpu_hal`]: hal
/// [`ResourceInfo::submission_index`]: crate::resource::ResourceInfo
struct ActiveSubmission {
/// The index of the submission we track.
///
/// When `Device::fence`'s value is greater than or equal to this, our queue
/// submission has completed.
index: SubmissionIndex,
/// Temporary resources to be freed once this queue submission has completed.
temp_resources: Vec<TempResource>,
/// Buffers to be mapped once this submission has completed.
mapped: Vec<Arc<Buffer>>,
/// Command buffers used by this submission, and the encoder that owns them.
///
/// [`wgpu_hal::Queue::submit`] requires the submitted command buffers to
/// remain alive until the submission has completed execution. Command
/// encoders double as allocation pools for command buffers, so holding them
/// here and cleaning them up in [`LifetimeTracker::triage_submissions`]
/// satisfies that requirement.
///
/// Once this submission has completed, the command buffers are reset and
/// the command encoder is recycled.
///
/// [`wgpu_hal::Queue::submit`]: hal::Queue::submit
encoders: Vec<EncoderInFlight>,
/// List of queue "on_submitted_work_done" closures to be called once this
/// submission has completed.
work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>,
}
impl ActiveSubmission {
/// Returns true if this submission contains the given buffer.
///
/// This only uses constant-time operations.
pub fn contains_buffer(&self, buffer: &Buffer) -> bool {
for encoder in &self.encoders {
// The ownership location of buffers depends on where the command encoder
// came from. If it is the staging command encoder on the queue, it is
// in the pending buffer list. If it came from a user command encoder,
// it is in the tracker.
if encoder.trackers.buffers.contains(buffer) {
return true;
}
if encoder
.pending_buffers
.contains_key(&buffer.tracker_index())
{
return true;
}
}
false
}
/// Returns true if this submission contains the given texture.
///
/// This only uses constant-time operations.
pub fn contains_texture(&self, texture: &Texture) -> bool {
for encoder in &self.encoders {
// The ownership location of textures depends on where the command encoder
// came from. If it is the staging command encoder on the queue, it is
// in the pending buffer list. If it came from a user command encoder,
// it is in the tracker.
if encoder.trackers.textures.contains(texture) {
return true;
}
if encoder
.pending_textures
.contains_key(&texture.tracker_index())
{
return true;
}
}
false
}
pub fn contains_blas(&self, blas: &Blas) -> bool {
for encoder in &self.encoders {
// The ownership location of blas's depends on where the command encoder
// came from. If it is the staging command encoder on the queue, it is
// in the pending buffer list. If it came from a user command encoder,
// it is in the tracker.
if encoder.trackers.blas_s.contains(blas) {
return true;
}
if encoder.pending_blas_s.contains_key(&blas.tracker_index()) {
return true;
}
}
false
}
pub fn contains_tlas(&self, tlas: &Tlas) -> bool {
for encoder in &self.encoders {
// The ownership location of tlas's depends on where the command encoder
// came from. If it is the staging command encoder on the queue, it is
// in the pending buffer list. If it came from a user command encoder,
// it is in the tracker.
if encoder.trackers.tlas_s.contains(tlas) {
return true;
}
if encoder.pending_tlas_s.contains_key(&tlas.tracker_index()) {
return true;
}
}
false
}
}
#[derive(Clone, Debug, Error)]
#[non_exhaustive]
pub enum WaitIdleError {
#[error(transparent)]
Device(#[from] DeviceError),
#[error("Tried to wait using a submission index ({0}) that has not been returned by a successful submission (last successful submission: {1})")]
WrongSubmissionIndex(SubmissionIndex, SubmissionIndex),
}
/// Resource tracking for a device.
///
/// ## Host mapping buffers
///
/// A buffer cannot be mapped until all active queue submissions that use it
/// have completed. To that end:
///
/// - Each buffer's `ResourceInfo::submission_index` records the index of the
/// most recent queue submission that uses that buffer.
///
/// - When the device is polled, the following `LifetimeTracker` methods decide
/// what should happen next:
///
/// 1) `triage_submissions` moves entries in `self.active[i]` for completed
/// submissions to `self.ready_to_map`. At this point, both
/// `self.active` and `self.ready_to_map` are up to date with the given
/// submission index.
///
/// 2) `handle_mapping` drains `self.ready_to_map` and actually maps the
/// buffers, collecting a list of notification closures to call.
///
/// Only calling `Global::buffer_map_async` clones a new `Arc` for the
/// buffer. This new `Arc` is only dropped by `handle_mapping`.
pub(crate) struct LifetimeTracker {
/// Resources used by queue submissions still in flight. One entry per
/// submission, with older submissions appearing before younger.
///
/// Entries are added by `track_submission` and drained by
/// `LifetimeTracker::triage_submissions`. Lots of methods contribute data
/// to particular entries.
active: Vec<ActiveSubmission>,
/// Buffers the user has asked us to map, and which are not used by any
/// queue submission still in flight.
ready_to_map: Vec<Arc<Buffer>>,
/// Queue "on_submitted_work_done" closures that were initiated for while there is no
/// currently pending submissions. These cannot be immediately invoked as they
/// must happen _after_ all mapped buffer callbacks are mapped, so we defer them
/// here until the next time the device is maintained.
work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>,
}
impl LifetimeTracker {
pub fn new() -> Self {
Self {
active: Vec::new(),
ready_to_map: Vec::new(),
work_done_closures: SmallVec::new(),
}
}
/// Return true if there are no queue submissions still in flight.
pub fn queue_empty(&self) -> bool {
self.active.is_empty()
}
/// Start tracking resources associated with a new queue submission.
pub fn track_submission(
&mut self,
index: SubmissionIndex,
temp_resources: impl Iterator<Item = TempResource>,
encoders: Vec<EncoderInFlight>,
) {
self.active.push(ActiveSubmission {
index,
temp_resources: temp_resources.collect(),
mapped: Vec::new(),
encoders,
work_done_closures: SmallVec::new(),
});
}
pub(crate) fn map(&mut self, buffer: &Arc<Buffer>) -> Option<SubmissionIndex> {
// Determine which buffers are ready to map, and which must wait for the GPU.
let submission = self
.active
.iter_mut()
.rev()
.find(|a| a.contains_buffer(buffer));
let maybe_submission_index = submission.as_ref().map(|s| s.index);
submission
.map_or(&mut self.ready_to_map, |a| &mut a.mapped)
.push(buffer.clone());
maybe_submission_index
}
/// Returns the submission index of the most recent submission that uses the
/// given buffer.
pub fn get_buffer_latest_submission_index(&self, buffer: &Buffer) -> Option<SubmissionIndex> {
// We iterate in reverse order, so that we can bail out early as soon
// as we find a hit.
self.active.iter().rev().find_map(|submission| {
if submission.contains_buffer(buffer) {
Some(submission.index)
} else {
None
}
})
}
/// Returns the submission index of the most recent submission that uses the
/// given blas.
pub fn get_blas_latest_submission_index(&self, blas: &Blas) -> Option<SubmissionIndex> {
// We iterate in reverse order, so that we can bail out early as soon
// as we find a hit.
self.active.iter().rev().find_map(|submission| {
if submission.contains_blas(blas) {
Some(submission.index)
} else {
None
}
})
}
/// Returns the submission index of the most recent submission that uses the
/// given tlas.
pub fn get_tlas_latest_submission_index(&self, tlas: &Tlas) -> Option<SubmissionIndex> {
// We iterate in reverse order, so that we can bail out early as soon
// as we find a hit.
self.active.iter().rev().find_map(|submission| {
if submission.contains_tlas(tlas) {
Some(submission.index)
} else {
None
}
})
}
/// Returns the submission index of the most recent submission that uses the
/// given texture.
pub fn get_texture_latest_submission_index(
&self,
texture: &Texture,
) -> Option<SubmissionIndex> {
// We iterate in reverse order, so that we can bail out early as soon
// as we find a hit.
self.active.iter().rev().find_map(|submission| {
if submission.contains_texture(texture) {
Some(submission.index)
} else {
None
}
})
}
/// Sort out the consequences of completed submissions.
///
/// Assume that all submissions up through `last_done` have completed.
///
/// - Buffers used by those submissions are now ready to map, if requested.
/// Add any buffers in the submission's [`mapped`] list to
/// [`self.ready_to_map`], where [`LifetimeTracker::handle_mapping`]
/// will find them.
///
/// Return a list of [`SubmittedWorkDoneClosure`]s to run.
///
/// [`mapped`]: ActiveSubmission::mapped
/// [`self.ready_to_map`]: LifetimeTracker::ready_to_map
/// [`SubmittedWorkDoneClosure`]: crate::device::queue::SubmittedWorkDoneClosure
#[must_use]
pub fn triage_submissions(
&mut self,
last_done: SubmissionIndex,
) -> SmallVec<[SubmittedWorkDoneClosure; 1]> {
profiling::scope!("triage_submissions");
//TODO: enable when `is_sorted_by_key` is stable
//debug_assert!(self.active.is_sorted_by_key(|a| a.index));
let done_count = self
.active
.iter()
.position(|a| a.index > last_done)
.unwrap_or(self.active.len());
let mut work_done_closures: SmallVec<_> = self.work_done_closures.drain(..).collect();
for a in self.active.drain(..done_count) {
self.ready_to_map.extend(a.mapped);
for encoder in a.encoders {
// This involves actually decrementing the ref count of all command buffer
// resources, so can be _very_ expensive.
profiling::scope!("drop command buffer trackers");
drop(encoder);
}
drop(a.temp_resources);
work_done_closures.extend(a.work_done_closures);
}
work_done_closures
}
pub fn schedule_resource_destruction(
&mut self,
temp_resource: TempResource,
last_submit_index: SubmissionIndex,
) {
let resources = self
.active
.iter_mut()
.find(|a| a.index == last_submit_index)
.map(|a| &mut a.temp_resources);
if let Some(resources) = resources {
resources.push(temp_resource);
}
}
pub fn add_work_done_closure(
&mut self,
closure: SubmittedWorkDoneClosure,
) -> Option<SubmissionIndex> {
match self.active.last_mut() {
Some(active) => {
active.work_done_closures.push(closure);
Some(active.index)
}
// We must defer the closure until all previously occurring map_async closures
// have fired. This is required by the spec.
None => {
self.work_done_closures.push(closure);
None
}
}
}
/// Map the buffers in `self.ready_to_map`.
///
/// Return a list of mapping notifications to send.
///
/// See the documentation for [`LifetimeTracker`] for details.
#[must_use]
pub(crate) fn handle_mapping(
&mut self,
snatch_guard: &SnatchGuard,
) -> Vec<super::BufferMapPendingClosure> {
if self.ready_to_map.is_empty() {
return Vec::new();
}
let mut pending_callbacks: Vec<super::BufferMapPendingClosure> =
Vec::with_capacity(self.ready_to_map.len());
for buffer in self.ready_to_map.drain(..) {
match buffer.map(snatch_guard) {
Some(cb) => pending_callbacks.push(cb),
None => continue,
}
}
pending_callbacks
}
}