1use alloc::{sync::Arc, vec::Vec};
2use core::{
3 cmp::max,
4 mem,
5 num::NonZeroU64,
6 ops::{Deref, Range},
7};
8
9use wgt::{math::align_to, BufferUsages, BufferUses, Features};
10
11use crate::{
12 command::encoder::EncodingState,
13 ray_tracing::{AsAction, AsBuild, TlasBuild, ValidateAsActionsError},
14 resource::{Buffer, InvalidResourceError},
15};
16use crate::{command::EncoderStateError, device::resource::CommandIndices};
17use crate::{
18 command::{ArcCommand, ArcReferences, CommandBufferMutable},
19 device::queue::TempResource,
20 init_tracker::MemoryInitKind,
21 ray_tracing::{
22 ArcBlasAabbGeometry, ArcBlasBuildEntry, ArcBlasGeometries, ArcBlasTriangleGeometry,
23 ArcTlasPackage, BlasBuildEntry, BlasGeometries, BuildAccelerationStructureError,
24 OwnedBlasBuildEntry, OwnedTlasPackage,
25 },
26 resource::{Blas, BlasCompactState, Labeled, StagingBuffer, Tlas, Trackable},
27 scratch::ScratchBuffer,
28 snatch::SnatchGuard,
29 track::TrackerIndex,
30 FastHashSet,
31};
32use crate::{lock::RwLockWriteGuard, resource::RawResourceAccess};
33
34struct BlasStore<'a> {
35 blas: Arc<Blas>,
36 entries: hal::AccelerationStructureEntries<'a, dyn hal::DynBuffer>,
37 scratch_buffer_offset: u64,
38}
39
40struct UnsafeTlasStore<'a> {
41 tlas: Arc<Tlas>,
42 entries: hal::AccelerationStructureEntries<'a, dyn hal::DynBuffer>,
43 scratch_buffer_offset: u64,
44}
45
46struct TlasStore<'a> {
47 internal: UnsafeTlasStore<'a>,
48 range: Range<usize>,
49}
50
51impl super::CommandEncoder {
52 fn mark_acceleration_structures_built_inner(
53 self: &Arc<Self>,
54 blases: &[Arc<Blas>],
55 tlases: &[Arc<Tlas>],
56 ) -> Result<(), EncoderStateError> {
57 profiling::scope!("CommandEncoder::mark_acceleration_structures_built");
58
59 let mut cmd_buf_data = self.data.lock();
60 cmd_buf_data.with_buffer(
61 crate::command::EncodingApi::Raw,
62 |cmd_buf_data| -> Result<(), BuildAccelerationStructureError> {
63 let device = &self.device;
64 device.check_is_valid()?;
65 device.require_acceleration_structures()?;
66
67 let mut build_command = AsBuild::with_capacity(blases.len(), tlases.len());
68
69 for blas in blases {
70 let blas = blas.clone();
71 blas.check_is_valid()?;
72 build_command.blas_s_built.push(blas);
73 }
74
75 for tlas in tlases {
76 let tlas = tlas.clone();
77 tlas.check_is_valid()?;
78 build_command.tlas_s_built.push(TlasBuild {
79 tlas,
80 dependencies: Vec::new(),
81 max_intersection_idx: None,
82 required_intersection_types: Vec::new(),
83 });
84 }
85
86 cmd_buf_data.as_actions.push(AsAction::Build(build_command));
87 Ok(())
88 },
89 )
90 }
91
92 pub fn mark_acceleration_structures_built(
93 self: &Arc<Self>,
94 blases: &[Arc<Blas>],
95 tlases: &[Arc<Tlas>],
96 ) {
97 if let Err(err) = self.mark_acceleration_structures_built_inner(blases, tlases) {
98 self.device.handle_error(
99 err,
100 Some(self.label()),
101 "CommandEncoder::mark_acceleration_structures_built",
102 );
103 }
104 }
105
106 fn build_acceleration_structures_inner<'a>(
107 self: &Arc<Self>,
108 blas_iter: impl Iterator<Item = BlasBuildEntry<'a, Arc<Blas>, Arc<Buffer>>>,
109 tlases: Vec<ArcTlasPackage>,
110 ) -> Result<(), EncoderStateError> {
111 profiling::scope!("CommandEncoder::build_acceleration_structures");
112
113 let mut cmd_buf_data = self.data.lock();
114
115 cmd_buf_data.push_with(|| -> Result<_, BuildAccelerationStructureError> {
116 let blas = blas_iter
117 .map(|blas_entry| {
118 let geometries = match blas_entry.geometries {
119 BlasGeometries::TriangleGeometries(triangle_geometries) => {
120 let tri_geo = triangle_geometries
121 .map(|tg| {
122 let vertex_buffer = tg.vertex_buffer;
123 vertex_buffer.check_is_valid()?;
124 Ok(ArcBlasTriangleGeometry {
125 size: tg.size.clone(),
126 vertex_buffer,
127 index_buffer: tg
128 .index_buffer
129 .map(|index_buffer| -> Result<_, InvalidResourceError> {
130 index_buffer.check_is_valid()?;
131 Ok(index_buffer)
132 })
133 .transpose()?,
134 transform_buffer: tg
135 .transform_buffer
136 .map(|transform_buffer| -> Result<_, InvalidResourceError> {
137 transform_buffer.check_is_valid()?;
138 Ok(transform_buffer)
139 })
140 .transpose()?,
141 first_vertex: tg.first_vertex,
142 vertex_stride: tg.vertex_stride,
143 first_index: tg.first_index,
144 transform_buffer_offset: tg.transform_buffer_offset,
145 })
146 })
147 .collect::<Result<_, BuildAccelerationStructureError>>()?;
148 ArcBlasGeometries::TriangleGeometries(tri_geo)
149 }
150 BlasGeometries::AabbGeometries(aabb_geometries) => {
151 let aabb_geo = aabb_geometries
152 .map(|ag| {
153 ag.aabb_buffer.check_is_valid()?;
154 Ok(ArcBlasAabbGeometry {
155 size: ag.size.clone(),
156 stride: ag.stride,
157 aabb_buffer: ag.aabb_buffer,
158 primitive_offset: ag.primitive_offset,
159 })
160 })
161 .collect::<Result<_, BuildAccelerationStructureError>>()?;
162 ArcBlasGeometries::AabbGeometries(aabb_geo)
163 }
164 };
165 let blas = blas_entry.blas;
166 blas.check_is_valid()?;
167 Ok(ArcBlasBuildEntry { blas, geometries })
168 })
169 .collect::<Result<_, BuildAccelerationStructureError>>()?;
170
171 for tlas_package in &tlases {
172 tlas_package.tlas.check_is_valid()?;
173 for instance in tlas_package.instances.iter().flatten() {
174 instance.blas.check_is_valid()?;
175 }
176 }
177
178 Ok(ArcCommand::BuildAccelerationStructures { blas, tlas: tlases })
179 })
180 }
181
182 pub fn build_acceleration_structures<'a>(
183 self: &Arc<Self>,
184 blas_iter: impl Iterator<Item = BlasBuildEntry<'a, Arc<Blas>, Arc<Buffer>>>,
185 tlases: Vec<ArcTlasPackage>,
186 ) {
187 if let Err(err) = self.build_acceleration_structures_inner(blas_iter, tlases) {
188 self.device.handle_error(
189 err,
190 Some(self.label()),
191 "CommandEncoder::build_acceleration_structures",
192 );
193 }
194 }
195}
196
197pub(crate) fn build_acceleration_structures(
198 state: &mut EncodingState,
199 blases: Vec<OwnedBlasBuildEntry<ArcReferences>>,
200 mut tlases: Vec<OwnedTlasPackage<ArcReferences>>,
201) -> Result<(), BuildAccelerationStructureError> {
202 profiling::scope!("build_acceleration_structures");
203 state.device.require_acceleration_structures()?;
204
205 let mut build_command = AsBuild::with_capacity(blases.len(), tlases.len());
206 let mut input_barriers = Vec::<hal::BufferBarrier<dyn hal::DynBuffer>>::new();
207 let mut scratch_buffer_blas_size = 0;
208 let mut blas_storage = Vec::with_capacity(blases.len());
209 iter_blas(
210 blases.iter(),
211 &mut build_command,
212 &mut input_barriers,
213 &mut scratch_buffer_blas_size,
214 &mut blas_storage,
215 state,
216 )?;
217
218 let mut scratch_buffer_tlas_size: u64 = 0;
219 let mut tlas_storage = Vec::<TlasStore>::with_capacity(tlases.len());
220 let mut instance_buffer_staging_source = Vec::<u8>::new();
221
222 for package in tlases.iter_mut() {
225 profiling::scope!("tlas validation");
226 let tlas = &package.tlas;
227 state.tracker.tlas_s.insert_single(tlas.clone());
228
229 let scratch_buffer_offset = scratch_buffer_tlas_size;
230 scratch_buffer_tlas_size = scratch_buffer_tlas_size.saturating_add(align_to(
231 tlas.size_info.build_scratch_size,
232 u64::from(state.device.alignments.ray_tracing_scratch_buffer_alignment),
233 ));
234
235 let first_byte_index = instance_buffer_staging_source.len();
236
237 let mut dependencies = Vec::new();
238 let mut seen_dependencies = FastHashSet::<TrackerIndex>::default();
239
240 let mut required_intersection_types = Vec::new();
241 let mut seen_intersection_types =
242 FastHashSet::<crate::resource::TlasIntersectionType>::default();
243
244 let mut instance_count = 0;
245
246 let mut max_intersection_idx = None;
247 for (instance_idx, instance) in mem::take(&mut package.instances)
248 .into_iter()
249 .enumerate()
250 .flat_map(|(idx, instance)| instance.map(|inst| (idx, inst)))
251 {
252 if instance.custom_data >= (1u32 << 24u32) {
253 return Err(BuildAccelerationStructureError::TlasInvalidCustomData(
254 tlas.error_ident(),
255 instance_idx,
256 ));
257 }
258
259 let max_intersection_idx = max_intersection_idx.get_or_insert(0);
260
261 *max_intersection_idx = (*max_intersection_idx).max(instance.intersection_index);
262
263 let blas = instance.blas;
264 let is_new_dependency = seen_dependencies.insert(blas.tracker_index());
265
266 if is_new_dependency {
267 state.tracker.blas_s.insert_single(blas.clone());
268 }
269
270 if instance.intersection_index >= (1u32 << 24u32) {
271 return Err(
272 BuildAccelerationStructureError::TlasInvalidIntersectionIndex(
273 tlas.error_ident(),
274 instance_idx,
275 instance.intersection_index,
276 ),
277 );
278 } else if instance.intersection_index != 0 {
279 state
280 .device
281 .require_features(Features::EXPERIMENTAL_RAY_TRACING_PIPELINES)?;
282 }
283
284 state.device.raw().tlas_instance_to_bytes(
285 hal::TlasInstance {
286 transform: instance.transform,
287 custom_data: instance.custom_data,
288 mask: instance.mask,
289 blas_address: blas.handle,
290 pipeline_intersection_data_offset: instance.intersection_index,
291 },
292 &mut instance_buffer_staging_source,
293 );
294
295 if tlas
296 .flags
297 .contains(wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN)
298 && !blas
299 .flags
300 .contains(wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN)
301 {
302 return Err(
303 BuildAccelerationStructureError::TlasDependentMissingVertexReturn(
304 tlas.error_ident(),
305 blas.error_ident(),
306 ),
307 );
308 }
309
310 instance_count += 1;
311
312 let intersection_ty = crate::resource::TlasIntersectionType {
313 index: instance.intersection_index,
314 required_intersection_ty: match blas.sizes {
315 wgt::BlasGeometrySizeDescriptors::Triangles { .. } => {
316 crate::ray_tracing_pipeline::RayTracingIntersectionType::Triangle
317 }
318 wgt::BlasGeometrySizeDescriptors::AABBs { .. } => {
319 crate::ray_tracing_pipeline::RayTracingIntersectionType::AABB
320 }
321 },
322 };
323
324 let new_intersection_ty = seen_intersection_types.insert(intersection_ty);
325
326 if new_intersection_ty {
327 required_intersection_types.push(intersection_ty);
328 }
329
330 if is_new_dependency {
331 dependencies.push(blas);
332 }
333 }
334
335 build_command.tlas_s_built.push(TlasBuild {
336 tlas: tlas.clone(),
337 dependencies,
338 max_intersection_idx,
339 required_intersection_types,
340 });
341
342 if instance_count > tlas.max_instance_count {
343 return Err(BuildAccelerationStructureError::TlasInstanceCountExceeded(
344 tlas.error_ident(),
345 instance_count,
346 tlas.max_instance_count,
347 ));
348 }
349
350 tlas_storage.push(TlasStore {
351 internal: UnsafeTlasStore {
352 tlas: tlas.clone(),
353 entries: hal::AccelerationStructureEntries::Instances(
354 hal::AccelerationStructureInstances {
355 buffer: Some(tlas.state()?.instance_buffer.as_ref()),
356 offset: 0,
357 count: instance_count,
358 },
359 ),
360 scratch_buffer_offset,
361 },
362 range: first_byte_index..instance_buffer_staging_source.len(),
363 });
364 }
365
366 let Some(scratch_size) =
367 wgt::BufferSize::new(max(scratch_buffer_blas_size, scratch_buffer_tlas_size))
368 else {
369 return Ok(());
371 };
372
373 if scratch_size.get() == u64::MAX {
374 return Err(crate::device::DeviceError::OutOfMemory.into());
375 }
376
377 let scratch_buffer = ScratchBuffer::new(state.device, scratch_size)?;
378
379 let scratch_buffer_barrier = hal::BufferBarrier::<dyn hal::DynBuffer> {
380 buffer: scratch_buffer.raw(),
381 usage: hal::StateTransition {
382 from: BufferUses::ACCELERATION_STRUCTURE_SCRATCH,
383 to: BufferUses::ACCELERATION_STRUCTURE_SCRATCH,
384 },
385 };
386
387 let mut tlas_descriptors = Vec::with_capacity(tlas_storage.len());
388
389 for &TlasStore {
390 internal:
391 UnsafeTlasStore {
392 ref tlas,
393 ref entries,
394 ref scratch_buffer_offset,
395 },
396 ..
397 } in &tlas_storage
398 {
399 if tlas.update_mode == wgt::AccelerationStructureUpdateMode::PreferUpdate {
400 log::warn!("build_acceleration_structures called with PreferUpdate, but only rebuild is implemented");
401 }
402 tlas_descriptors.push(hal::BuildAccelerationStructureDescriptor {
403 entries,
404 mode: hal::AccelerationStructureBuildMode::Build,
405 flags: tlas.flags,
406 source_acceleration_structure: None,
407 destination_acceleration_structure: tlas.try_raw(state.snatch_guard)?,
408 scratch_buffer: scratch_buffer.raw(),
409 scratch_buffer_offset: *scratch_buffer_offset,
410 })
411 }
412
413 let blas_present = !blas_storage.is_empty();
414 let tlas_present = !tlas_storage.is_empty();
415
416 let raw_encoder = &mut state.raw_encoder;
417
418 let mut blas_s_compactable = Vec::new();
419 let mut descriptors = Vec::with_capacity(blases.len());
420
421 for storage in &blas_storage {
422 descriptors.push(map_blas(
423 storage,
424 scratch_buffer.raw(),
425 state.snatch_guard,
426 &mut blas_s_compactable,
427 )?);
428 }
429
430 build_blas(
431 *raw_encoder,
432 blas_present,
433 tlas_present,
434 input_barriers,
435 &descriptors,
436 scratch_buffer_barrier,
437 blas_s_compactable,
438 );
439
440 if tlas_present {
441 let staging_buffer = if !instance_buffer_staging_source.is_empty() {
442 let mut staging_buffer = StagingBuffer::new(
443 state.device,
444 wgt::BufferSize::new(instance_buffer_staging_source.len() as u64).unwrap(),
445 )?;
446 staging_buffer.write_exact(&instance_buffer_staging_source);
447 let flushed = staging_buffer.flush();
448 Some(flushed)
449 } else {
450 None
451 };
452
453 unsafe {
454 if let Some(ref staging_buffer) = staging_buffer {
455 raw_encoder.transition_buffers(&[hal::BufferBarrier::<dyn hal::DynBuffer> {
456 buffer: staging_buffer.raw(),
457 usage: hal::StateTransition {
458 from: BufferUses::MAP_WRITE,
459 to: BufferUses::COPY_SRC,
460 },
461 }]);
462 }
463 }
464
465 let mut instance_buffer_barriers = Vec::new();
466 for &TlasStore {
467 internal: UnsafeTlasStore { ref tlas, .. },
468 ref range,
469 } in &tlas_storage
470 {
471 let size = match wgt::BufferSize::new((range.end - range.start) as u64) {
472 None => continue,
473 Some(size) => size,
474 };
475 let tlas_state = tlas.state()?;
476 instance_buffer_barriers.push(hal::BufferBarrier::<dyn hal::DynBuffer> {
477 buffer: tlas_state.instance_buffer.as_ref(),
478 usage: hal::StateTransition {
479 from: BufferUses::COPY_DST,
480 to: BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT,
481 },
482 });
483 unsafe {
484 raw_encoder.transition_buffers(&[hal::BufferBarrier::<dyn hal::DynBuffer> {
485 buffer: tlas_state.instance_buffer.as_ref(),
486 usage: hal::StateTransition {
487 from: BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT,
488 to: BufferUses::COPY_DST,
489 },
490 }]);
491 let temp = hal::BufferCopy {
492 src_offset: range.start as u64,
493 dst_offset: 0,
494 size,
495 };
496 raw_encoder.copy_buffer_to_buffer(
497 staging_buffer.as_ref().unwrap().raw(),
498 tlas_state.instance_buffer.as_ref(),
499 &[temp],
500 );
501 }
502 }
503
504 unsafe {
505 raw_encoder.transition_buffers(&instance_buffer_barriers);
506
507 raw_encoder.build_acceleration_structures(&tlas_descriptors);
508
509 raw_encoder.place_acceleration_structure_barrier(hal::AccelerationStructureBarrier {
510 usage: hal::StateTransition {
511 from: hal::AccelerationStructureUses::BUILD_OUTPUT,
512 to: hal::AccelerationStructureUses::SHADER_INPUT,
513 },
514 });
515 }
516
517 if let Some(staging_buffer) = staging_buffer {
518 state
519 .temp_resources
520 .push(TempResource::StagingBuffer(staging_buffer));
521 }
522 }
523
524 state
525 .temp_resources
526 .push(TempResource::ScratchBuffer(scratch_buffer));
527
528 state.as_actions.push(AsAction::Build(build_command));
529
530 Ok(())
531}
532
533impl CommandBufferMutable {
534 pub(crate) fn validate_acceleration_structure_actions(
535 &self,
536 snatch_guard: &SnatchGuard,
537 command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
538 ) -> Result<(), ValidateAsActionsError> {
539 profiling::scope!("CommandEncoder::[submission]::validate_as_actions");
540 for action in &self.as_actions {
541 match action {
542 AsAction::Build(build) => {
543 let build_command_index = NonZeroU64::new(
544 command_index_guard.next_acceleration_structure_build_command_index,
545 )
546 .unwrap();
547
548 command_index_guard.next_acceleration_structure_build_command_index += 1;
549 for blas in build.blas_s_built.iter() {
550 let mut state_lock = blas.compacted_state.lock();
551 *state_lock = match *state_lock {
552 BlasCompactState::Compacted => {
553 unreachable!("Should be validated out in build.")
554 }
555 _ => BlasCompactState::Idle,
558 };
559 *blas.built_index.write() = Some(build_command_index);
560 }
561
562 for tlas_build in build.tlas_s_built.iter() {
563 for blas in &tlas_build.dependencies {
564 if blas.built_index.read().is_none() {
565 return Err(ValidateAsActionsError::UsedUnbuiltBlas(
566 blas.error_ident(),
567 tlas_build.tlas.error_ident(),
568 ));
569 }
570 }
571 *tlas_build.tlas.built_index.write() = Some(build_command_index);
572 tlas_build
573 .tlas
574 .dependencies
575 .write()
576 .clone_from(&tlas_build.dependencies);
577 *tlas_build.tlas.max_intersection_index.write() =
578 tlas_build.max_intersection_idx;
579 *tlas_build.tlas.required_intersection_types.write() =
580 tlas_build.required_intersection_types.clone();
581 }
582 }
583 AsAction::BindTlas(tlas) => {
584 let tlas_build_index = tlas.built_index.read();
585 let dependencies = tlas.dependencies.read();
586
587 if (*tlas_build_index).is_none() {
588 return Err(ValidateAsActionsError::UsedUnbuiltTlas(tlas.error_ident()));
589 }
590 for blas in dependencies.deref() {
591 let blas_build_index = *blas.built_index.read();
592 if blas_build_index.is_none() {
593 return Err(ValidateAsActionsError::UsedUnbuiltBlas(
594 tlas.error_ident(),
595 blas.error_ident(),
596 ));
597 }
598 if blas_build_index.unwrap() > tlas_build_index.unwrap() {
599 return Err(ValidateAsActionsError::BlasNewerThenTlas(
600 blas.error_ident(),
601 tlas.error_ident(),
602 ));
603 }
604 blas.try_raw(snatch_guard)?;
605 }
606 }
607 AsAction::TraceTlas(tlas, intersection_types) => {
608 let current_max = tlas.max_intersection_index.read();
609
610 let max_intersection_len = u32::try_from(intersection_types.len())
611 .expect("should be smaller than a u32");
612
613 if let Some(current_max) = *current_max {
614 if current_max >= max_intersection_len {
615 return Err(ValidateAsActionsError::TlasIntersectionInvalid(
616 tlas.error_ident(),
617 current_max,
618 max_intersection_len,
619 ));
620 }
621 }
622 drop(current_max);
623
624 let required_intersection_types = tlas.required_intersection_types.read();
625
626 for intersection_ty in required_intersection_types.iter() {
627 let Ok(index) = usize::try_from(intersection_ty.index) else {
628 return Err(
629 ValidateAsActionsError::TlasInstancesIntersectionIndicesDiffer(
630 tlas.error_ident(),
631 intersection_ty.required_intersection_ty,
632 intersection_ty.index,
633 None,
634 ),
635 );
636 };
637 match intersection_types.get(index) {
638 Some(ty) => {
639 if *ty != intersection_ty.required_intersection_ty {
640 return Err(ValidateAsActionsError::TlasInstancesIntersectionIndicesDiffer(tlas.error_ident(), intersection_ty.required_intersection_ty, intersection_ty.index, Some(*ty)));
641 }
642 }
643 None => {
644 return Err(
645 ValidateAsActionsError::TlasInstancesIntersectionIndicesDiffer(
646 tlas.error_ident(),
647 intersection_ty.required_intersection_ty,
648 intersection_ty.index,
649 None,
650 ),
651 );
652 }
653 }
654 }
655 }
656 }
657 }
658 Ok(())
659 }
660
661 pub(crate) fn set_acceleration_structure_dependencies(&self, snatch_guard: &SnatchGuard) {
662 profiling::scope!("CommandEncoder::[submission]::set_acceleration_structure_dependencies");
663 let mut dependencies = Vec::new();
664 for action in &self.as_actions {
665 match action {
666 AsAction::Build(build) => {
667 dependencies.extend(build.blas_s_built.iter().map(Arc::clone));
668 }
669 AsAction::BindTlas(tlas) => {
670 dependencies.extend(tlas.dependencies.read().iter().map(Arc::clone));
671 }
672 AsAction::TraceTlas(..) => {}
674 }
675 }
676
677 let raw_dependencies = dependencies
678 .iter()
679 .flat_map(|blas| blas.raw(snatch_guard))
680 .collect::<Vec<_>>();
681
682 if !dependencies.is_empty() {
683 unsafe {
684 self.encoder
685 .raw
686 .set_acceleration_structure_dependencies(&self.encoder.list, &raw_dependencies);
687 }
688 }
689 }
690}
691
692fn iter_blas<'snatch_guard: 'buffers, 'buffers>(
694 blas_iter: impl Iterator<Item = &'buffers OwnedBlasBuildEntry<ArcReferences>>,
695 build_command: &mut AsBuild,
696 input_barriers: &mut Vec<hal::BufferBarrier<'buffers, dyn hal::DynBuffer>>,
697 scratch_buffer_blas_size: &mut u64,
698 blas_storage: &mut Vec<BlasStore<'buffers>>,
699 state: &mut EncodingState<'snatch_guard, '_>,
700) -> Result<(), BuildAccelerationStructureError> {
701 for entry in blas_iter {
702 let blas = &entry.blas;
703 state.tracker.blas_s.insert_single(blas.clone());
704
705 build_command.blas_s_built.push(blas.clone());
706
707 match &entry.geometries {
708 ArcBlasGeometries::TriangleGeometries(triangle_geometries) => {
709 let mut triangle_entries =
710 Vec::<hal::AccelerationStructureTriangles<dyn hal::DynBuffer>>::new();
711
712 for (i, mesh) in triangle_geometries.iter().enumerate() {
713 let size_desc = match &blas.sizes {
714 wgt::BlasGeometrySizeDescriptors::Triangles { descriptors } => descriptors,
715 _ => {
716 return Err(BuildAccelerationStructureError::BlasGeometryKindMismatch(
717 blas.error_ident(),
718 ));
719 }
720 };
721 if i >= size_desc.len() {
722 return Err(BuildAccelerationStructureError::IncompatibleBlasBuildSizes(
723 blas.error_ident(),
724 ));
725 }
726 let size_desc = &size_desc[i];
727
728 if size_desc.flags != mesh.size.flags {
729 return Err(BuildAccelerationStructureError::IncompatibleBlasFlags(
730 blas.error_ident(),
731 size_desc.flags,
732 mesh.size.flags,
733 ));
734 }
735
736 if size_desc.vertex_count < mesh.size.vertex_count {
737 return Err(
738 BuildAccelerationStructureError::IncompatibleBlasVertexCount(
739 blas.error_ident(),
740 size_desc.vertex_count,
741 mesh.size.vertex_count,
742 ),
743 );
744 }
745
746 if size_desc.vertex_format != mesh.size.vertex_format {
747 return Err(BuildAccelerationStructureError::DifferentBlasVertexFormats(
748 blas.error_ident(),
749 size_desc.vertex_format,
750 mesh.size.vertex_format,
751 ));
752 }
753
754 if size_desc
755 .vertex_format
756 .min_acceleration_structure_vertex_stride()
757 > mesh.vertex_stride
758 {
759 return Err(BuildAccelerationStructureError::VertexStrideTooSmall(
760 blas.error_ident(),
761 size_desc
762 .vertex_format
763 .min_acceleration_structure_vertex_stride(),
764 mesh.vertex_stride,
765 ));
766 }
767
768 if mesh.vertex_stride
769 % size_desc
770 .vertex_format
771 .acceleration_structure_stride_alignment()
772 != 0
773 {
774 return Err(BuildAccelerationStructureError::VertexStrideUnaligned(
775 blas.error_ident(),
776 size_desc
777 .vertex_format
778 .acceleration_structure_stride_alignment(),
779 mesh.vertex_stride,
780 ));
781 }
782
783 match (size_desc.index_count, mesh.size.index_count) {
784 (Some(_), None) | (None, Some(_)) => {
785 return Err(
786 BuildAccelerationStructureError::BlasIndexCountProvidedMismatch(
787 blas.error_ident(),
788 ),
789 )
790 }
791 (Some(create), Some(build)) if create < build => {
792 return Err(
793 BuildAccelerationStructureError::IncompatibleBlasIndexCount(
794 blas.error_ident(),
795 create,
796 build,
797 ),
798 )
799 }
800 _ => {}
801 }
802
803 if size_desc.index_format != mesh.size.index_format {
804 return Err(BuildAccelerationStructureError::DifferentBlasIndexFormats(
805 blas.error_ident(),
806 size_desc.index_format,
807 mesh.size.index_format,
808 ));
809 }
810
811 if size_desc.index_count.is_some() && mesh.index_buffer.is_none() {
812 return Err(BuildAccelerationStructureError::MissingIndexBuffer(
813 blas.error_ident(),
814 ));
815 }
816 let vertex_buffer = mesh.vertex_buffer.clone();
817 let vertex_pending = state.tracker.buffers.set_single(
818 &vertex_buffer,
819 BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT,
820 );
821 let vertex_buffer = {
822 let vertex_raw = mesh.vertex_buffer.as_ref().try_raw(state.snatch_guard)?;
823 let vertex_buffer = &mesh.vertex_buffer;
824 vertex_buffer.check_usage(BufferUsages::BLAS_INPUT)?;
825
826 if let Some(barrier) = vertex_pending.map(|pending| {
827 pending.into_hal(vertex_buffer.as_ref(), state.snatch_guard)
828 }) {
829 input_barriers.push(barrier);
830 }
831 if u64::from(mesh.size.vertex_count)
832 .checked_add(u64::from(mesh.first_vertex))
833 .and_then(|end_vertex| end_vertex.checked_mul(mesh.vertex_stride))
834 .is_none_or(|end| vertex_buffer.size < end)
835 {
836 return Err(BuildAccelerationStructureError::InsufficientBufferSize {
837 buffer_ident: vertex_buffer.error_ident(),
838 offset: u64::from(mesh.first_vertex)
839 .saturating_mul(mesh.vertex_stride),
840 region_size: u64::from(mesh.size.vertex_count)
841 .saturating_mul(mesh.vertex_stride),
842 buffer_size: vertex_buffer.size,
843 });
844 }
845 let vertex_buffer_offset = mesh.first_vertex as u64 * mesh.vertex_stride;
846 state.buffer_memory_init_actions.extend(
847 vertex_buffer.initialization_status.read().create_action(
848 vertex_buffer,
849 vertex_buffer_offset
850 ..(vertex_buffer_offset
851 + mesh.size.vertex_count as u64 * mesh.vertex_stride),
852 MemoryInitKind::NeedsInitializedMemory,
853 ),
854 );
855 vertex_raw
856 };
857 let index_buffer = if let Some(ref index_buffer) = mesh.index_buffer {
858 if mesh.first_index.is_none()
859 || mesh.size.index_count.is_none()
860 || mesh.size.index_count.is_none()
861 {
862 return Err(BuildAccelerationStructureError::MissingAssociatedData(
863 index_buffer.error_ident(),
864 ));
865 }
866 let index_pending = state.tracker.buffers.set_single(
867 index_buffer,
868 BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT,
869 );
870 let index_raw = index_buffer.try_raw(state.snatch_guard)?;
871 index_buffer.check_usage(BufferUsages::BLAS_INPUT)?;
872
873 if let Some(barrier) = index_pending.map(|pending| {
874 pending.into_hal(index_buffer.as_ref(), state.snatch_guard)
875 }) {
876 input_barriers.push(barrier);
877 }
878 let index_stride = mesh.size.index_format.unwrap().byte_size();
879 let Some(vertex_offset) =
881 mesh.first_index.unwrap().checked_mul(index_stride)
882 else {
883 return Err(BuildAccelerationStructureError::OffsetLimitedTo4GB {
884 buffer_ident: index_buffer.error_ident(),
885 offset: u64::from(mesh.first_index.unwrap())
886 .saturating_mul(u64::from(index_stride)),
887 count: u64::from(mesh.first_index.unwrap()),
888 stride: u64::from(index_stride),
889 });
890 };
891 let Some(indexes_size) =
892 mesh.size.index_count.unwrap().checked_mul(index_stride)
893 else {
894 return Err(BuildAccelerationStructureError::OffsetLimitedTo4GB {
895 buffer_ident: index_buffer.error_ident(),
896 offset: u64::from(mesh.size.index_count.unwrap())
897 .saturating_mul(u64::from(index_stride)),
898 count: u64::from(mesh.size.index_count.unwrap()),
899 stride: u64::from(index_stride),
900 });
901 };
902
903 if mesh.size.index_count.unwrap() % 3 != 0 {
904 return Err(BuildAccelerationStructureError::InvalidIndexCount(
905 index_buffer.error_ident(),
906 mesh.size.index_count.unwrap(),
907 ));
908 }
909 if index_buffer.size < u64::from(vertex_offset)
910 || index_buffer.size - u64::from(vertex_offset)
911 < u64::from(indexes_size)
912 {
913 return Err(BuildAccelerationStructureError::InsufficientBufferSize {
914 buffer_ident: index_buffer.error_ident(),
915 offset: u64::from(vertex_offset),
916 region_size: u64::from(indexes_size),
917 buffer_size: index_buffer.size,
918 });
919 }
920
921 state.buffer_memory_init_actions.extend(
922 index_buffer.initialization_status.read().create_action(
923 index_buffer,
924 u64::from(vertex_offset)
925 ..(u64::from(vertex_offset) + u64::from(indexes_size)),
926 MemoryInitKind::NeedsInitializedMemory,
927 ),
928 );
929 Some((index_raw, vertex_offset))
930 } else {
931 None
932 };
933 let transform_buffer = if let Some(ref transform_buffer) = mesh.transform_buffer
934 {
935 if !blas
936 .flags
937 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
938 {
939 return Err(BuildAccelerationStructureError::UseTransformMissing(
940 blas.error_ident(),
941 ));
942 }
943 if mesh.transform_buffer_offset.is_none() {
944 return Err(BuildAccelerationStructureError::MissingAssociatedData(
945 transform_buffer.error_ident(),
946 ));
947 }
948 let transform_pending = state.tracker.buffers.set_single(
949 transform_buffer,
950 BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT,
951 );
952 if mesh.transform_buffer_offset.is_none() {
953 return Err(BuildAccelerationStructureError::MissingAssociatedData(
954 transform_buffer.error_ident(),
955 ));
956 }
957 let transform_raw = transform_buffer.try_raw(state.snatch_guard)?;
958 transform_buffer.check_usage(BufferUsages::BLAS_INPUT)?;
959
960 if let Some(barrier) = transform_pending.map(|pending| {
961 pending.into_hal(transform_buffer.as_ref(), state.snatch_guard)
962 }) {
963 input_barriers.push(barrier);
964 }
965
966 let Ok(offset) = u32::try_from(mesh.transform_buffer_offset.unwrap())
968 else {
969 return Err(BuildAccelerationStructureError::OffsetLimitedTo4GB {
970 buffer_ident: transform_buffer.error_ident(),
971 offset: mesh.transform_buffer_offset.unwrap(),
972 count: mesh.transform_buffer_offset.unwrap(),
973 stride: 1,
974 });
975 };
976
977 if offset % wgt::TRANSFORM_BUFFER_ALIGNMENT as u32 != 0 {
978 return Err(
979 BuildAccelerationStructureError::UnalignedTransformBufferOffset(
980 transform_buffer.error_ident(),
981 ),
982 );
983 }
984 if transform_buffer.size < 48
985 || transform_buffer.size - 48 < u64::from(offset)
986 {
987 return Err(BuildAccelerationStructureError::InsufficientBufferSize {
988 buffer_ident: transform_buffer.error_ident(),
989 region_size: 48,
990 offset: u64::from(offset),
991 buffer_size: transform_buffer.size,
992 });
993 }
994 state.buffer_memory_init_actions.extend(
995 transform_buffer.initialization_status.read().create_action(
996 transform_buffer,
997 u64::from(offset)..(u64::from(offset) + 48),
998 MemoryInitKind::NeedsInitializedMemory,
999 ),
1000 );
1001 Some((transform_raw, offset))
1002 } else {
1003 if blas
1004 .flags
1005 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
1006 {
1007 return Err(BuildAccelerationStructureError::TransformMissing(
1008 blas.error_ident(),
1009 ));
1010 }
1011 None
1012 };
1013
1014 let triangles = hal::AccelerationStructureTriangles {
1015 vertex_buffer: Some(vertex_buffer),
1016 vertex_format: mesh.size.vertex_format,
1017 first_vertex: mesh.first_vertex,
1018 vertex_count: mesh.size.vertex_count,
1019 vertex_stride: mesh.vertex_stride,
1020 indices: index_buffer.map(|(index_raw, vertex_offset)| {
1021 hal::AccelerationStructureTriangleIndices::<dyn hal::DynBuffer> {
1022 format: mesh.size.index_format.unwrap(),
1023 buffer: Some(index_raw),
1024 offset: vertex_offset,
1025 count: mesh.size.index_count.unwrap(),
1026 }
1027 }),
1028 transform: transform_buffer.map(|(transform_raw, offset)| {
1029 hal::AccelerationStructureTriangleTransform {
1030 buffer: transform_raw,
1031 offset,
1032 }
1033 }),
1034 flags: mesh.size.flags,
1035 };
1036 triangle_entries.push(triangles);
1037 }
1038
1039 {
1040 let scratch_buffer_offset = *scratch_buffer_blas_size;
1041 *scratch_buffer_blas_size = scratch_buffer_blas_size.saturating_add(align_to(
1042 blas.size_info.build_scratch_size,
1043 u64::from(state.device.alignments.ray_tracing_scratch_buffer_alignment),
1044 ));
1045
1046 blas_storage.push(BlasStore {
1047 blas: blas.clone(),
1048 entries: hal::AccelerationStructureEntries::Triangles(triangle_entries),
1049 scratch_buffer_offset,
1050 });
1051 }
1052 }
1053 ArcBlasGeometries::AabbGeometries(aabb_geometries) => {
1054 let mut aabb_entries =
1055 Vec::<hal::AccelerationStructureAABBs<dyn hal::DynBuffer>>::new();
1056
1057 for (i, aabb) in aabb_geometries.iter().enumerate() {
1058 let size_desc = match &blas.sizes {
1059 wgt::BlasGeometrySizeDescriptors::AABBs { descriptors } => descriptors,
1060 _ => {
1061 return Err(BuildAccelerationStructureError::BlasGeometryKindMismatch(
1062 blas.error_ident(),
1063 ));
1064 }
1065 };
1066 if i >= size_desc.len() {
1067 return Err(BuildAccelerationStructureError::IncompatibleBlasBuildSizes(
1068 blas.error_ident(),
1069 ));
1070 }
1071 let size_desc = &size_desc[i];
1072
1073 if size_desc.flags != aabb.size.flags {
1074 return Err(BuildAccelerationStructureError::IncompatibleBlasFlags(
1075 blas.error_ident(),
1076 size_desc.flags,
1077 aabb.size.flags,
1078 ));
1079 }
1080
1081 if size_desc.primitive_count < aabb.size.primitive_count {
1082 return Err(
1083 BuildAccelerationStructureError::IncompatibleBlasAabbPrimitiveCount(
1084 blas.error_ident(),
1085 size_desc.primitive_count,
1086 aabb.size.primitive_count,
1087 ),
1088 );
1089 }
1090
1091 if aabb.primitive_offset % 8 != 0 {
1092 return Err(
1093 BuildAccelerationStructureError::UnalignedAabbPrimitiveOffset(
1094 blas.error_ident(),
1095 ),
1096 );
1097 }
1098
1099 if aabb.stride < wgt::AABB_GEOMETRY_MIN_STRIDE || aabb.stride % 8 != 0 {
1100 return Err(BuildAccelerationStructureError::InvalidAabbStride(
1101 blas.error_ident(),
1102 aabb.stride,
1103 ));
1104 }
1105
1106 let aabb_buffer = aabb.aabb_buffer.clone();
1107 let aabb_pending = state.tracker.buffers.set_single(
1108 &aabb_buffer,
1109 BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT,
1110 );
1111 let aabb_raw = {
1112 let aabb_raw = aabb.aabb_buffer.as_ref().try_raw(state.snatch_guard)?;
1113 let aabb_buffer = &aabb.aabb_buffer;
1114 aabb_buffer.check_usage(BufferUsages::BLAS_INPUT)?;
1115
1116 if let Some(barrier) = aabb_pending.map(|pending| {
1117 pending.into_hal(aabb_buffer.as_ref(), state.snatch_guard)
1118 }) {
1119 input_barriers.push(barrier);
1120 }
1121
1122 let Some(aabb_size) = u32::try_from(aabb.stride)
1124 .ok()
1125 .and_then(|stride| aabb.size.primitive_count.checked_mul(stride))
1126 else {
1127 return Err(BuildAccelerationStructureError::OffsetLimitedTo4GB {
1128 buffer_ident: aabb_buffer.error_ident(),
1129 offset: u64::from(aabb.size.primitive_count)
1130 .saturating_mul(aabb.stride),
1131 count: u64::from(aabb.size.primitive_count),
1132 stride: aabb.stride,
1133 });
1134 };
1135
1136 if aabb_buffer.size < u64::from(aabb.primitive_offset)
1137 || aabb_buffer.size - u64::from(aabb.primitive_offset)
1138 < u64::from(aabb_size)
1139 {
1140 return Err(BuildAccelerationStructureError::InsufficientBufferSize {
1141 buffer_ident: aabb_buffer.error_ident(),
1142 region_size: u64::from(aabb_size),
1143 offset: u64::from(aabb.primitive_offset),
1144 buffer_size: aabb_buffer.size,
1145 });
1146 }
1147
1148 state.buffer_memory_init_actions.extend(
1149 aabb_buffer.initialization_status.read().create_action(
1150 aabb_buffer,
1151 u64::from(aabb.primitive_offset)
1152 ..u64::from(aabb.primitive_offset) + u64::from(aabb_size),
1153 MemoryInitKind::NeedsInitializedMemory,
1154 ),
1155 );
1156 aabb_raw
1157 };
1158
1159 aabb_entries.push(hal::AccelerationStructureAABBs {
1160 buffer: Some(aabb_raw),
1161 offset: aabb.primitive_offset,
1162 count: aabb.size.primitive_count,
1163 stride: aabb.stride,
1164 flags: aabb.size.flags,
1165 });
1166 }
1167
1168 {
1169 let scratch_buffer_offset = *scratch_buffer_blas_size;
1170 *scratch_buffer_blas_size = scratch_buffer_blas_size.saturating_add(align_to(
1171 blas.size_info.build_scratch_size,
1172 u64::from(state.device.alignments.ray_tracing_scratch_buffer_alignment),
1173 ));
1174
1175 blas_storage.push(BlasStore {
1176 blas: blas.clone(),
1177 entries: hal::AccelerationStructureEntries::AABBs(aabb_entries),
1178 scratch_buffer_offset,
1179 });
1180 }
1181 }
1182 }
1183 }
1184 Ok(())
1185}
1186
1187fn map_blas<'a>(
1188 storage: &'a BlasStore<'_>,
1189 scratch_buffer: &'a dyn hal::DynBuffer,
1190 snatch_guard: &'a SnatchGuard,
1191 blases_compactable: &mut Vec<(
1192 &'a dyn hal::DynBuffer,
1193 &'a dyn hal::DynAccelerationStructure,
1194 )>,
1195) -> Result<
1196 hal::BuildAccelerationStructureDescriptor<
1197 'a,
1198 dyn hal::DynBuffer,
1199 dyn hal::DynAccelerationStructure,
1200 >,
1201 BuildAccelerationStructureError,
1202> {
1203 let BlasStore {
1204 blas,
1205 entries,
1206 scratch_buffer_offset,
1207 } = storage;
1208 if blas.update_mode == wgt::AccelerationStructureUpdateMode::PreferUpdate {
1209 log::debug!("only rebuild implemented")
1210 }
1211 let raw = blas.try_raw(snatch_guard)?;
1212
1213 let state_lock = blas.compacted_state.lock();
1214 if let BlasCompactState::Compacted = *state_lock {
1215 return Err(BuildAccelerationStructureError::CompactedBlas(
1216 blas.error_ident(),
1217 ));
1218 }
1219
1220 if blas
1221 .flags
1222 .contains(wgpu_types::AccelerationStructureFlags::ALLOW_COMPACTION)
1223 {
1224 blases_compactable.push((blas.compaction_buffer.as_ref().unwrap().as_ref(), raw));
1225 }
1226 Ok(hal::BuildAccelerationStructureDescriptor {
1227 entries,
1228 mode: hal::AccelerationStructureBuildMode::Build,
1229 flags: blas.flags,
1230 source_acceleration_structure: None,
1231 destination_acceleration_structure: raw,
1232 scratch_buffer,
1233 scratch_buffer_offset: *scratch_buffer_offset,
1234 })
1235}
1236
1237fn build_blas<'a>(
1238 cmd_buf_raw: &mut dyn hal::DynCommandEncoder,
1239 blas_present: bool,
1240 tlas_present: bool,
1241 input_barriers: Vec<hal::BufferBarrier<dyn hal::DynBuffer>>,
1242 blas_descriptors: &[hal::BuildAccelerationStructureDescriptor<
1243 'a,
1244 dyn hal::DynBuffer,
1245 dyn hal::DynAccelerationStructure,
1246 >],
1247 scratch_buffer_barrier: hal::BufferBarrier<dyn hal::DynBuffer>,
1248 blas_s_for_compaction: Vec<(
1249 &'a dyn hal::DynBuffer,
1250 &'a dyn hal::DynAccelerationStructure,
1251 )>,
1252) {
1253 unsafe {
1254 cmd_buf_raw.transition_buffers(&input_barriers);
1255 }
1256
1257 if blas_present {
1258 unsafe {
1259 cmd_buf_raw.place_acceleration_structure_barrier(hal::AccelerationStructureBarrier {
1260 usage: hal::StateTransition {
1261 from: hal::AccelerationStructureUses::BUILD_INPUT,
1262 to: hal::AccelerationStructureUses::BUILD_OUTPUT,
1263 },
1264 });
1265
1266 cmd_buf_raw.build_acceleration_structures(blas_descriptors);
1267 }
1268 }
1269
1270 if blas_present && tlas_present {
1271 unsafe {
1272 cmd_buf_raw.transition_buffers(&[scratch_buffer_barrier]);
1273 }
1274 }
1275
1276 let mut source_usage = hal::AccelerationStructureUses::empty();
1277 let mut destination_usage = hal::AccelerationStructureUses::empty();
1278 for &(buf, blas) in blas_s_for_compaction.iter() {
1279 unsafe {
1280 cmd_buf_raw.transition_buffers(&[hal::BufferBarrier {
1281 buffer: buf,
1282 usage: hal::StateTransition {
1283 from: BufferUses::ACCELERATION_STRUCTURE_QUERY,
1284 to: BufferUses::ACCELERATION_STRUCTURE_QUERY,
1285 },
1286 }])
1287 }
1288 unsafe { cmd_buf_raw.read_acceleration_structure_compact_size(blas, buf) }
1289 destination_usage |= hal::AccelerationStructureUses::COPY_SRC;
1290 }
1291
1292 if blas_present {
1293 source_usage |= hal::AccelerationStructureUses::BUILD_OUTPUT;
1294 destination_usage |= hal::AccelerationStructureUses::BUILD_INPUT
1295 }
1296 if tlas_present {
1297 source_usage |= hal::AccelerationStructureUses::SHADER_INPUT;
1298 destination_usage |= hal::AccelerationStructureUses::BUILD_OUTPUT;
1299 }
1300 unsafe {
1301 cmd_buf_raw.place_acceleration_structure_barrier(hal::AccelerationStructureBarrier {
1302 usage: hal::StateTransition {
1303 from: source_usage,
1304 to: destination_usage,
1305 },
1306 });
1307 }
1308}