1use alloc::{string::ToString, sync::Arc, vec::Vec};
2use core::mem::{size_of, ManuallyDrop};
3
4use crate::device::stage_compiler::StageCompiler;
5#[cfg(feature = "trace")]
6use crate::device::trace::{Action, IntoTrace};
7use crate::device::{DeviceError, ENTRYPOINT_FAILURE_ERROR};
8use crate::pipeline::CreatePipelineError;
9use crate::ray_tracing_pipeline;
10use crate::resource::{ParentDevice, ResourceState};
11use crate::{
12 api_log,
13 device::Device,
14 hal_label,
15 lock::RwLock,
16 lock::{rank, Mutex},
17 ray_tracing::{CreateBlasError, CreateTlasError},
18 resource,
19 resource::{BlasCompactState, TrackingData},
20 snatch::Snatchable,
21 LabelHelpers,
22};
23use hal::AccelerationStructureTriangleIndices;
24use wgt::{Features, AABB_GEOMETRY_MIN_STRIDE};
25
26impl Device {
27 pub fn require_acceleration_structures(&self) -> Result<(), crate::device::MissingFeatures> {
29 self.require_features(Features::EXPERIMENTAL_RAY_QUERY)
30 .or_else(|_| self.require_features(Features::EXPERIMENTAL_RAY_TRACING_PIPELINES))
31 }
32
33 pub fn create_blas(
34 self: &Arc<Self>,
35 blas_desc: &resource::BlasDescriptor,
36 sizes: wgt::BlasGeometrySizeDescriptors,
37 ) -> (Arc<resource::Blas>, Option<CreateBlasError>) {
38 #[cfg(feature = "trace")]
39 let trace_sizes = sizes.clone();
40
41 let (blas, error) = match self.create_blas_inner(blas_desc, sizes) {
42 Ok(blas) => (blas, None),
43 Err(err) => (resource::Blas::invalid(self.clone(), blas_desc), Some(err)),
44 };
45
46 #[cfg(feature = "trace")]
47 if let Some(trace) = self.trace.lock().as_mut() {
48 trace.add(Action::CreateBlas {
49 id: blas.to_trace(),
50 desc: blas_desc.clone(),
51 sizes: trace_sizes,
52 });
53 }
54
55 api_log!("Device::create_blas -> {:?}", Arc::as_ptr(&blas));
56 (blas, error)
57 }
58 pub(crate) fn create_blas_inner(
59 self: &Arc<Self>,
60 blas_desc: &resource::BlasDescriptor,
61 sizes: wgt::BlasGeometrySizeDescriptors,
62 ) -> Result<Arc<resource::Blas>, CreateBlasError> {
63 self.check_is_valid()?;
64 self.require_acceleration_structures()?;
65
66 if blas_desc
67 .flags
68 .contains(wgt::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN)
69 {
70 self.require_features(Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)?;
71 }
72
73 let size_info = match &sizes {
74 wgt::BlasGeometrySizeDescriptors::Triangles { descriptors } => {
75 if descriptors.len() as u32 > self.limits.max_blas_geometry_count {
76 return Err(CreateBlasError::TooManyGeometries(
77 self.limits.max_blas_geometry_count,
78 descriptors.len() as u32,
79 ));
80 }
81
82 let mut entries =
83 Vec::<hal::AccelerationStructureTriangles<dyn hal::DynBuffer>>::with_capacity(
84 descriptors.len(),
85 );
86 for desc in descriptors {
87 if desc.index_count.is_some() != desc.index_format.is_some() {
88 return Err(CreateBlasError::MissingIndexData);
89 }
90 let indices =
91 desc.index_count
92 .map(|count| AccelerationStructureTriangleIndices::<
93 dyn hal::DynBuffer,
94 > {
95 format: desc.index_format.unwrap(),
96 buffer: Some(self.zero_buffer.as_ref()),
97 offset: 0,
98 count,
99 });
100 if !self
101 .features
102 .allowed_vertex_formats_for_blas()
103 .contains(&desc.vertex_format)
104 {
105 return Err(CreateBlasError::InvalidVertexFormat(
106 desc.vertex_format,
107 self.features.allowed_vertex_formats_for_blas(),
108 ));
109 }
110
111 let mut transform = None;
112
113 if blas_desc
114 .flags
115 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
116 {
117 transform = Some(wgpu_hal::AccelerationStructureTriangleTransform {
118 buffer: self.zero_buffer.as_ref(),
119 offset: 0,
120 })
121 }
122
123 if desc.vertex_count > self.limits.max_blas_primitive_count {
124 return Err(CreateBlasError::TooManyPrimitives(
125 self.limits.max_blas_primitive_count,
126 desc.vertex_count,
127 ));
128 }
129
130 entries.push(hal::AccelerationStructureTriangles::<dyn hal::DynBuffer> {
131 vertex_buffer: Some(self.zero_buffer.as_ref()),
132 vertex_format: desc.vertex_format,
133 first_vertex: 0,
134 vertex_count: desc.vertex_count,
135 vertex_stride: 0,
136 indices,
137 transform,
138 flags: desc.flags,
139 });
140 }
141 unsafe {
142 self.raw().get_acceleration_structure_build_sizes(
143 &hal::GetAccelerationStructureBuildSizesDescriptor {
144 entries: &hal::AccelerationStructureEntries::Triangles(entries),
145 flags: blas_desc.flags,
146 },
147 )
148 }
149 }
150 wgt::BlasGeometrySizeDescriptors::AABBs { descriptors } => {
151 if descriptors.len() as u32 > self.limits.max_blas_geometry_count {
152 return Err(CreateBlasError::TooManyGeometries(
153 self.limits.max_blas_geometry_count,
154 descriptors.len() as u32,
155 ));
156 }
157
158 let mut entries =
159 Vec::<hal::AccelerationStructureAABBs<dyn hal::DynBuffer>>::with_capacity(
160 descriptors.len(),
161 );
162 for desc in descriptors {
163 if desc.primitive_count > self.limits.max_blas_primitive_count {
164 return Err(CreateBlasError::TooManyPrimitives(
165 self.limits.max_blas_primitive_count,
166 desc.primitive_count,
167 ));
168 }
169
170 entries.push(hal::AccelerationStructureAABBs::<dyn hal::DynBuffer> {
171 buffer: Some(self.zero_buffer.as_ref()),
172 offset: 0,
173 count: desc.primitive_count,
174 stride: AABB_GEOMETRY_MIN_STRIDE,
175 flags: desc.flags,
176 });
177 }
178 unsafe {
179 self.raw().get_acceleration_structure_build_sizes(
180 &hal::GetAccelerationStructureBuildSizesDescriptor {
181 entries: &hal::AccelerationStructureEntries::AABBs(entries),
182 flags: blas_desc.flags,
183 },
184 )
185 }
186 }
187 };
188
189 let raw = unsafe {
190 self.raw()
191 .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
192 label: hal_label(blas_desc.label.as_deref(), self.instance_flags),
193 size: size_info.acceleration_structure_size,
194 format: hal::AccelerationStructureFormat::BottomLevel,
195 allow_compaction: blas_desc
196 .flags
197 .contains(wgpu_types::AccelerationStructureFlags::ALLOW_COMPACTION),
198 })
199 }
200 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
201
202 let compaction_buffer = if blas_desc
203 .flags
204 .contains(wgpu_types::AccelerationStructureFlags::ALLOW_COMPACTION)
205 {
206 Some(ManuallyDrop::new(unsafe {
207 self.raw()
208 .create_buffer(&hal::BufferDescriptor {
209 label: hal_label(
210 Some("(wgpu internal) compaction read-back buffer"),
211 self.instance_flags,
212 ),
213 size: size_of::<wgpu_types::BufferAddress>() as wgpu_types::BufferAddress,
214 usage: wgpu_types::BufferUses::ACCELERATION_STRUCTURE_QUERY
215 | wgpu_types::BufferUses::MAP_READ,
216 memory_flags: hal::MemoryFlags::PREFER_COHERENT,
217 })
218 .map_err(DeviceError::from_hal)?
219 .0
220 }))
221 } else {
222 None
223 };
224
225 let handle = unsafe {
226 self.raw()
227 .get_acceleration_structure_device_address(raw.as_ref())
228 };
229
230 Ok(Arc::new(resource::Blas {
231 state: ResourceState::Valid(resource::BlasState {
232 raw: Snatchable::new(raw),
233 }),
234 device: self.clone(),
235 size_info,
236 sizes,
237 flags: blas_desc.flags,
238 update_mode: blas_desc.update_mode,
239 handle,
240 label: blas_desc.label.to_string(),
241 built_index: RwLock::new(rank::BLAS_BUILT_INDEX, None),
242 tracking_data: TrackingData::new(self.tracker_indices.blas_s.clone()),
243 compaction_buffer,
244 compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Idle),
245 }))
246 }
247
248 pub fn create_tlas(
249 self: &Arc<Self>,
250 desc: &resource::TlasDescriptor,
251 ) -> (Arc<resource::Tlas>, Option<CreateTlasError>) {
252 let (tlas, error) = match self.create_tlas_inner(desc) {
253 Ok(tlas) => (tlas, None),
254 Err(e) => (resource::Tlas::invalid(Arc::clone(self), desc), Some(e)),
255 };
256 #[cfg(feature = "trace")]
257 if let Some(trace) = self.trace.lock().as_mut() {
258 trace.add(Action::CreateTlas {
259 id: tlas.to_trace(),
260 desc: desc.clone(),
261 });
262 }
263
264 api_log!("Device::create_tlas -> {:?}", Arc::as_ptr(&tlas));
265
266 (tlas, error)
267 }
268
269 pub(crate) fn create_tlas_inner(
270 self: &Arc<Self>,
271 desc: &resource::TlasDescriptor,
272 ) -> Result<Arc<resource::Tlas>, CreateTlasError> {
273 self.check_is_valid()?;
274 self.require_acceleration_structures()?;
275
276 if desc.max_instances > self.limits.max_tlas_instance_count {
277 return Err(CreateTlasError::TooManyInstances(
278 self.limits.max_tlas_instance_count,
279 desc.max_instances,
280 ));
281 }
282
283 if desc
284 .flags
285 .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
286 {
287 return Err(CreateTlasError::DisallowedFlag(
288 wgt::AccelerationStructureFlags::USE_TRANSFORM,
289 ));
290 }
291
292 if desc
293 .flags
294 .contains(wgt::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN)
295 {
296 self.require_features(Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)?;
297 }
298
299 let size_info = unsafe {
300 self.raw().get_acceleration_structure_build_sizes(
301 &hal::GetAccelerationStructureBuildSizesDescriptor {
302 entries: &hal::AccelerationStructureEntries::Instances(
303 hal::AccelerationStructureInstances {
304 buffer: Some(self.zero_buffer.as_ref()),
305 offset: 0,
306 count: desc.max_instances,
307 },
308 ),
309 flags: desc.flags,
310 },
311 )
312 };
313
314 let raw = unsafe {
315 self.raw()
316 .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
317 label: hal_label(desc.label.as_deref(), self.instance_flags),
318 size: size_info.acceleration_structure_size,
319 format: hal::AccelerationStructureFormat::TopLevel,
320 allow_compaction: false,
321 })
322 }
323 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
324
325 let instance_buffer_size = self
326 .alignments
327 .raw_tlas_instance_size
328 .checked_mul(desc.max_instances.max(1))
329 .expect("max_tlas_instance_count should not allow excessive buffer size");
330 let (instance_buffer, _) = unsafe {
331 self.raw().create_buffer(&hal::BufferDescriptor {
332 label: hal_label(Some("(wgpu-core) instances_buffer"), self.instance_flags),
333 size: u64::from(instance_buffer_size),
334 usage: wgt::BufferUses::COPY_DST
335 | wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT,
336 memory_flags: hal::MemoryFlags::PREFER_COHERENT,
337 })
338 }
339 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
340
341 Ok(Arc::new(resource::Tlas {
342 state: ResourceState::Valid(resource::TlasState {
343 raw: Snatchable::new(raw),
344 instance_buffer,
345 }),
346 device: self.clone(),
347 size_info,
348 flags: desc.flags,
349 update_mode: desc.update_mode,
350 built_index: RwLock::new(rank::TLAS_BUILT_INDEX, None),
351 dependencies: RwLock::new(rank::TLAS_DEPENDENCIES, Vec::new()),
352 label: desc.label.to_string(),
353 max_instance_count: desc.max_instances,
354 tracking_data: TrackingData::new(self.tracker_indices.tlas_s.clone()),
355 max_intersection_index: RwLock::new(rank::TLAS_MAX_INTERSECTION_IDX, None),
356 required_intersection_types: RwLock::new(
357 rank::TLAS_REQUIRED_INTERSECTION_TYPES,
358 Vec::new(),
359 ),
360 }))
361 }
362
363 pub fn create_ray_tracing_pipeline(
364 self: &Arc<Self>,
365 desc: ray_tracing_pipeline::RayTracingPipelineDescriptor,
366 ) -> (
367 Arc<ray_tracing_pipeline::RayTracingPipeline>,
368 Option<CreatePipelineError>,
369 ) {
370 let (ray_tracing_pipeline, error) =
371 match self.create_ray_tracing_pipeline_inner(desc.clone()) {
372 Ok(ray_tracing_pipeline) => (ray_tracing_pipeline, None),
373 Err(error) => (
374 ray_tracing_pipeline::RayTracingPipeline::invalid(
375 self.clone(),
376 desc.label.to_string(),
377 ),
378 Some(error),
379 ),
380 };
381 #[cfg(feature = "trace")]
382 if let Some(ref mut trace) = *self.trace.lock() {
383 use crate::device::trace::IntoTrace;
384 trace.add(Action::CreateRayTracingPipeline {
385 id: ray_tracing_pipeline.to_trace(),
386 desc: desc.to_trace(),
387 });
388 }
389 (ray_tracing_pipeline, error)
390 }
391
392 pub fn create_ray_tracing_pipeline_inner(
393 self: &Arc<Self>,
394 desc: ray_tracing_pipeline::RayTracingPipelineDescriptor,
395 ) -> Result<Arc<ray_tracing_pipeline::RayTracingPipeline>, CreatePipelineError> {
396 use crate::validation;
397
398 self.check_is_valid()?;
399 self.require_features(Features::EXPERIMENTAL_RAY_TRACING_PIPELINES)?;
400
401 let io = validation::StageIo::default();
402
403 let is_auto_layout = desc.layout.is_none();
404
405 let pipeline_layout = match desc.layout {
407 Some(pipeline_layout) => {
408 pipeline_layout.same_device(self)?;
409 Some(pipeline_layout)
410 }
411 None => None,
412 };
413
414 let mut compiler = StageCompiler::new(self, pipeline_layout.clone(), io, None);
415
416 let ray_generation = {
417 let stage = validation::ShaderStageForValidation::RayGeneration;
418
419 compiler.compile(&desc.ray_generation, stage)?
420 };
421
422 let miss = {
423 let stage = validation::ShaderStageForValidation::Miss;
424
425 compiler.compile(&desc.miss, stage)?
426 };
427
428 if desc.intersections.len() > 1 << 24 {
429 return Err(
430 ray_tracing_pipeline::RayTracingPipelineError::TooManyIntersectionGroups(
431 desc.intersections.len(),
432 )
433 .into(),
434 );
435 }
436
437 if desc.max_recursion_depth > self.limits.max_ray_recursion_depth {
438 return Err(
439 ray_tracing_pipeline::RayTracingPipelineError::TooHighRayRecursionDepth(
440 desc.max_recursion_depth,
441 self.limits.max_ray_recursion_depth,
442 )
443 .into(),
444 );
445 }
446
447 let mut intersections = Vec::with_capacity(desc.intersections.len());
448 let mut compiled_intersections = Vec::with_capacity(desc.intersections.len());
449
450 for intersection in &desc.intersections {
451 match intersection {
452 ray_tracing_pipeline::RayTracingIntersectionDescriptor::Triangle {
453 closest_hit,
454 any_hit,
455 } => {
456 let closest_hit = {
457 let stage =
458 validation::ShaderStageForValidation::ClosestHit { triangle: true };
459
460 compiler.compile(closest_hit, stage)?
461 };
462
463 let any_hit = match any_hit {
464 Some(any_hit) => {
465 let stage =
466 validation::ShaderStageForValidation::AnyHit { triangle: true };
467
468 Some(compiler.compile(any_hit, stage)?)
469 }
470 None => None,
471 };
472
473 compiled_intersections.push((closest_hit, any_hit));
474 }
475 }
476 }
477
478 for (closest_hit, any_hit) in compiled_intersections.iter() {
479 intersections.push(hal::RayObjectIntersectionState {
480 closest_hit: closest_hit.to_hal(),
481 any_hit: any_hit.as_ref().map(|any| any.to_hal()),
482 });
483 }
484
485 let StageCompiler { resources, .. } = compiler;
486
487 let pipeline_layout = match resources.layout {
488 validation::BindingLayoutSource::Provided(layout) => layout,
489 validation::BindingLayoutSource::Derived(entries) => {
490 self.create_derived_pipeline_layout(entries, resources.immediates.size())?
491 }
492 };
493
494 let late_sized_buffer_groups = Device::make_late_sized_buffer_groups(
495 &resources.minimum_binding_sizes,
496 &pipeline_layout,
497 );
498
499 let cache = match desc.cache {
500 Some(cache) => {
501 cache.same_device(self)?;
502 Some(cache)
503 }
504 None => None,
505 };
506
507 let raw = {
508 let pipeline_desc = hal::RayTracingPipelineDescriptor {
509 label: desc.label.to_hal(self.instance_flags),
510 layout: pipeline_layout.raw()?,
511 ray_generation: ray_generation.to_hal(),
512 miss: miss.to_hal(),
513 intersection: &intersections,
514 max_recursion_depth: desc.max_recursion_depth,
515 cache: cache.as_ref().map(|it| it.raw()).transpose()?,
516 };
517 unsafe { self.raw().create_ray_tracing_pipeline(&pipeline_desc) }.map_err(|err| {
518 match err {
519 hal::PipelineError::Device(error) => {
520 CreatePipelineError::Device(self.handle_hal_error(error))
521 }
522 hal::PipelineError::Linkage(stage, message) => {
523 CreatePipelineError::Internal { stage, message }
524 }
525 hal::PipelineError::EntryPoint(stage) => CreatePipelineError::Internal {
526 stage: hal::auxil::map_naga_stage(stage),
527 message: ENTRYPOINT_FAILURE_ERROR.to_string(),
528 },
529 hal::PipelineError::PipelineConstants(stage, message) => {
530 CreatePipelineError::PipelineConstants { stage, message }
531 }
532 }
533 })?
534 };
535
536 let (shader_modules, intersection_types) = {
537 let mut shader_modules = Vec::new();
538 shader_modules.push(desc.ray_generation.module.clone());
539 shader_modules.push(desc.miss.module.clone());
540 shader_modules.reserve(desc.intersections.len());
541 let mut intersection_types = Vec::with_capacity(desc.intersections.len());
542 for intersection in &desc.intersections {
543 match intersection {
544 ray_tracing_pipeline::RayTracingIntersectionDescriptor::Triangle {
545 closest_hit,
546 any_hit,
547 } => {
548 shader_modules.push(closest_hit.module.clone());
549 if let Some(any) = any_hit {
550 shader_modules.push(any.module.clone());
551 }
552 intersection_types
553 .push(ray_tracing_pipeline::RayTracingIntersectionType::Triangle);
554 }
555 }
556 }
557 (
558 shader_modules,
559 Arc::from(intersection_types.into_boxed_slice()),
560 )
561 };
562
563 let shader_binding_data = match ray_tracing_pipeline::ShaderBindingData::from_raw_pipeline(
566 self.clone(),
567 raw.as_ref(),
568 desc.intersections.len(),
569 ) {
570 Ok(sbd) => sbd,
571 Err(e) => {
572 unsafe { self.raw().destroy_ray_tracing_pipeline(raw) };
574 return Err(e);
575 }
576 };
577
578 let naga::valid::ImmediateUsage::Valid {
579 slots: immediate_slots_required,
580 size: _,
581 } = resources.immediates
582 else {
583 unreachable!("Immediates exceeding maxImmediateSize should have been rejected");
584 };
585
586 let pipeline = ray_tracing_pipeline::RayTracingPipeline {
587 state: ResourceState::Valid(ray_tracing_pipeline::RayTracingPipelineState {
588 raw: ManuallyDrop::new(raw),
589 layout: pipeline_layout.clone(),
590 _shader_modules: shader_modules,
591 shader_binding_data,
592 intersection_types,
593 }),
594 device: self.clone(),
595 late_sized_buffer_groups,
596 immediate_slots_required,
597 label: desc.label.to_string(),
598 tracking_data: TrackingData::new(self.tracker_indices.ray_tracing_pipelines.clone()),
599 };
600
601 let pipeline = Arc::new(pipeline);
602
603 if is_auto_layout {
604 for bgl in pipeline_layout.bind_group_layouts.iter() {
605 let Some(bgl) = bgl else {
606 continue;
607 };
608
609 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
611 }
612 }
613
614 Ok(pipeline)
615 }
616}