Skip to main content

wgpu_core/device/
ray_tracing.rs

1use alloc::{string::ToString, sync::Arc, vec::Vec};
2use core::mem::{size_of, ManuallyDrop};
3
4use crate::device::stage_compiler::StageCompiler;
5#[cfg(feature = "trace")]
6use crate::device::trace::{Action, IntoTrace};
7use crate::device::{DeviceError, ENTRYPOINT_FAILURE_ERROR};
8use crate::pipeline::CreatePipelineError;
9use crate::ray_tracing_pipeline;
10use crate::resource::{ParentDevice, ResourceState};
11use crate::{
12    api_log,
13    device::Device,
14    hal_label,
15    lock::RwLock,
16    lock::{rank, Mutex},
17    ray_tracing::{CreateBlasError, CreateTlasError},
18    resource,
19    resource::{BlasCompactState, TrackingData},
20    snatch::Snatchable,
21    LabelHelpers,
22};
23use hal::AccelerationStructureTriangleIndices;
24use wgt::{Features, AABB_GEOMETRY_MIN_STRIDE};
25
26impl Device {
27    /// Requires any feature that allows acceleration structures as they can be provided by multiple extensions.
28    pub fn require_acceleration_structures(&self) -> Result<(), crate::device::MissingFeatures> {
29        self.require_features(Features::EXPERIMENTAL_RAY_QUERY)
30            .or_else(|_| self.require_features(Features::EXPERIMENTAL_RAY_TRACING_PIPELINES))
31    }
32
33    pub fn create_blas(
34        self: &Arc<Self>,
35        blas_desc: &resource::BlasDescriptor,
36        sizes: wgt::BlasGeometrySizeDescriptors,
37    ) -> (Arc<resource::Blas>, Option<CreateBlasError>) {
38        #[cfg(feature = "trace")]
39        let trace_sizes = sizes.clone();
40
41        let (blas, error) = match self.create_blas_inner(blas_desc, sizes) {
42            Ok(blas) => (blas, None),
43            Err(err) => (resource::Blas::invalid(self.clone(), blas_desc), Some(err)),
44        };
45
46        #[cfg(feature = "trace")]
47        if let Some(trace) = self.trace.lock().as_mut() {
48            trace.add(Action::CreateBlas {
49                id: blas.to_trace(),
50                desc: blas_desc.clone(),
51                sizes: trace_sizes,
52            });
53        }
54
55        api_log!("Device::create_blas -> {:?}", Arc::as_ptr(&blas));
56        (blas, error)
57    }
58    pub(crate) fn create_blas_inner(
59        self: &Arc<Self>,
60        blas_desc: &resource::BlasDescriptor,
61        sizes: wgt::BlasGeometrySizeDescriptors,
62    ) -> Result<Arc<resource::Blas>, CreateBlasError> {
63        self.check_is_valid()?;
64        self.require_acceleration_structures()?;
65
66        if blas_desc
67            .flags
68            .contains(wgt::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN)
69        {
70            self.require_features(Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)?;
71        }
72
73        let size_info = match &sizes {
74            wgt::BlasGeometrySizeDescriptors::Triangles { descriptors } => {
75                if descriptors.len() as u32 > self.limits.max_blas_geometry_count {
76                    return Err(CreateBlasError::TooManyGeometries(
77                        self.limits.max_blas_geometry_count,
78                        descriptors.len() as u32,
79                    ));
80                }
81
82                let mut entries =
83                    Vec::<hal::AccelerationStructureTriangles<dyn hal::DynBuffer>>::with_capacity(
84                        descriptors.len(),
85                    );
86                for desc in descriptors {
87                    if desc.index_count.is_some() != desc.index_format.is_some() {
88                        return Err(CreateBlasError::MissingIndexData);
89                    }
90                    let indices =
91                        desc.index_count
92                            .map(|count| AccelerationStructureTriangleIndices::<
93                                dyn hal::DynBuffer,
94                            > {
95                                format: desc.index_format.unwrap(),
96                                buffer: Some(self.zero_buffer.as_ref()),
97                                offset: 0,
98                                count,
99                            });
100                    if !self
101                        .features
102                        .allowed_vertex_formats_for_blas()
103                        .contains(&desc.vertex_format)
104                    {
105                        return Err(CreateBlasError::InvalidVertexFormat(
106                            desc.vertex_format,
107                            self.features.allowed_vertex_formats_for_blas(),
108                        ));
109                    }
110
111                    let mut transform = None;
112
113                    if blas_desc
114                        .flags
115                        .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
116                    {
117                        transform = Some(wgpu_hal::AccelerationStructureTriangleTransform {
118                            buffer: self.zero_buffer.as_ref(),
119                            offset: 0,
120                        })
121                    }
122
123                    if desc.vertex_count > self.limits.max_blas_primitive_count {
124                        return Err(CreateBlasError::TooManyPrimitives(
125                            self.limits.max_blas_primitive_count,
126                            desc.vertex_count,
127                        ));
128                    }
129
130                    entries.push(hal::AccelerationStructureTriangles::<dyn hal::DynBuffer> {
131                        vertex_buffer: Some(self.zero_buffer.as_ref()),
132                        vertex_format: desc.vertex_format,
133                        first_vertex: 0,
134                        vertex_count: desc.vertex_count,
135                        vertex_stride: 0,
136                        indices,
137                        transform,
138                        flags: desc.flags,
139                    });
140                }
141                unsafe {
142                    self.raw().get_acceleration_structure_build_sizes(
143                        &hal::GetAccelerationStructureBuildSizesDescriptor {
144                            entries: &hal::AccelerationStructureEntries::Triangles(entries),
145                            flags: blas_desc.flags,
146                        },
147                    )
148                }
149            }
150            wgt::BlasGeometrySizeDescriptors::AABBs { descriptors } => {
151                if descriptors.len() as u32 > self.limits.max_blas_geometry_count {
152                    return Err(CreateBlasError::TooManyGeometries(
153                        self.limits.max_blas_geometry_count,
154                        descriptors.len() as u32,
155                    ));
156                }
157
158                let mut entries =
159                    Vec::<hal::AccelerationStructureAABBs<dyn hal::DynBuffer>>::with_capacity(
160                        descriptors.len(),
161                    );
162                for desc in descriptors {
163                    if desc.primitive_count > self.limits.max_blas_primitive_count {
164                        return Err(CreateBlasError::TooManyPrimitives(
165                            self.limits.max_blas_primitive_count,
166                            desc.primitive_count,
167                        ));
168                    }
169
170                    entries.push(hal::AccelerationStructureAABBs::<dyn hal::DynBuffer> {
171                        buffer: Some(self.zero_buffer.as_ref()),
172                        offset: 0,
173                        count: desc.primitive_count,
174                        stride: AABB_GEOMETRY_MIN_STRIDE,
175                        flags: desc.flags,
176                    });
177                }
178                unsafe {
179                    self.raw().get_acceleration_structure_build_sizes(
180                        &hal::GetAccelerationStructureBuildSizesDescriptor {
181                            entries: &hal::AccelerationStructureEntries::AABBs(entries),
182                            flags: blas_desc.flags,
183                        },
184                    )
185                }
186            }
187        };
188
189        let raw = unsafe {
190            self.raw()
191                .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
192                    label: hal_label(blas_desc.label.as_deref(), self.instance_flags),
193                    size: size_info.acceleration_structure_size,
194                    format: hal::AccelerationStructureFormat::BottomLevel,
195                    allow_compaction: blas_desc
196                        .flags
197                        .contains(wgpu_types::AccelerationStructureFlags::ALLOW_COMPACTION),
198                })
199        }
200        .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
201
202        let compaction_buffer = if blas_desc
203            .flags
204            .contains(wgpu_types::AccelerationStructureFlags::ALLOW_COMPACTION)
205        {
206            Some(ManuallyDrop::new(unsafe {
207                self.raw()
208                    .create_buffer(&hal::BufferDescriptor {
209                        label: hal_label(
210                            Some("(wgpu internal) compaction read-back buffer"),
211                            self.instance_flags,
212                        ),
213                        size: size_of::<wgpu_types::BufferAddress>() as wgpu_types::BufferAddress,
214                        usage: wgpu_types::BufferUses::ACCELERATION_STRUCTURE_QUERY
215                            | wgpu_types::BufferUses::MAP_READ,
216                        memory_flags: hal::MemoryFlags::PREFER_COHERENT,
217                    })
218                    .map_err(DeviceError::from_hal)?
219                    .0
220            }))
221        } else {
222            None
223        };
224
225        let handle = unsafe {
226            self.raw()
227                .get_acceleration_structure_device_address(raw.as_ref())
228        };
229
230        Ok(Arc::new(resource::Blas {
231            state: ResourceState::Valid(resource::BlasState {
232                raw: Snatchable::new(raw),
233            }),
234            device: self.clone(),
235            size_info,
236            sizes,
237            flags: blas_desc.flags,
238            update_mode: blas_desc.update_mode,
239            handle,
240            label: blas_desc.label.to_string(),
241            built_index: RwLock::new(rank::BLAS_BUILT_INDEX, None),
242            tracking_data: TrackingData::new(self.tracker_indices.blas_s.clone()),
243            compaction_buffer,
244            compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Idle),
245        }))
246    }
247
248    pub fn create_tlas(
249        self: &Arc<Self>,
250        desc: &resource::TlasDescriptor,
251    ) -> (Arc<resource::Tlas>, Option<CreateTlasError>) {
252        let (tlas, error) = match self.create_tlas_inner(desc) {
253            Ok(tlas) => (tlas, None),
254            Err(e) => (resource::Tlas::invalid(Arc::clone(self), desc), Some(e)),
255        };
256        #[cfg(feature = "trace")]
257        if let Some(trace) = self.trace.lock().as_mut() {
258            trace.add(Action::CreateTlas {
259                id: tlas.to_trace(),
260                desc: desc.clone(),
261            });
262        }
263
264        api_log!("Device::create_tlas -> {:?}", Arc::as_ptr(&tlas));
265
266        (tlas, error)
267    }
268
269    pub(crate) fn create_tlas_inner(
270        self: &Arc<Self>,
271        desc: &resource::TlasDescriptor,
272    ) -> Result<Arc<resource::Tlas>, CreateTlasError> {
273        self.check_is_valid()?;
274        self.require_acceleration_structures()?;
275
276        if desc.max_instances > self.limits.max_tlas_instance_count {
277            return Err(CreateTlasError::TooManyInstances(
278                self.limits.max_tlas_instance_count,
279                desc.max_instances,
280            ));
281        }
282
283        if desc
284            .flags
285            .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM)
286        {
287            return Err(CreateTlasError::DisallowedFlag(
288                wgt::AccelerationStructureFlags::USE_TRANSFORM,
289            ));
290        }
291
292        if desc
293            .flags
294            .contains(wgt::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN)
295        {
296            self.require_features(Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)?;
297        }
298
299        let size_info = unsafe {
300            self.raw().get_acceleration_structure_build_sizes(
301                &hal::GetAccelerationStructureBuildSizesDescriptor {
302                    entries: &hal::AccelerationStructureEntries::Instances(
303                        hal::AccelerationStructureInstances {
304                            buffer: Some(self.zero_buffer.as_ref()),
305                            offset: 0,
306                            count: desc.max_instances,
307                        },
308                    ),
309                    flags: desc.flags,
310                },
311            )
312        };
313
314        let raw = unsafe {
315            self.raw()
316                .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
317                    label: hal_label(desc.label.as_deref(), self.instance_flags),
318                    size: size_info.acceleration_structure_size,
319                    format: hal::AccelerationStructureFormat::TopLevel,
320                    allow_compaction: false,
321                })
322        }
323        .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
324
325        let instance_buffer_size = self
326            .alignments
327            .raw_tlas_instance_size
328            .checked_mul(desc.max_instances.max(1))
329            .expect("max_tlas_instance_count should not allow excessive buffer size");
330        let (instance_buffer, _) = unsafe {
331            self.raw().create_buffer(&hal::BufferDescriptor {
332                label: hal_label(Some("(wgpu-core) instances_buffer"), self.instance_flags),
333                size: u64::from(instance_buffer_size),
334                usage: wgt::BufferUses::COPY_DST
335                    | wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT,
336                memory_flags: hal::MemoryFlags::PREFER_COHERENT,
337            })
338        }
339        .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
340
341        Ok(Arc::new(resource::Tlas {
342            state: ResourceState::Valid(resource::TlasState {
343                raw: Snatchable::new(raw),
344                instance_buffer,
345            }),
346            device: self.clone(),
347            size_info,
348            flags: desc.flags,
349            update_mode: desc.update_mode,
350            built_index: RwLock::new(rank::TLAS_BUILT_INDEX, None),
351            dependencies: RwLock::new(rank::TLAS_DEPENDENCIES, Vec::new()),
352            label: desc.label.to_string(),
353            max_instance_count: desc.max_instances,
354            tracking_data: TrackingData::new(self.tracker_indices.tlas_s.clone()),
355            max_intersection_index: RwLock::new(rank::TLAS_MAX_INTERSECTION_IDX, None),
356            required_intersection_types: RwLock::new(
357                rank::TLAS_REQUIRED_INTERSECTION_TYPES,
358                Vec::new(),
359            ),
360        }))
361    }
362
363    pub fn create_ray_tracing_pipeline(
364        self: &Arc<Self>,
365        desc: ray_tracing_pipeline::RayTracingPipelineDescriptor,
366    ) -> (
367        Arc<ray_tracing_pipeline::RayTracingPipeline>,
368        Option<CreatePipelineError>,
369    ) {
370        let (ray_tracing_pipeline, error) =
371            match self.create_ray_tracing_pipeline_inner(desc.clone()) {
372                Ok(ray_tracing_pipeline) => (ray_tracing_pipeline, None),
373                Err(error) => (
374                    ray_tracing_pipeline::RayTracingPipeline::invalid(
375                        self.clone(),
376                        desc.label.to_string(),
377                    ),
378                    Some(error),
379                ),
380            };
381        #[cfg(feature = "trace")]
382        if let Some(ref mut trace) = *self.trace.lock() {
383            use crate::device::trace::IntoTrace;
384            trace.add(Action::CreateRayTracingPipeline {
385                id: ray_tracing_pipeline.to_trace(),
386                desc: desc.to_trace(),
387            });
388        }
389        (ray_tracing_pipeline, error)
390    }
391
392    pub fn create_ray_tracing_pipeline_inner(
393        self: &Arc<Self>,
394        desc: ray_tracing_pipeline::RayTracingPipelineDescriptor,
395    ) -> Result<Arc<ray_tracing_pipeline::RayTracingPipeline>, CreatePipelineError> {
396        use crate::validation;
397
398        self.check_is_valid()?;
399        self.require_features(Features::EXPERIMENTAL_RAY_TRACING_PIPELINES)?;
400
401        let io = validation::StageIo::default();
402
403        let is_auto_layout = desc.layout.is_none();
404
405        // Get the pipeline layout from the desc if it is provided.
406        let pipeline_layout = match desc.layout {
407            Some(pipeline_layout) => {
408                pipeline_layout.same_device(self)?;
409                Some(pipeline_layout)
410            }
411            None => None,
412        };
413
414        let mut compiler = StageCompiler::new(self, pipeline_layout.clone(), io, None);
415
416        let ray_generation = {
417            let stage = validation::ShaderStageForValidation::RayGeneration;
418
419            compiler.compile(&desc.ray_generation, stage)?
420        };
421
422        let miss = {
423            let stage = validation::ShaderStageForValidation::Miss;
424
425            compiler.compile(&desc.miss, stage)?
426        };
427
428        if desc.intersections.len() > 1 << 24 {
429            return Err(
430                ray_tracing_pipeline::RayTracingPipelineError::TooManyIntersectionGroups(
431                    desc.intersections.len(),
432                )
433                .into(),
434            );
435        }
436
437        if desc.max_recursion_depth > self.limits.max_ray_recursion_depth {
438            return Err(
439                ray_tracing_pipeline::RayTracingPipelineError::TooHighRayRecursionDepth(
440                    desc.max_recursion_depth,
441                    self.limits.max_ray_recursion_depth,
442                )
443                .into(),
444            );
445        }
446
447        let mut intersections = Vec::with_capacity(desc.intersections.len());
448        let mut compiled_intersections = Vec::with_capacity(desc.intersections.len());
449
450        for intersection in &desc.intersections {
451            match intersection {
452                ray_tracing_pipeline::RayTracingIntersectionDescriptor::Triangle {
453                    closest_hit,
454                    any_hit,
455                } => {
456                    let closest_hit = {
457                        let stage =
458                            validation::ShaderStageForValidation::ClosestHit { triangle: true };
459
460                        compiler.compile(closest_hit, stage)?
461                    };
462
463                    let any_hit = match any_hit {
464                        Some(any_hit) => {
465                            let stage =
466                                validation::ShaderStageForValidation::AnyHit { triangle: true };
467
468                            Some(compiler.compile(any_hit, stage)?)
469                        }
470                        None => None,
471                    };
472
473                    compiled_intersections.push((closest_hit, any_hit));
474                }
475            }
476        }
477
478        for (closest_hit, any_hit) in compiled_intersections.iter() {
479            intersections.push(hal::RayObjectIntersectionState {
480                closest_hit: closest_hit.to_hal(),
481                any_hit: any_hit.as_ref().map(|any| any.to_hal()),
482            });
483        }
484
485        let StageCompiler { resources, .. } = compiler;
486
487        let pipeline_layout = match resources.layout {
488            validation::BindingLayoutSource::Provided(layout) => layout,
489            validation::BindingLayoutSource::Derived(entries) => {
490                self.create_derived_pipeline_layout(entries, resources.immediates.size())?
491            }
492        };
493
494        let late_sized_buffer_groups = Device::make_late_sized_buffer_groups(
495            &resources.minimum_binding_sizes,
496            &pipeline_layout,
497        );
498
499        let cache = match desc.cache {
500            Some(cache) => {
501                cache.same_device(self)?;
502                Some(cache)
503            }
504            None => None,
505        };
506
507        let raw = {
508            let pipeline_desc = hal::RayTracingPipelineDescriptor {
509                label: desc.label.to_hal(self.instance_flags),
510                layout: pipeline_layout.raw()?,
511                ray_generation: ray_generation.to_hal(),
512                miss: miss.to_hal(),
513                intersection: &intersections,
514                max_recursion_depth: desc.max_recursion_depth,
515                cache: cache.as_ref().map(|it| it.raw()).transpose()?,
516            };
517            unsafe { self.raw().create_ray_tracing_pipeline(&pipeline_desc) }.map_err(|err| {
518                match err {
519                    hal::PipelineError::Device(error) => {
520                        CreatePipelineError::Device(self.handle_hal_error(error))
521                    }
522                    hal::PipelineError::Linkage(stage, message) => {
523                        CreatePipelineError::Internal { stage, message }
524                    }
525                    hal::PipelineError::EntryPoint(stage) => CreatePipelineError::Internal {
526                        stage: hal::auxil::map_naga_stage(stage),
527                        message: ENTRYPOINT_FAILURE_ERROR.to_string(),
528                    },
529                    hal::PipelineError::PipelineConstants(stage, message) => {
530                        CreatePipelineError::PipelineConstants { stage, message }
531                    }
532                }
533            })?
534        };
535
536        let (shader_modules, intersection_types) = {
537            let mut shader_modules = Vec::new();
538            shader_modules.push(desc.ray_generation.module.clone());
539            shader_modules.push(desc.miss.module.clone());
540            shader_modules.reserve(desc.intersections.len());
541            let mut intersection_types = Vec::with_capacity(desc.intersections.len());
542            for intersection in &desc.intersections {
543                match intersection {
544                    ray_tracing_pipeline::RayTracingIntersectionDescriptor::Triangle {
545                        closest_hit,
546                        any_hit,
547                    } => {
548                        shader_modules.push(closest_hit.module.clone());
549                        if let Some(any) = any_hit {
550                            shader_modules.push(any.module.clone());
551                        }
552                        intersection_types
553                            .push(ray_tracing_pipeline::RayTracingIntersectionType::Triangle);
554                    }
555                }
556            }
557            (
558                shader_modules,
559                Arc::from(intersection_types.into_boxed_slice()),
560            )
561        };
562
563        // Won't panic because the length of `desc.intersections` is required to be below 2^24
564        // (see `CreateRayTracingPipelineError::TooManyIntersectionGroups`)
565        let shader_binding_data = match ray_tracing_pipeline::ShaderBindingData::from_raw_pipeline(
566            self.clone(),
567            raw.as_ref(),
568            desc.intersections.len(),
569        ) {
570            Ok(sbd) => sbd,
571            Err(e) => {
572                // We need to destroy the raw ray tracing pipeline first.
573                unsafe { self.raw().destroy_ray_tracing_pipeline(raw) };
574                return Err(e);
575            }
576        };
577
578        let naga::valid::ImmediateUsage::Valid {
579            slots: immediate_slots_required,
580            size: _,
581        } = resources.immediates
582        else {
583            unreachable!("Immediates exceeding maxImmediateSize should have been rejected");
584        };
585
586        let pipeline = ray_tracing_pipeline::RayTracingPipeline {
587            state: ResourceState::Valid(ray_tracing_pipeline::RayTracingPipelineState {
588                raw: ManuallyDrop::new(raw),
589                layout: pipeline_layout.clone(),
590                _shader_modules: shader_modules,
591                shader_binding_data,
592                intersection_types,
593            }),
594            device: self.clone(),
595            late_sized_buffer_groups,
596            immediate_slots_required,
597            label: desc.label.to_string(),
598            tracking_data: TrackingData::new(self.tracker_indices.ray_tracing_pipelines.clone()),
599        };
600
601        let pipeline = Arc::new(pipeline);
602
603        if is_auto_layout {
604            for bgl in pipeline_layout.bind_group_layouts.iter() {
605                let Some(bgl) = bgl else {
606                    continue;
607                };
608
609                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the result.
610                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
611            }
612        }
613
614        Ok(pipeline)
615    }
616}