Skip to main content

naga/back/spv/ray/
hit_object.rs

1/*!
2Generating SPIR-V for ray tracing invocation reordering (hit objects).
3
4This implements `SPV_EXT_shader_invocation_reorder`, which is exposed in WGSL by
5the `wgpu_ray_tracing_invocation_reorder` enable-extension.
6*/
7
8use super::write_ray_flags_contains_flags;
9use crate::back::spv::{
10    Block, BlockContext, Instruction, LocalType, LookupRaytracingFunction, NumericType, Writer,
11    WriterFlags,
12};
13use crate::back::RayQueryPoint;
14
15/// Values of `OpHitObjectGetHitKindEXT` greater than or equal to this indicate a
16/// triangle hit; smaller values come from intersection shaders.
17///
18/// `0xFE` is a front-facing triangle and `0xFF` is a back-facing one.
19const HIT_KIND_FRONT_FACING_TRIANGLE: u32 = 0xFE;
20
21impl Writer {
22    /// Write a helper function that traces a ray into a hit object.
23    ///
24    /// The generated function takes a pointer to the hit object, the
25    /// acceleration structure and a `RayDesc`. The payload is baked into the
26    /// function, exactly like [`Writer::write_trace_ray`].
27    fn write_hit_object_trace_ray(
28        &mut self,
29        ir_module: &crate::Module,
30        payload: crate::Handle<crate::GlobalVariable>,
31    ) -> Result<spirv::Word, super::super::Error> {
32        if let Some(&word) = self
33            .ray_tracing_functions
34            .get(&LookupRaytracingFunction::HitObjectTraceRay { payload })
35        {
36            return Ok(word);
37        }
38
39        self.require_hit_objects()?;
40
41        let hit_object_pointer_type_id = self.get_hit_object_pointer_id();
42        let acceleration_structure_type_id =
43            self.get_localtype_id(LocalType::AccelerationStructure);
44        let ray_desc_type_id = self.get_handle_type_id(
45            ir_module
46                .special_types
47                .ray_desc
48                .expect("ray desc should be set if `hitObjectTraceRay` is called"),
49        );
50
51        let (func_id, mut function, arg_ids) = self.write_function_signature(
52            &[
53                hit_object_pointer_type_id,
54                acceleration_structure_type_id,
55                ray_desc_type_id,
56            ],
57            self.void_type,
58        );
59
60        let hit_object_id = arg_ids[0];
61        let acceleration_structure_id = arg_ids[1];
62        let desc_id = arg_ids[2];
63        let payload_id = self.global_variables[payload].access_id;
64
65        let label_id = self.id_gen.next();
66        let mut block = Block::new(label_id);
67
68        let super::ExtractedRayDesc {
69            ray_flags_id,
70            cull_mask_id,
71            tmin_id,
72            tmax_id,
73            ray_origin_id,
74            ray_dir_id,
75            valid_id,
76        } = self.write_extract_ray_desc(&mut block, desc_id, self.trace_ray_argument_validation);
77
78        let merge_label_id = self.id_gen.next();
79        let merge_block = Block::new(merge_label_id);
80
81        // NOTE: this block will be unreachable if trace ray validation is disabled.
82        let invalid_label_id = self.id_gen.next();
83        let mut invalid_block = Block::new(invalid_label_id);
84
85        let valid_label_id = self.id_gen.next();
86        let mut valid_block = Block::new(valid_label_id);
87
88        match valid_id {
89            Some(all_valid_id) => {
90                block.body.push(Instruction::selection_merge(
91                    merge_label_id,
92                    spirv::SelectionControl::NONE,
93                ));
94                function.consume(
95                    block,
96                    Instruction::branch_conditional(all_valid_id, valid_label_id, invalid_label_id),
97                );
98            }
99            None => {
100                function.consume(block, Instruction::branch(valid_label_id));
101            }
102        }
103
104        let zero = self.get_constant_scalar(crate::Literal::U32(0));
105
106        valid_block.body.push(Instruction::hit_object_trace_ray(
107            hit_object_id,
108            acceleration_structure_id,
109            ray_flags_id,
110            cull_mask_id,
111            zero,
112            zero,
113            zero,
114            ray_origin_id,
115            tmin_id,
116            ray_dir_id,
117            tmax_id,
118            payload_id,
119        ));
120
121        function.consume(valid_block, Instruction::branch(merge_label_id));
122
123        if self.flags.contains(WriterFlags::PRINT_ON_TRACE_RAYS_FAIL) {
124            self.write_debug_printf(
125                &mut invalid_block,
126                "Naga ignored invalid arguments to hitObjectTraceRay with flags: %u t_min: %f t_max: %f origin: %v3f dir: %v3f",
127                &[
128                    ray_flags_id,
129                    tmin_id,
130                    tmax_id,
131                    ray_origin_id,
132                    ray_dir_id,
133                ],
134            );
135        }
136
137        // Leave the hit object in a well-defined state even when we skipped the
138        // trace, so that later queries on it are not reading uninitialized data.
139        invalid_block
140            .body
141            .push(Instruction::hit_object_record_empty(hit_object_id));
142
143        function.consume(invalid_block, Instruction::branch(merge_label_id));
144
145        function.consume(merge_block, Instruction::return_void());
146
147        function.to_words(&mut self.logical_layout.function_definitions);
148
149        self.ray_tracing_functions.insert(
150            LookupRaytracingFunction::HitObjectTraceRay { payload },
151            func_id,
152        );
153
154        Ok(func_id)
155    }
156
157    /// Write a helper function that records a miss in a hit object.
158    ///
159    /// The generated function takes a pointer to the hit object and a
160    /// `RayDesc`. The descriptor's cull mask is unused, and the miss index is
161    /// always zero.
162    fn write_hit_object_record_miss(
163        &mut self,
164        ir_module: &crate::Module,
165    ) -> Result<spirv::Word, super::super::Error> {
166        if let Some(&word) = self
167            .ray_tracing_functions
168            .get(&LookupRaytracingFunction::HitObjectRecordMiss)
169        {
170            return Ok(word);
171        }
172
173        self.require_hit_objects()?;
174
175        let hit_object_pointer_type_id = self.get_hit_object_pointer_id();
176        let ray_desc_type_id = self.get_handle_type_id(
177            ir_module
178                .special_types
179                .ray_desc
180                .expect("ray desc should be set if `hitObjectRecordMiss` is called"),
181        );
182
183        let (func_id, mut function, arg_ids) = self.write_function_signature(
184            &[hit_object_pointer_type_id, ray_desc_type_id],
185            self.void_type,
186        );
187
188        let hit_object_id = arg_ids[0];
189        let desc_id = arg_ids[1];
190
191        let label_id = self.id_gen.next();
192        let mut block = Block::new(label_id);
193
194        // Vulkan has no runtime requirements on the ray operands of
195        // `OpHitObjectRecordMissEXT`, so the ray is not validated.
196        let super::ExtractedRayDesc {
197            ray_flags_id,
198            cull_mask_id: _,
199            tmin_id,
200            tmax_id,
201            ray_origin_id,
202            ray_dir_id,
203            valid_id: _,
204        } = self.write_extract_ray_desc(&mut block, desc_id, false);
205
206        let zero = self.get_constant_scalar(crate::Literal::U32(0));
207
208        block.body.push(Instruction::hit_object_record_miss(
209            hit_object_id,
210            ray_flags_id,
211            zero,
212            ray_origin_id,
213            tmin_id,
214            ray_dir_id,
215            tmax_id,
216        ));
217
218        function.consume(block, Instruction::return_void());
219        function.to_words(&mut self.logical_layout.function_definitions);
220
221        self.ray_tracing_functions
222            .insert(LookupRaytracingFunction::HitObjectRecordMiss, func_id);
223
224        Ok(func_id)
225    }
226
227    /// Write a helper function that records a ray query's committed
228    /// intersection in a hit object.
229    ///
230    /// The generated function takes a pointer to the hit object, a pointer to
231    /// the ray query, a pointer to that query's initialization tracker, and a
232    /// pointer to that query's `t_max` tracker.
233    ///
234    /// `OpRayQueryGetIntersection*` may only be used on a query that has
235    /// finished traversal, so the tracker is checked first exactly as
236    /// [`Writer::write_ray_query_get_intersection_function`] does for committed
237    /// intersections; if the query is not usable the hit object is recorded as
238    /// empty instead.
239    ///
240    /// `OpHitObjectRecordFromQueryEXT` is only defined for a query whose
241    /// committed intersection is a hit. When the query has no committed
242    /// intersection we instead record a miss with `OpHitObjectRecordMissEXT`,
243    /// using the ray the query was initialized with and a miss index of zero,
244    /// which is what `hitObjectTraceRay` would have recorded.
245    ///
246    /// For a hit, the attribute variable is the source of the recorded
247    /// attributes when the committed intersection is procedural (an AABB), so
248    /// it is zeroed before recording; for triangles the barycentrics come from
249    /// the query. The `Hit Kind` operand is omitted because `rspirv`, which our
250    /// tests use to disassemble the output, doesn't support it yet.
251    fn write_hit_object_record_from_query(&mut self) -> Result<spirv::Word, super::super::Error> {
252        if let Some(&word) = self
253            .ray_tracing_functions
254            .get(&LookupRaytracingFunction::HitObjectRecordFromQuery)
255        {
256            return Ok(word);
257        }
258
259        self.require_hit_objects()?;
260
261        let hit_object_pointer_type_id = self.get_hit_object_pointer_id();
262        let ray_query_pointer_type_id = self.get_ray_query_pointer_id();
263        let u32_type_id = self.get_u32_type_id();
264        let u32_pointer_type_id =
265            self.get_pointer_type_id(u32_type_id, spirv::StorageClass::Function);
266        let f32_type_id = self.get_f32_type_id();
267        let f32_pointer_type_id =
268            self.get_pointer_type_id(f32_type_id, spirv::StorageClass::Function);
269        let vec3_type_id = self.get_numeric_type_id(NumericType::Vector {
270            size: crate::VectorSize::Tri,
271            scalar: crate::Scalar::F32,
272        });
273        let barycentrics_type_id = self.get_numeric_type_id(NumericType::Vector {
274            size: crate::VectorSize::Bi,
275            scalar: crate::Scalar::F32,
276        });
277        let bool_type_id = self.get_bool_type_id();
278
279        let attribute_var_id = self.get_hit_object_attribute_var();
280
281        let (func_id, mut function, arg_ids) = self.write_function_signature(
282            &[
283                hit_object_pointer_type_id,
284                ray_query_pointer_type_id,
285                u32_pointer_type_id,
286                f32_pointer_type_id,
287            ],
288            self.void_type,
289        );
290
291        let hit_object_id = arg_ids[0];
292        let query_id = arg_ids[1];
293        let tracker_id = arg_ids[2];
294        let t_max_tracker_id = arg_ids[3];
295
296        let entry_label_id = self.id_gen.next();
297        let mut entry_block = Block::new(entry_label_id);
298
299        let loaded_tracker_id = self.id_gen.next();
300        entry_block.body.push(Instruction::load(
301            u32_type_id,
302            loaded_tracker_id,
303            tracker_id,
304            None,
305        ));
306        let finished_traversal_id = write_ray_flags_contains_flags(
307            self,
308            &mut entry_block,
309            loaded_tracker_id,
310            RayQueryPoint::FINISHED_TRAVERSAL.bits(),
311        );
312
313        let merge_label_id = self.id_gen.next();
314        let merge_block = Block::new(merge_label_id);
315
316        let unusable_label_id = self.id_gen.next();
317        let mut unusable_block = Block::new(unusable_label_id);
318
319        let usable_label_id = self.id_gen.next();
320        let mut usable_block = Block::new(usable_label_id);
321
322        entry_block.body.push(Instruction::selection_merge(
323            merge_label_id,
324            spirv::SelectionControl::NONE,
325        ));
326        function.consume(
327            entry_block,
328            Instruction::branch_conditional(
329                finished_traversal_id,
330                usable_label_id,
331                unusable_label_id,
332            ),
333        );
334
335        let committed_id = self.get_constant_scalar(crate::Literal::U32(
336            spirv::RayQueryIntersection::RayQueryCommittedIntersectionKHR as _,
337        ));
338        let kind_id = self.id_gen.next();
339        usable_block
340            .body
341            .push(Instruction::ray_query_get_intersection(
342                spirv::Op::RayQueryGetIntersectionTypeKHR,
343                u32_type_id,
344                kind_id,
345                query_id,
346                committed_id,
347            ));
348        let none_id = self.get_constant_scalar(crate::Literal::U32(
349            spirv::RayQueryCommittedIntersectionType::RayQueryCommittedIntersectionNoneKHR as _,
350        ));
351        let is_hit_id = self.id_gen.next();
352        usable_block.body.push(Instruction::binary(
353            spirv::Op::INotEqual,
354            bool_type_id,
355            is_hit_id,
356            kind_id,
357            none_id,
358        ));
359
360        let hit_label_id = self.id_gen.next();
361        let mut hit_block = Block::new(hit_label_id);
362
363        let miss_label_id = self.id_gen.next();
364        let mut miss_block = Block::new(miss_label_id);
365
366        let recorded_label_id = self.id_gen.next();
367        let recorded_block = Block::new(recorded_label_id);
368
369        usable_block.body.push(Instruction::selection_merge(
370            recorded_label_id,
371            spirv::SelectionControl::NONE,
372        ));
373        function.consume(
374            usable_block,
375            Instruction::branch_conditional(is_hit_id, hit_label_id, miss_label_id),
376        );
377
378        let zero_id = self.get_constant_scalar(crate::Literal::U32(0));
379
380        // A hit. Like `traceRay`, the shader binding table offset and stride
381        // are zero, so the hit group is just the instance's record offset,
382        // which is only defined when there is a committed intersection.
383        let hit_sbt_id = self.id_gen.next();
384        hit_block.body.push(Instruction::ray_query_get_intersection(
385            spirv::Op::RayQueryGetIntersectionInstanceShaderBindingTableRecordOffsetKHR,
386            u32_type_id,
387            hit_sbt_id,
388            query_id,
389            committed_id,
390        ));
391        // For procedural hits the attribute variable is the source of the
392        // recorded attributes, so give it a defined value. For triangle hits
393        // it is ignored and the barycentrics come from the query.
394        let zero_barycentrics_id = self.get_constant_null(barycentrics_type_id);
395        hit_block.body.push(Instruction::store(
396            attribute_var_id,
397            zero_barycentrics_id,
398            None,
399        ));
400        // Revision 3 of the extension requires a `Hit Kind` operand for
401        // procedural hits. We cannot emit one yet: the `rspirv` grammar we
402        // disassemble with in tests predates that revision and rejects the
403        // extra operand. The specification's issue 1 notes that, for
404        // compatibility with such SPIR-V, implementations should accept the
405        // instruction for AABB intersections without a hit kind.
406        //
407        // TODO: pass `Some(zero_id)` once `rspirv` ships a grammar that knows
408        // about the operand.
409        hit_block
410            .body
411            .push(Instruction::hit_object_record_from_query(
412                hit_object_id,
413                query_id,
414                hit_sbt_id,
415                attribute_var_id,
416                None,
417            ));
418        function.consume(hit_block, Instruction::branch(recorded_label_id));
419
420        // No committed intersection: record a miss along the query's ray.
421        let ray_flags_id = self.id_gen.next();
422        miss_block.body.push(Instruction::ray_query_get(
423            spirv::Op::RayQueryGetRayFlagsKHR,
424            u32_type_id,
425            ray_flags_id,
426            query_id,
427        ));
428        let ray_origin_id = self.id_gen.next();
429        miss_block.body.push(Instruction::ray_query_get(
430            spirv::Op::RayQueryGetWorldRayOriginKHR,
431            vec3_type_id,
432            ray_origin_id,
433            query_id,
434        ));
435        let tmin_id = self.id_gen.next();
436        miss_block.body.push(Instruction::ray_query_get_t_min(
437            f32_type_id,
438            tmin_id,
439            query_id,
440        ));
441        let ray_dir_id = self.id_gen.next();
442        miss_block.body.push(Instruction::ray_query_get(
443            spirv::Op::RayQueryGetWorldRayDirectionKHR,
444            vec3_type_id,
445            ray_dir_id,
446            query_id,
447        ));
448        // SPIR-V has no getter for the `t_max` a query was initialized with,
449        // so use the value the initialization tracked for us.
450        let tmax_id = self.id_gen.next();
451        miss_block.body.push(Instruction::load(
452            f32_type_id,
453            tmax_id,
454            t_max_tracker_id,
455            None,
456        ));
457        miss_block.body.push(Instruction::hit_object_record_miss(
458            hit_object_id,
459            ray_flags_id,
460            zero_id,
461            ray_origin_id,
462            tmin_id,
463            ray_dir_id,
464            tmax_id,
465        ));
466        function.consume(miss_block, Instruction::branch(recorded_label_id));
467
468        function.consume(recorded_block, Instruction::branch(merge_label_id));
469
470        // A query that has not finished traversal has no committed
471        // intersection to read, so leave the hit object in a well-defined
472        // state instead.
473        unusable_block
474            .body
475            .push(Instruction::hit_object_record_empty(hit_object_id));
476        function.consume(unusable_block, Instruction::branch(merge_label_id));
477
478        function.consume(merge_block, Instruction::return_void());
479        function.to_words(&mut self.logical_layout.function_definitions);
480
481        self.ray_tracing_functions
482            .insert(LookupRaytracingFunction::HitObjectRecordFromQuery, func_id);
483
484        Ok(func_id)
485    }
486
487    /// Write a helper function that runs the closest hit or miss shader
488    /// recorded in a hit object.
489    ///
490    /// Like [`Writer::write_trace_ray`], the payload is baked into the function.
491    fn write_hit_object_execute_shader(
492        &mut self,
493        payload: crate::Handle<crate::GlobalVariable>,
494    ) -> Result<spirv::Word, super::super::Error> {
495        if let Some(&word) = self
496            .ray_tracing_functions
497            .get(&LookupRaytracingFunction::HitObjectExecuteShader { payload })
498        {
499            return Ok(word);
500        }
501
502        self.require_hit_objects()?;
503
504        let hit_object_pointer_type_id = self.get_hit_object_pointer_id();
505
506        let (func_id, mut function, arg_ids) =
507            self.write_function_signature(&[hit_object_pointer_type_id], self.void_type);
508
509        let hit_object_id = arg_ids[0];
510        let payload_id = self.global_variables[payload].access_id;
511
512        let label_id = self.id_gen.next();
513        let mut block = Block::new(label_id);
514
515        block.body.push(Instruction::hit_object_execute_shader(
516            hit_object_id,
517            payload_id,
518        ));
519
520        function.consume(block, Instruction::return_void());
521        function.to_words(&mut self.logical_layout.function_definitions);
522
523        self.ray_tracing_functions.insert(
524            LookupRaytracingFunction::HitObjectExecuteShader { payload },
525            func_id,
526        );
527
528        Ok(func_id)
529    }
530
531    /// Write a helper function that builds a `RayIntersection` value out of a
532    /// hit object.
533    ///
534    /// The member indices and types used here must match
535    /// [`Module::generate_ray_intersection_type`].
536    ///
537    /// Note that the hit object getters may only be executed when the hit
538    /// object actually records a hit, and `OpHitObjectGetAttributesEXT` only
539    /// when that hit is a triangle. The generated function therefore keeps
540    /// those getters inside guarded blocks and merges the results with `OpPhi`,
541    /// rather than accumulating them into a local variable: a `Function`
542    /// storage class variable of the `RayIntersection` struct type would carry
543    /// that struct's explicit layout decorations, which Vulkan forbids
544    /// (`VUID-StandaloneSpirv-None-10684`).
545    ///
546    /// [`Module::generate_ray_intersection_type`]: crate::Module::generate_ray_intersection_type
547    pub(in super::super) fn write_hit_object_get_intersection_function(
548        &mut self,
549        ir_module: &crate::Module,
550    ) -> Result<spirv::Word, super::super::Error> {
551        if let Some(&word) = self
552            .ray_tracing_functions
553            .get(&LookupRaytracingFunction::HitObjectGetIntersection)
554        {
555            return Ok(word);
556        }
557
558        self.require_hit_objects()?;
559
560        let ray_intersection = ir_module
561            .special_types
562            .ray_intersection
563            .expect("ray intersection should be set if `hitObjectGetIntersection` is called");
564        let intersection_type_id = self.get_handle_type_id(ray_intersection);
565
566        let flag_type_id = self.get_u32_type_id();
567
568        let transform_type_id = self.get_numeric_type_id(NumericType::Matrix {
569            columns: crate::VectorSize::Quad,
570            rows: crate::VectorSize::Tri,
571            scalar: crate::Scalar::F32,
572        });
573
574        let barycentrics_type_id = self.get_numeric_type_id(NumericType::Vector {
575            size: crate::VectorSize::Bi,
576            scalar: crate::Scalar::F32,
577        });
578
579        let bool_type_id = self.get_bool_type_id();
580
581        let scalar_type_id = self.get_f32_type_id();
582
583        let attribute_var_id = self.get_hit_object_attribute_var();
584
585        let argument_type_id = self.get_hit_object_pointer_id();
586
587        let (func_id, mut function, arg_ids) =
588            self.write_function_signature(&[argument_type_id], intersection_type_id);
589
590        let hit_object_id = arg_ids[0];
591
592        let entry_label_id = self.id_gen.next();
593        let mut block = Block::new(entry_label_id);
594
595        let is_hit_id = self.id_gen.next();
596        block.body.push(Instruction::hit_object_get(
597            spirv::Op::HitObjectIsHitEXT,
598            bool_type_id,
599            is_hit_id,
600            hit_object_id,
601        ));
602
603        let hit_label_id = self.id_gen.next();
604        let mut hit_block = Block::new(hit_label_id);
605
606        let final_label_id = self.id_gen.next();
607        let mut final_block = Block::new(final_label_id);
608
609        block.body.push(Instruction::selection_merge(
610            final_label_id,
611            spirv::SelectionControl::NONE,
612        ));
613        function.consume(
614            block,
615            Instruction::branch_conditional(is_hit_id, hit_label_id, final_label_id),
616        );
617
618        // `kind`: the hit kind tells us whether this was a triangle or a
619        // generated (procedural) intersection.
620        let hit_kind_id = self.id_gen.next();
621        hit_block.body.push(Instruction::hit_object_get(
622            spirv::Op::HitObjectGetHitKindEXT,
623            flag_type_id,
624            hit_kind_id,
625            hit_object_id,
626        ));
627
628        let front_facing_triangle_id =
629            self.get_constant_scalar(crate::Literal::U32(HIT_KIND_FRONT_FACING_TRIANGLE));
630
631        let is_triangle_id = self.id_gen.next();
632        hit_block.body.push(Instruction::binary(
633            spirv::Op::UGreaterThanEqual,
634            bool_type_id,
635            is_triangle_id,
636            hit_kind_id,
637            front_facing_triangle_id,
638        ));
639
640        let triangle_kind_id = self.get_constant_scalar(crate::Literal::U32(
641            crate::RayQueryIntersection::Triangle as _,
642        ));
643        let generated_kind_id = self.get_constant_scalar(crate::Literal::U32(
644            crate::RayQueryIntersection::Generated as _,
645        ));
646        let kind_id = self.id_gen.next();
647        hit_block.body.push(Instruction::select(
648            flag_type_id,
649            kind_id,
650            is_triangle_id,
651            triangle_kind_id,
652            generated_kind_id,
653        ));
654
655        let front_face_id = self.id_gen.next();
656        hit_block.body.push(Instruction::binary(
657            spirv::Op::IEqual,
658            bool_type_id,
659            front_face_id,
660            hit_kind_id,
661            front_facing_triangle_id,
662        ));
663
664        let t_id = self.id_gen.next();
665        hit_block.body.push(Instruction::hit_object_get(
666            spirv::Op::HitObjectGetRayTMaxEXT,
667            scalar_type_id,
668            t_id,
669            hit_object_id,
670        ));
671
672        // The plain `u32` members.
673        let mut simple_getter = |writer: &mut Self, op| {
674            let value_id = writer.id_gen.next();
675            hit_block.body.push(Instruction::hit_object_get(
676                op,
677                flag_type_id,
678                value_id,
679                hit_object_id,
680            ));
681            value_id
682        };
683        let instance_custom_data_id =
684            simple_getter(self, spirv::Op::HitObjectGetInstanceCustomIndexEXT);
685        let instance_index_id = simple_getter(self, spirv::Op::HitObjectGetInstanceIdEXT);
686        // Ray queries report the instance's shader binding table record
687        // offset. The record index is equal to it only because naga always
688        // traces with a shader binding table offset and stride of zero.
689        let sbt_record_offset_id = simple_getter(
690            self,
691            spirv::Op::HitObjectGetShaderBindingTableRecordIndexEXT,
692        );
693        let geometry_index_id = simple_getter(self, spirv::Op::HitObjectGetGeometryIndexEXT);
694        let primitive_index_id = simple_getter(self, spirv::Op::HitObjectGetPrimitiveIndexEXT);
695
696        let object_to_world_id = self.id_gen.next();
697        hit_block.body.push(Instruction::hit_object_get(
698            spirv::Op::HitObjectGetObjectToWorldEXT,
699            transform_type_id,
700            object_to_world_id,
701            hit_object_id,
702        ));
703        let world_to_object_id = self.id_gen.next();
704        hit_block.body.push(Instruction::hit_object_get(
705            spirv::Op::HitObjectGetWorldToObjectEXT,
706            transform_type_id,
707            world_to_object_id,
708            hit_object_id,
709        ));
710
711        // Barycentrics are only meaningful for triangle hits; for anything else
712        // they stay at zero.
713        let triangle_label_id = self.id_gen.next();
714        let mut triangle_block = Block::new(triangle_label_id);
715
716        let triangle_merge_label_id = self.id_gen.next();
717        let mut triangle_merge_block = Block::new(triangle_merge_label_id);
718
719        hit_block.body.push(Instruction::selection_merge(
720            triangle_merge_label_id,
721            spirv::SelectionControl::NONE,
722        ));
723        function.consume(
724            hit_block,
725            Instruction::branch_conditional(
726                is_triangle_id,
727                triangle_label_id,
728                triangle_merge_label_id,
729            ),
730        );
731
732        triangle_block
733            .body
734            .push(Instruction::hit_object_get_attributes(
735                hit_object_id,
736                attribute_var_id,
737            ));
738        let triangle_barycentrics_id = self.id_gen.next();
739        triangle_block.body.push(Instruction::load(
740            barycentrics_type_id,
741            triangle_barycentrics_id,
742            attribute_var_id,
743            None,
744        ));
745
746        function.consume(triangle_block, Instruction::branch(triangle_merge_label_id));
747
748        // `OpPhi` must come first in the block, before the composite that uses it.
749        let zero_barycentrics_id = self.get_constant_null(barycentrics_type_id);
750        let barycentrics_id = self.id_gen.next();
751        triangle_merge_block.body.push(Instruction::phi(
752            barycentrics_type_id,
753            barycentrics_id,
754            &[
755                (triangle_barycentrics_id, triangle_label_id),
756                (zero_barycentrics_id, hit_label_id),
757            ],
758        ));
759
760        let intersection_id = self.id_gen.next();
761        triangle_merge_block
762            .body
763            .push(Instruction::composite_construct(
764                intersection_type_id,
765                intersection_id,
766                &[
767                    kind_id,
768                    t_id,
769                    instance_custom_data_id,
770                    instance_index_id,
771                    sbt_record_offset_id,
772                    geometry_index_id,
773                    primitive_index_id,
774                    barycentrics_id,
775                    front_face_id,
776                    object_to_world_id,
777                    world_to_object_id,
778                ],
779            ));
780
781        function.consume(triangle_merge_block, Instruction::branch(final_label_id));
782
783        let blank_intersection_id = self.get_constant_null(intersection_type_id);
784        let result_id = self.id_gen.next();
785        final_block.body.push(Instruction::phi(
786            intersection_type_id,
787            result_id,
788            &[
789                (blank_intersection_id, entry_label_id),
790                (intersection_id, triangle_merge_label_id),
791            ],
792        ));
793        function.consume(final_block, Instruction::return_value(result_id));
794
795        function.to_words(&mut self.logical_layout.function_definitions);
796        self.ray_tracing_functions
797            .insert(LookupRaytracingFunction::HitObjectGetIntersection, func_id);
798
799        Ok(func_id)
800    }
801}
802
803impl BlockContext<'_> {
804    pub(in super::super) fn write_hit_object_function(
805        &mut self,
806        hit_object: crate::Handle<crate::Expression>,
807        fun: &crate::HitObjectFunction,
808        block: &mut Block,
809    ) -> Result<(), super::super::Error> {
810        let hit_object_id = self.cached[hit_object];
811
812        match *fun {
813            crate::HitObjectFunction::TraceRay {
814                acceleration_structure,
815                descriptor,
816                payload,
817            } => {
818                // Checked for when validating the module in `validate_block_impl`.
819                let crate::Expression::GlobalVariable(payload) =
820                    self.ir_function.expressions[payload]
821                else {
822                    unreachable!()
823                };
824
825                let desc_id = self.cached[descriptor];
826                let acc_struct_id = self.get_handle_id(acceleration_structure);
827
828                let func = self
829                    .writer
830                    .write_hit_object_trace_ray(self.ir_module, payload)?;
831
832                let func_id = self.gen_id();
833                block.body.push(Instruction::function_call(
834                    self.writer.void_type,
835                    func_id,
836                    func,
837                    &[hit_object_id, acc_struct_id, desc_id],
838                ));
839            }
840            crate::HitObjectFunction::RecordMiss { descriptor } => {
841                let desc_id = self.cached[descriptor];
842
843                let func = self.writer.write_hit_object_record_miss(self.ir_module)?;
844
845                let func_id = self.gen_id();
846                block.body.push(Instruction::function_call(
847                    self.writer.void_type,
848                    func_id,
849                    func,
850                    &[hit_object_id, desc_id],
851                ));
852            }
853            crate::HitObjectFunction::RecordFromQuery { query } => {
854                let query_id = self.cached[query];
855                let trackers = *self
856                    .ray_query_tracker_expr
857                    .get(&query)
858                    .expect("not a cached ray query");
859
860                let func = self.writer.write_hit_object_record_from_query()?;
861
862                let func_id = self.gen_id();
863                block.body.push(Instruction::function_call(
864                    self.writer.void_type,
865                    func_id,
866                    func,
867                    &[
868                        hit_object_id,
869                        query_id,
870                        trackers.initialized_tracker,
871                        trackers.t_max_tracker,
872                    ],
873                ));
874            }
875            crate::HitObjectFunction::RecordEmpty => {
876                self.writer.require_hit_objects()?;
877                block
878                    .body
879                    .push(Instruction::hit_object_record_empty(hit_object_id));
880            }
881            crate::HitObjectFunction::ExecuteShader { payload } => {
882                // Checked for when validating the module in `validate_block_impl`.
883                let crate::Expression::GlobalVariable(payload) =
884                    self.ir_function.expressions[payload]
885                else {
886                    unreachable!()
887                };
888
889                let func = self.writer.write_hit_object_execute_shader(payload)?;
890
891                let func_id = self.gen_id();
892                block.body.push(Instruction::function_call(
893                    self.writer.void_type,
894                    func_id,
895                    func,
896                    &[hit_object_id],
897                ));
898            }
899            crate::HitObjectFunction::Reorder { hint } => {
900                self.writer.require_hit_objects()?;
901                let hint = hint.map(|crate::ReorderHint { hint, bits }| {
902                    (self.cached[hint], self.cached[bits])
903                });
904                block.body.push(Instruction::reorder_thread_with_hit_object(
905                    hit_object_id,
906                    hint,
907                ));
908            }
909        }
910
911        Ok(())
912    }
913
914    /// Write an [`Expression::HitObjectQuery`].
915    ///
916    /// [`Expression::HitObjectQuery`]: crate::Expression::HitObjectQuery
917    pub(in super::super) fn write_hit_object_query(
918        &mut self,
919        hit_object: crate::Handle<crate::Expression>,
920        query: crate::HitObjectQuery,
921        block: &mut Block,
922    ) -> Result<spirv::Word, super::super::Error> {
923        self.writer.require_hit_objects()?;
924        let hit_object_id = self.cached[hit_object];
925
926        let op = match query {
927            crate::HitObjectQuery::IsEmpty => spirv::Op::HitObjectIsEmptyEXT,
928            crate::HitObjectQuery::IsHit => spirv::Op::HitObjectIsHitEXT,
929            crate::HitObjectQuery::IsMiss => spirv::Op::HitObjectIsMissEXT,
930            crate::HitObjectQuery::Intersection => {
931                let func = self
932                    .writer
933                    .write_hit_object_get_intersection_function(self.ir_module)?;
934                let ray_intersection = self.ir_module.special_types.ray_intersection.unwrap();
935                let intersection_type_id = self.get_handle_type_id(ray_intersection);
936                let id = self.gen_id();
937                block.body.push(Instruction::function_call(
938                    intersection_type_id,
939                    id,
940                    func,
941                    &[hit_object_id],
942                ));
943                return Ok(id);
944            }
945        };
946
947        let bool_type_id = self.writer.get_bool_type_id();
948        let id = self.gen_id();
949        block.body.push(Instruction::hit_object_get(
950            op,
951            bool_type_id,
952            id,
953            hit_object_id,
954        ));
955        Ok(id)
956    }
957}