Skip to main content

naga/back/spv/
writer.rs

1use alloc::{format, string::String, vec, vec::Vec};
2
3use arrayvec::ArrayVec;
4use hashbrown::hash_map::Entry;
5use spirv::Word;
6
7use super::{
8    block::DebugInfoInner,
9    helpers::{contains_builtin, global_needs_wrapper, map_storage_class},
10    Block, BlockContext, CachedConstant, CachedExpressions, CooperativeType, DebugInfo,
11    EntryPointContext, Error, Function, FunctionArgument, GlobalVariable, IdGenerator, Instruction,
12    LocalImageType, LocalType, LocalVariable, LogicalLayout, LookupFunctionType, LookupType,
13    NumericType, Options, PhysicalLayout, PipelineOptions, ResultMember, Writer, WriterFlags,
14    BITS_PER_BYTE,
15};
16use crate::{
17    arena::{Handle, HandleVec, UniqueArena},
18    back::spv::{
19        helpers::{is_uniform_matcx2_struct_member_access, BindingDecorations},
20        BindingInfo, Std140CompatTypeInfo, WrappedFunction,
21    },
22    common::ForDebugWithTypes as _,
23    proc::{Alignment, TypeResolution},
24    valid::{FunctionInfo, ModuleInfo},
25};
26
27pub struct FunctionInterface<'a> {
28    pub varying_ids: &'a mut Vec<Word>,
29    pub stage: crate::ShaderStage,
30    pub task_payload: Option<Handle<crate::GlobalVariable>>,
31    pub mesh_info: Option<crate::MeshStageInfo>,
32    pub workgroup_size: [u32; 3],
33}
34
35impl Function {
36    pub(super) fn to_words(&self, sink: &mut impl Extend<Word>) {
37        self.signature.as_ref().unwrap().to_words(sink);
38        for argument in self.parameters.iter() {
39            argument.instruction.to_words(sink);
40        }
41        for (index, block) in self.blocks.iter().enumerate() {
42            Instruction::label(block.label_id).to_words(sink);
43            if index == 0 {
44                for local_var in self.variables.values() {
45                    local_var.instruction.to_words(sink);
46                }
47                for local_var in self.ray_query_initialization_tracker_variables.values() {
48                    local_var.instruction.to_words(sink);
49                }
50                for local_var in self.ray_query_t_max_tracker_variables.values() {
51                    local_var.instruction.to_words(sink);
52                }
53                for local_var in self.force_loop_bounding_vars.iter() {
54                    local_var.instruction.to_words(sink);
55                }
56                for internal_var in self.spilled_composites.values() {
57                    internal_var.instruction.to_words(sink);
58                }
59            }
60            for instruction in block.body.iter() {
61                instruction.to_words(sink);
62            }
63        }
64        Instruction::function_end().to_words(sink);
65    }
66}
67
68impl Writer {
69    pub fn new(options: &Options) -> Result<Self, Error> {
70        let (major, minor) = options.lang_version;
71        if major != 1 {
72            return Err(Error::UnsupportedVersion(major, minor));
73        }
74
75        let mut capabilities_used = crate::FastIndexSet::default();
76        capabilities_used.insert(spirv::Capability::Shader);
77
78        let mut id_gen = IdGenerator::default();
79        let gl450_ext_inst_id = id_gen.next();
80        let void_type = id_gen.next();
81
82        Ok(Writer {
83            physical_layout: PhysicalLayout::new(major, minor),
84            logical_layout: LogicalLayout::default(),
85            id_gen,
86            capabilities_available: options.capabilities.clone(),
87            capabilities_used,
88            extensions_used: crate::FastIndexSet::default(),
89            debug_strings: vec![],
90            debugs: vec![],
91            annotations: vec![],
92            flags: options.flags,
93            bounds_check_policies: options.bounds_check_policies,
94            zero_initialize_workgroup_memory: options.zero_initialize_workgroup_memory,
95            force_loop_bounding: options.force_loop_bounding,
96            ray_query_initialization_tracking: options.ray_query_initialization_tracking,
97            trace_ray_argument_validation: options.trace_ray_argument_validation,
98            use_storage_input_output_16: options.use_storage_input_output_16,
99            emit_int_div_checks: options.emit_int_div_checks,
100            void_type,
101            tuple_of_u32s_ty_id: None,
102            lookup_type: crate::FastHashMap::default(),
103            lookup_function: crate::FastHashMap::default(),
104            lookup_function_type: crate::FastHashMap::default(),
105            wrapped_functions: crate::FastHashMap::default(),
106            constant_ids: HandleVec::new(),
107            cached_constants: crate::FastHashMap::default(),
108            global_variables: HandleVec::new(),
109            std140_compat_uniform_types: crate::FastHashMap::default(),
110            fake_missing_bindings: options.fake_missing_bindings,
111            binding_map: options.binding_map.clone(),
112            saved_cached: CachedExpressions::default(),
113            gl450_ext_inst_id,
114            temp_list: Vec::new(),
115            ray_query_functions: crate::FastHashMap::default(),
116            ray_tracing_functions: crate::FastHashMap::default(),
117            has_ray_tracing_pipeline: false,
118            hit_object_attribute_var: None,
119            io_f16_polyfills: super::f16_polyfill::F16IoPolyfill::new(
120                options.use_storage_input_output_16,
121            ),
122            debug_printf: None,
123            task_dispatch_limits: options.task_dispatch_limits,
124            mesh_shader_primitive_indices_clamp: options.mesh_shader_primitive_indices_clamp,
125        })
126    }
127
128    pub fn set_options(&mut self, options: &Options) -> Result<(), Error> {
129        let (major, minor) = options.lang_version;
130        if major != 1 {
131            return Err(Error::UnsupportedVersion(major, minor));
132        }
133        self.physical_layout = PhysicalLayout::new(major, minor);
134        self.capabilities_available = options.capabilities.clone();
135        self.flags = options.flags;
136        self.bounds_check_policies = options.bounds_check_policies;
137        self.zero_initialize_workgroup_memory = options.zero_initialize_workgroup_memory;
138        self.force_loop_bounding = options.force_loop_bounding;
139        self.use_storage_input_output_16 = options.use_storage_input_output_16;
140        self.binding_map = options.binding_map.clone();
141        self.io_f16_polyfills =
142            super::f16_polyfill::F16IoPolyfill::new(options.use_storage_input_output_16);
143        self.task_dispatch_limits = options.task_dispatch_limits;
144        self.mesh_shader_primitive_indices_clamp = options.mesh_shader_primitive_indices_clamp;
145        Ok(())
146    }
147
148    /// Returns `(major, minor)` of the SPIR-V language version.
149    pub const fn lang_version(&self) -> (u8, u8) {
150        self.physical_layout.lang_version()
151    }
152
153    /// Reset `Writer` to its initial state, retaining any allocations.
154    ///
155    /// Why not just implement `Reclaimable` for `Writer`? By design,
156    /// `Reclaimable::reclaim` requires ownership of the value, not just
157    /// `&mut`; see the trait documentation. But we need to use this method
158    /// from functions like `Writer::write`, which only have `&mut Writer`.
159    /// Workarounds include unsafe code (`core::ptr::read`, then `write`, ugh)
160    /// or something like a `Default` impl that returns an oddly-initialized
161    /// `Writer`, which is worse.
162    fn reset(&mut self) {
163        use super::reclaimable::Reclaimable;
164        use core::mem::take;
165
166        let mut id_gen = IdGenerator::default();
167        let gl450_ext_inst_id = id_gen.next();
168        let void_type = id_gen.next();
169
170        // Every field of the old writer that is not determined by the `Options`
171        // passed to `Writer::new` should be reset somehow.
172        let fresh = Writer {
173            // Copied from the old Writer:
174            flags: self.flags,
175            bounds_check_policies: self.bounds_check_policies,
176            zero_initialize_workgroup_memory: self.zero_initialize_workgroup_memory,
177            force_loop_bounding: self.force_loop_bounding,
178            ray_query_initialization_tracking: self.ray_query_initialization_tracking,
179            trace_ray_argument_validation: self.trace_ray_argument_validation,
180            use_storage_input_output_16: self.use_storage_input_output_16,
181            capabilities_available: take(&mut self.capabilities_available),
182            fake_missing_bindings: self.fake_missing_bindings,
183            binding_map: take(&mut self.binding_map),
184            task_dispatch_limits: self.task_dispatch_limits,
185            mesh_shader_primitive_indices_clamp: self.mesh_shader_primitive_indices_clamp,
186            emit_int_div_checks: self.emit_int_div_checks,
187
188            // Initialized afresh:
189            id_gen,
190            void_type,
191            tuple_of_u32s_ty_id: None,
192            gl450_ext_inst_id,
193
194            // Reclaimed:
195            capabilities_used: take(&mut self.capabilities_used).reclaim(),
196            extensions_used: take(&mut self.extensions_used).reclaim(),
197            physical_layout: self.physical_layout.clone().reclaim(),
198            logical_layout: take(&mut self.logical_layout).reclaim(),
199            debug_strings: take(&mut self.debug_strings).reclaim(),
200            debugs: take(&mut self.debugs).reclaim(),
201            annotations: take(&mut self.annotations).reclaim(),
202            lookup_type: take(&mut self.lookup_type).reclaim(),
203            lookup_function: take(&mut self.lookup_function).reclaim(),
204            lookup_function_type: take(&mut self.lookup_function_type).reclaim(),
205            wrapped_functions: take(&mut self.wrapped_functions).reclaim(),
206            constant_ids: take(&mut self.constant_ids).reclaim(),
207            cached_constants: take(&mut self.cached_constants).reclaim(),
208            global_variables: take(&mut self.global_variables).reclaim(),
209            std140_compat_uniform_types: take(&mut self.std140_compat_uniform_types).reclaim(),
210            saved_cached: take(&mut self.saved_cached).reclaim(),
211            temp_list: take(&mut self.temp_list).reclaim(),
212            ray_query_functions: take(&mut self.ray_query_functions).reclaim(),
213            ray_tracing_functions: take(&mut self.ray_tracing_functions).reclaim(),
214            has_ray_tracing_pipeline: false,
215            hit_object_attribute_var: None,
216            io_f16_polyfills: take(&mut self.io_f16_polyfills).reclaim(),
217            debug_printf: None,
218        };
219
220        *self = fresh;
221
222        self.capabilities_used.insert(spirv::Capability::Shader);
223    }
224
225    /// Indicate that the code requires any one of the listed capabilities.
226    ///
227    /// If nothing in `capabilities` appears in the available capabilities
228    /// specified in the [`Options`] from which this `Writer` was created,
229    /// return an error. The `what` string is used in the error message to
230    /// explain what provoked the requirement. (If no available capabilities were
231    /// given, assume everything is available.)
232    ///
233    /// The first acceptable capability will be added to this `Writer`'s
234    /// [`capabilities_used`] table, and an `OpCapability` emitted for it in the
235    /// result. For this reason, more specific capabilities should be listed
236    /// before more general.
237    ///
238    /// [`capabilities_used`]: Writer::capabilities_used
239    pub(super) fn require_any(
240        &mut self,
241        what: &'static str,
242        capabilities: &[spirv::Capability],
243    ) -> Result<(), Error> {
244        match *capabilities {
245            [] => Ok(()),
246            [first, ..] => {
247                // Find the first acceptable capability, or return an error if
248                // there is none.
249                let selected = match self.capabilities_available {
250                    None => first,
251                    Some(ref available) => {
252                        match capabilities
253                            .iter()
254                            // need explicit type for hashbrown::HashSet::contains fn call to keep rustc happy
255                            .find(|cap| available.contains::<spirv::Capability>(cap))
256                        {
257                            Some(&cap) => cap,
258                            None => {
259                                return Err(Error::MissingCapabilities(what, capabilities.to_vec()))
260                            }
261                        }
262                    }
263                };
264                self.capabilities_used.insert(selected);
265                Ok(())
266            }
267        }
268    }
269
270    /// Indicate that the code requires all of the listed capabilities.
271    ///
272    /// If all entries of `capabilities` appear in the available capabilities
273    /// specified in the [`Options`] from which this `Writer` was created
274    /// (including the case where [`Options::capabilities`] is `None`), add
275    /// them all to this `Writer`'s [`capabilities_used`] table, and return
276    /// `Ok(())`. If at least one of the listed capabilities is not available,
277    /// do not add anything to the `capabilities_used` table, and return the
278    /// first unavailable requested capability, wrapped in `Err()`.
279    ///
280    /// This method is does not return an [`enum@Error`] in case of failure
281    /// because it may be used in cases where the caller can recover (e.g.,
282    /// with a polyfill) if the requested capabilities are not available. In
283    /// this case, it would be unnecessary work to find *all* the unavailable
284    /// requested capabilities, and to allocate a `Vec` for them, just so we
285    /// could return an [`Error::MissingCapabilities`]).
286    ///
287    /// [`capabilities_used`]: Writer::capabilities_used
288    pub(super) fn require_all(
289        &mut self,
290        capabilities: &[spirv::Capability],
291    ) -> Result<(), spirv::Capability> {
292        if let Some(ref available) = self.capabilities_available {
293            for requested in capabilities {
294                if !available.contains(requested) {
295                    return Err(*requested);
296                }
297            }
298        }
299
300        for requested in capabilities {
301            self.capabilities_used.insert(*requested);
302        }
303
304        Ok(())
305    }
306
307    /// Indicate that the code uses the given extension.
308    pub(super) fn use_extension(&mut self, extension: &'static str) {
309        self.extensions_used.insert(extension);
310    }
311
312    pub(super) fn get_type_id(&mut self, lookup_ty: LookupType) -> Word {
313        match self.lookup_type.entry(lookup_ty) {
314            Entry::Occupied(e) => *e.get(),
315            Entry::Vacant(e) => {
316                let local = match lookup_ty {
317                    LookupType::Handle(_handle) => unreachable!("Handles are populated at start"),
318                    LookupType::Local(local) => local,
319                };
320
321                let id = self.id_gen.next();
322                e.insert(id);
323                self.write_type_declaration_local(id, local);
324                id
325            }
326        }
327    }
328
329    pub(super) fn get_handle_type_id(&mut self, handle: Handle<crate::Type>) -> Word {
330        self.get_type_id(LookupType::Handle(handle))
331    }
332
333    pub(super) fn get_expression_lookup_type(&mut self, tr: &TypeResolution) -> LookupType {
334        match *tr {
335            TypeResolution::Handle(ty_handle) => LookupType::Handle(ty_handle),
336            TypeResolution::Value(ref inner) => {
337                let inner_local_type = self.localtype_from_inner(inner).unwrap();
338                LookupType::Local(inner_local_type)
339            }
340        }
341    }
342
343    pub(super) fn get_expression_type_id(&mut self, tr: &TypeResolution) -> Word {
344        let lookup_ty = self.get_expression_lookup_type(tr);
345        self.get_type_id(lookup_ty)
346    }
347
348    pub(super) fn get_localtype_id(&mut self, local: LocalType) -> Word {
349        self.get_type_id(LookupType::Local(local))
350    }
351
352    pub(super) fn get_pointer_type_id(&mut self, base: Word, class: spirv::StorageClass) -> Word {
353        self.get_type_id(LookupType::Local(LocalType::Pointer { base, class }))
354    }
355
356    pub(super) fn get_handle_pointer_type_id(
357        &mut self,
358        base: Handle<crate::Type>,
359        class: spirv::StorageClass,
360    ) -> Word {
361        let base_id = self.get_handle_type_id(base);
362        self.get_pointer_type_id(base_id, class)
363    }
364
365    pub(super) fn get_ray_query_pointer_id(&mut self) -> Word {
366        let rq_id = self.get_type_id(LookupType::Local(LocalType::RayQuery));
367        self.get_pointer_type_id(rq_id, spirv::StorageClass::Function)
368    }
369
370    pub(super) fn get_hit_object_pointer_id(&mut self) -> Word {
371        let ho_id = self.get_type_id(LookupType::Local(LocalType::HitObject));
372        self.get_pointer_type_id(ho_id, spirv::StorageClass::Function)
373    }
374
375    /// Request everything needed to use `SPV_EXT_shader_invocation_reorder`.
376    pub(super) fn require_hit_objects(&mut self) -> Result<(), Error> {
377        self.require_any(
378            "hit objects",
379            &[spirv::Capability::ShaderInvocationReorderEXT],
380        )?;
381        self.use_extension("SPV_EXT_shader_invocation_reorder");
382        Ok(())
383    }
384
385    /// Return the id of the module-scope `HitObjectAttributeEXT` variable used
386    /// to read triangle barycentrics out of a hit object, creating it if needed.
387    pub(super) fn get_hit_object_attribute_var(&mut self) -> Word {
388        if let Some(id) = self.hit_object_attribute_var {
389            return id;
390        }
391
392        let barycentrics_type_id = self.get_numeric_type_id(NumericType::Vector {
393            size: crate::VectorSize::Bi,
394            scalar: crate::Scalar::F32,
395        });
396        let pointer_type_id = self.get_pointer_type_id(
397            barycentrics_type_id,
398            spirv::StorageClass::HitObjectAttributeEXT,
399        );
400
401        let id = self.id_gen.next();
402        Instruction::variable(
403            pointer_type_id,
404            id,
405            spirv::StorageClass::HitObjectAttributeEXT,
406            None,
407        )
408        .to_words(&mut self.logical_layout.declarations);
409
410        if self.flags.contains(WriterFlags::DEBUG) {
411            self.debugs
412                .push(Instruction::name(id, "naga_hit_object_attributes"));
413        }
414
415        self.hit_object_attribute_var = Some(id);
416        id
417    }
418
419    /// Return a SPIR-V type for a pointer to `resolution`.
420    ///
421    /// The given `resolution` must be one that we can represent
422    /// either as a `LocalType::Pointer` or `LocalType::LocalPointer`.
423    pub(super) fn get_resolution_pointer_id(
424        &mut self,
425        resolution: &TypeResolution,
426        class: spirv::StorageClass,
427    ) -> Word {
428        let resolution_type_id = self.get_expression_type_id(resolution);
429        self.get_pointer_type_id(resolution_type_id, class)
430    }
431
432    pub(super) fn get_numeric_type_id(&mut self, numeric: NumericType) -> Word {
433        self.get_type_id(LocalType::Numeric(numeric).into())
434    }
435
436    pub(super) fn get_u32_type_id(&mut self) -> Word {
437        self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32))
438    }
439
440    pub(super) fn get_f32_type_id(&mut self) -> Word {
441        self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::F32))
442    }
443
444    pub(super) fn get_vec2u_type_id(&mut self) -> Word {
445        self.get_numeric_type_id(NumericType::Vector {
446            size: crate::VectorSize::Bi,
447            scalar: crate::Scalar::U32,
448        })
449    }
450
451    pub(super) fn get_vec2f_type_id(&mut self) -> Word {
452        self.get_numeric_type_id(NumericType::Vector {
453            size: crate::VectorSize::Bi,
454            scalar: crate::Scalar::F32,
455        })
456    }
457
458    pub(super) fn get_vec3u_type_id(&mut self) -> Word {
459        self.get_numeric_type_id(NumericType::Vector {
460            size: crate::VectorSize::Tri,
461            scalar: crate::Scalar::U32,
462        })
463    }
464
465    pub(super) fn get_f32_pointer_type_id(&mut self, class: spirv::StorageClass) -> Word {
466        let f32_id = self.get_f32_type_id();
467        self.get_pointer_type_id(f32_id, class)
468    }
469
470    pub(super) fn get_vec2u_pointer_type_id(&mut self, class: spirv::StorageClass) -> Word {
471        let vec2u_id = self.get_numeric_type_id(NumericType::Vector {
472            size: crate::VectorSize::Bi,
473            scalar: crate::Scalar::U32,
474        });
475        self.get_pointer_type_id(vec2u_id, class)
476    }
477
478    pub(super) fn get_bool_type_id(&mut self) -> Word {
479        self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::BOOL))
480    }
481
482    pub(super) fn get_vec2_bool_type_id(&mut self) -> Word {
483        self.get_numeric_type_id(NumericType::Vector {
484            size: crate::VectorSize::Bi,
485            scalar: crate::Scalar::BOOL,
486        })
487    }
488
489    pub(super) fn get_vec3_bool_type_id(&mut self) -> Word {
490        self.get_numeric_type_id(NumericType::Vector {
491            size: crate::VectorSize::Tri,
492            scalar: crate::Scalar::BOOL,
493        })
494    }
495
496    /// Used for "mulhi" to get the upper bits of multiplication.
497    ///
498    /// More specifically, `OpUMulExtended` multiplies 2 numbers and returns the lower and upper bits of the result
499    /// as a user-defined struct type with 2 u32s. This defines that struct.
500    pub(super) fn get_tuple_of_u32s_ty_id(&mut self) -> Word {
501        if let Some(val) = self.tuple_of_u32s_ty_id {
502            val
503        } else {
504            let id = self.id_gen.next();
505            let u32_id = self.get_u32_type_id();
506            let ins = Instruction::type_struct(id, &[u32_id, u32_id]);
507            ins.to_words(&mut self.logical_layout.declarations);
508            self.tuple_of_u32s_ty_id = Some(id);
509            id
510        }
511    }
512
513    pub(super) fn decorate(&mut self, id: Word, decoration: spirv::Decoration, operands: &[Word]) {
514        self.annotations
515            .push(Instruction::decorate(id, decoration, operands));
516    }
517
518    /// Return `inner` as a `LocalType`, if that's possible.
519    ///
520    /// If `inner` can be represented as a `LocalType`, return
521    /// `Some(local_type)`.
522    ///
523    /// Otherwise, return `None`. In this case, the type must always be looked
524    /// up using a `LookupType::Handle`.
525    fn localtype_from_inner(&mut self, inner: &crate::TypeInner) -> Option<LocalType> {
526        Some(match *inner {
527            crate::TypeInner::Scalar(_)
528            | crate::TypeInner::Atomic(_)
529            | crate::TypeInner::Vector { .. }
530            | crate::TypeInner::Matrix { .. } => {
531                // We expect `NumericType::from_inner` to handle all
532                // these cases, so unwrap.
533                LocalType::Numeric(NumericType::from_inner(inner).unwrap())
534            }
535            crate::TypeInner::CooperativeMatrix { .. } => {
536                LocalType::Cooperative(CooperativeType::from_inner(inner).unwrap())
537            }
538            crate::TypeInner::Pointer { base, space } => {
539                let base_type_id = self.get_handle_type_id(base);
540                LocalType::Pointer {
541                    base: base_type_id,
542                    class: map_storage_class(space),
543                }
544            }
545            crate::TypeInner::ValuePointer {
546                size,
547                scalar,
548                space,
549            } => {
550                let base_numeric_type = match size {
551                    Some(size) => NumericType::Vector { size, scalar },
552                    None => NumericType::Scalar(scalar),
553                };
554                LocalType::Pointer {
555                    base: self.get_numeric_type_id(base_numeric_type),
556                    class: map_storage_class(space),
557                }
558            }
559            crate::TypeInner::Image {
560                dim,
561                arrayed,
562                class,
563            } => LocalType::Image(LocalImageType::from_inner(dim, arrayed, class)),
564            crate::TypeInner::Sampler { comparison: _ } => LocalType::Sampler,
565            crate::TypeInner::AccelerationStructure { .. } => LocalType::AccelerationStructure,
566            crate::TypeInner::RayQuery { .. } => LocalType::RayQuery,
567            crate::TypeInner::HitObject => LocalType::HitObject,
568            crate::TypeInner::Array { .. }
569            | crate::TypeInner::Struct { .. }
570            | crate::TypeInner::BindingArray { .. } => return None,
571        })
572    }
573
574    /// Resolve the [`BindingInfo`] for a [`crate::ResourceBinding`] from the
575    /// provided [`Writer::binding_map`].
576    ///
577    /// If the specified resource is not present in the binding map this will
578    /// return an error, unless [`Writer::fake_missing_bindings`] is set.
579    pub(super) fn resolve_resource_binding(
580        &self,
581        res_binding: &crate::ResourceBinding,
582    ) -> Result<BindingInfo, Error> {
583        match self.binding_map.get(res_binding) {
584            Some(target) => Ok(*target),
585            None if self.fake_missing_bindings => Ok(BindingInfo {
586                descriptor_set: res_binding.group,
587                binding: res_binding.binding,
588                binding_array_size: None,
589                external_texture: false,
590            }),
591            None => Err(Error::MissingBinding(*res_binding)),
592        }
593    }
594
595    /// Emits code for any wrapper functions required by the expressions in ir_function.
596    /// The IDs of any emitted functions will be stored in [`Self::wrapped_functions`].
597    fn write_wrapped_functions(
598        &mut self,
599        ir_function: &crate::Function,
600        info: &FunctionInfo,
601        ir_module: &crate::Module,
602    ) -> Result<(), Error> {
603        log::trace!("Generating wrapped functions for {:?}", ir_function.name);
604
605        for (expr_handle, expr) in ir_function.expressions.iter() {
606            match *expr {
607                crate::Expression::Binary { op, left, right } => {
608                    let expr_ty_inner = info[expr_handle].ty.inner_with(&ir_module.types);
609                    if let Some(expr_ty) = NumericType::from_inner(expr_ty_inner) {
610                        match (op, expr_ty.scalar().kind) {
611                            // Division and modulo are undefined behaviour when the
612                            // dividend is the minimum representable value and the divisor
613                            // is negative one, or when the divisor is zero. These wrapped
614                            // functions override the divisor to one in these cases,
615                            // matching the WGSL spec.
616                            //
617                            // Signed `%` is additionally always wrapped (even without
618                            // `emit_int_div_checks`) so it can be lowered to
619                            // `a - b * (a / b)`: `OpSRem` produces a poison result for
620                            // negative operands in the Vulkan SPIR-V environment unless
621                            // `VK_KHR_maintenance8` is enabled. See
622                            // <https://github.com/gfx-rs/wgpu/issues/8191>.
623                            (
624                                crate::BinaryOperator::Divide | crate::BinaryOperator::Modulo,
625                                crate::ScalarKind::Sint | crate::ScalarKind::Uint,
626                            ) if self.emit_int_div_checks
627                                || matches!(
628                                    (op, expr_ty.scalar().kind),
629                                    (crate::BinaryOperator::Modulo, crate::ScalarKind::Sint)
630                                ) =>
631                            {
632                                self.write_wrapped_binary_op(
633                                    op,
634                                    expr_ty,
635                                    &info[left].ty,
636                                    &info[right].ty,
637                                )?;
638                            }
639                            _ => {}
640                        }
641                    }
642                }
643                crate::Expression::Load { pointer } => {
644                    if let crate::TypeInner::Pointer {
645                        base: pointer_type,
646                        space: crate::AddressSpace::Uniform,
647                    } = *info[pointer].ty.inner_with(&ir_module.types)
648                    {
649                        if self.std140_compat_uniform_types.contains_key(&pointer_type) {
650                            // Loading a std140 compat type requires the wrapper function
651                            // to convert to the regular type.
652                            self.write_wrapped_convert_from_std140_compat_type(
653                                ir_module,
654                                pointer_type,
655                            )?;
656                        }
657                    }
658                }
659                crate::Expression::Access { base, .. } => {
660                    if let crate::TypeInner::Pointer {
661                        base: base_type,
662                        space: crate::AddressSpace::Uniform,
663                    } = *info[base].ty.inner_with(&ir_module.types)
664                    {
665                        // Dynamic accesses of a two-row matrix's columns require a
666                        // wrapper function.
667                        if let crate::TypeInner::Matrix {
668                            rows: crate::VectorSize::Bi,
669                            ..
670                        } = ir_module.types[base_type].inner
671                        {
672                            self.write_wrapped_matcx2_get_column(ir_module, base_type)?;
673                            // If the matrix is *not* directly a member of a struct, then
674                            // we additionally require a wrapper function to convert from
675                            // the std140 compat type to the regular type.
676                            if !is_uniform_matcx2_struct_member_access(
677                                ir_function,
678                                info,
679                                ir_module,
680                                base,
681                            ) {
682                                self.write_wrapped_convert_from_std140_compat_type(
683                                    ir_module, base_type,
684                                )?;
685                            }
686                        }
687                    }
688                }
689                _ => {}
690            }
691        }
692
693        Ok(())
694    }
695
696    /// Write a SPIR-V function that performs the operator `op` with Naga IR semantics.
697    ///
698    /// Define a function that performs an integer division or modulo operation,
699    /// except that using a divisor of zero or causing signed overflow with a
700    /// divisor of -1 returns the numerator unchanged, rather than exhibiting
701    /// undefined behavior.
702    ///
703    /// Store the generated function's id in the [`wrapped_functions`] table.
704    ///
705    /// The operator `op` must be either [`Divide`] or [`Modulo`].
706    ///
707    /// # Panics
708    ///
709    /// The `return_type`, `left_type` or `right_type` arguments must all be
710    /// integer scalars or vectors. If not, this function panics.
711    ///
712    /// [`wrapped_functions`]: Writer::wrapped_functions
713    /// [`Divide`]: crate::BinaryOperator::Divide
714    /// [`Modulo`]: crate::BinaryOperator::Modulo
715    fn write_wrapped_binary_op(
716        &mut self,
717        op: crate::BinaryOperator,
718        return_type: NumericType,
719        left_type: &TypeResolution,
720        right_type: &TypeResolution,
721    ) -> Result<(), Error> {
722        let return_type_id = self.get_localtype_id(LocalType::Numeric(return_type));
723        let left_type_id = self.get_expression_type_id(left_type);
724        let right_type_id = self.get_expression_type_id(right_type);
725
726        // Check if we've already emitted this function.
727        let wrapped = WrappedFunction::BinaryOp {
728            op,
729            left_type_id,
730            right_type_id,
731        };
732        let function_id = match self.wrapped_functions.entry(wrapped) {
733            Entry::Occupied(_) => return Ok(()),
734            Entry::Vacant(e) => *e.insert(self.id_gen.next()),
735        };
736
737        let scalar = return_type.scalar();
738
739        if self.flags.contains(WriterFlags::DEBUG) {
740            let function_name = match op {
741                crate::BinaryOperator::Divide => "naga_div",
742                crate::BinaryOperator::Modulo => "naga_mod",
743                _ => unreachable!(),
744            };
745            self.debugs
746                .push(Instruction::name(function_id, function_name));
747        }
748        let mut function = Function::default();
749
750        let function_type_id = self.get_function_type(LookupFunctionType {
751            parameter_type_ids: vec![left_type_id, right_type_id],
752            return_type_id,
753        });
754        function.signature = Some(Instruction::function(
755            return_type_id,
756            function_id,
757            spirv::FunctionControl::empty(),
758            function_type_id,
759        ));
760
761        let lhs_id = self.id_gen.next();
762        let rhs_id = self.id_gen.next();
763        if self.flags.contains(WriterFlags::DEBUG) {
764            self.debugs.push(Instruction::name(lhs_id, "lhs"));
765            self.debugs.push(Instruction::name(rhs_id, "rhs"));
766        }
767        let left_par = Instruction::function_parameter(left_type_id, lhs_id);
768        let right_par = Instruction::function_parameter(right_type_id, rhs_id);
769        for instruction in [left_par, right_par] {
770            function.parameters.push(FunctionArgument {
771                instruction,
772                handle_id: 0,
773            });
774        }
775
776        let label_id = self.id_gen.next();
777        let mut block = Block::new(label_id);
778
779        let bool_type = return_type.with_scalar(crate::Scalar::BOOL);
780        let bool_type_id = self.get_numeric_type_id(bool_type);
781
782        let maybe_splat_const = |writer: &mut Self, const_id| match return_type {
783            NumericType::Scalar(_) => const_id,
784            NumericType::Vector { size, .. } => {
785                let constituent_ids = [const_id; crate::VectorSize::MAX];
786                writer.get_constant_composite(
787                    LookupType::Local(LocalType::Numeric(return_type)),
788                    &constituent_ids[..size as usize],
789                )
790            }
791            NumericType::Matrix { .. } => unreachable!(),
792        };
793
794        let const_zero_id = self.get_constant_scalar_with(0, scalar)?;
795        let composite_zero_id = maybe_splat_const(self, const_zero_id);
796        let rhs_eq_zero_id = self.id_gen.next();
797        block.body.push(Instruction::binary(
798            spirv::Op::IEqual,
799            bool_type_id,
800            rhs_eq_zero_id,
801            rhs_id,
802            composite_zero_id,
803        ));
804        let divisor_selector_id = match scalar.kind {
805            crate::ScalarKind::Sint => {
806                let (const_min_id, const_neg_one_id) = match scalar.width {
807                    2 => Ok((
808                        self.get_constant_scalar(crate::Literal::I16(i16::MIN)),
809                        self.get_constant_scalar(crate::Literal::I16(-1i16)),
810                    )),
811                    4 => Ok((
812                        self.get_constant_scalar(crate::Literal::I32(i32::MIN)),
813                        self.get_constant_scalar(crate::Literal::I32(-1i32)),
814                    )),
815                    8 => Ok((
816                        self.get_constant_scalar(crate::Literal::I64(i64::MIN)),
817                        self.get_constant_scalar(crate::Literal::I64(-1i64)),
818                    )),
819                    _ => Err(Error::Validation("Unexpected scalar width")),
820                }?;
821                let composite_min_id = maybe_splat_const(self, const_min_id);
822                let composite_neg_one_id = maybe_splat_const(self, const_neg_one_id);
823
824                let lhs_eq_int_min_id = self.id_gen.next();
825                block.body.push(Instruction::binary(
826                    spirv::Op::IEqual,
827                    bool_type_id,
828                    lhs_eq_int_min_id,
829                    lhs_id,
830                    composite_min_id,
831                ));
832                let rhs_eq_neg_one_id = self.id_gen.next();
833                block.body.push(Instruction::binary(
834                    spirv::Op::IEqual,
835                    bool_type_id,
836                    rhs_eq_neg_one_id,
837                    rhs_id,
838                    composite_neg_one_id,
839                ));
840                let lhs_eq_int_min_and_rhs_eq_neg_one_id = self.id_gen.next();
841                block.body.push(Instruction::binary(
842                    spirv::Op::LogicalAnd,
843                    bool_type_id,
844                    lhs_eq_int_min_and_rhs_eq_neg_one_id,
845                    lhs_eq_int_min_id,
846                    rhs_eq_neg_one_id,
847                ));
848                let rhs_eq_zero_or_lhs_eq_int_min_and_rhs_eq_neg_one_id = self.id_gen.next();
849                block.body.push(Instruction::binary(
850                    spirv::Op::LogicalOr,
851                    bool_type_id,
852                    rhs_eq_zero_or_lhs_eq_int_min_and_rhs_eq_neg_one_id,
853                    rhs_eq_zero_id,
854                    lhs_eq_int_min_and_rhs_eq_neg_one_id,
855                ));
856                rhs_eq_zero_or_lhs_eq_int_min_and_rhs_eq_neg_one_id
857            }
858            crate::ScalarKind::Uint => rhs_eq_zero_id,
859            _ => unreachable!(),
860        };
861
862        let const_one_id = self.get_constant_scalar_with(1, scalar)?;
863        let composite_one_id = maybe_splat_const(self, const_one_id);
864        let divisor_id = self.id_gen.next();
865        block.body.push(Instruction::select(
866            right_type_id,
867            divisor_id,
868            divisor_selector_id,
869            composite_one_id,
870            rhs_id,
871        ));
872        let return_id = if matches!(op, crate::BinaryOperator::Modulo)
873            && matches!(scalar.kind, crate::ScalarKind::Sint)
874        {
875            // `OpSRem` produces a poison result for negative operands in the Vulkan
876            // environment without the `maintenance8` feature. `OpSDiv` is not poisoned, so
877            // reconstruct the remainder as `a - b * (a / b)`, which is well-defined for
878            // negative operands. `divisor_id` is the zero/overflow-guarded divisor selected
879            // above, so the degenerate cases still match `OpSRem`'s guarded result (0).
880            let quotient_id = self.id_gen.next();
881            block.body.push(Instruction::binary(
882                spirv::Op::SDiv,
883                return_type_id,
884                quotient_id,
885                lhs_id,
886                divisor_id,
887            ));
888            let product_id = self.id_gen.next();
889            block.body.push(Instruction::binary(
890                spirv::Op::IMul,
891                return_type_id,
892                product_id,
893                quotient_id,
894                divisor_id,
895            ));
896            let remainder_id = self.id_gen.next();
897            block.body.push(Instruction::binary(
898                spirv::Op::ISub,
899                return_type_id,
900                remainder_id,
901                lhs_id,
902                product_id,
903            ));
904            remainder_id
905        } else {
906            let spv_op = match (op, scalar.kind) {
907                (crate::BinaryOperator::Divide, crate::ScalarKind::Sint) => spirv::Op::SDiv,
908                (crate::BinaryOperator::Divide, crate::ScalarKind::Uint) => spirv::Op::UDiv,
909                (crate::BinaryOperator::Modulo, crate::ScalarKind::Sint) => spirv::Op::SRem,
910                (crate::BinaryOperator::Modulo, crate::ScalarKind::Uint) => spirv::Op::UMod,
911                _ => unreachable!(),
912            };
913            let return_id = self.id_gen.next();
914            block.body.push(Instruction::binary(
915                spv_op,
916                return_type_id,
917                return_id,
918                lhs_id,
919                divisor_id,
920            ));
921            return_id
922        };
923
924        function.consume(block, Instruction::return_value(return_id));
925        function.to_words(&mut self.logical_layout.function_definitions);
926        Ok(())
927    }
928
929    /// Writes a wrapper function to convert from a std140 compat type to its
930    /// corresponding regular type.
931    ///
932    /// See [`Self::write_std140_compat_type_declaration`] for more details.
933    fn write_wrapped_convert_from_std140_compat_type(
934        &mut self,
935        ir_module: &crate::Module,
936        r#type: Handle<crate::Type>,
937    ) -> Result<(), Error> {
938        if !self.std140_compat_uniform_types.contains_key(&r#type) {
939            return Ok(());
940        }
941        // Check if we've already emitted this function.
942        let wrapped = WrappedFunction::ConvertFromStd140CompatType { r#type };
943        let function_id = match self.wrapped_functions.entry(wrapped) {
944            Entry::Occupied(_) => return Ok(()),
945            Entry::Vacant(e) => *e.insert(self.id_gen.next()),
946        };
947        if self.flags.contains(WriterFlags::DEBUG) {
948            self.debugs.push(Instruction::name(
949                function_id,
950                &format!("{:?}_from_std140", r#type.for_debug(&ir_module.types)),
951            ));
952        }
953        let param_type_id = self.std140_compat_uniform_types[&r#type].type_id;
954        let return_type_id = self.get_handle_type_id(r#type);
955
956        let mut function = Function::default();
957        let function_type_id = self.get_function_type(LookupFunctionType {
958            parameter_type_ids: vec![param_type_id],
959            return_type_id,
960        });
961        function.signature = Some(Instruction::function(
962            return_type_id,
963            function_id,
964            spirv::FunctionControl::empty(),
965            function_type_id,
966        ));
967        let param_id = self.id_gen.next();
968        function.parameters.push(FunctionArgument {
969            instruction: Instruction::function_parameter(param_type_id, param_id),
970            handle_id: 0,
971        });
972
973        let label_id = self.id_gen.next();
974        let mut block = Block::new(label_id);
975
976        let result_id = match ir_module.types[r#type].inner {
977            // Param is struct containing a vector member for each of the
978            // matrix's columns. Extract each column from the struct then
979            // composite into a matrix.
980            crate::TypeInner::Matrix {
981                columns,
982                rows: rows @ crate::VectorSize::Bi,
983                scalar,
984            } => {
985                let column_type_id =
986                    self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
987
988                let mut column_ids: ArrayVec<Word, 4> = ArrayVec::new();
989                for column in 0..columns as u32 {
990                    let column_id = self.id_gen.next();
991                    block.body.push(Instruction::composite_extract(
992                        column_type_id,
993                        column_id,
994                        param_id,
995                        &[column],
996                    ));
997                    column_ids.push(column_id);
998                }
999                let result_id = self.id_gen.next();
1000                block.body.push(Instruction::composite_construct(
1001                    return_type_id,
1002                    result_id,
1003                    &column_ids,
1004                ));
1005                result_id
1006            }
1007            // Param is an array where the base type is the std140 compatible
1008            // type corresponding to `base`. Iterate through each element and
1009            // call its conversion function, then composite into a new array.
1010            crate::TypeInner::Array { base, size, .. } => {
1011                // Ensure the conversion function for the array's base type is
1012                // declared.
1013                self.write_wrapped_convert_from_std140_compat_type(ir_module, base)?;
1014
1015                let element_type_id = self.get_handle_type_id(base);
1016                let std140_info = self.std140_compat_uniform_types.get(&base);
1017                let mut element_ids = Vec::new();
1018                let size = match size.resolve(ir_module.to_ctx())? {
1019                    crate::proc::IndexableLength::Known(size) => size,
1020                    crate::proc::IndexableLength::Dynamic => {
1021                        return Err(Error::Validation(
1022                            "Uniform buffers cannot contain dynamic arrays",
1023                        ))
1024                    }
1025                };
1026                for i in 0..size {
1027                    let std140_element_id = self.id_gen.next();
1028                    let std140_element_type_id =
1029                        std140_info.map_or(element_type_id, |info| info.type_id);
1030                    block.body.push(Instruction::composite_extract(
1031                        std140_element_type_id,
1032                        std140_element_id,
1033                        param_id,
1034                        &[i],
1035                    ));
1036
1037                    // Only call the conversion function if a compatibility mapping actually exists.
1038                    let final_element_id = if std140_info.is_some() {
1039                        let conversion_fn_id = self.wrapped_functions
1040                            [&WrappedFunction::ConvertFromStd140CompatType { r#type: base }];
1041                        let id = self.id_gen.next();
1042                        block.body.push(Instruction::function_call(
1043                            element_type_id,
1044                            id,
1045                            conversion_fn_id,
1046                            &[std140_element_id],
1047                        ));
1048                        id
1049                    } else {
1050                        std140_element_type_id
1051                    };
1052                    element_ids.push(final_element_id);
1053                }
1054                let result_id = self.id_gen.next();
1055                block.body.push(Instruction::composite_construct(
1056                    return_type_id,
1057                    result_id,
1058                    &element_ids,
1059                ));
1060                result_id
1061            }
1062            // Param is a struct where each two-row matrix member has been
1063            // decomposed in to separate vector members for each column.
1064            // Other members use their std140 compatible type if one exists, or
1065            // else their regular type. Iterate through each member, converting
1066            // or composing any matrices if required, then finally compose into
1067            // the struct.
1068            crate::TypeInner::Struct { ref members, .. } => {
1069                let mut member_ids = Vec::new();
1070                let mut next_index = 0;
1071                for member in members {
1072                    let member_id = self.id_gen.next();
1073                    let member_type_id = self.get_handle_type_id(member.ty);
1074                    match ir_module.types[member.ty].inner {
1075                        crate::TypeInner::Matrix {
1076                            columns,
1077                            rows: rows @ crate::VectorSize::Bi,
1078                            scalar,
1079                        } => {
1080                            let mut column_ids: ArrayVec<Word, 4> = ArrayVec::new();
1081                            let column_type_id = self
1082                                .get_numeric_type_id(NumericType::Vector { size: rows, scalar });
1083                            for _ in 0..columns as u32 {
1084                                let column_id = self.id_gen.next();
1085                                block.body.push(Instruction::composite_extract(
1086                                    column_type_id,
1087                                    column_id,
1088                                    param_id,
1089                                    &[next_index],
1090                                ));
1091                                column_ids.push(column_id);
1092                                next_index += 1;
1093                            }
1094                            block.body.push(Instruction::composite_construct(
1095                                member_type_id,
1096                                member_id,
1097                                &column_ids,
1098                            ));
1099                        }
1100                        _ => {
1101                            // Ensure the conversion function for the member's
1102                            // type is declared.
1103                            self.write_wrapped_convert_from_std140_compat_type(
1104                                ir_module, member.ty,
1105                            )?;
1106                            match self.std140_compat_uniform_types.get(&member.ty) {
1107                                Some(std140_type_info) => {
1108                                    let std140_member_id = self.id_gen.next();
1109                                    block.body.push(Instruction::composite_extract(
1110                                        std140_type_info.type_id,
1111                                        std140_member_id,
1112                                        param_id,
1113                                        &[next_index],
1114                                    ));
1115                                    let function_id = self.wrapped_functions
1116                                        [&WrappedFunction::ConvertFromStd140CompatType {
1117                                            r#type: member.ty,
1118                                        }];
1119                                    block.body.push(Instruction::function_call(
1120                                        member_type_id,
1121                                        member_id,
1122                                        function_id,
1123                                        &[std140_member_id],
1124                                    ));
1125                                    next_index += 1;
1126                                }
1127                                None => {
1128                                    block.body.push(Instruction::composite_extract(
1129                                        member_type_id,
1130                                        member_id,
1131                                        param_id,
1132                                        &[next_index],
1133                                    ));
1134                                    next_index += 1;
1135                                }
1136                            }
1137                        }
1138                    }
1139                    member_ids.push(member_id);
1140                }
1141                let result_id = self.id_gen.next();
1142                block.body.push(Instruction::composite_construct(
1143                    return_type_id,
1144                    result_id,
1145                    &member_ids,
1146                ));
1147                result_id
1148            }
1149            _ => unreachable!(),
1150        };
1151
1152        function.consume(block, Instruction::return_value(result_id));
1153        function.to_words(&mut self.logical_layout.function_definitions);
1154        Ok(())
1155    }
1156
1157    /// Writes a wrapper function to get an `OpTypeVector` column from an
1158    /// `OpTypeMatrix` with a dynamic index.
1159    ///
1160    /// This is used when accessing a column of a [`TypeInner::Matrix`] through
1161    /// a [`Uniform`] address space pointer. In such cases, the matrix will have
1162    /// been declared in SPIR-V using an alternative type where each column is a
1163    /// member of a containing struct. SPIR-V is unable to dynamically access
1164    /// struct members, so instead we load the matrix then call this function to
1165    /// access a column from the loaded value.
1166    ///
1167    /// [`TypeInner::Matrix`]: crate::TypeInner::Matrix
1168    /// [`Uniform`]: crate::AddressSpace::Uniform
1169    fn write_wrapped_matcx2_get_column(
1170        &mut self,
1171        ir_module: &crate::Module,
1172        r#type: Handle<crate::Type>,
1173    ) -> Result<(), Error> {
1174        let wrapped = WrappedFunction::MatCx2GetColumn { r#type };
1175        let function_id = match self.wrapped_functions.entry(wrapped) {
1176            Entry::Occupied(_) => return Ok(()),
1177            Entry::Vacant(e) => *e.insert(self.id_gen.next()),
1178        };
1179        if self.flags.contains(WriterFlags::DEBUG) {
1180            self.debugs.push(Instruction::name(
1181                function_id,
1182                &format!("{:?}_get_column", r#type.for_debug(&ir_module.types)),
1183            ));
1184        }
1185
1186        let crate::TypeInner::Matrix {
1187            columns,
1188            rows: rows @ crate::VectorSize::Bi,
1189            scalar,
1190        } = ir_module.types[r#type].inner
1191        else {
1192            unreachable!();
1193        };
1194
1195        let mut function = Function::default();
1196        let matrix_type_id = self.get_handle_type_id(r#type);
1197        let column_index_type_id = self.get_u32_type_id();
1198        let column_type_id = self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
1199        let matrix_param_id = self.id_gen.next();
1200        let column_index_param_id = self.id_gen.next();
1201        function.parameters.push(FunctionArgument {
1202            instruction: Instruction::function_parameter(matrix_type_id, matrix_param_id),
1203            handle_id: 0,
1204        });
1205        function.parameters.push(FunctionArgument {
1206            instruction: Instruction::function_parameter(
1207                column_index_type_id,
1208                column_index_param_id,
1209            ),
1210            handle_id: 0,
1211        });
1212        let function_type_id = self.get_function_type(LookupFunctionType {
1213            parameter_type_ids: vec![matrix_type_id, column_index_type_id],
1214            return_type_id: column_type_id,
1215        });
1216        function.signature = Some(Instruction::function(
1217            column_type_id,
1218            function_id,
1219            spirv::FunctionControl::empty(),
1220            function_type_id,
1221        ));
1222
1223        let label_id = self.id_gen.next();
1224        let mut block = Block::new(label_id);
1225
1226        // Create a switch case for each column in the matrix, where each case
1227        // extracts its column from the matrix. Finally we use OpPhi to return
1228        // the correct column.
1229        let merge_id = self.id_gen.next();
1230        block.body.push(Instruction::selection_merge(
1231            merge_id,
1232            spirv::SelectionControl::NONE,
1233        ));
1234        let cases = (0..columns as u32)
1235            .map(|i| super::instructions::Case {
1236                value: i,
1237                label_id: self.id_gen.next(),
1238            })
1239            .collect::<ArrayVec<_, 4>>();
1240
1241        // Which label we branch to in the default (column index out-of-bounds)
1242        // case depends on our bounds check policy.
1243        let default_id = match self.bounds_check_policies.index {
1244            // For `Restrict`, treat the same as the final column.
1245            crate::proc::BoundsCheckPolicy::Restrict => cases.last().unwrap().label_id,
1246            // For `ReadZeroSkipWrite`, branch directly to the merge block. This
1247            // will be handled in the `OpPhi` below to produce a zero value.
1248            crate::proc::BoundsCheckPolicy::ReadZeroSkipWrite => merge_id,
1249            // For `Unchecked` we create a new block containing an
1250            // `OpUnreachable`.
1251            crate::proc::BoundsCheckPolicy::Unchecked => self.id_gen.next(),
1252        };
1253        function.consume(
1254            block,
1255            Instruction::switch(column_index_param_id, default_id, &cases),
1256        );
1257
1258        // Emit a block for each case, and produce a list of variable and parent
1259        // block IDs that will be used in an `OpPhi` below to select the right
1260        // value.
1261        let mut var_parent_pairs = cases
1262            .into_iter()
1263            .map(|case| {
1264                let mut block = Block::new(case.label_id);
1265                let column_id = self.id_gen.next();
1266                block.body.push(Instruction::composite_extract(
1267                    column_type_id,
1268                    column_id,
1269                    matrix_param_id,
1270                    &[case.value],
1271                ));
1272                function.consume(block, Instruction::branch(merge_id));
1273                (column_id, case.label_id)
1274            })
1275            // Need capacity for up to 4 columns plus possibly a default case.
1276            .collect::<ArrayVec<_, 5>>();
1277
1278        // Emit a block or append the variable and parent `OpPhi` pair for the
1279        // column index out-of-bounds case, if required.
1280        match self.bounds_check_policies.index {
1281            // Don't need to do anything for `Restrict` as we have branched from
1282            // the final column case's block.
1283            crate::proc::BoundsCheckPolicy::Restrict => {}
1284            // For `ReadZeroSkipWrite` we have branched directly from the block
1285            // containing the `OpSwitch`. The `OpPhi` should produce a zero
1286            // value.
1287            crate::proc::BoundsCheckPolicy::ReadZeroSkipWrite => {
1288                var_parent_pairs.push((self.get_constant_null(column_type_id), label_id));
1289            }
1290            // For `Unchecked` create a new block containing `OpUnreachable`.
1291            // This does not need to be handled by the `OpPhi`.
1292            crate::proc::BoundsCheckPolicy::Unchecked => {
1293                function.consume(
1294                    Block::new(default_id),
1295                    Instruction::new(spirv::Op::Unreachable),
1296                );
1297            }
1298        }
1299
1300        let mut block = Block::new(merge_id);
1301        let result_id = self.id_gen.next();
1302        block.body.push(Instruction::phi(
1303            column_type_id,
1304            result_id,
1305            &var_parent_pairs,
1306        ));
1307
1308        function.consume(block, Instruction::return_value(result_id));
1309        function.to_words(&mut self.logical_layout.function_definitions);
1310        Ok(())
1311    }
1312
1313    fn write_function(
1314        &mut self,
1315        ir_function: &crate::Function,
1316        info: &FunctionInfo,
1317        ir_module: &crate::Module,
1318        mut interface: Option<FunctionInterface>,
1319        debug_info: &Option<DebugInfoInner>,
1320    ) -> Result<Word, Error> {
1321        self.write_wrapped_functions(ir_function, info, ir_module)?;
1322
1323        log::trace!("Generating code for {:?}", ir_function.name);
1324        let mut function = Function::default();
1325
1326        let prelude_id = self.id_gen.next();
1327        let mut prelude = Block::new(prelude_id);
1328        let mut ep_context = EntryPointContext {
1329            argument_ids: Vec::new(),
1330            results: Vec::new(),
1331            task_payload_variable_id: if let Some(ref i) = interface {
1332                i.task_payload.map(|a| self.global_variables[a].var_id)
1333            } else {
1334                None
1335            },
1336            mesh_state: None,
1337        };
1338
1339        let mut parameter_type_ids = Vec::with_capacity(ir_function.arguments.len());
1340
1341        let mut local_invocation_index_var_id = None;
1342        let mut local_invocation_index_id = None;
1343
1344        for argument in ir_function.arguments.iter() {
1345            let class = spirv::StorageClass::Input;
1346            let handle_ty = ir_module.types[argument.ty].inner.is_handle();
1347            let argument_type_id = if handle_ty {
1348                self.get_handle_pointer_type_id(argument.ty, spirv::StorageClass::UniformConstant)
1349            } else {
1350                self.get_handle_type_id(argument.ty)
1351            };
1352
1353            if let Some(ref mut iface) = interface {
1354                let id = if let Some(ref binding) = argument.binding {
1355                    let name = argument.name.as_deref();
1356
1357                    let varying_id = self.write_varying(
1358                        ir_module,
1359                        iface.stage,
1360                        class,
1361                        name,
1362                        argument.ty,
1363                        binding,
1364                    )?;
1365                    iface.varying_ids.push(varying_id);
1366                    let id = self.load_io_with_f16_polyfill(
1367                        &mut prelude.body,
1368                        varying_id,
1369                        argument_type_id,
1370                    );
1371                    if binding == &crate::Binding::BuiltIn(crate::BuiltIn::LocalInvocationIndex) {
1372                        local_invocation_index_id = Some(id);
1373                        local_invocation_index_var_id = Some(varying_id);
1374                    }
1375
1376                    id
1377                } else if let crate::TypeInner::Struct { ref members, .. } =
1378                    ir_module.types[argument.ty].inner
1379                {
1380                    let struct_id = self.id_gen.next();
1381                    let mut constituent_ids = Vec::with_capacity(members.len());
1382                    for member in members {
1383                        let type_id = self.get_handle_type_id(member.ty);
1384                        let name = member.name.as_deref();
1385                        let binding = member.binding.as_ref().unwrap();
1386                        let varying_id = self.write_varying(
1387                            ir_module,
1388                            iface.stage,
1389                            class,
1390                            name,
1391                            member.ty,
1392                            binding,
1393                        )?;
1394                        iface.varying_ids.push(varying_id);
1395                        let id =
1396                            self.load_io_with_f16_polyfill(&mut prelude.body, varying_id, type_id);
1397                        constituent_ids.push(id);
1398                        if binding == &crate::Binding::BuiltIn(crate::BuiltIn::LocalInvocationIndex)
1399                        {
1400                            local_invocation_index_id = Some(id);
1401                            local_invocation_index_var_id = Some(varying_id);
1402                        }
1403                    }
1404                    prelude.body.push(Instruction::composite_construct(
1405                        argument_type_id,
1406                        struct_id,
1407                        &constituent_ids,
1408                    ));
1409                    struct_id
1410                } else {
1411                    unreachable!("Missing argument binding on an entry point");
1412                };
1413                ep_context.argument_ids.push(id);
1414            } else {
1415                let argument_id = self.id_gen.next();
1416                let instruction = Instruction::function_parameter(argument_type_id, argument_id);
1417                if self.flags.contains(WriterFlags::DEBUG) {
1418                    if let Some(ref name) = argument.name {
1419                        self.debugs.push(Instruction::name(argument_id, name));
1420                    }
1421                }
1422                function.parameters.push(FunctionArgument {
1423                    instruction,
1424                    handle_id: if handle_ty {
1425                        let id = self.id_gen.next();
1426                        prelude.body.push(Instruction::load(
1427                            self.get_handle_type_id(argument.ty),
1428                            id,
1429                            argument_id,
1430                            None,
1431                        ));
1432                        id
1433                    } else {
1434                        0
1435                    },
1436                });
1437                parameter_type_ids.push(argument_type_id);
1438            };
1439        }
1440
1441        let return_type_id = match ir_function.result {
1442            Some(ref result) => {
1443                if let Some(ref mut iface) = interface {
1444                    let mut has_point_size = false;
1445                    let class = spirv::StorageClass::Output;
1446                    if let Some(ref binding) = result.binding {
1447                        has_point_size |=
1448                            *binding == crate::Binding::BuiltIn(crate::BuiltIn::PointSize);
1449                        let type_id = self.get_handle_type_id(result.ty);
1450                        let varying_id =
1451                            if *binding == crate::Binding::BuiltIn(crate::BuiltIn::MeshTaskSize) {
1452                                0
1453                            } else {
1454                                let varying_id = self.write_varying(
1455                                    ir_module,
1456                                    iface.stage,
1457                                    class,
1458                                    None,
1459                                    result.ty,
1460                                    binding,
1461                                )?;
1462                                iface.varying_ids.push(varying_id);
1463                                varying_id
1464                            };
1465                        ep_context.results.push(ResultMember {
1466                            id: varying_id,
1467                            type_id,
1468                            built_in: binding.to_built_in(),
1469                        });
1470                    } else if let crate::TypeInner::Struct { ref members, .. } =
1471                        ir_module.types[result.ty].inner
1472                    {
1473                        for member in members {
1474                            let type_id = self.get_handle_type_id(member.ty);
1475                            let name = member.name.as_deref();
1476                            let binding = member.binding.as_ref().unwrap();
1477                            has_point_size |=
1478                                *binding == crate::Binding::BuiltIn(crate::BuiltIn::PointSize);
1479                            // This isn't an actual builtin in SPIR-V. It can only appear as the
1480                            // output of a task shader and the output is used when writing the
1481                            // entry point return, in which case the id is ignored anyway.
1482                            let varying_id = if *binding
1483                                == crate::Binding::BuiltIn(crate::BuiltIn::MeshTaskSize)
1484                            {
1485                                0
1486                            } else {
1487                                let varying_id = self.write_varying(
1488                                    ir_module,
1489                                    iface.stage,
1490                                    class,
1491                                    name,
1492                                    member.ty,
1493                                    binding,
1494                                )?;
1495                                iface.varying_ids.push(varying_id);
1496                                varying_id
1497                            };
1498                            ep_context.results.push(ResultMember {
1499                                id: varying_id,
1500                                type_id,
1501                                built_in: binding.to_built_in(),
1502                            });
1503                        }
1504                    } else {
1505                        unreachable!("Missing result binding on an entry point");
1506                    }
1507
1508                    if self.flags.contains(WriterFlags::FORCE_POINT_SIZE)
1509                        && iface.stage == crate::ShaderStage::Vertex
1510                        && !has_point_size
1511                    {
1512                        // add point size artificially
1513                        let varying_id = self.id_gen.next();
1514                        let pointer_type_id = self.get_f32_pointer_type_id(class);
1515                        Instruction::variable(pointer_type_id, varying_id, class, None)
1516                            .to_words(&mut self.logical_layout.declarations);
1517                        self.decorate(
1518                            varying_id,
1519                            spirv::Decoration::BuiltIn,
1520                            &[spirv::BuiltIn::PointSize as u32],
1521                        );
1522                        iface.varying_ids.push(varying_id);
1523
1524                        let default_value_id = self.get_constant_scalar(crate::Literal::F32(1.0));
1525                        prelude
1526                            .body
1527                            .push(Instruction::store(varying_id, default_value_id, None));
1528                    }
1529                    if iface.stage == crate::ShaderStage::Task {
1530                        self.get_vec3u_type_id()
1531                    } else {
1532                        self.void_type
1533                    }
1534                } else {
1535                    self.get_handle_type_id(result.ty)
1536                }
1537            }
1538            None => self.void_type,
1539        };
1540
1541        if let Some(ref mut iface) = interface {
1542            if let Some(task_payload) = iface.task_payload {
1543                iface
1544                    .varying_ids
1545                    .push(self.global_variables[task_payload].var_id);
1546            }
1547            // The mesh output copy and the workgroup memory polyfill both read
1548            // `LocalInvocationIndex`, which an entry point may only declare once.
1549            if iface.mesh_info.is_some() && local_invocation_index_var_id.is_none() {
1550                let u32_type_id = self.get_u32_type_id();
1551                let pointer_type_id =
1552                    self.get_pointer_type_id(u32_type_id, spirv::StorageClass::Input);
1553                let var_id = self.id_gen.next();
1554                Instruction::variable(pointer_type_id, var_id, spirv::StorageClass::Input, None)
1555                    .to_words(&mut self.logical_layout.declarations);
1556                self.decorate(
1557                    var_id,
1558                    spirv::Decoration::BuiltIn,
1559                    &[spirv::BuiltIn::LocalInvocationIndex as u32],
1560                );
1561                iface.varying_ids.push(var_id);
1562
1563                let id = self.id_gen.next();
1564                prelude
1565                    .body
1566                    .push(Instruction::load(u32_type_id, id, var_id, None));
1567                local_invocation_index_var_id = Some(var_id);
1568                local_invocation_index_id = Some(id);
1569            }
1570            self.write_entry_point_mesh_shader_info(
1571                iface,
1572                local_invocation_index_var_id,
1573                ir_module,
1574                &mut ep_context,
1575            )?;
1576        }
1577
1578        let lookup_function_type = LookupFunctionType {
1579            parameter_type_ids,
1580            return_type_id,
1581        };
1582
1583        let function_id = self.id_gen.next();
1584        if self.flags.contains(WriterFlags::DEBUG) {
1585            if let Some(ref name) = ir_function.name {
1586                self.debugs.push(Instruction::name(function_id, name));
1587            }
1588        }
1589
1590        let function_type = self.get_function_type(lookup_function_type);
1591        function.signature = Some(Instruction::function(
1592            return_type_id,
1593            function_id,
1594            spirv::FunctionControl::empty(),
1595            function_type,
1596        ));
1597
1598        if interface.is_some() {
1599            function.entry_point_context = Some(ep_context);
1600        }
1601
1602        // fill up the `GlobalVariable::access_id`
1603        for gv in self.global_variables.iter_mut() {
1604            gv.reset_for_function();
1605        }
1606        for (handle, var) in ir_module.global_variables.iter() {
1607            if info[handle].is_empty() {
1608                continue;
1609            }
1610
1611            let mut gv = self.global_variables[handle].clone();
1612            if let Some(ref mut iface) = interface {
1613                // Have to include global variables in the interface
1614                if self.physical_layout.version >= 0x10400 && iface.task_payload != Some(handle) {
1615                    iface.varying_ids.push(gv.var_id);
1616                }
1617            }
1618
1619            match ir_module.types[var.ty].inner {
1620                // Any that are binding arrays we skip as we cannot load the array, we must load the result after indexing.
1621                crate::TypeInner::BindingArray { .. } => {
1622                    gv.access_id = gv.var_id;
1623                }
1624                _ => {
1625                    // Handle globals are pre-emitted and should be loaded automatically.
1626                    if var.space == crate::AddressSpace::Handle {
1627                        let var_type_id = self.get_handle_type_id(var.ty);
1628                        let id = self.id_gen.next();
1629                        prelude
1630                            .body
1631                            .push(Instruction::load(var_type_id, id, gv.var_id, None));
1632                        gv.access_id = gv.var_id;
1633                        gv.handle_id = id;
1634                    } else if global_needs_wrapper(ir_module, var) {
1635                        let class = map_storage_class(var.space);
1636                        let pointer_type_id = match self.std140_compat_uniform_types.get(&var.ty) {
1637                            Some(std140_type_info) if var.space == crate::AddressSpace::Uniform => {
1638                                self.get_pointer_type_id(std140_type_info.type_id, class)
1639                            }
1640                            _ => self.get_handle_pointer_type_id(var.ty, class),
1641                        };
1642                        let index_id = self.get_index_constant(0);
1643                        let id = self.id_gen.next();
1644                        prelude.body.push(Instruction::access_chain(
1645                            pointer_type_id,
1646                            id,
1647                            gv.var_id,
1648                            &[index_id],
1649                        ));
1650                        gv.access_id = id;
1651                    } else {
1652                        // by default, the variable ID is accessed as is
1653                        gv.access_id = gv.var_id;
1654                    };
1655                }
1656            }
1657
1658            // work around borrow checking in the presence of `self.xxx()` calls
1659            self.global_variables[handle] = gv;
1660        }
1661
1662        // Create a `BlockContext` for generating SPIR-V for the function's
1663        // body.
1664        let mut context = BlockContext {
1665            ir_module,
1666            ir_function,
1667            fun_info: info,
1668            function: &mut function,
1669            // Re-use the cached expression table from prior functions.
1670            cached: core::mem::take(&mut self.saved_cached),
1671
1672            // Steal the Writer's temp list for a bit.
1673            temp_list: core::mem::take(&mut self.temp_list),
1674            force_loop_bounding: self.force_loop_bounding,
1675            writer: self,
1676            expression_constness: super::ExpressionConstnessTracker::from_arena(
1677                &ir_function.expressions,
1678            ),
1679            ray_query_tracker_expr: crate::FastHashMap::default(),
1680        };
1681
1682        // fill up the pre-emitted and const expressions
1683        context.cached.reset(ir_function.expressions.len());
1684        for (handle, expr) in ir_function.expressions.iter() {
1685            if (expr.needs_pre_emit() && !matches!(*expr, crate::Expression::LocalVariable(_)))
1686                || context.expression_constness.is_const(handle)
1687            {
1688                context.cache_expression_value(handle, &mut prelude)?;
1689            }
1690        }
1691
1692        for (handle, variable) in ir_function.local_variables.iter() {
1693            let id = context.gen_id();
1694
1695            if context.writer.flags.contains(WriterFlags::DEBUG) {
1696                if let Some(ref name) = variable.name {
1697                    context.writer.debugs.push(Instruction::name(id, name));
1698                }
1699            }
1700
1701            let init_word = variable.init.map(|constant| context.cached[constant]);
1702            let pointer_type_id = context
1703                .writer
1704                .get_handle_pointer_type_id(variable.ty, spirv::StorageClass::Function);
1705            let instruction = Instruction::variable(
1706                pointer_type_id,
1707                id,
1708                spirv::StorageClass::Function,
1709                init_word.or_else(|| match ir_module.types[variable.ty].inner {
1710                    crate::TypeInner::RayQuery { .. } | crate::TypeInner::HitObject => None,
1711                    _ => {
1712                        let type_id = context.get_handle_type_id(variable.ty);
1713                        Some(context.writer.write_constant_null(type_id))
1714                    }
1715                }),
1716            );
1717
1718            context
1719                .function
1720                .variables
1721                .insert(handle, LocalVariable { id, instruction });
1722
1723            if let crate::TypeInner::RayQuery { .. } = ir_module.types[variable.ty].inner {
1724                // Don't refactor this into a struct: Although spirv itself allows opaque types in structs,
1725                // the vulkan environment for spirv does not. Putting ray queries into structs can cause
1726                // confusing bugs.
1727                let u32_type_id = context.writer.get_u32_type_id();
1728                let ptr_u32_type_id = context
1729                    .writer
1730                    .get_pointer_type_id(u32_type_id, spirv::StorageClass::Function);
1731                let tracker_id = context.gen_id();
1732                let tracker_init_id = context.writer.get_constant_scalar(crate::Literal::U32(
1733                    crate::back::RayQueryPoint::empty().bits(),
1734                ));
1735                let tracker_instruction = Instruction::variable(
1736                    ptr_u32_type_id,
1737                    tracker_id,
1738                    spirv::StorageClass::Function,
1739                    Some(tracker_init_id),
1740                );
1741
1742                context
1743                    .function
1744                    .ray_query_initialization_tracker_variables
1745                    .insert(
1746                        handle,
1747                        LocalVariable {
1748                            id: tracker_id,
1749                            instruction: tracker_instruction,
1750                        },
1751                    );
1752                let f32_type_id = context.writer.get_f32_type_id();
1753                let ptr_f32_type_id = context
1754                    .writer
1755                    .get_pointer_type_id(f32_type_id, spirv::StorageClass::Function);
1756                let t_max_tracker_id = context.gen_id();
1757                let t_max_tracker_init_id =
1758                    context.writer.get_constant_scalar(crate::Literal::F32(0.0));
1759                let t_max_tracker_instruction = Instruction::variable(
1760                    ptr_f32_type_id,
1761                    t_max_tracker_id,
1762                    spirv::StorageClass::Function,
1763                    Some(t_max_tracker_init_id),
1764                );
1765
1766                context.function.ray_query_t_max_tracker_variables.insert(
1767                    handle,
1768                    LocalVariable {
1769                        id: t_max_tracker_id,
1770                        instruction: t_max_tracker_instruction,
1771                    },
1772                );
1773            }
1774
1775            if let crate::TypeInner::HitObject = ir_module.types[variable.ty].inner {
1776                // SPIR-V gives a fresh hit object no defined state, so record it
1777                // as empty. This lands in the first block right after the
1778                // `OpVariable`s, so an untouched hit object reliably reports
1779                // `hitObjectIsEmpty`.
1780                prelude.body.push(Instruction::hit_object_record_empty(id));
1781            }
1782        }
1783
1784        for (handle, expr) in ir_function.expressions.iter() {
1785            match *expr {
1786                crate::Expression::LocalVariable(_) => {
1787                    // Cache the `OpVariable` instruction we generated above as
1788                    // the value of this expression.
1789                    context.cache_expression_value(handle, &mut prelude)?;
1790                }
1791                crate::Expression::Access { base, .. }
1792                | crate::Expression::AccessIndex { base, .. } => {
1793                    // Count references to `base` by `Access` and `AccessIndex`
1794                    // instructions. See `access_uses` for details.
1795                    *context.function.access_uses.entry(base).or_insert(0) += 1;
1796                }
1797                _ => {}
1798            }
1799        }
1800
1801        let next_id = context.gen_id();
1802
1803        context
1804            .function
1805            .consume(prelude, Instruction::branch(next_id));
1806
1807        let workgroup_vars_init_exit_block_id =
1808            match (context.writer.zero_initialize_workgroup_memory, interface) {
1809                (
1810                    super::ZeroInitializeWorkgroupMemoryMode::Polyfill,
1811                    Some(
1812                        ref mut interface @ FunctionInterface {
1813                            stage:
1814                                crate::ShaderStage::Compute
1815                                | crate::ShaderStage::Mesh
1816                                | crate::ShaderStage::Task,
1817                            ..
1818                        },
1819                    ),
1820                ) => context.writer.generate_workgroup_vars_init_block(
1821                    next_id,
1822                    ir_module,
1823                    info,
1824                    local_invocation_index_id,
1825                    interface,
1826                    context.function,
1827                ),
1828                _ => None,
1829            };
1830
1831        let main_id = if let Some(exit_id) = workgroup_vars_init_exit_block_id {
1832            exit_id
1833        } else {
1834            next_id
1835        };
1836
1837        context.write_function_body(main_id, debug_info.as_ref())?;
1838
1839        // Consume the `BlockContext`, ending its borrows and letting the
1840        // `Writer` steal back its cached expression table and temp_list.
1841        let BlockContext {
1842            cached, temp_list, ..
1843        } = context;
1844        self.saved_cached = cached;
1845        self.temp_list = temp_list;
1846
1847        function.to_words(&mut self.logical_layout.function_definitions);
1848
1849        if let Some(EntryPointContext {
1850            mesh_state: Some(ref mesh_state),
1851            ..
1852        }) = function.entry_point_context
1853        {
1854            self.write_mesh_shader_wrapper(mesh_state, function_id)
1855        } else if let Some(EntryPointContext {
1856            task_payload_variable_id: Some(tp),
1857            ..
1858        }) = function.entry_point_context
1859        {
1860            self.write_task_shader_wrapper(tp, function_id)
1861        } else {
1862            Ok(function_id)
1863        }
1864    }
1865
1866    fn write_execution_mode(
1867        &mut self,
1868        function_id: Word,
1869        mode: spirv::ExecutionMode,
1870    ) -> Result<(), Error> {
1871        //self.check(mode.required_capabilities())?;
1872        Instruction::execution_mode(function_id, mode, &[])
1873            .to_words(&mut self.logical_layout.execution_modes);
1874        Ok(())
1875    }
1876
1877    // TODO Move to instructions module
1878    fn write_entry_point(
1879        &mut self,
1880        entry_point: &crate::EntryPoint,
1881        info: &FunctionInfo,
1882        ir_module: &crate::Module,
1883        debug_info: &Option<DebugInfoInner>,
1884    ) -> Result<Instruction, Error> {
1885        let mut interface_ids = Vec::new();
1886
1887        let function_id = self.write_function(
1888            &entry_point.function,
1889            info,
1890            ir_module,
1891            Some(FunctionInterface {
1892                varying_ids: &mut interface_ids,
1893                stage: entry_point.stage,
1894                task_payload: entry_point.task_payload,
1895                mesh_info: entry_point.mesh_info.clone(),
1896                workgroup_size: entry_point.workgroup_size,
1897            }),
1898            debug_info,
1899        )?;
1900
1901        let exec_model = match entry_point.stage {
1902            crate::ShaderStage::Vertex => spirv::ExecutionModel::Vertex,
1903            crate::ShaderStage::Fragment => {
1904                self.write_execution_mode(function_id, spirv::ExecutionMode::OriginUpperLeft)?;
1905                match entry_point.early_depth_test {
1906                    Some(crate::EarlyDepthTest::Force) => {
1907                        self.write_execution_mode(
1908                            function_id,
1909                            spirv::ExecutionMode::EarlyFragmentTests,
1910                        )?;
1911                    }
1912                    Some(crate::EarlyDepthTest::Allow { conservative }) => {
1913                        // TODO: Consider emitting EarlyAndLateFragmentTestsAMD here, if available.
1914                        // https://github.khronos.org/SPIRV-Registry/extensions/AMD/SPV_AMD_shader_early_and_late_fragment_tests.html
1915                        // This permits early depth tests even if the shader writes to a storage
1916                        // binding
1917                        match conservative {
1918                            crate::ConservativeDepth::GreaterEqual => self.write_execution_mode(
1919                                function_id,
1920                                spirv::ExecutionMode::DepthGreater,
1921                            )?,
1922                            crate::ConservativeDepth::LessEqual => self.write_execution_mode(
1923                                function_id,
1924                                spirv::ExecutionMode::DepthLess,
1925                            )?,
1926                            crate::ConservativeDepth::Unchanged => self.write_execution_mode(
1927                                function_id,
1928                                spirv::ExecutionMode::DepthUnchanged,
1929                            )?,
1930                        }
1931                    }
1932                    None => {}
1933                }
1934                if let Some(ref result) = entry_point.function.result {
1935                    if contains_builtin(
1936                        result.binding.as_ref(),
1937                        result.ty,
1938                        &ir_module.types,
1939                        crate::BuiltIn::FragDepth,
1940                    ) {
1941                        self.write_execution_mode(
1942                            function_id,
1943                            spirv::ExecutionMode::DepthReplacing,
1944                        )?;
1945                    }
1946                }
1947                spirv::ExecutionModel::Fragment
1948            }
1949            crate::ShaderStage::Compute => {
1950                let execution_mode = spirv::ExecutionMode::LocalSize;
1951                Instruction::execution_mode(
1952                    function_id,
1953                    execution_mode,
1954                    &entry_point.workgroup_size,
1955                )
1956                .to_words(&mut self.logical_layout.execution_modes);
1957                spirv::ExecutionModel::GLCompute
1958            }
1959            crate::ShaderStage::Task => {
1960                let execution_mode = spirv::ExecutionMode::LocalSize;
1961                Instruction::execution_mode(
1962                    function_id,
1963                    execution_mode,
1964                    &entry_point.workgroup_size,
1965                )
1966                .to_words(&mut self.logical_layout.execution_modes);
1967                spirv::ExecutionModel::TaskEXT
1968            }
1969            crate::ShaderStage::Mesh => {
1970                let execution_mode = spirv::ExecutionMode::LocalSize;
1971                Instruction::execution_mode(
1972                    function_id,
1973                    execution_mode,
1974                    &entry_point.workgroup_size,
1975                )
1976                .to_words(&mut self.logical_layout.execution_modes);
1977                let mesh_info = entry_point.mesh_info.as_ref().unwrap();
1978                Instruction::execution_mode(
1979                    function_id,
1980                    match mesh_info.topology {
1981                        crate::MeshOutputTopology::Points => spirv::ExecutionMode::OutputPoints,
1982                        crate::MeshOutputTopology::Lines => spirv::ExecutionMode::OutputLinesEXT,
1983                        crate::MeshOutputTopology::Triangles => {
1984                            spirv::ExecutionMode::OutputTrianglesEXT
1985                        }
1986                    },
1987                    &[],
1988                )
1989                .to_words(&mut self.logical_layout.execution_modes);
1990                Instruction::execution_mode(
1991                    function_id,
1992                    spirv::ExecutionMode::OutputVertices,
1993                    core::slice::from_ref(&mesh_info.max_vertices),
1994                )
1995                .to_words(&mut self.logical_layout.execution_modes);
1996                Instruction::execution_mode(
1997                    function_id,
1998                    spirv::ExecutionMode::OutputPrimitivesEXT,
1999                    core::slice::from_ref(&mesh_info.max_primitives),
2000                )
2001                .to_words(&mut self.logical_layout.execution_modes);
2002                spirv::ExecutionModel::MeshEXT
2003            }
2004            crate::ShaderStage::RayGeneration => {
2005                self.require_any("ray tracing pipelines", &[spirv::Capability::RayTracingKHR])?;
2006                spirv::ExecutionModel::RayGenerationKHR
2007            }
2008            crate::ShaderStage::AnyHit => {
2009                self.require_any("ray tracing pipelines", &[spirv::Capability::RayTracingKHR])?;
2010                spirv::ExecutionModel::AnyHitKHR
2011            }
2012            crate::ShaderStage::ClosestHit => {
2013                self.require_any("ray tracing pipelines", &[spirv::Capability::RayTracingKHR])?;
2014                spirv::ExecutionModel::ClosestHitKHR
2015            }
2016            crate::ShaderStage::Miss => {
2017                self.require_any("ray tracing pipelines", &[spirv::Capability::RayTracingKHR])?;
2018                spirv::ExecutionModel::MissKHR
2019            }
2020        };
2021        //self.check(exec_model.required_capabilities())?;
2022
2023        // `HitObjectAttributeEXT` variables are only permitted in these stages.
2024        if matches!(
2025            exec_model,
2026            spirv::ExecutionModel::RayGenerationKHR
2027                | spirv::ExecutionModel::ClosestHitKHR
2028                | spirv::ExecutionModel::MissKHR
2029        ) {
2030            if let Some(id) = self.hit_object_attribute_var {
2031                interface_ids.push(id);
2032            }
2033        }
2034
2035        Ok(Instruction::entry_point(
2036            exec_model,
2037            function_id,
2038            &entry_point.name,
2039            interface_ids.as_slice(),
2040        ))
2041    }
2042
2043    fn make_scalar(&mut self, id: Word, scalar: crate::Scalar) -> Instruction {
2044        use crate::ScalarKind as Sk;
2045
2046        let bits = (scalar.width * BITS_PER_BYTE) as u32;
2047        match scalar.kind {
2048            Sk::Sint | Sk::Uint => {
2049                let signedness = if scalar.kind == Sk::Sint {
2050                    super::instructions::Signedness::Signed
2051                } else {
2052                    super::instructions::Signedness::Unsigned
2053                };
2054                let cap = match bits {
2055                    8 => Some(spirv::Capability::Int8),
2056                    16 => Some(spirv::Capability::Int16),
2057                    64 => Some(spirv::Capability::Int64),
2058                    _ => None,
2059                };
2060                if let Some(cap) = cap {
2061                    self.capabilities_used.insert(cap);
2062                }
2063                if bits == 16 {
2064                    self.capabilities_used
2065                        .insert(spirv::Capability::StorageBuffer16BitAccess);
2066                    self.capabilities_used
2067                        .insert(spirv::Capability::UniformAndStorageBuffer16BitAccess);
2068                    if self.use_storage_input_output_16 {
2069                        self.capabilities_used
2070                            .insert(spirv::Capability::StorageInputOutput16);
2071                    }
2072                }
2073                Instruction::type_int(id, bits, signedness)
2074            }
2075            Sk::Float => {
2076                if bits == 64 {
2077                    self.capabilities_used.insert(spirv::Capability::Float64);
2078                }
2079                if bits == 16 {
2080                    self.capabilities_used.insert(spirv::Capability::Float16);
2081                    self.capabilities_used
2082                        .insert(spirv::Capability::StorageBuffer16BitAccess);
2083                    self.capabilities_used
2084                        .insert(spirv::Capability::UniformAndStorageBuffer16BitAccess);
2085                    if self.use_storage_input_output_16 {
2086                        self.capabilities_used
2087                            .insert(spirv::Capability::StorageInputOutput16);
2088                    }
2089                }
2090                Instruction::type_float(id, bits)
2091            }
2092            Sk::Bool => Instruction::type_bool(id),
2093            Sk::AbstractInt | Sk::AbstractFloat => {
2094                unreachable!("abstract types should never reach the backend");
2095            }
2096        }
2097    }
2098
2099    fn request_type_capabilities(&mut self, inner: &crate::TypeInner) -> Result<(), Error> {
2100        match *inner {
2101            crate::TypeInner::Image {
2102                dim,
2103                arrayed,
2104                class,
2105            } => {
2106                let sampled = match class {
2107                    crate::ImageClass::Sampled { .. } => true,
2108                    crate::ImageClass::Depth { .. } => true,
2109                    crate::ImageClass::Storage { format, .. } => {
2110                        self.request_image_format_capabilities(format.into())?;
2111                        false
2112                    }
2113                    crate::ImageClass::External => unimplemented!(),
2114                };
2115
2116                match dim {
2117                    crate::ImageDimension::D1 => {
2118                        if sampled {
2119                            self.require_any("sampled 1D images", &[spirv::Capability::Sampled1D])?;
2120                        } else {
2121                            self.require_any("1D storage images", &[spirv::Capability::Image1D])?;
2122                        }
2123                    }
2124                    crate::ImageDimension::Cube if arrayed => {
2125                        if sampled {
2126                            self.require_any(
2127                                "sampled cube array images",
2128                                &[spirv::Capability::SampledCubeArray],
2129                            )?;
2130                        } else {
2131                            self.require_any(
2132                                "cube array storage images",
2133                                &[spirv::Capability::ImageCubeArray],
2134                            )?;
2135                        }
2136                    }
2137                    _ => {}
2138                }
2139            }
2140            crate::TypeInner::AccelerationStructure { .. } => {
2141                self.require_any(
2142                    "Acceleration Structure",
2143                    // unless we use this conditional, the ray query snapshot
2144                    // tests pick the wrong capability
2145                    &[if self.has_ray_tracing_pipeline {
2146                        spirv::Capability::RayTracingKHR
2147                    } else {
2148                        spirv::Capability::RayQueryKHR
2149                    }],
2150                )?;
2151            }
2152            crate::TypeInner::RayQuery { .. } => {
2153                self.require_any("Ray Query", &[spirv::Capability::RayQueryKHR])?;
2154            }
2155            crate::TypeInner::HitObject => {
2156                self.require_hit_objects()?;
2157            }
2158            crate::TypeInner::Atomic(crate::Scalar { width: 8, kind: _ }) => {
2159                self.require_any("64 bit integer atomics", &[spirv::Capability::Int64Atomics])?;
2160            }
2161            crate::TypeInner::Atomic(crate::Scalar {
2162                width: 4,
2163                kind: crate::ScalarKind::Float,
2164            }) => {
2165                self.require_any(
2166                    "32 bit floating-point atomics",
2167                    &[spirv::Capability::AtomicFloat32AddEXT],
2168                )?;
2169                self.use_extension("SPV_EXT_shader_atomic_float_add");
2170            }
2171            // 16 bit floating-point support requires Float16 capability
2172            crate::TypeInner::Matrix {
2173                scalar: crate::Scalar::F16,
2174                ..
2175            }
2176            | crate::TypeInner::Vector {
2177                scalar: crate::Scalar::F16,
2178                ..
2179            }
2180            | crate::TypeInner::Scalar(crate::Scalar::F16) => {
2181                self.require_any("16 bit floating-point", &[spirv::Capability::Float16])?;
2182                self.use_extension("SPV_KHR_16bit_storage");
2183            }
2184            // 16 bit integer support requires Int16 capability
2185            crate::TypeInner::Vector {
2186                scalar:
2187                    crate::Scalar {
2188                        kind: crate::ScalarKind::Sint | crate::ScalarKind::Uint,
2189                        width: 2,
2190                    },
2191                ..
2192            }
2193            | crate::TypeInner::Scalar(crate::Scalar {
2194                kind: crate::ScalarKind::Sint | crate::ScalarKind::Uint,
2195                width: 2,
2196            }) => {
2197                self.require_any("16 bit integer", &[spirv::Capability::Int16])?;
2198                self.use_extension("SPV_KHR_16bit_storage");
2199            }
2200            // Cooperative types and ops
2201            crate::TypeInner::CooperativeMatrix { .. } => {
2202                self.require_any(
2203                    "cooperative matrix",
2204                    &[spirv::Capability::CooperativeMatrixKHR],
2205                )?;
2206                self.require_any("memory model", &[spirv::Capability::VulkanMemoryModel])?;
2207                self.use_extension("SPV_KHR_cooperative_matrix");
2208                self.use_extension("SPV_KHR_vulkan_memory_model");
2209            }
2210            _ => {}
2211        }
2212        Ok(())
2213    }
2214
2215    fn write_numeric_type_declaration_local(&mut self, id: Word, numeric: NumericType) {
2216        let instruction = match numeric {
2217            NumericType::Scalar(scalar) => self.make_scalar(id, scalar),
2218            NumericType::Vector { size, scalar } => {
2219                let scalar_id = self.get_numeric_type_id(NumericType::Scalar(scalar));
2220                Instruction::type_vector(id, scalar_id, size)
2221            }
2222            NumericType::Matrix {
2223                columns,
2224                rows,
2225                scalar,
2226            } => {
2227                let column_id =
2228                    self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
2229                Instruction::type_matrix(id, column_id, columns)
2230            }
2231        };
2232
2233        instruction.to_words(&mut self.logical_layout.declarations);
2234    }
2235
2236    fn write_cooperative_type_declaration_local(&mut self, id: Word, coop: CooperativeType) {
2237        let instruction = match coop {
2238            CooperativeType::Matrix {
2239                columns,
2240                rows,
2241                scalar,
2242                role,
2243            } => {
2244                let scalar_id =
2245                    self.get_localtype_id(LocalType::Numeric(NumericType::Scalar(scalar)));
2246                let scope_id = self.get_index_constant(spirv::Scope::Subgroup as u32);
2247                let columns_id = self.get_index_constant(columns as u32);
2248                let rows_id = self.get_index_constant(rows as u32);
2249                let role_id =
2250                    self.get_index_constant(spirv::CooperativeMatrixUse::from(role) as u32);
2251                Instruction::type_coop_matrix(id, scalar_id, scope_id, rows_id, columns_id, role_id)
2252            }
2253        };
2254
2255        instruction.to_words(&mut self.logical_layout.declarations);
2256    }
2257
2258    fn write_type_declaration_local(&mut self, id: Word, local_ty: LocalType) {
2259        let instruction = match local_ty {
2260            LocalType::Numeric(numeric) => {
2261                self.write_numeric_type_declaration_local(id, numeric);
2262                return;
2263            }
2264            LocalType::Cooperative(coop) => {
2265                self.write_cooperative_type_declaration_local(id, coop);
2266                return;
2267            }
2268            LocalType::Pointer { base, class } => Instruction::type_pointer(id, class, base),
2269            LocalType::Image(image) => {
2270                let local_type = LocalType::Numeric(NumericType::Scalar(image.sampled_type));
2271                let type_id = self.get_localtype_id(local_type);
2272                Instruction::type_image(id, type_id, image.dim, image.flags, image.image_format)
2273            }
2274            LocalType::Sampler => Instruction::type_sampler(id),
2275            LocalType::SampledImage { image_type_id } => {
2276                Instruction::type_sampled_image(id, image_type_id)
2277            }
2278            LocalType::BindingArray { base, size } => {
2279                let inner_ty = self.get_handle_type_id(base);
2280                let scalar_id = self.get_constant_scalar(crate::Literal::U32(size));
2281                Instruction::type_array(id, inner_ty, scalar_id)
2282            }
2283            LocalType::AccelerationStructure => Instruction::type_acceleration_structure(id),
2284            LocalType::RayQuery => Instruction::type_ray_query(id),
2285            LocalType::HitObject => Instruction::type_hit_object(id),
2286        };
2287
2288        instruction.to_words(&mut self.logical_layout.declarations);
2289    }
2290
2291    fn write_type_declaration_arena(
2292        &mut self,
2293        module: &crate::Module,
2294        handle: Handle<crate::Type>,
2295    ) -> Result<Word, Error> {
2296        let ty = &module.types[handle];
2297        // If it's a type that needs SPIR-V capabilities, request them now.
2298        // This needs to happen regardless of the LocalType lookup succeeding,
2299        // because some types which map to the same LocalType have different
2300        // capability requirements. See https://github.com/gfx-rs/wgpu/issues/5569
2301        self.request_type_capabilities(&ty.inner)?;
2302        let id = if let Some(local) = self.localtype_from_inner(&ty.inner) {
2303            // This type can be represented as a `LocalType`, so check if we've
2304            // already written an instruction for it. If not, do so now, with
2305            // `write_type_declaration_local`.
2306            match self.lookup_type.entry(LookupType::Local(local)) {
2307                // We already have an id for this `LocalType`.
2308                Entry::Occupied(e) => *e.get(),
2309
2310                // It's a type we haven't seen before.
2311                Entry::Vacant(e) => {
2312                    let id = self.id_gen.next();
2313                    e.insert(id);
2314
2315                    self.write_type_declaration_local(id, local);
2316
2317                    id
2318                }
2319            }
2320        } else {
2321            use spirv::Decoration;
2322
2323            let id = self.id_gen.next();
2324            let instruction = match ty.inner {
2325                crate::TypeInner::Array { base, size, stride } => {
2326                    self.decorate(id, Decoration::ArrayStride, &[stride]);
2327
2328                    let type_id = self.get_handle_type_id(base);
2329                    match size.resolve(module.to_ctx())? {
2330                        crate::proc::IndexableLength::Known(length) => {
2331                            let length_id = self.get_index_constant(length);
2332                            Instruction::type_array(id, type_id, length_id)
2333                        }
2334                        crate::proc::IndexableLength::Dynamic => {
2335                            Instruction::type_runtime_array(id, type_id)
2336                        }
2337                    }
2338                }
2339                crate::TypeInner::BindingArray { base, size } => {
2340                    let type_id = self.get_handle_type_id(base);
2341                    match size.resolve(module.to_ctx())? {
2342                        crate::proc::IndexableLength::Known(length) => {
2343                            let length_id = self.get_index_constant(length);
2344                            Instruction::type_array(id, type_id, length_id)
2345                        }
2346                        crate::proc::IndexableLength::Dynamic => {
2347                            Instruction::type_runtime_array(id, type_id)
2348                        }
2349                    }
2350                }
2351                crate::TypeInner::Struct {
2352                    ref members,
2353                    span: _,
2354                } => {
2355                    let mut has_runtime_array = false;
2356                    let mut member_ids = Vec::with_capacity(members.len());
2357                    for (index, member) in members.iter().enumerate() {
2358                        let member_ty = &module.types[member.ty];
2359                        match member_ty.inner {
2360                            crate::TypeInner::Array {
2361                                base: _,
2362                                size: crate::ArraySize::Dynamic,
2363                                stride: _,
2364                            } => {
2365                                has_runtime_array = true;
2366                            }
2367                            _ => (),
2368                        }
2369                        self.decorate_struct_member(id, index, member, &module.types)?;
2370                        let member_id = self.get_handle_type_id(member.ty);
2371                        member_ids.push(member_id);
2372                    }
2373                    if has_runtime_array {
2374                        self.decorate(id, Decoration::Block, &[]);
2375                    }
2376                    Instruction::type_struct(id, member_ids.as_slice())
2377                }
2378
2379                // These all have TypeLocal representations, so they should have been
2380                // handled by `write_type_declaration_local` above.
2381                crate::TypeInner::Scalar(_)
2382                | crate::TypeInner::Atomic(_)
2383                | crate::TypeInner::Vector { .. }
2384                | crate::TypeInner::Matrix { .. }
2385                | crate::TypeInner::CooperativeMatrix { .. }
2386                | crate::TypeInner::Pointer { .. }
2387                | crate::TypeInner::ValuePointer { .. }
2388                | crate::TypeInner::Image { .. }
2389                | crate::TypeInner::Sampler { .. }
2390                | crate::TypeInner::AccelerationStructure { .. }
2391                | crate::TypeInner::RayQuery { .. }
2392                | crate::TypeInner::HitObject => unreachable!(),
2393            };
2394
2395            instruction.to_words(&mut self.logical_layout.declarations);
2396            id
2397        };
2398
2399        // Add this handle as a new alias for that type.
2400        self.lookup_type.insert(LookupType::Handle(handle), id);
2401
2402        if self.flags.contains(WriterFlags::DEBUG) {
2403            if let Some(ref name) = ty.name {
2404                self.debugs.push(Instruction::name(id, name));
2405            }
2406        }
2407
2408        Ok(id)
2409    }
2410
2411    /// Writes a std140 layout compatible type declaration for a type. Returns
2412    /// the ID of the declared type, or None if no declaration is required.
2413    ///
2414    /// This should be called for any type for which there exists a
2415    /// [`GlobalVariable`] in the [`Uniform`] address space. If the type already
2416    /// adheres to std140 layout rules it will return without declaring any
2417    /// types. If the type contains another type which requires a std140
2418    /// compatible type declaration, it will recursively call itself.
2419    ///
2420    /// When `handle` refers to a [`TypeInner::Matrix`] with 2 rows, the
2421    /// declared type will be an `OpTypeStruct` containing an `OpVector` for
2422    /// each of the matrix's columns.
2423    ///
2424    /// When `handle` refers to a [`TypeInner::Array`] whose base type is a
2425    /// matrix with 2 rows, this will declare an `OpTypeArray` whose element
2426    /// type is the matrix's corresponding std140 compatible type.
2427    ///
2428    /// When `handle` refers to a [`TypeInner::Struct`] and any of its members
2429    /// require a std140 compatible type declaration, this will declare a new
2430    /// struct with the following rules:
2431    /// * Struct or array members will be declared with their std140 compatible
2432    ///   type declaration, if one is required.
2433    /// * Two-row matrix members will have each of their columns hoisted
2434    ///   directly into the struct as 2-component vector members.
2435    /// * All other members will be declared with their normal type.
2436    ///
2437    /// Note that this means the Naga IR index of a struct member may not match
2438    /// the index in the generated SPIR-V. The mapping can be obtained via
2439    /// `Std140TypeInfo::member_indices`.
2440    ///
2441    /// [`GlobalVariable`]: crate::GlobalVariable
2442    /// [`Uniform`]: crate::AddressSpace::Uniform
2443    /// [`TypeInner::Matrix`]: crate::TypeInner::Matrix
2444    /// [`TypeInner::Array`]: crate::TypeInner::Array
2445    /// [`TypeInner::Struct`]: crate::TypeInner::Struct
2446    fn write_std140_compat_type_declaration(
2447        &mut self,
2448        module: &crate::Module,
2449        handle: Handle<crate::Type>,
2450    ) -> Result<Option<Word>, Error> {
2451        if let Some(std140_type_info) = self.std140_compat_uniform_types.get(&handle) {
2452            return Ok(Some(std140_type_info.type_id));
2453        }
2454
2455        let type_inner = &module.types[handle].inner;
2456        let std140_type_id = match *type_inner {
2457            crate::TypeInner::Matrix {
2458                columns,
2459                rows: rows @ crate::VectorSize::Bi,
2460                scalar,
2461            } => {
2462                let std140_type_id = self.id_gen.next();
2463                let mut member_type_ids: ArrayVec<Word, 4> = ArrayVec::new();
2464                let column_type_id =
2465                    self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
2466                for column in 0..columns as u32 {
2467                    member_type_ids.push(column_type_id);
2468                    self.annotations.push(Instruction::member_decorate(
2469                        std140_type_id,
2470                        column,
2471                        spirv::Decoration::Offset,
2472                        &[column * rows as u32 * scalar.width as u32],
2473                    ));
2474                    if self.flags.contains(WriterFlags::DEBUG) {
2475                        self.debugs.push(Instruction::member_name(
2476                            std140_type_id,
2477                            column,
2478                            &format!("col{column}"),
2479                        ));
2480                    }
2481                }
2482                Instruction::type_struct(std140_type_id, &member_type_ids)
2483                    .to_words(&mut self.logical_layout.declarations);
2484                self.std140_compat_uniform_types.insert(
2485                    handle,
2486                    Std140CompatTypeInfo {
2487                        type_id: std140_type_id,
2488                        member_indices: Vec::new(),
2489                    },
2490                );
2491                Some(std140_type_id)
2492            }
2493            crate::TypeInner::Array { base, size, stride } => {
2494                match self.write_std140_compat_type_declaration(module, base)? {
2495                    Some(std140_base_type_id) => {
2496                        let std140_type_id = self.id_gen.next();
2497                        self.decorate(std140_type_id, spirv::Decoration::ArrayStride, &[stride]);
2498                        let instruction = match size.resolve(module.to_ctx())? {
2499                            crate::proc::IndexableLength::Known(length) => {
2500                                let length_id = self.get_index_constant(length);
2501                                Instruction::type_array(
2502                                    std140_type_id,
2503                                    std140_base_type_id,
2504                                    length_id,
2505                                )
2506                            }
2507                            crate::proc::IndexableLength::Dynamic => {
2508                                unreachable!()
2509                            }
2510                        };
2511                        instruction.to_words(&mut self.logical_layout.declarations);
2512                        self.std140_compat_uniform_types.insert(
2513                            handle,
2514                            Std140CompatTypeInfo {
2515                                type_id: std140_type_id,
2516                                member_indices: Vec::new(),
2517                            },
2518                        );
2519                        Some(std140_type_id)
2520                    }
2521                    None => None,
2522                }
2523            }
2524            crate::TypeInner::Struct { ref members, .. } => {
2525                let mut needs_std140_type = false;
2526                for member in members {
2527                    match module.types[member.ty].inner {
2528                        // We don't need to write a std140 type for the matrix itself as
2529                        // it will be decomposed into the parent struct. As a result, the
2530                        // struct does need a std140 type, however.
2531                        crate::TypeInner::Matrix {
2532                            rows: crate::VectorSize::Bi,
2533                            ..
2534                        } => needs_std140_type = true,
2535                        // If an array or struct member needs a std140 type, because it
2536                        // (transitively) contains a `matCx2`, then the struct also needs
2537                        // a std140 type which uses the std140 type for this member.
2538                        crate::TypeInner::Array { .. } | crate::TypeInner::Struct { .. }
2539                            if self
2540                                .write_std140_compat_type_declaration(module, member.ty)?
2541                                .is_some() =>
2542                        {
2543                            needs_std140_type = true;
2544                        }
2545                        _ => {}
2546                    }
2547                }
2548
2549                if needs_std140_type {
2550                    let std140_type_id = self.id_gen.next();
2551                    let mut member_ids = Vec::new();
2552                    let mut member_indices = Vec::new();
2553                    let mut next_index = 0;
2554
2555                    for member in members {
2556                        member_indices.push(next_index);
2557                        match module.types[member.ty].inner {
2558                            crate::TypeInner::Matrix {
2559                                columns,
2560                                rows: rows @ crate::VectorSize::Bi,
2561                                scalar,
2562                            } => {
2563                                let vector_type_id =
2564                                    self.get_numeric_type_id(NumericType::Vector {
2565                                        size: rows,
2566                                        scalar,
2567                                    });
2568                                for column in 0..columns as u32 {
2569                                    self.annotations.push(Instruction::member_decorate(
2570                                        std140_type_id,
2571                                        next_index,
2572                                        spirv::Decoration::Offset,
2573                                        &[member.offset
2574                                            + column * rows as u32 * scalar.width as u32],
2575                                    ));
2576                                    if self.flags.contains(WriterFlags::DEBUG) {
2577                                        if let Some(ref name) = member.name {
2578                                            self.debugs.push(Instruction::member_name(
2579                                                std140_type_id,
2580                                                next_index,
2581                                                &format!("{name}_col{column}"),
2582                                            ));
2583                                        }
2584                                    }
2585                                    member_ids.push(vector_type_id);
2586                                    next_index += 1;
2587                                }
2588                            }
2589                            _ => {
2590                                let member_id =
2591                                    match self.std140_compat_uniform_types.get(&member.ty) {
2592                                        Some(std140_member_type_info) => {
2593                                            self.annotations.push(Instruction::member_decorate(
2594                                                std140_type_id,
2595                                                next_index,
2596                                                spirv::Decoration::Offset,
2597                                                &[member.offset],
2598                                            ));
2599                                            if self.flags.contains(WriterFlags::DEBUG) {
2600                                                if let Some(ref name) = member.name {
2601                                                    self.debugs.push(Instruction::member_name(
2602                                                        std140_type_id,
2603                                                        next_index,
2604                                                        name,
2605                                                    ));
2606                                                }
2607                                            }
2608                                            std140_member_type_info.type_id
2609                                        }
2610                                        None => {
2611                                            self.decorate_struct_member(
2612                                                std140_type_id,
2613                                                next_index as usize,
2614                                                member,
2615                                                &module.types,
2616                                            )?;
2617                                            self.get_handle_type_id(member.ty)
2618                                        }
2619                                    };
2620                                member_ids.push(member_id);
2621                                next_index += 1;
2622                            }
2623                        }
2624                    }
2625
2626                    Instruction::type_struct(std140_type_id, &member_ids)
2627                        .to_words(&mut self.logical_layout.declarations);
2628                    self.std140_compat_uniform_types.insert(
2629                        handle,
2630                        Std140CompatTypeInfo {
2631                            type_id: std140_type_id,
2632                            member_indices,
2633                        },
2634                    );
2635                    Some(std140_type_id)
2636                } else {
2637                    None
2638                }
2639            }
2640            _ => None,
2641        };
2642
2643        if let Some(std140_type_id) = std140_type_id {
2644            if self.flags.contains(WriterFlags::DEBUG) {
2645                let name = format!("std140_{:?}", handle.for_debug(&module.types));
2646                self.debugs.push(Instruction::name(std140_type_id, &name));
2647            }
2648        }
2649        Ok(std140_type_id)
2650    }
2651
2652    fn request_image_format_capabilities(
2653        &mut self,
2654        format: spirv::ImageFormat,
2655    ) -> Result<(), Error> {
2656        use spirv::ImageFormat as If;
2657        match format {
2658            If::Rg32f
2659            | If::Rg16f
2660            | If::R11fG11fB10f
2661            | If::R16f
2662            | If::Rgba16
2663            | If::Rgb10A2
2664            | If::Rg16
2665            | If::Rg8
2666            | If::R16
2667            | If::R8
2668            | If::Rgba16Snorm
2669            | If::Rg16Snorm
2670            | If::Rg8Snorm
2671            | If::R16Snorm
2672            | If::R8Snorm
2673            | If::Rg32i
2674            | If::Rg16i
2675            | If::Rg8i
2676            | If::R16i
2677            | If::R8i
2678            | If::Rgb10a2ui
2679            | If::Rg32ui
2680            | If::Rg16ui
2681            | If::Rg8ui
2682            | If::R16ui
2683            | If::R8ui => self.require_any(
2684                "storage image format",
2685                &[spirv::Capability::StorageImageExtendedFormats],
2686            ),
2687            If::R64ui | If::R64i => {
2688                self.use_extension("SPV_EXT_shader_image_int64");
2689                self.require_any(
2690                    "64-bit integer storage image format",
2691                    &[spirv::Capability::Int64ImageEXT],
2692                )
2693            }
2694            If::Unknown
2695            | If::Rgba32f
2696            | If::Rgba16f
2697            | If::R32f
2698            | If::Rgba8
2699            | If::Rgba8Snorm
2700            | If::Rgba32i
2701            | If::Rgba16i
2702            | If::Rgba8i
2703            | If::R32i
2704            | If::Rgba32ui
2705            | If::Rgba16ui
2706            | If::Rgba8ui
2707            | If::R32ui => Ok(()),
2708        }
2709    }
2710
2711    pub(super) fn get_index_constant(&mut self, index: Word) -> Word {
2712        self.get_constant_scalar(crate::Literal::U32(index))
2713    }
2714
2715    pub(super) fn get_constant_scalar_with(
2716        &mut self,
2717        value: u8,
2718        scalar: crate::Scalar,
2719    ) -> Result<Word, Error> {
2720        Ok(
2721            self.get_constant_scalar(crate::Literal::new(value, scalar).ok_or(
2722                Error::Validation("Unexpected kind and/or width for Literal"),
2723            )?),
2724        )
2725    }
2726
2727    pub(super) fn get_constant_scalar(&mut self, value: crate::Literal) -> Word {
2728        let scalar = CachedConstant::Literal(value.into());
2729        if let Some(&id) = self.cached_constants.get(&scalar) {
2730            return id;
2731        }
2732        let id = self.id_gen.next();
2733        self.write_constant_scalar(id, &value, None);
2734        self.cached_constants.insert(scalar, id);
2735        id
2736    }
2737
2738    fn write_constant_scalar(
2739        &mut self,
2740        id: Word,
2741        value: &crate::Literal,
2742        debug_name: Option<&String>,
2743    ) {
2744        if self.flags.contains(WriterFlags::DEBUG) {
2745            if let Some(name) = debug_name {
2746                self.debugs.push(Instruction::name(id, name));
2747            }
2748        }
2749        let type_id = self.get_numeric_type_id(NumericType::Scalar(value.scalar()));
2750        let instruction = match *value {
2751            crate::Literal::F64(value) => {
2752                let bits = value.to_bits();
2753                Instruction::constant_64bit(type_id, id, bits as u32, (bits >> 32) as u32)
2754            }
2755            crate::Literal::F32(value) => Instruction::constant_32bit(type_id, id, value.to_bits()),
2756            crate::Literal::F16(value) => {
2757                let low = value.to_bits();
2758                Instruction::constant_16bit(type_id, id, low as u32)
2759            }
2760            crate::Literal::U16(value) => Instruction::constant_16bit(type_id, id, value as u32),
2761            crate::Literal::I16(value) => {
2762                // Sign-extend into the 32-bit word so that `spirv-as` can
2763                // round-trip the disassembly (it expects signed values for
2764                // signed types).
2765                Instruction::constant_16bit(type_id, id, value as i32 as u32)
2766            }
2767            crate::Literal::U32(value) => Instruction::constant_32bit(type_id, id, value),
2768            crate::Literal::I32(value) => Instruction::constant_32bit(type_id, id, value as u32),
2769            crate::Literal::U64(value) => {
2770                Instruction::constant_64bit(type_id, id, value as u32, (value >> 32) as u32)
2771            }
2772            crate::Literal::I64(value) => {
2773                Instruction::constant_64bit(type_id, id, value as u32, (value >> 32) as u32)
2774            }
2775            crate::Literal::Bool(true) => Instruction::constant_true(type_id, id),
2776            crate::Literal::Bool(false) => Instruction::constant_false(type_id, id),
2777            crate::Literal::AbstractInt(_) | crate::Literal::AbstractFloat(_) => {
2778                unreachable!("Abstract types should not appear in IR presented to backends");
2779            }
2780        };
2781
2782        instruction.to_words(&mut self.logical_layout.declarations);
2783    }
2784
2785    pub(super) fn get_constant_composite(
2786        &mut self,
2787        ty: LookupType,
2788        constituent_ids: &[Word],
2789    ) -> Word {
2790        let composite = CachedConstant::Composite {
2791            ty,
2792            constituent_ids: constituent_ids.to_vec(),
2793        };
2794        if let Some(&id) = self.cached_constants.get(&composite) {
2795            return id;
2796        }
2797        let id = self.id_gen.next();
2798        self.write_constant_composite(id, ty, constituent_ids, None);
2799        self.cached_constants.insert(composite, id);
2800        id
2801    }
2802
2803    fn write_constant_composite(
2804        &mut self,
2805        id: Word,
2806        ty: LookupType,
2807        constituent_ids: &[Word],
2808        debug_name: Option<&String>,
2809    ) {
2810        if self.flags.contains(WriterFlags::DEBUG) {
2811            if let Some(name) = debug_name {
2812                self.debugs.push(Instruction::name(id, name));
2813            }
2814        }
2815        let type_id = self.get_type_id(ty);
2816        Instruction::constant_composite(type_id, id, constituent_ids)
2817            .to_words(&mut self.logical_layout.declarations);
2818    }
2819
2820    pub(super) fn get_constant_null(&mut self, type_id: Word) -> Word {
2821        let null = CachedConstant::ZeroValue(type_id);
2822        if let Some(&id) = self.cached_constants.get(&null) {
2823            return id;
2824        }
2825        let id = self.write_constant_null(type_id);
2826        self.cached_constants.insert(null, id);
2827        id
2828    }
2829
2830    pub(super) fn write_constant_null(&mut self, type_id: Word) -> Word {
2831        let null_id = self.id_gen.next();
2832        Instruction::constant_null(type_id, null_id)
2833            .to_words(&mut self.logical_layout.declarations);
2834        null_id
2835    }
2836
2837    fn write_constant_expr(
2838        &mut self,
2839        handle: Handle<crate::Expression>,
2840        ir_module: &crate::Module,
2841        mod_info: &ModuleInfo,
2842    ) -> Result<Word, Error> {
2843        let id = match ir_module.global_expressions[handle] {
2844            crate::Expression::Literal(literal) => self.get_constant_scalar(literal),
2845            crate::Expression::Constant(constant) => {
2846                let constant = &ir_module.constants[constant];
2847                self.constant_ids[constant.init]
2848            }
2849            crate::Expression::ZeroValue(ty) => {
2850                let type_id = self.get_handle_type_id(ty);
2851                self.get_constant_null(type_id)
2852            }
2853            crate::Expression::Compose { ty, ref components } => {
2854                let component_ids: Vec<_> = crate::proc::flatten_compose(
2855                    ty,
2856                    components,
2857                    &ir_module.global_expressions,
2858                    &ir_module.types,
2859                )
2860                .map(|component| self.constant_ids[component])
2861                .collect();
2862                self.get_constant_composite(LookupType::Handle(ty), component_ids.as_slice())
2863            }
2864            crate::Expression::Splat { size, value } => {
2865                let value_id = self.constant_ids[value];
2866                let component_ids = &[value_id; 4][..size as usize];
2867
2868                let ty = self.get_expression_lookup_type(&mod_info[handle]);
2869
2870                self.get_constant_composite(ty, component_ids)
2871            }
2872            _ => {
2873                return Err(Error::Override);
2874            }
2875        };
2876
2877        self.constant_ids[handle] = id;
2878
2879        Ok(id)
2880    }
2881
2882    pub(super) fn write_control_barrier(
2883        &mut self,
2884        flags: crate::Barrier,
2885        body: &mut Vec<Instruction>,
2886    ) {
2887        let memory_scope = if flags.contains(crate::Barrier::STORAGE) {
2888            spirv::Scope::Device
2889        } else if flags.contains(crate::Barrier::SUB_GROUP) {
2890            spirv::Scope::Subgroup
2891        } else {
2892            spirv::Scope::Workgroup
2893        };
2894        let mut semantics = spirv::MemorySemantics::ACQUIRE_RELEASE;
2895        semantics.set(
2896            spirv::MemorySemantics::UNIFORM_MEMORY,
2897            flags.contains(crate::Barrier::STORAGE),
2898        );
2899        semantics.set(
2900            spirv::MemorySemantics::WORKGROUP_MEMORY,
2901            flags.contains(crate::Barrier::WORK_GROUP),
2902        );
2903        semantics.set(
2904            spirv::MemorySemantics::SUBGROUP_MEMORY,
2905            flags.contains(crate::Barrier::SUB_GROUP),
2906        );
2907        semantics.set(
2908            spirv::MemorySemantics::IMAGE_MEMORY,
2909            flags.contains(crate::Barrier::TEXTURE),
2910        );
2911        let exec_scope_id = if flags.contains(crate::Barrier::SUB_GROUP) {
2912            self.get_index_constant(spirv::Scope::Subgroup as u32)
2913        } else {
2914            self.get_index_constant(spirv::Scope::Workgroup as u32)
2915        };
2916        let mem_scope_id = self.get_index_constant(memory_scope as u32);
2917        let semantics_id = self.get_index_constant(semantics.bits());
2918        body.push(Instruction::control_barrier(
2919            exec_scope_id,
2920            mem_scope_id,
2921            semantics_id,
2922        ));
2923    }
2924
2925    pub(super) fn write_memory_barrier(&mut self, flags: crate::Barrier, block: &mut Block) {
2926        let mut semantics = spirv::MemorySemantics::ACQUIRE_RELEASE;
2927        semantics.set(
2928            spirv::MemorySemantics::UNIFORM_MEMORY,
2929            flags.contains(crate::Barrier::STORAGE),
2930        );
2931        semantics.set(
2932            spirv::MemorySemantics::WORKGROUP_MEMORY,
2933            flags.contains(crate::Barrier::WORK_GROUP),
2934        );
2935        semantics.set(
2936            spirv::MemorySemantics::SUBGROUP_MEMORY,
2937            flags.contains(crate::Barrier::SUB_GROUP),
2938        );
2939        semantics.set(
2940            spirv::MemorySemantics::IMAGE_MEMORY,
2941            flags.contains(crate::Barrier::TEXTURE),
2942        );
2943        let mem_scope_id = if flags.contains(crate::Barrier::STORAGE) {
2944            self.get_index_constant(spirv::Scope::Device as u32)
2945        } else if flags.contains(crate::Barrier::SUB_GROUP) {
2946            self.get_index_constant(spirv::Scope::Subgroup as u32)
2947        } else {
2948            self.get_index_constant(spirv::Scope::Workgroup as u32)
2949        };
2950        let semantics_id = self.get_index_constant(semantics.bits());
2951        block
2952            .body
2953            .push(Instruction::memory_barrier(mem_scope_id, semantics_id));
2954    }
2955
2956    fn generate_workgroup_vars_init_block(
2957        &mut self,
2958        entry_id: Word,
2959        ir_module: &crate::Module,
2960        info: &FunctionInfo,
2961        local_invocation_index: Option<Word>,
2962        interface: &mut FunctionInterface,
2963        function: &mut Function,
2964    ) -> Option<Word> {
2965        let body = ir_module
2966            .global_variables
2967            .iter()
2968            .filter(|&(handle, var)| {
2969                let task_exception = (var.space == crate::AddressSpace::TaskPayload)
2970                    && interface.stage == crate::ShaderStage::Task;
2971                !info[handle].is_empty()
2972                    && (var.space == crate::AddressSpace::WorkGroup || task_exception)
2973            })
2974            .map(|(handle, var)| {
2975                // It's safe to use `var_id` here, not `access_id`, because only
2976                // variables in the `Uniform` and `StorageBuffer` address spaces
2977                // get wrapped, and we're initializing `WorkGroup` variables.
2978                let var_id = self.global_variables[handle].var_id;
2979                let var_type_id = self.get_handle_type_id(var.ty);
2980                let init_word = self.get_constant_null(var_type_id);
2981                Instruction::store(var_id, init_word, None)
2982            })
2983            .collect::<Vec<_>>();
2984
2985        if body.is_empty() {
2986            return None;
2987        }
2988
2989        let mut pre_if_block = Block::new(entry_id);
2990
2991        let local_invocation_index = if let Some(local_invocation_index) = local_invocation_index {
2992            local_invocation_index
2993        } else {
2994            let varying_id = self.id_gen.next();
2995            let class = spirv::StorageClass::Input;
2996            let u32_ty_id = self.get_u32_type_id();
2997            let pointer_type_id = self.get_pointer_type_id(u32_ty_id, class);
2998
2999            Instruction::variable(pointer_type_id, varying_id, class, None)
3000                .to_words(&mut self.logical_layout.declarations);
3001
3002            self.decorate(
3003                varying_id,
3004                spirv::Decoration::BuiltIn,
3005                &[spirv::BuiltIn::LocalInvocationIndex as u32],
3006            );
3007
3008            interface.varying_ids.push(varying_id);
3009            let id = self.id_gen.next();
3010            pre_if_block
3011                .body
3012                .push(Instruction::load(u32_ty_id, id, varying_id, None));
3013
3014            id
3015        };
3016
3017        let zero_id = self.get_constant_scalar(crate::Literal::U32(0));
3018
3019        let eq_id = self.id_gen.next();
3020        pre_if_block.body.push(Instruction::binary(
3021            spirv::Op::IEqual,
3022            self.get_bool_type_id(),
3023            eq_id,
3024            local_invocation_index,
3025            zero_id,
3026        ));
3027
3028        let merge_id = self.id_gen.next();
3029        pre_if_block.body.push(Instruction::selection_merge(
3030            merge_id,
3031            spirv::SelectionControl::NONE,
3032        ));
3033
3034        let accept_id = self.id_gen.next();
3035        function.consume(
3036            pre_if_block,
3037            Instruction::branch_conditional(eq_id, accept_id, merge_id),
3038        );
3039
3040        let accept_block = Block {
3041            label_id: accept_id,
3042            body,
3043        };
3044        function.consume(accept_block, Instruction::branch(merge_id));
3045
3046        let mut post_if_block = Block::new(merge_id);
3047
3048        self.write_control_barrier(crate::Barrier::WORK_GROUP, &mut post_if_block.body);
3049
3050        let next_id = self.id_gen.next();
3051        function.consume(post_if_block, Instruction::branch(next_id));
3052        Some(next_id)
3053    }
3054
3055    /// Generate an `OpVariable` for one value in an [`EntryPoint`]'s IO interface.
3056    ///
3057    /// The [`Binding`]s of the arguments and result of an [`EntryPoint`]'s
3058    /// [`Function`] describe a SPIR-V shader interface. In SPIR-V, the
3059    /// interface is represented by global variables in the `Input` and `Output`
3060    /// storage classes, with decorations indicating which builtin or location
3061    /// each variable corresponds to.
3062    ///
3063    /// This function emits a single global `OpVariable` for a single value from
3064    /// the interface, and adds appropriate decorations to indicate which
3065    /// builtin or location it represents, how it should be interpolated, and so
3066    /// on. The `class` argument gives the variable's SPIR-V storage class,
3067    /// which should be either [`Input`] or [`Output`]. The one exception is
3068    /// `hit_barycentrics`, which overrides `class` to `HitAttributeKHR`.
3069    ///
3070    /// [`Binding`]: crate::Binding
3071    /// [`Function`]: crate::Function
3072    /// [`EntryPoint`]: crate::EntryPoint
3073    /// [`Input`]: spirv::StorageClass::Input
3074    /// [`Output`]: spirv::StorageClass::Output
3075    fn write_varying(
3076        &mut self,
3077        ir_module: &crate::Module,
3078        stage: crate::ShaderStage,
3079        class: spirv::StorageClass,
3080        debug_name: Option<&str>,
3081        ty: Handle<crate::Type>,
3082        binding: &crate::Binding,
3083    ) -> Result<Word, Error> {
3084        // Triangle barycentrics are supplied as a `HitAttributeKHR` variable, not as an `Input`
3085        // builtin: when a hit group has no intersection shader, the hit attribute an any-hit or
3086        // closest-hit shader reads is a two-component float vector holding the barycentric
3087        // coordinates of the hit, and hit attributes carry no `BuiltIn` decoration.
3088        //
3089        // SPEC: https://registry.khronos.org/vulkan/specs/latest/html/vkspec.html#interfaces-raypipeline
3090        //
3091        // Declaring it here, alongside the entry point's other varyings, works because ray tracing
3092        // requires SPIR-V 1.4, where the `OpEntryPoint` interface lists global variables from every
3093        // storage class rather than just `Input` and `Output`.
3094        //
3095        // SPEC: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpEntryPoint
3096        let class = match *binding {
3097            crate::Binding::BuiltIn(crate::BuiltIn::HitBarycentrics) => {
3098                spirv::StorageClass::HitAttributeKHR
3099            }
3100            _ => class,
3101        };
3102
3103        let id = self.id_gen.next();
3104        let ty_inner = &ir_module.types[ty].inner;
3105        let needs_polyfill = self.needs_f16_polyfill(ty_inner);
3106
3107        let pointer_type_id = if needs_polyfill {
3108            let f32_value_local =
3109                super::f16_polyfill::F16IoPolyfill::create_polyfill_type(ty_inner)
3110                    .expect("needs_polyfill returned true but create_polyfill_type returned None");
3111
3112            let f32_type_id = self.get_localtype_id(f32_value_local);
3113            let ptr_id = self.get_pointer_type_id(f32_type_id, class);
3114            self.io_f16_polyfills.register_io_var(id, f32_type_id);
3115
3116            ptr_id
3117        } else {
3118            self.get_handle_pointer_type_id(ty, class)
3119        };
3120
3121        Instruction::variable(pointer_type_id, id, class, None)
3122            .to_words(&mut self.logical_layout.declarations);
3123
3124        if self
3125            .flags
3126            .contains(WriterFlags::DEBUG | WriterFlags::LABEL_VARYINGS)
3127        {
3128            if let Some(name) = debug_name {
3129                self.debugs.push(Instruction::name(id, name));
3130            }
3131        }
3132
3133        let binding = self.map_binding(ir_module, stage, class, ty, binding)?;
3134        self.write_binding(id, binding);
3135
3136        Ok(id)
3137    }
3138
3139    pub fn write_binding(&mut self, id: Word, binding: BindingDecorations) {
3140        match binding {
3141            BindingDecorations::None => (),
3142            BindingDecorations::BuiltIn(bi, others) => {
3143                self.decorate(id, spirv::Decoration::BuiltIn, &[bi as u32]);
3144                for other in others {
3145                    self.decorate(id, other, &[]);
3146                }
3147            }
3148            BindingDecorations::Location {
3149                location,
3150                others,
3151                blend_src,
3152            } => {
3153                self.decorate(id, spirv::Decoration::Location, &[location]);
3154                for other in others {
3155                    self.decorate(id, other, &[]);
3156                }
3157                if let Some(blend_src) = blend_src {
3158                    self.decorate(id, spirv::Decoration::Index, &[blend_src]);
3159                }
3160            }
3161        }
3162    }
3163
3164    pub fn write_binding_struct_member(
3165        &mut self,
3166        struct_id: Word,
3167        member_idx: Word,
3168        binding_info: BindingDecorations,
3169    ) {
3170        match binding_info {
3171            BindingDecorations::None => (),
3172            BindingDecorations::BuiltIn(bi, others) => {
3173                self.annotations.push(Instruction::member_decorate(
3174                    struct_id,
3175                    member_idx,
3176                    spirv::Decoration::BuiltIn,
3177                    &[bi as Word],
3178                ));
3179                for other in others {
3180                    self.annotations.push(Instruction::member_decorate(
3181                        struct_id,
3182                        member_idx,
3183                        other,
3184                        &[],
3185                    ));
3186                }
3187            }
3188            BindingDecorations::Location {
3189                location,
3190                others,
3191                blend_src,
3192            } => {
3193                self.annotations.push(Instruction::member_decorate(
3194                    struct_id,
3195                    member_idx,
3196                    spirv::Decoration::Location,
3197                    &[location],
3198                ));
3199                for other in others {
3200                    self.annotations.push(Instruction::member_decorate(
3201                        struct_id,
3202                        member_idx,
3203                        other,
3204                        &[],
3205                    ));
3206                }
3207                if let Some(blend_src) = blend_src {
3208                    self.annotations.push(Instruction::member_decorate(
3209                        struct_id,
3210                        member_idx,
3211                        spirv::Decoration::Index,
3212                        &[blend_src],
3213                    ));
3214                }
3215            }
3216        }
3217    }
3218
3219    pub fn map_binding(
3220        &mut self,
3221        ir_module: &crate::Module,
3222        stage: crate::ShaderStage,
3223        class: spirv::StorageClass,
3224        ty: Handle<crate::Type>,
3225        binding: &crate::Binding,
3226    ) -> Result<BindingDecorations, Error> {
3227        use spirv::BuiltIn;
3228        use spirv::Decoration;
3229        match *binding {
3230            crate::Binding::Location {
3231                location,
3232                interpolation,
3233                sampling,
3234                blend_src,
3235                per_primitive,
3236            } => {
3237                let mut others = ArrayVec::new();
3238
3239                let no_decorations =
3240                    // VUID-StandaloneSpirv-Flat-06202
3241                    // > The Flat, NoPerspective, Sample, and Centroid decorations
3242                    // > must not be used on variables with the Input storage class in a vertex shader
3243                    (class == spirv::StorageClass::Input && stage == crate::ShaderStage::Vertex) ||
3244                    // VUID-StandaloneSpirv-Flat-06201
3245                    // > The Flat, NoPerspective, Sample, and Centroid decorations
3246                    // > must not be used on variables with the Output storage class in a fragment shader
3247                    (class == spirv::StorageClass::Output && stage == crate::ShaderStage::Fragment);
3248
3249                if !no_decorations {
3250                    match interpolation {
3251                        // Perspective-correct interpolation is the default in SPIR-V.
3252                        None | Some(crate::Interpolation::Perspective) => (),
3253                        Some(crate::Interpolation::Flat) => {
3254                            others.push(Decoration::Flat);
3255                        }
3256                        Some(crate::Interpolation::Linear) => {
3257                            others.push(Decoration::NoPerspective);
3258                        }
3259                        Some(crate::Interpolation::PerVertex) => {
3260                            others.push(Decoration::PerVertexKHR);
3261                            self.require_any(
3262                                "`per_vertex` interpolation",
3263                                &[spirv::Capability::FragmentBarycentricKHR],
3264                            )?;
3265                            self.use_extension("SPV_KHR_fragment_shader_barycentric");
3266                        }
3267                    }
3268                    match sampling {
3269                        // Center sampling is the default in SPIR-V.
3270                        None
3271                        | Some(
3272                            crate::Sampling::Center
3273                            | crate::Sampling::First
3274                            | crate::Sampling::Either,
3275                        ) => (),
3276                        Some(crate::Sampling::Centroid) => {
3277                            others.push(Decoration::Centroid);
3278                        }
3279                        Some(crate::Sampling::Sample) => {
3280                            self.require_any(
3281                                "per-sample interpolation",
3282                                &[spirv::Capability::SampleRateShading],
3283                            )?;
3284                            others.push(Decoration::Sample);
3285                        }
3286                    }
3287                }
3288                if per_primitive && stage == crate::ShaderStage::Fragment {
3289                    others.push(Decoration::PerPrimitiveEXT);
3290                }
3291                Ok(BindingDecorations::Location {
3292                    location,
3293                    others,
3294                    blend_src,
3295                })
3296            }
3297            crate::Binding::BuiltIn(built_in) => {
3298                use crate::BuiltIn as Bi;
3299                let mut others = ArrayVec::new();
3300
3301                let built_in = match built_in {
3302                    Bi::Position { invariant } => {
3303                        if invariant {
3304                            others.push(Decoration::Invariant);
3305                        }
3306
3307                        if class == spirv::StorageClass::Output {
3308                            BuiltIn::Position
3309                        } else {
3310                            BuiltIn::FragCoord
3311                        }
3312                    }
3313                    Bi::ViewIndex => {
3314                        self.require_any("`view_index` built-in", &[spirv::Capability::MultiView])?;
3315                        BuiltIn::ViewIndex
3316                    }
3317                    // vertex
3318                    Bi::BaseInstance => BuiltIn::BaseInstance,
3319                    Bi::BaseVertex => BuiltIn::BaseVertex,
3320                    Bi::ClipDistances => {
3321                        self.require_any(
3322                            "`clip_distances` built-in",
3323                            &[spirv::Capability::ClipDistance],
3324                        )?;
3325                        BuiltIn::ClipDistance
3326                    }
3327                    Bi::CullDistance => {
3328                        self.require_any(
3329                            "`cull_distance` built-in",
3330                            &[spirv::Capability::CullDistance],
3331                        )?;
3332                        BuiltIn::CullDistance
3333                    }
3334                    Bi::InstanceIndex => match stage {
3335                        crate::ShaderStage::AnyHit | crate::ShaderStage::ClosestHit => {
3336                            BuiltIn::InstanceId
3337                        }
3338                        _ => BuiltIn::InstanceIndex,
3339                    },
3340                    Bi::PointSize => BuiltIn::PointSize,
3341                    Bi::VertexIndex => BuiltIn::VertexIndex,
3342                    Bi::DrawIndex => {
3343                        self.use_extension("SPV_KHR_shader_draw_parameters");
3344                        self.require_any(
3345                            "`draw_index built-in",
3346                            &[spirv::Capability::DrawParameters],
3347                        )?;
3348                        BuiltIn::DrawIndex
3349                    }
3350                    // fragment
3351                    Bi::FragDepth => BuiltIn::FragDepth,
3352                    Bi::PointCoord => BuiltIn::PointCoord,
3353                    Bi::FrontFacing => BuiltIn::FrontFacing,
3354                    Bi::PrimitiveIndex => {
3355                        // `PrimitiveId` is enabled by any of `Geometry`, `Tessellation`,
3356                        // `RayTracingKHR` or `MeshShadingEXT`. `require_any` picks the first one the target
3357                        // allows.
3358                        //
3359                        // SPEC: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#:~:text=PrimitiveId
3360                        let enabled_by: &[spirv::Capability] = match stage {
3361                            // The stage itself already requires `RayTracingKHR`.
3362                            crate::ShaderStage::AnyHit | crate::ShaderStage::ClosestHit => &[],
3363                            // The stage itself already requires `MeshShadingEXT`.
3364                            crate::ShaderStage::Mesh => &[],
3365                            // A fragment shader can be fed primitive IDs by a geometry,
3366                            // tessellation or mesh pipeline.
3367                            crate::ShaderStage::Fragment => &[
3368                                spirv::Capability::Geometry,
3369                                spirv::Capability::Tessellation,
3370                                spirv::Capability::MeshShadingEXT,
3371                            ],
3372                            // `PrimitiveId` isn't permitted in these execution models.
3373                            _ => return Err(Error::Validation(
3374                                "`primitive_index` built-in is not allowed in this shader stage",
3375                            )),
3376                        };
3377                        self.require_any("`primitive_index` built-in", enabled_by)?;
3378
3379                        if stage == crate::ShaderStage::Mesh {
3380                            others.push(Decoration::PerPrimitiveEXT);
3381                        }
3382
3383                        BuiltIn::PrimitiveId
3384                    }
3385                    Bi::Barycentric { perspective } => {
3386                        self.require_any(
3387                            "`barycentric` built-in",
3388                            &[spirv::Capability::FragmentBarycentricKHR],
3389                        )?;
3390                        self.use_extension("SPV_KHR_fragment_shader_barycentric");
3391                        if perspective {
3392                            BuiltIn::BaryCoordKHR
3393                        } else {
3394                            BuiltIn::BaryCoordNoPerspKHR
3395                        }
3396                    }
3397                    Bi::SampleIndex => {
3398                        self.require_any(
3399                            "`sample_index` built-in",
3400                            &[spirv::Capability::SampleRateShading],
3401                        )?;
3402
3403                        BuiltIn::SampleId
3404                    }
3405                    Bi::SampleMask => BuiltIn::SampleMask,
3406                    // compute
3407                    Bi::GlobalInvocationId => BuiltIn::GlobalInvocationId,
3408                    Bi::LocalInvocationId => BuiltIn::LocalInvocationId,
3409                    Bi::LocalInvocationIndex => BuiltIn::LocalInvocationIndex,
3410                    Bi::WorkGroupId => BuiltIn::WorkgroupId,
3411                    Bi::WorkGroupSize => BuiltIn::WorkgroupSize,
3412                    Bi::NumWorkGroups => BuiltIn::NumWorkgroups,
3413                    // Subgroup
3414                    Bi::NumSubgroups => {
3415                        self.require_any(
3416                            "`num_subgroups` built-in",
3417                            &[spirv::Capability::GroupNonUniform],
3418                        )?;
3419                        BuiltIn::NumSubgroups
3420                    }
3421                    Bi::SubgroupId => {
3422                        self.require_any(
3423                            "`subgroup_id` built-in",
3424                            &[spirv::Capability::GroupNonUniform],
3425                        )?;
3426                        BuiltIn::SubgroupId
3427                    }
3428                    Bi::SubgroupSize => {
3429                        self.require_any(
3430                            "`subgroup_size` built-in",
3431                            &[
3432                                spirv::Capability::GroupNonUniform,
3433                                spirv::Capability::SubgroupBallotKHR,
3434                            ],
3435                        )?;
3436                        BuiltIn::SubgroupSize
3437                    }
3438                    Bi::SubgroupInvocationId => {
3439                        self.require_any(
3440                            "`subgroup_invocation_id` built-in",
3441                            &[
3442                                spirv::Capability::GroupNonUniform,
3443                                spirv::Capability::SubgroupBallotKHR,
3444                            ],
3445                        )?;
3446                        BuiltIn::SubgroupLocalInvocationId
3447                    }
3448                    Bi::CullPrimitive => {
3449                        others.push(Decoration::PerPrimitiveEXT);
3450                        BuiltIn::CullPrimitiveEXT
3451                    }
3452                    Bi::PointIndex => BuiltIn::PrimitivePointIndicesEXT,
3453                    Bi::LineIndices => BuiltIn::PrimitiveLineIndicesEXT,
3454                    Bi::TriangleIndices => BuiltIn::PrimitiveTriangleIndicesEXT,
3455                    // No decoration, this EmitMeshTasksEXT is called at function return
3456                    Bi::MeshTaskSize => return Ok(BindingDecorations::None),
3457                    // These aren't normal builtins and don't occur in function output
3458                    Bi::VertexCount | Bi::Vertices | Bi::PrimitiveCount | Bi::Primitives => {
3459                        unreachable!()
3460                    }
3461                    // ray tracing pipeline
3462                    Bi::RayInvocationId => BuiltIn::LaunchIdKHR,
3463                    Bi::NumRayInvocations => BuiltIn::LaunchSizeKHR,
3464                    Bi::InstanceCustomData => BuiltIn::InstanceCustomIndexKHR,
3465                    Bi::GeometryIndex => BuiltIn::RayGeometryIndexKHR,
3466                    Bi::WorldRayOrigin => BuiltIn::WorldRayOriginKHR,
3467                    Bi::WorldRayDirection => BuiltIn::WorldRayDirectionKHR,
3468                    Bi::ObjectRayOrigin => BuiltIn::ObjectRayOriginKHR,
3469                    Bi::ObjectRayDirection => BuiltIn::ObjectRayDirectionKHR,
3470                    Bi::RayTmin => BuiltIn::RayTminKHR,
3471                    Bi::RayTCurrentMax => BuiltIn::RayTmaxKHR,
3472                    Bi::ObjectToWorld => BuiltIn::ObjectToWorldKHR,
3473                    Bi::WorldToObject => BuiltIn::WorldToObjectKHR,
3474                    Bi::HitKind => BuiltIn::HitKindKHR,
3475                    // Read from the entry point's `HitAttributeKHR` variable instead.
3476                    Bi::HitBarycentrics => return Ok(BindingDecorations::None),
3477                };
3478
3479                use crate::ScalarKind as Sk;
3480
3481                // Per the Vulkan spec, `VUID-StandaloneSpirv-Flat-04744`:
3482                //
3483                // > Any variable with integer or double-precision floating-
3484                // > point type and with Input storage class in a fragment
3485                // > shader, must be decorated Flat
3486                if class == spirv::StorageClass::Input && stage == crate::ShaderStage::Fragment {
3487                    let is_flat = match ir_module.types[ty].inner {
3488                        crate::TypeInner::Scalar(scalar)
3489                        | crate::TypeInner::Vector { scalar, .. } => match scalar.kind {
3490                            Sk::Uint | Sk::Sint | Sk::Bool => true,
3491                            Sk::Float => false,
3492                            Sk::AbstractInt | Sk::AbstractFloat => {
3493                                return Err(Error::Validation(
3494                                    "Abstract types should not appear in IR presented to backends",
3495                                ))
3496                            }
3497                        },
3498                        _ => false,
3499                    };
3500
3501                    if is_flat {
3502                        others.push(Decoration::Flat);
3503                    }
3504                }
3505                Ok(BindingDecorations::BuiltIn(built_in, others))
3506            }
3507        }
3508    }
3509
3510    /// Load an IO variable, converting from `f32` to `f16` if polyfill is active.
3511    /// Returns the id of the loaded value matching `target_type_id`.
3512    pub(super) fn load_io_with_f16_polyfill(
3513        &mut self,
3514        body: &mut Vec<Instruction>,
3515        varying_id: Word,
3516        target_type_id: Word,
3517    ) -> Word {
3518        let tmp = self.id_gen.next();
3519        if let Some(f32_ty) = self.io_f16_polyfills.get_f32_io_type(varying_id) {
3520            body.push(Instruction::load(f32_ty, tmp, varying_id, None));
3521            let converted = self.id_gen.next();
3522            super::f16_polyfill::F16IoPolyfill::emit_f32_to_f16_conversion(
3523                tmp,
3524                target_type_id,
3525                converted,
3526                body,
3527            );
3528            converted
3529        } else {
3530            body.push(Instruction::load(target_type_id, tmp, varying_id, None));
3531            tmp
3532        }
3533    }
3534
3535    /// Store an IO variable, converting from `f16` to `f32` if polyfill is active.
3536    pub(super) fn store_io_with_f16_polyfill(
3537        &mut self,
3538        body: &mut Vec<Instruction>,
3539        varying_id: Word,
3540        value_id: Word,
3541    ) {
3542        if let Some(f32_ty) = self.io_f16_polyfills.get_f32_io_type(varying_id) {
3543            let converted = self.id_gen.next();
3544            super::f16_polyfill::F16IoPolyfill::emit_f16_to_f32_conversion(
3545                value_id, f32_ty, converted, body,
3546            );
3547            body.push(Instruction::store(varying_id, converted, None));
3548        } else {
3549            body.push(Instruction::store(varying_id, value_id, None));
3550        }
3551    }
3552
3553    fn write_global_variable(
3554        &mut self,
3555        ir_module: &crate::Module,
3556        global_variable: &crate::GlobalVariable,
3557    ) -> Result<Word, Error> {
3558        use spirv::Decoration;
3559
3560        let id = self.id_gen.next();
3561        let class = map_storage_class(global_variable.space);
3562
3563        if let crate::AddressSpace::RayPayload | crate::AddressSpace::IncomingRayPayload =
3564            global_variable.space
3565        {
3566            self.require_any("ray tracing pipelines", &[spirv::Capability::RayTracingKHR])?;
3567        }
3568
3569        //self.check(class.required_capabilities())?;
3570
3571        if global_variable
3572            .memory_decorations
3573            .contains(crate::MemoryDecorations::COHERENT)
3574        {
3575            self.decorate(id, Decoration::Coherent, &[]);
3576        }
3577        if global_variable
3578            .memory_decorations
3579            .contains(crate::MemoryDecorations::VOLATILE)
3580        {
3581            self.decorate(id, Decoration::Volatile, &[]);
3582        }
3583
3584        if self.flags.contains(WriterFlags::DEBUG) {
3585            if let Some(ref name) = global_variable.name {
3586                self.debugs.push(Instruction::name(id, name));
3587            }
3588        }
3589
3590        let storage_access = match global_variable.space {
3591            crate::AddressSpace::Storage { access } => Some(access),
3592            _ => match ir_module.types[global_variable.ty].inner {
3593                crate::TypeInner::Image {
3594                    class: crate::ImageClass::Storage { access, .. },
3595                    ..
3596                } => Some(access),
3597                _ => None,
3598            },
3599        };
3600        if let Some(storage_access) = storage_access {
3601            if !storage_access.contains(crate::StorageAccess::LOAD) {
3602                self.decorate(id, Decoration::NonReadable, &[]);
3603            }
3604            if !storage_access.contains(crate::StorageAccess::STORE) {
3605                self.decorate(id, Decoration::NonWritable, &[]);
3606            }
3607        }
3608
3609        // Note: we should be able to substitute `binding_array<Foo, 0>`,
3610        // but there is still code that tries to register the pre-substituted type,
3611        // and it is failing on 0.
3612        let mut substitute_inner_type_lookup = None;
3613        if let Some(ref res_binding) = global_variable.binding {
3614            let bind_target = self.resolve_resource_binding(res_binding)?;
3615            self.decorate(id, Decoration::DescriptorSet, &[bind_target.descriptor_set]);
3616            self.decorate(id, Decoration::Binding, &[bind_target.binding]);
3617
3618            if let Some(remapped_binding_array_size) = bind_target.binding_array_size {
3619                if let crate::TypeInner::BindingArray { base, .. } =
3620                    ir_module.types[global_variable.ty].inner
3621                {
3622                    let binding_array_type_id =
3623                        self.get_type_id(LookupType::Local(LocalType::BindingArray {
3624                            base,
3625                            size: remapped_binding_array_size,
3626                        }));
3627                    substitute_inner_type_lookup = Some(LookupType::Local(LocalType::Pointer {
3628                        base: binding_array_type_id,
3629                        class,
3630                    }));
3631                }
3632            }
3633        };
3634
3635        let init_word = global_variable
3636            .init
3637            .map(|constant| self.constant_ids[constant]);
3638        let inner_type_id = self.get_type_id(
3639            substitute_inner_type_lookup.unwrap_or(LookupType::Handle(global_variable.ty)),
3640        );
3641
3642        // generate the wrapping structure if needed
3643        let pointer_type_id = if global_needs_wrapper(ir_module, global_variable) {
3644            let wrapper_type_id = self.id_gen.next();
3645
3646            self.decorate(wrapper_type_id, Decoration::Block, &[]);
3647
3648            match self.std140_compat_uniform_types.get(&global_variable.ty) {
3649                Some(std140_type_info) if global_variable.space == crate::AddressSpace::Uniform => {
3650                    self.annotations.push(Instruction::member_decorate(
3651                        wrapper_type_id,
3652                        0,
3653                        Decoration::Offset,
3654                        &[0],
3655                    ));
3656                    Instruction::type_struct(wrapper_type_id, &[std140_type_info.type_id])
3657                        .to_words(&mut self.logical_layout.declarations);
3658                }
3659                _ => {
3660                    let member = crate::StructMember {
3661                        name: None,
3662                        ty: global_variable.ty,
3663                        binding: None,
3664                        offset: 0,
3665                    };
3666                    self.decorate_struct_member(wrapper_type_id, 0, &member, &ir_module.types)?;
3667
3668                    Instruction::type_struct(wrapper_type_id, &[inner_type_id])
3669                        .to_words(&mut self.logical_layout.declarations);
3670                }
3671            }
3672
3673            let pointer_type_id = self.id_gen.next();
3674            Instruction::type_pointer(pointer_type_id, class, wrapper_type_id)
3675                .to_words(&mut self.logical_layout.declarations);
3676
3677            pointer_type_id
3678        } else {
3679            // This is a global variable in the Storage address space. The only
3680            // way it could have `global_needs_wrapper() == false` is if it has
3681            // a runtime-sized or binding array.
3682            // Runtime-sized arrays were decorated when iterating through struct content.
3683            // Now binding arrays require Block decorating.
3684            if let crate::AddressSpace::Storage { .. } = global_variable.space {
3685                match ir_module.types[global_variable.ty].inner {
3686                    crate::TypeInner::BindingArray { base, .. } => {
3687                        let ty = &ir_module.types[base];
3688                        let mut should_decorate = true;
3689                        // Check if the type has a runtime array.
3690                        // A normal runtime array gets validated out,
3691                        // so only structs can be with runtime arrays
3692                        if let crate::TypeInner::Struct { ref members, .. } = ty.inner {
3693                            // only the last member in a struct can be dynamically sized
3694                            if let Some(last_member) = members.last() {
3695                                if let &crate::TypeInner::Array {
3696                                    size: crate::ArraySize::Dynamic,
3697                                    ..
3698                                } = &ir_module.types[last_member.ty].inner
3699                                {
3700                                    should_decorate = false;
3701                                }
3702                            }
3703                        }
3704                        if should_decorate {
3705                            let decorated_id = self.get_handle_type_id(base);
3706                            self.decorate(decorated_id, Decoration::Block, &[]);
3707                        }
3708                    }
3709                    _ => (),
3710                };
3711            }
3712            if substitute_inner_type_lookup.is_some() {
3713                inner_type_id
3714            } else {
3715                self.get_handle_pointer_type_id(global_variable.ty, class)
3716            }
3717        };
3718
3719        let init_word = match (global_variable.space, self.zero_initialize_workgroup_memory) {
3720            (crate::AddressSpace::Private, _)
3721            | (crate::AddressSpace::WorkGroup, super::ZeroInitializeWorkgroupMemoryMode::Native) => {
3722                init_word.or_else(|| Some(self.get_constant_null(inner_type_id)))
3723            }
3724            _ => init_word,
3725        };
3726
3727        Instruction::variable(pointer_type_id, id, class, init_word)
3728            .to_words(&mut self.logical_layout.declarations);
3729        Ok(id)
3730    }
3731
3732    /// Write the necessary decorations for a struct member.
3733    ///
3734    /// Emit decorations for the `index`'th member of the struct type
3735    /// designated by `struct_id`, described by `member`.
3736    fn decorate_struct_member(
3737        &mut self,
3738        struct_id: Word,
3739        index: usize,
3740        member: &crate::StructMember,
3741        arena: &UniqueArena<crate::Type>,
3742    ) -> Result<(), Error> {
3743        use spirv::Decoration;
3744
3745        self.annotations.push(Instruction::member_decorate(
3746            struct_id,
3747            index as u32,
3748            Decoration::Offset,
3749            &[member.offset],
3750        ));
3751
3752        if self.flags.contains(WriterFlags::DEBUG) {
3753            if let Some(ref name) = member.name {
3754                self.debugs
3755                    .push(Instruction::member_name(struct_id, index as u32, name));
3756            }
3757        }
3758
3759        // Matrices and (potentially nested) arrays of matrices both require decorations,
3760        // so "see through" any arrays to determine if they're needed.
3761        let mut member_array_subty_inner = &arena[member.ty].inner;
3762        while let crate::TypeInner::Array { base, .. } = *member_array_subty_inner {
3763            member_array_subty_inner = &arena[base].inner;
3764        }
3765
3766        if let crate::TypeInner::Matrix {
3767            columns: _,
3768            rows,
3769            scalar,
3770        } = *member_array_subty_inner
3771        {
3772            let byte_stride = Alignment::from(rows) * scalar.width as u32;
3773            self.annotations.push(Instruction::member_decorate(
3774                struct_id,
3775                index as u32,
3776                Decoration::ColMajor,
3777                &[],
3778            ));
3779            self.annotations.push(Instruction::member_decorate(
3780                struct_id,
3781                index as u32,
3782                Decoration::MatrixStride,
3783                &[byte_stride],
3784            ));
3785        }
3786
3787        Ok(())
3788    }
3789
3790    pub(super) fn get_function_type(&mut self, lookup_function_type: LookupFunctionType) -> Word {
3791        match self
3792            .lookup_function_type
3793            .entry(lookup_function_type.clone())
3794        {
3795            Entry::Occupied(e) => *e.get(),
3796            Entry::Vacant(_) => {
3797                let id = self.id_gen.next();
3798                let instruction = Instruction::type_function(
3799                    id,
3800                    lookup_function_type.return_type_id,
3801                    &lookup_function_type.parameter_type_ids,
3802                );
3803                instruction.to_words(&mut self.logical_layout.declarations);
3804                self.lookup_function_type.insert(lookup_function_type, id);
3805                id
3806            }
3807        }
3808    }
3809
3810    const fn write_physical_layout(&mut self) {
3811        self.physical_layout.bound = self.id_gen.0 + 1;
3812    }
3813
3814    fn write_logical_layout(
3815        &mut self,
3816        ir_module: &crate::Module,
3817        mod_info: &ModuleInfo,
3818        ep_index: Option<usize>,
3819        debug_info: &Option<DebugInfo>,
3820    ) -> Result<(), Error> {
3821        fn has_view_index_check(
3822            ir_module: &crate::Module,
3823            binding: Option<&crate::Binding>,
3824            ty: Handle<crate::Type>,
3825        ) -> bool {
3826            match ir_module.types[ty].inner {
3827                crate::TypeInner::Struct { ref members, .. } => members.iter().any(|member| {
3828                    has_view_index_check(ir_module, member.binding.as_ref(), member.ty)
3829                }),
3830                _ => binding == Some(&crate::Binding::BuiltIn(crate::BuiltIn::ViewIndex)),
3831            }
3832        }
3833
3834        let has_storage_buffers =
3835            ir_module
3836                .global_variables
3837                .iter()
3838                .any(|(_, var)| match var.space {
3839                    crate::AddressSpace::Storage { .. } => true,
3840                    _ => false,
3841                });
3842        let has_view_index = ir_module
3843            .entry_points
3844            .iter()
3845            .flat_map(|entry| entry.function.arguments.iter())
3846            .any(|arg| has_view_index_check(ir_module, arg.binding.as_ref(), arg.ty));
3847        let has_vertex_return = ir_module.special_types.ray_vertex_return.is_some();
3848
3849        let rt_uses = ir_module.uses_ray_tracing(ep_index);
3850        let has_ray_query = rt_uses.queries;
3851        let has_ray_tracing_pipeline = rt_uses.pipelines;
3852
3853        self.has_ray_tracing_pipeline = has_ray_tracing_pipeline;
3854
3855        if self.physical_layout.version < 0x10300 && has_storage_buffers {
3856            // enable the storage buffer class on < SPV-1.3
3857            Instruction::extension("SPV_KHR_storage_buffer_storage_class")
3858                .to_words(&mut self.logical_layout.extensions);
3859        }
3860        if has_view_index {
3861            Instruction::extension("SPV_KHR_multiview")
3862                .to_words(&mut self.logical_layout.extensions)
3863        }
3864        if has_ray_query {
3865            Instruction::extension("SPV_KHR_ray_query")
3866                .to_words(&mut self.logical_layout.extensions)
3867        }
3868        if has_vertex_return {
3869            Instruction::extension("SPV_KHR_ray_tracing_position_fetch")
3870                .to_words(&mut self.logical_layout.extensions);
3871        }
3872        if ir_module.uses_mesh_shaders() {
3873            self.use_extension("SPV_EXT_mesh_shader");
3874            self.require_any("Mesh Shaders", &[spirv::Capability::MeshShadingEXT])?;
3875            let lang_version = self.lang_version();
3876            if lang_version.0 <= 1 && lang_version.1 < 4 {
3877                return Err(Error::SpirvVersionTooLow(1, 4));
3878            }
3879        }
3880        if has_ray_tracing_pipeline {
3881            // `SPV_KHR_ray_tracing` requires SPIR-V 1.4. That is also what lets `write_varying`
3882            // emit `hit_barycentrics` as a `HitAttributeKHR` variable, since only from 1.4 on does
3883            // the `OpEntryPoint` interface cover storage classes other than `Input` and `Output`.
3884            //
3885            // SPEC: https://github.khronos.org/SPIRV-Registry/extensions/KHR/SPV_KHR_ray_tracing.html
3886            let lang_version = self.lang_version();
3887            if lang_version.0 <= 1 && lang_version.1 < 4 {
3888                return Err(Error::SpirvVersionTooLow(1, 4));
3889            }
3890            Instruction::extension("SPV_KHR_ray_tracing")
3891                .to_words(&mut self.logical_layout.extensions)
3892        }
3893        Instruction::type_void(self.void_type).to_words(&mut self.logical_layout.declarations);
3894
3895        if ir_module.uses_invocation_reorder() {
3896            // Create the `HitObjectAttributeEXT` variable up front, so that we
3897            // can list it in the interface of every ray tracing entry point,
3898            // whether or not that entry point happens to use it.
3899            self.require_hit_objects()?;
3900            self.get_hit_object_attribute_var();
3901        }
3902        Instruction::ext_inst_import(self.gl450_ext_inst_id, "GLSL.std.450")
3903            .to_words(&mut self.logical_layout.ext_inst_imports);
3904
3905        let mut debug_info_inner = None;
3906        if self.flags.contains(WriterFlags::DEBUG) {
3907            if let Some(debug_info) = debug_info.as_ref() {
3908                let source_file_id = self.id_gen.next();
3909                self.debugs
3910                    .push(Instruction::string(debug_info.file_name, source_file_id));
3911
3912                debug_info_inner = Some(DebugInfoInner {
3913                    source_code: debug_info.source_code,
3914                    source_file_id,
3915                });
3916                self.debugs.append(&mut Instruction::source_auto_continued(
3917                    debug_info.language,
3918                    0,
3919                    &debug_info_inner,
3920                ));
3921            }
3922        }
3923
3924        // write all types
3925        for (handle, _) in ir_module.types.iter() {
3926            self.write_type_declaration_arena(ir_module, handle)?;
3927        }
3928
3929        // write std140 layout compatible types required by uniforms
3930        for (_, var) in ir_module.global_variables.iter() {
3931            if var.space == crate::AddressSpace::Uniform {
3932                self.write_std140_compat_type_declaration(ir_module, var.ty)?;
3933            }
3934        }
3935
3936        // write all const-expressions as constants
3937        self.constant_ids
3938            .resize(ir_module.global_expressions.len(), 0);
3939        for (handle, _) in ir_module.global_expressions.iter() {
3940            self.write_constant_expr(handle, ir_module, mod_info)?;
3941        }
3942        debug_assert!(self.constant_ids.iter().all(|&id| id != 0));
3943
3944        // write the name of constants on their respective const-expression initializer
3945        if self.flags.contains(WriterFlags::DEBUG) {
3946            for (_, constant) in ir_module.constants.iter() {
3947                if let Some(ref name) = constant.name {
3948                    let id = self.constant_ids[constant.init];
3949                    self.debugs.push(Instruction::name(id, name));
3950                }
3951            }
3952        }
3953
3954        // write all global variables
3955        for (handle, var) in ir_module.global_variables.iter() {
3956            // If a single entry point was specified, only write `OpVariable` instructions
3957            // for the globals it actually uses. Emit dummies for the others,
3958            // to preserve the indices in `global_variables`.
3959            let gvar = match ep_index {
3960                Some(index) if mod_info.get_entry_point(index)[handle].is_empty() => {
3961                    GlobalVariable::dummy()
3962                }
3963                _ => {
3964                    let id = self.write_global_variable(ir_module, var)?;
3965                    GlobalVariable::new(id)
3966                }
3967            };
3968            self.global_variables.insert(handle, gvar);
3969        }
3970
3971        // write all functions
3972        for (handle, ir_function) in ir_module.functions.iter() {
3973            let info = &mod_info[handle];
3974            if let Some(index) = ep_index {
3975                let ep_info = mod_info.get_entry_point(index);
3976                // If this function uses globals that we omitted from the SPIR-V
3977                // because the entry point and its callees didn't use them,
3978                // then we must skip it.
3979                if !ep_info.dominates_global_use(info) {
3980                    log::debug!("Skip function {:?}", ir_function.name);
3981                    continue;
3982                }
3983
3984                // Skip functions that that are not compatible with this entry point's stage.
3985                //
3986                // When validation is enabled, it rejects modules whose entry points try to call
3987                // incompatible functions, so if we got this far, then any functions incompatible
3988                // with our selected entry point must not be used.
3989                //
3990                // When validation is disabled, `fun_info.available_stages` is always just
3991                // `ShaderStages::all()`, so this will write all functions in the module, and
3992                // the downstream GLSL compiler will catch any problems.
3993                if !info.available_stages.contains(ep_info.available_stages) {
3994                    continue;
3995                }
3996            }
3997            let id = self.write_function(ir_function, info, ir_module, None, &debug_info_inner)?;
3998            self.lookup_function.insert(handle, id);
3999        }
4000
4001        // write all or one entry points
4002        for (index, ir_ep) in ir_module.entry_points.iter().enumerate() {
4003            if ep_index.is_some() && ep_index != Some(index) {
4004                continue;
4005            }
4006            let info = mod_info.get_entry_point(index);
4007            let ep_instruction =
4008                self.write_entry_point(ir_ep, info, ir_module, &debug_info_inner)?;
4009            ep_instruction.to_words(&mut self.logical_layout.entry_points);
4010        }
4011
4012        for capability in self.capabilities_used.iter() {
4013            Instruction::capability(*capability).to_words(&mut self.logical_layout.capabilities);
4014        }
4015        for extension in self.extensions_used.iter() {
4016            Instruction::extension(extension).to_words(&mut self.logical_layout.extensions);
4017        }
4018        if ir_module.entry_points.is_empty() {
4019            // SPIR-V doesn't like modules without entry points
4020            Instruction::capability(spirv::Capability::Linkage)
4021                .to_words(&mut self.logical_layout.capabilities);
4022        }
4023
4024        let addressing_model = spirv::AddressingModel::Logical;
4025        let memory_model = if self
4026            .capabilities_used
4027            .contains(&spirv::Capability::VulkanMemoryModel)
4028        {
4029            spirv::MemoryModel::Vulkan
4030        } else {
4031            spirv::MemoryModel::GLSL450
4032        };
4033        //self.check(addressing_model.required_capabilities())?;
4034        //self.check(memory_model.required_capabilities())?;
4035
4036        Instruction::memory_model(addressing_model, memory_model)
4037            .to_words(&mut self.logical_layout.memory_model);
4038
4039        for debug_string in self.debug_strings.iter() {
4040            debug_string.to_words(&mut self.logical_layout.debugs);
4041        }
4042
4043        if self.flags.contains(WriterFlags::DEBUG) {
4044            for debug in self.debugs.iter() {
4045                debug.to_words(&mut self.logical_layout.debugs);
4046            }
4047        }
4048
4049        for annotation in self.annotations.iter() {
4050            annotation.to_words(&mut self.logical_layout.annotations);
4051        }
4052
4053        Ok(())
4054    }
4055
4056    pub fn write(
4057        &mut self,
4058        ir_module: &crate::Module,
4059        info: &ModuleInfo,
4060        pipeline_options: Option<&PipelineOptions>,
4061        debug_info: &Option<DebugInfo>,
4062        words: &mut Vec<Word>,
4063    ) -> Result<(), Error> {
4064        self.reset();
4065
4066        // Try to find the entry point and corresponding index
4067        let ep_index = match pipeline_options {
4068            Some(po) => {
4069                let index = ir_module
4070                    .entry_points
4071                    .iter()
4072                    .position(|ep| po.shader_stage == ep.stage && po.entry_point == ep.name)
4073                    .ok_or(Error::EntryPointNotFound)?;
4074                Some(index)
4075            }
4076            None => None,
4077        };
4078
4079        self.write_logical_layout(ir_module, info, ep_index, debug_info)?;
4080        self.write_physical_layout();
4081
4082        self.physical_layout.in_words(words);
4083        self.logical_layout.in_words(words);
4084        Ok(())
4085    }
4086
4087    /// Return the set of capabilities the last module written used.
4088    pub const fn get_capabilities_used(&self) -> &crate::FastIndexSet<spirv::Capability> {
4089        &self.capabilities_used
4090    }
4091
4092    pub fn decorate_non_uniform_binding_array_access(&mut self, id: Word) -> Result<(), Error> {
4093        self.require_any("NonUniformEXT", &[spirv::Capability::ShaderNonUniform])?;
4094        self.use_extension("SPV_EXT_descriptor_indexing");
4095        self.decorate(id, spirv::Decoration::NonUniform, &[]);
4096        Ok(())
4097    }
4098
4099    pub(super) fn needs_f16_polyfill(&self, ty_inner: &crate::TypeInner) -> bool {
4100        self.io_f16_polyfills.needs_polyfill(ty_inner)
4101    }
4102
4103    pub(super) fn write_debug_printf(
4104        &mut self,
4105        block: &mut Block,
4106        string: &str,
4107        format_params: &[Word],
4108    ) {
4109        if self.debug_printf.is_none() {
4110            self.use_extension("SPV_KHR_non_semantic_info");
4111            let import_id = self.id_gen.next();
4112            Instruction::ext_inst_import(import_id, "NonSemantic.DebugPrintf")
4113                .to_words(&mut self.logical_layout.ext_inst_imports);
4114            self.debug_printf = Some(import_id)
4115        }
4116
4117        let import_id = self.debug_printf.unwrap();
4118
4119        let string_id = self.id_gen.next();
4120        self.debug_strings
4121            .push(Instruction::string(string, string_id));
4122
4123        let mut operands = Vec::with_capacity(1 + format_params.len());
4124        operands.push(string_id);
4125        operands.extend(format_params.iter());
4126
4127        let print_id = self.id_gen.next();
4128        block.body.push(Instruction::ext_inst(
4129            import_id,
4130            1,
4131            self.void_type,
4132            print_id,
4133            &operands,
4134        ));
4135    }
4136}
4137
4138#[test]
4139fn test_write_physical_layout() {
4140    let mut writer = Writer::new(&Options::default()).unwrap();
4141    assert_eq!(writer.physical_layout.bound, 0);
4142    writer.write_physical_layout();
4143    assert_eq!(writer.physical_layout.bound, 3);
4144}