Skip to main content

naga/back/spv/
block.rs

1/*!
2Implementations for `BlockContext` methods.
3*/
4
5use alloc::vec::Vec;
6
7use arrayvec::ArrayVec;
8use spirv::Word;
9
10use super::{
11    helpers::map_storage_class, index::BoundsCheckResult, selection::Selection, Block,
12    BlockContext, Dimension, Error, IdGenerator, Instruction, LocalType, LookupType, NumericType,
13    ResultMember, WrappedFunction, Writer, WriterFlags,
14};
15use crate::{
16    arena::Handle, back::spv::helpers::is_uniform_matcx2_struct_member_access,
17    proc::index::GuardedIndex, Statement,
18};
19
20fn get_dimension(type_inner: &crate::TypeInner) -> Dimension {
21    match *type_inner {
22        crate::TypeInner::Scalar(_) => Dimension::Scalar,
23        crate::TypeInner::Vector { .. } => Dimension::Vector,
24        crate::TypeInner::Matrix { .. } => Dimension::Matrix,
25        crate::TypeInner::CooperativeMatrix { .. } => Dimension::CooperativeMatrix,
26        _ => unreachable!(),
27    }
28}
29
30/// How to derive the type of `OpAccessChain` instructions from Naga IR.
31///
32/// Most of the time, we compile Naga IR to SPIR-V instructions whose result
33/// types are simply the direct SPIR-V analog of the Naga IR's. But in some
34/// cases, the Naga IR and SPIR-V types need to diverge.
35///
36/// This enum specifies how [`BlockContext::write_access_chain`] should
37/// choose a SPIR-V result type for the `OpAccessChain` it generates, based on
38/// the type of the given Naga IR [`Expression`] it's generating code for.
39///
40/// [`Expression`]: crate::Expression
41#[derive(Copy, Clone)]
42enum AccessTypeAdjustment {
43    /// No adjustment needed: the SPIR-V type should be the direct
44    /// analog of the Naga IR expression type.
45    ///
46    /// For most access chains, this is the right thing: the Naga IR access
47    /// expression produces a [`Pointer`] to the element / component, and the
48    /// SPIR-V `OpAccessChain` instruction does the same.
49    ///
50    /// [`Pointer`]: crate::TypeInner::Pointer
51    None,
52
53    /// The SPIR-V type should be an `OpPointer` to the direct analog of the
54    /// Naga IR expression's type.
55    ///
56    /// This is necessary for indexing binding arrays in the [`Handle`] address
57    /// space:
58    ///
59    /// - In Naga IR, referencing a binding array [`GlobalVariable`] in the
60    ///   [`Handle`] address space produces a value of type [`BindingArray`],
61    ///   not a pointer to such. And [`Access`] and [`AccessIndex`] expressions
62    ///   operate on handle binding arrays by value, and produce handle values,
63    ///   not pointers.
64    ///
65    /// - In SPIR-V, a binding array `OpVariable` produces a pointer to an
66    ///   array, and `OpAccessChain` instructions operate on pointers,
67    ///   regardless of whether the elements are opaque types or not.
68    ///
69    /// See also the documentation for [`BindingArray`].
70    ///
71    /// [`Handle`]: crate::AddressSpace::Handle
72    /// [`GlobalVariable`]: crate::GlobalVariable
73    /// [`BindingArray`]: crate::TypeInner::BindingArray
74    /// [`Access`]: crate::Expression::Access
75    /// [`AccessIndex`]: crate::Expression::AccessIndex
76    IntroducePointer(spirv::StorageClass),
77
78    /// The SPIR-V type should be an `OpPointer` to the std140 layout
79    /// compatible variant of the Naga IR expression's base type.
80    ///
81    /// This is used when accessing a type through an [`AddressSpace::Uniform`]
82    /// pointer in cases where the original type is incompatible with std140
83    /// layout requirements and we have therefore declared the uniform to be of
84    /// an alternative std140 compliant type.
85    ///
86    /// [`AddressSpace::Uniform`]: crate::AddressSpace::Uniform
87    UseStd140CompatType,
88}
89
90/// The results of emitting code for a left-hand-side expression.
91///
92/// On success, `write_access_chain` returns one of these.
93enum ExpressionPointer {
94    /// The pointer to the expression's value is available, as the value of the
95    /// expression with the given id.
96    Ready { pointer_id: Word },
97
98    /// The access expression must be conditional on the value of `condition`, a boolean
99    /// expression that is true if all indices are in bounds. If `condition` is true, then
100    /// `access` is an `OpAccessChain` instruction that will compute a pointer to the
101    /// expression's value. If `condition` is false, then executing `access` would be
102    /// undefined behavior.
103    Conditional {
104        condition: Word,
105        access: Instruction,
106    },
107}
108
109/// The termination statement to be added to the end of the block
110enum BlockExit {
111    /// Generates an OpReturn (void return)
112    Return,
113    /// Generates an OpBranch to the specified block
114    Branch {
115        /// The branch target block
116        target: Word,
117    },
118    /// Translates a loop `break if` into an `OpBranchConditional` to the
119    /// merge block if true (the merge block is passed through [`LoopContext::break_id`]
120    /// or else to the loop header (passed through [`preamble_id`])
121    ///
122    /// [`preamble_id`]: Self::BreakIf::preamble_id
123    BreakIf {
124        /// The condition of the `break if`
125        condition: Handle<crate::Expression>,
126        /// The loop header block id
127        preamble_id: Word,
128    },
129}
130
131/// What code generation did with a provided [`BlockExit`] value.
132///
133/// A function that accepts a [`BlockExit`] argument should return a value of
134/// this type, to indicate whether the code it generated ended up using the
135/// provided exit, or ignored it and did a non-local exit of some other kind
136/// (say, [`Break`] or [`Continue`]). Some callers must use this information to
137/// decide whether to generate the target block at all.
138///
139/// [`Break`]: Statement::Break
140/// [`Continue`]: Statement::Continue
141#[must_use]
142enum BlockExitDisposition {
143    /// The generated code used the provided `BlockExit` value. If it included a
144    /// block label, the caller should be sure to actually emit the block it
145    /// refers to.
146    Used,
147
148    /// The generated code did not use the provided `BlockExit` value. If it
149    /// included a block label, the caller should not bother to actually emit
150    /// the block it refers to, unless it knows the block is needed for
151    /// something else.
152    Discarded,
153}
154
155#[derive(Clone, Copy, Default)]
156struct LoopContext {
157    continuing_id: Option<Word>,
158    break_id: Option<Word>,
159}
160
161#[derive(Debug)]
162pub(crate) struct DebugInfoInner<'a> {
163    pub source_code: &'a str,
164    pub source_file_id: Word,
165}
166
167impl Writer {
168    // Flip Y coordinate to adjust for coordinate space difference
169    // between SPIR-V and our IR.
170    // The `position_id` argument is a pointer to a `vecN<f32>`,
171    // whose `y` component we will negate.
172    fn write_epilogue_position_y_flip(
173        &mut self,
174        position_id: Word,
175        body: &mut Vec<Instruction>,
176    ) -> Result<(), Error> {
177        let float_ptr_type_id = self.get_f32_pointer_type_id(spirv::StorageClass::Output);
178        let index_y_id = self.get_index_constant(1);
179        let access_id = self.id_gen.next();
180        body.push(Instruction::access_chain(
181            float_ptr_type_id,
182            access_id,
183            position_id,
184            &[index_y_id],
185        ));
186
187        let float_type_id = self.get_f32_type_id();
188        let load_id = self.id_gen.next();
189        body.push(Instruction::load(float_type_id, load_id, access_id, None));
190
191        let neg_id = self.id_gen.next();
192        body.push(Instruction::unary(
193            spirv::Op::FNegate,
194            float_type_id,
195            neg_id,
196            load_id,
197        ));
198
199        body.push(Instruction::store(access_id, neg_id, None));
200        Ok(())
201    }
202
203    // Clamp fragment depth between 0 and 1.
204    fn write_epilogue_frag_depth_clamp(
205        &mut self,
206        frag_depth_id: Word,
207        body: &mut Vec<Instruction>,
208    ) -> Result<(), Error> {
209        let float_type_id = self.get_f32_type_id();
210        let zero_scalar_id = self.get_constant_scalar(crate::Literal::F32(0.0));
211        let one_scalar_id = self.get_constant_scalar(crate::Literal::F32(1.0));
212
213        let original_id = self.id_gen.next();
214        body.push(Instruction::load(
215            float_type_id,
216            original_id,
217            frag_depth_id,
218            None,
219        ));
220
221        let clamp_id = self.id_gen.next();
222        body.push(Instruction::ext_inst_gl_op(
223            self.gl450_ext_inst_id,
224            spirv::GlslStd450Op::FClamp,
225            float_type_id,
226            clamp_id,
227            &[original_id, zero_scalar_id, one_scalar_id],
228        ));
229
230        body.push(Instruction::store(frag_depth_id, clamp_id, None));
231        Ok(())
232    }
233
234    fn write_entry_point_return(
235        &mut self,
236        value_id: Word,
237        ir_result: &crate::FunctionResult,
238        result_members: &[ResultMember],
239        body: &mut Vec<Instruction>,
240    ) -> Result<Instruction, Error> {
241        for (index, res_member) in result_members.iter().enumerate() {
242            // This isn't a real builtin, and is handled elsewhere
243            if res_member.built_in == Some(crate::BuiltIn::MeshTaskSize) {
244                return Ok(Instruction::return_value(value_id));
245            }
246            let member_value_id = match ir_result.binding {
247                Some(_) => value_id,
248                None => {
249                    let member_value_id = self.id_gen.next();
250                    body.push(Instruction::composite_extract(
251                        res_member.type_id,
252                        member_value_id,
253                        value_id,
254                        &[index as u32],
255                    ));
256                    member_value_id
257                }
258            };
259
260            self.store_io_with_f16_polyfill(body, res_member.id, member_value_id);
261
262            match res_member.built_in {
263                Some(crate::BuiltIn::Position { .. })
264                    if self.flags.contains(WriterFlags::ADJUST_COORDINATE_SPACE) =>
265                {
266                    self.write_epilogue_position_y_flip(res_member.id, body)?;
267                }
268                Some(crate::BuiltIn::FragDepth)
269                    if self.flags.contains(WriterFlags::CLAMP_FRAG_DEPTH) =>
270                {
271                    self.write_epilogue_frag_depth_clamp(res_member.id, body)?;
272                }
273                _ => {}
274            }
275        }
276        Ok(Instruction::return_void())
277    }
278}
279
280impl BlockContext<'_> {
281    /// Generates code to ensure that a loop is bounded. Should be called immediately
282    /// after adding the OpLoopMerge instruction to `block`. This function will
283    /// [`consume()`](crate::back::spv::Function::consume) `block` and append its
284    /// instructions to a new [`Block`], which will be returned to the caller for it to
285    /// consumed prior to writing the loop body.
286    ///
287    /// Additionally this function will populate [`force_loop_bounding_vars`](crate::back::spv::Function::force_loop_bounding_vars),
288    /// ensuring that [`Function::to_words()`](crate::back::spv::Function::to_words) will
289    /// declare the required variables.
290    ///
291    /// See [`crate::back::msl::Writer::gen_force_bounded_loop_statements`] for details
292    /// of why this is required.
293    fn write_force_bounded_loop_instructions(&mut self, mut block: Block, merge_id: Word) -> Block {
294        let uint_type_id = self.writer.get_u32_type_id();
295        let uint2_type_id = self.writer.get_vec2u_type_id();
296        let uint2_ptr_type_id = self
297            .writer
298            .get_vec2u_pointer_type_id(spirv::StorageClass::Function);
299        let bool_type_id = self.writer.get_bool_type_id();
300        let bool2_type_id = self.writer.get_vec2_bool_type_id();
301        let zero_uint_const_id = self.writer.get_constant_scalar(crate::Literal::U32(0));
302        let zero_uint2_const_id = self.writer.get_constant_composite(
303            LookupType::Local(LocalType::Numeric(NumericType::Vector {
304                size: crate::VectorSize::Bi,
305                scalar: crate::Scalar::U32,
306            })),
307            &[zero_uint_const_id, zero_uint_const_id],
308        );
309        let one_uint_const_id = self.writer.get_constant_scalar(crate::Literal::U32(1));
310        let max_uint_const_id = self
311            .writer
312            .get_constant_scalar(crate::Literal::U32(u32::MAX));
313        let max_uint2_const_id = self.writer.get_constant_composite(
314            LookupType::Local(LocalType::Numeric(NumericType::Vector {
315                size: crate::VectorSize::Bi,
316                scalar: crate::Scalar::U32,
317            })),
318            &[max_uint_const_id, max_uint_const_id],
319        );
320
321        let loop_counter_var_id = self.gen_id();
322        if self.writer.flags.contains(WriterFlags::DEBUG) {
323            self.writer
324                .debugs
325                .push(Instruction::name(loop_counter_var_id, "loop_bound"));
326        }
327        let var = super::LocalVariable {
328            id: loop_counter_var_id,
329            instruction: Instruction::variable(
330                uint2_ptr_type_id,
331                loop_counter_var_id,
332                spirv::StorageClass::Function,
333                Some(max_uint2_const_id),
334            ),
335        };
336        self.function.force_loop_bounding_vars.push(var);
337
338        let break_if_block = self.gen_id();
339
340        self.function
341            .consume(block, Instruction::branch(break_if_block));
342        block = Block::new(break_if_block);
343
344        // Load the current loop counter value from its variable. We use a vec2<u32> to
345        // simulate a 64-bit counter.
346        let load_id = self.gen_id();
347        block.body.push(Instruction::load(
348            uint2_type_id,
349            load_id,
350            loop_counter_var_id,
351            None,
352        ));
353
354        // If both the high and low u32s have reached 0 then break. ie
355        // if (all(eq(loop_counter, vec2(0)))) { break; }
356        let eq_id = self.gen_id();
357        block.body.push(Instruction::binary(
358            spirv::Op::IEqual,
359            bool2_type_id,
360            eq_id,
361            zero_uint2_const_id,
362            load_id,
363        ));
364        let all_eq_id = self.gen_id();
365        block.body.push(Instruction::relational(
366            spirv::Op::All,
367            bool_type_id,
368            all_eq_id,
369            eq_id,
370        ));
371
372        let inc_counter_block_id = self.gen_id();
373        block.body.push(Instruction::selection_merge(
374            inc_counter_block_id,
375            spirv::SelectionControl::empty(),
376        ));
377        self.function.consume(
378            block,
379            Instruction::branch_conditional(all_eq_id, merge_id, inc_counter_block_id),
380        );
381        block = Block::new(inc_counter_block_id);
382
383        // To simulate a 64-bit counter we always decrement the low u32, and decrement
384        // the high u32 when the low u32 overflows. ie
385        // counter -= vec2(select(0u, 1u, counter.y == 0), 1u);
386        // Count down from u32::MAX rather than up from 0 to avoid hang on
387        // certain Intel drivers. See <https://github.com/gfx-rs/wgpu/issues/7319>.
388        let low_id = self.gen_id();
389        block.body.push(Instruction::composite_extract(
390            uint_type_id,
391            low_id,
392            load_id,
393            &[1],
394        ));
395        let low_overflow_id = self.gen_id();
396        block.body.push(Instruction::binary(
397            spirv::Op::IEqual,
398            bool_type_id,
399            low_overflow_id,
400            low_id,
401            zero_uint_const_id,
402        ));
403        let carry_bit_id = self.gen_id();
404        block.body.push(Instruction::select(
405            uint_type_id,
406            carry_bit_id,
407            low_overflow_id,
408            one_uint_const_id,
409            zero_uint_const_id,
410        ));
411        let decrement_id = self.gen_id();
412        block.body.push(Instruction::composite_construct(
413            uint2_type_id,
414            decrement_id,
415            &[carry_bit_id, one_uint_const_id],
416        ));
417        let result_id = self.gen_id();
418        block.body.push(Instruction::binary(
419            spirv::Op::ISub,
420            uint2_type_id,
421            result_id,
422            load_id,
423            decrement_id,
424        ));
425        block
426            .body
427            .push(Instruction::store(loop_counter_var_id, result_id, None));
428
429        block
430    }
431
432    /// If `pointer` refers to a scalar reached by a dynamic (non-constant)
433    /// index into a vector in the [`Immediate`] (push-constant) address
434    /// space, write code to access the value, returning the ID of the
435    /// result. Else return `None`.
436    ///
437    /// `VUID-RuntimeSpirv-None-04745` requires:
438    ///
439    /// > All block members in a variable with a Storage Class of PushConstant
440    /// > declared as an array must only be accessed by dynamically uniform
441    /// > indices
442    ///
443    /// According to [upstream discussion], this requirement was intended to
444    /// allow drivers to always compile loads from push constants as scalar
445    /// loads; the omission of vectors seems to be an oversight.
446    ///
447    /// Naga IR, however, has no such restriction on indexing vectors in the
448    /// `Immediate` address space. So Naga must not emit `OpAccessChain` with
449    /// such an index directly into a `PushConstant` vector.
450    ///
451    /// Instead, this loads the whole vector -- a plain `OpLoad`, which isn't
452    /// subject to the restriction above -- and then extracts the desired
453    /// component from that loaded value with `OpVectorExtractDynamic`, exactly
454    /// as [`Self::write_vector_access()`] already does for by-value vectors.
455    ///
456    /// [`Immediate`]: crate::AddressSpace::Immediate
457    /// [upstream discussion]: https://gitlab.freedesktop.org/mesa/mesa/-/work_items/15891#note_3573369
458    fn maybe_write_immediate_vector_dynamic_access(
459        &mut self,
460        pointer: Handle<crate::Expression>,
461        block: &mut Block,
462    ) -> Result<Option<Word>, Error> {
463        // We're only interested in a scalar reached by indexing into a
464        // vector with a computed (not compile-time-constant) index.
465        let crate::Expression::Access {
466            base: vector_pointer,
467            index,
468        } = self.ir_function.expressions[pointer]
469        else {
470            return Ok(None);
471        };
472
473        // If the index is actually a compile-time constant, the plain
474        // access-chain path is fine: constants are as uniform as can be.
475        if let GuardedIndex::Known(_) =
476            GuardedIndex::from_expression(index, &self.ir_function.expressions, self.ir_module)
477        {
478            return Ok(None);
479        }
480
481        // Ensure `vector_pointer` is a pointer to a vector in the Immediate
482        // (push-constant) address space.
483        let vector_pointer_ty = self.fun_info[vector_pointer]
484            .ty
485            .inner_with(&self.ir_module.types);
486        if vector_pointer_ty.pointer_space() != Some(crate::AddressSpace::Immediate) {
487            return Ok(None);
488        }
489        let Some(vector_base_ty) = vector_pointer_ty.pointer_base_type() else {
490            return Ok(None);
491        };
492        let crate::TypeInner::Vector { size, scalar } =
493            *vector_base_ty.inner_with(&self.ir_module.types)
494        else {
495            return Ok(None);
496        };
497
498        let vector_type_id = self.get_numeric_type_id(NumericType::Vector { size, scalar });
499        let component_type_id = self.get_numeric_type_id(NumericType::Scalar(scalar));
500
501        let vector_load_id = self.write_checked_load(
502            vector_pointer,
503            block,
504            AccessTypeAdjustment::None,
505            vector_type_id,
506        )?;
507
508        let result_id = self.write_vector_access(
509            component_type_id,
510            vector_pointer,
511            Some(vector_load_id),
512            GuardedIndex::Expression(index),
513            block,
514        )?;
515
516        Ok(Some(result_id))
517    }
518
519    /// If `pointer` refers to an access chain that contains a dynamic indexing
520    /// of a two-row matrix in the [`Uniform`] address space, write code to
521    /// access the value returning the ID of the result. Else return None.
522    ///
523    /// Two-row matrices in the uniform address space will have been declared
524    /// using a alternative std140 layout compatible type, where each column is
525    /// a member of a containing struct. As a result, SPIR-V is unable to access
526    /// its columns with a non-constant index. To work around this limitation
527    /// this function will call [`Self::write_checked_load()`] to load the
528    /// matrix itself, which handles conversion from the std140 compatible type
529    /// to the real matrix type. It then calls a [`wrapper function`] to obtain
530    /// the correct column from the matrix, and possibly extracts a component
531    /// from the vector too.
532    ///
533    /// [`Uniform`]: crate::AddressSpace::Uniform
534    /// [`wrapper function`]: super::Writer::write_wrapped_matcx2_get_column
535    fn maybe_write_uniform_matcx2_dynamic_access(
536        &mut self,
537        pointer: Handle<crate::Expression>,
538        block: &mut Block,
539    ) -> Result<Option<Word>, Error> {
540        // If this access chain contains a dynamic matrix access, `pointer` is
541        // either a pointer to a vector (the column) or a scalar (a component
542        // within the column). In either case grab the pointer to the column,
543        // and remember the component index if there is one. If `pointer`
544        // points to any other type we're not interested.
545        let (column_pointer, component_index) = match self.fun_info[pointer]
546            .ty
547            .inner_with(&self.ir_module.types)
548            .pointer_base_type()
549        {
550            Some(resolution) => match *resolution.inner_with(&self.ir_module.types) {
551                crate::TypeInner::Scalar(_) => match self.ir_function.expressions[pointer] {
552                    crate::Expression::Access { base, index } => {
553                        (base, Some(GuardedIndex::Expression(index)))
554                    }
555                    crate::Expression::AccessIndex { base, index } => {
556                        (base, Some(GuardedIndex::Known(index)))
557                    }
558                    _ => return Ok(None),
559                },
560                crate::TypeInner::Vector { .. } => (pointer, None),
561                _ => return Ok(None),
562            },
563            None => return Ok(None),
564        };
565
566        // Ensure the column is accessed with a dynamic index (i.e.
567        // `Expression::Access`), and grab the pointer to the matrix.
568        let crate::Expression::Access {
569            base: matrix_pointer,
570            index: column_index,
571        } = self.ir_function.expressions[column_pointer]
572        else {
573            return Ok(None);
574        };
575
576        // Ensure the matrix pointer is in the uniform address space.
577        let crate::TypeInner::Pointer {
578            base: matrix_pointer_base_type,
579            space: crate::AddressSpace::Uniform,
580        } = *self.fun_info[matrix_pointer]
581            .ty
582            .inner_with(&self.ir_module.types)
583        else {
584            return Ok(None);
585        };
586
587        // Ensure the matrix pointer actually points to a Cx2 matrix.
588        let crate::TypeInner::Matrix {
589            columns,
590            rows: rows @ crate::VectorSize::Bi,
591            scalar,
592        } = self.ir_module.types[matrix_pointer_base_type].inner
593        else {
594            return Ok(None);
595        };
596
597        let matrix_type_id = self.get_numeric_type_id(NumericType::Matrix {
598            columns,
599            rows,
600            scalar,
601        });
602        let column_type_id = self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
603        let component_type_id = self.get_numeric_type_id(NumericType::Scalar(scalar));
604        let get_column_function_id = self.writer.wrapped_functions
605            [&WrappedFunction::MatCx2GetColumn {
606                r#type: matrix_pointer_base_type,
607            }];
608
609        let matrix_load_id = self.write_checked_load(
610            matrix_pointer,
611            block,
612            AccessTypeAdjustment::None,
613            matrix_type_id,
614        )?;
615
616        // Naga IR allows the index to be either an I32 or U32 but our wrapper
617        // function expects a U32 argument, so convert it if required.
618        let column_index_id = match *self.fun_info[column_index]
619            .ty
620            .inner_with(&self.ir_module.types)
621        {
622            crate::TypeInner::Scalar(crate::Scalar {
623                kind: crate::ScalarKind::Uint,
624                ..
625            }) => self.cached[column_index],
626            crate::TypeInner::Scalar(crate::Scalar {
627                kind: crate::ScalarKind::Sint,
628                ..
629            }) => {
630                let cast_id = self.gen_id();
631                let u32_type_id = self.writer.get_u32_type_id();
632                block.body.push(Instruction::unary(
633                    spirv::Op::Bitcast,
634                    u32_type_id,
635                    cast_id,
636                    self.cached[column_index],
637                ));
638                cast_id
639            }
640            _ => return Err(Error::Validation("Matrix access index must be u32 or i32")),
641        };
642        let column_id = self.gen_id();
643        block.body.push(Instruction::function_call(
644            column_type_id,
645            column_id,
646            get_column_function_id,
647            &[matrix_load_id, column_index_id],
648        ));
649        let result_id = match component_index {
650            Some(index) => self.write_vector_access(
651                component_type_id,
652                column_pointer,
653                Some(column_id),
654                index,
655                block,
656            )?,
657            None => column_id,
658        };
659
660        Ok(Some(result_id))
661    }
662
663    /// If `pointer` refers to two-row matrix that is a member of a struct in
664    /// the [`Uniform`] address space, write code to load the matrix returning
665    /// the ID of the result. Else return None.
666    ///
667    /// Two-row matrices that are struct members in the uniform address space
668    /// will have been decomposed such that the struct contains a separate
669    /// vector member for each column of the matrix. This function will load
670    /// each column separately from the containing struct, then composite them
671    /// into the real matrix type.
672    ///
673    /// [`Uniform`]: crate::AddressSpace::Uniform
674    fn maybe_write_load_uniform_matcx2_struct_member(
675        &mut self,
676        pointer: Handle<crate::Expression>,
677        block: &mut Block,
678    ) -> Result<Option<Word>, Error> {
679        // Check this is a uniform address space pointer to a two-row matrix.
680        let crate::TypeInner::Pointer {
681            base: matrix_type,
682            space: space @ crate::AddressSpace::Uniform,
683        } = *self.fun_info[pointer].ty.inner_with(&self.ir_module.types)
684        else {
685            return Ok(None);
686        };
687
688        let crate::TypeInner::Matrix {
689            columns,
690            rows: rows @ crate::VectorSize::Bi,
691            scalar,
692        } = self.ir_module.types[matrix_type].inner
693        else {
694            return Ok(None);
695        };
696
697        // Check this is a struct member. Note struct members can only be
698        // accessed with `AccessIndex`.
699        let crate::Expression::AccessIndex {
700            base: struct_pointer,
701            index: member_index,
702        } = self.ir_function.expressions[pointer]
703        else {
704            return Ok(None);
705        };
706
707        let crate::TypeInner::Pointer {
708            base: struct_type, ..
709        } = *self.fun_info[struct_pointer]
710            .ty
711            .inner_with(&self.ir_module.types)
712        else {
713            return Ok(None);
714        };
715
716        let crate::TypeInner::Struct { .. } = self.ir_module.types[struct_type].inner else {
717            return Ok(None);
718        };
719
720        let matrix_type_id = self.get_numeric_type_id(NumericType::Matrix {
721            columns,
722            rows,
723            scalar,
724        });
725        let column_type_id = self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
726        let column_pointer_type_id =
727            self.get_pointer_type_id(column_type_id, map_storage_class(space));
728        let column0_index = self.writer.std140_compat_uniform_types[&struct_type].member_indices
729            [member_index as usize];
730        let column_indices = (0..columns as u32)
731            .map(|c| self.get_index_constant(column0_index + c))
732            .collect::<ArrayVec<_, 4>>();
733
734        // Load each column from the struct, then composite into the real
735        // matrix type.
736        let load_mat_from_struct =
737            |struct_pointer_id: Word, id_gen: &mut IdGenerator, block: &mut Block| -> Word {
738                let mut column_ids: ArrayVec<Word, 4> = ArrayVec::new();
739                for index in &column_indices {
740                    let column_pointer_id = id_gen.next();
741                    block.body.push(Instruction::access_chain(
742                        column_pointer_type_id,
743                        column_pointer_id,
744                        struct_pointer_id,
745                        &[*index],
746                    ));
747                    let column_id = id_gen.next();
748                    block.body.push(Instruction::load(
749                        column_type_id,
750                        column_id,
751                        column_pointer_id,
752                        None,
753                    ));
754                    column_ids.push(column_id);
755                }
756                let result_id = id_gen.next();
757                block.body.push(Instruction::composite_construct(
758                    matrix_type_id,
759                    result_id,
760                    &column_ids,
761                ));
762                result_id
763            };
764
765        let result_id = match self.write_access_chain(
766            struct_pointer,
767            block,
768            AccessTypeAdjustment::UseStd140CompatType,
769        )? {
770            ExpressionPointer::Ready { pointer_id } => {
771                load_mat_from_struct(pointer_id, &mut self.writer.id_gen, block)
772            }
773            ExpressionPointer::Conditional { condition, access } => self
774                .write_conditional_indexed_load(
775                    matrix_type_id,
776                    condition,
777                    block,
778                    |id_gen, block| {
779                        let pointer_id = access.result_id.unwrap();
780                        block.body.push(access);
781                        load_mat_from_struct(pointer_id, id_gen, block)
782                    },
783                ),
784        };
785
786        Ok(Some(result_id))
787    }
788
789    /// Cache an expression for a value.
790    pub(super) fn cache_expression_value(
791        &mut self,
792        expr_handle: Handle<crate::Expression>,
793        block: &mut Block,
794    ) -> Result<(), Error> {
795        let is_named_expression = self
796            .ir_function
797            .named_expressions
798            .contains_key(&expr_handle);
799
800        if self.fun_info[expr_handle].ref_count == 0 && !is_named_expression {
801            return Ok(());
802        }
803
804        let result_type_id = self.get_expression_type_id(&self.fun_info[expr_handle].ty);
805        let id = match self.ir_function.expressions[expr_handle] {
806            crate::Expression::Literal(literal) => self.writer.get_constant_scalar(literal),
807            crate::Expression::Constant(handle) => {
808                let init = self.ir_module.constants[handle].init;
809                self.writer.constant_ids[init]
810            }
811            crate::Expression::Override(_) => return Err(Error::Override),
812            crate::Expression::ZeroValue(_) => self.writer.get_constant_null(result_type_id),
813            crate::Expression::Compose { ty, ref components } => {
814                self.temp_list.clear();
815                if self.expression_constness.is_const(expr_handle) {
816                    self.temp_list.extend(
817                        crate::proc::flatten_compose(
818                            ty,
819                            components,
820                            &self.ir_function.expressions,
821                            &self.ir_module.types,
822                        )
823                        .map(|component| self.cached[component]),
824                    );
825                    self.writer
826                        .get_constant_composite(LookupType::Handle(ty), &self.temp_list)
827                } else {
828                    self.temp_list
829                        .extend(components.iter().map(|&component| self.cached[component]));
830
831                    let id = self.gen_id();
832                    block.body.push(Instruction::composite_construct(
833                        result_type_id,
834                        id,
835                        &self.temp_list,
836                    ));
837                    id
838                }
839            }
840            crate::Expression::Splat { size, value } => {
841                let value_id = self.cached[value];
842                let components = &[value_id; 4][..size as usize];
843
844                if self.expression_constness.is_const(expr_handle) {
845                    let ty = self
846                        .writer
847                        .get_expression_lookup_type(&self.fun_info[expr_handle].ty);
848                    self.writer.get_constant_composite(ty, components)
849                } else {
850                    let id = self.gen_id();
851                    block.body.push(Instruction::composite_construct(
852                        result_type_id,
853                        id,
854                        components,
855                    ));
856                    id
857                }
858            }
859            crate::Expression::Access { base, index } => {
860                let base_ty_inner = self.fun_info[base].ty.inner_with(&self.ir_module.types);
861                match *base_ty_inner {
862                    crate::TypeInner::Pointer { .. } | crate::TypeInner::ValuePointer { .. } => {
863                        // When we have a chain of `Access` and `AccessIndex` expressions
864                        // operating on pointers, we want to generate a single
865                        // `OpAccessChain` instruction for the whole chain. Put off
866                        // generating any code for this until we find the `Expression`
867                        // that actually dereferences the pointer.
868                        0
869                    }
870                    _ if self.function.spilled_accesses.contains(base) => {
871                        // As far as Naga IR is concerned, this expression does not yield
872                        // a pointer (we just checked, above), but this backend spilled it
873                        // to a temporary variable, so SPIR-V thinks we're accessing it
874                        // via a pointer.
875
876                        // Since the base expression was spilled, mark this access to it
877                        // as spilled, too.
878                        self.function.spilled_accesses.insert(expr_handle);
879                        self.maybe_access_spilled_composite(expr_handle, block, result_type_id)?
880                    }
881                    crate::TypeInner::Vector { .. } => self.write_vector_access(
882                        result_type_id,
883                        base,
884                        None,
885                        GuardedIndex::Expression(index),
886                        block,
887                    )?,
888                    crate::TypeInner::Array { .. } | crate::TypeInner::Matrix { .. } => {
889                        // See if `index` is known at compile time.
890                        match GuardedIndex::from_expression(
891                            index,
892                            &self.ir_function.expressions,
893                            self.ir_module,
894                        ) {
895                            GuardedIndex::Known(value) => {
896                                // If `index` is known and in bounds, we can just use
897                                // `OpCompositeExtract`.
898                                //
899                                // At the moment, validation rejects programs if this
900                                // index is out of bounds, so we don't need bounds checks.
901                                // However, that rejection is incorrect, since WGSL says
902                                // that `let` bindings are not constant expressions
903                                // (#6396). So eventually we will need to emulate bounds
904                                // checks here.
905                                let id = self.gen_id();
906                                let base_id = self.cached[base];
907                                block.body.push(Instruction::composite_extract(
908                                    result_type_id,
909                                    id,
910                                    base_id,
911                                    &[value],
912                                ));
913                                id
914                            }
915                            GuardedIndex::Expression(_) => {
916                                // We are subscripting an array or matrix that is not
917                                // behind a pointer, using an index computed at runtime.
918                                // SPIR-V has no instructions that do this, so the best we
919                                // can do is spill the value to a new temporary variable,
920                                // at which point we can get a pointer to that and just
921                                // use `OpAccessChain` in the usual way.
922                                self.spill_to_internal_variable(base, block);
923
924                                // Since the base was spilled, mark this access to it as
925                                // spilled, too.
926                                self.function.spilled_accesses.insert(expr_handle);
927                                self.maybe_access_spilled_composite(
928                                    expr_handle,
929                                    block,
930                                    result_type_id,
931                                )?
932                            }
933                        }
934                    }
935                    crate::TypeInner::BindingArray {
936                        base: binding_type, ..
937                    } => {
938                        // Only binding arrays in the `Handle` address space will take
939                        // this path, since we handled the `Pointer` case above.
940                        let result_id = match self.write_access_chain(
941                            expr_handle,
942                            block,
943                            AccessTypeAdjustment::IntroducePointer(
944                                spirv::StorageClass::UniformConstant,
945                            ),
946                        )? {
947                            ExpressionPointer::Ready { pointer_id } => pointer_id,
948                            ExpressionPointer::Conditional { .. } => {
949                                return Err(Error::FeatureNotImplemented(
950                                    "Texture array out-of-bounds handling",
951                                ));
952                            }
953                        };
954
955                        let binding_type_id = self.get_handle_type_id(binding_type);
956
957                        let load_id = self.gen_id();
958                        block.body.push(Instruction::load(
959                            binding_type_id,
960                            load_id,
961                            result_id,
962                            None,
963                        ));
964
965                        // Subsequent image operations require the image/sampler to be decorated as NonUniform
966                        // if the image/sampler binding array was accessed with a non-uniform index
967                        // see VUID-RuntimeSpirv-NonUniform-06274
968                        if self.fun_info[index].uniformity.non_uniform_result.is_some() {
969                            self.writer
970                                .decorate_non_uniform_binding_array_access(load_id)?;
971                        }
972
973                        load_id
974                    }
975                    ref other => {
976                        log::error!(
977                            "Unable to access base {:?} of type {:?}",
978                            self.ir_function.expressions[base],
979                            other
980                        );
981                        return Err(Error::Validation(
982                            "only vectors and arrays may be dynamically indexed by value",
983                        ));
984                    }
985                }
986            }
987            crate::Expression::AccessIndex { base, index } => {
988                match *self.fun_info[base].ty.inner_with(&self.ir_module.types) {
989                    crate::TypeInner::Pointer { .. } | crate::TypeInner::ValuePointer { .. } => {
990                        // When we have a chain of `Access` and `AccessIndex` expressions
991                        // operating on pointers, we want to generate a single
992                        // `OpAccessChain` instruction for the whole chain. Put off
993                        // generating any code for this until we find the `Expression`
994                        // that actually dereferences the pointer.
995                        0
996                    }
997                    _ if self.function.spilled_accesses.contains(base) => {
998                        // As far as Naga IR is concerned, this expression does not yield
999                        // a pointer (we just checked, above), but this backend spilled it
1000                        // to a temporary variable, so SPIR-V thinks we're accessing it
1001                        // via a pointer.
1002
1003                        // Since the base expression was spilled, mark this access to it
1004                        // as spilled, too.
1005                        self.function.spilled_accesses.insert(expr_handle);
1006                        self.maybe_access_spilled_composite(expr_handle, block, result_type_id)?
1007                    }
1008                    crate::TypeInner::Vector { .. }
1009                    | crate::TypeInner::Matrix { .. }
1010                    | crate::TypeInner::Array { .. }
1011                    | crate::TypeInner::Struct { .. } => {
1012                        // We never need bounds checks here: dynamically sized arrays can
1013                        // only appear behind pointers, and are thus handled by the
1014                        // `is_intermediate` case above. Everything else's size is
1015                        // statically known and checked in validation.
1016                        let id = self.gen_id();
1017                        let base_id = self.cached[base];
1018                        block.body.push(Instruction::composite_extract(
1019                            result_type_id,
1020                            id,
1021                            base_id,
1022                            &[index],
1023                        ));
1024                        id
1025                    }
1026                    crate::TypeInner::BindingArray {
1027                        base: binding_type, ..
1028                    } => {
1029                        // Only binding arrays in the `Handle` address space will take
1030                        // this path, since we handled the `Pointer` case above.
1031                        let result_id = match self.write_access_chain(
1032                            expr_handle,
1033                            block,
1034                            AccessTypeAdjustment::IntroducePointer(
1035                                spirv::StorageClass::UniformConstant,
1036                            ),
1037                        )? {
1038                            ExpressionPointer::Ready { pointer_id } => pointer_id,
1039                            ExpressionPointer::Conditional { .. } => {
1040                                return Err(Error::FeatureNotImplemented(
1041                                    "Texture array out-of-bounds handling",
1042                                ));
1043                            }
1044                        };
1045
1046                        let binding_type_id = self.get_handle_type_id(binding_type);
1047
1048                        let load_id = self.gen_id();
1049                        block.body.push(Instruction::load(
1050                            binding_type_id,
1051                            load_id,
1052                            result_id,
1053                            None,
1054                        ));
1055
1056                        load_id
1057                    }
1058                    ref other => {
1059                        log::error!("Unable to access index of {other:?}");
1060                        return Err(Error::FeatureNotImplemented("access index for type"));
1061                    }
1062                }
1063            }
1064            crate::Expression::GlobalVariable(handle) => {
1065                self.writer.global_variables[handle].access_id
1066            }
1067            crate::Expression::Swizzle {
1068                size,
1069                vector,
1070                pattern,
1071            } => {
1072                let vector_id = self.cached[vector];
1073                self.temp_list.clear();
1074                for &sc in pattern[..size as usize].iter() {
1075                    self.temp_list.push(sc as Word);
1076                }
1077                let id = self.gen_id();
1078                block.body.push(Instruction::vector_shuffle(
1079                    result_type_id,
1080                    id,
1081                    vector_id,
1082                    vector_id,
1083                    &self.temp_list,
1084                ));
1085                id
1086            }
1087            crate::Expression::Unary { op, expr } => {
1088                let id = self.gen_id();
1089                let expr_id = self.cached[expr];
1090                let expr_ty_inner = self.fun_info[expr].ty.inner_with(&self.ir_module.types);
1091
1092                let spirv_op = match op {
1093                    crate::UnaryOperator::Negate => match expr_ty_inner.scalar_kind() {
1094                        Some(crate::ScalarKind::Float) => spirv::Op::FNegate,
1095                        Some(crate::ScalarKind::Sint) => spirv::Op::SNegate,
1096                        _ => return Err(Error::Validation("Unexpected kind for negation")),
1097                    },
1098                    crate::UnaryOperator::LogicalNot => spirv::Op::LogicalNot,
1099                    crate::UnaryOperator::BitwiseNot => spirv::Op::Not,
1100                };
1101
1102                block
1103                    .body
1104                    .push(Instruction::unary(spirv_op, result_type_id, id, expr_id));
1105                id
1106            }
1107            crate::Expression::Binary { op, left, right } => {
1108                let id = self.gen_id();
1109                let left_id = self.cached[left];
1110                let right_id = self.cached[right];
1111                let left_type_id = self.get_expression_type_id(&self.fun_info[left].ty);
1112                let right_type_id = self.get_expression_type_id(&self.fun_info[right].ty);
1113
1114                if let Some(function_id) =
1115                    self.writer
1116                        .wrapped_functions
1117                        .get(&WrappedFunction::BinaryOp {
1118                            op,
1119                            left_type_id,
1120                            right_type_id,
1121                        })
1122                {
1123                    block.body.push(Instruction::function_call(
1124                        result_type_id,
1125                        id,
1126                        *function_id,
1127                        &[left_id, right_id],
1128                    ));
1129                } else {
1130                    let left_ty_inner = self.fun_info[left].ty.inner_with(&self.ir_module.types);
1131                    let right_ty_inner = self.fun_info[right].ty.inner_with(&self.ir_module.types);
1132
1133                    let left_dimension = get_dimension(left_ty_inner);
1134                    let right_dimension = get_dimension(right_ty_inner);
1135
1136                    let mut reverse_operands = false;
1137
1138                    let spirv_op = match op {
1139                        crate::BinaryOperator::Add => match *left_ty_inner {
1140                            crate::TypeInner::Scalar(scalar)
1141                            | crate::TypeInner::Vector { scalar, .. } => match scalar.kind {
1142                                crate::ScalarKind::Float => spirv::Op::FAdd,
1143                                _ => spirv::Op::IAdd,
1144                            },
1145                            crate::TypeInner::Matrix {
1146                                columns,
1147                                rows,
1148                                scalar,
1149                            } => {
1150                                //TODO: why not just rely on `Fadd` for matrices?
1151                                self.write_matrix_matrix_column_op(
1152                                    block,
1153                                    id,
1154                                    result_type_id,
1155                                    left_id,
1156                                    right_id,
1157                                    columns,
1158                                    rows,
1159                                    scalar.width,
1160                                    spirv::Op::FAdd,
1161                                );
1162
1163                                self.cached[expr_handle] = id;
1164                                return Ok(());
1165                            }
1166                            crate::TypeInner::CooperativeMatrix { .. } => spirv::Op::FAdd,
1167                            _ => unimplemented!(),
1168                        },
1169                        crate::BinaryOperator::Subtract => match *left_ty_inner {
1170                            crate::TypeInner::Scalar(scalar)
1171                            | crate::TypeInner::Vector { scalar, .. } => match scalar.kind {
1172                                crate::ScalarKind::Float => spirv::Op::FSub,
1173                                _ => spirv::Op::ISub,
1174                            },
1175                            crate::TypeInner::Matrix {
1176                                columns,
1177                                rows,
1178                                scalar,
1179                            } => {
1180                                self.write_matrix_matrix_column_op(
1181                                    block,
1182                                    id,
1183                                    result_type_id,
1184                                    left_id,
1185                                    right_id,
1186                                    columns,
1187                                    rows,
1188                                    scalar.width,
1189                                    spirv::Op::FSub,
1190                                );
1191
1192                                self.cached[expr_handle] = id;
1193                                return Ok(());
1194                            }
1195                            crate::TypeInner::CooperativeMatrix { .. } => spirv::Op::FSub,
1196                            _ => unimplemented!(),
1197                        },
1198                        crate::BinaryOperator::Multiply => {
1199                            match (left_dimension, right_dimension) {
1200                                (Dimension::Scalar, Dimension::Vector) => {
1201                                    self.write_vector_scalar_mult(
1202                                        block,
1203                                        id,
1204                                        result_type_id,
1205                                        right_id,
1206                                        left_id,
1207                                        right_ty_inner,
1208                                    );
1209
1210                                    self.cached[expr_handle] = id;
1211                                    return Ok(());
1212                                }
1213                                (Dimension::Vector, Dimension::Scalar) => {
1214                                    self.write_vector_scalar_mult(
1215                                        block,
1216                                        id,
1217                                        result_type_id,
1218                                        left_id,
1219                                        right_id,
1220                                        left_ty_inner,
1221                                    );
1222
1223                                    self.cached[expr_handle] = id;
1224                                    return Ok(());
1225                                }
1226                                (Dimension::Vector, Dimension::Matrix) => {
1227                                    spirv::Op::VectorTimesMatrix
1228                                }
1229                                (Dimension::Matrix, Dimension::Scalar)
1230                                | (Dimension::CooperativeMatrix, Dimension::Scalar) => {
1231                                    spirv::Op::MatrixTimesScalar
1232                                }
1233                                (Dimension::Scalar, Dimension::Matrix)
1234                                | (Dimension::Scalar, Dimension::CooperativeMatrix) => {
1235                                    reverse_operands = true;
1236                                    spirv::Op::MatrixTimesScalar
1237                                }
1238                                (Dimension::Matrix, Dimension::Vector) => {
1239                                    spirv::Op::MatrixTimesVector
1240                                }
1241                                (Dimension::Matrix, Dimension::Matrix) => {
1242                                    spirv::Op::MatrixTimesMatrix
1243                                }
1244                                (Dimension::Vector, Dimension::Vector)
1245                                | (Dimension::Scalar, Dimension::Scalar)
1246                                    if left_ty_inner.scalar_kind()
1247                                        == Some(crate::ScalarKind::Float) =>
1248                                {
1249                                    spirv::Op::FMul
1250                                }
1251                                (Dimension::Vector, Dimension::Vector)
1252                                | (Dimension::Scalar, Dimension::Scalar) => spirv::Op::IMul,
1253                                (Dimension::CooperativeMatrix, Dimension::CooperativeMatrix)
1254                                //Note: technically can do `FMul` but IR doesn't have matrix per-component multiplication
1255                                | (Dimension::CooperativeMatrix, _)
1256                                | (_, Dimension::CooperativeMatrix) => {
1257                                    unimplemented!()
1258                                }
1259                            }
1260                        }
1261                        crate::BinaryOperator::Divide => match left_ty_inner.scalar_kind() {
1262                            Some(crate::ScalarKind::Sint) => spirv::Op::SDiv,
1263                            Some(crate::ScalarKind::Uint) => spirv::Op::UDiv,
1264                            Some(crate::ScalarKind::Float) => spirv::Op::FDiv,
1265                            _ => unimplemented!(),
1266                        },
1267                        crate::BinaryOperator::Modulo => match left_ty_inner.scalar_kind() {
1268                            Some(crate::ScalarKind::Float) => spirv::Op::FRem,
1269                            Some(crate::ScalarKind::Sint) => {
1270                                // Signed `%` is always lowered to `a - b * (a / b)`
1271                                // through a wrapper function (see
1272                                // `write_wrapped_functions`), because `OpSRem` with a
1273                                // negative operand is poison in the Vulkan SPIR-V
1274                                // environment without `VK_KHR_maintenance8`. So this raw
1275                                // path is never reached for signed modulo.
1276                                unreachable!("signed modulo must be lowered via the wrapped path")
1277                            }
1278                            Some(crate::ScalarKind::Uint) => {
1279                                assert!(!self.writer.emit_int_div_checks);
1280                                spirv::Op::UMod
1281                            }
1282                            _ => unimplemented!(),
1283                        },
1284                        crate::BinaryOperator::Equal => match left_ty_inner.scalar_kind() {
1285                            Some(crate::ScalarKind::Sint | crate::ScalarKind::Uint) => {
1286                                spirv::Op::IEqual
1287                            }
1288                            Some(crate::ScalarKind::Float) => spirv::Op::FOrdEqual,
1289                            Some(crate::ScalarKind::Bool) => spirv::Op::LogicalEqual,
1290                            _ => unimplemented!(),
1291                        },
1292                        crate::BinaryOperator::NotEqual => match left_ty_inner.scalar_kind() {
1293                            Some(crate::ScalarKind::Sint | crate::ScalarKind::Uint) => {
1294                                spirv::Op::INotEqual
1295                            }
1296                            Some(crate::ScalarKind::Float) => spirv::Op::FOrdNotEqual,
1297                            Some(crate::ScalarKind::Bool) => spirv::Op::LogicalNotEqual,
1298                            _ => unimplemented!(),
1299                        },
1300                        crate::BinaryOperator::Less => match left_ty_inner.scalar_kind() {
1301                            Some(crate::ScalarKind::Sint) => spirv::Op::SLessThan,
1302                            Some(crate::ScalarKind::Uint) => spirv::Op::ULessThan,
1303                            Some(crate::ScalarKind::Float) => spirv::Op::FOrdLessThan,
1304                            _ => unimplemented!(),
1305                        },
1306                        crate::BinaryOperator::LessEqual => match left_ty_inner.scalar_kind() {
1307                            Some(crate::ScalarKind::Sint) => spirv::Op::SLessThanEqual,
1308                            Some(crate::ScalarKind::Uint) => spirv::Op::ULessThanEqual,
1309                            Some(crate::ScalarKind::Float) => spirv::Op::FOrdLessThanEqual,
1310                            _ => unimplemented!(),
1311                        },
1312                        crate::BinaryOperator::Greater => match left_ty_inner.scalar_kind() {
1313                            Some(crate::ScalarKind::Sint) => spirv::Op::SGreaterThan,
1314                            Some(crate::ScalarKind::Uint) => spirv::Op::UGreaterThan,
1315                            Some(crate::ScalarKind::Float) => spirv::Op::FOrdGreaterThan,
1316                            _ => unimplemented!(),
1317                        },
1318                        crate::BinaryOperator::GreaterEqual => match left_ty_inner.scalar_kind() {
1319                            Some(crate::ScalarKind::Sint) => spirv::Op::SGreaterThanEqual,
1320                            Some(crate::ScalarKind::Uint) => spirv::Op::UGreaterThanEqual,
1321                            Some(crate::ScalarKind::Float) => spirv::Op::FOrdGreaterThanEqual,
1322                            _ => unimplemented!(),
1323                        },
1324                        crate::BinaryOperator::And => match left_ty_inner.scalar_kind() {
1325                            Some(crate::ScalarKind::Bool) => spirv::Op::LogicalAnd,
1326                            _ => spirv::Op::BitwiseAnd,
1327                        },
1328                        crate::BinaryOperator::ExclusiveOr => spirv::Op::BitwiseXor,
1329                        crate::BinaryOperator::InclusiveOr => match left_ty_inner.scalar_kind() {
1330                            Some(crate::ScalarKind::Bool) => spirv::Op::LogicalOr,
1331                            _ => spirv::Op::BitwiseOr,
1332                        },
1333                        crate::BinaryOperator::LogicalAnd => spirv::Op::LogicalAnd,
1334                        crate::BinaryOperator::LogicalOr => spirv::Op::LogicalOr,
1335                        crate::BinaryOperator::ShiftLeft => spirv::Op::ShiftLeftLogical,
1336                        crate::BinaryOperator::ShiftRight => match left_ty_inner.scalar_kind() {
1337                            Some(crate::ScalarKind::Sint) => spirv::Op::ShiftRightArithmetic,
1338                            Some(crate::ScalarKind::Uint) => spirv::Op::ShiftRightLogical,
1339                            _ => unimplemented!(),
1340                        },
1341                    };
1342
1343                    block.body.push(Instruction::binary(
1344                        spirv_op,
1345                        result_type_id,
1346                        id,
1347                        if reverse_operands { right_id } else { left_id },
1348                        if reverse_operands { left_id } else { right_id },
1349                    ));
1350                }
1351                id
1352            }
1353            crate::Expression::Math {
1354                fun,
1355                arg,
1356                arg1,
1357                arg2,
1358                arg3,
1359            } => {
1360                use crate::MathFunction as Mf;
1361                enum MathOp {
1362                    Ext(spirv::GlslStd450Op),
1363                    Custom(Instruction),
1364                }
1365
1366                let arg0_id = self.cached[arg];
1367                let arg_ty = self.fun_info[arg].ty.inner_with(&self.ir_module.types);
1368                let arg_scalar_kind = arg_ty.scalar_kind();
1369                let arg1_id = match arg1 {
1370                    Some(handle) => self.cached[handle],
1371                    None => 0,
1372                };
1373                let arg2_id = match arg2 {
1374                    Some(handle) => self.cached[handle],
1375                    None => 0,
1376                };
1377                let arg3_id = match arg3 {
1378                    Some(handle) => self.cached[handle],
1379                    None => 0,
1380                };
1381
1382                let id = self.gen_id();
1383                let math_op = match fun {
1384                    // comparison
1385                    Mf::Abs => {
1386                        match arg_scalar_kind {
1387                            Some(crate::ScalarKind::Float) => {
1388                                MathOp::Ext(spirv::GlslStd450Op::FAbs)
1389                            }
1390                            Some(crate::ScalarKind::Sint) => MathOp::Ext(spirv::GlslStd450Op::SAbs),
1391                            Some(crate::ScalarKind::Uint) => {
1392                                MathOp::Custom(Instruction::unary(
1393                                    spirv::Op::CopyObject, // do nothing
1394                                    result_type_id,
1395                                    id,
1396                                    arg0_id,
1397                                ))
1398                            }
1399                            other => unimplemented!("Unexpected abs({:?})", other),
1400                        }
1401                    }
1402                    Mf::Min => MathOp::Ext(match arg_scalar_kind {
1403                        Some(crate::ScalarKind::Float) => spirv::GlslStd450Op::FMin,
1404                        Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::SMin,
1405                        Some(crate::ScalarKind::Uint) => spirv::GlslStd450Op::UMin,
1406                        other => unimplemented!("Unexpected min({:?})", other),
1407                    }),
1408                    Mf::Max => MathOp::Ext(match arg_scalar_kind {
1409                        Some(crate::ScalarKind::Float) => spirv::GlslStd450Op::FMax,
1410                        Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::SMax,
1411                        Some(crate::ScalarKind::Uint) => spirv::GlslStd450Op::UMax,
1412                        other => unimplemented!("Unexpected max({:?})", other),
1413                    }),
1414                    Mf::Clamp => match arg_scalar_kind {
1415                        // Clamp is undefined if min > max. In practice this means it can use a median-of-three
1416                        // instruction to determine the value. This is fine according to the WGSL spec for float
1417                        // clamp, but integer clamp _must_ use min-max. As such we write out min/max.
1418                        Some(crate::ScalarKind::Float) => MathOp::Ext(spirv::GlslStd450Op::FClamp),
1419                        Some(_) => {
1420                            let (min_op, max_op) = match arg_scalar_kind {
1421                                Some(crate::ScalarKind::Sint) => {
1422                                    (spirv::GlslStd450Op::SMin, spirv::GlslStd450Op::SMax)
1423                                }
1424                                Some(crate::ScalarKind::Uint) => {
1425                                    (spirv::GlslStd450Op::UMin, spirv::GlslStd450Op::UMax)
1426                                }
1427                                _ => unreachable!(),
1428                            };
1429
1430                            let max_id = self.gen_id();
1431                            block.body.push(Instruction::ext_inst_gl_op(
1432                                self.writer.gl450_ext_inst_id,
1433                                max_op,
1434                                result_type_id,
1435                                max_id,
1436                                &[arg0_id, arg1_id],
1437                            ));
1438
1439                            MathOp::Custom(Instruction::ext_inst_gl_op(
1440                                self.writer.gl450_ext_inst_id,
1441                                min_op,
1442                                result_type_id,
1443                                id,
1444                                &[max_id, arg2_id],
1445                            ))
1446                        }
1447                        other => unimplemented!("Unexpected max({:?})", other),
1448                    },
1449                    Mf::Saturate => {
1450                        let (maybe_size, scalar) = match *arg_ty {
1451                            crate::TypeInner::Vector { size, scalar } => (Some(size), scalar),
1452                            crate::TypeInner::Scalar(scalar) => (None, scalar),
1453                            ref other => unimplemented!("Unexpected saturate({:?})", other),
1454                        };
1455                        let scalar = crate::Scalar::float(scalar.width);
1456                        let mut arg1_id = self.writer.get_constant_scalar_with(0, scalar)?;
1457                        let mut arg2_id = self.writer.get_constant_scalar_with(1, scalar)?;
1458
1459                        if let Some(size) = maybe_size {
1460                            let ty =
1461                                LocalType::Numeric(NumericType::Vector { size, scalar }).into();
1462
1463                            self.temp_list.clear();
1464                            self.temp_list.resize(size as _, arg1_id);
1465
1466                            arg1_id = self.writer.get_constant_composite(ty, &self.temp_list);
1467
1468                            self.temp_list.fill(arg2_id);
1469
1470                            arg2_id = self.writer.get_constant_composite(ty, &self.temp_list);
1471                        }
1472
1473                        MathOp::Custom(Instruction::ext_inst_gl_op(
1474                            self.writer.gl450_ext_inst_id,
1475                            spirv::GlslStd450Op::FClamp,
1476                            result_type_id,
1477                            id,
1478                            &[arg0_id, arg1_id, arg2_id],
1479                        ))
1480                    }
1481                    // trigonometry
1482                    Mf::Sin => MathOp::Ext(spirv::GlslStd450Op::Sin),
1483                    Mf::Sinh => MathOp::Ext(spirv::GlslStd450Op::Sinh),
1484                    Mf::Asin => MathOp::Ext(spirv::GlslStd450Op::Asin),
1485                    Mf::Cos => MathOp::Ext(spirv::GlslStd450Op::Cos),
1486                    Mf::Cosh => MathOp::Ext(spirv::GlslStd450Op::Cosh),
1487                    Mf::Acos => MathOp::Ext(spirv::GlslStd450Op::Acos),
1488                    Mf::Tan => MathOp::Ext(spirv::GlslStd450Op::Tan),
1489                    Mf::Tanh => MathOp::Ext(spirv::GlslStd450Op::Tanh),
1490                    Mf::Atan => MathOp::Ext(spirv::GlslStd450Op::Atan),
1491                    Mf::Atan2 => MathOp::Ext(spirv::GlslStd450Op::Atan2),
1492                    Mf::Asinh => MathOp::Ext(spirv::GlslStd450Op::Asinh),
1493                    Mf::Acosh => MathOp::Ext(spirv::GlslStd450Op::Acosh),
1494                    Mf::Atanh => MathOp::Ext(spirv::GlslStd450Op::Atanh),
1495                    Mf::Radians => MathOp::Ext(spirv::GlslStd450Op::Radians),
1496                    Mf::Degrees => MathOp::Ext(spirv::GlslStd450Op::Degrees),
1497                    // decomposition
1498                    Mf::Ceil => MathOp::Ext(spirv::GlslStd450Op::Ceil),
1499                    Mf::Round => MathOp::Ext(spirv::GlslStd450Op::RoundEven),
1500                    Mf::Floor => MathOp::Ext(spirv::GlslStd450Op::Floor),
1501                    Mf::Fract => MathOp::Ext(spirv::GlslStd450Op::Fract),
1502                    Mf::Trunc => MathOp::Ext(spirv::GlslStd450Op::Trunc),
1503                    Mf::Modf => MathOp::Ext(spirv::GlslStd450Op::ModfStruct),
1504                    Mf::Frexp => MathOp::Ext(spirv::GlslStd450Op::FrexpStruct),
1505                    Mf::Ldexp => MathOp::Ext(spirv::GlslStd450Op::Ldexp),
1506                    // geometry
1507                    Mf::Dot => match *self.fun_info[arg].ty.inner_with(&self.ir_module.types) {
1508                        crate::TypeInner::Vector {
1509                            scalar:
1510                                crate::Scalar {
1511                                    kind: crate::ScalarKind::Float,
1512                                    ..
1513                                },
1514                            ..
1515                        } => MathOp::Custom(Instruction::binary(
1516                            spirv::Op::Dot,
1517                            result_type_id,
1518                            id,
1519                            arg0_id,
1520                            arg1_id,
1521                        )),
1522                        // TODO: consider using integer dot product if VK_KHR_shader_integer_dot_product is available
1523                        crate::TypeInner::Vector { size, .. } => {
1524                            self.write_dot_product(
1525                                id,
1526                                result_type_id,
1527                                arg0_id,
1528                                arg1_id,
1529                                size as u32,
1530                                block,
1531                                |result_id, composite_id, index| {
1532                                    Instruction::composite_extract(
1533                                        result_type_id,
1534                                        result_id,
1535                                        composite_id,
1536                                        &[index],
1537                                    )
1538                                },
1539                            );
1540                            self.cached[expr_handle] = id;
1541                            return Ok(());
1542                        }
1543                        _ => unreachable!(
1544                            "Correct TypeInner for dot product should be already validated"
1545                        ),
1546                    },
1547                    fun @ (Mf::Dot4I8Packed | Mf::Dot4U8Packed) => {
1548                        if self
1549                            .writer
1550                            .require_all(&[
1551                                spirv::Capability::DotProduct,
1552                                spirv::Capability::DotProductInput4x8BitPacked,
1553                            ])
1554                            .is_ok()
1555                        {
1556                            // Write optimized code using `PackedVectorFormat4x8Bit`.
1557                            if self.writer.lang_version() < (1, 6) {
1558                                // SPIR-V 1.6 supports the required capabilities natively, so the extension
1559                                // is only required for earlier versions. See right column of
1560                                // <https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpSDot>.
1561                                self.writer.use_extension("SPV_KHR_integer_dot_product");
1562                            }
1563
1564                            let op = match fun {
1565                                Mf::Dot4I8Packed => spirv::Op::SDot,
1566                                Mf::Dot4U8Packed => spirv::Op::UDot,
1567                                _ => unreachable!(),
1568                            };
1569
1570                            block.body.push(Instruction::ternary(
1571                                op,
1572                                result_type_id,
1573                                id,
1574                                arg0_id,
1575                                arg1_id,
1576                                spirv::PackedVectorFormat::PackedVectorFormat4x8Bit as Word,
1577                            ));
1578                        } else {
1579                            // Fall back to a polyfill since `PackedVectorFormat4x8Bit` is not available.
1580                            let (extract_op, arg0_id, arg1_id) = match fun {
1581                                Mf::Dot4U8Packed => (spirv::Op::BitFieldUExtract, arg0_id, arg1_id),
1582                                Mf::Dot4I8Packed => {
1583                                    // Convert both packed arguments to signed integers so that we can apply the
1584                                    // `BitFieldSExtract` operation on them in `write_dot_product` below.
1585                                    let new_arg0_id = self.gen_id();
1586                                    block.body.push(Instruction::unary(
1587                                        spirv::Op::Bitcast,
1588                                        result_type_id,
1589                                        new_arg0_id,
1590                                        arg0_id,
1591                                    ));
1592
1593                                    let new_arg1_id = self.gen_id();
1594                                    block.body.push(Instruction::unary(
1595                                        spirv::Op::Bitcast,
1596                                        result_type_id,
1597                                        new_arg1_id,
1598                                        arg1_id,
1599                                    ));
1600
1601                                    (spirv::Op::BitFieldSExtract, new_arg0_id, new_arg1_id)
1602                                }
1603                                _ => unreachable!(),
1604                            };
1605
1606                            let eight = self.writer.get_constant_scalar(crate::Literal::U32(8));
1607
1608                            const VEC_LENGTH: u8 = 4;
1609                            let bit_shifts: [_; VEC_LENGTH as usize] =
1610                                core::array::from_fn(|index| {
1611                                    self.writer
1612                                        .get_constant_scalar(crate::Literal::U32(index as u32 * 8))
1613                                });
1614
1615                            self.write_dot_product(
1616                                id,
1617                                result_type_id,
1618                                arg0_id,
1619                                arg1_id,
1620                                VEC_LENGTH as Word,
1621                                block,
1622                                |result_id, composite_id, index| {
1623                                    Instruction::ternary(
1624                                        extract_op,
1625                                        result_type_id,
1626                                        result_id,
1627                                        composite_id,
1628                                        bit_shifts[index as usize],
1629                                        eight,
1630                                    )
1631                                },
1632                            );
1633                        }
1634
1635                        self.cached[expr_handle] = id;
1636                        return Ok(());
1637                    }
1638                    Mf::Outer => MathOp::Custom(Instruction::binary(
1639                        spirv::Op::OuterProduct,
1640                        result_type_id,
1641                        id,
1642                        arg0_id,
1643                        arg1_id,
1644                    )),
1645                    Mf::Cross => MathOp::Ext(spirv::GlslStd450Op::Cross),
1646                    Mf::Distance => MathOp::Ext(spirv::GlslStd450Op::Distance),
1647                    Mf::Length => MathOp::Ext(spirv::GlslStd450Op::Length),
1648                    Mf::Normalize => MathOp::Ext(spirv::GlslStd450Op::Normalize),
1649                    Mf::FaceForward => MathOp::Ext(spirv::GlslStd450Op::FaceForward),
1650                    Mf::Reflect => MathOp::Ext(spirv::GlslStd450Op::Reflect),
1651                    Mf::Refract => MathOp::Ext(spirv::GlslStd450Op::Refract),
1652                    // exponent
1653                    Mf::Exp => MathOp::Ext(spirv::GlslStd450Op::Exp),
1654                    Mf::Exp2 => MathOp::Ext(spirv::GlslStd450Op::Exp2),
1655                    Mf::Log => MathOp::Ext(spirv::GlslStd450Op::Log),
1656                    Mf::Log2 => MathOp::Ext(spirv::GlslStd450Op::Log2),
1657                    Mf::Pow => MathOp::Ext(spirv::GlslStd450Op::Pow),
1658                    // computational
1659                    Mf::Sign => MathOp::Ext(match arg_scalar_kind {
1660                        Some(crate::ScalarKind::Float) => spirv::GlslStd450Op::FSign,
1661                        Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::SSign,
1662                        other => unimplemented!("Unexpected sign({:?})", other),
1663                    }),
1664                    Mf::Fma => MathOp::Ext(spirv::GlslStd450Op::Fma),
1665                    Mf::Mix => {
1666                        let selector = arg2.unwrap();
1667                        let selector_ty =
1668                            self.fun_info[selector].ty.inner_with(&self.ir_module.types);
1669                        match (arg_ty, selector_ty) {
1670                            // if the selector is a scalar, we need to splat it
1671                            (
1672                                &crate::TypeInner::Vector { size, .. },
1673                                &crate::TypeInner::Scalar(scalar),
1674                            ) => {
1675                                let selector_type_id =
1676                                    self.get_numeric_type_id(NumericType::Vector { size, scalar });
1677                                self.temp_list.clear();
1678                                self.temp_list.resize(size as usize, arg2_id);
1679
1680                                let selector_id = self.gen_id();
1681                                block.body.push(Instruction::composite_construct(
1682                                    selector_type_id,
1683                                    selector_id,
1684                                    &self.temp_list,
1685                                ));
1686
1687                                MathOp::Custom(Instruction::ext_inst_gl_op(
1688                                    self.writer.gl450_ext_inst_id,
1689                                    spirv::GlslStd450Op::FMix,
1690                                    result_type_id,
1691                                    id,
1692                                    &[arg0_id, arg1_id, selector_id],
1693                                ))
1694                            }
1695                            _ => MathOp::Ext(spirv::GlslStd450Op::FMix),
1696                        }
1697                    }
1698                    Mf::Step => MathOp::Ext(spirv::GlslStd450Op::Step),
1699                    Mf::SmoothStep => MathOp::Ext(spirv::GlslStd450Op::SmoothStep),
1700                    Mf::Sqrt => MathOp::Ext(spirv::GlslStd450Op::Sqrt),
1701                    Mf::InverseSqrt => MathOp::Ext(spirv::GlslStd450Op::InverseSqrt),
1702                    Mf::Inverse => MathOp::Ext(spirv::GlslStd450Op::MatrixInverse),
1703                    Mf::Transpose => MathOp::Custom(Instruction::unary(
1704                        spirv::Op::Transpose,
1705                        result_type_id,
1706                        id,
1707                        arg0_id,
1708                    )),
1709                    Mf::Determinant => MathOp::Ext(spirv::GlslStd450Op::Determinant),
1710                    Mf::QuantizeToF16 => MathOp::Custom(Instruction::unary(
1711                        spirv::Op::QuantizeToF16,
1712                        result_type_id,
1713                        id,
1714                        arg0_id,
1715                    )),
1716                    Mf::ReverseBits => MathOp::Custom(Instruction::unary(
1717                        spirv::Op::BitReverse,
1718                        result_type_id,
1719                        id,
1720                        arg0_id,
1721                    )),
1722                    Mf::CountTrailingZeros => {
1723                        let uint_id = match *arg_ty {
1724                            crate::TypeInner::Vector { size, scalar } => {
1725                                let ty =
1726                                    LocalType::Numeric(NumericType::Vector { size, scalar }).into();
1727
1728                                self.temp_list.clear();
1729                                self.temp_list.resize(
1730                                    size as _,
1731                                    self.writer
1732                                        .get_constant_scalar_with(scalar.width * 8, scalar)?,
1733                                );
1734
1735                                self.writer.get_constant_composite(ty, &self.temp_list)
1736                            }
1737                            crate::TypeInner::Scalar(scalar) => self
1738                                .writer
1739                                .get_constant_scalar_with(scalar.width * 8, scalar)?,
1740                            _ => unreachable!(),
1741                        };
1742
1743                        let lsb_id = self.gen_id();
1744                        block.body.push(Instruction::ext_inst_gl_op(
1745                            self.writer.gl450_ext_inst_id,
1746                            spirv::GlslStd450Op::FindILsb,
1747                            result_type_id,
1748                            lsb_id,
1749                            &[arg0_id],
1750                        ));
1751
1752                        MathOp::Custom(Instruction::ext_inst_gl_op(
1753                            self.writer.gl450_ext_inst_id,
1754                            spirv::GlslStd450Op::UMin,
1755                            result_type_id,
1756                            id,
1757                            &[uint_id, lsb_id],
1758                        ))
1759                    }
1760                    Mf::CountLeadingZeros => {
1761                        let (int_type_id, int_id, width) = match *arg_ty {
1762                            crate::TypeInner::Vector { size, scalar } => {
1763                                let ty =
1764                                    LocalType::Numeric(NumericType::Vector { size, scalar }).into();
1765
1766                                self.temp_list.clear();
1767                                self.temp_list.resize(
1768                                    size as _,
1769                                    self.writer
1770                                        .get_constant_scalar_with(scalar.width * 8 - 1, scalar)?,
1771                                );
1772
1773                                (
1774                                    self.get_type_id(ty),
1775                                    self.writer.get_constant_composite(ty, &self.temp_list),
1776                                    scalar.width,
1777                                )
1778                            }
1779                            crate::TypeInner::Scalar(scalar) => (
1780                                self.get_numeric_type_id(NumericType::Scalar(scalar)),
1781                                self.writer
1782                                    .get_constant_scalar_with(scalar.width * 8 - 1, scalar)?,
1783                                scalar.width,
1784                            ),
1785                            _ => unreachable!(),
1786                        };
1787
1788                        if width != 4 {
1789                            unreachable!("This is validated out until a polyfill is implemented. https://github.com/gfx-rs/wgpu/issues/5276");
1790                        };
1791
1792                        let msb_id = self.gen_id();
1793                        block.body.push(Instruction::ext_inst_gl_op(
1794                            self.writer.gl450_ext_inst_id,
1795                            if width != 4 {
1796                                spirv::GlslStd450Op::FindILsb
1797                            } else {
1798                                spirv::GlslStd450Op::FindUMsb
1799                            },
1800                            int_type_id,
1801                            msb_id,
1802                            &[arg0_id],
1803                        ));
1804
1805                        MathOp::Custom(Instruction::binary(
1806                            spirv::Op::ISub,
1807                            result_type_id,
1808                            id,
1809                            int_id,
1810                            msb_id,
1811                        ))
1812                    }
1813                    Mf::CountOneBits => MathOp::Custom(Instruction::unary(
1814                        spirv::Op::BitCount,
1815                        result_type_id,
1816                        id,
1817                        arg0_id,
1818                    )),
1819                    Mf::ExtractBits => {
1820                        let op = match arg_scalar_kind {
1821                            Some(crate::ScalarKind::Uint) => spirv::Op::BitFieldUExtract,
1822                            Some(crate::ScalarKind::Sint) => spirv::Op::BitFieldSExtract,
1823                            other => unimplemented!("Unexpected sign({:?})", other),
1824                        };
1825
1826                        // The behavior of ExtractBits is undefined when offset + count > bit_width. We need
1827                        // to first sanitize the offset and count first. If we don't do this, AMD and Intel
1828                        // will return out-of-spec values if the extracted range is not within the bit width.
1829                        //
1830                        // This encodes the exact formula specified by the wgsl spec:
1831                        // https://gpuweb.github.io/gpuweb/wgsl/#extractBits-unsigned-builtin
1832                        //
1833                        // w = sizeof(x) * 8
1834                        // o = min(offset, w)
1835                        // tmp = w - o
1836                        // c = min(count, tmp)
1837                        //
1838                        // bitfieldExtract(x, o, c)
1839
1840                        let bit_width = arg_ty.scalar_width().unwrap() * 8;
1841                        let width_constant = self
1842                            .writer
1843                            .get_constant_scalar(crate::Literal::U32(bit_width as u32));
1844
1845                        let u32_type =
1846                            self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32));
1847
1848                        // o = min(offset, w)
1849                        let offset_id = self.gen_id();
1850                        block.body.push(Instruction::ext_inst_gl_op(
1851                            self.writer.gl450_ext_inst_id,
1852                            spirv::GlslStd450Op::UMin,
1853                            u32_type,
1854                            offset_id,
1855                            &[arg1_id, width_constant],
1856                        ));
1857
1858                        // tmp = w - o
1859                        let max_count_id = self.gen_id();
1860                        block.body.push(Instruction::binary(
1861                            spirv::Op::ISub,
1862                            u32_type,
1863                            max_count_id,
1864                            width_constant,
1865                            offset_id,
1866                        ));
1867
1868                        // c = min(count, tmp)
1869                        let count_id = self.gen_id();
1870                        block.body.push(Instruction::ext_inst_gl_op(
1871                            self.writer.gl450_ext_inst_id,
1872                            spirv::GlslStd450Op::UMin,
1873                            u32_type,
1874                            count_id,
1875                            &[arg2_id, max_count_id],
1876                        ));
1877
1878                        MathOp::Custom(Instruction::ternary(
1879                            op,
1880                            result_type_id,
1881                            id,
1882                            arg0_id,
1883                            offset_id,
1884                            count_id,
1885                        ))
1886                    }
1887                    Mf::InsertBits => {
1888                        // The behavior of InsertBits has the same undefined behavior as ExtractBits.
1889
1890                        let bit_width = arg_ty.scalar_width().unwrap() * 8;
1891                        let width_constant = self
1892                            .writer
1893                            .get_constant_scalar(crate::Literal::U32(bit_width as u32));
1894
1895                        let u32_type =
1896                            self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32));
1897
1898                        // o = min(offset, w)
1899                        let offset_id = self.gen_id();
1900                        block.body.push(Instruction::ext_inst_gl_op(
1901                            self.writer.gl450_ext_inst_id,
1902                            spirv::GlslStd450Op::UMin,
1903                            u32_type,
1904                            offset_id,
1905                            &[arg2_id, width_constant],
1906                        ));
1907
1908                        // tmp = w - o
1909                        let max_count_id = self.gen_id();
1910                        block.body.push(Instruction::binary(
1911                            spirv::Op::ISub,
1912                            u32_type,
1913                            max_count_id,
1914                            width_constant,
1915                            offset_id,
1916                        ));
1917
1918                        // c = min(count, tmp)
1919                        let count_id = self.gen_id();
1920                        block.body.push(Instruction::ext_inst_gl_op(
1921                            self.writer.gl450_ext_inst_id,
1922                            spirv::GlslStd450Op::UMin,
1923                            u32_type,
1924                            count_id,
1925                            &[arg3_id, max_count_id],
1926                        ));
1927
1928                        MathOp::Custom(Instruction::quaternary(
1929                            spirv::Op::BitFieldInsert,
1930                            result_type_id,
1931                            id,
1932                            arg0_id,
1933                            arg1_id,
1934                            offset_id,
1935                            count_id,
1936                        ))
1937                    }
1938                    Mf::FirstTrailingBit => MathOp::Ext(spirv::GlslStd450Op::FindILsb),
1939                    Mf::FirstLeadingBit => {
1940                        if arg_ty.scalar_width() == Some(4) {
1941                            let thing = match arg_scalar_kind {
1942                                Some(crate::ScalarKind::Uint) => spirv::GlslStd450Op::FindUMsb,
1943                                Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::FindSMsb,
1944                                other => unimplemented!("Unexpected firstLeadingBit({:?})", other),
1945                            };
1946                            MathOp::Ext(thing)
1947                        } else {
1948                            unreachable!("This is validated out until a polyfill is implemented. https://github.com/gfx-rs/wgpu/issues/5276");
1949                        }
1950                    }
1951                    Mf::Pack4x8unorm => MathOp::Ext(spirv::GlslStd450Op::PackUnorm4x8),
1952                    Mf::Pack4x8snorm => MathOp::Ext(spirv::GlslStd450Op::PackSnorm4x8),
1953                    Mf::Pack2x16float => MathOp::Ext(spirv::GlslStd450Op::PackHalf2x16),
1954                    Mf::Pack2x16unorm => MathOp::Ext(spirv::GlslStd450Op::PackUnorm2x16),
1955                    Mf::Pack2x16snorm => MathOp::Ext(spirv::GlslStd450Op::PackSnorm2x16),
1956                    fun @ (Mf::Pack4xI8 | Mf::Pack4xU8 | Mf::Pack4xI8Clamp | Mf::Pack4xU8Clamp) => {
1957                        let is_signed = matches!(fun, Mf::Pack4xI8 | Mf::Pack4xI8Clamp);
1958                        let should_clamp = matches!(fun, Mf::Pack4xI8Clamp | Mf::Pack4xU8Clamp);
1959
1960                        let last_instruction =
1961                            if self.writer.require_all(&[spirv::Capability::Int8]).is_ok() {
1962                                self.write_pack4x8_optimized(
1963                                    block,
1964                                    result_type_id,
1965                                    arg0_id,
1966                                    id,
1967                                    is_signed,
1968                                    should_clamp,
1969                                )
1970                            } else {
1971                                self.write_pack4x8_polyfill(
1972                                    block,
1973                                    result_type_id,
1974                                    arg0_id,
1975                                    id,
1976                                    is_signed,
1977                                    should_clamp,
1978                                )
1979                            };
1980
1981                        MathOp::Custom(last_instruction)
1982                    }
1983                    Mf::Unpack4x8unorm => MathOp::Ext(spirv::GlslStd450Op::UnpackUnorm4x8),
1984                    Mf::Unpack4x8snorm => MathOp::Ext(spirv::GlslStd450Op::UnpackSnorm4x8),
1985                    Mf::Unpack2x16float => MathOp::Ext(spirv::GlslStd450Op::UnpackHalf2x16),
1986                    Mf::Unpack2x16unorm => MathOp::Ext(spirv::GlslStd450Op::UnpackUnorm2x16),
1987                    Mf::Unpack2x16snorm => MathOp::Ext(spirv::GlslStd450Op::UnpackSnorm2x16),
1988                    fun @ (Mf::Unpack4xI8 | Mf::Unpack4xU8) => {
1989                        let is_signed = matches!(fun, Mf::Unpack4xI8);
1990
1991                        let last_instruction =
1992                            if self.writer.require_all(&[spirv::Capability::Int8]).is_ok() {
1993                                self.write_unpack4x8_optimized(
1994                                    block,
1995                                    result_type_id,
1996                                    arg0_id,
1997                                    id,
1998                                    is_signed,
1999                                )
2000                            } else {
2001                                self.write_unpack4x8_polyfill(
2002                                    block,
2003                                    result_type_id,
2004                                    arg0_id,
2005                                    id,
2006                                    is_signed,
2007                                )
2008                            };
2009
2010                        MathOp::Custom(last_instruction)
2011                    }
2012                };
2013
2014                block.body.push(match math_op {
2015                    MathOp::Ext(op) => Instruction::ext_inst_gl_op(
2016                        self.writer.gl450_ext_inst_id,
2017                        op,
2018                        result_type_id,
2019                        id,
2020                        &[arg0_id, arg1_id, arg2_id, arg3_id][..fun.argument_count()],
2021                    ),
2022                    MathOp::Custom(inst) => inst,
2023                });
2024                id
2025            }
2026            crate::Expression::LocalVariable(variable) => {
2027                if let Some(rq_tracker) = self
2028                    .function
2029                    .ray_query_initialization_tracker_variables
2030                    .get(&variable)
2031                {
2032                    self.ray_query_tracker_expr.insert(
2033                        expr_handle,
2034                        super::RayQueryTrackers {
2035                            initialized_tracker: rq_tracker.id,
2036                            t_max_tracker: self
2037                                .function
2038                                .ray_query_t_max_tracker_variables
2039                                .get(&variable)
2040                                .expect("Both trackers are set at the same time.")
2041                                .id,
2042                        },
2043                    );
2044                }
2045                self.function.variables[&variable].id
2046            }
2047            crate::Expression::Load { pointer } => {
2048                self.write_checked_load(pointer, block, AccessTypeAdjustment::None, result_type_id)?
2049            }
2050            crate::Expression::FunctionArgument(index) => self.function.parameter_id(index),
2051            crate::Expression::CallResult(_)
2052            | crate::Expression::AtomicResult { .. }
2053            | crate::Expression::WorkGroupUniformLoadResult { .. }
2054            | crate::Expression::RayQueryProceedResult
2055            | crate::Expression::SubgroupBallotResult
2056            | crate::Expression::SubgroupOperationResult { .. } => self.cached[expr_handle],
2057            crate::Expression::As { expr, target } => {
2058                self.write_as_expression(expr, target, block, result_type_id)?
2059            }
2060            crate::Expression::Bitcast { expr, ty } => {
2061                self.write_bitcast_expression(expr, ty, block, result_type_id)
2062            }
2063            crate::Expression::ImageLoad {
2064                image,
2065                coordinate,
2066                array_index,
2067                sample,
2068                level,
2069            } => self.write_image_load(
2070                result_type_id,
2071                image,
2072                coordinate,
2073                array_index,
2074                level,
2075                sample,
2076                block,
2077            )?,
2078            crate::Expression::ImageSample {
2079                image,
2080                sampler,
2081                gather,
2082                coordinate,
2083                array_index,
2084                offset,
2085                level,
2086                depth_ref,
2087                clamp_to_edge,
2088            } => self.write_image_sample(
2089                result_type_id,
2090                image,
2091                sampler,
2092                gather,
2093                coordinate,
2094                array_index,
2095                offset,
2096                level,
2097                depth_ref,
2098                clamp_to_edge,
2099                block,
2100            )?,
2101            crate::Expression::Select {
2102                condition,
2103                accept,
2104                reject,
2105            } => {
2106                let id = self.gen_id();
2107                let mut condition_id = self.cached[condition];
2108                let accept_id = self.cached[accept];
2109                let reject_id = self.cached[reject];
2110
2111                let condition_ty = self.fun_info[condition]
2112                    .ty
2113                    .inner_with(&self.ir_module.types);
2114                let object_ty = self.fun_info[accept].ty.inner_with(&self.ir_module.types);
2115
2116                if let (
2117                    &crate::TypeInner::Scalar(
2118                        condition_scalar @ crate::Scalar {
2119                            kind: crate::ScalarKind::Bool,
2120                            ..
2121                        },
2122                    ),
2123                    &crate::TypeInner::Vector { size, .. },
2124                ) = (condition_ty, object_ty)
2125                {
2126                    self.temp_list.clear();
2127                    self.temp_list.resize(size as usize, condition_id);
2128
2129                    let bool_vector_type_id = self.get_numeric_type_id(NumericType::Vector {
2130                        size,
2131                        scalar: condition_scalar,
2132                    });
2133
2134                    let id = self.gen_id();
2135                    block.body.push(Instruction::composite_construct(
2136                        bool_vector_type_id,
2137                        id,
2138                        &self.temp_list,
2139                    ));
2140                    condition_id = id
2141                }
2142
2143                let instruction =
2144                    Instruction::select(result_type_id, id, condition_id, accept_id, reject_id);
2145                block.body.push(instruction);
2146                id
2147            }
2148            crate::Expression::Derivative { axis, ctrl, expr } => {
2149                use crate::{DerivativeAxis as Axis, DerivativeControl as Ctrl};
2150                match ctrl {
2151                    Ctrl::Coarse | Ctrl::Fine => {
2152                        self.writer.require_any(
2153                            "DerivativeControl",
2154                            &[spirv::Capability::DerivativeControl],
2155                        )?;
2156                    }
2157                    Ctrl::None => {}
2158                }
2159                let id = self.gen_id();
2160                let expr_id = self.cached[expr];
2161                let op = match (axis, ctrl) {
2162                    (Axis::X, Ctrl::Coarse) => spirv::Op::DPdxCoarse,
2163                    (Axis::X, Ctrl::Fine) => spirv::Op::DPdxFine,
2164                    (Axis::X, Ctrl::None) => spirv::Op::DPdx,
2165                    (Axis::Y, Ctrl::Coarse) => spirv::Op::DPdyCoarse,
2166                    (Axis::Y, Ctrl::Fine) => spirv::Op::DPdyFine,
2167                    (Axis::Y, Ctrl::None) => spirv::Op::DPdy,
2168                    (Axis::Width, Ctrl::Coarse) => spirv::Op::FwidthCoarse,
2169                    (Axis::Width, Ctrl::Fine) => spirv::Op::FwidthFine,
2170                    (Axis::Width, Ctrl::None) => spirv::Op::Fwidth,
2171                };
2172                block
2173                    .body
2174                    .push(Instruction::derivative(op, result_type_id, id, expr_id));
2175                id
2176            }
2177            crate::Expression::ImageQuery { image, query } => {
2178                self.write_image_query(result_type_id, image, query, block)?
2179            }
2180            crate::Expression::Relational { fun, argument } => {
2181                use crate::RelationalFunction as Rf;
2182                let arg_id = self.cached[argument];
2183                let op = match fun {
2184                    Rf::All => spirv::Op::All,
2185                    Rf::Any => spirv::Op::Any,
2186                    Rf::IsNan => spirv::Op::IsNan,
2187                    Rf::IsInf => spirv::Op::IsInf,
2188                };
2189                let id = self.gen_id();
2190                block
2191                    .body
2192                    .push(Instruction::relational(op, result_type_id, id, arg_id));
2193                id
2194            }
2195            crate::Expression::ArrayLength(expr) => self.write_runtime_array_length(expr, block)?,
2196            crate::Expression::RayQueryGetIntersection { query, committed } => {
2197                let query_id = self.cached[query];
2198                let init_tracker_id = *self
2199                    .ray_query_tracker_expr
2200                    .get(&query)
2201                    .expect("not a cached ray query");
2202                let func_id = self
2203                    .writer
2204                    .write_ray_query_get_intersection_function(committed, self.ir_module);
2205                let ray_intersection = self.ir_module.special_types.ray_intersection.unwrap();
2206                let intersection_type_id = self.get_handle_type_id(ray_intersection);
2207                let id = self.gen_id();
2208                block.body.push(Instruction::function_call(
2209                    intersection_type_id,
2210                    id,
2211                    func_id,
2212                    &[query_id, init_tracker_id.initialized_tracker],
2213                ));
2214                id
2215            }
2216            crate::Expression::HitObjectQuery { hit_object, query } => {
2217                self.write_hit_object_query(hit_object, query, block)?
2218            }
2219            crate::Expression::RayQueryVertexPositions { query, committed } => {
2220                self.writer.require_any(
2221                    "RayQueryVertexPositions",
2222                    &[spirv::Capability::RayQueryPositionFetchKHR],
2223                )?;
2224                self.write_ray_query_return_vertex_position(query, block, committed)
2225            }
2226            crate::Expression::CooperativeLoad { ref data, .. } => {
2227                self.writer.require_any(
2228                    "CooperativeMatrix",
2229                    &[spirv::Capability::CooperativeMatrixKHR],
2230                )?;
2231                let layout = if data.row_major {
2232                    spirv::CooperativeMatrixLayout::RowMajorKHR
2233                } else {
2234                    spirv::CooperativeMatrixLayout::ColumnMajorKHR
2235                };
2236                let layout_id = self.get_index_constant(layout as u32);
2237                let stride_id = self.cached[data.stride];
2238                match self.write_access_chain(data.pointer, block, AccessTypeAdjustment::None)? {
2239                    ExpressionPointer::Ready { pointer_id } => {
2240                        let id = self.gen_id();
2241                        block.body.push(Instruction::coop_load(
2242                            result_type_id,
2243                            id,
2244                            pointer_id,
2245                            layout_id,
2246                            stride_id,
2247                        ));
2248                        id
2249                    }
2250                    ExpressionPointer::Conditional { condition, access } => self
2251                        .write_conditional_indexed_load(
2252                            result_type_id,
2253                            condition,
2254                            block,
2255                            |id_gen, block| {
2256                                let pointer_id = access.result_id.unwrap();
2257                                block.body.push(access);
2258                                let id = id_gen.next();
2259                                block.body.push(Instruction::coop_load(
2260                                    result_type_id,
2261                                    id,
2262                                    pointer_id,
2263                                    layout_id,
2264                                    stride_id,
2265                                ));
2266                                id
2267                            },
2268                        ),
2269                }
2270            }
2271            crate::Expression::CooperativeMultiplyAdd { a, b, c } => {
2272                self.writer.require_any(
2273                    "CooperativeMatrix",
2274                    &[spirv::Capability::CooperativeMatrixKHR],
2275                )?;
2276                let a_id = self.cached[a];
2277                let b_id = self.cached[b];
2278                let c_id = self.cached[c];
2279                let id = self.gen_id();
2280                block.body.push(Instruction::coop_mul_add(
2281                    result_type_id,
2282                    id,
2283                    a_id,
2284                    b_id,
2285                    c_id,
2286                ));
2287                id
2288            }
2289        };
2290
2291        self.cached[expr_handle] = id;
2292        Ok(())
2293    }
2294
2295    /// Helper which generates [`Bitcast`] expressions.
2296    ///
2297    /// [`Bitcast`]: crate::Expression::Bitcast
2298    fn write_bitcast_expression(
2299        &mut self,
2300        expr: Handle<crate::Expression>,
2301        ty: Handle<crate::Type>,
2302        block: &mut Block,
2303        result_type_id: u32,
2304    ) -> Word {
2305        let expr_id = self.cached[expr];
2306
2307        // Filter out identity casts. Some Adreno drivers are
2308        // confused by no-op OpBitCast instructions.
2309        let source_inner = self.fun_info[expr].ty.inner_with(&self.ir_module.types);
2310        if *source_inner == self.ir_module.types[ty].inner {
2311            return expr_id;
2312        }
2313
2314        let id = self.gen_id();
2315        block.body.push(Instruction::unary(
2316            spirv::Op::Bitcast,
2317            result_type_id,
2318            id,
2319            expr_id,
2320        ));
2321        id
2322    }
2323
2324    /// Helper which focuses on generating the `As` expressions and the various conversions
2325    /// that need to happen because of that.
2326    fn write_as_expression(
2327        &mut self,
2328        expr: Handle<crate::Expression>,
2329        target: crate::Scalar,
2330        block: &mut Block,
2331        result_type_id: u32,
2332    ) -> Result<u32, Error> {
2333        use crate::ScalarKind as Sk;
2334        let crate::Scalar {
2335            kind,
2336            width: dst_width,
2337        } = target;
2338        let expr_id = self.cached[expr];
2339        let ty = self.fun_info[expr].ty.inner_with(&self.ir_module.types);
2340
2341        // Matrix casts needs special treatment in SPIR-V, as the cast functions
2342        // can take vectors or scalars, but not matrices. In order to cast a matrix
2343        // we need to cast each column of the matrix individually and construct a new
2344        // matrix from the converted columns.
2345        if let crate::TypeInner::Matrix {
2346            columns,
2347            rows,
2348            scalar,
2349        } = *ty
2350        {
2351            if dst_width == scalar.width {
2352                // No conversion needs to be done, passes through.
2353                return Ok(expr_id);
2354            }
2355
2356            if kind != Sk::Float {
2357                // Only float conversions are supported for matrices.
2358                return Err(Error::Validation("Matrices must be floats"));
2359            }
2360
2361            // Type of each extracted column
2362            let column_src_ty =
2363                self.get_type_id(LookupType::Local(LocalType::Numeric(NumericType::Vector {
2364                    size: rows,
2365                    scalar,
2366                })));
2367
2368            // Type of the column after conversion
2369            let column_dst_ty =
2370                self.get_type_id(LookupType::Local(LocalType::Numeric(NumericType::Vector {
2371                    size: rows,
2372                    scalar: crate::Scalar {
2373                        kind,
2374                        width: dst_width,
2375                    },
2376                })));
2377
2378            let mut components = ArrayVec::<Word, 4>::new();
2379
2380            for column in 0..columns as usize {
2381                let column_id = self.gen_id();
2382                block.body.push(Instruction::composite_extract(
2383                    column_src_ty,
2384                    column_id,
2385                    expr_id,
2386                    &[column as u32],
2387                ));
2388
2389                let column_conv_id = self.gen_id();
2390                block.body.push(Instruction::unary(
2391                    spirv::Op::FConvert,
2392                    column_dst_ty,
2393                    column_conv_id,
2394                    column_id,
2395                ));
2396
2397                components.push(column_conv_id);
2398            }
2399
2400            let construct_id = self.gen_id();
2401
2402            block.body.push(Instruction::composite_construct(
2403                result_type_id,
2404                construct_id,
2405                &components,
2406            ));
2407
2408            return Ok(construct_id);
2409        }
2410
2411        let (src_scalar, src_size) = match *ty {
2412            crate::TypeInner::Scalar(scalar) => (scalar, None),
2413            crate::TypeInner::Vector { scalar, size } => (scalar, Some(size)),
2414            ref other => {
2415                log::error!("As source {other:?}");
2416                return Err(Error::Validation("Unexpected Expression::As source"));
2417            }
2418        };
2419
2420        enum Cast {
2421            Identity(Word),
2422            Unary(spirv::Op, Word),
2423            Binary(spirv::Op, Word, Word),
2424            Ternary(spirv::Op, Word, Word, Word),
2425        }
2426        let cast = match (src_scalar.kind, kind, dst_width) {
2427            // Filter out identity casts. Some Adreno drivers are
2428            // confused by no-op OpBitCast instructions.
2429            (src_kind, kind, dst_width) if src_kind == kind && dst_width == src_scalar.width => {
2430                Cast::Identity(expr_id)
2431            }
2432            (Sk::Bool, Sk::Bool, _) => Cast::Unary(spirv::Op::CopyObject, expr_id),
2433            // casting to a bool - generate `OpXxxNotEqual`
2434            (_, Sk::Bool, _) => {
2435                let op = match src_scalar.kind {
2436                    Sk::Sint | Sk::Uint => spirv::Op::INotEqual,
2437                    Sk::Float => spirv::Op::FUnordNotEqual,
2438                    Sk::Bool | Sk::AbstractInt | Sk::AbstractFloat => unreachable!(),
2439                };
2440                let zero_scalar_id = self.writer.get_constant_scalar_with(0, src_scalar)?;
2441                let zero_id = match src_size {
2442                    Some(size) => {
2443                        let ty = LocalType::Numeric(NumericType::Vector {
2444                            size,
2445                            scalar: src_scalar,
2446                        })
2447                        .into();
2448
2449                        self.temp_list.clear();
2450                        self.temp_list.resize(size as _, zero_scalar_id);
2451
2452                        self.writer.get_constant_composite(ty, &self.temp_list)
2453                    }
2454                    None => zero_scalar_id,
2455                };
2456
2457                Cast::Binary(op, expr_id, zero_id)
2458            }
2459            // casting from a bool - generate `OpSelect`
2460            (Sk::Bool, _, dst_width) => {
2461                let dst_scalar = crate::Scalar {
2462                    kind,
2463                    width: dst_width,
2464                };
2465                let zero_scalar_id = self.writer.get_constant_scalar_with(0, dst_scalar)?;
2466                let one_scalar_id = self.writer.get_constant_scalar_with(1, dst_scalar)?;
2467                let (accept_id, reject_id) = match src_size {
2468                    Some(size) => {
2469                        let ty = LocalType::Numeric(NumericType::Vector {
2470                            size,
2471                            scalar: dst_scalar,
2472                        })
2473                        .into();
2474
2475                        self.temp_list.clear();
2476                        self.temp_list.resize(size as _, zero_scalar_id);
2477
2478                        let vec0_id = self.writer.get_constant_composite(ty, &self.temp_list);
2479
2480                        self.temp_list.fill(one_scalar_id);
2481
2482                        let vec1_id = self.writer.get_constant_composite(ty, &self.temp_list);
2483
2484                        (vec1_id, vec0_id)
2485                    }
2486                    None => (one_scalar_id, zero_scalar_id),
2487                };
2488
2489                Cast::Ternary(spirv::Op::Select, expr_id, accept_id, reject_id)
2490            }
2491            // Avoid undefined behaviour when casting from a float to integer
2492            // when the value is out of range for the target type. Additionally
2493            // ensure we clamp to the correct value as per the WGSL spec.
2494            //
2495            // https://www.w3.org/TR/WGSL/#floating-point-conversion:
2496            // * If X is exactly representable in the target type T, then the
2497            //   result is that value.
2498            // * Otherwise, the result is the value in T closest to
2499            //   truncate(X) and also exactly representable in the original
2500            //   floating point type.
2501            (Sk::Float, Sk::Sint | Sk::Uint, width) => {
2502                let dst_scalar = crate::Scalar { kind, width };
2503                let (min, max) =
2504                    crate::proc::min_max_float_representable_by(src_scalar, dst_scalar);
2505                let expr_type_id = self.get_expression_type_id(&self.fun_info[expr].ty);
2506
2507                let maybe_splat_const = |writer: &mut Writer, const_id| match src_size {
2508                    None => const_id,
2509                    Some(size) => {
2510                        let constituent_ids = [const_id; crate::VectorSize::MAX];
2511                        writer.get_constant_composite(
2512                            LookupType::Local(LocalType::Numeric(NumericType::Vector {
2513                                size,
2514                                scalar: src_scalar,
2515                            })),
2516                            &constituent_ids[..size as usize],
2517                        )
2518                    }
2519                };
2520                let min_const_id = self.writer.get_constant_scalar(min);
2521                let min_const_id = maybe_splat_const(self.writer, min_const_id);
2522                let max_const_id = self.writer.get_constant_scalar(max);
2523                let max_const_id = maybe_splat_const(self.writer, max_const_id);
2524
2525                let clamp_id = self.gen_id();
2526                block.body.push(Instruction::ext_inst_gl_op(
2527                    self.writer.gl450_ext_inst_id,
2528                    spirv::GlslStd450Op::FClamp,
2529                    expr_type_id,
2530                    clamp_id,
2531                    &[expr_id, min_const_id, max_const_id],
2532                ));
2533
2534                let op = match dst_scalar.kind {
2535                    crate::ScalarKind::Sint => spirv::Op::ConvertFToS,
2536                    crate::ScalarKind::Uint => spirv::Op::ConvertFToU,
2537                    _ => unreachable!(),
2538                };
2539                Cast::Unary(op, clamp_id)
2540            }
2541            (Sk::Float, Sk::Float, dst_width) if src_scalar.width != dst_width => {
2542                Cast::Unary(spirv::Op::FConvert, expr_id)
2543            }
2544            (Sk::Sint, Sk::Float, _) => Cast::Unary(spirv::Op::ConvertSToF, expr_id),
2545            (Sk::Sint, Sk::Sint, dst_width) if src_scalar.width != dst_width => {
2546                Cast::Unary(spirv::Op::SConvert, expr_id)
2547            }
2548            (Sk::Uint, Sk::Float, _) => Cast::Unary(spirv::Op::ConvertUToF, expr_id),
2549            (Sk::Uint, Sk::Uint, dst_width) if src_scalar.width != dst_width => {
2550                Cast::Unary(spirv::Op::UConvert, expr_id)
2551            }
2552            (Sk::Uint, Sk::Sint, dst_width) if src_scalar.width != dst_width => {
2553                Cast::Unary(spirv::Op::SConvert, expr_id)
2554            }
2555            (Sk::Sint, Sk::Uint, dst_width) if src_scalar.width != dst_width => {
2556                Cast::Unary(spirv::Op::UConvert, expr_id)
2557            }
2558            // Converting between signed and unsigned integers of the same
2559            // width preserves the bits.
2560            _ => Cast::Unary(spirv::Op::Bitcast, expr_id),
2561        };
2562        Ok(match cast {
2563            Cast::Identity(expr) => expr,
2564            Cast::Unary(op, op1) => {
2565                let id = self.gen_id();
2566                block
2567                    .body
2568                    .push(Instruction::unary(op, result_type_id, id, op1));
2569                id
2570            }
2571            Cast::Binary(op, op1, op2) => {
2572                let id = self.gen_id();
2573                block
2574                    .body
2575                    .push(Instruction::binary(op, result_type_id, id, op1, op2));
2576                id
2577            }
2578            Cast::Ternary(op, op1, op2, op3) => {
2579                let id = self.gen_id();
2580                block
2581                    .body
2582                    .push(Instruction::ternary(op, result_type_id, id, op1, op2, op3));
2583                id
2584            }
2585        })
2586    }
2587
2588    /// Build an `OpAccessChain` instruction.
2589    ///
2590    /// Emit any needed bounds-checking expressions to `block`.
2591    ///
2592    /// Give the `OpAccessChain` a result type based on `expr_handle`, adjusted
2593    /// according to `type_adjustment`; see the documentation for
2594    /// [`AccessTypeAdjustment`] for details.
2595    ///
2596    /// On success, the return value is an [`ExpressionPointer`] value; see the
2597    /// documentation for that type.
2598    fn write_access_chain(
2599        &mut self,
2600        mut expr_handle: Handle<crate::Expression>,
2601        block: &mut Block,
2602        type_adjustment: AccessTypeAdjustment,
2603    ) -> Result<ExpressionPointer, Error> {
2604        let result_type_id = {
2605            let resolution = &self.fun_info[expr_handle].ty;
2606            match type_adjustment {
2607                AccessTypeAdjustment::None => self.writer.get_expression_type_id(resolution),
2608                AccessTypeAdjustment::IntroducePointer(class) => {
2609                    self.writer.get_resolution_pointer_id(resolution, class)
2610                }
2611                AccessTypeAdjustment::UseStd140CompatType => {
2612                    match *resolution.inner_with(&self.ir_module.types) {
2613                        crate::TypeInner::Pointer {
2614                            base,
2615                            space: space @ crate::AddressSpace::Uniform,
2616                        } => self.writer.get_pointer_type_id(
2617                            self.writer.std140_compat_uniform_types[&base].type_id,
2618                            map_storage_class(space),
2619                        ),
2620                        _ => unreachable!(
2621                            "`UseStd140CompatType` must only be used with uniform pointer types"
2622                        ),
2623                    }
2624                }
2625            }
2626        };
2627
2628        // The id of the boolean `and` of all dynamic bounds checks up to this point.
2629        //
2630        // See `extend_bounds_check_condition_chain` for a full explanation.
2631        let mut accumulated_checks = None;
2632
2633        // Is true if we are accessing into a binding array with a non-uniform index.
2634        let mut is_non_uniform_binding_array = false;
2635
2636        // The index value if the previously encountered expression was an
2637        // `AccessIndex` of a matrix which has been decomposed into individual
2638        // column vectors directly in the containing struct. The subsequent
2639        // iteration will append the correct index to the list for accessing
2640        // said column from the containing struct.
2641        let mut prev_decomposed_matrix_index = None;
2642
2643        self.temp_list.clear();
2644        let root_id = loop {
2645            // If `expr_handle` was spilled, then the temporary variable has exactly
2646            // the value we want to start from.
2647            if let Some(spilled) = self.function.spilled_composites.get(&expr_handle) {
2648                // The root id of the `OpAccessChain` instruction is the temporary
2649                // variable we spilled the composite to.
2650                break spilled.id;
2651            }
2652
2653            expr_handle = match self.ir_function.expressions[expr_handle] {
2654                crate::Expression::Access { base, index } => {
2655                    is_non_uniform_binding_array |=
2656                        self.is_nonuniform_binding_array_access(base, index);
2657
2658                    let index = GuardedIndex::Expression(index);
2659                    let index_id =
2660                        self.write_access_chain_index(base, index, &mut accumulated_checks, block)?;
2661                    self.temp_list.push(index_id);
2662
2663                    base
2664                }
2665                crate::Expression::AccessIndex { base, index } => {
2666                    // Decide whether we're indexing a struct (bounds checks
2667                    // forbidden) or anything else (bounds checks required).
2668                    let mut base_ty = self.fun_info[base].ty.inner_with(&self.ir_module.types);
2669                    let mut base_ty_handle = self.fun_info[base].ty.handle();
2670                    let mut pointer_space = None;
2671                    if let crate::TypeInner::Pointer { base, space } = *base_ty {
2672                        base_ty = &self.ir_module.types[base].inner;
2673                        base_ty_handle = Some(base);
2674                        pointer_space = Some(space);
2675                    }
2676                    match *base_ty {
2677                        // When indexing a struct bounds checks are forbidden. If accessing the
2678                        // struct through a uniform address space pointer, where the struct has
2679                        // been declared with an alternative std140 compatible layout, we must use
2680                        // the remapped member index. Additionally if the previous iteration was
2681                        // accessing a column of a matrix member which has been decomposed directly
2682                        // into the struct, we must ensure we access the correct column.
2683                        crate::TypeInner::Struct { .. } => {
2684                            let index = match base_ty_handle.and_then(|handle| {
2685                                self.writer.std140_compat_uniform_types.get(&handle)
2686                            }) {
2687                                Some(std140_type_info)
2688                                    if pointer_space == Some(crate::AddressSpace::Uniform) =>
2689                                {
2690                                    std140_type_info.member_indices[index as usize]
2691                                        + prev_decomposed_matrix_index.take().unwrap_or(0)
2692                                }
2693                                _ => index,
2694                            };
2695                            let index_id = self.get_index_constant(index);
2696                            self.temp_list.push(index_id);
2697                        }
2698                        // Bounds checks are not required when indexing a matrix. If indexing a
2699                        // two-row matrix contained within a struct through a uniform address space
2700                        // pointer then the matrix' columns will have been decomposed directly into
2701                        // the containing struct. We skip adding an index to the list on this
2702                        // iteration and instead adjust the index on the next iteration when
2703                        // accessing the struct member.
2704                        _ if is_uniform_matcx2_struct_member_access(
2705                            self.ir_function,
2706                            self.fun_info,
2707                            self.ir_module,
2708                            base,
2709                        ) =>
2710                        {
2711                            assert!(prev_decomposed_matrix_index.is_none());
2712                            prev_decomposed_matrix_index = Some(index);
2713                        }
2714                        _ => {
2715                            // `index` is constant, so this can't possibly require
2716                            // setting `is_nonuniform_binding_array_access`.
2717
2718                            // Even though the index value is statically known, `base`
2719                            // may be a runtime-sized array, so we still need to go
2720                            // through the bounds check process.
2721                            let index_id = self.write_access_chain_index(
2722                                base,
2723                                GuardedIndex::Known(index),
2724                                &mut accumulated_checks,
2725                                block,
2726                            )?;
2727                            self.temp_list.push(index_id);
2728                        }
2729                    }
2730                    base
2731                }
2732                crate::Expression::GlobalVariable(handle) => {
2733                    let gv = &self.writer.global_variables[handle];
2734                    break gv.access_id;
2735                }
2736                crate::Expression::LocalVariable(variable) => {
2737                    let local_var = &self.function.variables[&variable];
2738                    break local_var.id;
2739                }
2740                crate::Expression::FunctionArgument(index) => {
2741                    break self.function.parameter_id(index);
2742                }
2743                ref other => unimplemented!("Unexpected pointer expression {:?}", other),
2744            }
2745        };
2746
2747        let (pointer_id, expr_pointer) = if self.temp_list.is_empty() {
2748            (
2749                root_id,
2750                ExpressionPointer::Ready {
2751                    pointer_id: root_id,
2752                },
2753            )
2754        } else {
2755            self.temp_list.reverse();
2756            let pointer_id = self.gen_id();
2757            let access =
2758                Instruction::access_chain(result_type_id, pointer_id, root_id, &self.temp_list);
2759
2760            // If we generated some bounds checks, we need to leave it to our
2761            // caller to generate the branch, the access, the load or store, and
2762            // the zero value (for loads). Otherwise, we can emit the access
2763            // ourselves, and just hand them the id of the pointer.
2764            let expr_pointer = match accumulated_checks {
2765                Some(condition) => ExpressionPointer::Conditional { condition, access },
2766                None => {
2767                    block.body.push(access);
2768                    ExpressionPointer::Ready { pointer_id }
2769                }
2770            };
2771            (pointer_id, expr_pointer)
2772        };
2773        // Subsequent load, store and atomic operations require the pointer to be decorated as NonUniform
2774        // if the binding array was accessed with a non-uniform index
2775        // see VUID-RuntimeSpirv-NonUniform-06274
2776        if is_non_uniform_binding_array {
2777            self.writer
2778                .decorate_non_uniform_binding_array_access(pointer_id)?;
2779        }
2780
2781        Ok(expr_pointer)
2782    }
2783
2784    fn is_nonuniform_binding_array_access(
2785        &mut self,
2786        base: Handle<crate::Expression>,
2787        index: Handle<crate::Expression>,
2788    ) -> bool {
2789        let crate::Expression::GlobalVariable(var_handle) = self.ir_function.expressions[base]
2790        else {
2791            return false;
2792        };
2793
2794        // The access chain needs to be decorated as NonUniform
2795        // see VUID-RuntimeSpirv-NonUniform-06274
2796        let gvar = &self.ir_module.global_variables[var_handle];
2797        let crate::TypeInner::BindingArray { .. } = self.ir_module.types[gvar.ty].inner else {
2798            return false;
2799        };
2800
2801        self.fun_info[index].uniformity.non_uniform_result.is_some()
2802    }
2803
2804    /// Compute a single index operand to an `OpAccessChain` instruction.
2805    ///
2806    /// Given that we are indexing `base` with `index`, apply the appropriate
2807    /// bounds check policies, emitting code to `block` to clamp `index` or
2808    /// determine whether it's in bounds. Return the SPIR-V instruction id of
2809    /// the index value we should actually use.
2810    ///
2811    /// Extend `accumulated_checks` to include the results of any needed bounds
2812    /// checks. See [`BlockContext::extend_bounds_check_condition_chain`].
2813    fn write_access_chain_index(
2814        &mut self,
2815        base: Handle<crate::Expression>,
2816        index: GuardedIndex,
2817        accumulated_checks: &mut Option<Word>,
2818        block: &mut Block,
2819    ) -> Result<Word, Error> {
2820        match self.write_bounds_check(base, index, block)? {
2821            BoundsCheckResult::KnownInBounds(known_index) => {
2822                // Even if the index is known, `OpAccessChain`
2823                // requires expression operands, not literals.
2824                let scalar = crate::Literal::U32(known_index);
2825                Ok(self.writer.get_constant_scalar(scalar))
2826            }
2827            BoundsCheckResult::Computed(computed_index_id) => Ok(computed_index_id),
2828            BoundsCheckResult::Conditional {
2829                condition_id: condition,
2830                index_id: index,
2831            } => {
2832                self.extend_bounds_check_condition_chain(accumulated_checks, condition, block);
2833
2834                // Use the index from the `Access` expression unchanged.
2835                Ok(index)
2836            }
2837        }
2838    }
2839
2840    /// Add a condition to a chain of bounds checks.
2841    ///
2842    /// As we build an `OpAccessChain` instruction govered by
2843    /// [`BoundsCheckPolicy::ReadZeroSkipWrite`], we accumulate a chain of
2844    /// dynamic bounds checks, one for each index in the chain, which must all
2845    /// be true for that `OpAccessChain`'s execution to be well-defined. This
2846    /// function adds the boolean instruction id `comparison_id` to `chain`.
2847    ///
2848    /// If `chain` is `None`, that means there are no bounds checks in the chain
2849    /// yet. If chain is `Some(id)`, then `id` is the conjunction of all the
2850    /// bounds checks in the chain.
2851    ///
2852    /// When we have multiple bounds checks, we combine them with
2853    /// `OpLogicalAnd`, not a short-circuit branch. This means we might do
2854    /// comparisons we don't need to, but we expect these checks to almost
2855    /// always succeed, and keeping branches to a minimum is essential.
2856    ///
2857    /// [`BoundsCheckPolicy::ReadZeroSkipWrite`]: crate::proc::BoundsCheckPolicy
2858    fn extend_bounds_check_condition_chain(
2859        &mut self,
2860        chain: &mut Option<Word>,
2861        comparison_id: Word,
2862        block: &mut Block,
2863    ) {
2864        match *chain {
2865            Some(ref mut prior_checks) => {
2866                let combined = self.gen_id();
2867                block.body.push(Instruction::binary(
2868                    spirv::Op::LogicalAnd,
2869                    self.writer.get_bool_type_id(),
2870                    combined,
2871                    *prior_checks,
2872                    comparison_id,
2873                ));
2874                *prior_checks = combined;
2875            }
2876            None => {
2877                // Start a fresh chain of checks.
2878                *chain = Some(comparison_id);
2879            }
2880        }
2881    }
2882
2883    fn write_checked_load(
2884        &mut self,
2885        pointer: Handle<crate::Expression>,
2886        block: &mut Block,
2887        access_type_adjustment: AccessTypeAdjustment,
2888        result_type_id: Word,
2889    ) -> Result<Word, Error> {
2890        if let Some(result_id) = self.maybe_write_immediate_vector_dynamic_access(pointer, block)? {
2891            Ok(result_id)
2892        } else if let Some(result_id) =
2893            self.maybe_write_uniform_matcx2_dynamic_access(pointer, block)?
2894        {
2895            Ok(result_id)
2896        } else if let Some(result_id) =
2897            self.maybe_write_load_uniform_matcx2_struct_member(pointer, block)?
2898        {
2899            Ok(result_id)
2900        } else {
2901            // If `pointer` refers to a uniform address space pointer to a type
2902            // which was declared using a std140 compatible type variant (i.e.
2903            // is a two-row matrix, or a struct or array containing such a
2904            // matrix) we must ensure the access chain and the type of the load
2905            // instruction use the std140 compatible type variant.
2906            struct WrappedLoad {
2907                access_type_adjustment: AccessTypeAdjustment,
2908                r#type: Handle<crate::Type>,
2909            }
2910            let mut wrapped_load = None;
2911            if let crate::TypeInner::Pointer {
2912                base: pointer_base_type,
2913                space: crate::AddressSpace::Uniform,
2914            } = *self.fun_info[pointer].ty.inner_with(&self.ir_module.types)
2915            {
2916                if self
2917                    .writer
2918                    .std140_compat_uniform_types
2919                    .contains_key(&pointer_base_type)
2920                {
2921                    wrapped_load = Some(WrappedLoad {
2922                        access_type_adjustment: AccessTypeAdjustment::UseStd140CompatType,
2923                        r#type: pointer_base_type,
2924                    });
2925                };
2926            };
2927
2928            let (load_type_id, access_type_adjustment) = match wrapped_load {
2929                Some(ref wrapped_load) => (
2930                    self.writer.std140_compat_uniform_types[&wrapped_load.r#type].type_id,
2931                    wrapped_load.access_type_adjustment,
2932                ),
2933                None => (result_type_id, access_type_adjustment),
2934            };
2935
2936            let load_id = match self.write_access_chain(pointer, block, access_type_adjustment)? {
2937                ExpressionPointer::Ready { pointer_id } => {
2938                    let id = self.gen_id();
2939                    let atomic_space =
2940                        match *self.fun_info[pointer].ty.inner_with(&self.ir_module.types) {
2941                            crate::TypeInner::Pointer { base, space } => {
2942                                match self.ir_module.types[base].inner {
2943                                    crate::TypeInner::Atomic { .. } => Some(space),
2944                                    _ => None,
2945                                }
2946                            }
2947                            _ => None,
2948                        };
2949                    let instruction = if let Some(space) = atomic_space {
2950                        let (semantics, scope) = space.to_spirv_semantics_and_scope();
2951                        let scope_constant_id = self.get_scope_constant(scope as u32);
2952                        let semantics_id = self.get_index_constant(semantics.bits());
2953                        Instruction::atomic_load(
2954                            result_type_id,
2955                            id,
2956                            pointer_id,
2957                            scope_constant_id,
2958                            semantics_id,
2959                        )
2960                    } else {
2961                        Instruction::load(load_type_id, id, pointer_id, None)
2962                    };
2963                    block.body.push(instruction);
2964                    id
2965                }
2966                ExpressionPointer::Conditional { condition, access } => {
2967                    //TODO: support atomics?
2968                    self.write_conditional_indexed_load(
2969                        load_type_id,
2970                        condition,
2971                        block,
2972                        move |id_gen, block| {
2973                            // The in-bounds path. Perform the access and the load.
2974                            let pointer_id = access.result_id.unwrap();
2975                            let value_id = id_gen.next();
2976                            block.body.push(access);
2977                            block.body.push(Instruction::load(
2978                                load_type_id,
2979                                value_id,
2980                                pointer_id,
2981                                None,
2982                            ));
2983                            value_id
2984                        },
2985                    )
2986                }
2987            };
2988
2989            match wrapped_load {
2990                Some(ref wrapped_load) => {
2991                    // If we loaded a std140 compat type then we must call the
2992                    // function to convert the loaded value to the regular type.
2993                    let result_id = self.gen_id();
2994                    let function_id = self.writer.wrapped_functions
2995                        [&WrappedFunction::ConvertFromStd140CompatType {
2996                            r#type: wrapped_load.r#type,
2997                        }];
2998                    block.body.push(Instruction::function_call(
2999                        result_type_id,
3000                        result_id,
3001                        function_id,
3002                        &[load_id],
3003                    ));
3004                    Ok(result_id)
3005                }
3006                None => Ok(load_id),
3007            }
3008        }
3009    }
3010
3011    fn spill_to_internal_variable(&mut self, base: Handle<crate::Expression>, block: &mut Block) {
3012        use indexmap::map::Entry;
3013
3014        // Make sure we have an internal variable to spill `base` to.
3015        let spill_variable_id = match self.function.spilled_composites.entry(base) {
3016            Entry::Occupied(preexisting) => preexisting.get().id,
3017            Entry::Vacant(vacant) => {
3018                // Generate a new internal variable of the appropriate
3019                // type for `base`.
3020                let pointer_type_id = self.writer.get_resolution_pointer_id(
3021                    &self.fun_info[base].ty,
3022                    spirv::StorageClass::Function,
3023                );
3024                let id = self.writer.id_gen.next();
3025                vacant.insert(super::LocalVariable {
3026                    id,
3027                    instruction: Instruction::variable(
3028                        pointer_type_id,
3029                        id,
3030                        spirv::StorageClass::Function,
3031                        None,
3032                    ),
3033                });
3034                id
3035            }
3036        };
3037
3038        // Perform the store even if we already had a spill variable for `base`.
3039        // Consider this code:
3040        //
3041        // var x = ...;
3042        // var y = ...;
3043        // var z = ...;
3044        // for (i = 0; i<2; i++) {
3045        //     let a = array(i, i, i);
3046        //     if (i == 0) {
3047        //         x += a[y];
3048        //     } else [
3049        //         x += a[z];
3050        //     }
3051        // }
3052        //
3053        // The value of `a` needs to be spilled so we can subscript it with `y` and `z`.
3054        //
3055        // When we generate SPIR-V for `a[y]`, we will create the spill
3056        // variable, and store `a`'s value in it.
3057        //
3058        // When we generate SPIR-V for `a[z]`, we will notice that the spill
3059        // variable for `a` has already been declared, but it is still essential
3060        // that we store `a` into it, so that `a[z]` sees this iteration's value
3061        // of `a`.
3062        let base_id = self.cached[base];
3063        block
3064            .body
3065            .push(Instruction::store(spill_variable_id, base_id, None));
3066    }
3067
3068    /// Generate an access to a spilled temporary, if necessary.
3069    ///
3070    /// Given `access`, an [`Access`] or [`AccessIndex`] expression that refers
3071    /// to a component of a composite value that has been spilled to a temporary
3072    /// variable, determine whether other expressions are going to use
3073    /// `access`'s value:
3074    ///
3075    /// - If so, perform the access and cache that as the value of `access`.
3076    ///
3077    /// - Otherwise, generate no code and cache no value for `access`.
3078    ///
3079    /// Return `Ok(0)` if no value was fetched, or `Ok(id)` if we loaded it into
3080    /// the instruction given by `id`.
3081    ///
3082    /// [`Access`]: crate::Expression::Access
3083    /// [`AccessIndex`]: crate::Expression::AccessIndex
3084    fn maybe_access_spilled_composite(
3085        &mut self,
3086        access: Handle<crate::Expression>,
3087        block: &mut Block,
3088        result_type_id: Word,
3089    ) -> Result<Word, Error> {
3090        let access_uses = self.function.access_uses.get(&access).map_or(0, |r| *r);
3091        if access_uses == self.fun_info[access].ref_count {
3092            // This expression is only used by other `Access` and
3093            // `AccessIndex` expressions, so we don't need to cache a
3094            // value for it yet.
3095            Ok(0)
3096        } else {
3097            // There are other expressions that are going to expect this
3098            // expression's value to be cached, not just other `Access` or
3099            // `AccessIndex` expressions. We must actually perform the
3100            // access on the spill variable now.
3101            self.write_checked_load(
3102                access,
3103                block,
3104                AccessTypeAdjustment::IntroducePointer(spirv::StorageClass::Function),
3105                result_type_id,
3106            )
3107        }
3108    }
3109
3110    /// Build the instructions for matrix - matrix column operations
3111    #[allow(clippy::too_many_arguments)]
3112    fn write_matrix_matrix_column_op(
3113        &mut self,
3114        block: &mut Block,
3115        result_id: Word,
3116        result_type_id: Word,
3117        left_id: Word,
3118        right_id: Word,
3119        columns: crate::VectorSize,
3120        rows: crate::VectorSize,
3121        width: u8,
3122        op: spirv::Op,
3123    ) {
3124        self.temp_list.clear();
3125
3126        let vector_type_id = self.get_numeric_type_id(NumericType::Vector {
3127            size: rows,
3128            scalar: crate::Scalar::float(width),
3129        });
3130
3131        for index in 0..columns as u32 {
3132            let column_id_left = self.gen_id();
3133            let column_id_right = self.gen_id();
3134            let column_id_res = self.gen_id();
3135
3136            block.body.push(Instruction::composite_extract(
3137                vector_type_id,
3138                column_id_left,
3139                left_id,
3140                &[index],
3141            ));
3142            block.body.push(Instruction::composite_extract(
3143                vector_type_id,
3144                column_id_right,
3145                right_id,
3146                &[index],
3147            ));
3148            block.body.push(Instruction::binary(
3149                op,
3150                vector_type_id,
3151                column_id_res,
3152                column_id_left,
3153                column_id_right,
3154            ));
3155
3156            self.temp_list.push(column_id_res);
3157        }
3158
3159        block.body.push(Instruction::composite_construct(
3160            result_type_id,
3161            result_id,
3162            &self.temp_list,
3163        ));
3164    }
3165
3166    /// Build the instructions for vector - scalar multiplication
3167    fn write_vector_scalar_mult(
3168        &mut self,
3169        block: &mut Block,
3170        result_id: Word,
3171        result_type_id: Word,
3172        vector_id: Word,
3173        scalar_id: Word,
3174        vector: &crate::TypeInner,
3175    ) {
3176        let (size, kind) = match *vector {
3177            crate::TypeInner::Vector {
3178                size,
3179                scalar: crate::Scalar { kind, .. },
3180            } => (size, kind),
3181            _ => unreachable!(),
3182        };
3183
3184        let (op, operand_id) = match kind {
3185            crate::ScalarKind::Float => (spirv::Op::VectorTimesScalar, scalar_id),
3186            _ => {
3187                let operand_id = self.gen_id();
3188                self.temp_list.clear();
3189                self.temp_list.resize(size as usize, scalar_id);
3190                block.body.push(Instruction::composite_construct(
3191                    result_type_id,
3192                    operand_id,
3193                    &self.temp_list,
3194                ));
3195                (spirv::Op::IMul, operand_id)
3196            }
3197        };
3198
3199        block.body.push(Instruction::binary(
3200            op,
3201            result_type_id,
3202            result_id,
3203            vector_id,
3204            operand_id,
3205        ));
3206    }
3207
3208    /// Build the instructions for the arithmetic expression of a dot product
3209    ///
3210    /// The argument `extractor` is a function that maps `(result_id,
3211    /// composite_id, index)` to an instruction that extracts the `index`th
3212    /// entry of the value with ID `composite_id` and assigns it to the slot
3213    /// with id `result_id` (which must have type `result_type_id`).
3214    #[expect(clippy::too_many_arguments)]
3215    fn write_dot_product(
3216        &mut self,
3217        result_id: Word,
3218        result_type_id: Word,
3219        arg0_id: Word,
3220        arg1_id: Word,
3221        size: u32,
3222        block: &mut Block,
3223        extractor: impl Fn(Word, Word, Word) -> Instruction,
3224    ) {
3225        let mut partial_sum = self.writer.get_constant_null(result_type_id);
3226        let last_component = size - 1;
3227        for index in 0..=last_component {
3228            // compute the product of the current components
3229            let a_id = self.gen_id();
3230            block.body.push(extractor(a_id, arg0_id, index));
3231            let b_id = self.gen_id();
3232            block.body.push(extractor(b_id, arg1_id, index));
3233            let prod_id = self.gen_id();
3234            block.body.push(Instruction::binary(
3235                spirv::Op::IMul,
3236                result_type_id,
3237                prod_id,
3238                a_id,
3239                b_id,
3240            ));
3241
3242            // choose the id for the next sum, depending on current index
3243            let id = if index == last_component {
3244                result_id
3245            } else {
3246                self.gen_id()
3247            };
3248
3249            // sum the computed product with the partial sum
3250            block.body.push(Instruction::binary(
3251                spirv::Op::IAdd,
3252                result_type_id,
3253                id,
3254                partial_sum,
3255                prod_id,
3256            ));
3257            // set the id of the result as the previous partial sum
3258            partial_sum = id;
3259        }
3260    }
3261
3262    /// Emit code for `pack4x{I,U}8[Clamp]` if capability "Int8" is available.
3263    fn write_pack4x8_optimized(
3264        &mut self,
3265        block: &mut Block,
3266        result_type_id: u32,
3267        arg0_id: u32,
3268        id: u32,
3269        is_signed: bool,
3270        should_clamp: bool,
3271    ) -> Instruction {
3272        let int_type = if is_signed {
3273            crate::ScalarKind::Sint
3274        } else {
3275            crate::ScalarKind::Uint
3276        };
3277        let wide_vector_type = NumericType::Vector {
3278            size: crate::VectorSize::Quad,
3279            scalar: crate::Scalar {
3280                kind: int_type,
3281                width: 4,
3282            },
3283        };
3284        let wide_vector_type_id = self.get_numeric_type_id(wide_vector_type);
3285        let packed_vector_type_id = self.get_numeric_type_id(NumericType::Vector {
3286            size: crate::VectorSize::Quad,
3287            scalar: crate::Scalar {
3288                kind: crate::ScalarKind::Uint,
3289                width: 1,
3290            },
3291        });
3292
3293        let mut wide_vector = arg0_id;
3294        if should_clamp {
3295            let (min, max, clamp_op) = if is_signed {
3296                (
3297                    crate::Literal::I32(-128),
3298                    crate::Literal::I32(127),
3299                    spirv::GlslStd450Op::SClamp,
3300                )
3301            } else {
3302                (
3303                    crate::Literal::U32(0),
3304                    crate::Literal::U32(255),
3305                    spirv::GlslStd450Op::UClamp,
3306                )
3307            };
3308            let [min, max] = [min, max].map(|lit| {
3309                let scalar = self.writer.get_constant_scalar(lit);
3310                self.writer.get_constant_composite(
3311                    LookupType::Local(LocalType::Numeric(wide_vector_type)),
3312                    &[scalar; 4],
3313                )
3314            });
3315
3316            let clamp_id = self.gen_id();
3317            block.body.push(Instruction::ext_inst_gl_op(
3318                self.writer.gl450_ext_inst_id,
3319                clamp_op,
3320                wide_vector_type_id,
3321                clamp_id,
3322                &[wide_vector, min, max],
3323            ));
3324
3325            wide_vector = clamp_id;
3326        }
3327
3328        let packed_vector = self.gen_id();
3329        block.body.push(Instruction::unary(
3330            spirv::Op::UConvert, // We truncate, so `UConvert` and `SConvert` behave identically.
3331            packed_vector_type_id,
3332            packed_vector,
3333            wide_vector,
3334        ));
3335
3336        // The SPIR-V spec [1] defines the bit order for bit casting between a vector
3337        // and a scalar precisely as required by the WGSL spec [2].
3338        // [1]: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpBitcast
3339        // [2]: https://www.w3.org/TR/WGSL/#pack4xI8-builtin
3340        Instruction::unary(spirv::Op::Bitcast, result_type_id, id, packed_vector)
3341    }
3342
3343    /// Emit code for `pack4x{I,U}8[Clamp]` if capability "Int8" is not available.
3344    fn write_pack4x8_polyfill(
3345        &mut self,
3346        block: &mut Block,
3347        result_type_id: u32,
3348        arg0_id: u32,
3349        id: u32,
3350        is_signed: bool,
3351        should_clamp: bool,
3352    ) -> Instruction {
3353        let int_type = if is_signed {
3354            crate::ScalarKind::Sint
3355        } else {
3356            crate::ScalarKind::Uint
3357        };
3358        let uint_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32));
3359        let int_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar {
3360            kind: int_type,
3361            width: 4,
3362        }));
3363
3364        let mut last_instruction = Instruction::new(spirv::Op::Nop);
3365
3366        let zero = self.writer.get_constant_scalar(crate::Literal::U32(0));
3367        let mut preresult = zero;
3368        block
3369            .body
3370            .reserve(usize::from(VEC_LENGTH) * (2 + usize::from(is_signed)));
3371
3372        let eight = self.writer.get_constant_scalar(crate::Literal::U32(8));
3373        const VEC_LENGTH: u8 = 4;
3374        for i in 0..u32::from(VEC_LENGTH) {
3375            let offset = self.writer.get_constant_scalar(crate::Literal::U32(i * 8));
3376            let mut extracted = self.gen_id();
3377            block.body.push(Instruction::binary(
3378                spirv::Op::CompositeExtract,
3379                int_type_id,
3380                extracted,
3381                arg0_id,
3382                i,
3383            ));
3384            if is_signed {
3385                let casted = self.gen_id();
3386                block.body.push(Instruction::unary(
3387                    spirv::Op::Bitcast,
3388                    uint_type_id,
3389                    casted,
3390                    extracted,
3391                ));
3392                extracted = casted;
3393            }
3394            if should_clamp {
3395                let (min, max, clamp_op) = if is_signed {
3396                    (
3397                        crate::Literal::I32(-128),
3398                        crate::Literal::I32(127),
3399                        spirv::GlslStd450Op::SClamp,
3400                    )
3401                } else {
3402                    (
3403                        crate::Literal::U32(0),
3404                        crate::Literal::U32(255),
3405                        spirv::GlslStd450Op::UClamp,
3406                    )
3407                };
3408                let [min, max] = [min, max].map(|lit| self.writer.get_constant_scalar(lit));
3409
3410                let clamp_id = self.gen_id();
3411                block.body.push(Instruction::ext_inst_gl_op(
3412                    self.writer.gl450_ext_inst_id,
3413                    clamp_op,
3414                    result_type_id,
3415                    clamp_id,
3416                    &[extracted, min, max],
3417                ));
3418
3419                extracted = clamp_id;
3420            }
3421            let is_last = i == u32::from(VEC_LENGTH - 1);
3422            if is_last {
3423                last_instruction = Instruction::quaternary(
3424                    spirv::Op::BitFieldInsert,
3425                    result_type_id,
3426                    id,
3427                    preresult,
3428                    extracted,
3429                    offset,
3430                    eight,
3431                )
3432            } else {
3433                let new_preresult = self.gen_id();
3434                block.body.push(Instruction::quaternary(
3435                    spirv::Op::BitFieldInsert,
3436                    result_type_id,
3437                    new_preresult,
3438                    preresult,
3439                    extracted,
3440                    offset,
3441                    eight,
3442                ));
3443                preresult = new_preresult;
3444            }
3445        }
3446        last_instruction
3447    }
3448
3449    /// Emit code for `unpack4x{I,U}8` if capability "Int8" is available.
3450    fn write_unpack4x8_optimized(
3451        &mut self,
3452        block: &mut Block,
3453        result_type_id: u32,
3454        arg0_id: u32,
3455        id: u32,
3456        is_signed: bool,
3457    ) -> Instruction {
3458        let (int_type, convert_op) = if is_signed {
3459            (crate::ScalarKind::Sint, spirv::Op::SConvert)
3460        } else {
3461            (crate::ScalarKind::Uint, spirv::Op::UConvert)
3462        };
3463
3464        let packed_vector_type_id = self.get_numeric_type_id(NumericType::Vector {
3465            size: crate::VectorSize::Quad,
3466            scalar: crate::Scalar {
3467                kind: int_type,
3468                width: 1,
3469            },
3470        });
3471
3472        // The SPIR-V spec [1] defines the bit order for bit casting between a vector
3473        // and a scalar precisely as required by the WGSL spec [2].
3474        // [1]: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpBitcast
3475        // [2]: https://www.w3.org/TR/WGSL/#pack4xI8-builtin
3476        let packed_vector = self.gen_id();
3477        block.body.push(Instruction::unary(
3478            spirv::Op::Bitcast,
3479            packed_vector_type_id,
3480            packed_vector,
3481            arg0_id,
3482        ));
3483
3484        Instruction::unary(convert_op, result_type_id, id, packed_vector)
3485    }
3486
3487    /// Emit code for `unpack4x{I,U}8` if capability "Int8" is not available.
3488    fn write_unpack4x8_polyfill(
3489        &mut self,
3490        block: &mut Block,
3491        result_type_id: u32,
3492        arg0_id: u32,
3493        id: u32,
3494        is_signed: bool,
3495    ) -> Instruction {
3496        let (int_type, extract_op) = if is_signed {
3497            (crate::ScalarKind::Sint, spirv::Op::BitFieldSExtract)
3498        } else {
3499            (crate::ScalarKind::Uint, spirv::Op::BitFieldUExtract)
3500        };
3501
3502        let sint_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::I32));
3503
3504        let eight = self.writer.get_constant_scalar(crate::Literal::U32(8));
3505        let int_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar {
3506            kind: int_type,
3507            width: 4,
3508        }));
3509        block
3510            .body
3511            .reserve(usize::from(VEC_LENGTH) * 2 + usize::from(is_signed));
3512        let arg_id = if is_signed {
3513            let new_arg_id = self.gen_id();
3514            block.body.push(Instruction::unary(
3515                spirv::Op::Bitcast,
3516                sint_type_id,
3517                new_arg_id,
3518                arg0_id,
3519            ));
3520            new_arg_id
3521        } else {
3522            arg0_id
3523        };
3524
3525        const VEC_LENGTH: u8 = 4;
3526        let parts: [_; VEC_LENGTH as usize] = core::array::from_fn(|_| self.gen_id());
3527        for (i, part_id) in parts.into_iter().enumerate() {
3528            let index = self
3529                .writer
3530                .get_constant_scalar(crate::Literal::U32(i as u32 * 8));
3531            block.body.push(Instruction::ternary(
3532                extract_op,
3533                int_type_id,
3534                part_id,
3535                arg_id,
3536                index,
3537                eight,
3538            ));
3539        }
3540
3541        Instruction::composite_construct(result_type_id, id, &parts)
3542    }
3543
3544    /// Generate one or more SPIR-V blocks for `naga_block`.
3545    ///
3546    /// Use `label_id` as the label for the SPIR-V entry point block.
3547    ///
3548    /// If control reaches the end of the SPIR-V block, terminate it according
3549    /// to `exit`. This function's return value indicates whether it acted on
3550    /// this parameter or not; see [`BlockExitDisposition`].
3551    ///
3552    /// If the block contains [`Break`] or [`Continue`] statements,
3553    /// `loop_context` supplies the labels of the SPIR-V blocks to jump to. If
3554    /// either of these labels are `None`, then it should have been a Naga
3555    /// validation error for the corresponding statement to occur in this
3556    /// context.
3557    ///
3558    /// [`Break`]: Statement::Break
3559    /// [`Continue`]: Statement::Continue
3560    fn write_block(
3561        &mut self,
3562        label_id: Word,
3563        naga_block: &crate::Block,
3564        exit: BlockExit,
3565        loop_context: LoopContext,
3566        debug_info: Option<&DebugInfoInner>,
3567    ) -> Result<BlockExitDisposition, Error> {
3568        let mut block = Block::new(label_id);
3569        for (statement, span) in naga_block.span_iter() {
3570            if let (Some(debug_info), false) = (
3571                debug_info,
3572                matches!(
3573                    statement,
3574                    &(Statement::Block(..)
3575                        | Statement::Break
3576                        | Statement::Continue
3577                        | Statement::Kill
3578                        | Statement::Return { .. }
3579                        | Statement::Loop { .. })
3580                ),
3581            ) {
3582                let loc: crate::SourceLocation = span.location(debug_info.source_code);
3583                block.body.push(Instruction::line(
3584                    debug_info.source_file_id,
3585                    loc.line_number,
3586                    loc.line_position,
3587                ));
3588            };
3589            match *statement {
3590                Statement::Emit(ref range) => {
3591                    for handle in range.clone() {
3592                        // omit const expressions as we've already cached those
3593                        if !self.expression_constness.is_const(handle) {
3594                            self.cache_expression_value(handle, &mut block)?;
3595                        }
3596                    }
3597                }
3598                Statement::Block(ref block_statements) => {
3599                    let scope_id = self.gen_id();
3600                    self.function.consume(block, Instruction::branch(scope_id));
3601
3602                    let merge_id = self.gen_id();
3603                    let merge_used = self.write_block(
3604                        scope_id,
3605                        block_statements,
3606                        BlockExit::Branch { target: merge_id },
3607                        loop_context,
3608                        debug_info,
3609                    )?;
3610
3611                    match merge_used {
3612                        BlockExitDisposition::Used => {
3613                            block = Block::new(merge_id);
3614                        }
3615                        BlockExitDisposition::Discarded => {
3616                            return Ok(BlockExitDisposition::Discarded);
3617                        }
3618                    }
3619                }
3620                Statement::If {
3621                    condition,
3622                    ref accept,
3623                    ref reject,
3624                } => {
3625                    // In spirv 1.6, in a conditional branch the two block ids
3626                    // of the branches can't have the same label. If `accept`
3627                    // and `reject` are both empty (e.g. in `if (condition) {}`)
3628                    // merge id will be both labels. Because both branches are
3629                    // empty, we can skip the if statement.
3630                    if !(accept.is_empty() && reject.is_empty()) {
3631                        let condition_id = self.cached[condition];
3632
3633                        let merge_id = self.gen_id();
3634                        block.body.push(Instruction::selection_merge(
3635                            merge_id,
3636                            spirv::SelectionControl::NONE,
3637                        ));
3638
3639                        let accept_id = if accept.is_empty() {
3640                            None
3641                        } else {
3642                            Some(self.gen_id())
3643                        };
3644                        let reject_id = if reject.is_empty() {
3645                            None
3646                        } else {
3647                            Some(self.gen_id())
3648                        };
3649
3650                        self.function.consume(
3651                            block,
3652                            Instruction::branch_conditional(
3653                                condition_id,
3654                                accept_id.unwrap_or(merge_id),
3655                                reject_id.unwrap_or(merge_id),
3656                            ),
3657                        );
3658
3659                        if let Some(block_id) = accept_id {
3660                            // We can ignore the `BlockExitDisposition` returned here because,
3661                            // even if `merge_id` is not actually reachable, it is always
3662                            // referred to by the `OpSelectionMerge` instruction we emitted
3663                            // earlier.
3664                            let _ = self.write_block(
3665                                block_id,
3666                                accept,
3667                                BlockExit::Branch { target: merge_id },
3668                                loop_context,
3669                                debug_info,
3670                            )?;
3671                        }
3672                        if let Some(block_id) = reject_id {
3673                            // We can ignore the `BlockExitDisposition` returned here because,
3674                            // even if `merge_id` is not actually reachable, it is always
3675                            // referred to by the `OpSelectionMerge` instruction we emitted
3676                            // earlier.
3677                            let _ = self.write_block(
3678                                block_id,
3679                                reject,
3680                                BlockExit::Branch { target: merge_id },
3681                                loop_context,
3682                                debug_info,
3683                            )?;
3684                        }
3685
3686                        block = Block::new(merge_id);
3687                    }
3688                }
3689                Statement::Switch {
3690                    selector,
3691                    ref cases,
3692                } => {
3693                    let selector_id = self.cached[selector];
3694
3695                    let merge_id = self.gen_id();
3696                    block.body.push(Instruction::selection_merge(
3697                        merge_id,
3698                        spirv::SelectionControl::NONE,
3699                    ));
3700
3701                    let mut default_id = None;
3702                    // id of previous empty fall-through case
3703                    let mut last_id = None;
3704
3705                    let mut raw_cases = Vec::with_capacity(cases.len());
3706                    let mut case_ids = Vec::with_capacity(cases.len());
3707                    for case in cases.iter() {
3708                        // take id of previous empty fall-through case or generate a new one
3709                        let label_id = last_id.take().unwrap_or_else(|| self.gen_id());
3710
3711                        if case.fall_through && case.body.is_empty() {
3712                            last_id = Some(label_id);
3713                        }
3714
3715                        case_ids.push(label_id);
3716
3717                        match case.value {
3718                            crate::SwitchValue::I32(value) => {
3719                                raw_cases.push(super::instructions::Case {
3720                                    value: value as Word,
3721                                    label_id,
3722                                });
3723                            }
3724                            crate::SwitchValue::U32(value) => {
3725                                raw_cases.push(super::instructions::Case { value, label_id });
3726                            }
3727                            crate::SwitchValue::Default => {
3728                                default_id = Some(label_id);
3729                            }
3730                        }
3731                    }
3732
3733                    let default_id = default_id.unwrap();
3734
3735                    self.function.consume(
3736                        block,
3737                        Instruction::switch(selector_id, default_id, &raw_cases),
3738                    );
3739
3740                    let inner_context = LoopContext {
3741                        break_id: Some(merge_id),
3742                        ..loop_context
3743                    };
3744
3745                    for (i, (case, label_id)) in cases
3746                        .iter()
3747                        .zip(case_ids.iter())
3748                        .filter(|&(case, _)| !(case.fall_through && case.body.is_empty()))
3749                        .enumerate()
3750                    {
3751                        let case_finish_id = if case.fall_through {
3752                            case_ids[i + 1]
3753                        } else {
3754                            merge_id
3755                        };
3756                        // We can ignore the `BlockExitDisposition` returned here because
3757                        // `case_finish_id` is always referred to by either:
3758                        //
3759                        // - the `OpSwitch`, if it's the next case's label for a
3760                        //   fall-through, or
3761                        //
3762                        // - the `OpSelectionMerge`, if it's the switch's overall merge
3763                        //   block because there's no fall-through.
3764                        let _ = self.write_block(
3765                            *label_id,
3766                            &case.body,
3767                            BlockExit::Branch {
3768                                target: case_finish_id,
3769                            },
3770                            inner_context,
3771                            debug_info,
3772                        )?;
3773                    }
3774
3775                    block = Block::new(merge_id);
3776                }
3777                Statement::Loop {
3778                    ref body,
3779                    ref continuing,
3780                    break_if,
3781                } => {
3782                    let preamble_id = self.gen_id();
3783                    self.function
3784                        .consume(block, Instruction::branch(preamble_id));
3785
3786                    let merge_id = self.gen_id();
3787                    let body_id = self.gen_id();
3788                    let continuing_id = self.gen_id();
3789
3790                    // SPIR-V requires the continuing to the `OpLoopMerge`,
3791                    // so we have to start a new block with it.
3792                    block = Block::new(preamble_id);
3793                    // HACK the loop statement is begin with branch instruction,
3794                    // so we need to put `OpLine` debug info before merge instruction
3795                    if let Some(debug_info) = debug_info {
3796                        let loc: crate::SourceLocation = span.location(debug_info.source_code);
3797                        block.body.push(Instruction::line(
3798                            debug_info.source_file_id,
3799                            loc.line_number,
3800                            loc.line_position,
3801                        ))
3802                    }
3803                    block.body.push(Instruction::loop_merge(
3804                        merge_id,
3805                        continuing_id,
3806                        spirv::SelectionControl::NONE,
3807                    ));
3808
3809                    if self.force_loop_bounding {
3810                        block = self.write_force_bounded_loop_instructions(block, merge_id);
3811                    }
3812                    self.function.consume(block, Instruction::branch(body_id));
3813
3814                    // We can ignore the `BlockExitDisposition` returned here because,
3815                    // even if `continuing_id` is not actually reachable, it is always
3816                    // referred to by the `OpLoopMerge` instruction we emitted earlier.
3817                    let _ = self.write_block(
3818                        body_id,
3819                        body,
3820                        BlockExit::Branch {
3821                            target: continuing_id,
3822                        },
3823                        LoopContext {
3824                            continuing_id: Some(continuing_id),
3825                            break_id: Some(merge_id),
3826                        },
3827                        debug_info,
3828                    )?;
3829
3830                    let exit = match break_if {
3831                        Some(condition) => BlockExit::BreakIf {
3832                            condition,
3833                            preamble_id,
3834                        },
3835                        None => BlockExit::Branch {
3836                            target: preamble_id,
3837                        },
3838                    };
3839
3840                    // We can ignore the `BlockExitDisposition` returned here because,
3841                    // even if `merge_id` is not actually reachable, it is always referred
3842                    // to by the `OpLoopMerge` instruction we emitted earlier.
3843                    let _ = self.write_block(
3844                        continuing_id,
3845                        continuing,
3846                        exit,
3847                        LoopContext {
3848                            continuing_id: None,
3849                            break_id: Some(merge_id),
3850                        },
3851                        debug_info,
3852                    )?;
3853
3854                    block = Block::new(merge_id);
3855                }
3856                Statement::Break => {
3857                    self.function
3858                        .consume(block, Instruction::branch(loop_context.break_id.unwrap()));
3859                    return Ok(BlockExitDisposition::Discarded);
3860                }
3861                Statement::Continue => {
3862                    self.function.consume(
3863                        block,
3864                        Instruction::branch(loop_context.continuing_id.unwrap()),
3865                    );
3866                    return Ok(BlockExitDisposition::Discarded);
3867                }
3868                Statement::Return { value: Some(value) } => {
3869                    let value_id = self.cached[value];
3870                    let instruction = match self.function.entry_point_context {
3871                        // If this is an entry point, and we need to return anything,
3872                        // let's instead store the output variables and return `void`.
3873                        Some(ref context) => self.writer.write_entry_point_return(
3874                            value_id,
3875                            self.ir_function.result.as_ref().unwrap(),
3876                            &context.results,
3877                            &mut block.body,
3878                        )?,
3879                        None => Instruction::return_value(value_id),
3880                    };
3881                    self.function.consume(block, instruction);
3882                    return Ok(BlockExitDisposition::Discarded);
3883                }
3884                Statement::Return { value: None } => {
3885                    self.function.consume(block, Instruction::return_void());
3886                    return Ok(BlockExitDisposition::Discarded);
3887                }
3888                Statement::Kill => {
3889                    self.function.consume(block, Instruction::kill());
3890                    return Ok(BlockExitDisposition::Discarded);
3891                }
3892                Statement::ControlBarrier(flags) => {
3893                    self.writer.write_control_barrier(flags, &mut block.body);
3894                }
3895                Statement::MemoryBarrier(flags) => {
3896                    self.writer.write_memory_barrier(flags, &mut block);
3897                }
3898                Statement::Store { pointer, value } => {
3899                    let value_id = self.cached[value];
3900                    match self.write_access_chain(
3901                        pointer,
3902                        &mut block,
3903                        AccessTypeAdjustment::None,
3904                    )? {
3905                        ExpressionPointer::Ready { pointer_id } => {
3906                            let atomic_space = match *self.fun_info[pointer]
3907                                .ty
3908                                .inner_with(&self.ir_module.types)
3909                            {
3910                                crate::TypeInner::Pointer { base, space } => {
3911                                    match self.ir_module.types[base].inner {
3912                                        crate::TypeInner::Atomic { .. } => Some(space),
3913                                        _ => None,
3914                                    }
3915                                }
3916                                _ => None,
3917                            };
3918                            let instruction = if let Some(space) = atomic_space {
3919                                let (semantics, scope) = space.to_spirv_semantics_and_scope();
3920                                let scope_constant_id = self.get_scope_constant(scope as u32);
3921                                let semantics_id = self.get_index_constant(semantics.bits());
3922                                Instruction::atomic_store(
3923                                    pointer_id,
3924                                    scope_constant_id,
3925                                    semantics_id,
3926                                    value_id,
3927                                )
3928                            } else {
3929                                Instruction::store(pointer_id, value_id, None)
3930                            };
3931                            block.body.push(instruction);
3932                        }
3933                        ExpressionPointer::Conditional { condition, access } => {
3934                            let mut selection = Selection::start(&mut block, ());
3935                            selection.if_true(self, condition, ());
3936
3937                            // The in-bounds path. Perform the access and the store.
3938                            let pointer_id = access.result_id.unwrap();
3939                            selection.block().body.push(access);
3940                            selection
3941                                .block()
3942                                .body
3943                                .push(Instruction::store(pointer_id, value_id, None));
3944
3945                            // Finish the in-bounds block and start the merge block. This
3946                            // is the block we'll leave current on return.
3947                            selection.finish(self, ());
3948                        }
3949                    };
3950                }
3951                Statement::ImageStore {
3952                    image,
3953                    coordinate,
3954                    array_index,
3955                    value,
3956                } => self.write_image_store(image, coordinate, array_index, value, &mut block)?,
3957                Statement::Call {
3958                    function: local_function,
3959                    ref arguments,
3960                    result,
3961                } => {
3962                    let id = self.gen_id();
3963                    self.temp_list.clear();
3964                    for &argument in arguments {
3965                        self.temp_list.push(self.cached[argument]);
3966                    }
3967
3968                    let type_id = match result {
3969                        Some(expr) => {
3970                            self.cached[expr] = id;
3971                            self.get_expression_type_id(&self.fun_info[expr].ty)
3972                        }
3973                        None => self.writer.void_type,
3974                    };
3975
3976                    block.body.push(Instruction::function_call(
3977                        type_id,
3978                        id,
3979                        self.writer.lookup_function[&local_function],
3980                        &self.temp_list,
3981                    ));
3982                }
3983                Statement::Atomic {
3984                    pointer,
3985                    ref fun,
3986                    value,
3987                    result,
3988                } => {
3989                    let id = self.gen_id();
3990                    // Compare-and-exchange operations produce a struct result,
3991                    // so use `result`'s type if it is available. For no-result
3992                    // operations, fall back to `value`'s type.
3993                    let result_type_id =
3994                        self.get_expression_type_id(&self.fun_info[result.unwrap_or(value)].ty);
3995
3996                    if let Some(result) = result {
3997                        self.cached[result] = id;
3998                    }
3999
4000                    let pointer_id = match self.write_access_chain(
4001                        pointer,
4002                        &mut block,
4003                        AccessTypeAdjustment::None,
4004                    )? {
4005                        ExpressionPointer::Ready { pointer_id } => pointer_id,
4006                        ExpressionPointer::Conditional { .. } => {
4007                            return Err(Error::FeatureNotImplemented(
4008                                "Atomics out-of-bounds handling",
4009                            ));
4010                        }
4011                    };
4012
4013                    let space = self.fun_info[pointer]
4014                        .ty
4015                        .inner_with(&self.ir_module.types)
4016                        .pointer_space()
4017                        .unwrap();
4018                    let (semantics, scope) = space.to_spirv_semantics_and_scope();
4019                    let scope_constant_id = self.get_scope_constant(scope as u32);
4020                    let semantics_id = self.get_index_constant(semantics.bits());
4021                    let value_id = self.cached[value];
4022                    let value_inner = self.fun_info[value].ty.inner_with(&self.ir_module.types);
4023
4024                    let crate::TypeInner::Scalar(scalar) = *value_inner else {
4025                        return Err(Error::FeatureNotImplemented(
4026                            "Atomics with non-scalar values",
4027                        ));
4028                    };
4029
4030                    let instruction = match *fun {
4031                        crate::AtomicFunction::Add => {
4032                            let spirv_op = match scalar.kind {
4033                                crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4034                                    spirv::Op::AtomicIAdd
4035                                }
4036                                crate::ScalarKind::Float => spirv::Op::AtomicFAddEXT,
4037                                _ => unimplemented!(),
4038                            };
4039                            Instruction::atomic_binary(
4040                                spirv_op,
4041                                result_type_id,
4042                                id,
4043                                pointer_id,
4044                                scope_constant_id,
4045                                semantics_id,
4046                                value_id,
4047                            )
4048                        }
4049                        crate::AtomicFunction::Subtract => {
4050                            let (spirv_op, value_id) = match scalar.kind {
4051                                crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4052                                    (spirv::Op::AtomicISub, value_id)
4053                                }
4054                                crate::ScalarKind::Float => {
4055                                    // HACK: SPIR-V doesn't have a atomic subtraction,
4056                                    // so we add the negated value instead.
4057                                    let neg_result_id = self.gen_id();
4058                                    block.body.push(Instruction::unary(
4059                                        spirv::Op::FNegate,
4060                                        result_type_id,
4061                                        neg_result_id,
4062                                        value_id,
4063                                    ));
4064                                    (spirv::Op::AtomicFAddEXT, neg_result_id)
4065                                }
4066                                _ => unimplemented!(),
4067                            };
4068                            Instruction::atomic_binary(
4069                                spirv_op,
4070                                result_type_id,
4071                                id,
4072                                pointer_id,
4073                                scope_constant_id,
4074                                semantics_id,
4075                                value_id,
4076                            )
4077                        }
4078                        crate::AtomicFunction::And => {
4079                            let spirv_op = match scalar.kind {
4080                                crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4081                                    spirv::Op::AtomicAnd
4082                                }
4083                                _ => unimplemented!(),
4084                            };
4085                            Instruction::atomic_binary(
4086                                spirv_op,
4087                                result_type_id,
4088                                id,
4089                                pointer_id,
4090                                scope_constant_id,
4091                                semantics_id,
4092                                value_id,
4093                            )
4094                        }
4095                        crate::AtomicFunction::InclusiveOr => {
4096                            let spirv_op = match scalar.kind {
4097                                crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4098                                    spirv::Op::AtomicOr
4099                                }
4100                                _ => unimplemented!(),
4101                            };
4102                            Instruction::atomic_binary(
4103                                spirv_op,
4104                                result_type_id,
4105                                id,
4106                                pointer_id,
4107                                scope_constant_id,
4108                                semantics_id,
4109                                value_id,
4110                            )
4111                        }
4112                        crate::AtomicFunction::ExclusiveOr => {
4113                            let spirv_op = match scalar.kind {
4114                                crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4115                                    spirv::Op::AtomicXor
4116                                }
4117                                _ => unimplemented!(),
4118                            };
4119                            Instruction::atomic_binary(
4120                                spirv_op,
4121                                result_type_id,
4122                                id,
4123                                pointer_id,
4124                                scope_constant_id,
4125                                semantics_id,
4126                                value_id,
4127                            )
4128                        }
4129                        crate::AtomicFunction::Min => {
4130                            let spirv_op = match scalar.kind {
4131                                crate::ScalarKind::Sint => spirv::Op::AtomicSMin,
4132                                crate::ScalarKind::Uint => spirv::Op::AtomicUMin,
4133                                _ => unimplemented!(),
4134                            };
4135                            Instruction::atomic_binary(
4136                                spirv_op,
4137                                result_type_id,
4138                                id,
4139                                pointer_id,
4140                                scope_constant_id,
4141                                semantics_id,
4142                                value_id,
4143                            )
4144                        }
4145                        crate::AtomicFunction::Max => {
4146                            let spirv_op = match scalar.kind {
4147                                crate::ScalarKind::Sint => spirv::Op::AtomicSMax,
4148                                crate::ScalarKind::Uint => spirv::Op::AtomicUMax,
4149                                _ => unimplemented!(),
4150                            };
4151                            Instruction::atomic_binary(
4152                                spirv_op,
4153                                result_type_id,
4154                                id,
4155                                pointer_id,
4156                                scope_constant_id,
4157                                semantics_id,
4158                                value_id,
4159                            )
4160                        }
4161                        crate::AtomicFunction::Exchange { compare: None } => {
4162                            Instruction::atomic_binary(
4163                                spirv::Op::AtomicExchange,
4164                                result_type_id,
4165                                id,
4166                                pointer_id,
4167                                scope_constant_id,
4168                                semantics_id,
4169                                value_id,
4170                            )
4171                        }
4172                        crate::AtomicFunction::Exchange { compare: Some(cmp) } => {
4173                            let scalar_type_id =
4174                                self.get_numeric_type_id(NumericType::Scalar(scalar));
4175                            let bool_type_id =
4176                                self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::BOOL));
4177
4178                            let cas_result_id = self.gen_id();
4179                            let equality_result_id = self.gen_id();
4180                            let equality_operator = match scalar.kind {
4181                                crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4182                                    spirv::Op::IEqual
4183                                }
4184                                _ => unimplemented!(),
4185                            };
4186
4187                            let mut cas_instr = Instruction::new(spirv::Op::AtomicCompareExchange);
4188                            cas_instr.set_type(scalar_type_id);
4189                            cas_instr.set_result(cas_result_id);
4190                            cas_instr.add_operand(pointer_id);
4191                            cas_instr.add_operand(scope_constant_id);
4192                            cas_instr.add_operand(semantics_id); // semantics if equal
4193                            cas_instr.add_operand(semantics_id); // semantics if not equal
4194                            cas_instr.add_operand(value_id);
4195                            cas_instr.add_operand(self.cached[cmp]);
4196                            block.body.push(cas_instr);
4197                            block.body.push(Instruction::binary(
4198                                equality_operator,
4199                                bool_type_id,
4200                                equality_result_id,
4201                                cas_result_id,
4202                                self.cached[cmp],
4203                            ));
4204                            Instruction::composite_construct(
4205                                result_type_id,
4206                                id,
4207                                &[cas_result_id, equality_result_id],
4208                            )
4209                        }
4210                    };
4211
4212                    block.body.push(instruction);
4213                }
4214                Statement::ImageAtomic {
4215                    image,
4216                    coordinate,
4217                    array_index,
4218                    fun,
4219                    value,
4220                } => {
4221                    self.write_image_atomic(
4222                        image,
4223                        coordinate,
4224                        array_index,
4225                        fun,
4226                        value,
4227                        &mut block,
4228                    )?;
4229                }
4230                Statement::WorkGroupUniformLoad { pointer, result } => {
4231                    self.writer
4232                        .write_control_barrier(crate::Barrier::WORK_GROUP, &mut block.body);
4233                    let result_type_id = self.get_expression_type_id(&self.fun_info[result].ty);
4234                    // Match `Expression::Load` behavior, including `OpAtomicLoad` when
4235                    // loading from a pointer to `atomic<T>`.
4236                    let id = self.write_checked_load(
4237                        pointer,
4238                        &mut block,
4239                        AccessTypeAdjustment::None,
4240                        result_type_id,
4241                    )?;
4242                    self.cached[result] = id;
4243                    self.writer
4244                        .write_control_barrier(crate::Barrier::WORK_GROUP, &mut block.body);
4245                }
4246                Statement::RayQuery { query, ref fun } => {
4247                    self.write_ray_query_function(query, fun, &mut block);
4248                }
4249                Statement::SubgroupBallot {
4250                    result,
4251                    ref predicate,
4252                } => {
4253                    self.write_subgroup_ballot(predicate, result, &mut block)?;
4254                }
4255                Statement::SubgroupCollectiveOperation {
4256                    ref op,
4257                    ref collective_op,
4258                    argument,
4259                    result,
4260                } => {
4261                    self.write_subgroup_operation(op, collective_op, argument, result, &mut block)?;
4262                }
4263                Statement::SubgroupGather {
4264                    ref mode,
4265                    argument,
4266                    result,
4267                } => {
4268                    self.write_subgroup_gather(mode, argument, result, &mut block)?;
4269                }
4270                Statement::SubgroupBallotFindBit {
4271                    order,
4272                    argument,
4273                    result,
4274                } => {
4275                    self.write_subgroup_ballot_find_bit(order, argument, result, &mut block)?;
4276                }
4277                Statement::CooperativeStore { target, ref data } => {
4278                    let target_id = self.cached[target];
4279                    let layout = if data.row_major {
4280                        spirv::CooperativeMatrixLayout::RowMajorKHR
4281                    } else {
4282                        spirv::CooperativeMatrixLayout::ColumnMajorKHR
4283                    };
4284                    let layout_id = self.get_index_constant(layout as u32);
4285                    let stride_id = self.cached[data.stride];
4286                    match self.write_access_chain(
4287                        data.pointer,
4288                        &mut block,
4289                        AccessTypeAdjustment::None,
4290                    )? {
4291                        ExpressionPointer::Ready { pointer_id } => {
4292                            block.body.push(Instruction::coop_store(
4293                                target_id, pointer_id, layout_id, stride_id,
4294                            ));
4295                        }
4296                        ExpressionPointer::Conditional { condition, access } => {
4297                            let mut selection = Selection::start(&mut block, ());
4298                            selection.if_true(self, condition, ());
4299
4300                            // The in-bounds path. Perform the access and the store.
4301                            let pointer_id = access.result_id.unwrap();
4302                            selection.block().body.push(access);
4303                            selection.block().body.push(Instruction::coop_store(
4304                                target_id, pointer_id, layout_id, stride_id,
4305                            ));
4306
4307                            // Finish the in-bounds block and start the merge block. This
4308                            // is the block we'll leave current on return.
4309                            selection.finish(self, ());
4310                        }
4311                    };
4312                }
4313                Statement::RayPipelineFunction(ref fun) => {
4314                    self.write_ray_tracing_pipeline_function(fun, &mut block)?;
4315                }
4316                Statement::HitObject {
4317                    hit_object,
4318                    ref fun,
4319                } => {
4320                    self.write_hit_object_function(hit_object, fun, &mut block)?;
4321                }
4322                Statement::DebugPrintf {
4323                    ref format,
4324                    ref arguments,
4325                } => {
4326                    let mut format_params = Vec::with_capacity(arguments.len());
4327                    for &arg in arguments {
4328                        let word = self.cached[arg];
4329                        format_params.push(word);
4330                    }
4331
4332                    self.writer
4333                        .write_debug_printf(&mut block, format, &format_params);
4334                }
4335            }
4336        }
4337
4338        let termination = match exit {
4339            // We're generating code for the top-level Block of the function, so we
4340            // need to end it with some kind of return instruction.
4341            BlockExit::Return => match self.ir_function.result {
4342                Some(ref result) if self.function.entry_point_context.is_none() => {
4343                    let type_id = self.get_handle_type_id(result.ty);
4344                    let null_id = self.writer.get_constant_null(type_id);
4345                    Instruction::return_value(null_id)
4346                }
4347                _ => Instruction::return_void(),
4348            },
4349            BlockExit::Branch { target } => Instruction::branch(target),
4350            BlockExit::BreakIf {
4351                condition,
4352                preamble_id,
4353            } => {
4354                let condition_id = self.cached[condition];
4355
4356                Instruction::branch_conditional(
4357                    condition_id,
4358                    loop_context.break_id.unwrap(),
4359                    preamble_id,
4360                )
4361            }
4362        };
4363
4364        self.function.consume(block, termination);
4365        Ok(BlockExitDisposition::Used)
4366    }
4367
4368    pub(super) fn write_function_body(
4369        &mut self,
4370        entry_id: Word,
4371        debug_info: Option<&DebugInfoInner>,
4372    ) -> Result<(), Error> {
4373        // We can ignore the `BlockExitDisposition` returned here because
4374        // `BlockExit::Return` doesn't refer to a block.
4375        let _ = self.write_block(
4376            entry_id,
4377            &self.ir_function.body,
4378            BlockExit::Return,
4379            LoopContext::default(),
4380            debug_info,
4381        )?;
4382
4383        Ok(())
4384    }
4385}