Skip to main content

naga/front/wgsl/lower/
mod.rs

1use alloc::{
2    borrow::ToOwned,
3    boxed::Box,
4    format,
5    string::{String, ToString},
6    vec::Vec,
7};
8use core::{matches, num::NonZeroU32};
9
10use crate::front::wgsl::error::{Error, ExpectedToken, InvalidAssignmentType};
11use crate::front::wgsl::index::Index;
12use crate::front::wgsl::parse::directive::enable_extension::{
13    EnableExtensions, ImplementedEnableExtension,
14};
15use crate::front::wgsl::parse::number::Number;
16use crate::front::wgsl::parse::{ast, conv};
17use crate::front::wgsl::Result;
18use crate::front::Typifier;
19use crate::{
20    common::wgsl::{TryToWgsl, TypeContext},
21    compact::KeepUnused,
22};
23use crate::{common::ForDebugWithTypes, proc::LayoutErrorInner};
24use crate::{ir, proc};
25use crate::{Arena, FastHashMap, FastIndexMap, Handle, Span};
26
27use construction::Constructor;
28use template_list::TemplateListIter;
29
30mod construction;
31mod conversion;
32mod template_list;
33
34/// Resolves the inner type of a given expression.
35///
36/// Expects a &mut [`ExpressionContext`] and a [`Handle<Expression>`].
37///
38/// Returns a &[`ir::TypeInner`].
39///
40/// Ideally, we would simply have a function that takes a `&mut ExpressionContext`
41/// and returns a `&TypeResolution`. Unfortunately, this leads the borrow checker
42/// to conclude that the mutable borrow lasts for as long as we are using the
43/// `&TypeResolution`, so we can't use the `ExpressionContext` for anything else -
44/// like, say, resolving another operand's type. Using a macro that expands to
45/// two separate calls, only the first of which needs a `&mut`,
46/// lets the borrow checker see that the mutable borrow is over.
47macro_rules! resolve_inner {
48    ($ctx:ident, $expr:expr) => {{
49        $ctx.grow_types($expr)?;
50        $ctx.typifier()[$expr].inner_with(&$ctx.module.types)
51    }};
52}
53pub(super) use resolve_inner;
54
55/// Resolves the inner types of two given expressions.
56///
57/// Expects a &mut [`ExpressionContext`] and two [`Handle<Expression>`]s.
58///
59/// Returns a tuple containing two &[`ir::TypeInner`].
60///
61/// See the documentation of [`resolve_inner!`] for why this macro is necessary.
62macro_rules! resolve_inner_binary {
63    ($ctx:ident, $left:expr, $right:expr) => {{
64        $ctx.grow_types($left)?;
65        $ctx.grow_types($right)?;
66        (
67            $ctx.typifier()[$left].inner_with(&$ctx.module.types),
68            $ctx.typifier()[$right].inner_with(&$ctx.module.types),
69        )
70    }};
71}
72
73/// Resolves the type of a given expression.
74///
75/// Expects a &mut [`ExpressionContext`] and a [`Handle<Expression>`].
76///
77/// Returns a &[`TypeResolution`].
78///
79/// See the documentation of [`resolve_inner!`] for why this macro is necessary.
80///
81/// [`TypeResolution`]: proc::TypeResolution
82macro_rules! resolve {
83    ($ctx:ident, $expr:expr) => {{
84        let expr = $expr;
85        $ctx.grow_types(expr)?;
86        &$ctx.typifier()[expr]
87    }};
88}
89pub(super) use resolve;
90
91/// State for constructing a `ir::Module`.
92pub struct GlobalContext<'source, 'temp, 'out> {
93    enable_extensions: EnableExtensions,
94
95    /// The `TranslationUnit`'s expressions arena.
96    ast_expressions: &'temp Arena<ast::Expression<'source>>,
97
98    // Naga IR values.
99    /// The map from the names of module-scope declarations to the Naga IR
100    /// `Handle`s we have built for them, owned by `Lowerer::lower`.
101    globals: &'temp mut FastHashMap<&'source str, LoweredGlobalDecl>,
102
103    /// The module we're constructing.
104    module: &'out mut ir::Module,
105
106    const_typifier: &'temp mut Typifier,
107
108    layouter: &'temp mut proc::Layouter,
109
110    global_expression_kind_tracker: &'temp mut proc::ExpressionKindTracker,
111}
112
113impl<'source> GlobalContext<'source, '_, '_> {
114    const fn as_const(&mut self) -> ExpressionContext<'source, '_, '_> {
115        ExpressionContext {
116            enable_extensions: self.enable_extensions,
117            ast_expressions: self.ast_expressions,
118            globals: self.globals,
119            module: self.module,
120            const_typifier: self.const_typifier,
121            layouter: self.layouter,
122            expr_type: ExpressionContextType::Constant(None),
123            global_expression_kind_tracker: self.global_expression_kind_tracker,
124        }
125    }
126
127    const fn as_override(&mut self) -> ExpressionContext<'source, '_, '_> {
128        ExpressionContext {
129            enable_extensions: self.enable_extensions,
130            ast_expressions: self.ast_expressions,
131            globals: self.globals,
132            module: self.module,
133            const_typifier: self.const_typifier,
134            layouter: self.layouter,
135            expr_type: ExpressionContextType::Override,
136            global_expression_kind_tracker: self.global_expression_kind_tracker,
137        }
138    }
139
140    fn ensure_type_exists(
141        &mut self,
142        name: Option<String>,
143        inner: ir::TypeInner,
144    ) -> Handle<ir::Type> {
145        self.module
146            .types
147            .insert(ir::Type { inner, name }, Span::UNDEFINED)
148    }
149}
150
151/// State for lowering a statement within a function.
152pub struct StatementContext<'source, 'temp, 'out> {
153    enable_extensions: EnableExtensions,
154
155    // WGSL AST values.
156    /// A reference to [`TranslationUnit::expressions`] for the translation unit
157    /// we're lowering.
158    ///
159    /// [`TranslationUnit::expressions`]: ast::TranslationUnit::expressions
160    ast_expressions: &'temp Arena<ast::Expression<'source>>,
161
162    // Naga IR values.
163    /// The map from the names of module-scope declarations to the Naga IR
164    /// `Handle`s we have built for them, owned by `Lowerer::lower`.
165    globals: &'temp mut FastHashMap<&'source str, LoweredGlobalDecl>,
166
167    /// A map from each `ast::Local` handle to the Naga expression
168    /// we've built for it:
169    ///
170    /// - WGSL function arguments become Naga [`FunctionArgument`] expressions.
171    ///
172    /// - WGSL `var` declarations become Naga [`LocalVariable`] expressions.
173    ///
174    /// - WGSL `let` declararations become arbitrary Naga expressions.
175    ///
176    /// This always borrows the `local_table` local variable in
177    /// [`Lowerer::function`].
178    ///
179    /// [`LocalVariable`]: ir::Expression::LocalVariable
180    /// [`FunctionArgument`]: ir::Expression::FunctionArgument
181    local_table:
182        &'temp mut FastHashMap<Handle<ast::Local>, Declared<Typed<Handle<ir::Expression>>>>,
183
184    const_typifier: &'temp mut Typifier,
185    typifier: &'temp mut Typifier,
186    layouter: &'temp mut proc::Layouter,
187    function: &'out mut ir::Function,
188    /// Stores the names of expressions that are assigned in `let` statement
189    /// Also stores the spans of the names, for use in errors.
190    named_expressions: &'out mut FastIndexMap<Handle<ir::Expression>, (String, Span)>,
191    module: &'out mut ir::Module,
192
193    /// Which `Expression`s in `self.naga_expressions` are const expressions, in
194    /// the WGSL sense.
195    ///
196    /// According to the WGSL spec, a const expression must not refer to any
197    /// `let` declarations, even if those declarations' initializers are
198    /// themselves const expressions. So this tracker is not simply concerned
199    /// with the form of the expressions; it is also tracking whether WGSL says
200    /// we should consider them to be const. See the use of `force_non_const` in
201    /// the code for lowering `let` bindings.
202    local_expression_kind_tracker: &'temp mut proc::ExpressionKindTracker,
203    global_expression_kind_tracker: &'temp mut proc::ExpressionKindTracker,
204}
205
206impl<'a, 'temp> StatementContext<'a, 'temp, '_> {
207    const fn as_const<'t>(
208        &'t mut self,
209        block: &'t mut ir::Block,
210        emitter: &'t mut proc::Emitter,
211    ) -> ExpressionContext<'a, 't, 't>
212    where
213        'temp: 't,
214    {
215        ExpressionContext {
216            enable_extensions: self.enable_extensions,
217            globals: self.globals,
218            ast_expressions: self.ast_expressions,
219            const_typifier: self.const_typifier,
220            layouter: self.layouter,
221            global_expression_kind_tracker: self.global_expression_kind_tracker,
222            module: self.module,
223            expr_type: ExpressionContextType::Constant(Some(LocalExpressionContext {
224                local_table: self.local_table,
225                function: self.function,
226                block,
227                emitter,
228                typifier: self.typifier,
229                local_expression_kind_tracker: self.local_expression_kind_tracker,
230            })),
231        }
232    }
233
234    const fn as_expression<'t>(
235        &'t mut self,
236        block: &'t mut ir::Block,
237        emitter: &'t mut proc::Emitter,
238    ) -> ExpressionContext<'a, 't, 't>
239    where
240        'temp: 't,
241    {
242        ExpressionContext {
243            enable_extensions: self.enable_extensions,
244            globals: self.globals,
245            ast_expressions: self.ast_expressions,
246            const_typifier: self.const_typifier,
247            layouter: self.layouter,
248            global_expression_kind_tracker: self.global_expression_kind_tracker,
249            module: self.module,
250            expr_type: ExpressionContextType::Runtime(LocalExpressionContext {
251                local_table: self.local_table,
252                function: self.function,
253                block,
254                emitter,
255                typifier: self.typifier,
256                local_expression_kind_tracker: self.local_expression_kind_tracker,
257            }),
258        }
259    }
260
261    #[allow(dead_code)]
262    const fn as_global(&mut self) -> GlobalContext<'a, '_, '_> {
263        GlobalContext {
264            enable_extensions: self.enable_extensions,
265            ast_expressions: self.ast_expressions,
266            globals: self.globals,
267            module: self.module,
268            const_typifier: self.const_typifier,
269            layouter: self.layouter,
270            global_expression_kind_tracker: self.global_expression_kind_tracker,
271        }
272    }
273
274    fn invalid_assignment_type(&self, expr: Handle<ir::Expression>) -> InvalidAssignmentType {
275        if let Some(&(_, span)) = self.named_expressions.get(&expr) {
276            InvalidAssignmentType::ImmutableBinding(span)
277        } else {
278            match self.function.expressions[expr] {
279                ir::Expression::Swizzle { .. } => InvalidAssignmentType::Swizzle,
280                ir::Expression::Access { base, .. } => self.invalid_assignment_type(base),
281                ir::Expression::AccessIndex { base, .. } => self.invalid_assignment_type(base),
282                _ => InvalidAssignmentType::Other,
283            }
284        }
285    }
286}
287
288pub struct LocalExpressionContext<'temp, 'out> {
289    /// A map from [`ast::Local`] handles to the Naga expressions we've built for them.
290    ///
291    /// This is always [`StatementContext::local_table`] for the
292    /// enclosing statement; see that documentation for details.
293    local_table: &'temp FastHashMap<Handle<ast::Local>, Declared<Typed<Handle<ir::Expression>>>>,
294
295    function: &'out mut ir::Function,
296    block: &'temp mut ir::Block,
297    emitter: &'temp mut proc::Emitter,
298    typifier: &'temp mut Typifier,
299
300    /// Which `Expression`s in `self.naga_expressions` are const expressions, in
301    /// the WGSL sense.
302    ///
303    /// See [`StatementContext::local_expression_kind_tracker`] for details.
304    local_expression_kind_tracker: &'temp mut proc::ExpressionKindTracker,
305}
306
307/// The type of Naga IR expression we are lowering an [`ast::Expression`] to.
308pub enum ExpressionContextType<'temp, 'out> {
309    /// We are lowering to an arbitrary runtime expression, to be
310    /// included in a function's body.
311    ///
312    /// The given [`LocalExpressionContext`] holds information about local
313    /// variables, arguments, and other definitions available only to runtime
314    /// expressions, not constant or override expressions.
315    Runtime(LocalExpressionContext<'temp, 'out>),
316
317    /// We are lowering to a constant expression, to be included in the module's
318    /// constant expression arena.
319    ///
320    /// Everything global constant expressions are allowed to refer to is
321    /// available in the [`ExpressionContext`], but local constant expressions can
322    /// also refer to other
323    Constant(Option<LocalExpressionContext<'temp, 'out>>),
324
325    /// We are lowering to an override expression, to be included in the module's
326    /// constant expression arena.
327    ///
328    /// Everything override expressions are allowed to refer to is
329    /// available in the [`ExpressionContext`], so this variant
330    /// carries no further information.
331    Override,
332}
333
334/// State for lowering an [`ast::Expression`] to Naga IR.
335///
336/// [`ExpressionContext`]s come in two kinds, distinguished by
337/// the value of the [`expr_type`] field:
338///
339/// - A [`Runtime`] context contributes [`naga::Expression`]s to a [`naga::Function`]'s
340///   runtime expression arena.
341///
342/// - A [`Constant`] context contributes [`naga::Expression`]s to a [`naga::Module`]'s
343///   constant expression arena.
344///
345/// [`ExpressionContext`]s are constructed in restricted ways:
346///
347/// - To get a [`Runtime`] [`ExpressionContext`], call
348///   [`StatementContext::as_expression`].
349///
350/// - To get a [`Constant`] [`ExpressionContext`], call
351///   [`GlobalContext::as_const`].
352///
353/// - You can demote a [`Runtime`] context to a [`Constant`] context
354///   by calling [`as_const`], but there's no way to go in the other
355///   direction, producing a runtime context from a constant one. This
356///   is because runtime expressions can refer to constant
357///   expressions, via [`Expression::Constant`], but constant
358///   expressions can't refer to a function's expressions.
359///
360/// Not to be confused with `wgsl::parse::ExpressionContext`, which is
361/// for parsing the `ast::Expression` in the first place.
362///
363/// [`expr_type`]: ExpressionContext::expr_type
364/// [`Runtime`]: ExpressionContextType::Runtime
365/// [`naga::Expression`]: ir::Expression
366/// [`naga::Function`]: ir::Function
367/// [`Constant`]: ExpressionContextType::Constant
368/// [`naga::Module`]: ir::Module
369/// [`as_const`]: ExpressionContext::as_const
370/// [`Expression::Constant`]: ir::Expression::Constant
371pub struct ExpressionContext<'source, 'temp, 'out> {
372    enable_extensions: EnableExtensions,
373
374    // WGSL AST values.
375    ast_expressions: &'temp Arena<ast::Expression<'source>>,
376
377    // Naga IR values.
378    /// The map from the names of module-scope declarations to the Naga IR
379    /// `Handle`s we have built for them, owned by `Lowerer::lower`.
380    globals: &'temp mut FastHashMap<&'source str, LoweredGlobalDecl>,
381
382    /// The IR [`Module`] we're constructing.
383    ///
384    /// [`Module`]: ir::Module
385    module: &'out mut ir::Module,
386
387    /// Type judgments for [`module::global_expressions`].
388    ///
389    /// [`module::global_expressions`]: ir::Module::global_expressions
390    const_typifier: &'temp mut Typifier,
391    layouter: &'temp mut proc::Layouter,
392    global_expression_kind_tracker: &'temp mut proc::ExpressionKindTracker,
393
394    /// Whether we are lowering a constant expression or a general
395    /// runtime expression, and the data needed in each case.
396    expr_type: ExpressionContextType<'temp, 'out>,
397}
398
399impl TypeContext for ExpressionContext<'_, '_, '_> {
400    fn lookup_type(&self, handle: Handle<ir::Type>) -> &ir::Type {
401        &self.module.types[handle]
402    }
403
404    fn type_name(&self, handle: Handle<ir::Type>) -> &str {
405        self.module.types[handle]
406            .name
407            .as_deref()
408            .unwrap_or("{anonymous type}")
409    }
410
411    fn write_override<W: core::fmt::Write>(
412        &self,
413        handle: Handle<ir::Override>,
414        out: &mut W,
415    ) -> core::fmt::Result {
416        match self.module.overrides[handle].name {
417            Some(ref name) => out.write_str(name),
418            None => write!(out, "{{anonymous override {handle:?}}}"),
419        }
420    }
421
422    fn write_unnamed_struct<W: core::fmt::Write>(
423        &self,
424        _: &ir::TypeInner,
425        _: &mut W,
426    ) -> core::fmt::Result {
427        unreachable!("the WGSL front end should always know the type name");
428    }
429}
430
431impl<'source, 'temp, 'out> ExpressionContext<'source, 'temp, 'out> {
432    const fn is_runtime(&self) -> bool {
433        match self.expr_type {
434            ExpressionContextType::Runtime(_) => true,
435            ExpressionContextType::Constant(_) | ExpressionContextType::Override => false,
436        }
437    }
438
439    #[allow(dead_code)]
440    const fn as_const(&mut self) -> ExpressionContext<'source, '_, '_> {
441        ExpressionContext {
442            enable_extensions: self.enable_extensions,
443            globals: self.globals,
444            ast_expressions: self.ast_expressions,
445            const_typifier: self.const_typifier,
446            layouter: self.layouter,
447            module: self.module,
448            expr_type: ExpressionContextType::Constant(match self.expr_type {
449                ExpressionContextType::Runtime(ref mut local_expression_context)
450                | ExpressionContextType::Constant(Some(ref mut local_expression_context)) => {
451                    Some(LocalExpressionContext {
452                        local_table: local_expression_context.local_table,
453                        function: local_expression_context.function,
454                        block: local_expression_context.block,
455                        emitter: local_expression_context.emitter,
456                        typifier: local_expression_context.typifier,
457                        local_expression_kind_tracker: local_expression_context
458                            .local_expression_kind_tracker,
459                    })
460                }
461                ExpressionContextType::Constant(None) | ExpressionContextType::Override => None,
462            }),
463            global_expression_kind_tracker: self.global_expression_kind_tracker,
464        }
465    }
466
467    const fn as_global(&mut self) -> GlobalContext<'source, '_, '_> {
468        GlobalContext {
469            enable_extensions: self.enable_extensions,
470            ast_expressions: self.ast_expressions,
471            globals: self.globals,
472            module: self.module,
473            const_typifier: self.const_typifier,
474            layouter: self.layouter,
475            global_expression_kind_tracker: self.global_expression_kind_tracker,
476        }
477    }
478
479    const fn as_const_evaluator(&mut self) -> proc::ConstantEvaluator<'_> {
480        match self.expr_type {
481            ExpressionContextType::Runtime(ref mut rctx) => {
482                proc::ConstantEvaluator::for_wgsl_function(
483                    self.module,
484                    &mut rctx.function.expressions,
485                    rctx.local_expression_kind_tracker,
486                    self.layouter,
487                    rctx.emitter,
488                    rctx.block,
489                    false,
490                )
491            }
492            ExpressionContextType::Constant(Some(ref mut rctx)) => {
493                proc::ConstantEvaluator::for_wgsl_function(
494                    self.module,
495                    &mut rctx.function.expressions,
496                    rctx.local_expression_kind_tracker,
497                    self.layouter,
498                    rctx.emitter,
499                    rctx.block,
500                    true,
501                )
502            }
503            ExpressionContextType::Constant(None) => proc::ConstantEvaluator::for_wgsl_module(
504                self.module,
505                self.global_expression_kind_tracker,
506                self.layouter,
507                false,
508            ),
509            ExpressionContextType::Override => proc::ConstantEvaluator::for_wgsl_module(
510                self.module,
511                self.global_expression_kind_tracker,
512                self.layouter,
513                true,
514            ),
515        }
516    }
517
518    /// Return a wrapper around `value` suitable for formatting.
519    ///
520    /// Return a wrapper around `value` that implements
521    /// [`core::fmt::Display`] in a form suitable for use in
522    /// diagnostic messages.
523    const fn as_diagnostic_display<T>(
524        &self,
525        value: T,
526    ) -> crate::common::DiagnosticDisplay<(T, proc::GlobalCtx<'_>)> {
527        let ctx = self.module.to_ctx();
528        crate::common::DiagnosticDisplay((value, ctx))
529    }
530
531    fn append_expression(
532        &mut self,
533        expr: ir::Expression,
534        span: Span,
535    ) -> Result<'source, Handle<ir::Expression>> {
536        let mut eval = self.as_const_evaluator();
537        eval.try_eval_and_append(expr, span)
538            .map_err(|e| Box::new(Error::ConstantEvaluatorError(e.into(), span)))
539    }
540
541    fn get_const_val<T: TryFrom<crate::Literal, Error = proc::ConstValueError>>(
542        &self,
543        handle: Handle<ir::Expression>,
544    ) -> core::result::Result<T, proc::ConstValueError> {
545        match self.expr_type {
546            ExpressionContextType::Runtime(ref ctx) => {
547                if !ctx.local_expression_kind_tracker.is_const(handle) {
548                    return Err(proc::ConstValueError::NonConst);
549                }
550
551                self.module
552                    .to_ctx()
553                    .get_const_val_from(handle, &ctx.function.expressions)
554            }
555            ExpressionContextType::Constant(Some(ref ctx)) => {
556                assert!(ctx.local_expression_kind_tracker.is_const(handle));
557                self.module
558                    .to_ctx()
559                    .get_const_val_from(handle, &ctx.function.expressions)
560            }
561            ExpressionContextType::Constant(None) => self.module.to_ctx().get_const_val(handle),
562            ExpressionContextType::Override => Err(proc::ConstValueError::NonConst),
563        }
564    }
565
566    /// Return `true` if `handle` is a constant expression.
567    fn is_const(&self, handle: Handle<ir::Expression>) -> bool {
568        use ExpressionContextType as Ect;
569        match self.expr_type {
570            Ect::Runtime(ref ctx) | Ect::Constant(Some(ref ctx)) => {
571                ctx.local_expression_kind_tracker.is_const(handle)
572            }
573            Ect::Constant(None) | Ect::Override => {
574                self.global_expression_kind_tracker.is_const(handle)
575            }
576        }
577    }
578
579    fn get_expression_span(&self, handle: Handle<ir::Expression>) -> Span {
580        match self.expr_type {
581            ExpressionContextType::Runtime(ref ctx)
582            | ExpressionContextType::Constant(Some(ref ctx)) => {
583                ctx.function.expressions.get_span(handle)
584            }
585            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {
586                self.module.global_expressions.get_span(handle)
587            }
588        }
589    }
590
591    const fn typifier(&self) -> &Typifier {
592        match self.expr_type {
593            ExpressionContextType::Runtime(ref ctx)
594            | ExpressionContextType::Constant(Some(ref ctx)) => ctx.typifier,
595            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {
596                self.const_typifier
597            }
598        }
599    }
600
601    fn get(&self, handle: Handle<crate::Expression>) -> &crate::Expression {
602        match self.expr_type {
603            ExpressionContextType::Runtime(ref ctx)
604            | ExpressionContextType::Constant(Some(ref ctx)) => &ctx.function.expressions[handle],
605            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {
606                &self.module.global_expressions[handle]
607            }
608        }
609    }
610
611    fn local(
612        &mut self,
613        local: &Handle<ast::Local>,
614        span: Span,
615    ) -> Result<'source, Typed<Handle<ir::Expression>>> {
616        match self.expr_type {
617            ExpressionContextType::Runtime(ref ctx) => Ok(ctx.local_table[local].runtime()),
618            ExpressionContextType::Constant(Some(ref ctx)) => ctx.local_table[local]
619                .const_time()
620                .ok_or(Box::new(Error::UnexpectedOperationInConstContext(span))),
621            _ => Err(Box::new(Error::UnexpectedOperationInConstContext(span))),
622        }
623    }
624
625    fn runtime_expression_ctx(
626        &mut self,
627        span: Span,
628    ) -> Result<'source, &mut LocalExpressionContext<'temp, 'out>> {
629        match self.expr_type {
630            ExpressionContextType::Runtime(ref mut ctx) => Ok(ctx),
631            ExpressionContextType::Constant(_) | ExpressionContextType::Override => {
632                Err(Box::new(Error::UnexpectedOperationInConstContext(span)))
633            }
634        }
635    }
636
637    fn with_nested_runtime_expression_ctx<'a, F, T>(
638        &mut self,
639        span: Span,
640        f: F,
641    ) -> Result<'source, (T, crate::Block)>
642    where
643        for<'t> F: FnOnce(&mut ExpressionContext<'source, 't, 't>) -> Result<'source, T>,
644    {
645        let mut block = crate::Block::new();
646        let rctx = match self.expr_type {
647            ExpressionContextType::Runtime(ref mut rctx) => Ok(rctx),
648            ExpressionContextType::Constant(_) | ExpressionContextType::Override => {
649                Err(Error::UnexpectedOperationInConstContext(span))
650            }
651        }?;
652
653        rctx.block
654            .extend(rctx.emitter.finish(&rctx.function.expressions));
655        rctx.emitter.start(&rctx.function.expressions);
656
657        let nested_rctx = LocalExpressionContext {
658            local_table: rctx.local_table,
659            function: rctx.function,
660            block: &mut block,
661            emitter: rctx.emitter,
662            typifier: rctx.typifier,
663            local_expression_kind_tracker: rctx.local_expression_kind_tracker,
664        };
665        let mut nested_ctx = ExpressionContext {
666            enable_extensions: self.enable_extensions,
667            expr_type: ExpressionContextType::Runtime(nested_rctx),
668            ast_expressions: self.ast_expressions,
669            globals: self.globals,
670            module: self.module,
671            const_typifier: self.const_typifier,
672            layouter: self.layouter,
673            global_expression_kind_tracker: self.global_expression_kind_tracker,
674        };
675        let ret = f(&mut nested_ctx)?;
676
677        block.extend(rctx.emitter.finish(&rctx.function.expressions));
678        rctx.emitter.start(&rctx.function.expressions);
679
680        Ok((ret, block))
681    }
682
683    fn gather_component(
684        &mut self,
685        expr: Handle<ir::Expression>,
686        component_span: Span,
687        gather_span: Span,
688    ) -> Result<'source, ir::SwizzleComponent> {
689        match self.expr_type {
690            ExpressionContextType::Runtime(ref rctx) => {
691                if !rctx.local_expression_kind_tracker.is_const(expr) {
692                    return Err(Box::new(Error::ExpectedConstExprConcreteIntegerScalar(
693                        component_span,
694                    )));
695                }
696
697                let index = self
698                    .module
699                    .to_ctx()
700                    .get_const_val_from::<u32, _>(expr, &rctx.function.expressions)
701                    .map_err(|err| match err {
702                        proc::ConstValueError::NonConst | proc::ConstValueError::InvalidType => {
703                            Error::ExpectedConstExprConcreteIntegerScalar(component_span)
704                        }
705                        proc::ConstValueError::Negative => {
706                            Error::ExpectedNonNegative(component_span)
707                        }
708                    })?;
709                ir::SwizzleComponent::XYZW
710                    .get(index as usize)
711                    .copied()
712                    .ok_or(Box::new(Error::InvalidGatherComponent(component_span)))
713            }
714            // This means a `gather` operation appeared in a constant expression.
715            // This error refers to the `gather` itself, not its "component" argument.
716            ExpressionContextType::Constant(_) | ExpressionContextType::Override => Err(Box::new(
717                Error::UnexpectedOperationInConstContext(gather_span),
718            )),
719        }
720    }
721
722    /// Determine the type of `handle`, and add it to the module's arena.
723    ///
724    /// If you just need a `TypeInner` for `handle`'s type, use the
725    /// [`resolve_inner!`] macro instead. This function
726    /// should only be used when the type of `handle` needs to appear
727    /// in the module's final `Arena<Type>`, for example, if you're
728    /// creating a [`LocalVariable`] whose type is inferred from its
729    /// initializer.
730    ///
731    /// [`LocalVariable`]: ir::LocalVariable
732    fn register_type(
733        &mut self,
734        handle: Handle<ir::Expression>,
735    ) -> Result<'source, Handle<ir::Type>> {
736        self.grow_types(handle)?;
737        // This is equivalent to calling ExpressionContext::typifier(),
738        // except that this lets the borrow checker see that it's okay
739        // to also borrow self.module.types mutably below.
740        let typifier = match self.expr_type {
741            ExpressionContextType::Runtime(ref ctx)
742            | ExpressionContextType::Constant(Some(ref ctx)) => ctx.typifier,
743            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {
744                &*self.const_typifier
745            }
746        };
747        Ok(typifier.register_type(handle, &mut self.module.types))
748    }
749
750    /// Resolve the types of all expressions up through `handle`.
751    ///
752    /// Ensure that [`self.typifier`] has a [`TypeResolution`] for
753    /// every expression in `self.function.expressions`.
754    ///
755    /// This does not add types to any arena. The [`Typifier`]
756    /// documentation explains the steps we take to avoid filling
757    /// arenas with intermediate types.
758    ///
759    /// This function takes `&mut self`, so it can't conveniently
760    /// return a shared reference to the resulting `TypeResolution`:
761    /// the shared reference would extend the mutable borrow, and you
762    /// wouldn't be able to use `self` for anything else. Instead, you
763    /// should use [`register_type`] or one of [`resolve!`],
764    /// [`resolve_inner!`] or [`resolve_inner_binary!`].
765    ///
766    /// [`self.typifier`]: ExpressionContext::typifier
767    /// [`TypeResolution`]: proc::TypeResolution
768    /// [`register_type`]: Self::register_type
769    /// [`Typifier`]: Typifier
770    fn grow_types(&mut self, handle: Handle<ir::Expression>) -> Result<'source, &mut Self> {
771        let empty_arena = Arena::new();
772        let resolve_ctx;
773        let typifier;
774        let expressions;
775        match self.expr_type {
776            ExpressionContextType::Runtime(ref mut ctx)
777            | ExpressionContextType::Constant(Some(ref mut ctx)) => {
778                resolve_ctx = proc::ResolveContext::with_locals(
779                    self.module,
780                    &ctx.function.local_variables,
781                    &ctx.function.arguments,
782                );
783                typifier = &mut *ctx.typifier;
784                expressions = &ctx.function.expressions;
785            }
786            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {
787                resolve_ctx = proc::ResolveContext::with_locals(self.module, &empty_arena, &[]);
788                typifier = self.const_typifier;
789                expressions = &self.module.global_expressions;
790            }
791        };
792        typifier
793            .grow(handle, expressions, &resolve_ctx)
794            .map_err(Error::InvalidResolve)?;
795
796        Ok(self)
797    }
798
799    fn image_data(
800        &mut self,
801        image: Handle<ir::Expression>,
802        span: Span,
803    ) -> Result<'source, (ir::ImageClass, bool)> {
804        match *resolve_inner!(self, image) {
805            ir::TypeInner::Image { class, arrayed, .. } => Ok((class, arrayed)),
806            _ => Err(Box::new(Error::BadTexture(span))),
807        }
808    }
809
810    fn prepare_args<'b>(
811        &mut self,
812        args: &'b [Handle<ast::Expression<'source>>],
813        min_args: u32,
814        span: Span,
815    ) -> ArgumentContext<'b, 'source> {
816        ArgumentContext {
817            args: args.iter(),
818            min_args,
819            args_used: 0,
820            total_args: args.len() as u32,
821            span,
822        }
823    }
824
825    /// Insert splats, if needed by the non-'*' operations.
826    ///
827    /// See the "Binary arithmetic expressions with mixed scalar and vector operands"
828    /// table in the WebGPU Shading Language specification for relevant operators.
829    ///
830    /// Multiply is not handled here as backends are expected to handle vec*scalar
831    /// operations, so inserting splats into the IR increases size needlessly.
832    fn binary_op_splat(
833        &mut self,
834        op: ir::BinaryOperator,
835        left: &mut Handle<ir::Expression>,
836        right: &mut Handle<ir::Expression>,
837    ) -> Result<'source, ()> {
838        if matches!(
839            op,
840            ir::BinaryOperator::Add
841                | ir::BinaryOperator::Subtract
842                | ir::BinaryOperator::Divide
843                | ir::BinaryOperator::Modulo
844        ) {
845            match resolve_inner_binary!(self, *left, *right) {
846                (&ir::TypeInner::Vector { size, .. }, &ir::TypeInner::Scalar { .. }) => {
847                    *right = self.append_expression(
848                        ir::Expression::Splat {
849                            size,
850                            value: *right,
851                        },
852                        self.get_expression_span(*right),
853                    )?;
854                }
855                (&ir::TypeInner::Scalar { .. }, &ir::TypeInner::Vector { size, .. }) => {
856                    *left = self.append_expression(
857                        ir::Expression::Splat { size, value: *left },
858                        self.get_expression_span(*left),
859                    )?;
860                }
861                _ => {}
862            }
863        }
864
865        Ok(())
866    }
867
868    /// Add a single expression to the expression table that is not covered by `self.emitter`.
869    ///
870    /// This is useful for `CallResult` and `AtomicResult` expressions, which should not be covered by
871    /// `Emit` statements.
872    fn interrupt_emitter(
873        &mut self,
874        expression: ir::Expression,
875        span: Span,
876    ) -> Result<'source, Handle<ir::Expression>> {
877        match self.expr_type {
878            ExpressionContextType::Runtime(ref mut rctx)
879            | ExpressionContextType::Constant(Some(ref mut rctx)) => {
880                rctx.block
881                    .extend(rctx.emitter.finish(&rctx.function.expressions));
882            }
883            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {}
884        }
885        let result = self.append_expression(expression, span);
886        match self.expr_type {
887            ExpressionContextType::Runtime(ref mut rctx)
888            | ExpressionContextType::Constant(Some(ref mut rctx)) => {
889                rctx.emitter.start(&rctx.function.expressions);
890            }
891            ExpressionContextType::Constant(None) | ExpressionContextType::Override => {}
892        }
893        result
894    }
895
896    /// Apply the WGSL Load Rule to `expr`.
897    ///
898    /// If `expr` is has type `ref<SC, T, A>`, perform a load to produce a value of type
899    /// `T`. Otherwise, return `expr` unchanged.
900    fn apply_load_rule(
901        &mut self,
902        expr: Typed<Handle<ir::Expression>>,
903    ) -> Result<'source, Handle<ir::Expression>> {
904        match expr {
905            Typed::Reference(pointer) => {
906                let span = self.get_expression_span(pointer);
907
908                // Reject direct access to atomic variables that does not go
909                // through a built-in function.
910                if let ir::TypeInner::Pointer { base, .. } = *resolve_inner!(self, pointer) {
911                    fn type_contains_atomic(
912                        ty: Handle<ir::Type>,
913                        types: &crate::UniqueArena<ir::Type>,
914                    ) -> bool {
915                        match types[ty].inner {
916                            ir::TypeInner::Atomic(_) => true,
917                            ir::TypeInner::Array { base, .. } => type_contains_atomic(base, types),
918                            ir::TypeInner::Struct { ref members, .. } => members
919                                .iter()
920                                .any(|member| type_contains_atomic(member.ty, types)),
921                            _ => false,
922                        }
923                    }
924
925                    if type_contains_atomic(base, &self.module.types) {
926                        return Err(Box::new(Error::InvalidAtomicAccess(span)));
927                    }
928                }
929
930                let load = ir::Expression::Load { pointer };
931                self.append_expression(load, span)
932            }
933            Typed::Plain(handle) => Ok(handle),
934        }
935    }
936
937    fn ensure_type_exists(&mut self, inner: ir::TypeInner) -> Handle<ir::Type> {
938        self.as_global().ensure_type_exists(None, inner)
939    }
940
941    /// Check that `expr` is an identifier resolving to a predeclared enumerant.
942    ///
943    /// The identifier must not have any template parameters.
944    ///
945    /// Return the name of the identifier, together with its span.
946    ///
947    /// Actually, this only checks that the identifier refers to some
948    /// predeclared object, not necessarily an enumerant. This should be good
949    /// enough, since the caller is going to compare the name against some list
950    /// of permitted enumerants anyway.
951    fn enumerant(
952        &self,
953        expr: Handle<ast::Expression<'source>>,
954    ) -> Result<'source, (&'source str, Span)> {
955        let span = self.ast_expressions.get_span(expr);
956        let expr = &self.ast_expressions[expr];
957
958        let ast::Expression::Ident(ref ident) = *expr else {
959            return Err(Box::new(Error::UnexpectedExprForEnumerant(span)));
960        };
961
962        let ast::TemplateElaboratedIdent {
963            ident: ast::IdentExpr::Unresolved(name),
964            ref template_list,
965            ..
966        } = *ident
967        else {
968            return Err(Box::new(Error::UnexpectedIdentForEnumerant(span)));
969        };
970
971        if self.globals.get(name).is_some() {
972            return Err(Box::new(Error::UnexpectedIdentForEnumerant(span)));
973        }
974
975        if !template_list.is_empty() {
976            return Err(Box::new(Error::UnexpectedTemplate(span)));
977        }
978
979        Ok((name, span))
980    }
981
982    fn var_address_space(
983        &self,
984        template_list: &[Handle<ast::Expression<'source>>],
985    ) -> Result<'source, ir::AddressSpace> {
986        let mut tl = TemplateListIter::new(Span::UNDEFINED, template_list);
987        let mut address_space = tl.maybe_address_space(self)?;
988        if let Some(ref mut address_space) = address_space {
989            tl.maybe_access_mode(address_space, self)?;
990        }
991        tl.finish(self)?;
992        Ok(address_space.unwrap_or(ir::AddressSpace::Handle))
993    }
994}
995
996struct ArgumentContext<'ctx, 'source> {
997    args: core::slice::Iter<'ctx, Handle<ast::Expression<'source>>>,
998    min_args: u32,
999    args_used: u32,
1000    total_args: u32,
1001    span: Span,
1002}
1003
1004impl<'source> ArgumentContext<'_, 'source> {
1005    pub fn finish(self) -> Result<'source, ()> {
1006        if self.args.len() == 0 {
1007            Ok(())
1008        } else {
1009            Err(Box::new(Error::WrongArgumentCount {
1010                found: self.total_args,
1011                expected: self.min_args..self.args_used + 1,
1012                span: self.span,
1013            }))
1014        }
1015    }
1016
1017    pub fn next(&mut self) -> Result<'source, Handle<ast::Expression<'source>>> {
1018        match self.args.next().copied() {
1019            Some(arg) => {
1020                self.args_used += 1;
1021                Ok(arg)
1022            }
1023            None => Err(Box::new(Error::WrongArgumentCount {
1024                found: self.total_args,
1025                expected: self.min_args..self.args_used + 1,
1026                span: self.span,
1027            })),
1028        }
1029    }
1030}
1031
1032#[derive(Debug, Copy, Clone)]
1033enum Declared<T> {
1034    /// Value declared as const
1035    Const(T),
1036
1037    /// Value declared as non-const
1038    Runtime(T),
1039}
1040
1041impl<T> Declared<T> {
1042    fn runtime(self) -> T {
1043        match self {
1044            Declared::Const(t) | Declared::Runtime(t) => t,
1045        }
1046    }
1047
1048    fn const_time(self) -> Option<T> {
1049        match self {
1050            Declared::Const(t) => Some(t),
1051            Declared::Runtime(_) => None,
1052        }
1053    }
1054}
1055
1056/// WGSL type annotations on expressions, types, values, etc.
1057///
1058/// Naga and WGSL types are very close, but Naga lacks WGSL's `ref` types, which
1059/// we need to know to apply the Load Rule. This enum carries some WGSL or Naga
1060/// datum along with enough information to determine its corresponding WGSL
1061/// type.
1062///
1063/// The `T` type parameter can be any expression-like thing:
1064///
1065/// - `Typed<Handle<ir::Type>>` can represent a full WGSL type. For example,
1066///   given some Naga `Pointer` type `ptr`, a WGSL reference type is a
1067///   `Typed::Reference(ptr)` whereas a WGSL pointer type is a
1068///   `Typed::Plain(ptr)`.
1069///
1070/// - `Typed<ir::Expression>` or `Typed<Handle<ir::Expression>>` can
1071///   represent references similarly.
1072///
1073/// Use the `map` and `try_map` methods to convert from one expression
1074/// representation to another.
1075///
1076/// [`Expression`]: ir::Expression
1077#[derive(Debug, Copy, Clone)]
1078enum Typed<T> {
1079    /// A WGSL reference.
1080    Reference(T),
1081
1082    /// A WGSL plain type.
1083    Plain(T),
1084}
1085
1086impl<T> Typed<T> {
1087    fn map<U>(self, mut f: impl FnMut(T) -> U) -> Typed<U> {
1088        match self {
1089            Self::Reference(v) => Typed::Reference(f(v)),
1090            Self::Plain(v) => Typed::Plain(f(v)),
1091        }
1092    }
1093
1094    fn try_map<U, E>(
1095        self,
1096        mut f: impl FnMut(T) -> core::result::Result<U, E>,
1097    ) -> core::result::Result<Typed<U>, E> {
1098        Ok(match self {
1099            Self::Reference(expr) => Typed::Reference(f(expr)?),
1100            Self::Plain(expr) => Typed::Plain(f(expr)?),
1101        })
1102    }
1103
1104    fn ref_or<E>(self, error: E) -> core::result::Result<T, E> {
1105        match self {
1106            Self::Reference(v) => Ok(v),
1107            Self::Plain(_) => Err(error),
1108        }
1109    }
1110}
1111
1112/// A single vector component or swizzle.
1113///
1114/// This represents the things that can appear after the `.` in a vector access
1115/// expression: either a single component name, or a series of them,
1116/// representing a swizzle.
1117enum Components {
1118    Single(u32),
1119    Swizzle {
1120        size: ir::VectorSize,
1121        pattern: [ir::SwizzleComponent; 4],
1122    },
1123}
1124
1125impl Components {
1126    const fn letter_component(letter: char) -> Option<ir::SwizzleComponent> {
1127        use ir::SwizzleComponent as Sc;
1128        match letter {
1129            'x' | 'r' => Some(Sc::X),
1130            'y' | 'g' => Some(Sc::Y),
1131            'z' | 'b' => Some(Sc::Z),
1132            'w' | 'a' => Some(Sc::W),
1133            _ => None,
1134        }
1135    }
1136
1137    fn single_component(name: &str, name_span: Span) -> Result<'_, u32> {
1138        let ch = name.chars().next().ok_or(Error::BadAccessor(name_span))?;
1139        match Self::letter_component(ch) {
1140            Some(sc) => Ok(sc as u32),
1141            None => Err(Box::new(Error::BadAccessor(name_span))),
1142        }
1143    }
1144
1145    /// Construct a `Components` value from a 'member' name, like `"wzy"` or `"x"`.
1146    ///
1147    /// Use `name_span` for reporting errors in parsing the component string.
1148    fn new(name: &str, name_span: Span) -> Result<'_, Self> {
1149        let size = match name.len() {
1150            1 => return Ok(Components::Single(Self::single_component(name, name_span)?)),
1151            2 => ir::VectorSize::Bi,
1152            3 => ir::VectorSize::Tri,
1153            4 => ir::VectorSize::Quad,
1154            _ => return Err(Box::new(Error::BadAccessor(name_span))),
1155        };
1156
1157        let mut pattern = [ir::SwizzleComponent::X; 4];
1158        for (comp, ch) in pattern.iter_mut().zip(name.chars()) {
1159            *comp = Self::letter_component(ch).ok_or(Error::BadAccessor(name_span))?;
1160        }
1161
1162        if name.chars().all(|c| matches!(c, 'x' | 'y' | 'z' | 'w'))
1163            || name.chars().all(|c| matches!(c, 'r' | 'g' | 'b' | 'a'))
1164        {
1165            Ok(Components::Swizzle { size, pattern })
1166        } else {
1167            Err(Box::new(Error::BadAccessor(name_span)))
1168        }
1169    }
1170}
1171
1172/// An `ast::GlobalDecl` for which we have built the Naga IR equivalent.
1173enum LoweredGlobalDecl {
1174    Function {
1175        handle: Handle<ir::Function>,
1176        must_use: bool,
1177    },
1178    Var(Handle<ir::GlobalVariable>),
1179    Const(Handle<ir::Constant>),
1180    Override(Handle<ir::Override>),
1181    Type(Handle<ir::Type>),
1182    EntryPoint(usize),
1183}
1184
1185enum Texture {
1186    Gather,
1187    GatherCompare,
1188
1189    Sample,
1190    SampleBias,
1191    SampleCompare,
1192    SampleCompareLevel,
1193    SampleGrad,
1194    SampleLevel,
1195    SampleBaseClampToEdge,
1196}
1197
1198impl Texture {
1199    pub fn map(word: &str) -> Option<Self> {
1200        Some(match word {
1201            "textureGather" => Self::Gather,
1202            "textureGatherCompare" => Self::GatherCompare,
1203
1204            "textureSample" => Self::Sample,
1205            "textureSampleBias" => Self::SampleBias,
1206            "textureSampleCompare" => Self::SampleCompare,
1207            "textureSampleCompareLevel" => Self::SampleCompareLevel,
1208            "textureSampleGrad" => Self::SampleGrad,
1209            "textureSampleLevel" => Self::SampleLevel,
1210            "textureSampleBaseClampToEdge" => Self::SampleBaseClampToEdge,
1211            _ => return None,
1212        })
1213    }
1214
1215    pub const fn min_argument_count(&self) -> u32 {
1216        match *self {
1217            Self::Gather => 3,
1218            Self::GatherCompare => 4,
1219
1220            Self::Sample => 3,
1221            Self::SampleBias => 5,
1222            Self::SampleCompare => 5,
1223            Self::SampleCompareLevel => 5,
1224            Self::SampleGrad => 6,
1225            Self::SampleLevel => 5,
1226            Self::SampleBaseClampToEdge => 3,
1227        }
1228    }
1229}
1230
1231enum SubgroupGather {
1232    BroadcastFirst,
1233    Broadcast,
1234    Shuffle,
1235    ShuffleDown,
1236    ShuffleUp,
1237    ShuffleXor,
1238    QuadBroadcast,
1239}
1240
1241impl SubgroupGather {
1242    pub fn map(word: &str) -> Option<Self> {
1243        Some(match word {
1244            "subgroupBroadcastFirst" => Self::BroadcastFirst,
1245            "subgroupBroadcast" => Self::Broadcast,
1246            "subgroupShuffle" => Self::Shuffle,
1247            "subgroupShuffleDown" => Self::ShuffleDown,
1248            "subgroupShuffleUp" => Self::ShuffleUp,
1249            "subgroupShuffleXor" => Self::ShuffleXor,
1250            "quadBroadcast" => Self::QuadBroadcast,
1251            _ => return None,
1252        })
1253    }
1254}
1255
1256/// Whether a declaration accepts abstract types, or concretizes.
1257enum AbstractRule {
1258    /// This declaration concretizes its initialization expression.
1259    Concretize,
1260
1261    /// This declaration can accept initializers with abstract types.
1262    Allow,
1263}
1264
1265/// Whether `@must_use` applies to a call expression.
1266#[derive(Debug, Copy, Clone)]
1267enum MustUse {
1268    Yes,
1269    No,
1270}
1271
1272impl From<bool> for MustUse {
1273    fn from(value: bool) -> Self {
1274        if value {
1275            MustUse::Yes
1276        } else {
1277            MustUse::No
1278        }
1279    }
1280}
1281
1282pub struct Lowerer<'source, 'temp> {
1283    index: &'temp Index<'source>,
1284}
1285
1286impl<'source, 'temp> Lowerer<'source, 'temp> {
1287    pub const fn new(index: &'temp Index<'source>) -> Self {
1288        Self { index }
1289    }
1290
1291    pub fn lower(&mut self, tu: ast::TranslationUnit<'source>) -> Result<'source, ir::Module> {
1292        let mut module = ir::Module {
1293            diagnostic_filters: tu.diagnostic_filters,
1294            diagnostic_filter_leaf: tu.diagnostic_filter_leaf,
1295            ..Default::default()
1296        };
1297
1298        let mut ctx = GlobalContext {
1299            enable_extensions: tu.enable_extensions,
1300            ast_expressions: &tu.expressions,
1301            globals: &mut FastHashMap::default(),
1302            module: &mut module,
1303            const_typifier: &mut Typifier::new(),
1304            layouter: &mut proc::Layouter::default(),
1305            global_expression_kind_tracker: &mut proc::ExpressionKindTracker::new(),
1306        };
1307        if !tu.doc_comments.is_empty() {
1308            ctx.module.get_or_insert_default_doc_comments().module =
1309                tu.doc_comments.iter().map(|s| s.to_string()).collect();
1310        }
1311
1312        for decl_handle in self.index.visit_ordered() {
1313            let span = tu.decls.get_span(decl_handle);
1314            let decl = &tu.decls[decl_handle];
1315
1316            match decl.kind {
1317                ast::GlobalDeclKind::Fn(ref f) => {
1318                    let lowered_decl = self.function(f, span, &mut ctx)?;
1319                    if !f.doc_comments.is_empty() {
1320                        match lowered_decl {
1321                            LoweredGlobalDecl::Function { handle, .. } => {
1322                                ctx.module
1323                                    .get_or_insert_default_doc_comments()
1324                                    .functions
1325                                    .insert(
1326                                        handle,
1327                                        f.doc_comments.iter().map(|s| s.to_string()).collect(),
1328                                    );
1329                            }
1330                            LoweredGlobalDecl::EntryPoint(index) => {
1331                                ctx.module
1332                                    .get_or_insert_default_doc_comments()
1333                                    .entry_points
1334                                    .insert(
1335                                        index,
1336                                        f.doc_comments.iter().map(|s| s.to_string()).collect(),
1337                                    );
1338                            }
1339                            _ => {}
1340                        }
1341                    }
1342                    ctx.globals.insert(f.name.name, lowered_decl);
1343                }
1344                ast::GlobalDeclKind::Var(ref v) => {
1345                    let explicit_ty =
1346                        v.ty.as_ref()
1347                            .map(|ast| self.resolve_ast_type(ast, &mut ctx.as_const()))
1348                            .transpose()?;
1349
1350                    let (ty, initializer) = self.type_and_init(
1351                        v.name,
1352                        v.init,
1353                        explicit_ty,
1354                        AbstractRule::Concretize,
1355                        &mut ctx.as_override(),
1356                    )?;
1357
1358                    let binding = if let Some(ref binding) = v.binding {
1359                        Some(ir::ResourceBinding {
1360                            group: self.const_u32(binding.group, &mut ctx.as_const())?.0,
1361                            binding: self.const_u32(binding.binding, &mut ctx.as_const())?.0,
1362                        })
1363                    } else {
1364                        None
1365                    };
1366
1367                    let space = ctx.as_const().var_address_space(&v.template_list)?;
1368
1369                    let handle = ctx.module.global_variables.append(
1370                        ir::GlobalVariable {
1371                            name: Some(v.name.name.to_string()),
1372                            space,
1373                            binding,
1374                            ty,
1375                            init: initializer,
1376                            memory_decorations: v.memory_decorations,
1377                        },
1378                        span,
1379                    );
1380
1381                    if !v.doc_comments.is_empty() {
1382                        ctx.module
1383                            .get_or_insert_default_doc_comments()
1384                            .global_variables
1385                            .insert(
1386                                handle,
1387                                v.doc_comments.iter().map(|s| s.to_string()).collect(),
1388                            );
1389                    }
1390                    ctx.globals
1391                        .insert(v.name.name, LoweredGlobalDecl::Var(handle));
1392                }
1393                ast::GlobalDeclKind::Const(ref c) => {
1394                    let mut ectx = ctx.as_const();
1395
1396                    let explicit_ty =
1397                        c.ty.as_ref()
1398                            .map(|ast| self.resolve_ast_type(ast, &mut ectx))
1399                            .transpose()?;
1400
1401                    let (ty, init) = self.type_and_init(
1402                        c.name,
1403                        Some(c.init),
1404                        explicit_ty,
1405                        AbstractRule::Allow,
1406                        &mut ectx,
1407                    )?;
1408                    let init = init.expect("Global const must have init");
1409
1410                    let handle = ctx.module.constants.append(
1411                        ir::Constant {
1412                            name: Some(c.name.name.to_string()),
1413                            ty,
1414                            init,
1415                        },
1416                        span,
1417                    );
1418
1419                    ctx.globals
1420                        .insert(c.name.name, LoweredGlobalDecl::Const(handle));
1421                    if !c.doc_comments.is_empty() {
1422                        ctx.module
1423                            .get_or_insert_default_doc_comments()
1424                            .constants
1425                            .insert(
1426                                handle,
1427                                c.doc_comments.iter().map(|s| s.to_string()).collect(),
1428                            );
1429                    }
1430                }
1431                ast::GlobalDeclKind::Override(ref o) => {
1432                    let explicit_ty =
1433                        o.ty.as_ref()
1434                            .map(|ast| self.resolve_ast_type(ast, &mut ctx.as_const()))
1435                            .transpose()?;
1436
1437                    let mut ectx = ctx.as_override();
1438
1439                    let (ty, init) = self.type_and_init(
1440                        o.name,
1441                        o.init,
1442                        explicit_ty,
1443                        AbstractRule::Concretize,
1444                        &mut ectx,
1445                    )?;
1446
1447                    let id =
1448                        o.id.map(|id| self.const_u32(id, &mut ctx.as_const()))
1449                            .transpose()?;
1450
1451                    let id = if let Some((id, id_span)) = id {
1452                        Some(
1453                            u16::try_from(id)
1454                                .map_err(|_| Error::PipelineConstantIDValue(id_span))?,
1455                        )
1456                    } else {
1457                        None
1458                    };
1459
1460                    let handle = ctx.module.overrides.append(
1461                        ir::Override {
1462                            name: Some(o.name.name.to_string()),
1463                            id,
1464                            ty,
1465                            init,
1466                        },
1467                        span,
1468                    );
1469
1470                    ctx.globals
1471                        .insert(o.name.name, LoweredGlobalDecl::Override(handle));
1472                }
1473                ast::GlobalDeclKind::Struct(ref s) => {
1474                    let handle = self.r#struct(s, span, &mut ctx)?;
1475                    ctx.globals
1476                        .insert(s.name.name, LoweredGlobalDecl::Type(handle));
1477                    if !s.doc_comments.is_empty() {
1478                        ctx.module
1479                            .get_or_insert_default_doc_comments()
1480                            .types
1481                            .insert(
1482                                handle,
1483                                s.doc_comments.iter().map(|s| s.to_string()).collect(),
1484                            );
1485                    }
1486                }
1487                ast::GlobalDeclKind::Type(ref alias) => {
1488                    let ty = self.resolve_named_ast_type(
1489                        &alias.ty,
1490                        alias.name.name.to_string(),
1491                        &mut ctx.as_const(),
1492                    )?;
1493                    ctx.globals
1494                        .insert(alias.name.name, LoweredGlobalDecl::Type(ty));
1495                }
1496                ast::GlobalDeclKind::ConstAssert(condition) => {
1497                    let condition = self.expression(condition, &mut ctx.as_const())?;
1498
1499                    let span = ctx.module.global_expressions.get_span(condition);
1500                    match ctx
1501                        .module
1502                        .to_ctx()
1503                        .get_const_val_from(condition, &ctx.module.global_expressions)
1504                    {
1505                        Ok(true) => Ok(()),
1506                        Ok(false) => Err(Error::ConstAssertFailed(span)),
1507                        Err(proc::ConstValueError::Negative) => unreachable!(),
1508                        Err(
1509                            proc::ConstValueError::NonConst | proc::ConstValueError::InvalidType,
1510                        ) => Err(Error::NotBool(span)),
1511                    }?;
1512                }
1513            }
1514        }
1515
1516        // Constant evaluation may leave abstract-typed literals and
1517        // compositions in expression arenas, so we need to compact the module
1518        // to remove unused expressions and types.
1519        crate::compact::compact(&mut module, KeepUnused::Yes);
1520
1521        Ok(module)
1522    }
1523
1524    /// Obtain (inferred) type and initializer after automatic conversion
1525    fn type_and_init(
1526        &mut self,
1527        name: ast::Ident<'source>,
1528        init: Option<Handle<ast::Expression<'source>>>,
1529        explicit_ty: Option<Handle<ir::Type>>,
1530        abstract_rule: AbstractRule,
1531        ectx: &mut ExpressionContext<'source, '_, '_>,
1532    ) -> Result<'source, (Handle<ir::Type>, Option<Handle<ir::Expression>>)> {
1533        let ty;
1534        let initializer;
1535        match (init, explicit_ty) {
1536            (Some(init), Some(explicit_ty)) => {
1537                let init = self.expression_for_abstract(init, ectx)?;
1538                let ty_res = proc::TypeResolution::Handle(explicit_ty);
1539                let init = ectx
1540                    .try_automatic_conversions(init, &ty_res, name.span)
1541                    .map_err(|error| match *error {
1542                        // Both of these mean the same thing to the reader of a
1543                        // `var`/`let` declaration: the initializer's type isn't
1544                        // the declared one.
1545                        Error::AutoConversion(e) => Box::new(Error::InitializationTypeMismatch {
1546                            name: name.span,
1547                            expected: e.dest_type,
1548                            got: e.source_type,
1549                        }),
1550                        Error::TypeMismatch(e) => Box::new(Error::InitializationTypeMismatch {
1551                            name: name.span,
1552                            expected: e.dest_type,
1553                            got: e.source_type,
1554                        }),
1555                        _ => error,
1556                    })?;
1557
1558                ty = explicit_ty;
1559                initializer = Some(init);
1560            }
1561            (Some(init), None) => {
1562                let mut init = self.expression_for_abstract(init, ectx)?;
1563                if let AbstractRule::Concretize = abstract_rule {
1564                    init = ectx.concretize(init)?;
1565                }
1566                ty = ectx.register_type(init)?;
1567                initializer = Some(init);
1568            }
1569            (None, Some(explicit_ty)) => {
1570                ty = explicit_ty;
1571                initializer = None;
1572            }
1573            (None, None) => return Err(Box::new(Error::DeclMissingTypeAndInit(name.span))),
1574        }
1575        Ok((ty, initializer))
1576    }
1577
1578    fn function(
1579        &mut self,
1580        f: &ast::Function<'source>,
1581        span: Span,
1582        ctx: &mut GlobalContext<'source, '_, '_>,
1583    ) -> Result<'source, LoweredGlobalDecl> {
1584        let mut local_table = FastHashMap::default();
1585        let mut expressions = Arena::new();
1586        let mut named_expressions = FastIndexMap::default();
1587        let mut local_expression_kind_tracker = proc::ExpressionKindTracker::new();
1588
1589        let arguments = f
1590            .arguments
1591            .iter()
1592            .enumerate()
1593            .map(|(i, arg)| -> Result<'_, _> {
1594                let ty = self.resolve_ast_type(&arg.ty, &mut ctx.as_const())?;
1595                let expr =
1596                    expressions.append(ir::Expression::FunctionArgument(i as u32), arg.name.span);
1597                local_table.insert(arg.handle, Declared::Runtime(Typed::Plain(expr)));
1598                named_expressions.insert(expr, (arg.name.name.to_string(), arg.name.span));
1599                local_expression_kind_tracker.insert(expr, proc::ExpressionKind::Runtime);
1600
1601                Ok(ir::FunctionArgument {
1602                    name: Some(arg.name.name.to_string()),
1603                    ty,
1604                    binding: self.binding(&arg.binding, ty, ctx)?,
1605                })
1606            })
1607            .collect::<Result<Vec<_>>>()?;
1608
1609        let result = f
1610            .result
1611            .as_ref()
1612            .map(|res| -> Result<'_, _> {
1613                let ty = self.resolve_ast_type(&res.ty, &mut ctx.as_const())?;
1614                Ok(ir::FunctionResult {
1615                    ty,
1616                    binding: self.binding(&res.binding, ty, ctx)?,
1617                })
1618            })
1619            .transpose()?;
1620
1621        let mut function = ir::Function {
1622            name: Some(f.name.name.to_string()),
1623            arguments,
1624            result,
1625            local_variables: Arena::new(),
1626            expressions,
1627            named_expressions: crate::NamedExpressions::default(),
1628            body: ir::Block::default(),
1629            diagnostic_filter_leaf: f.diagnostic_filter_leaf,
1630        };
1631
1632        let mut typifier = Typifier::default();
1633        let mut stmt_ctx = StatementContext {
1634            enable_extensions: ctx.enable_extensions,
1635            local_table: &mut local_table,
1636            globals: ctx.globals,
1637            ast_expressions: ctx.ast_expressions,
1638            const_typifier: ctx.const_typifier,
1639            typifier: &mut typifier,
1640            layouter: ctx.layouter,
1641            function: &mut function,
1642            named_expressions: &mut named_expressions,
1643            module: ctx.module,
1644            local_expression_kind_tracker: &mut local_expression_kind_tracker,
1645            global_expression_kind_tracker: ctx.global_expression_kind_tracker,
1646        };
1647        let mut body = self.block(&f.body, false, &mut stmt_ctx)?;
1648        proc::ensure_block_returns(&mut body);
1649
1650        function.body = body;
1651        function.named_expressions = named_expressions
1652            .into_iter()
1653            .map(|(key, (name, _))| (key, name))
1654            .collect();
1655
1656        if let Some(ref entry) = f.entry_point {
1657            let (workgroup_size, workgroup_size_overrides) =
1658                if let Some(workgroup_size) = entry.workgroup_size {
1659                    // TODO: replace with try_map once stabilized
1660                    let mut workgroup_size_out = [1; 3];
1661                    let mut workgroup_size_overrides_out = [None; 3];
1662                    for (i, size) in workgroup_size.into_iter().enumerate() {
1663                        if let Some(size_expr) = size {
1664                            match self.const_u32(size_expr, &mut ctx.as_const()) {
1665                                Ok(value) => {
1666                                    workgroup_size_out[i] = value.0;
1667                                }
1668                                Err(err) => {
1669                                    if let Error::ConstantEvaluatorError(ref ty, _) = *err {
1670                                        match **ty {
1671                                            proc::ConstantEvaluatorError::OverrideExpr => {
1672                                                workgroup_size_overrides_out[i] =
1673                                                    Some(self.workgroup_size_override(
1674                                                        size_expr,
1675                                                        &mut ctx.as_override(),
1676                                                    )?);
1677                                            }
1678                                            _ => {
1679                                                return Err(err);
1680                                            }
1681                                        }
1682                                    } else {
1683                                        return Err(err);
1684                                    }
1685                                }
1686                            }
1687                        }
1688                    }
1689                    if workgroup_size_overrides_out.iter().all(|x| x.is_none()) {
1690                        (workgroup_size_out, None)
1691                    } else {
1692                        (workgroup_size_out, Some(workgroup_size_overrides_out))
1693                    }
1694                } else {
1695                    ([0; 3], None)
1696                };
1697
1698            let mesh_info = if let Some((var_name, var_span)) = entry.mesh_output_variable {
1699                let var = match ctx.globals.get(var_name) {
1700                    Some(&LoweredGlobalDecl::Var(handle)) => handle,
1701                    Some(_) => {
1702                        return Err(Box::new(Error::ExpectedGlobalVariable {
1703                            name_span: var_span,
1704                        }))
1705                    }
1706                    None => return Err(Box::new(Error::UnknownIdent(var_span, var_name))),
1707                };
1708
1709                let mut info = ctx.module.analyze_mesh_shader_info(var);
1710                if let Some(h) = info.1[0] {
1711                    info.0.max_vertices_override = Some(
1712                        ctx.module
1713                            .global_expressions
1714                            .append(crate::Expression::Override(h), Span::UNDEFINED),
1715                    );
1716                }
1717                if let Some(h) = info.1[1] {
1718                    info.0.max_primitives_override = Some(
1719                        ctx.module
1720                            .global_expressions
1721                            .append(crate::Expression::Override(h), Span::UNDEFINED),
1722                    );
1723                }
1724
1725                Some(info.0)
1726            } else {
1727                None
1728            };
1729
1730            let task_payload = if let Some((var_name, var_span)) = entry.task_payload {
1731                Some(match ctx.globals.get(var_name) {
1732                    Some(&LoweredGlobalDecl::Var(handle)) => handle,
1733                    Some(_) => {
1734                        return Err(Box::new(Error::ExpectedGlobalVariable {
1735                            name_span: var_span,
1736                        }))
1737                    }
1738                    None => return Err(Box::new(Error::UnknownIdent(var_span, var_name))),
1739                })
1740            } else {
1741                None
1742            };
1743
1744            let incoming_ray_payload =
1745                if let Some((var_name, var_span)) = entry.ray_incoming_payload {
1746                    Some(match ctx.globals.get(var_name) {
1747                        Some(&LoweredGlobalDecl::Var(handle)) => handle,
1748                        Some(_) => {
1749                            return Err(Box::new(Error::ExpectedGlobalVariable {
1750                                name_span: var_span,
1751                            }))
1752                        }
1753                        None => return Err(Box::new(Error::UnknownIdent(var_span, var_name))),
1754                    })
1755                } else {
1756                    None
1757                };
1758
1759            ctx.module.entry_points.push(ir::EntryPoint {
1760                name: f.name.name.to_string(),
1761                stage: entry.stage,
1762                early_depth_test: entry.early_depth_test,
1763                workgroup_size,
1764                workgroup_size_overrides,
1765                function,
1766                mesh_info,
1767                task_payload,
1768                incoming_ray_payload,
1769            });
1770            Ok(LoweredGlobalDecl::EntryPoint(
1771                ctx.module.entry_points.len() - 1,
1772            ))
1773        } else {
1774            let handle = ctx.module.functions.append(function, span);
1775            Ok(LoweredGlobalDecl::Function {
1776                handle,
1777                must_use: f.result.as_ref().is_some_and(|res| res.must_use),
1778            })
1779        }
1780    }
1781
1782    fn workgroup_size_override(
1783        &mut self,
1784        size_expr: Handle<ast::Expression<'source>>,
1785        ctx: &mut ExpressionContext<'source, '_, '_>,
1786    ) -> Result<'source, Handle<ir::Expression>> {
1787        let span = ctx.ast_expressions.get_span(size_expr);
1788        let expr = self.expression(size_expr, ctx)?;
1789        match resolve_inner!(ctx, expr).scalar_kind().ok_or(0) {
1790            Ok(ir::ScalarKind::Sint) | Ok(ir::ScalarKind::Uint) => Ok(expr),
1791            _ => Err(Box::new(Error::ExpectedConstExprConcreteIntegerScalar(
1792                span,
1793            ))),
1794        }
1795    }
1796
1797    fn block(
1798        &mut self,
1799        b: &ast::Block<'source>,
1800        is_inside_loop: bool,
1801        ctx: &mut StatementContext<'source, '_, '_>,
1802    ) -> Result<'source, ir::Block> {
1803        let mut block = ir::Block::default();
1804
1805        for stmt in b.stmts.iter() {
1806            self.statement(stmt, &mut block, is_inside_loop, ctx)?;
1807        }
1808
1809        Ok(block)
1810    }
1811
1812    fn statement(
1813        &mut self,
1814        stmt: &ast::Statement<'source>,
1815        block: &mut ir::Block,
1816        is_inside_loop: bool,
1817        ctx: &mut StatementContext<'source, '_, '_>,
1818    ) -> Result<'source, ()> {
1819        let out = match stmt.kind {
1820            ast::StatementKind::Block(ref block) => {
1821                let block = self.block(block, is_inside_loop, ctx)?;
1822                ir::Statement::Block(block)
1823            }
1824            ast::StatementKind::LocalDecl(ref decl) => match *decl {
1825                ast::LocalDecl::Let(ref l) => {
1826                    let mut emitter = proc::Emitter::default();
1827                    emitter.start(&ctx.function.expressions);
1828
1829                    let explicit_ty = l
1830                        .ty
1831                        .as_ref()
1832                        .map(|ty| self.resolve_ast_type(ty, &mut ctx.as_const(block, &mut emitter)))
1833                        .transpose()?;
1834
1835                    let mut ectx = ctx.as_expression(block, &mut emitter);
1836
1837                    let (ty, initializer) = self.type_and_init(
1838                        l.name,
1839                        Some(l.init),
1840                        explicit_ty,
1841                        AbstractRule::Concretize,
1842                        &mut ectx,
1843                    )?;
1844
1845                    // We have this special check here for `let` declarations because the
1846                    // validator doesn't check them (they are comingled with other things in
1847                    // `named_expressions`; see <https://github.com/gfx-rs/wgpu/issues/7393>).
1848                    // The check could go in `type_and_init`, but then we'd have to
1849                    // distinguish whether override-sized is allowed. The error ought to use
1850                    // the type's span, but `module.types.get_span(ty)` is `Span::UNDEFINED`
1851                    // (see <https://github.com/gfx-rs/wgpu/issues/7951>).
1852                    if ctx.module.types[ty]
1853                        .inner
1854                        .is_dynamically_sized(&ctx.module.types)
1855                    {
1856                        return Err(Box::new(Error::TypeNotConstructible(l.name.span)));
1857                    }
1858
1859                    // We passed `Some()` to `type_and_init`, so we
1860                    // will get a lowered initializer expression back.
1861                    let initializer =
1862                        initializer.expect("type_and_init did not return an initializer");
1863
1864                    // The WGSL spec says that any expression that refers to a
1865                    // `let`-bound variable is not a const expression. This
1866                    // affects when errors must be reported, so we can't even
1867                    // treat suitable `let` bindings as constant as an
1868                    // optimization.
1869                    ctx.local_expression_kind_tracker
1870                        .force_non_const(initializer);
1871
1872                    block.extend(emitter.finish(&ctx.function.expressions));
1873                    ctx.local_table
1874                        .insert(l.handle, Declared::Runtime(Typed::Plain(initializer)));
1875                    ctx.named_expressions
1876                        .insert(initializer, (l.name.name.to_string(), l.name.span));
1877
1878                    if matches!(
1879                        ctx.module.types[ty].inner,
1880                        crate::TypeInner::RayQuery { .. }
1881                    ) {
1882                        // If a `let` variable is a ray query, it must be invalid as a `let`
1883                        // must have an initializer (it is also pretty useless as all other
1884                        // operations are disallowed, or require write-able variables).
1885                        return Err(Box::new(Error::RayQueryWithInitializer(
1886                            ctx.function.expressions.get_span(initializer),
1887                        )));
1888                    }
1889
1890                    if matches!(ctx.module.types[ty].inner, crate::TypeInner::HitObject) {
1891                        // Hit objects are like ray queries: a `let` must have an
1892                        // initializer, but hit objects cannot be initialized.
1893                        return Err(Box::new(Error::HitObjectWithInitializer(
1894                            ctx.function.expressions.get_span(initializer),
1895                        )));
1896                    }
1897
1898                    return Ok(());
1899                }
1900                ast::LocalDecl::Var(ref v) => {
1901                    let mut emitter = proc::Emitter::default();
1902                    emitter.start(&ctx.function.expressions);
1903
1904                    let explicit_ty =
1905                        v.ty.as_ref()
1906                            .map(|ast| {
1907                                self.resolve_ast_type(ast, &mut ctx.as_const(block, &mut emitter))
1908                            })
1909                            .transpose()?;
1910
1911                    let mut ectx = ctx.as_expression(block, &mut emitter);
1912                    let (ty, initializer) = self.type_and_init(
1913                        v.name,
1914                        v.init,
1915                        explicit_ty,
1916                        AbstractRule::Concretize,
1917                        &mut ectx,
1918                    )?;
1919
1920                    let (const_initializer, initializer) = {
1921                        match initializer {
1922                            Some(init) => {
1923                                // It's not correct to hoist the initializer up
1924                                // to the top of the function if:
1925                                // - the initialization is inside a loop, and should
1926                                //   take place on every iteration, or
1927                                // - the initialization is not a constant
1928                                //   expression, so its value depends on the
1929                                //   state at the point of initialization.
1930                                if is_inside_loop
1931                                    || !ctx.local_expression_kind_tracker.is_const_or_override(init)
1932                                {
1933                                    (None, Some(init))
1934                                } else {
1935                                    (Some(init), None)
1936                                }
1937                            }
1938                            None => (None, None),
1939                        }
1940                    };
1941
1942                    let var = ctx.function.local_variables.append(
1943                        ir::LocalVariable {
1944                            name: Some(v.name.name.to_string()),
1945                            ty,
1946                            init: const_initializer,
1947                        },
1948                        stmt.span,
1949                    );
1950
1951                    let handle = ctx
1952                        .as_expression(block, &mut emitter)
1953                        .interrupt_emitter(ir::Expression::LocalVariable(var), Span::UNDEFINED)?;
1954
1955                    block.extend(emitter.finish(&ctx.function.expressions));
1956                    ctx.local_table
1957                        .insert(v.handle, Declared::Runtime(Typed::Reference(handle)));
1958
1959                    match ctx.module.types[ty].inner {
1960                        crate::TypeInner::RayQuery { .. } => {
1961                            // Initializers are disallowed for ray queries as any store is disallowed.
1962                            // However, in loops ray queries need to be reset using a special piece of
1963                            // IR.
1964
1965                            // Because we have a special case for ray queries, and initializers are always
1966                            // disallowed for ray queries, we remove them here. This prevents having to
1967                            // special-case them and then just emitting invalid IR anyway and gives a
1968                            // clearer error message.
1969                            if let Some(expr) = initializer {
1970                                return Err(Box::new(Error::RayQueryWithInitializer(
1971                                    ctx.function.expressions.get_span(expr),
1972                                )));
1973                            }
1974
1975                            if is_inside_loop {
1976                                ir::Statement::RayQuery {
1977                                    query: handle,
1978                                    fun: ir::RayQueryFunction::Begin,
1979                                }
1980                            } else {
1981                                return Ok(());
1982                            }
1983                        }
1984                        crate::TypeInner::HitObject => {
1985                            // Initializers are disallowed for hit objects as any store is
1986                            // disallowed.
1987                            if let Some(expr) = initializer {
1988                                return Err(Box::new(Error::HitObjectWithInitializer(
1989                                    ctx.function.expressions.get_span(expr),
1990                                )));
1991                            }
1992
1993                            // Backends record every hit object as empty when its
1994                            // function starts, but a declaration inside a loop must
1995                            // start out empty on every iteration, so reset it here.
1996                            if is_inside_loop {
1997                                ir::Statement::HitObject {
1998                                    hit_object: handle,
1999                                    fun: ir::HitObjectFunction::RecordEmpty,
2000                                }
2001                            } else {
2002                                return Ok(());
2003                            }
2004                        }
2005                        _ => {
2006                            let initializer = if is_inside_loop {
2007                                match initializer {
2008                                    Some(initializer) => Some(initializer),
2009                                    None => Some(
2010                                        ctx.as_expression(block, &mut emitter).append_expression(
2011                                            ir::Expression::ZeroValue(ty),
2012                                            stmt.span,
2013                                        )?,
2014                                    ),
2015                                }
2016                            } else {
2017                                initializer
2018                            };
2019
2020                            match initializer {
2021                                Some(initializer) => ir::Statement::Store {
2022                                    pointer: handle,
2023                                    value: initializer,
2024                                },
2025                                None => return Ok(()),
2026                            }
2027                        }
2028                    }
2029                }
2030                ast::LocalDecl::Const(ref c) => {
2031                    let mut emitter = proc::Emitter::default();
2032                    emitter.start(&ctx.function.expressions);
2033
2034                    let ectx = &mut ctx.as_const(block, &mut emitter);
2035
2036                    let explicit_ty =
2037                        c.ty.as_ref()
2038                            .map(|ast| self.resolve_ast_type(ast, &mut ectx.as_const()))
2039                            .transpose()?;
2040
2041                    let (_ty, init) = self.type_and_init(
2042                        c.name,
2043                        Some(c.init),
2044                        explicit_ty,
2045                        AbstractRule::Allow,
2046                        &mut ectx.as_const(),
2047                    )?;
2048                    let init = init.expect("Local const must have init");
2049
2050                    block.extend(emitter.finish(&ctx.function.expressions));
2051                    ctx.local_table
2052                        .insert(c.handle, Declared::Const(Typed::Plain(init)));
2053                    return Ok(());
2054                }
2055            },
2056            ast::StatementKind::If {
2057                condition,
2058                ref accept,
2059                ref reject,
2060            } => {
2061                let mut emitter = proc::Emitter::default();
2062                emitter.start(&ctx.function.expressions);
2063
2064                let condition =
2065                    self.expression(condition, &mut ctx.as_expression(block, &mut emitter))?;
2066                block.extend(emitter.finish(&ctx.function.expressions));
2067
2068                let accept = self.block(accept, is_inside_loop, ctx)?;
2069                let reject = self.block(reject, is_inside_loop, ctx)?;
2070
2071                ir::Statement::If {
2072                    condition,
2073                    accept,
2074                    reject,
2075                }
2076            }
2077            ast::StatementKind::Switch {
2078                selector,
2079                ref cases,
2080            } => {
2081                let mut emitter = proc::Emitter::default();
2082                emitter.start(&ctx.function.expressions);
2083
2084                let mut ectx = ctx.as_expression(block, &mut emitter);
2085
2086                // Determine the scalar type of the selector and case expressions, find the
2087                // consensus type for automatic conversion, then convert them.
2088                let (mut exprs, spans) = core::iter::once(selector)
2089                    .chain(cases.iter().filter_map(|case| match case.value {
2090                        ast::SwitchValue::Expr(expr) => Some(expr),
2091                        ast::SwitchValue::Default => None,
2092                    }))
2093                    .enumerate()
2094                    .map(|(i, expr)| {
2095                        let span = ectx.ast_expressions.get_span(expr);
2096                        let expr = self.expression_for_abstract(expr, &mut ectx)?;
2097                        let ty = resolve_inner!(ectx, expr);
2098                        match *ty {
2099                            ir::TypeInner::Scalar(
2100                                ir::Scalar::I32 | ir::Scalar::U32 | ir::Scalar::ABSTRACT_INT,
2101                            ) => Ok((expr, span)),
2102                            _ => match i {
2103                                0 => Err(Box::new(Error::InvalidSwitchSelector { span })),
2104                                _ => Err(Box::new(Error::InvalidSwitchCase { span })),
2105                            },
2106                        }
2107                    })
2108                    .collect::<Result<(Vec<_>, Vec<_>)>>()?;
2109
2110                let mut consensus =
2111                    ectx.automatic_conversion_consensus(None, &exprs)
2112                        .map_err(|span_idx| Error::SwitchCaseTypeMismatch {
2113                            span: spans[span_idx],
2114                        })?;
2115                // Concretize to I32 if the selector and all cases were abstract
2116                if consensus == ir::Scalar::ABSTRACT_INT {
2117                    consensus = ir::Scalar::I32;
2118                }
2119                for expr in &mut exprs {
2120                    ectx.convert_to_leaf_scalar(expr, consensus)?;
2121                }
2122
2123                block.extend(emitter.finish(&ctx.function.expressions));
2124
2125                let mut exprs = exprs.into_iter();
2126                let selector = exprs
2127                    .next()
2128                    .expect("First element should be selector expression");
2129
2130                let cases = cases
2131                    .iter()
2132                    .map(|case| {
2133                        Ok(ir::SwitchCase {
2134                            value: match case.value {
2135                                ast::SwitchValue::Expr(expr) => {
2136                                    let span = ctx.ast_expressions.get_span(expr);
2137                                    let expr = exprs.next().expect(
2138                                        "Should yield expression for each SwitchValue::Expr case",
2139                                    );
2140                                    match ctx
2141                                        .module
2142                                        .to_ctx()
2143                                        .get_const_val_from(expr, &ctx.function.expressions)
2144                                    {
2145                                        Ok(ir::Literal::I32(value)) => ir::SwitchValue::I32(value),
2146                                        Ok(ir::Literal::U32(value)) => ir::SwitchValue::U32(value),
2147                                        _ => {
2148                                            return Err(Box::new(Error::InvalidSwitchCase {
2149                                                span,
2150                                            }));
2151                                        }
2152                                    }
2153                                }
2154                                ast::SwitchValue::Default => ir::SwitchValue::Default,
2155                            },
2156                            body: self.block(&case.body, is_inside_loop, ctx)?,
2157                            fall_through: case.fall_through,
2158                        })
2159                    })
2160                    .collect::<Result<_>>()?;
2161
2162                ir::Statement::Switch { selector, cases }
2163            }
2164            ast::StatementKind::Loop {
2165                ref body,
2166                ref continuing,
2167                break_if,
2168            } => {
2169                let body = self.block(body, true, ctx)?;
2170                let mut continuing = self.block(continuing, true, ctx)?;
2171
2172                let mut emitter = proc::Emitter::default();
2173                emitter.start(&ctx.function.expressions);
2174                let break_if = break_if
2175                    .map(|expr| {
2176                        self.expression(expr, &mut ctx.as_expression(&mut continuing, &mut emitter))
2177                    })
2178                    .transpose()?;
2179                continuing.extend(emitter.finish(&ctx.function.expressions));
2180
2181                ir::Statement::Loop {
2182                    body,
2183                    continuing,
2184                    break_if,
2185                }
2186            }
2187            ast::StatementKind::Break => ir::Statement::Break,
2188            ast::StatementKind::Continue => ir::Statement::Continue,
2189            ast::StatementKind::Return { value: ast_value } => {
2190                let mut emitter = proc::Emitter::default();
2191                emitter.start(&ctx.function.expressions);
2192
2193                let value;
2194                if let Some(ast_expr) = ast_value {
2195                    let value_span = ctx.ast_expressions.get_span(ast_expr);
2196                    let result_ty = ctx.function.result.as_ref().map(|r| r.ty);
2197                    let mut ectx = ctx.as_expression(block, &mut emitter);
2198                    let expr = self.expression_for_abstract(ast_expr, &mut ectx)?;
2199
2200                    if let Some(result_ty) = result_ty {
2201                        let mut ectx = ctx.as_expression(block, &mut emitter);
2202                        let resolution = proc::TypeResolution::Handle(result_ty);
2203                        let converted =
2204                            ectx.try_automatic_conversions(expr, &resolution, value_span)?;
2205                        value = Some(converted);
2206                    } else {
2207                        value = Some(expr);
2208                    }
2209                } else {
2210                    value = None;
2211                }
2212                block.extend(emitter.finish(&ctx.function.expressions));
2213
2214                ir::Statement::Return { value }
2215            }
2216            ast::StatementKind::Kill => ir::Statement::Kill,
2217            ast::StatementKind::Call(ref call_phrase) => {
2218                let mut emitter = proc::Emitter::default();
2219                emitter.start(&ctx.function.expressions);
2220
2221                let _ = self.call(
2222                    call_phrase,
2223                    stmt.span,
2224                    &mut ctx.as_expression(block, &mut emitter),
2225                    true,
2226                )?;
2227                block.extend(emitter.finish(&ctx.function.expressions));
2228                return Ok(());
2229            }
2230            ast::StatementKind::Assign {
2231                target: ast_target,
2232                op,
2233                value,
2234            } => {
2235                let mut emitter = proc::Emitter::default();
2236                emitter.start(&ctx.function.expressions);
2237                let target_span = ctx.ast_expressions.get_span(ast_target);
2238
2239                let mut ectx = ctx.as_expression(block, &mut emitter);
2240                let target = self.expression_for_reference(ast_target, &mut ectx)?;
2241                let target_handle = match target {
2242                    Typed::Reference(handle) => handle,
2243                    Typed::Plain(handle) => {
2244                        let ty = ctx.invalid_assignment_type(handle);
2245                        return Err(Box::new(Error::InvalidAssignment {
2246                            span: target_span,
2247                            ty,
2248                        }));
2249                    }
2250                };
2251
2252                // Usually the value needs to be converted to match the type of
2253                // the memory view you're assigning it to. The bit shift
2254                // operators are exceptions, in that the right operand is always
2255                // a `u32` or `vecN<u32>`.
2256                let target_scalar = match op {
2257                    Some(ir::BinaryOperator::ShiftLeft | ir::BinaryOperator::ShiftRight) => {
2258                        Some(ir::Scalar::U32)
2259                    }
2260                    _ => resolve_inner!(ectx, target_handle)
2261                        .pointer_automatically_convertible_scalar(&ectx.module.types),
2262                };
2263
2264                // Need to emit the LHS _before_ the RHS so that it is evaluated first.
2265                let op_assign = if let Some(op) = op {
2266                    Some((op, ectx.apply_load_rule(target)?))
2267                } else {
2268                    None
2269                };
2270
2271                let value = self.expression_for_abstract(value, &mut ectx)?;
2272                let mut value = match target_scalar {
2273                    Some(target_scalar) => ectx.try_automatic_conversion_for_leaf_scalar(
2274                        value,
2275                        target_scalar,
2276                        target_span,
2277                    )?,
2278                    None => value,
2279                };
2280
2281                let value = match op_assign {
2282                    Some((op, mut left)) => {
2283                        ectx.binary_op_splat(op, &mut left, &mut value)?;
2284                        ectx.append_expression(
2285                            ir::Expression::Binary {
2286                                op,
2287                                left,
2288                                right: value,
2289                            },
2290                            stmt.span,
2291                        )?
2292                    }
2293                    None => value,
2294                };
2295                block.extend(emitter.finish(&ctx.function.expressions));
2296
2297                ir::Statement::Store {
2298                    pointer: target_handle,
2299                    value,
2300                }
2301            }
2302            ast::StatementKind::Increment(value) | ast::StatementKind::Decrement(value) => {
2303                let mut emitter = proc::Emitter::default();
2304                emitter.start(&ctx.function.expressions);
2305
2306                let op = match stmt.kind {
2307                    ast::StatementKind::Increment(_) => ir::BinaryOperator::Add,
2308                    ast::StatementKind::Decrement(_) => ir::BinaryOperator::Subtract,
2309                    _ => unreachable!(),
2310                };
2311
2312                let value_span = ctx.ast_expressions.get_span(value);
2313                let target = self
2314                    .expression_for_reference(value, &mut ctx.as_expression(block, &mut emitter))?;
2315                let target_handle = target.ref_or(Error::BadIncrDecrReferenceType(value_span))?;
2316
2317                let mut ectx = ctx.as_expression(block, &mut emitter);
2318                let scalar = match *resolve_inner!(ectx, target_handle) {
2319                    ir::TypeInner::ValuePointer {
2320                        size: None, scalar, ..
2321                    } => scalar,
2322                    ir::TypeInner::Pointer { base, .. } => match ectx.module.types[base].inner {
2323                        ir::TypeInner::Scalar(scalar) => scalar,
2324                        _ => return Err(Box::new(Error::BadIncrDecrReferenceType(value_span))),
2325                    },
2326                    _ => return Err(Box::new(Error::BadIncrDecrReferenceType(value_span))),
2327                };
2328                let literal = match scalar.kind {
2329                    ir::ScalarKind::Sint | ir::ScalarKind::Uint => ir::Literal::one(scalar)
2330                        .ok_or(Error::BadIncrDecrReferenceType(value_span))?,
2331                    _ => return Err(Box::new(Error::BadIncrDecrReferenceType(value_span))),
2332                };
2333
2334                let right =
2335                    ectx.interrupt_emitter(ir::Expression::Literal(literal), Span::UNDEFINED)?;
2336                let rctx = ectx.runtime_expression_ctx(stmt.span)?;
2337                let left = rctx.function.expressions.append(
2338                    ir::Expression::Load {
2339                        pointer: target_handle,
2340                    },
2341                    value_span,
2342                );
2343                let value = rctx
2344                    .function
2345                    .expressions
2346                    .append(ir::Expression::Binary { op, left, right }, stmt.span);
2347                rctx.local_expression_kind_tracker
2348                    .insert(left, proc::ExpressionKind::Runtime);
2349                rctx.local_expression_kind_tracker
2350                    .insert(value, proc::ExpressionKind::Runtime);
2351
2352                block.extend(emitter.finish(&ctx.function.expressions));
2353                ir::Statement::Store {
2354                    pointer: target_handle,
2355                    value,
2356                }
2357            }
2358            ast::StatementKind::ConstAssert(condition) => {
2359                let mut emitter = proc::Emitter::default();
2360                emitter.start(&ctx.function.expressions);
2361
2362                let condition =
2363                    self.expression(condition, &mut ctx.as_const(block, &mut emitter))?;
2364
2365                let span = ctx.function.expressions.get_span(condition);
2366                match ctx
2367                    .module
2368                    .to_ctx()
2369                    .get_const_val_from(condition, &ctx.function.expressions)
2370                {
2371                    Ok(true) => Ok(()),
2372                    Ok(false) => Err(Error::ConstAssertFailed(span)),
2373                    Err(proc::ConstValueError::Negative) => unreachable!(),
2374                    Err(proc::ConstValueError::NonConst | proc::ConstValueError::InvalidType) => {
2375                        Err(Error::NotBool(span))
2376                    }
2377                }?;
2378
2379                block.extend(emitter.finish(&ctx.function.expressions));
2380
2381                return Ok(());
2382            }
2383            ast::StatementKind::Phony(expr) => {
2384                // Remembered the RHS of the phony assignment as a named expression. This
2385                // is important (1) to preserve the RHS for validation, (2) to track any
2386                // referenced globals.
2387                let mut emitter = proc::Emitter::default();
2388                emitter.start(&ctx.function.expressions);
2389
2390                let value = self.expression(expr, &mut ctx.as_expression(block, &mut emitter))?;
2391                block.extend(emitter.finish(&ctx.function.expressions));
2392                ctx.named_expressions
2393                    .insert(value, ("phony".to_string(), stmt.span));
2394                return Ok(());
2395            }
2396        };
2397
2398        block.push(out, stmt.span);
2399
2400        Ok(())
2401    }
2402
2403    /// Lower `expr` and apply the Load Rule if possible.
2404    ///
2405    /// For the time being, this concretizes abstract values, to support
2406    /// consumers that haven't been adapted to consume them yet. Consumers
2407    /// prepared for abstract values can call [`expression_for_abstract`].
2408    ///
2409    /// [`expression_for_abstract`]: Lowerer::expression_for_abstract
2410    fn expression(
2411        &mut self,
2412        expr: Handle<ast::Expression<'source>>,
2413        ctx: &mut ExpressionContext<'source, '_, '_>,
2414    ) -> Result<'source, Handle<ir::Expression>> {
2415        let expr = self.expression_for_abstract(expr, ctx)?;
2416        ctx.concretize(expr)
2417    }
2418
2419    fn expression_for_abstract(
2420        &mut self,
2421        expr: Handle<ast::Expression<'source>>,
2422        ctx: &mut ExpressionContext<'source, '_, '_>,
2423    ) -> Result<'source, Handle<ir::Expression>> {
2424        let expr = self.expression_for_reference(expr, ctx)?;
2425        ctx.apply_load_rule(expr)
2426    }
2427
2428    fn expression_with_leaf_scalar(
2429        &mut self,
2430        expr: Handle<ast::Expression<'source>>,
2431        scalar: ir::Scalar,
2432        ctx: &mut ExpressionContext<'source, '_, '_>,
2433    ) -> Result<'source, Handle<ir::Expression>> {
2434        let unconverted = self.expression_for_abstract(expr, ctx)?;
2435        ctx.try_automatic_conversion_for_leaf_scalar(unconverted, scalar, Span::default())
2436    }
2437
2438    fn expression_for_reference(
2439        &mut self,
2440        expr: Handle<ast::Expression<'source>>,
2441        ctx: &mut ExpressionContext<'source, '_, '_>,
2442    ) -> Result<'source, Typed<Handle<ir::Expression>>> {
2443        let span = ctx.ast_expressions.get_span(expr);
2444        let expr = &ctx.ast_expressions[expr];
2445
2446        let expr: Typed<ir::Expression> = match *expr {
2447            ast::Expression::Literal(literal) => {
2448                let literal = match literal {
2449                    ast::Literal::Number(Number::F16(f)) => ir::Literal::F16(f),
2450                    ast::Literal::Number(Number::F32(f)) => ir::Literal::F32(f),
2451                    ast::Literal::Number(Number::I32(i)) => ir::Literal::I32(i),
2452                    ast::Literal::Number(Number::U32(u)) => ir::Literal::U32(u),
2453                    ast::Literal::Number(Number::I64(i)) => ir::Literal::I64(i),
2454                    ast::Literal::Number(Number::U64(u)) => ir::Literal::U64(u),
2455                    ast::Literal::Number(Number::F64(f)) => ir::Literal::F64(f),
2456                    ast::Literal::Number(Number::AbstractInt(i)) => ir::Literal::AbstractInt(i),
2457                    ast::Literal::Number(Number::AbstractFloat(f)) => ir::Literal::AbstractFloat(f),
2458                    ast::Literal::Bool(b) => ir::Literal::Bool(b),
2459                };
2460                let handle = ctx.interrupt_emitter(ir::Expression::Literal(literal), span)?;
2461                return Ok(Typed::Plain(handle));
2462            }
2463            ast::Expression::Ident(ast::TemplateElaboratedIdent {
2464                ref template_list, ..
2465            }) if !template_list.is_empty() => {
2466                return Err(Box::new(Error::UnexpectedTemplate(span)))
2467            }
2468            ast::Expression::Ident(ast::TemplateElaboratedIdent {
2469                ident: ast::IdentExpr::Local(local),
2470                ..
2471            }) => {
2472                return ctx.local(&local, span);
2473            }
2474            ast::Expression::Ident(ast::TemplateElaboratedIdent {
2475                ident: ast::IdentExpr::Unresolved(name),
2476                ..
2477            }) => {
2478                let global = ctx
2479                    .globals
2480                    .get(name)
2481                    .ok_or(Error::UnknownIdent(span, name))?;
2482                let expr = match *global {
2483                    LoweredGlobalDecl::Var(handle) => {
2484                        let expr = ir::Expression::GlobalVariable(handle);
2485                        let v = &ctx.module.global_variables[handle];
2486                        match v.space {
2487                            ir::AddressSpace::Handle => Typed::Plain(expr),
2488                            _ => Typed::Reference(expr),
2489                        }
2490                    }
2491                    LoweredGlobalDecl::Const(handle) => {
2492                        Typed::Plain(ir::Expression::Constant(handle))
2493                    }
2494                    LoweredGlobalDecl::Override(handle) => {
2495                        Typed::Plain(ir::Expression::Override(handle))
2496                    }
2497                    LoweredGlobalDecl::Function { .. }
2498                    | LoweredGlobalDecl::Type(_)
2499                    | LoweredGlobalDecl::EntryPoint(_) => {
2500                        return Err(Box::new(Error::Unexpected(span, ExpectedToken::Variable)));
2501                    }
2502                };
2503
2504                return expr.try_map(|handle| ctx.interrupt_emitter(handle, span));
2505            }
2506            ast::Expression::Unary { op, expr } => self.unary(op, expr, span, ctx)?,
2507            ast::Expression::AddrOf(expr) => {
2508                // The `&` operator simply converts a reference to a pointer. And since a
2509                // reference is required, the Load Rule is not applied.
2510                match self.expression_for_reference(expr, ctx)? {
2511                    Typed::Reference(handle) => {
2512                        let expr = &ctx.runtime_expression_ctx(span)?.function.expressions[handle];
2513                        if let &ir::Expression::Access { base, .. }
2514                        | &ir::Expression::AccessIndex { base, .. } = expr
2515                        {
2516                            if let Some(ty) = resolve_inner!(ctx, base).pointer_base_type() {
2517                                if matches!(
2518                                    *ty.inner_with(&ctx.module.types),
2519                                    ir::TypeInner::Vector { .. },
2520                                ) {
2521                                    return Err(Box::new(Error::InvalidAddrOfOperand(
2522                                        ctx.get_expression_span(handle),
2523                                    )));
2524                                }
2525                            }
2526                        }
2527                        // No code is generated. We just declare the reference a pointer now.
2528                        return Ok(Typed::Plain(handle));
2529                    }
2530                    Typed::Plain(_) => {
2531                        return Err(Box::new(Error::NotReference(
2532                            "the operand of the `&` operator",
2533                            span,
2534                        )));
2535                    }
2536                }
2537            }
2538            ast::Expression::Deref(expr) => {
2539                // The pointer we dereference must be loaded.
2540                let pointer = self.expression(expr, ctx)?;
2541
2542                if resolve_inner!(ctx, pointer).pointer_space().is_none() {
2543                    return Err(Box::new(Error::NotPointer(span)));
2544                }
2545
2546                // No code is generated. We just declare the pointer a reference now.
2547                return Ok(Typed::Reference(pointer));
2548            }
2549            ast::Expression::Binary { op, left, right } => {
2550                self.binary(op, left, right, span, ctx)?
2551            }
2552            ast::Expression::Call(ref call_phrase) => {
2553                let handle = self
2554                    .call(call_phrase, span, ctx, false)?
2555                    .ok_or(Error::FunctionReturnsVoid(span))?;
2556                return Ok(Typed::Plain(handle));
2557            }
2558            ast::Expression::Index { base, index } => {
2559                let mut lowered_base = self.expression_for_reference(base, ctx)?;
2560                let index = self.expression(index, ctx)?;
2561
2562                // <https://www.w3.org/TR/WGSL/#language_extension-pointer_composite_access>
2563                // Declare pointer as reference
2564                if let Typed::Plain(handle) = lowered_base {
2565                    if resolve_inner!(ctx, handle).pointer_space().is_some() {
2566                        lowered_base = Typed::Reference(handle);
2567                    }
2568                }
2569
2570                lowered_base.try_map(|base| match ctx.get_const_val(index).ok() {
2571                    Some(index) => Ok::<_, Box<Error>>(ir::Expression::AccessIndex { base, index }),
2572                    None => {
2573                        // When an abstract array value e is indexed by an expression
2574                        // that is not a const-expression, then the array is concretized
2575                        // before the index is applied.
2576                        // https://www.w3.org/TR/WGSL/#array-access-expr
2577                        // Also applies to vectors and matrices.
2578                        let base = ctx.concretize(base)?;
2579                        Ok(ir::Expression::Access { base, index })
2580                    }
2581                })?
2582            }
2583            ast::Expression::Member { base, ref field } => {
2584                let mut lowered_base = self.expression_for_reference(base, ctx)?;
2585
2586                // <https://www.w3.org/TR/WGSL/#language_extension-pointer_composite_access>
2587                // Declare pointer as reference
2588                if let Typed::Plain(handle) = lowered_base {
2589                    if resolve_inner!(ctx, handle).pointer_space().is_some() {
2590                        lowered_base = Typed::Reference(handle);
2591                    }
2592                }
2593
2594                let temp_ty;
2595                let composite_type: &ir::TypeInner = match lowered_base {
2596                    Typed::Reference(handle) => {
2597                        temp_ty = resolve_inner!(ctx, handle)
2598                            .pointer_base_type()
2599                            .expect("In Typed::Reference(handle), handle must be a Naga pointer");
2600                        temp_ty.inner_with(&ctx.module.types)
2601                    }
2602
2603                    Typed::Plain(handle) => {
2604                        resolve_inner!(ctx, handle)
2605                    }
2606                };
2607
2608                let access = match *composite_type {
2609                    ir::TypeInner::Struct { ref members, .. } => {
2610                        let index = members
2611                            .iter()
2612                            .position(|m| m.name.as_deref() == Some(field.name))
2613                            .ok_or(Error::BadAccessor(field.span))?
2614                            as u32;
2615
2616                        lowered_base.map(|base| ir::Expression::AccessIndex { base, index })
2617                    }
2618                    ir::TypeInner::Vector { size: vec_size, .. } => {
2619                        match Components::new(field.name, field.span)? {
2620                            Components::Swizzle { size, pattern } => {
2621                                for &component in pattern[..size as usize].iter() {
2622                                    if component as u8 >= vec_size as u8 {
2623                                        return Err(Box::new(Error::BadAccessor(field.span)));
2624                                    }
2625                                }
2626                                Typed::Plain(ir::Expression::Swizzle {
2627                                    size,
2628                                    vector: ctx.apply_load_rule(lowered_base)?,
2629                                    pattern,
2630                                })
2631                            }
2632                            Components::Single(index) => {
2633                                if index >= vec_size as u32 {
2634                                    return Err(Box::new(Error::BadAccessor(field.span)));
2635                                }
2636                                lowered_base.map(|base| ir::Expression::AccessIndex { base, index })
2637                            }
2638                        }
2639                    }
2640                    _ => return Err(Box::new(Error::BadAccessor(field.span))),
2641                };
2642
2643                access
2644            }
2645            ast::Expression::String(_) => {
2646                return Err(Box::new(Error::InvalidStringLiteral {
2647                    span,
2648                    description: "String literals are only supported in debugPrintf",
2649                }))
2650            }
2651        };
2652
2653        expr.try_map(|handle| ctx.append_expression(handle, span))
2654    }
2655
2656    /// Generate IR for the short-circuiting operators `&&` and `||`.
2657    ///
2658    /// `binary` has already lowered the LHS expression and resolved its type.
2659    fn logical(
2660        &mut self,
2661        op: crate::BinaryOperator,
2662        left: Handle<crate::Expression>,
2663        right: Handle<ast::Expression<'source>>,
2664        span: Span,
2665        ctx: &mut ExpressionContext<'source, '_, '_>,
2666    ) -> Result<'source, Typed<crate::Expression>> {
2667        debug_assert!(
2668            op == crate::BinaryOperator::LogicalAnd || op == crate::BinaryOperator::LogicalOr
2669        );
2670
2671        if ctx.is_runtime() {
2672            // To simulate short-circuiting behavior, we want to generate IR
2673            // like the following for `&&`. For `||`, the condition is `!_lhs`
2674            // and the else value is `true`.
2675            //
2676            // var _e0: bool;
2677            // if _lhs {
2678            //     _e0 = _rhs;
2679            // } else {
2680            //     _e0 = false;
2681            // }
2682
2683            let (condition, else_val) = if op == crate::BinaryOperator::LogicalAnd {
2684                let condition = left;
2685                let else_val = ctx.append_expression(
2686                    crate::Expression::Literal(crate::Literal::Bool(false)),
2687                    span,
2688                )?;
2689                (condition, else_val)
2690            } else {
2691                let condition = ctx.append_expression(
2692                    crate::Expression::Unary {
2693                        op: crate::UnaryOperator::LogicalNot,
2694                        expr: left,
2695                    },
2696                    span,
2697                )?;
2698                let else_val = ctx.append_expression(
2699                    crate::Expression::Literal(crate::Literal::Bool(true)),
2700                    span,
2701                )?;
2702                (condition, else_val)
2703            };
2704
2705            let bool_ty = ctx.ensure_type_exists(crate::TypeInner::Scalar(crate::Scalar::BOOL));
2706
2707            let rctx = ctx.runtime_expression_ctx(span)?;
2708            let result_var = rctx.function.local_variables.append(
2709                crate::LocalVariable {
2710                    name: None,
2711                    ty: bool_ty,
2712                    init: None,
2713                },
2714                span,
2715            );
2716            let pointer =
2717                ctx.append_expression(crate::Expression::LocalVariable(result_var), span)?;
2718
2719            let (right, mut accept) = ctx.with_nested_runtime_expression_ctx(span, |ctx| {
2720                let right = self.expression_for_abstract(right, ctx)?;
2721                ctx.grow_types(right)?;
2722                Ok(right)
2723            })?;
2724
2725            accept.push(
2726                crate::Statement::Store {
2727                    pointer,
2728                    value: right,
2729                },
2730                span,
2731            );
2732
2733            let mut reject = crate::Block::with_capacity(1);
2734            reject.push(
2735                crate::Statement::Store {
2736                    pointer,
2737                    value: else_val,
2738                },
2739                span,
2740            );
2741
2742            let rctx = ctx.runtime_expression_ctx(span)?;
2743            rctx.block.push(
2744                crate::Statement::If {
2745                    condition,
2746                    accept,
2747                    reject,
2748                },
2749                span,
2750            );
2751
2752            Ok(Typed::Reference(crate::Expression::LocalVariable(
2753                result_var,
2754            )))
2755        } else {
2756            let left_val: Option<bool> = ctx.get_const_val(left).ok();
2757
2758            if left_val.is_some_and(|left_val| {
2759                op == crate::BinaryOperator::LogicalAnd && !left_val
2760                    || op == crate::BinaryOperator::LogicalOr && left_val
2761            }) {
2762                // Short-circuit behavior: don't evaluate the RHS.
2763
2764                // TODO(https://github.com/gfx-rs/wgpu/issues/8440): We shouldn't ignore the
2765                // RHS completely, it should still be type-checked. Preserving it for type
2766                // checking is a bit tricky, because we're trying to produce an expression
2767                // for a const context, but the RHS is allowed to have things that aren't
2768                // const.
2769
2770                Ok(Typed::Plain(ctx.get(left).clone()))
2771            } else {
2772                // Evaluate the RHS and construct the entire binary expression as we
2773                // normally would. This case applies to well-formed constant logical
2774                // expressions that don't short-circuit (handled by the constant evaluator
2775                // shortly), to override expressions (handled when overrides are processed)
2776                // and to non-well-formed expressions (rejected by type checking).
2777                let right = self.expression_for_abstract(right, ctx)?;
2778                ctx.grow_types(right)?;
2779
2780                Ok(Typed::Plain(crate::Expression::Binary { op, left, right }))
2781            }
2782        }
2783    }
2784
2785    fn type_expression(
2786        &mut self,
2787        expr: Handle<ast::Expression<'source>>,
2788        ctx: &mut ExpressionContext<'source, '_, '_>,
2789    ) -> Result<'source, Handle<ir::Type>> {
2790        let span = ctx.ast_expressions.get_span(expr);
2791        let expr = &ctx.ast_expressions[expr];
2792
2793        let ident = match *expr {
2794            ast::Expression::Ident(ref ident) => ident,
2795            _ => return Err(Box::new(Error::UnexpectedExprForTypeExpression(span))),
2796        };
2797
2798        self.type_specifier(ident, ctx, None)
2799    }
2800
2801    fn type_specifier(
2802        &mut self,
2803        ident: &ast::TemplateElaboratedIdent<'source>,
2804        ctx: &mut ExpressionContext<'source, '_, '_>,
2805        alias_name: Option<String>,
2806    ) -> Result<'source, Handle<ir::Type>> {
2807        let &ast::TemplateElaboratedIdent {
2808            ref ident,
2809            ident_span,
2810            ref template_list,
2811            ..
2812        } = ident;
2813
2814        let ident = match *ident {
2815            ast::IdentExpr::Unresolved(ident) => ident,
2816            ast::IdentExpr::Local(_) => {
2817                // Since WGSL only supports module-scope type definitions and
2818                // aliases, a local identifier can't possibly refer to a type.
2819                return Err(Box::new(Error::UnexpectedExprForTypeExpression(ident_span)));
2820            }
2821        };
2822
2823        let mut tl = TemplateListIter::new(ident_span, template_list);
2824
2825        if let Some(global) = ctx.globals.get(ident) {
2826            let &LoweredGlobalDecl::Type(handle) = global else {
2827                return Err(Box::new(Error::UnexpectedExprForTypeExpression(ident_span)));
2828            };
2829
2830            // Type generators can only be predeclared, so since `ident` refers
2831            // to a module-scope declaration, the template parameter list should
2832            // be empty.
2833            tl.finish(ctx)?;
2834            return Ok(handle);
2835        }
2836
2837        // If `ident` doesn't resolve to a module-scope declaration, then it
2838        // must resolve to a predeclared type or type generator.
2839        let ty = conv::map_predeclared_type(&ctx.enable_extensions, ident_span, ident)?
2840            .ok_or_else(|| Box::new(Error::UnknownIdent(ident_span, ident)))?;
2841        let ty = self.finalize_type(ctx, ty, &mut tl, alias_name)?;
2842
2843        tl.finish(ctx)?;
2844
2845        Ok(ty)
2846    }
2847
2848    /// Construct an [`ir::Type`] from a [`conv::PredeclaredType`] and a list of
2849    /// template parameters.
2850    ///
2851    /// If we're processing a type alias, then `alias_name` is the name we
2852    /// should use in the new `ir::Type`.
2853    ///
2854    /// For example, when parsing `vec3<f32>`, the caller would pass:
2855    ///
2856    /// - for `ty`, [`TypeGenerator::Vector`], and
2857    ///
2858    /// - for `tl`, an iterator producing a single [`Expression::Ident`] representing `f32`.
2859    ///
2860    /// From those arguments this function will return a handle for the
2861    /// [`ir::Type`] representing `vec3<f32>`.
2862    ///
2863    /// [`TypeGenerator::Vector`]: conv::TypeGenerator::Vector
2864    /// [`Expression::Ident`]: crate::front::wgsl::parse::ast::Expression::Ident
2865    fn finalize_type(
2866        &mut self,
2867        ctx: &mut ExpressionContext<'source, '_, '_>,
2868        ty: conv::PredeclaredType,
2869        tl: &mut TemplateListIter<'_, 'source>,
2870        alias_name: Option<String>,
2871    ) -> Result<'source, Handle<ir::Type>> {
2872        let ty = match ty {
2873            conv::PredeclaredType::TypeInner(ty_inner) => {
2874                if let ir::TypeInner::Image {
2875                    class: ir::ImageClass::External,
2876                    ..
2877                } = ty_inner
2878                {
2879                    // Other than the WGSL backend, every backend that supports
2880                    // external textures does so by lowering them to a set of
2881                    // ordinary textures and some parameters saying how to
2882                    // sample from them. We don't know which backend will
2883                    // consume the `Module` we're building, but in case it's not
2884                    // WGSL, populate `SpecialTypes::external_texture_params`
2885                    // and `SpecialTypes::external_texture_transfer_function`
2886                    // with the types the backend will use for the parameter
2887                    // buffer.
2888                    //
2889                    // Neither of these are the type we are lowering here:
2890                    // that's an ordinary `TypeInner::Image`. But the fact we
2891                    // are lowering a `texture_external` implies the backends
2892                    // may need these additional types too.
2893                    ctx.module.generate_external_texture_types();
2894                }
2895
2896                ctx.as_global().ensure_type_exists(alias_name, ty_inner)
2897            }
2898            conv::PredeclaredType::RayDesc => ctx.module.generate_ray_desc_type(),
2899            conv::PredeclaredType::RayIntersection => ctx.module.generate_ray_intersection_type(),
2900            conv::PredeclaredType::TypeGenerator(type_generator) => {
2901                let ty_inner = match type_generator {
2902                    conv::TypeGenerator::Vector { size } => {
2903                        let (scalar, _) = tl.scalar_ty(self, ctx)?;
2904                        ir::TypeInner::Vector { size, scalar }
2905                    }
2906                    conv::TypeGenerator::Matrix { columns, rows } => {
2907                        let (scalar, span) = tl.scalar_ty(self, ctx)?;
2908                        if scalar.kind != ir::ScalarKind::Float {
2909                            return Err(Box::new(Error::BadMatrixScalarKind(span, scalar)));
2910                        }
2911                        ir::TypeInner::Matrix {
2912                            columns,
2913                            rows,
2914                            scalar,
2915                        }
2916                    }
2917                    conv::TypeGenerator::Array => {
2918                        let base = tl.ty(self, ctx)?;
2919                        let size = tl.maybe_array_size(self, ctx)?;
2920
2921                        // Determine the size of the base type, if needed.
2922                        ctx.layouter.update(ctx.module.to_ctx()).map_err(|err| {
2923                            let LayoutErrorInner::TooLarge = err.inner else {
2924                                unreachable!("unexpected layout error: {err:?}");
2925                            };
2926                            // Lots of type definitions don't get spans, so this error
2927                            // message may not be very useful.
2928                            Box::new(Error::TypeTooLarge {
2929                                span: ctx.module.types.get_span(err.ty),
2930                            })
2931                        })?;
2932                        let stride = ctx.layouter[base].to_stride();
2933
2934                        ir::TypeInner::Array { base, size, stride }
2935                    }
2936                    conv::TypeGenerator::Atomic => {
2937                        let (scalar, _) = tl.scalar_ty(self, ctx)?;
2938                        ir::TypeInner::Atomic(scalar)
2939                    }
2940                    conv::TypeGenerator::Pointer => {
2941                        let mut space = tl.address_space(ctx)?;
2942                        let base = tl.ty(self, ctx)?;
2943                        tl.maybe_access_mode(&mut space, ctx)?;
2944                        ir::TypeInner::Pointer { base, space }
2945                    }
2946                    conv::TypeGenerator::SampledTexture {
2947                        dim,
2948                        arrayed,
2949                        multi,
2950                    } => {
2951                        let (scalar, span) = tl.scalar_ty(self, ctx)?;
2952                        let ir::Scalar { kind, width } = scalar;
2953                        if width != 4 {
2954                            return Err(Box::new(Error::BadTextureSampleType { span, scalar }));
2955                        }
2956                        ir::TypeInner::Image {
2957                            dim,
2958                            arrayed,
2959                            class: ir::ImageClass::Sampled { kind, multi },
2960                        }
2961                    }
2962                    conv::TypeGenerator::StorageTexture { dim, arrayed } => {
2963                        let format = tl.storage_format(ctx)?;
2964                        let access = tl.access_mode(ctx)?;
2965                        ir::TypeInner::Image {
2966                            dim,
2967                            arrayed,
2968                            class: ir::ImageClass::Storage { format, access },
2969                        }
2970                    }
2971                    conv::TypeGenerator::BindingArray => {
2972                        let base = tl.ty(self, ctx)?;
2973                        let size = tl.maybe_array_size(self, ctx)?;
2974                        ir::TypeInner::BindingArray { base, size }
2975                    }
2976                    conv::TypeGenerator::AccelerationStructure => {
2977                        let vertex_return = tl.maybe_vertex_return(ctx)?;
2978                        ir::TypeInner::AccelerationStructure { vertex_return }
2979                    }
2980                    conv::TypeGenerator::RayQuery => {
2981                        let vertex_return = tl.maybe_vertex_return(ctx)?;
2982                        ir::TypeInner::RayQuery { vertex_return }
2983                    }
2984                    conv::TypeGenerator::HitObject => ir::TypeInner::HitObject,
2985                    conv::TypeGenerator::CooperativeMatrix { columns, rows } => {
2986                        let (ty, span) = tl.ty_with_span(self, ctx)?;
2987                        let ir::TypeInner::Scalar(scalar) = ctx.module.types[ty].inner else {
2988                            return Err(Box::new(Error::UnsupportedCooperativeScalar(span)));
2989                        };
2990                        let role = tl.cooperative_role(ctx)?;
2991                        ir::TypeInner::CooperativeMatrix {
2992                            columns,
2993                            rows,
2994                            scalar,
2995                            role,
2996                        }
2997                    }
2998                };
2999                ctx.as_global().ensure_type_exists(alias_name, ty_inner)
3000            }
3001        };
3002        Ok(ty)
3003    }
3004
3005    fn unary(
3006        &mut self,
3007        op: ir::UnaryOperator,
3008        expr: Handle<ast::Expression<'source>>,
3009        span: Span,
3010        ctx: &mut ExpressionContext<'source, '_, '_>,
3011    ) -> Result<'source, Typed<ir::Expression>> {
3012        let make_error = |operand_type: String| Error::InvalidUnaryOperandType {
3013            span,
3014            op,
3015            operand_type,
3016        };
3017
3018        let expr = self.expression_for_abstract(expr, ctx)?;
3019        ctx.grow_types(expr)?;
3020        let expr_ty_resolution = resolve!(ctx, expr);
3021
3022        // All unary operators are only defined for scalars and vectors of scalars.
3023        let Some(kind) = expr_ty_resolution
3024            .inner_with(&ctx.module.types)
3025            .vector_size_and_scalar()
3026            .map(|(_, scalar)| scalar.kind)
3027        else {
3028            let operand_type = ctx.type_resolution_to_string(expr_ty_resolution);
3029            return Err(Box::new(make_error(operand_type)));
3030        };
3031        // validate preconditions
3032        match (op, kind) {
3033            // `T` is `bool` or `vecN<bool>`. These types have no automatic conversions.
3034            (ir::UnaryOperator::LogicalNot, ir::ScalarKind::Bool) => {}
3035
3036            // `T` is `AbstractInt`, `AbstractFloat`, `i32`, `f32`, `f16`,
3037            // `vecN<AbstractInt>`, `vecN<AbstractFloat>`, `vecN<i32>`, `vecN<f32>`, or `vecN<f16>`.
3038            (
3039                ir::UnaryOperator::Negate,
3040                ir::ScalarKind::AbstractInt
3041                | ir::ScalarKind::AbstractFloat
3042                | ir::ScalarKind::Sint
3043                | ir::ScalarKind::Float,
3044            ) => {}
3045
3046            // `S` is `AbstractInt`, `i32`, or `u32`.
3047            // `T` is `S` or `vecN<S>`.
3048            (
3049                ir::UnaryOperator::BitwiseNot,
3050                ir::ScalarKind::Sint | ir::ScalarKind::Uint | ir::ScalarKind::AbstractInt,
3051            ) => {}
3052
3053            _ => {
3054                let operand_type = ctx.type_resolution_to_string(expr_ty_resolution);
3055                return Err(Box::new(make_error(operand_type)));
3056            }
3057        }
3058
3059        Ok(Typed::Plain(ir::Expression::Unary { op, expr }))
3060    }
3061
3062    fn binary(
3063        &mut self,
3064        op: ir::BinaryOperator,
3065        left: Handle<ast::Expression<'source>>,
3066        right: Handle<ast::Expression<'source>>,
3067        span: Span,
3068        ctx: &mut ExpressionContext<'source, '_, '_>,
3069    ) -> Result<'source, Typed<ir::Expression>> {
3070        if op == ir::BinaryOperator::LogicalAnd || op == ir::BinaryOperator::LogicalOr {
3071            let left = self.expression_for_abstract(left, ctx)?;
3072            ctx.grow_types(left)?;
3073
3074            if !matches!(
3075                resolve_inner!(ctx, left),
3076                &ir::TypeInner::Scalar(ir::Scalar::BOOL)
3077            ) {
3078                // Pass it through as-is, will fail validation
3079                let right = self.expression_for_abstract(right, ctx)?;
3080                ctx.grow_types(right)?;
3081                Ok(Typed::Plain(crate::Expression::Binary { op, left, right }))
3082            } else {
3083                self.logical(op, left, right, span, ctx)
3084            }
3085        } else {
3086            // Load both operands.
3087            let mut left = self.expression_for_abstract(left, ctx)?;
3088            let mut right = self.expression_for_abstract(right, ctx)?;
3089
3090            // Convert `scalar op vector` to `vector op vector` by introducing
3091            // `Splat` expressions.
3092            ctx.binary_op_splat(op, &mut left, &mut right)?;
3093
3094            // Apply automatic conversions.
3095            match op {
3096                ir::BinaryOperator::ShiftLeft | ir::BinaryOperator::ShiftRight => {
3097                    // Shift operators require the right operand to be `u32` or
3098                    // `vecN<u32>`. We can let the validator sort out vector length
3099                    // issues, but the right operand must be, or convert to, a u32 leaf
3100                    // scalar.
3101                    right =
3102                        ctx.try_automatic_conversion_for_leaf_scalar(right, ir::Scalar::U32, span)?;
3103
3104                    // Additionally, we must concretize the left operand if the right operand
3105                    // is not a const-expression.
3106                    // See https://www.w3.org/TR/WGSL/#overload-resolution-section.
3107                    //
3108                    // 2. Eliminate any candidate where one of its subexpressions resolves to
3109                    // an abstract type after feasible automatic conversions, but another of
3110                    // the candidate’s subexpressions is not a const-expression.
3111                    //
3112                    // We only have to explicitly do so for shifts as their operands may be
3113                    // of different types - for other binary ops this is achieved by finding
3114                    // the conversion consensus for both operands.
3115                    if !ctx.is_const(right) {
3116                        left = ctx.concretize(left)?;
3117                    }
3118                }
3119
3120                // All other operators follow the same pattern: reconcile the
3121                // scalar leaf types. If there's no reconciliation possible,
3122                // leave the expressions as they are: validation will report the
3123                // problem.
3124                _ => {
3125                    ctx.grow_types(left)?;
3126                    ctx.grow_types(right)?;
3127                    if let Ok(consensus_scalar) =
3128                        ctx.automatic_conversion_consensus(None, [left, right].iter())
3129                    {
3130                        ctx.convert_to_leaf_scalar(&mut left, consensus_scalar)?;
3131                        ctx.convert_to_leaf_scalar(&mut right, consensus_scalar)?;
3132                    }
3133                }
3134            }
3135
3136            Ok(Typed::Plain(ir::Expression::Binary { op, left, right }))
3137        }
3138    }
3139
3140    /// Generate Naga IR for a call to a WGSL builtin function.
3141    #[allow(clippy::too_many_arguments)]
3142    fn call_builtin<'phrase>(
3143        &mut self,
3144        function_name: &'source str,
3145        function_span: Span,
3146        arguments: &[Handle<ast::Expression<'source>>],
3147        template_params: &mut TemplateListIter<'phrase, 'source>,
3148        call_span: Span,
3149        ctx: &mut ExpressionContext<'source, '_, '_>,
3150        is_statement: bool,
3151    ) -> Result<'source, Option<(Handle<ir::Expression>, MustUse)>> {
3152        let (expr, must_use) = if let Some(fun) = conv::map_relational_fun(function_name) {
3153            let mut args = ctx.prepare_args(arguments, 1, function_span);
3154            let argument = self.expression(args.next()?, ctx)?;
3155            args.finish()?;
3156
3157            // Check for no-op all(bool) and any(bool):
3158            let argument_unmodified = matches!(
3159                fun,
3160                ir::RelationalFunction::All | ir::RelationalFunction::Any
3161            ) && {
3162                matches!(
3163                    resolve_inner!(ctx, argument),
3164                    &ir::TypeInner::Scalar(ir::Scalar {
3165                        kind: ir::ScalarKind::Bool,
3166                        ..
3167                    })
3168                )
3169            };
3170
3171            if argument_unmodified {
3172                return Ok(Some((argument, MustUse::Yes)));
3173            } else {
3174                (ir::Expression::Relational { fun, argument }, MustUse::Yes)
3175            }
3176        } else if let Some((axis, ctrl)) = conv::map_derivative(function_name) {
3177            let mut args = ctx.prepare_args(arguments, 1, function_span);
3178            let expr = self.expression(args.next()?, ctx)?;
3179            args.finish()?;
3180
3181            (
3182                ir::Expression::Derivative { axis, ctrl, expr },
3183                MustUse::Yes,
3184            )
3185        } else if let Some(fun) = conv::map_standard_fun(function_name) {
3186            (
3187                self.math_function_helper(function_span, fun, arguments, ctx)?,
3188                MustUse::Yes,
3189            )
3190        } else if let Some(fun) = Texture::map(function_name) {
3191            (
3192                self.texture_sample_helper(fun, arguments, function_span, ctx)?,
3193                MustUse::Yes,
3194            )
3195        } else if let Some((op, cop)) = conv::map_subgroup_operation(function_name) {
3196            return Ok(Some((
3197                self.subgroup_operation_helper(function_span, op, cop, arguments, ctx)?,
3198                MustUse::Yes,
3199            )));
3200        } else if let Some(mode) = SubgroupGather::map(function_name) {
3201            return Ok(Some((
3202                self.subgroup_gather_helper(function_span, mode, arguments, ctx)?,
3203                MustUse::Yes,
3204            )));
3205        } else if let Some(fun) = ir::AtomicFunction::map(function_name) {
3206            return Ok(self
3207                .atomic_helper(function_span, fun, arguments, is_statement, ctx)?
3208                .map(|result| (result, MustUse::No)));
3209        } else {
3210            match function_name {
3211                "bitcast" => {
3212                    let ty = template_params.ty(self, ctx)?;
3213
3214                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3215                    let expr = self.expression_for_abstract(args.next()?, ctx)?;
3216                    args.finish()?;
3217                    ctx.grow_types(expr)?;
3218
3219                    let expr = self.bitcast_operand(function_span, ty, expr, ctx)?;
3220                    (ir::Expression::Bitcast { expr, ty }, MustUse::Yes)
3221                }
3222                "coopLoad" | "coopLoadT" => {
3223                    let row_major = function_name.ends_with("T");
3224                    let (matrix_ty, matrix_span) = template_params.ty_with_span(self, ctx)?;
3225
3226                    let mut args = ctx.prepare_args(arguments, 1, call_span);
3227                    let pointer = self.expression(args.next()?, ctx)?;
3228                    let (columns, rows, role) = match ctx.module.types[matrix_ty].inner {
3229                        ir::TypeInner::CooperativeMatrix {
3230                            columns,
3231                            rows,
3232                            role,
3233                            ..
3234                        } => (columns, rows, role),
3235                        _ => return Err(Box::new(Error::InvalidCooperativeLoadType(matrix_span))),
3236                    };
3237                    let stride = if args.total_args > 1 {
3238                        self.expression(args.next()?, ctx)?
3239                    } else {
3240                        // Infer the stride from the matrix type
3241                        let stride = if row_major {
3242                            columns as u32
3243                        } else {
3244                            rows as u32
3245                        };
3246                        ctx.append_expression(
3247                            ir::Expression::Literal(ir::Literal::U32(stride)),
3248                            Span::UNDEFINED,
3249                        )?
3250                    };
3251                    args.finish()?;
3252
3253                    (
3254                        crate::Expression::CooperativeLoad {
3255                            columns,
3256                            rows,
3257                            role,
3258                            data: crate::CooperativeData {
3259                                pointer,
3260                                stride,
3261                                row_major,
3262                            },
3263                        },
3264                        MustUse::Yes,
3265                    )
3266                }
3267                "select" => {
3268                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3269
3270                    let reject_orig = args.next()?;
3271                    let accept_orig = args.next()?;
3272                    let mut values = [
3273                        self.expression_for_abstract(reject_orig, ctx)?,
3274                        self.expression_for_abstract(accept_orig, ctx)?,
3275                    ];
3276                    let condition = self.expression(args.next()?, ctx)?;
3277
3278                    args.finish()?;
3279
3280                    let diagnostic_details =
3281                        |ctx: &ExpressionContext<'_, '_, '_>,
3282                         ty_res: &proc::TypeResolution,
3283                         orig_expr| {
3284                            (
3285                                ctx.ast_expressions.get_span(orig_expr),
3286                                format!("`{}`", ctx.as_diagnostic_display(ty_res)),
3287                            )
3288                        };
3289                    for (&value, orig_value) in values.iter().zip([reject_orig, accept_orig]) {
3290                        let value_ty_res = resolve!(ctx, value);
3291                        if value_ty_res
3292                            .inner_with(&ctx.module.types)
3293                            .vector_size_and_scalar()
3294                            .is_none()
3295                        {
3296                            let (arg_span, arg_type) =
3297                                diagnostic_details(ctx, value_ty_res, orig_value);
3298                            return Err(Box::new(Error::SelectUnexpectedArgumentType {
3299                                arg_span,
3300                                arg_type,
3301                            }));
3302                        }
3303                    }
3304                    let mut consensus_scalar = ctx
3305                        .automatic_conversion_consensus(None, &values)
3306                        .map_err(|_idx| {
3307                            let [reject, accept] = values;
3308                            let [(reject_span, reject_type), (accept_span, accept_type)] =
3309                                [(reject_orig, reject), (accept_orig, accept)].map(
3310                                    |(orig_expr, expr)| {
3311                                        let ty_res = &ctx.typifier()[expr];
3312                                        diagnostic_details(ctx, ty_res, orig_expr)
3313                                    },
3314                                );
3315                            Error::SelectRejectAndAcceptHaveNoCommonType {
3316                                reject_span,
3317                                reject_type,
3318                                accept_span,
3319                                accept_type,
3320                            }
3321                        })?;
3322                    if !ctx.is_const(condition) {
3323                        consensus_scalar = consensus_scalar.concretize();
3324                    }
3325
3326                    ctx.convert_slice_to_common_leaf_scalar(&mut values, consensus_scalar)?;
3327
3328                    let [reject, accept] = values;
3329
3330                    (
3331                        ir::Expression::Select {
3332                            reject,
3333                            accept,
3334                            condition,
3335                        },
3336                        MustUse::Yes,
3337                    )
3338                }
3339                "arrayLength" => {
3340                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3341                    let expr = self.expression(args.next()?, ctx)?;
3342                    args.finish()?;
3343
3344                    (ir::Expression::ArrayLength(expr), MustUse::Yes)
3345                }
3346                "atomicLoad" => {
3347                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3348                    let (pointer, _scalar) = self.atomic_pointer(args.next()?, ctx)?;
3349                    args.finish()?;
3350
3351                    (ir::Expression::Load { pointer }, MustUse::No)
3352                }
3353                "atomicStore" => {
3354                    let mut args = ctx.prepare_args(arguments, 2, function_span);
3355                    let (pointer, scalar) = self.atomic_pointer(args.next()?, ctx)?;
3356                    let value = self.expression_with_leaf_scalar(args.next()?, scalar, ctx)?;
3357                    args.finish()?;
3358
3359                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3360                    rctx.block
3361                        .extend(rctx.emitter.finish(&rctx.function.expressions));
3362                    rctx.emitter.start(&rctx.function.expressions);
3363                    rctx.block
3364                        .push(ir::Statement::Store { pointer, value }, function_span);
3365                    return Ok(None);
3366                }
3367                "atomicCompareExchangeWeak" => {
3368                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3369
3370                    let (pointer, scalar) = self.atomic_pointer(args.next()?, ctx)?;
3371
3372                    let compare = self.expression_with_leaf_scalar(args.next()?, scalar, ctx)?;
3373
3374                    let value = args.next()?;
3375                    let value_span = ctx.ast_expressions.get_span(value);
3376                    let value = self.expression_with_leaf_scalar(value, scalar, ctx)?;
3377
3378                    args.finish()?;
3379
3380                    let expression = match *resolve_inner!(ctx, value) {
3381                        ir::TypeInner::Scalar(scalar) => ir::Expression::AtomicResult {
3382                            ty: ctx.module.generate_predeclared_type(
3383                                ir::PredeclaredType::AtomicCompareExchangeWeakResult(scalar),
3384                            ),
3385                            comparison: true,
3386                        },
3387                        _ => return Err(Box::new(Error::InvalidAtomicOperandType(value_span))),
3388                    };
3389
3390                    let result = ctx.interrupt_emitter(expression, function_span)?;
3391                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3392                    rctx.block.push(
3393                        ir::Statement::Atomic {
3394                            pointer,
3395                            fun: ir::AtomicFunction::Exchange {
3396                                compare: Some(compare),
3397                            },
3398                            value,
3399                            result: Some(result),
3400                        },
3401                        function_span,
3402                    );
3403                    return Ok(Some((result, MustUse::No)));
3404                }
3405                "textureAtomicMin" | "textureAtomicMax" | "textureAtomicAdd"
3406                | "textureAtomicAnd" | "textureAtomicOr" | "textureAtomicXor" => {
3407                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3408
3409                    let image = args.next()?;
3410                    let image_span = ctx.ast_expressions.get_span(image);
3411                    let image = self.expression(image, ctx)?;
3412
3413                    let coordinate = self.expression(args.next()?, ctx)?;
3414
3415                    let (_, arrayed) = ctx.image_data(image, image_span)?;
3416                    let array_index = arrayed
3417                        .then(|| {
3418                            args.min_args += 1;
3419                            self.expression(args.next()?, ctx)
3420                        })
3421                        .transpose()?;
3422
3423                    let value = self.expression(args.next()?, ctx)?;
3424
3425                    args.finish()?;
3426
3427                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3428                    rctx.block
3429                        .extend(rctx.emitter.finish(&rctx.function.expressions));
3430                    rctx.emitter.start(&rctx.function.expressions);
3431                    let stmt = ir::Statement::ImageAtomic {
3432                        image,
3433                        coordinate,
3434                        array_index,
3435                        fun: match function_name {
3436                            "textureAtomicMin" => ir::AtomicFunction::Min,
3437                            "textureAtomicMax" => ir::AtomicFunction::Max,
3438                            "textureAtomicAdd" => ir::AtomicFunction::Add,
3439                            "textureAtomicAnd" => ir::AtomicFunction::And,
3440                            "textureAtomicOr" => ir::AtomicFunction::InclusiveOr,
3441                            "textureAtomicXor" => ir::AtomicFunction::ExclusiveOr,
3442                            _ => unreachable!(),
3443                        },
3444                        value,
3445                    };
3446                    rctx.block.push(stmt, function_span);
3447                    return Ok(None);
3448                }
3449                "storageBarrier" => {
3450                    ctx.prepare_args(arguments, 0, function_span).finish()?;
3451
3452                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3453                    rctx.block.push(
3454                        ir::Statement::ControlBarrier(ir::Barrier::STORAGE),
3455                        function_span,
3456                    );
3457                    return Ok(None);
3458                }
3459                "workgroupBarrier" => {
3460                    ctx.prepare_args(arguments, 0, function_span).finish()?;
3461
3462                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3463                    rctx.block.push(
3464                        ir::Statement::ControlBarrier(ir::Barrier::WORK_GROUP),
3465                        function_span,
3466                    );
3467                    return Ok(None);
3468                }
3469                "subgroupBarrier" => {
3470                    ctx.prepare_args(arguments, 0, function_span).finish()?;
3471
3472                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3473                    rctx.block.push(
3474                        ir::Statement::ControlBarrier(ir::Barrier::SUB_GROUP),
3475                        function_span,
3476                    );
3477                    return Ok(None);
3478                }
3479                "textureBarrier" => {
3480                    ctx.prepare_args(arguments, 0, function_span).finish()?;
3481
3482                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3483                    rctx.block.push(
3484                        ir::Statement::ControlBarrier(ir::Barrier::TEXTURE),
3485                        function_span,
3486                    );
3487                    return Ok(None);
3488                }
3489                "workgroupUniformLoad" => {
3490                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3491                    let expr = args.next()?;
3492                    args.finish()?;
3493
3494                    let pointer = self.expression(expr, ctx)?;
3495                    let result_ty = match *resolve_inner!(ctx, pointer) {
3496                        ir::TypeInner::Pointer {
3497                            base,
3498                            space: ir::AddressSpace::WorkGroup,
3499                        } => match ctx.module.types[base].inner {
3500                            // Match `Expression::Load` semantics:
3501                            // loading through a pointer to `atomic<T>` produces a `T`.
3502                            ir::TypeInner::Atomic(scalar) => ctx.module.types.insert(
3503                                ir::Type {
3504                                    name: None,
3505                                    inner: ir::TypeInner::Scalar(scalar),
3506                                },
3507                                function_span,
3508                            ),
3509                            _ => base,
3510                        },
3511                        ir::TypeInner::ValuePointer {
3512                            size,
3513                            scalar,
3514                            space: ir::AddressSpace::WorkGroup,
3515                        } => ctx.module.types.insert(
3516                            ir::Type {
3517                                name: None,
3518                                inner: match size {
3519                                    Some(size) => ir::TypeInner::Vector { size, scalar },
3520                                    None => ir::TypeInner::Scalar(scalar),
3521                                },
3522                            },
3523                            function_span,
3524                        ),
3525                        _ => {
3526                            let span = ctx.ast_expressions.get_span(expr);
3527                            return Err(Box::new(Error::InvalidWorkGroupUniformLoad(span)));
3528                        }
3529                    };
3530                    let result = ctx.interrupt_emitter(
3531                        ir::Expression::WorkGroupUniformLoadResult { ty: result_ty },
3532                        function_span,
3533                    )?;
3534                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3535                    rctx.block.push(
3536                        ir::Statement::WorkGroupUniformLoad { pointer, result },
3537                        function_span,
3538                    );
3539
3540                    return Ok(Some((result, MustUse::Yes)));
3541                }
3542                "textureStore" => {
3543                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3544
3545                    let image = args.next()?;
3546                    let image_span = ctx.ast_expressions.get_span(image);
3547                    let image = self.expression(image, ctx)?;
3548
3549                    let coordinate = self.expression(args.next()?, ctx)?;
3550
3551                    let (class, arrayed) = ctx.image_data(image, image_span)?;
3552                    let array_index = arrayed
3553                        .then(|| {
3554                            args.min_args += 1;
3555                            self.expression(args.next()?, ctx)
3556                        })
3557                        .transpose()?;
3558                    let scalar = if let ir::ImageClass::Storage { format, .. } = class {
3559                        format.into()
3560                    } else {
3561                        return Err(Box::new(Error::NotStorageTexture(image_span)));
3562                    };
3563
3564                    let value = self.expression_with_leaf_scalar(args.next()?, scalar, ctx)?;
3565
3566                    args.finish()?;
3567
3568                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3569                    rctx.block
3570                        .extend(rctx.emitter.finish(&rctx.function.expressions));
3571                    rctx.emitter.start(&rctx.function.expressions);
3572                    let stmt = ir::Statement::ImageStore {
3573                        image,
3574                        coordinate,
3575                        array_index,
3576                        value,
3577                    };
3578                    rctx.block.push(stmt, function_span);
3579                    return Ok(None);
3580                }
3581                "textureLoad" => {
3582                    let mut args = ctx.prepare_args(arguments, 2, function_span);
3583
3584                    let image = args.next()?;
3585                    let image_span = ctx.ast_expressions.get_span(image);
3586                    let image = self.expression(image, ctx)?;
3587
3588                    let coordinate = self.expression(args.next()?, ctx)?;
3589
3590                    let (class, arrayed) = ctx.image_data(image, image_span)?;
3591                    let array_index = arrayed
3592                        .then(|| {
3593                            args.min_args += 1;
3594                            self.expression(args.next()?, ctx)
3595                        })
3596                        .transpose()?;
3597
3598                    let level = class
3599                        .is_mipmapped()
3600                        .then(|| {
3601                            args.min_args += 1;
3602                            self.expression(args.next()?, ctx)
3603                        })
3604                        .transpose()?;
3605
3606                    let sample = class
3607                        .is_multisampled()
3608                        .then(|| self.expression(args.next()?, ctx))
3609                        .transpose()?;
3610
3611                    args.finish()?;
3612
3613                    (
3614                        ir::Expression::ImageLoad {
3615                            image,
3616                            coordinate,
3617                            array_index,
3618                            level,
3619                            sample,
3620                        },
3621                        MustUse::Yes,
3622                    )
3623                }
3624                "textureDimensions" => {
3625                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3626                    let image = self.expression(args.next()?, ctx)?;
3627                    let level = args
3628                        .next()
3629                        .map(|arg| self.expression(arg, ctx))
3630                        .ok()
3631                        .transpose()?;
3632                    args.finish()?;
3633
3634                    (
3635                        ir::Expression::ImageQuery {
3636                            image,
3637                            query: ir::ImageQuery::Size { level },
3638                        },
3639                        MustUse::Yes,
3640                    )
3641                }
3642                "textureNumLevels" => {
3643                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3644                    let image = self.expression(args.next()?, ctx)?;
3645                    args.finish()?;
3646
3647                    (
3648                        ir::Expression::ImageQuery {
3649                            image,
3650                            query: ir::ImageQuery::NumLevels,
3651                        },
3652                        MustUse::Yes,
3653                    )
3654                }
3655                "textureNumLayers" => {
3656                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3657                    let image = self.expression(args.next()?, ctx)?;
3658                    args.finish()?;
3659
3660                    (
3661                        ir::Expression::ImageQuery {
3662                            image,
3663                            query: ir::ImageQuery::NumLayers,
3664                        },
3665                        MustUse::Yes,
3666                    )
3667                }
3668                "textureNumSamples" => {
3669                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3670                    let image = self.expression(args.next()?, ctx)?;
3671                    args.finish()?;
3672
3673                    (
3674                        ir::Expression::ImageQuery {
3675                            image,
3676                            query: ir::ImageQuery::NumSamples,
3677                        },
3678                        MustUse::Yes,
3679                    )
3680                }
3681                "rayQueryInitialize" => {
3682                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3683                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3684                    let acceleration_structure = self.expression(args.next()?, ctx)?;
3685                    let descriptor = self.expression(args.next()?, ctx)?;
3686                    args.finish()?;
3687
3688                    let _ = ctx.module.generate_ray_desc_type();
3689                    let fun = ir::RayQueryFunction::Initialize {
3690                        acceleration_structure,
3691                        descriptor,
3692                    };
3693
3694                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3695                    rctx.block
3696                        .extend(rctx.emitter.finish(&rctx.function.expressions));
3697                    rctx.emitter.start(&rctx.function.expressions);
3698                    rctx.block
3699                        .push(ir::Statement::RayQuery { query, fun }, function_span);
3700                    return Ok(None);
3701                }
3702                "getCommittedHitVertexPositions" => {
3703                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3704                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3705                    args.finish()?;
3706
3707                    let _ = ctx.module.generate_vertex_return_type();
3708
3709                    (
3710                        ir::Expression::RayQueryVertexPositions {
3711                            query,
3712                            committed: true,
3713                        },
3714                        MustUse::No,
3715                    )
3716                }
3717                "getCandidateHitVertexPositions" => {
3718                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3719                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3720                    args.finish()?;
3721
3722                    let _ = ctx.module.generate_vertex_return_type();
3723
3724                    (
3725                        ir::Expression::RayQueryVertexPositions {
3726                            query,
3727                            committed: false,
3728                        },
3729                        MustUse::No,
3730                    )
3731                }
3732                "rayQueryProceed" => {
3733                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3734                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3735                    args.finish()?;
3736
3737                    let result = ctx
3738                        .interrupt_emitter(ir::Expression::RayQueryProceedResult, function_span)?;
3739                    let fun = ir::RayQueryFunction::Proceed { result };
3740                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3741                    rctx.block
3742                        .push(ir::Statement::RayQuery { query, fun }, function_span);
3743                    return Ok(Some((result, MustUse::No)));
3744                }
3745                "rayQueryGenerateIntersection" => {
3746                    let mut args = ctx.prepare_args(arguments, 2, function_span);
3747                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3748                    let hit_t = self.expression(args.next()?, ctx)?;
3749                    args.finish()?;
3750
3751                    let fun = ir::RayQueryFunction::GenerateIntersection { hit_t };
3752                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3753                    rctx.block
3754                        .push(ir::Statement::RayQuery { query, fun }, function_span);
3755                    return Ok(None);
3756                }
3757                "rayQueryConfirmIntersection" => {
3758                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3759                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3760                    args.finish()?;
3761
3762                    let fun = ir::RayQueryFunction::ConfirmIntersection;
3763                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3764                    rctx.block
3765                        .push(ir::Statement::RayQuery { query, fun }, function_span);
3766                    return Ok(None);
3767                }
3768                "rayQueryTerminate" => {
3769                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3770                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3771                    args.finish()?;
3772
3773                    let fun = ir::RayQueryFunction::Terminate;
3774                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3775                    rctx.block
3776                        .push(ir::Statement::RayQuery { query, fun }, function_span);
3777                    return Ok(None);
3778                }
3779                "rayQueryGetCommittedIntersection" => {
3780                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3781                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3782                    args.finish()?;
3783
3784                    let _ = ctx.module.generate_ray_intersection_type();
3785                    (
3786                        ir::Expression::RayQueryGetIntersection {
3787                            query,
3788                            committed: true,
3789                        },
3790                        MustUse::No,
3791                    )
3792                }
3793                "rayQueryGetCandidateIntersection" => {
3794                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3795                    let query = self.ray_query_pointer(args.next()?, ctx)?;
3796                    args.finish()?;
3797
3798                    let _ = ctx.module.generate_ray_intersection_type();
3799                    (
3800                        ir::Expression::RayQueryGetIntersection {
3801                            query,
3802                            committed: false,
3803                        },
3804                        MustUse::No,
3805                    )
3806                }
3807                "subgroupBallot" => {
3808                    let mut args = ctx.prepare_args(arguments, 0, function_span);
3809                    let predicate = if arguments.len() == 1 {
3810                        Some(self.expression(args.next()?, ctx)?)
3811                    } else {
3812                        None
3813                    };
3814                    args.finish()?;
3815
3816                    let result =
3817                        ctx.interrupt_emitter(ir::Expression::SubgroupBallotResult, function_span)?;
3818                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3819                    rctx.block.push(
3820                        ir::Statement::SubgroupBallot { result, predicate },
3821                        function_span,
3822                    );
3823                    return Ok(Some((result, MustUse::Yes)));
3824                }
3825                "quadSwapX" => {
3826                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3827
3828                    let argument = self.expression(args.next()?, ctx)?;
3829                    args.finish()?;
3830
3831                    let ty = ctx.register_type(argument)?;
3832
3833                    let result = ctx.interrupt_emitter(
3834                        crate::Expression::SubgroupOperationResult { ty },
3835                        function_span,
3836                    )?;
3837                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3838                    rctx.block.push(
3839                        crate::Statement::SubgroupGather {
3840                            mode: crate::GatherMode::QuadSwap(crate::Direction::X),
3841                            argument,
3842                            result,
3843                        },
3844                        function_span,
3845                    );
3846                    return Ok(Some((result, MustUse::Yes)));
3847                }
3848                "quadSwapY" => {
3849                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3850
3851                    let argument = self.expression(args.next()?, ctx)?;
3852                    args.finish()?;
3853
3854                    let ty = ctx.register_type(argument)?;
3855
3856                    let result = ctx.interrupt_emitter(
3857                        crate::Expression::SubgroupOperationResult { ty },
3858                        function_span,
3859                    )?;
3860                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3861                    rctx.block.push(
3862                        crate::Statement::SubgroupGather {
3863                            mode: crate::GatherMode::QuadSwap(crate::Direction::Y),
3864                            argument,
3865                            result,
3866                        },
3867                        function_span,
3868                    );
3869                    return Ok(Some((result, MustUse::Yes)));
3870                }
3871                "quadSwapDiagonal" => {
3872                    let mut args = ctx.prepare_args(arguments, 1, function_span);
3873
3874                    let argument = self.expression(args.next()?, ctx)?;
3875                    args.finish()?;
3876
3877                    let ty = ctx.register_type(argument)?;
3878
3879                    let result = ctx.interrupt_emitter(
3880                        crate::Expression::SubgroupOperationResult { ty },
3881                        function_span,
3882                    )?;
3883                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3884                    rctx.block.push(
3885                        crate::Statement::SubgroupGather {
3886                            mode: crate::GatherMode::QuadSwap(crate::Direction::Diagonal),
3887                            argument,
3888                            result,
3889                        },
3890                        function_span,
3891                    );
3892                    return Ok(Some((result, MustUse::Yes)));
3893                }
3894                "coopStore" | "coopStoreT" => {
3895                    let row_major = function_name.ends_with("T");
3896
3897                    let mut args = ctx.prepare_args(arguments, 2, function_span);
3898                    let target = self.expression(args.next()?, ctx)?;
3899                    let pointer = self.expression(args.next()?, ctx)?;
3900                    let stride = if args.total_args > 2 {
3901                        self.expression(args.next()?, ctx)?
3902                    } else {
3903                        // Infer the stride from the matrix type
3904                        let stride = match *resolve_inner!(ctx, target) {
3905                            ir::TypeInner::CooperativeMatrix { columns, rows, .. } => {
3906                                if row_major {
3907                                    columns as u32
3908                                } else {
3909                                    rows as u32
3910                                }
3911                            }
3912                            _ => 0,
3913                        };
3914                        ctx.append_expression(
3915                            ir::Expression::Literal(ir::Literal::U32(stride)),
3916                            Span::UNDEFINED,
3917                        )?
3918                    };
3919                    args.finish()?;
3920
3921                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3922                    rctx.block.push(
3923                        crate::Statement::CooperativeStore {
3924                            target,
3925                            data: crate::CooperativeData {
3926                                pointer,
3927                                stride,
3928                                row_major,
3929                            },
3930                        },
3931                        function_span,
3932                    );
3933                    return Ok(None);
3934                }
3935                "coopMultiplyAdd" => {
3936                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3937                    let a = self.expression(args.next()?, ctx)?;
3938                    let b = self.expression(args.next()?, ctx)?;
3939                    let c = self.expression(args.next()?, ctx)?;
3940                    args.finish()?;
3941
3942                    (
3943                        ir::Expression::CooperativeMultiplyAdd { a, b, c },
3944                        MustUse::Yes,
3945                    )
3946                }
3947                "hitObjectTraceRay"
3948                | "hitObjectRecordMiss"
3949                | "hitObjectRecordFromQuery"
3950                | "hitObjectRecordEmpty"
3951                | "hitObjectExecuteShader"
3952                | "hitObjectIsEmpty"
3953                | "hitObjectIsHit"
3954                | "hitObjectIsMiss"
3955                | "hitObjectGetIntersection"
3956                | "reorderThread" => {
3957                    match self.call_hit_object_builtin(
3958                        function_name,
3959                        function_span,
3960                        arguments,
3961                        ctx,
3962                    )? {
3963                        Some(expr) => (expr, MustUse::Yes),
3964                        None => return Ok(None),
3965                    }
3966                }
3967                "traceRay" => {
3968                    let mut args = ctx.prepare_args(arguments, 3, function_span);
3969                    let acceleration_structure = self.expression(args.next()?, ctx)?;
3970                    let descriptor = self.expression(args.next()?, ctx)?;
3971                    let payload = self.expression(args.next()?, ctx)?;
3972                    args.finish()?;
3973
3974                    let _ = ctx.module.generate_ray_desc_type();
3975                    let fun = ir::RayPipelineFunction::TraceRay {
3976                        acceleration_structure,
3977                        descriptor,
3978                        payload,
3979                    };
3980
3981                    let rctx = ctx.runtime_expression_ctx(function_span)?;
3982                    rctx.block
3983                        .extend(rctx.emitter.finish(&rctx.function.expressions));
3984                    rctx.emitter.start(&rctx.function.expressions);
3985                    rctx.block
3986                        .push(ir::Statement::RayPipelineFunction(fun), function_span);
3987                    return Ok(None);
3988                }
3989                "debugPrintf" => {
3990                    if !ctx
3991                        .enable_extensions
3992                        .contains(ImplementedEnableExtension::WgpuDebugPrintf)
3993                    {
3994                        return Err(Box::new(Error::EnableExtensionNotEnabled {
3995                            span: function_span,
3996                            kind: ImplementedEnableExtension::WgpuDebugPrintf.into(),
3997                        }));
3998                    }
3999
4000                    if arguments.is_empty() {
4001                        return Err(Box::new(Error::WrongArgumentCount {
4002                            expected: 1..u32::MAX,
4003                            found: 0,
4004                            span: function_span,
4005                        }));
4006                    }
4007
4008                    // extract the format string
4009                    let format_handle = arguments[0];
4010                    let format = match ctx.ast_expressions[format_handle] {
4011                        ast::Expression::String(s) => s.to_string(),
4012                        _ => {
4013                            return Err(Box::new(Error::ExpectedStringLiteral {
4014                                span: ctx.ast_expressions.get_span(format_handle),
4015                                description:
4016                                    "debugPrintf's first argument must be a string literal",
4017                            }))
4018                        }
4019                    };
4020
4021                    // extract remaining arguments (if any)
4022                    let mut ir_arguments = Vec::with_capacity(arguments.len().saturating_sub(1));
4023
4024                    for &ast_handle in &arguments[1..] {
4025                        let ir_handle = self.expression(ast_handle, ctx)?;
4026                        ir_arguments.push(ir_handle);
4027                    }
4028
4029                    let rctx = ctx.runtime_expression_ctx(function_span)?;
4030                    rctx.block
4031                        .extend(rctx.emitter.finish(&rctx.function.expressions));
4032                    rctx.emitter.start(&rctx.function.expressions);
4033                    rctx.block.push(
4034                        ir::Statement::DebugPrintf {
4035                            format,
4036                            arguments: ir_arguments,
4037                        },
4038                        function_span,
4039                    );
4040
4041                    return Ok(None);
4042                }
4043                _ => return Err(Box::new(Error::UnknownIdent(function_span, function_name))),
4044            }
4045        };
4046
4047        let expr = ctx.append_expression(expr, function_span)?;
4048        Ok(Some((expr, must_use)))
4049    }
4050
4051    /// Generate Naga IR for call expressions and statements, and type
4052    /// constructor expressions.
4053    ///
4054    /// The "function" being called is simply an `Ident` that we know refers to
4055    /// some module-scope definition.
4056    ///
4057    /// - If it is the name of a type, then the expression is a type constructor
4058    ///   expression: either constructing a value from components, a conversion
4059    ///   expression, or a zero value expression.
4060    ///
4061    /// - If it is the name of a function, then we're generating a [`Call`]
4062    ///   statement. We may be in the midst of generating code for an
4063    ///   expression, in which case we must generate an `Emit` statement to
4064    ///   force evaluation of the IR expressions we've generated so far, add the
4065    ///   `Call` statement to the current block, and then resume generating
4066    ///   expressions.
4067    ///
4068    /// [`Call`]: ir::Statement::Call
4069    fn call(
4070        &mut self,
4071        call_phrase: &ast::CallPhrase<'source>,
4072        span: Span,
4073        ctx: &mut ExpressionContext<'source, '_, '_>,
4074        is_statement: bool,
4075    ) -> Result<'source, Option<Handle<ir::Expression>>> {
4076        let function_name = match call_phrase.function.ident {
4077            ast::IdentExpr::Unresolved(name) => name,
4078            ast::IdentExpr::Local(_) => {
4079                return Err(Box::new(Error::CalledLocalDecl(
4080                    call_phrase.function.ident_span,
4081                )))
4082            }
4083        };
4084        let mut function_span = call_phrase.function.ident_span;
4085        function_span.subsume(call_phrase.function.template_list_span);
4086        let arguments = call_phrase.arguments.as_slice();
4087
4088        let mut tl = TemplateListIter::new(function_span, &call_phrase.function.template_list);
4089
4090        let result = match ctx.globals.get(function_name) {
4091            Some(&LoweredGlobalDecl::Type(ty)) => {
4092                // user-declared types can't make use of template lists
4093                tl.finish(ctx)?;
4094
4095                let handle =
4096                    self.construct(span, Constructor::Type(ty), function_span, arguments, ctx)?;
4097                Some((handle, MustUse::Yes))
4098            }
4099            Some(
4100                &LoweredGlobalDecl::Const(_)
4101                | &LoweredGlobalDecl::Override(_)
4102                | &LoweredGlobalDecl::Var(_),
4103            ) => {
4104                return Err(Box::new(Error::Unexpected(
4105                    function_span,
4106                    ExpectedToken::Function,
4107                )))
4108            }
4109            Some(&LoweredGlobalDecl::EntryPoint(_)) => {
4110                return Err(Box::new(Error::CalledEntryPoint(function_span)));
4111            }
4112            Some(&LoweredGlobalDecl::Function {
4113                handle: function,
4114                must_use,
4115            }) => {
4116                // user-declared functions can't make use of template lists
4117                tl.finish(ctx)?;
4118
4119                let arguments = arguments
4120                    .iter()
4121                    .enumerate()
4122                    .map(|(i, &arg)| {
4123                        // Try to convert abstract values to the known argument types
4124                        let Some(&ir::FunctionArgument {
4125                            ty: parameter_ty, ..
4126                        }) = ctx.module.functions[function].arguments.get(i)
4127                        else {
4128                            // Wrong number of arguments... just concretize the type here
4129                            // and let the validator report the error.
4130                            return self.expression(arg, ctx);
4131                        };
4132
4133                        let expr = self.expression_for_abstract(arg, ctx)?;
4134                        ctx.try_automatic_conversions(
4135                            expr,
4136                            &proc::TypeResolution::Handle(parameter_ty),
4137                            ctx.ast_expressions.get_span(arg),
4138                        )
4139                    })
4140                    .collect::<Result<Vec<_>>>()?;
4141
4142                let has_result = ctx.module.functions[function].result.is_some();
4143
4144                let rctx = ctx.runtime_expression_ctx(span)?;
4145                // we need to always do this before a fn call since all arguments need to be emitted before the fn call
4146                rctx.block
4147                    .extend(rctx.emitter.finish(&rctx.function.expressions));
4148                let result = has_result.then(|| {
4149                    let result = rctx
4150                        .function
4151                        .expressions
4152                        .append(ir::Expression::CallResult(function), span);
4153                    rctx.local_expression_kind_tracker
4154                        .insert(result, proc::ExpressionKind::Runtime);
4155                    (result, must_use.into())
4156                });
4157                rctx.emitter.start(&rctx.function.expressions);
4158                rctx.block.push(
4159                    ir::Statement::Call {
4160                        function,
4161                        arguments,
4162                        result: result.map(|(expr, _)| expr),
4163                    },
4164                    span,
4165                );
4166
4167                result
4168            }
4169            None => {
4170                // If the name refers to a predeclared type, this is a construction expression.
4171                let ty = conv::map_predeclared_type(
4172                    &ctx.enable_extensions,
4173                    function_span,
4174                    function_name,
4175                )?;
4176                if let Some(ty) = ty {
4177                    let empty_template_list = call_phrase.function.template_list.is_empty();
4178                    let constructor_ty = match ty {
4179                        conv::PredeclaredType::TypeGenerator(conv::TypeGenerator::Vector {
4180                            size,
4181                        }) if empty_template_list => Constructor::PartialVector { size },
4182                        conv::PredeclaredType::TypeGenerator(conv::TypeGenerator::Matrix {
4183                            columns,
4184                            rows,
4185                        }) if empty_template_list => Constructor::PartialMatrix { columns, rows },
4186                        conv::PredeclaredType::TypeGenerator(conv::TypeGenerator::Array)
4187                            if empty_template_list =>
4188                        {
4189                            Constructor::PartialArray
4190                        }
4191                        conv::PredeclaredType::TypeGenerator(
4192                            conv::TypeGenerator::CooperativeMatrix { .. },
4193                        ) if empty_template_list => {
4194                            return Err(Box::new(Error::UnderspecifiedCooperativeMatrix));
4195                        }
4196                        _ => Constructor::Type(self.finalize_type(ctx, ty, &mut tl, None)?),
4197                    };
4198                    tl.finish(ctx)?;
4199                    let handle =
4200                        self.construct(span, constructor_ty, function_span, arguments, ctx)?;
4201                    Some((handle, MustUse::Yes))
4202                } else {
4203                    // Otherwise, it must be a call to a builtin function.
4204                    let result = self.call_builtin(
4205                        function_name,
4206                        function_span,
4207                        arguments,
4208                        &mut tl,
4209                        span,
4210                        ctx,
4211                        is_statement,
4212                    )?;
4213                    tl.finish(ctx)?;
4214                    result
4215                }
4216            }
4217        };
4218
4219        let result_used = !is_statement;
4220        if matches!(result, Some((_, MustUse::Yes))) && !result_used {
4221            return Err(Box::new(Error::FunctionMustUseUnused(function_span)));
4222        }
4223        Ok(result.map(|(expr, _)| expr))
4224    }
4225
4226    /// Generate a Naga IR [`Math`] expression.
4227    ///
4228    /// Generate Naga IR for a call to the [`MathFunction`] `fun`, whose
4229    /// unlowered arguments are `ast_arguments`.
4230    ///
4231    /// The `span` argument should give the span of the function name in the
4232    /// call expression.
4233    ///
4234    /// [`Math`]: ir::Expression::Math
4235    /// [`MathFunction`]: ir::MathFunction
4236    fn math_function_helper(
4237        &mut self,
4238        span: Span,
4239        fun: ir::MathFunction,
4240        ast_arguments: &[Handle<ast::Expression<'source>>],
4241        ctx: &mut ExpressionContext<'source, '_, '_>,
4242    ) -> Result<'source, ir::Expression> {
4243        let mut lowered_arguments = Vec::with_capacity(ast_arguments.len());
4244        for &arg in ast_arguments {
4245            let lowered = self.expression_for_abstract(arg, ctx)?;
4246            ctx.grow_types(lowered)?;
4247            lowered_arguments.push(lowered);
4248        }
4249
4250        let fun_overloads = fun.overloads();
4251        let rule = self.resolve_overloads(span, fun, fun_overloads, &lowered_arguments, ctx)?;
4252        self.apply_automatic_conversions_for_call(&rule, &mut lowered_arguments, ctx)?;
4253
4254        self.check_bitfield_range(span, fun, &rule, &lowered_arguments, ctx)?;
4255
4256        // If this function returns a predeclared type, register it
4257        // in `Module::special_types`. The typifier will expect to
4258        // be able to find it there.
4259        if let proc::Conclusion::Predeclared(predeclared) = rule.conclusion {
4260            ctx.module.generate_predeclared_type(predeclared);
4261        }
4262
4263        Ok(ir::Expression::Math {
4264            fun,
4265            arg: lowered_arguments[0],
4266            arg1: lowered_arguments.get(1).cloned(),
4267            arg2: lowered_arguments.get(2).cloned(),
4268            arg3: lowered_arguments.get(3).cloned(),
4269        })
4270    }
4271
4272    /// Check the `offset` and `count` arguments of a call to [`ExtractBits`]
4273    /// or [`InsertBits`] to make sure it fits inside the number's `width`.
4274    ///
4275    /// If `offset` and `count` are fixed numbers and they pick bits outside
4276    /// the range we must return a shader creation error.
4277    ///
4278    /// If the pipeline overrides the numbers, we do not check them here.
4279    /// [`get_const_val`] reports them as non-const, so they fall through
4280    /// this check.
4281    ///
4282    /// [`ExtractBits`]: ir::MathFunction::ExtractBits
4283    /// [`InsertBits`]: ir::MathFunction::InsertBits
4284    /// [`get_const_val`]: ExpressionContext::get_const_val
4285    fn check_bitfield_range(
4286        &mut self,
4287        span: Span,
4288        fun: ir::MathFunction,
4289        rule: &proc::Rule,
4290        arguments: &[Handle<ir::Expression>],
4291        ctx: &ExpressionContext<'source, '_, '_>,
4292    ) -> Result<'source, ()> {
4293        let (builtin, offset, count) = match fun {
4294            ir::MathFunction::ExtractBits => ("extractBits", 1, 2),
4295            ir::MathFunction::InsertBits => ("insertBits", 2, 3),
4296            _ => return Ok(()),
4297        };
4298
4299        let (Ok(offset), Ok(count)) = (
4300            ctx.get_const_val::<u32>(arguments[offset]),
4301            ctx.get_const_val::<u32>(arguments[count]),
4302        ) else {
4303            return Ok(());
4304        };
4305
4306        let Some(scalar) = rule.arguments[0].inner_with(&ctx.module.types).scalar() else {
4307            return Ok(());
4308        };
4309
4310        proc::check_bitfield_range(builtin, offset, count, u32::from(scalar.width) * 8)
4311            .map_err(|e| Box::new(Error::ConstantEvaluatorError(Box::new(e), span)))
4312    }
4313
4314    /// Choose the right overload for a function call.
4315    ///
4316    /// Return a [`Rule`] representing the most preferred overload in
4317    /// `overloads` to apply to `arguments`, or return an error explaining why
4318    /// the call is not valid.
4319    ///
4320    /// Use `fun` to identify the function being called in error messages;
4321    /// `span` should be the span of the function name in the call expression.
4322    ///
4323    /// [`Rule`]: proc::Rule
4324    fn resolve_overloads<O, F>(
4325        &self,
4326        span: Span,
4327        fun: F,
4328        overloads: O,
4329        arguments: &[Handle<ir::Expression>],
4330        ctx: &ExpressionContext<'source, '_, '_>,
4331    ) -> Result<'source, proc::Rule>
4332    where
4333        O: proc::OverloadSet,
4334        F: TryToWgsl + core::fmt::Debug + Copy,
4335    {
4336        let mut remaining_overloads = overloads.clone();
4337        let min_arguments = remaining_overloads.min_arguments();
4338        let max_arguments = remaining_overloads.max_arguments();
4339        if arguments.len() < min_arguments {
4340            return Err(Box::new(Error::WrongArgumentCount {
4341                span,
4342                expected: min_arguments as u32..max_arguments as u32,
4343                found: arguments.len() as u32,
4344            }));
4345        }
4346        if arguments.len() > max_arguments {
4347            return Err(Box::new(Error::TooManyArguments {
4348                function: fun.to_wgsl_for_diagnostics(),
4349                call_span: span,
4350                arg_span: ctx.get_expression_span(arguments[max_arguments]),
4351                max_arguments: max_arguments as _,
4352            }));
4353        }
4354
4355        log::debug!(
4356            "Initial overloads: {:#?}",
4357            remaining_overloads.for_debug(&ctx.module.types)
4358        );
4359
4360        for (arg_index, &arg) in arguments.iter().enumerate() {
4361            let arg_type_resolution = &ctx.typifier()[arg];
4362            let arg_inner = arg_type_resolution.inner_with(&ctx.module.types);
4363            log::debug!(
4364                "Supplying argument {arg_index} of type {:?}",
4365                arg_type_resolution.for_debug(&ctx.module.types)
4366            );
4367            let next_remaining_overloads =
4368                remaining_overloads.arg(arg_index, arg_inner, &ctx.module.types);
4369
4370            // If any argument is not a constant expression, then no overloads
4371            // that accept abstract values should be considered.
4372            // (`OverloadSet::concrete_only` is supposed to help impose this
4373            // restriction.) However, no `MathFunction` accepts a mix of
4374            // abstract and concrete arguments, so we don't need to worry
4375            // about that here.
4376
4377            log::debug!(
4378                "Remaining overloads: {:#?}",
4379                next_remaining_overloads.for_debug(&ctx.module.types)
4380            );
4381
4382            // If the set of remaining overloads is empty, then this argument's type
4383            // was unacceptable. Diagnose the problem and produce an error message.
4384            if next_remaining_overloads.is_empty() {
4385                let function = fun.to_wgsl_for_diagnostics();
4386                let call_span = span;
4387                let arg_span = ctx.get_expression_span(arg);
4388                let arg_ty = ctx.as_diagnostic_display(arg_type_resolution).to_string();
4389
4390                // Is this type *ever* permitted for the arg_index'th argument?
4391                // For example, `bool` is never permitted for `max`.
4392                let only_this_argument = overloads.arg(arg_index, arg_inner, &ctx.module.types);
4393                if only_this_argument.is_empty() {
4394                    // No overload of `fun` accepts this type as the
4395                    // arg_index'th argument. Determine the set of types that
4396                    // would ever be allowed there.
4397                    let allowed: Vec<String> = overloads
4398                        .allowed_args(arg_index, &ctx.module.to_ctx())
4399                        .iter()
4400                        .map(|ty| ctx.type_resolution_to_string(ty))
4401                        .collect();
4402
4403                    if allowed.is_empty() {
4404                        // No overload of `fun` accepts any argument at this
4405                        // index, so it's a simple case of excess arguments.
4406                        // However, since each `MathFunction`'s overloads all
4407                        // have the same arity, we should have detected this
4408                        // earlier.
4409                        unreachable!("expected all overloads to have the same arity");
4410                    }
4411
4412                    // Some overloads of `fun` do accept this many arguments,
4413                    // but none accept one of this type.
4414                    return Err(Box::new(Error::WrongArgumentType {
4415                        function,
4416                        call_span,
4417                        arg_span,
4418                        arg_index: arg_index as u32,
4419                        arg_ty,
4420                        allowed,
4421                    }));
4422                }
4423
4424                // This argument's type is accepted by some overloads---just
4425                // not those overloads that remain, given the prior arguments.
4426                // For example, `max` accepts `f32` as its second argument -
4427                // but not if the first was `i32`.
4428
4429                // Build a list of the types that would have been accepted here,
4430                // given the prior arguments.
4431                let allowed: Vec<String> = remaining_overloads
4432                    .allowed_args(arg_index, &ctx.module.to_ctx())
4433                    .iter()
4434                    .map(|ty| ctx.type_resolution_to_string(ty))
4435                    .collect();
4436
4437                // Re-run the argument list to determine which prior argument
4438                // made this one unacceptable.
4439                let mut remaining_overloads = overloads;
4440                for (prior_index, &prior_expr) in arguments.iter().enumerate() {
4441                    let prior_type_resolution = &ctx.typifier()[prior_expr];
4442                    let prior_ty = prior_type_resolution.inner_with(&ctx.module.types);
4443                    remaining_overloads =
4444                        remaining_overloads.arg(prior_index, prior_ty, &ctx.module.types);
4445                    if remaining_overloads
4446                        .arg(arg_index, arg_inner, &ctx.module.types)
4447                        .is_empty()
4448                    {
4449                        // This is the argument that killed our dreams.
4450                        let inconsistent_span = ctx.get_expression_span(arguments[prior_index]);
4451                        let inconsistent_ty =
4452                            ctx.as_diagnostic_display(prior_type_resolution).to_string();
4453
4454                        if allowed.is_empty() {
4455                            // Some overloads did accept `ty` at `arg_index`, but
4456                            // given the arguments up through `prior_expr`, we see
4457                            // no types acceptable at `arg_index`. This means that some
4458                            // overloads expect fewer arguments than others. However,
4459                            // each `MathFunction`'s overloads have the same arity, so this
4460                            // should be impossible.
4461                            unreachable!("expected all overloads to have the same arity");
4462                        }
4463
4464                        // Report `arg`'s type as inconsistent with `prior_expr`'s
4465                        return Err(Box::new(Error::InconsistentArgumentType {
4466                            function,
4467                            call_span,
4468                            arg_span,
4469                            arg_index: arg_index as u32,
4470                            arg_ty,
4471                            inconsistent_span,
4472                            inconsistent_index: prior_index as u32,
4473                            inconsistent_ty,
4474                            allowed,
4475                        }));
4476                    }
4477                }
4478                unreachable!("Failed to eliminate argument type when re-tried");
4479            }
4480            remaining_overloads = next_remaining_overloads;
4481        }
4482
4483        // Select the most preferred type rule for this call,
4484        // given the argument types supplied above.
4485        Ok(remaining_overloads.most_preferred())
4486    }
4487
4488    /// Apply automatic type conversions for a function call.
4489    ///
4490    /// Apply whatever automatic conversions are needed to pass `arguments` to
4491    /// the function overload described by `rule`. Update `arguments` to refer
4492    /// to the converted arguments.
4493    fn apply_automatic_conversions_for_call(
4494        &self,
4495        rule: &proc::Rule,
4496        arguments: &mut [Handle<ir::Expression>],
4497        ctx: &mut ExpressionContext<'source, '_, '_>,
4498    ) -> Result<'source, ()> {
4499        for (i, argument) in arguments.iter_mut().enumerate() {
4500            let goal_inner = rule.arguments[i].inner_with(&ctx.module.types);
4501            let converted = match goal_inner.scalar_for_conversions(&ctx.module.types) {
4502                Some(goal_scalar) => {
4503                    let arg_span = ctx.get_expression_span(*argument);
4504                    ctx.try_automatic_conversion_for_leaf_scalar(*argument, goal_scalar, arg_span)?
4505                }
4506                // No conversion is necessary.
4507                None => *argument,
4508            };
4509
4510            *argument = converted;
4511        }
4512
4513        Ok(())
4514    }
4515
4516    fn atomic_pointer(
4517        &mut self,
4518        expr: Handle<ast::Expression<'source>>,
4519        ctx: &mut ExpressionContext<'source, '_, '_>,
4520    ) -> Result<'source, (Handle<ir::Expression>, ir::Scalar)> {
4521        let span = ctx.ast_expressions.get_span(expr);
4522        let pointer = self.expression(expr, ctx)?;
4523
4524        match *resolve_inner!(ctx, pointer) {
4525            ir::TypeInner::Pointer { base, .. } => match ctx.module.types[base].inner {
4526                ir::TypeInner::Atomic(scalar) => Ok((pointer, scalar)),
4527                ref other => {
4528                    log::error!("Pointer type to {other:?} passed to atomic op");
4529                    Err(Box::new(Error::InvalidAtomicPointer(span)))
4530                }
4531            },
4532            ref other => {
4533                log::error!("Type {other:?} passed to atomic op");
4534                Err(Box::new(Error::InvalidAtomicPointer(span)))
4535            }
4536        }
4537    }
4538
4539    fn atomic_helper(
4540        &mut self,
4541        span: Span,
4542        fun: ir::AtomicFunction,
4543        args: &[Handle<ast::Expression<'source>>],
4544        is_statement: bool,
4545        ctx: &mut ExpressionContext<'source, '_, '_>,
4546    ) -> Result<'source, Option<Handle<ir::Expression>>> {
4547        let mut args = ctx.prepare_args(args, 2, span);
4548
4549        let (pointer, scalar) = self.atomic_pointer(args.next()?, ctx)?;
4550        let value = self.expression_with_leaf_scalar(args.next()?, scalar, ctx)?;
4551        let value_inner = resolve_inner!(ctx, value);
4552        args.finish()?;
4553
4554        // If we don't use the return value of a 64-bit `min` or `max`
4555        // operation, generate a no-result form of the `Atomic` statement, so
4556        // that we can pass validation with only `SHADER_INT64_ATOMIC_MIN_MAX`
4557        // whenever possible.
4558        let is_64_bit_min_max = matches!(fun, ir::AtomicFunction::Min | ir::AtomicFunction::Max)
4559            && matches!(
4560                *value_inner,
4561                ir::TypeInner::Scalar(ir::Scalar { width: 8, .. })
4562            );
4563        let result = if is_64_bit_min_max && is_statement {
4564            let rctx = ctx.runtime_expression_ctx(span)?;
4565            rctx.block
4566                .extend(rctx.emitter.finish(&rctx.function.expressions));
4567            rctx.emitter.start(&rctx.function.expressions);
4568            None
4569        } else {
4570            let ty = ctx.register_type(value)?;
4571            Some(ctx.interrupt_emitter(
4572                ir::Expression::AtomicResult {
4573                    ty,
4574                    comparison: false,
4575                },
4576                span,
4577            )?)
4578        };
4579        let rctx = ctx.runtime_expression_ctx(span)?;
4580        rctx.block.push(
4581            ir::Statement::Atomic {
4582                pointer,
4583                fun,
4584                value,
4585                result,
4586            },
4587            span,
4588        );
4589        Ok(result)
4590    }
4591
4592    fn texture_sample_helper(
4593        &mut self,
4594        fun: Texture,
4595        args: &[Handle<ast::Expression<'source>>],
4596        span: Span,
4597        ctx: &mut ExpressionContext<'source, '_, '_>,
4598    ) -> Result<'source, ir::Expression> {
4599        let mut args = ctx.prepare_args(args, fun.min_argument_count(), span);
4600
4601        fn get_image_and_span<'source>(
4602            lowerer: &mut Lowerer<'source, '_>,
4603            args: &mut ArgumentContext<'_, 'source>,
4604            ctx: &mut ExpressionContext<'source, '_, '_>,
4605        ) -> Result<'source, (Handle<ir::Expression>, Span)> {
4606            let image = args.next()?;
4607            let image_span = ctx.ast_expressions.get_span(image);
4608            let image = lowerer.expression_for_abstract(image, ctx)?;
4609            Ok((image, image_span))
4610        }
4611
4612        let image;
4613        let image_span;
4614        let gather;
4615        match fun {
4616            Texture::Gather => {
4617                let image_or_component = args.next()?;
4618                let image_or_component_span = ctx.ast_expressions.get_span(image_or_component);
4619                // Gathers from depth textures don't take an initial `component` argument.
4620                let lowered_image_or_component = self.expression(image_or_component, ctx)?;
4621
4622                match *resolve_inner!(ctx, lowered_image_or_component) {
4623                    ir::TypeInner::Image {
4624                        class: ir::ImageClass::Depth { .. },
4625                        ..
4626                    } => {
4627                        image = lowered_image_or_component;
4628                        image_span = image_or_component_span;
4629                        gather = Some(ir::SwizzleComponent::X);
4630                    }
4631                    _ => {
4632                        (image, image_span) = get_image_and_span(self, &mut args, ctx)?;
4633                        gather = Some(ctx.gather_component(
4634                            lowered_image_or_component,
4635                            image_or_component_span,
4636                            span,
4637                        )?);
4638                    }
4639                }
4640            }
4641            Texture::GatherCompare => {
4642                (image, image_span) = get_image_and_span(self, &mut args, ctx)?;
4643                gather = Some(ir::SwizzleComponent::X);
4644            }
4645
4646            _ => {
4647                (image, image_span) = get_image_and_span(self, &mut args, ctx)?;
4648                gather = None;
4649            }
4650        };
4651
4652        let sampler = self.expression_for_abstract(args.next()?, ctx)?;
4653
4654        let coordinate = self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4655        let clamp_to_edge = matches!(fun, Texture::SampleBaseClampToEdge);
4656
4657        let (class, arrayed) = ctx.image_data(image, image_span)?;
4658        let array_index = arrayed
4659            .then(|| self.expression(args.next()?, ctx))
4660            .transpose()?;
4661
4662        let level;
4663        let depth_ref;
4664        match fun {
4665            Texture::Gather => {
4666                level = ir::SampleLevel::Zero;
4667                depth_ref = None;
4668            }
4669            Texture::GatherCompare => {
4670                let reference =
4671                    self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4672                level = ir::SampleLevel::Zero;
4673                depth_ref = Some(reference);
4674            }
4675
4676            Texture::Sample => {
4677                level = ir::SampleLevel::Auto;
4678                depth_ref = None;
4679            }
4680            Texture::SampleBias => {
4681                let bias = self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4682                level = ir::SampleLevel::Bias(bias);
4683                depth_ref = None;
4684            }
4685            Texture::SampleCompare => {
4686                let reference =
4687                    self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4688                level = ir::SampleLevel::Auto;
4689                depth_ref = Some(reference);
4690            }
4691            Texture::SampleCompareLevel => {
4692                let reference =
4693                    self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4694                level = ir::SampleLevel::Zero;
4695                depth_ref = Some(reference);
4696            }
4697            Texture::SampleGrad => {
4698                let x = self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4699                let y = self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?;
4700                level = ir::SampleLevel::Gradient { x, y };
4701                depth_ref = None;
4702            }
4703            Texture::SampleLevel => {
4704                let exact = match class {
4705                    // When applied to depth textures, `textureSampleLevel`'s
4706                    // `level` argument is an `i32` or `u32`.
4707                    ir::ImageClass::Depth { .. } => self.expression(args.next()?, ctx)?,
4708
4709                    // When applied to other sampled types, its `level` argument
4710                    // is an `f32`.
4711                    ir::ImageClass::Sampled { .. } => {
4712                        self.expression_with_leaf_scalar(args.next()?, ir::Scalar::F32, ctx)?
4713                    }
4714
4715                    // Sampling `External` textures with a specified level isn't
4716                    // allowed, and sampling `Storage` textures isn't allowed at
4717                    // all. Let the validator report the error.
4718                    ir::ImageClass::Storage { .. } | ir::ImageClass::External => {
4719                        self.expression(args.next()?, ctx)?
4720                    }
4721                };
4722                level = ir::SampleLevel::Exact(exact);
4723                depth_ref = None;
4724            }
4725            Texture::SampleBaseClampToEdge => {
4726                level = crate::SampleLevel::Zero;
4727                depth_ref = None;
4728            }
4729        };
4730
4731        let offset = args
4732            .next()
4733            .map(|arg| self.expression_with_leaf_scalar(arg, ir::Scalar::I32, &mut ctx.as_const()))
4734            .ok()
4735            .transpose()?;
4736
4737        args.finish()?;
4738
4739        Ok(ir::Expression::ImageSample {
4740            image,
4741            sampler,
4742            gather,
4743            coordinate,
4744            array_index,
4745            offset,
4746            level,
4747            depth_ref,
4748            clamp_to_edge,
4749        })
4750    }
4751
4752    fn subgroup_operation_helper(
4753        &mut self,
4754        span: Span,
4755        op: ir::SubgroupOperation,
4756        collective_op: ir::CollectiveOperation,
4757        arguments: &[Handle<ast::Expression<'source>>],
4758        ctx: &mut ExpressionContext<'source, '_, '_>,
4759    ) -> Result<'source, Handle<ir::Expression>> {
4760        let mut args = ctx.prepare_args(arguments, 1, span);
4761
4762        let argument = self.expression(args.next()?, ctx)?;
4763        args.finish()?;
4764
4765        let ty = ctx.register_type(argument)?;
4766
4767        let result = ctx.interrupt_emitter(ir::Expression::SubgroupOperationResult { ty }, span)?;
4768        let rctx = ctx.runtime_expression_ctx(span)?;
4769        rctx.block.push(
4770            ir::Statement::SubgroupCollectiveOperation {
4771                op,
4772                collective_op,
4773                argument,
4774                result,
4775            },
4776            span,
4777        );
4778        Ok(result)
4779    }
4780
4781    /// Check that `bitcast<ty>(expr)` is one of the overloads of
4782    /// [WGSL's `bitcast`][spec], and return `expr`, concretized if it was abstract.
4783    ///
4784    /// Naga also accepts same-shaped bitcasts between numeric types of the
4785    /// same width, such as `i64` and `u64`, or `f16` and `i16`.
4786    ///
4787    /// [spec]: https://www.w3.org/TR/WGSL/#bitcast-builtin
4788    fn bitcast_operand(
4789        &mut self,
4790        span: Span,
4791        ty: Handle<ir::Type>,
4792        mut expr: Handle<ir::Expression>,
4793        ctx: &mut ExpressionContext<'source, '_, '_>,
4794    ) -> Result<'source, Handle<ir::Expression>> {
4795        use ir::{Scalar, ScalarKind as Sk, TypeInner as Ti, VectorSize as Vs};
4796
4797        let bad_cast = |ctx: &ExpressionContext<'source, '_, '_>, expr| {
4798            Box::new(Error::BadTypeCast {
4799                from_type: ctx.type_resolution_to_string(&ctx.typifier()[expr]),
4800                span,
4801                to_type: ctx.type_to_string(ty),
4802            })
4803        };
4804
4805        let (to_size, to_scalar) = match ctx.module.types[ty].inner {
4806            Ti::Scalar(scalar) => (None, scalar),
4807            Ti::Vector { size, scalar } => (Some(size), scalar),
4808            _ => return Err(bad_cast(ctx, expr)),
4809        };
4810
4811        let (from_size, mut from_scalar) = match *resolve_inner!(ctx, expr) {
4812            Ti::Scalar(scalar) => (None, scalar),
4813            Ti::Vector { size, scalar } => (Some(size), scalar),
4814            _ => return Err(bad_cast(ctx, expr)),
4815        };
4816
4817        // Pick the overload with the best conversion rank for an abstract operand.
4818        if from_scalar.is_abstract() {
4819            let same_shape = from_size == to_size;
4820            let concrete = match from_scalar.kind {
4821                Sk::AbstractInt if same_shape && matches!(to_scalar.kind, Sk::Sint | Sk::Uint) => {
4822                    to_scalar
4823                }
4824                Sk::AbstractInt if same_shape && to_scalar == Scalar::F16 => to_scalar,
4825                Sk::AbstractInt => Scalar::I32,
4826                _ if same_shape && to_scalar.kind == Sk::Float => to_scalar,
4827                _ => Scalar::F32,
4828            };
4829            expr = ctx.try_automatic_conversion_for_leaf_scalar(expr, concrete, span)?;
4830            ctx.grow_types(expr)?;
4831            from_scalar = concrete;
4832        }
4833
4834        let is_bit_pattern =
4835            |scalar: Scalar| matches!(scalar.kind, Sk::Sint | Sk::Uint | Sk::Float);
4836        let is_32_bit = |scalar: Scalar| is_bit_pattern(scalar) && scalar.width == 4;
4837        let is_f16 = |scalar: Scalar| scalar == Scalar::F16;
4838
4839        let valid = match (from_size, from_scalar, to_size, to_scalar) {
4840            (Some(Vs::Bi), half, None, word) | (None, word, Some(Vs::Bi), half) if is_f16(half) => {
4841                is_32_bit(word)
4842            }
4843            (Some(Vs::Quad), half, Some(Vs::Bi), word)
4844            | (Some(Vs::Bi), word, Some(Vs::Quad), half)
4845                if is_f16(half) =>
4846            {
4847                is_32_bit(word)
4848            }
4849            _ => {
4850                from_size == to_size
4851                    && from_scalar.width == to_scalar.width
4852                    && is_bit_pattern(from_scalar)
4853                    && is_bit_pattern(to_scalar)
4854            }
4855        };
4856
4857        if valid {
4858            Ok(expr)
4859        } else {
4860            Err(bad_cast(ctx, expr))
4861        }
4862    }
4863
4864    fn subgroup_gather_helper(
4865        &mut self,
4866        span: Span,
4867        mode: SubgroupGather,
4868        arguments: &[Handle<ast::Expression<'source>>],
4869        ctx: &mut ExpressionContext<'source, '_, '_>,
4870    ) -> Result<'source, Handle<ir::Expression>> {
4871        let mut args = ctx.prepare_args(arguments, 2, span);
4872
4873        let argument = self.expression(args.next()?, ctx)?;
4874
4875        use SubgroupGather as Sg;
4876        let mode = if let Sg::BroadcastFirst = mode {
4877            ir::GatherMode::BroadcastFirst
4878        } else {
4879            let index = self.expression(args.next()?, ctx)?;
4880            match mode {
4881                Sg::BroadcastFirst => unreachable!(),
4882                Sg::Broadcast => ir::GatherMode::Broadcast(index),
4883                Sg::Shuffle => ir::GatherMode::Shuffle(index),
4884                Sg::ShuffleDown => ir::GatherMode::ShuffleDown(index),
4885                Sg::ShuffleUp => ir::GatherMode::ShuffleUp(index),
4886                Sg::ShuffleXor => ir::GatherMode::ShuffleXor(index),
4887                Sg::QuadBroadcast => ir::GatherMode::QuadBroadcast(index),
4888            }
4889        };
4890
4891        args.finish()?;
4892
4893        let ty = ctx.register_type(argument)?;
4894
4895        let result = ctx.interrupt_emitter(ir::Expression::SubgroupOperationResult { ty }, span)?;
4896        let rctx = ctx.runtime_expression_ctx(span)?;
4897        rctx.block.push(
4898            ir::Statement::SubgroupGather {
4899                mode,
4900                argument,
4901                result,
4902            },
4903            span,
4904        );
4905        Ok(result)
4906    }
4907
4908    fn r#struct(
4909        &mut self,
4910        s: &ast::Struct<'source>,
4911        span: Span,
4912        ctx: &mut GlobalContext<'source, '_, '_>,
4913    ) -> Result<'source, Handle<ir::Type>> {
4914        let mut offset = 0;
4915        let mut struct_alignment = proc::Alignment::ONE;
4916        let mut members = Vec::with_capacity(s.members.len());
4917
4918        let mut doc_comments: Vec<Option<Vec<String>>> = Vec::new();
4919
4920        for member in s.members.iter() {
4921            let ty = self.resolve_ast_type(&member.ty, &mut ctx.as_const())?;
4922
4923            ctx.layouter.update(ctx.module.to_ctx()).map_err(|err| {
4924                let LayoutErrorInner::TooLarge = err.inner else {
4925                    unreachable!("unexpected layout error: {err:?}");
4926                };
4927                // Since anonymous types of struct members don't get a span,
4928                // associate the error with the member. The layouter could have
4929                // failed on any type that was pending layout, but if it wasn't
4930                // the current struct member, it wasn't a struct member at all,
4931                // because we resolve struct members one-by-one.
4932                if ty == err.ty {
4933                    Box::new(Error::StructMemberTooLarge {
4934                        member_name_span: member.name.span,
4935                    })
4936                } else {
4937                    // Lots of type definitions don't get spans, so this error
4938                    // message may not be very useful.
4939                    Box::new(Error::TypeTooLarge {
4940                        span: ctx.module.types.get_span(err.ty),
4941                    })
4942                }
4943            })?;
4944
4945            let member_min_size = ctx.layouter[ty].size;
4946            let member_min_alignment = ctx.layouter[ty].alignment;
4947
4948            let member_size = if let Some(size_expr) = member.size {
4949                let (size, span) = self.const_u32(size_expr, &mut ctx.as_const())?;
4950                if let ir::TypeInner::Array {
4951                    size: ir::ArraySize::Dynamic | ir::ArraySize::Pending(_),
4952                    ..
4953                } = ctx.module.types[ty].inner
4954                {
4955                    return Err(Box::new(Error::SizeAttributeRequiresFixedFootprint(span)));
4956                }
4957                if size < member_min_size {
4958                    return Err(Box::new(Error::SizeAttributeTooLow(span, member_min_size)));
4959                } else {
4960                    size
4961                }
4962            } else {
4963                member_min_size
4964            };
4965
4966            let member_alignment = if let Some(align_expr) = member.align {
4967                let (align, span) = self.const_u32(align_expr, &mut ctx.as_const())?;
4968                if let Some(alignment) = proc::Alignment::new(align) {
4969                    if alignment < member_min_alignment {
4970                        return Err(Box::new(Error::AlignAttributeTooLow(
4971                            span,
4972                            member_min_alignment,
4973                        )));
4974                    } else {
4975                        alignment
4976                    }
4977                } else {
4978                    return Err(Box::new(Error::NonPowerOfTwoAlignAttribute(span)));
4979                }
4980            } else {
4981                member_min_alignment
4982            };
4983
4984            let binding = self.binding(&member.binding, ty, ctx)?;
4985
4986            offset = member_alignment.round_up(offset);
4987            struct_alignment = struct_alignment.max(member_alignment);
4988
4989            if !member.doc_comments.is_empty() {
4990                doc_comments.push(Some(
4991                    member.doc_comments.iter().map(|s| s.to_string()).collect(),
4992                ));
4993            } else {
4994                doc_comments.push(None);
4995            }
4996            members.push(ir::StructMember {
4997                name: Some(member.name.name.to_owned()),
4998                ty,
4999                binding,
5000                offset,
5001            });
5002
5003            offset += member_size;
5004            if offset > crate::valid::MAX_TYPE_SIZE {
5005                return Err(Box::new(Error::TypeTooLarge { span }));
5006            }
5007        }
5008
5009        let size = struct_alignment.round_up(offset);
5010        let inner = ir::TypeInner::Struct {
5011            members,
5012            span: size,
5013        };
5014
5015        let handle = ctx.module.types.insert(
5016            ir::Type {
5017                name: Some(s.name.name.to_string()),
5018                inner,
5019            },
5020            span,
5021        );
5022        for (i, c) in doc_comments.drain(..).enumerate() {
5023            if let Some(comment) = c {
5024                ctx.module
5025                    .get_or_insert_default_doc_comments()
5026                    .struct_members
5027                    .insert((handle, i), comment);
5028            }
5029        }
5030        Ok(handle)
5031    }
5032
5033    fn const_u32(
5034        &mut self,
5035        expr: Handle<ast::Expression<'source>>,
5036        ctx: &mut ExpressionContext<'source, '_, '_>,
5037    ) -> Result<'source, (u32, Span)> {
5038        let span = ctx.ast_expressions.get_span(expr);
5039        let expr = self.expression(expr, ctx)?;
5040        let value = ctx
5041            .module
5042            .to_ctx()
5043            .get_const_val(expr)
5044            .map_err(|err| match err {
5045                proc::ConstValueError::NonConst | proc::ConstValueError::InvalidType => {
5046                    Error::ExpectedConstExprConcreteIntegerScalar(span)
5047                }
5048                proc::ConstValueError::Negative => Error::ExpectedNonNegative(span),
5049            })?;
5050        Ok((value, span))
5051    }
5052
5053    fn array_size(
5054        &mut self,
5055        expr: Handle<ast::Expression<'source>>,
5056        ctx: &mut ExpressionContext<'source, '_, '_>,
5057    ) -> Result<'source, ir::ArraySize> {
5058        let span = ctx.ast_expressions.get_span(expr);
5059        let const_ctx = &mut ctx.as_const();
5060        let const_expr = self.expression(expr, const_ctx);
5061        match const_expr {
5062            Ok(value) => {
5063                let len = const_ctx.get_const_val(value).map_err(|err| {
5064                    Box::new(match err {
5065                        proc::ConstValueError::NonConst | proc::ConstValueError::InvalidType => {
5066                            Error::ExpectedConstExprConcreteIntegerScalar(span)
5067                        }
5068                        proc::ConstValueError::Negative => Error::ExpectedPositiveArrayLength(span),
5069                    })
5070                })?;
5071                let size = NonZeroU32::new(len).ok_or(Error::ExpectedPositiveArrayLength(span))?;
5072                Ok(ir::ArraySize::Constant(size))
5073            }
5074            Err(err) => {
5075                // If the error is simply that `expr` was an override expression, then we
5076                // can represent that as an array length.
5077                let Error::ConstantEvaluatorError(ref ty, _) = *err else {
5078                    return Err(err);
5079                };
5080
5081                let proc::ConstantEvaluatorError::OverrideExpr = **ty else {
5082                    return Err(err);
5083                };
5084
5085                Ok(ir::ArraySize::Pending(self.array_size_override(
5086                    expr,
5087                    &mut ctx.as_global().as_override(),
5088                    span,
5089                )?))
5090            }
5091        }
5092    }
5093
5094    fn array_size_override(
5095        &mut self,
5096        size_expr: Handle<ast::Expression<'source>>,
5097        ctx: &mut ExpressionContext<'source, '_, '_>,
5098        span: Span,
5099    ) -> Result<'source, Handle<ir::Override>> {
5100        let expr = self.expression(size_expr, ctx)?;
5101        match resolve_inner!(ctx, expr).scalar_kind().ok_or(0) {
5102            Ok(ir::ScalarKind::Sint) | Ok(ir::ScalarKind::Uint) => Ok({
5103                if let ir::Expression::Override(handle) = ctx.module.global_expressions[expr] {
5104                    handle
5105                } else {
5106                    let ty = ctx.register_type(expr)?;
5107                    ctx.module.overrides.append(
5108                        ir::Override {
5109                            name: None,
5110                            id: None,
5111                            ty,
5112                            init: Some(expr),
5113                        },
5114                        span,
5115                    )
5116                }
5117            }),
5118            _ => Err(Box::new(Error::ExpectedConstExprConcreteIntegerScalar(
5119                span,
5120            ))),
5121        }
5122    }
5123
5124    /// Build the Naga equivalent of a named AST type.
5125    ///
5126    /// Return a Naga `Handle<Type>` representing the front-end type
5127    /// `handle`, which should be named `name`, if given.
5128    ///
5129    /// If `handle` refers to a type cached in [`SpecialTypes`],
5130    /// `name` may be ignored.
5131    ///
5132    /// [`SpecialTypes`]: ir::SpecialTypes
5133    fn resolve_named_ast_type(
5134        &mut self,
5135        ident: &ast::TemplateElaboratedIdent<'source>,
5136        name: String,
5137        ctx: &mut ExpressionContext<'source, '_, '_>,
5138    ) -> Result<'source, Handle<ir::Type>> {
5139        self.type_specifier(ident, ctx, Some(name))
5140    }
5141
5142    /// Return a Naga `Handle<Type>` representing the front-end type `handle`.
5143    fn resolve_ast_type(
5144        &mut self,
5145        ident: &ast::TemplateElaboratedIdent<'source>,
5146        ctx: &mut ExpressionContext<'source, '_, '_>,
5147    ) -> Result<'source, Handle<ir::Type>> {
5148        self.type_specifier(ident, ctx, None)
5149    }
5150
5151    fn binding(
5152        &mut self,
5153        binding: &Option<ast::Binding<'source>>,
5154        ty: Handle<ir::Type>,
5155        ctx: &mut GlobalContext<'source, '_, '_>,
5156    ) -> Result<'source, Option<ir::Binding>> {
5157        Ok(match *binding {
5158            Some(ast::Binding::BuiltIn(b)) => Some(ir::Binding::BuiltIn(b)),
5159            Some(ast::Binding::Location {
5160                location,
5161                interpolation,
5162                sampling,
5163                blend_src,
5164                per_primitive,
5165            }) => {
5166                let blend_src = if let Some(blend_src) = blend_src {
5167                    Some(self.const_u32(blend_src, &mut ctx.as_const())?.0)
5168                } else {
5169                    None
5170                };
5171
5172                let mut binding = ir::Binding::Location {
5173                    location: self.const_u32(location, &mut ctx.as_const())?.0,
5174                    interpolation,
5175                    sampling,
5176                    blend_src,
5177                    per_primitive,
5178                };
5179                binding.apply_default_interpolation(&ctx.module.types[ty].inner);
5180                Some(binding)
5181            }
5182            None => None,
5183        })
5184    }
5185
5186    /// Generate Naga IR for a call to one of the `hitObject*` built-ins, or to
5187    /// `reorderThread`.
5188    ///
5189    /// Returns `Ok(None)` if the call lowered to a statement rather than an
5190    /// expression. Every hit object built-in that does produce a value is
5191    /// [`MustUse::Yes`], so the caller does not need to be told.
5192    ///
5193    /// This is deliberately kept out of [`Self::call_builtin`]: that function
5194    /// recurses through the whole expression tree, so its stack frame is
5195    /// budgeted (see `clippy.toml`'s `stack-size-threshold`), and these
5196    /// built-ins' locals do not need to be part of it.
5197    fn call_hit_object_builtin(
5198        &mut self,
5199        function_name: &'source str,
5200        function_span: Span,
5201        arguments: &[Handle<ast::Expression<'source>>],
5202        ctx: &mut ExpressionContext<'source, '_, '_>,
5203    ) -> Result<'source, Option<ir::Expression>> {
5204        ctx.enable_extensions.require(
5205            ImplementedEnableExtension::WgpuRayTracingInvocationReorder,
5206            function_span,
5207        )?;
5208
5209        Ok(Some(match function_name {
5210            "hitObjectTraceRay" => {
5211                let mut args = ctx.prepare_args(arguments, 4, function_span);
5212                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5213                let acceleration_structure = self.expression(args.next()?, ctx)?;
5214                let descriptor = self.expression(args.next()?, ctx)?;
5215                let payload = self.expression(args.next()?, ctx)?;
5216                args.finish()?;
5217
5218                let _ = ctx.module.generate_ray_desc_type();
5219                let fun = ir::HitObjectFunction::TraceRay {
5220                    acceleration_structure,
5221                    descriptor,
5222                    payload,
5223                };
5224
5225                let rctx = ctx.runtime_expression_ctx(function_span)?;
5226                rctx.block
5227                    .extend(rctx.emitter.finish(&rctx.function.expressions));
5228                rctx.emitter.start(&rctx.function.expressions);
5229                rctx.block
5230                    .push(ir::Statement::HitObject { hit_object, fun }, function_span);
5231                return Ok(None);
5232            }
5233            "hitObjectRecordMiss" => {
5234                let mut args = ctx.prepare_args(arguments, 2, function_span);
5235                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5236                let descriptor = self.expression(args.next()?, ctx)?;
5237                args.finish()?;
5238
5239                let _ = ctx.module.generate_ray_desc_type();
5240                let fun = ir::HitObjectFunction::RecordMiss { descriptor };
5241
5242                let rctx = ctx.runtime_expression_ctx(function_span)?;
5243                rctx.block
5244                    .extend(rctx.emitter.finish(&rctx.function.expressions));
5245                rctx.emitter.start(&rctx.function.expressions);
5246                rctx.block
5247                    .push(ir::Statement::HitObject { hit_object, fun }, function_span);
5248                return Ok(None);
5249            }
5250            "hitObjectRecordFromQuery" => {
5251                let mut args = ctx.prepare_args(arguments, 2, function_span);
5252                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5253                let query = self.ray_query_pointer(args.next()?, ctx)?;
5254                args.finish()?;
5255
5256                let fun = ir::HitObjectFunction::RecordFromQuery { query };
5257
5258                let rctx = ctx.runtime_expression_ctx(function_span)?;
5259                rctx.block
5260                    .extend(rctx.emitter.finish(&rctx.function.expressions));
5261                rctx.emitter.start(&rctx.function.expressions);
5262                rctx.block
5263                    .push(ir::Statement::HitObject { hit_object, fun }, function_span);
5264                return Ok(None);
5265            }
5266            "hitObjectRecordEmpty" => {
5267                let mut args = ctx.prepare_args(arguments, 1, function_span);
5268                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5269                args.finish()?;
5270
5271                let fun = ir::HitObjectFunction::RecordEmpty;
5272
5273                let rctx = ctx.runtime_expression_ctx(function_span)?;
5274                rctx.block
5275                    .extend(rctx.emitter.finish(&rctx.function.expressions));
5276                rctx.emitter.start(&rctx.function.expressions);
5277                rctx.block
5278                    .push(ir::Statement::HitObject { hit_object, fun }, function_span);
5279                return Ok(None);
5280            }
5281            "hitObjectExecuteShader" => {
5282                let mut args = ctx.prepare_args(arguments, 2, function_span);
5283                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5284                let payload = self.expression(args.next()?, ctx)?;
5285                args.finish()?;
5286
5287                let fun = ir::HitObjectFunction::ExecuteShader { payload };
5288
5289                let rctx = ctx.runtime_expression_ctx(function_span)?;
5290                rctx.block
5291                    .extend(rctx.emitter.finish(&rctx.function.expressions));
5292                rctx.emitter.start(&rctx.function.expressions);
5293                rctx.block
5294                    .push(ir::Statement::HitObject { hit_object, fun }, function_span);
5295                return Ok(None);
5296            }
5297            "hitObjectIsEmpty" | "hitObjectIsHit" | "hitObjectIsMiss" => {
5298                let mut args = ctx.prepare_args(arguments, 1, function_span);
5299                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5300                args.finish()?;
5301
5302                let query = match function_name {
5303                    "hitObjectIsEmpty" => ir::HitObjectQuery::IsEmpty,
5304                    "hitObjectIsHit" => ir::HitObjectQuery::IsHit,
5305                    _ => ir::HitObjectQuery::IsMiss,
5306                };
5307
5308                ir::Expression::HitObjectQuery { hit_object, query }
5309            }
5310            "hitObjectGetIntersection" => {
5311                let mut args = ctx.prepare_args(arguments, 1, function_span);
5312                let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5313                args.finish()?;
5314
5315                let _ = ctx.module.generate_ray_intersection_type();
5316
5317                ir::Expression::HitObjectQuery {
5318                    hit_object,
5319                    query: ir::HitObjectQuery::Intersection,
5320                }
5321            }
5322            "reorderThread" => {
5323                match arguments.len() {
5324                    // `reorderThread(hint, bits)`
5325                    2 => {
5326                        let mut args = ctx.prepare_args(arguments, 2, function_span);
5327                        let hint =
5328                            self.expression_with_leaf_scalar(args.next()?, ir::Scalar::U32, ctx)?;
5329                        let bits =
5330                            self.expression_with_leaf_scalar(args.next()?, ir::Scalar::U32, ctx)?;
5331                        args.finish()?;
5332
5333                        let fun = ir::RayPipelineFunction::ReorderThread { hint, bits };
5334
5335                        let rctx = ctx.runtime_expression_ctx(function_span)?;
5336                        rctx.block
5337                            .extend(rctx.emitter.finish(&rctx.function.expressions));
5338                        rctx.emitter.start(&rctx.function.expressions);
5339                        rctx.block
5340                            .push(ir::Statement::RayPipelineFunction(fun), function_span);
5341                    }
5342                    // `reorderThread(&hit_object)` and
5343                    // `reorderThread(&hit_object, hint, bits)`
5344                    arg_count @ (1 | 3) => {
5345                        let arg_count = arg_count as u32;
5346                        let mut args = ctx.prepare_args(arguments, arg_count, function_span);
5347                        let hit_object = self.hit_object_pointer(args.next()?, ctx)?;
5348                        let hint = self.reorder_hint(arg_count == 3, &mut args, ctx)?;
5349                        args.finish()?;
5350
5351                        let fun = ir::HitObjectFunction::Reorder { hint };
5352
5353                        let rctx = ctx.runtime_expression_ctx(function_span)?;
5354                        rctx.block
5355                            .extend(rctx.emitter.finish(&rctx.function.expressions));
5356                        rctx.emitter.start(&rctx.function.expressions);
5357                        rctx.block
5358                            .push(ir::Statement::HitObject { hit_object, fun }, function_span);
5359                    }
5360                    found => {
5361                        return Err(Box::new(Error::WrongArgumentCount {
5362                            span: function_span,
5363                            expected: 1..4,
5364                            found: found as u32,
5365                        }))
5366                    }
5367                }
5368
5369                return Ok(None);
5370            }
5371            _ => unreachable!("unhandled ray tracing invocation reorder built-in"),
5372        }))
5373    }
5374
5375    /// Lower the trailing `hint: u32, bits: u32` arguments of an invocation
5376    /// reordering built-in, if `present` says the call supplied them.
5377    fn reorder_hint(
5378        &mut self,
5379        present: bool,
5380        args: &mut ArgumentContext<'_, 'source>,
5381        ctx: &mut ExpressionContext<'source, '_, '_>,
5382    ) -> Result<'source, Option<ir::ReorderHint>> {
5383        if !present {
5384            return Ok(None);
5385        }
5386
5387        let hint = self.expression_with_leaf_scalar(args.next()?, ir::Scalar::U32, ctx)?;
5388        let bits = self.expression_with_leaf_scalar(args.next()?, ir::Scalar::U32, ctx)?;
5389        Ok(Some(ir::ReorderHint { hint, bits }))
5390    }
5391
5392    fn hit_object_pointer(
5393        &mut self,
5394        expr: Handle<ast::Expression<'source>>,
5395        ctx: &mut ExpressionContext<'source, '_, '_>,
5396    ) -> Result<'source, Handle<ir::Expression>> {
5397        let span = ctx.ast_expressions.get_span(expr);
5398        let pointer = self.expression(expr, ctx)?;
5399
5400        match *resolve_inner!(ctx, pointer) {
5401            ir::TypeInner::Pointer { base, .. } => match ctx.module.types[base].inner {
5402                ir::TypeInner::HitObject => Ok(pointer),
5403                ref other => {
5404                    log::error!("Pointer type to {other:?} passed to hit object op");
5405                    Err(Box::new(Error::InvalidHitObjectPointer(span)))
5406                }
5407            },
5408            ref other => {
5409                log::error!("Type {other:?} passed to hit object op");
5410                Err(Box::new(Error::InvalidHitObjectPointer(span)))
5411            }
5412        }
5413    }
5414
5415    fn ray_query_pointer(
5416        &mut self,
5417        expr: Handle<ast::Expression<'source>>,
5418        ctx: &mut ExpressionContext<'source, '_, '_>,
5419    ) -> Result<'source, Handle<ir::Expression>> {
5420        let span = ctx.ast_expressions.get_span(expr);
5421        let pointer = self.expression(expr, ctx)?;
5422
5423        match *resolve_inner!(ctx, pointer) {
5424            ir::TypeInner::Pointer { base, .. } => match ctx.module.types[base].inner {
5425                ir::TypeInner::RayQuery { .. } => Ok(pointer),
5426                ref other => {
5427                    log::error!("Pointer type to {other:?} passed to ray query op");
5428                    Err(Box::new(Error::InvalidRayQueryPointer(span)))
5429                }
5430            },
5431            ref other => {
5432                log::error!("Type {other:?} passed to ray query op");
5433                Err(Box::new(Error::InvalidRayQueryPointer(span)))
5434            }
5435        }
5436    }
5437}
5438
5439impl ir::AtomicFunction {
5440    pub fn map(word: &str) -> Option<Self> {
5441        Some(match word {
5442            "atomicAdd" => ir::AtomicFunction::Add,
5443            "atomicSub" => ir::AtomicFunction::Subtract,
5444            "atomicAnd" => ir::AtomicFunction::And,
5445            "atomicOr" => ir::AtomicFunction::InclusiveOr,
5446            "atomicXor" => ir::AtomicFunction::ExclusiveOr,
5447            "atomicMin" => ir::AtomicFunction::Min,
5448            "atomicMax" => ir::AtomicFunction::Max,
5449            "atomicExchange" => ir::AtomicFunction::Exchange { compare: None },
5450            _ => return None,
5451        })
5452    }
5453}