Skip to main content

naga/back/
pipeline_constants.rs

1use alloc::{
2    borrow::Cow,
3    boxed::Box,
4    string::{String, ToString},
5    vec::Vec,
6};
7use core::mem;
8
9use hashbrown::HashSet;
10use thiserror::Error;
11
12use super::PipelineConstants;
13use crate::{
14    arena::HandleVec,
15    compact::{compact, KeepUnused},
16    ir,
17    proc::{ConstantEvaluator, ConstantEvaluatorError, Emitter},
18    valid::{Capabilities, ModuleInfo, ValidationError, ValidationFlags, Validator},
19    Arena, Block, Constant, Expression, Function, Handle, Literal, Module, Override, Range, Scalar,
20    Span, Statement, TypeInner, WithSpan,
21};
22
23// Possibly unused if not compiled with no_std
24#[allow(unused_imports)]
25use num_traits::float::FloatCore as _;
26
27#[derive(Error, Debug, Clone)]
28#[cfg_attr(test, derive(PartialEq))]
29pub enum PipelineConstantError {
30    #[error("Missing value for pipeline-overridable constant with identifier string: '{0}'")]
31    MissingValue(String),
32    #[error("pipeline-overridable constant '{0}' not found in the shader")]
33    NotFound(String),
34    #[error(
35        "Source f64 value needs to be finite ({}) for number destinations",
36        "NaNs and Inifinites are not allowed"
37    )]
38    SrcNeedsToBeFinite,
39    #[error("Source f64 value doesn't fit in destination")]
40    DstRangeTooSmall,
41    #[error(transparent)]
42    ConstantEvaluatorError(#[from] ConstantEvaluatorError),
43    #[error(transparent)]
44    ValidationError(#[from] Box<WithSpan<ValidationError>>),
45    #[error("workgroup_size override isn't strictly positive")]
46    NegativeWorkgroupSize,
47    #[error("max vertices or max primitives is negative")]
48    NegativeMeshOutputMax,
49}
50
51/// Compact `module` and replace all overrides with constants.
52///
53/// `module` must be valid. Both compaction and constant evaluation may produce
54/// invalid results (e.g. replace an invalid expression with a constant) for
55/// invalid modules.
56///
57/// If `entry_point` is specified, remove all other entry points from the
58/// returned module. Without this, re-validation will fail if any entry point
59/// uses an override whose value wasn't provided.
60///
61/// If no changes are needed, this just returns `Cow::Borrowed` references to
62/// `module` and `module_info`. Otherwise, it clones `module`, retains only the
63/// selected entry point, compacts the module, edits its [`global_expressions`]
64/// arena to contain only fully-evaluated expressions, and returns the
65/// simplified module and its validation results.
66///
67/// The module returned has an empty `overrides` arena, and the
68/// `global_expressions` arena contains only fully-evaluated expressions.
69///
70/// [`global_expressions`]: Module::global_expressions
71pub fn process_overrides<'a>(
72    module: &'a Module,
73    module_info: &'a ModuleInfo,
74    entry_point: Option<(ir::ShaderStage, &str)>,
75    pipeline_constants: &PipelineConstants,
76) -> Result<(Cow<'a, Module>, Cow<'a, ModuleInfo>), PipelineConstantError> {
77    let mut handles = module
78        .overrides
79        .iter()
80        .map(|(handle, _)| handle)
81        .collect::<Vec<_>>();
82    for c in pipeline_constants.keys() {
83        let c_id = c.parse().ok();
84        if let Some((i, _)) = handles.iter().enumerate().find(|&(_, handle)| {
85            let o = &module.overrides[*handle];
86            if o.id.is_some() {
87                o.id == c_id
88            } else {
89                o.name.as_deref() == Some(c.as_str())
90            }
91        }) {
92            handles.swap_remove(i);
93        } else {
94            return Err(PipelineConstantError::NotFound(c.clone()));
95        }
96    }
97
98    if (entry_point.is_none() || module.entry_points.len() <= 1) && module.overrides.is_empty() {
99        // We skip compacting the module here mostly to reduce the risk of
100        // hitting corner cases like https://github.com/gfx-rs/wgpu/issues/7793.
101        // Compaction doesn't cost very much [1], so it would also be reasonable
102        // to do it unconditionally. Even when there is a single entry point or
103        // when no entry point is specified, it is still possible that there
104        // are unreferenced items in the module that would be removed by this
105        // compaction.
106        //
107        // [1]: https://github.com/gfx-rs/wgpu/pull/7703#issuecomment-2902153760
108        return Ok((Cow::Borrowed(module), Cow::Borrowed(module_info)));
109    }
110
111    let mut module = module.clone();
112    if let Some((ep_stage, ep_name)) = entry_point {
113        module
114            .entry_points
115            .retain(|ep| ep.stage == ep_stage && ep.name == ep_name);
116    }
117
118    // Compact the module to remove anything not reachable from an entry point.
119    // This is necessary because we may not have values for overrides that are
120    // not reachable from the/an entry point.
121    compact(&mut module, KeepUnused::No);
122
123    // If there are no overrides in the module, then we can skip the rest.
124    if module.overrides.is_empty() {
125        return revalidate(module);
126    }
127
128    // A map from override handles to the handles of the constants
129    // we've replaced them with.
130    let mut override_map = HandleVec::with_capacity(module.overrides.len());
131
132    // A map from `module`'s original global expression handles to
133    // handles in the new, simplified global expression arena.
134    let mut adjusted_global_expressions = HandleVec::with_capacity(module.global_expressions.len());
135
136    // The set of constants whose initializer handles we've already
137    // updated to refer to the newly built global expression arena.
138    //
139    // All constants in `module` must have their `init` handles
140    // updated to point into the new, simplified global expression
141    // arena. Some of these we can most easily handle as a side effect
142    // during the simplification process, but we must handle the rest
143    // in a final fixup pass, guided by `adjusted_global_expressions`. We
144    // add their handles to this set, so that the final fixup step can
145    // leave them alone.
146    let mut adjusted_constant_initializers = HashSet::with_capacity(module.constants.len());
147
148    let mut global_expression_kind_tracker = crate::proc::ExpressionKindTracker::new();
149    let mut layouter = crate::proc::Layouter::default();
150
151    // An iterator through the original overrides table, consumed in
152    // approximate tandem with the global expressions.
153    let mut overrides = module.overrides.take();
154    let mut override_iter = overrides.iter_mut_span();
155
156    // Do two things in tandem:
157    //
158    // - Rebuild the global expression arena from scratch, fully
159    //   evaluating all expressions, and replacing each `Override`
160    //   expression in `module.global_expressions` with a `Constant`
161    //   expression.
162    //
163    // - Build a new `Constant` in `module.constants` to take the
164    //   place of each `Override`.
165    //
166    // Build a map from old global expression handles to their
167    // fully-evaluated counterparts in `adjusted_global_expressions` as we
168    // go.
169    //
170    // Why in tandem? Overrides refer to expressions, and expressions
171    // refer to overrides, so we can't disentangle the two into
172    // separate phases. However, we can take advantage of the fact
173    // that the overrides and expressions must form a DAG, and work
174    // our way from the leaves to the roots, replacing and evaluating
175    // as we go.
176    //
177    // Although the two loops are nested, this is really two
178    // alternating phases: we adjust and evaluate constant expressions
179    // until we hit an `Override` expression, at which point we switch
180    // to building `Constant`s for `Overrides` until we've handled the
181    // one used by the expression. Then we switch back to processing
182    // expressions. Because we know they form a DAG, we know the
183    // `Override` expressions we encounter can only have initializers
184    // referring to global expressions we've already simplified.
185    for (old_h, expr, span) in module.global_expressions.drain() {
186        let mut expr = match expr {
187            Expression::Override(h) => {
188                let c_h = if let Some(new_h) = override_map.get(h) {
189                    *new_h
190                } else {
191                    let mut new_h = None;
192                    for entry in override_iter.by_ref() {
193                        let stop = entry.0 == h;
194                        new_h = Some(process_override(
195                            entry,
196                            pipeline_constants,
197                            &mut module,
198                            &mut override_map,
199                            &adjusted_global_expressions,
200                            &mut adjusted_constant_initializers,
201                            &mut global_expression_kind_tracker,
202                        )?);
203                        if stop {
204                            break;
205                        }
206                    }
207                    new_h.unwrap()
208                };
209                Expression::Constant(c_h)
210            }
211            Expression::Constant(c_h) => {
212                if adjusted_constant_initializers.insert(c_h) {
213                    let init = &mut module.constants[c_h].init;
214                    *init = adjusted_global_expressions[*init];
215                }
216                expr
217            }
218            expr => expr,
219        };
220        let mut evaluator = ConstantEvaluator::for_wgsl_module(
221            &mut module,
222            &mut global_expression_kind_tracker,
223            &mut layouter,
224            false,
225        );
226        adjust_expr(&adjusted_global_expressions, &mut expr);
227        let h = evaluator.try_eval_and_append(expr, span)?;
228        adjusted_global_expressions.insert(old_h, h);
229    }
230
231    // Finish processing any overrides we didn't visit in the loop above.
232    for entry in override_iter {
233        match *entry.1 {
234            Override { name: Some(_), .. } | Override { id: Some(_), .. } => {
235                process_override(
236                    entry,
237                    pipeline_constants,
238                    &mut module,
239                    &mut override_map,
240                    &adjusted_global_expressions,
241                    &mut adjusted_constant_initializers,
242                    &mut global_expression_kind_tracker,
243                )?;
244            }
245            Override {
246                init: Some(ref mut init),
247                ..
248            } => {
249                *init = adjusted_global_expressions[*init];
250            }
251            _ => {}
252        }
253    }
254
255    // Update the initialization expression handles of all `Constant`s
256    // and `GlobalVariable`s. Skip `Constant`s we'd already updated en
257    // passant.
258    for (_, c) in module
259        .constants
260        .iter_mut()
261        .filter(|&(c_h, _)| !adjusted_constant_initializers.contains(&c_h))
262    {
263        c.init = adjusted_global_expressions[c.init];
264    }
265
266    for (_, v) in module.global_variables.iter_mut() {
267        if let Some(ref mut init) = v.init {
268            *init = adjusted_global_expressions[*init];
269        }
270    }
271
272    let mut functions = module.functions.take();
273    for (_, function) in functions.iter_mut() {
274        process_function(&mut module, &override_map, &mut layouter, function)?;
275    }
276    module.functions = functions;
277
278    let mut entry_points = mem::take(&mut module.entry_points);
279    for ep in entry_points.iter_mut() {
280        process_function(&mut module, &override_map, &mut layouter, &mut ep.function)?;
281        process_workgroup_size_override(&mut module, &adjusted_global_expressions, ep)?;
282        process_mesh_shader_overrides(&mut module, &adjusted_global_expressions, ep)?;
283    }
284    module.entry_points = entry_points;
285    module.overrides = overrides;
286
287    // Now that we've rewritten all the expressions, we need to
288    // recompute their types and other metadata. For the time being,
289    // do a full re-validation.
290    revalidate(module)
291}
292
293fn revalidate(
294    module: Module,
295) -> Result<(Cow<'static, Module>, Cow<'static, ModuleInfo>), PipelineConstantError> {
296    let mut validator = Validator::new(ValidationFlags::all(), Capabilities::all());
297    let module_info = validator.validate_resolved_overrides(&module)?;
298    Ok((Cow::Owned(module), Cow::Owned(module_info)))
299}
300
301fn process_workgroup_size_override(
302    module: &mut Module,
303    adjusted_global_expressions: &HandleVec<Expression, Handle<Expression>>,
304    ep: &mut crate::EntryPoint,
305) -> Result<(), PipelineConstantError> {
306    match ep.workgroup_size_overrides {
307        None => {}
308        Some(overrides) => {
309            overrides.iter().enumerate().try_for_each(
310                |(i, overridden)| -> Result<(), PipelineConstantError> {
311                    match *overridden {
312                        None => Ok(()),
313                        Some(h) => {
314                            ep.workgroup_size[i] = module
315                                .to_ctx()
316                                .get_const_val(adjusted_global_expressions[h])
317                                .map(|n| {
318                                    if n == 0 {
319                                        Err(PipelineConstantError::NegativeWorkgroupSize)
320                                    } else {
321                                        Ok(n)
322                                    }
323                                })
324                                .map_err(|_| PipelineConstantError::NegativeWorkgroupSize)??;
325                            Ok(())
326                        }
327                    }
328                },
329            )?;
330            ep.workgroup_size_overrides = None;
331        }
332    }
333    Ok(())
334}
335
336fn process_mesh_shader_overrides(
337    module: &mut Module,
338    adjusted_global_expressions: &HandleVec<Expression, Handle<Expression>>,
339    ep: &mut crate::EntryPoint,
340) -> Result<(), PipelineConstantError> {
341    if let Some(ref mut mesh_info) = ep.mesh_info {
342        if let Some(r#override) = mesh_info.max_vertices_override {
343            mesh_info.max_vertices = module
344                .to_ctx()
345                .get_const_val(adjusted_global_expressions[r#override])
346                .map_err(|_| PipelineConstantError::NegativeMeshOutputMax)?;
347        }
348        if let Some(r#override) = mesh_info.max_primitives_override {
349            mesh_info.max_primitives = module
350                .to_ctx()
351                .get_const_val(adjusted_global_expressions[r#override])
352                .map_err(|_| PipelineConstantError::NegativeMeshOutputMax)?;
353        }
354    }
355    Ok(())
356}
357
358/// Add a [`Constant`] to `module` for the override `old_h`.
359///
360/// Add the new `Constant` to `override_map` and `adjusted_constant_initializers`.
361fn process_override(
362    (old_h, r#override, span): (Handle<Override>, &mut Override, &Span),
363    pipeline_constants: &PipelineConstants,
364    module: &mut Module,
365    override_map: &mut HandleVec<Override, Handle<Constant>>,
366    adjusted_global_expressions: &HandleVec<Expression, Handle<Expression>>,
367    adjusted_constant_initializers: &mut HashSet<Handle<Constant>>,
368    global_expression_kind_tracker: &mut crate::proc::ExpressionKindTracker,
369) -> Result<Handle<Constant>, PipelineConstantError> {
370    // Determine which key to use for `r#override` in `pipeline_constants`.
371    let key = if let Some(id) = r#override.id {
372        Cow::Owned(id.to_string())
373    } else if let Some(ref name) = r#override.name {
374        Cow::Borrowed(name)
375    } else {
376        unreachable!();
377    };
378
379    // Generate a global expression for `r#override`'s value, either
380    // from the provided `pipeline_constants` table or its initializer
381    // in the module.
382    let init = if let Some(value) = pipeline_constants.get::<str>(&key) {
383        let literal = match module.types[r#override.ty].inner {
384            TypeInner::Scalar(scalar) => map_value_to_literal(*value, scalar)?,
385            _ => unreachable!(),
386        };
387        let expr = module
388            .global_expressions
389            .append(Expression::Literal(literal), Span::UNDEFINED);
390        global_expression_kind_tracker.insert(expr, crate::proc::ExpressionKind::Const);
391        expr
392    } else if let Some(init) = r#override.init {
393        adjusted_global_expressions[init]
394    } else {
395        return Err(PipelineConstantError::MissingValue(key.to_string()));
396    };
397
398    // Generate a new `Constant` to represent the override's value.
399    let constant = Constant {
400        name: r#override.name.clone(),
401        ty: r#override.ty,
402        init,
403    };
404    let h = module.constants.append(constant, *span);
405    override_map.insert(old_h, h);
406    adjusted_constant_initializers.insert(h);
407    r#override.init = Some(init);
408    Ok(h)
409}
410
411/// Replace all override expressions in `function` with fully-evaluated constants.
412///
413/// Replace all `Expression::Override`s in `function`'s expression arena with
414/// the corresponding `Expression::Constant`s, as given in `override_map`.
415/// Replace any expressions whose values are now known with their fully
416/// evaluated form.
417///
418/// If `h` is a `Handle<Override>`, then `override_map[h]` is the
419/// `Handle<Constant>` for the override's final value.
420fn process_function(
421    module: &mut Module,
422    override_map: &HandleVec<Override, Handle<Constant>>,
423    layouter: &mut crate::proc::Layouter,
424    function: &mut Function,
425) -> Result<(), ConstantEvaluatorError> {
426    // A map from original local expression handles to
427    // handles in the new, local expression arena.
428    let mut adjusted_local_expressions = HandleVec::with_capacity(function.expressions.len());
429
430    let mut local_expression_kind_tracker = crate::proc::ExpressionKindTracker::new();
431
432    let mut expressions = function.expressions.take();
433
434    // Dummy `emitter` and `block` for the constant evaluator.
435    // We can ignore the concept of emitting expressions here since
436    // expressions have already been covered by a `Statement::Emit`
437    // in the frontend.
438    // The only thing we might have to do is remove some expressions
439    // that have been covered by a `Statement::Emit`. See the docs of
440    // `filter_emits_in_block` for the reasoning.
441    let mut emitter = Emitter::default();
442    let mut block = Block::new();
443
444    let mut evaluator = ConstantEvaluator::for_wgsl_function(
445        module,
446        &mut function.expressions,
447        &mut local_expression_kind_tracker,
448        layouter,
449        &mut emitter,
450        &mut block,
451        false,
452    );
453
454    for (old_h, mut expr, span) in expressions.drain() {
455        if let Expression::Override(h) = expr {
456            expr = Expression::Constant(override_map[h]);
457        }
458        adjust_expr(&adjusted_local_expressions, &mut expr);
459        let h = evaluator.try_eval_and_append(expr, span)?;
460        adjusted_local_expressions.insert(old_h, h);
461    }
462
463    adjust_block(&adjusted_local_expressions, &mut function.body);
464
465    filter_emits_in_block(&mut function.body, &function.expressions);
466
467    // Update local expression initializers.
468    for (_, local) in function.local_variables.iter_mut() {
469        if let &mut Some(ref mut init) = &mut local.init {
470            *init = adjusted_local_expressions[*init];
471        }
472    }
473
474    // We've changed the keys of `function.named_expression`, so we have to
475    // rebuild it from scratch.
476    let named_expressions = mem::take(&mut function.named_expressions);
477    for (expr_h, name) in named_expressions {
478        function
479            .named_expressions
480            .insert(adjusted_local_expressions[expr_h], name);
481    }
482
483    Ok(())
484}
485
486/// Replace every expression handle in `expr` with its counterpart
487/// given by `new_pos`.
488fn adjust_expr(new_pos: &HandleVec<Expression, Handle<Expression>>, expr: &mut Expression) {
489    let adjust = |expr: &mut Handle<Expression>| {
490        *expr = new_pos[*expr];
491    };
492    match *expr {
493        Expression::Compose {
494            ref mut components,
495            ty: _,
496        } => {
497            for c in components.iter_mut() {
498                adjust(c);
499            }
500        }
501        Expression::Access {
502            ref mut base,
503            ref mut index,
504        } => {
505            adjust(base);
506            adjust(index);
507        }
508        Expression::AccessIndex {
509            ref mut base,
510            index: _,
511        } => {
512            adjust(base);
513        }
514        Expression::Splat {
515            ref mut value,
516            size: _,
517        } => {
518            adjust(value);
519        }
520        Expression::Swizzle {
521            ref mut vector,
522            size: _,
523            pattern: _,
524        } => {
525            adjust(vector);
526        }
527        Expression::Load { ref mut pointer } => {
528            adjust(pointer);
529        }
530        Expression::ImageSample {
531            ref mut image,
532            ref mut sampler,
533            ref mut coordinate,
534            ref mut array_index,
535            ref mut offset,
536            ref mut level,
537            ref mut depth_ref,
538            gather: _,
539            clamp_to_edge: _,
540        } => {
541            adjust(image);
542            adjust(sampler);
543            adjust(coordinate);
544            if let Some(e) = array_index.as_mut() {
545                adjust(e);
546            }
547            if let Some(e) = offset.as_mut() {
548                adjust(e);
549            }
550            match *level {
551                crate::SampleLevel::Exact(ref mut expr)
552                | crate::SampleLevel::Bias(ref mut expr) => {
553                    adjust(expr);
554                }
555                crate::SampleLevel::Gradient {
556                    ref mut x,
557                    ref mut y,
558                } => {
559                    adjust(x);
560                    adjust(y);
561                }
562                _ => {}
563            }
564            if let Some(e) = depth_ref.as_mut() {
565                adjust(e);
566            }
567        }
568        Expression::ImageLoad {
569            ref mut image,
570            ref mut coordinate,
571            ref mut array_index,
572            ref mut sample,
573            ref mut level,
574        } => {
575            adjust(image);
576            adjust(coordinate);
577            if let Some(e) = array_index.as_mut() {
578                adjust(e);
579            }
580            if let Some(e) = sample.as_mut() {
581                adjust(e);
582            }
583            if let Some(e) = level.as_mut() {
584                adjust(e);
585            }
586        }
587        Expression::ImageQuery {
588            ref mut image,
589            ref mut query,
590        } => {
591            adjust(image);
592            match *query {
593                crate::ImageQuery::Size { ref mut level } => {
594                    if let Some(e) = level.as_mut() {
595                        adjust(e);
596                    }
597                }
598                crate::ImageQuery::NumLevels
599                | crate::ImageQuery::NumLayers
600                | crate::ImageQuery::NumSamples => {}
601            }
602        }
603        Expression::Unary {
604            ref mut expr,
605            op: _,
606        } => {
607            adjust(expr);
608        }
609        Expression::Binary {
610            ref mut left,
611            ref mut right,
612            op: _,
613        } => {
614            adjust(left);
615            adjust(right);
616        }
617        Expression::Select {
618            ref mut condition,
619            ref mut accept,
620            ref mut reject,
621        } => {
622            adjust(condition);
623            adjust(accept);
624            adjust(reject);
625        }
626        Expression::Derivative {
627            ref mut expr,
628            axis: _,
629            ctrl: _,
630        } => {
631            adjust(expr);
632        }
633        Expression::Relational {
634            ref mut argument,
635            fun: _,
636        } => {
637            adjust(argument);
638        }
639        Expression::Math {
640            ref mut arg,
641            ref mut arg1,
642            ref mut arg2,
643            ref mut arg3,
644            fun: _,
645        } => {
646            adjust(arg);
647            if let Some(e) = arg1.as_mut() {
648                adjust(e);
649            }
650            if let Some(e) = arg2.as_mut() {
651                adjust(e);
652            }
653            if let Some(e) = arg3.as_mut() {
654                adjust(e);
655            }
656        }
657        Expression::As {
658            ref mut expr,
659            target: _,
660        } => {
661            adjust(expr);
662        }
663        Expression::Bitcast {
664            ref mut expr,
665            ty: _,
666        } => {
667            adjust(expr);
668        }
669        Expression::ArrayLength(ref mut expr) => {
670            adjust(expr);
671        }
672        Expression::RayQueryGetIntersection {
673            ref mut query,
674            committed: _,
675        } => {
676            adjust(query);
677        }
678        Expression::Literal(_)
679        | Expression::FunctionArgument(_)
680        | Expression::GlobalVariable(_)
681        | Expression::LocalVariable(_)
682        | Expression::CallResult(_)
683        | Expression::RayQueryProceedResult
684        | Expression::Constant(_)
685        | Expression::Override(_)
686        | Expression::ZeroValue(_)
687        | Expression::AtomicResult {
688            ty: _,
689            comparison: _,
690        }
691        | Expression::WorkGroupUniformLoadResult { ty: _ }
692        | Expression::SubgroupBallotResult
693        | Expression::SubgroupOperationResult { .. } => {}
694        Expression::RayQueryVertexPositions {
695            ref mut query,
696            committed: _,
697        } => {
698            adjust(query);
699        }
700        Expression::HitObjectQuery {
701            ref mut hit_object,
702            query: _,
703        } => {
704            adjust(hit_object);
705        }
706        Expression::CooperativeLoad { ref mut data, .. } => {
707            adjust(&mut data.pointer);
708            adjust(&mut data.stride);
709        }
710        Expression::CooperativeMultiplyAdd {
711            ref mut a,
712            ref mut b,
713            ref mut c,
714        } => {
715            adjust(a);
716            adjust(b);
717            adjust(c);
718        }
719    }
720}
721
722/// Replace every expression handle in `block` with its counterpart
723/// given by `new_pos`.
724fn adjust_block(new_pos: &HandleVec<Expression, Handle<Expression>>, block: &mut Block) {
725    for stmt in block.iter_mut() {
726        adjust_stmt(new_pos, stmt);
727    }
728}
729
730/// Replace every expression handle in `stmt` with its counterpart
731/// given by `new_pos`.
732fn adjust_reorder_hint(
733    hint: &mut Option<crate::ReorderHint>,
734    adjust: impl Fn(&mut Handle<Expression>),
735) {
736    if let Some(crate::ReorderHint {
737        ref mut hint,
738        ref mut bits,
739    }) = *hint
740    {
741        adjust(hint);
742        adjust(bits);
743    }
744}
745
746fn adjust_stmt(new_pos: &HandleVec<Expression, Handle<Expression>>, stmt: &mut Statement) {
747    let adjust = |expr: &mut Handle<Expression>| {
748        *expr = new_pos[*expr];
749    };
750    match *stmt {
751        Statement::Emit(ref mut range) => {
752            if let Some((mut first, mut last)) = range.first_and_last() {
753                adjust(&mut first);
754                adjust(&mut last);
755                *range = Range::new_from_bounds(first, last);
756            }
757        }
758        Statement::Block(ref mut block) => {
759            adjust_block(new_pos, block);
760        }
761        Statement::If {
762            ref mut condition,
763            ref mut accept,
764            ref mut reject,
765        } => {
766            adjust(condition);
767            adjust_block(new_pos, accept);
768            adjust_block(new_pos, reject);
769        }
770        Statement::Switch {
771            ref mut selector,
772            ref mut cases,
773        } => {
774            adjust(selector);
775            for case in cases.iter_mut() {
776                adjust_block(new_pos, &mut case.body);
777            }
778        }
779        Statement::Loop {
780            ref mut body,
781            ref mut continuing,
782            ref mut break_if,
783        } => {
784            adjust_block(new_pos, body);
785            adjust_block(new_pos, continuing);
786            if let Some(e) = break_if.as_mut() {
787                adjust(e);
788            }
789        }
790        Statement::Return { ref mut value } => {
791            if let Some(e) = value.as_mut() {
792                adjust(e);
793            }
794        }
795        Statement::Store {
796            ref mut pointer,
797            ref mut value,
798        } => {
799            adjust(pointer);
800            adjust(value);
801        }
802        Statement::ImageStore {
803            ref mut image,
804            ref mut coordinate,
805            ref mut array_index,
806            ref mut value,
807        } => {
808            adjust(image);
809            adjust(coordinate);
810            if let Some(e) = array_index.as_mut() {
811                adjust(e);
812            }
813            adjust(value);
814        }
815        Statement::Atomic {
816            ref mut pointer,
817            ref mut value,
818            ref mut result,
819            ref mut fun,
820        } => {
821            adjust(pointer);
822            adjust(value);
823            if let Some(ref mut result) = *result {
824                adjust(result);
825            }
826            match *fun {
827                crate::AtomicFunction::Exchange {
828                    compare: Some(ref mut compare),
829                } => {
830                    adjust(compare);
831                }
832                crate::AtomicFunction::Add
833                | crate::AtomicFunction::Subtract
834                | crate::AtomicFunction::And
835                | crate::AtomicFunction::ExclusiveOr
836                | crate::AtomicFunction::InclusiveOr
837                | crate::AtomicFunction::Min
838                | crate::AtomicFunction::Max
839                | crate::AtomicFunction::Exchange { compare: None } => {}
840            }
841        }
842        Statement::ImageAtomic {
843            ref mut image,
844            ref mut coordinate,
845            ref mut array_index,
846            fun: _,
847            ref mut value,
848        } => {
849            adjust(image);
850            adjust(coordinate);
851            if let Some(ref mut array_index) = *array_index {
852                adjust(array_index);
853            }
854            adjust(value);
855        }
856        Statement::WorkGroupUniformLoad {
857            ref mut pointer,
858            ref mut result,
859        } => {
860            adjust(pointer);
861            adjust(result);
862        }
863        Statement::SubgroupBallot {
864            ref mut result,
865            ref mut predicate,
866        } => {
867            if let Some(ref mut predicate) = *predicate {
868                adjust(predicate);
869            }
870            adjust(result);
871        }
872        Statement::SubgroupCollectiveOperation {
873            ref mut argument,
874            ref mut result,
875            ..
876        } => {
877            adjust(argument);
878            adjust(result);
879        }
880        Statement::SubgroupGather {
881            ref mut mode,
882            ref mut argument,
883            ref mut result,
884        } => {
885            match *mode {
886                crate::GatherMode::BroadcastFirst => {}
887                crate::GatherMode::Broadcast(ref mut index)
888                | crate::GatherMode::Shuffle(ref mut index)
889                | crate::GatherMode::ShuffleDown(ref mut index)
890                | crate::GatherMode::ShuffleUp(ref mut index)
891                | crate::GatherMode::ShuffleXor(ref mut index)
892                | crate::GatherMode::QuadBroadcast(ref mut index) => {
893                    adjust(index);
894                }
895                crate::GatherMode::QuadSwap(_) => {}
896            }
897            adjust(argument);
898            adjust(result)
899        }
900        Statement::SubgroupBallotFindBit {
901            order: _,
902            ref mut argument,
903            ref mut result,
904        } => {
905            adjust(argument);
906            adjust(result);
907        }
908        Statement::Call {
909            ref mut arguments,
910            ref mut result,
911            function: _,
912        } => {
913            for argument in arguments.iter_mut() {
914                adjust(argument);
915            }
916            if let Some(e) = result.as_mut() {
917                adjust(e);
918            }
919        }
920        Statement::RayQuery {
921            ref mut query,
922            ref mut fun,
923        } => {
924            adjust(query);
925            match *fun {
926                crate::RayQueryFunction::Initialize {
927                    ref mut acceleration_structure,
928                    ref mut descriptor,
929                } => {
930                    adjust(acceleration_structure);
931                    adjust(descriptor);
932                }
933                crate::RayQueryFunction::Proceed { ref mut result } => {
934                    adjust(result);
935                }
936                crate::RayQueryFunction::GenerateIntersection { ref mut hit_t } => {
937                    adjust(hit_t);
938                }
939                crate::RayQueryFunction::ConfirmIntersection => {}
940                crate::RayQueryFunction::Terminate => {}
941                crate::RayQueryFunction::Begin => {}
942            }
943        }
944        Statement::HitObject {
945            ref mut hit_object,
946            ref mut fun,
947        } => {
948            adjust(hit_object);
949            match *fun {
950                crate::HitObjectFunction::TraceRay {
951                    ref mut acceleration_structure,
952                    ref mut descriptor,
953                    ref mut payload,
954                } => {
955                    adjust(acceleration_structure);
956                    adjust(descriptor);
957                    adjust(payload);
958                }
959                crate::HitObjectFunction::RecordMiss { ref mut descriptor } => {
960                    adjust(descriptor);
961                }
962                crate::HitObjectFunction::RecordFromQuery { ref mut query } => {
963                    adjust(query);
964                }
965                crate::HitObjectFunction::RecordEmpty => {}
966                crate::HitObjectFunction::ExecuteShader { ref mut payload } => {
967                    adjust(payload);
968                }
969                crate::HitObjectFunction::Reorder { ref mut hint } => {
970                    adjust_reorder_hint(hint, adjust);
971                }
972            }
973        }
974        Statement::CooperativeStore {
975            ref mut target,
976            ref mut data,
977        } => {
978            adjust(target);
979            adjust(&mut data.pointer);
980            adjust(&mut data.stride);
981        }
982        Statement::RayPipelineFunction(ref mut func) => match *func {
983            crate::RayPipelineFunction::ReorderThread {
984                ref mut hint,
985                ref mut bits,
986            } => {
987                adjust(hint);
988                adjust(bits);
989            }
990            crate::RayPipelineFunction::TraceRay {
991                ref mut acceleration_structure,
992                ref mut descriptor,
993                ref mut payload,
994            } => {
995                adjust(acceleration_structure);
996                adjust(descriptor);
997                adjust(payload);
998            }
999        },
1000        Statement::DebugPrintf {
1001            format: _,
1002            ref mut arguments,
1003        } => {
1004            for argument in arguments.iter_mut() {
1005                adjust(argument);
1006            }
1007        }
1008        Statement::Break
1009        | Statement::Continue
1010        | Statement::Kill
1011        | Statement::ControlBarrier(_)
1012        | Statement::MemoryBarrier(_) => {}
1013    }
1014}
1015
1016/// Adjust [`Emit`] statements in `block` to skip [`needs_pre_emit`] expressions we have introduced.
1017///
1018/// According to validation, [`Emit`] statements must not cover any expressions
1019/// for which [`Expression::needs_pre_emit`] returns true. All expressions built
1020/// by successful constant evaluation fall into that category, meaning that
1021/// `process_function` will usually rewrite [`Override`] expressions and those
1022/// that use their values into pre-emitted expressions, leaving any [`Emit`]
1023/// statements that cover them invalid.
1024///
1025/// This function rewrites all [`Emit`] statements into zero or more new
1026/// [`Emit`] statements covering only those expressions in the original range
1027/// that are not pre-emitted.
1028///
1029/// [`Emit`]: Statement::Emit
1030/// [`needs_pre_emit`]: Expression::needs_pre_emit
1031/// [`Override`]: Expression::Override
1032fn filter_emits_in_block(block: &mut Block, expressions: &Arena<Expression>) {
1033    let original = mem::replace(block, Block::with_capacity(block.len()));
1034    for (stmt, span) in original.span_into_iter() {
1035        match stmt {
1036            Statement::Emit(range) => {
1037                let mut current = None;
1038                for expr_h in range {
1039                    if expressions[expr_h].needs_pre_emit() {
1040                        if let Some((first, last)) = current {
1041                            block.push(Statement::Emit(Range::new_from_bounds(first, last)), span);
1042                        }
1043
1044                        current = None;
1045                    } else if let Some((_, ref mut last)) = current {
1046                        *last = expr_h;
1047                    } else {
1048                        current = Some((expr_h, expr_h));
1049                    }
1050                }
1051                if let Some((first, last)) = current {
1052                    block.push(Statement::Emit(Range::new_from_bounds(first, last)), span);
1053                }
1054            }
1055            Statement::Block(mut child) => {
1056                filter_emits_in_block(&mut child, expressions);
1057                block.push(Statement::Block(child), span);
1058            }
1059            Statement::If {
1060                condition,
1061                mut accept,
1062                mut reject,
1063            } => {
1064                filter_emits_in_block(&mut accept, expressions);
1065                filter_emits_in_block(&mut reject, expressions);
1066                block.push(
1067                    Statement::If {
1068                        condition,
1069                        accept,
1070                        reject,
1071                    },
1072                    span,
1073                );
1074            }
1075            Statement::Switch {
1076                selector,
1077                mut cases,
1078            } => {
1079                for case in &mut cases {
1080                    filter_emits_in_block(&mut case.body, expressions);
1081                }
1082                block.push(Statement::Switch { selector, cases }, span);
1083            }
1084            Statement::Loop {
1085                mut body,
1086                mut continuing,
1087                break_if,
1088            } => {
1089                filter_emits_in_block(&mut body, expressions);
1090                filter_emits_in_block(&mut continuing, expressions);
1091                block.push(
1092                    Statement::Loop {
1093                        body,
1094                        continuing,
1095                        break_if,
1096                    },
1097                    span,
1098                );
1099            }
1100            stmt => block.push(stmt.clone(), span),
1101        }
1102    }
1103}
1104
1105fn map_value_to_literal(value: f64, scalar: Scalar) -> Result<Literal, PipelineConstantError> {
1106    // note that in rust 0.0 == -0.0
1107    match scalar {
1108        Scalar::BOOL => {
1109            // https://webidl.spec.whatwg.org/#js-boolean
1110            let value = value != 0.0 && !value.is_nan();
1111            Ok(Literal::Bool(value))
1112        }
1113        Scalar::I16 => {
1114            if !value.is_finite() {
1115                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1116            }
1117
1118            let value = value.trunc();
1119            if value < f64::from(i16::MIN) || value > f64::from(i16::MAX) {
1120                return Err(PipelineConstantError::DstRangeTooSmall);
1121            }
1122
1123            let value = value as i16;
1124            Ok(Literal::I16(value))
1125        }
1126        Scalar::U16 => {
1127            if !value.is_finite() {
1128                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1129            }
1130
1131            let value = value.trunc();
1132            if value < f64::from(u16::MIN) || value > f64::from(u16::MAX) {
1133                return Err(PipelineConstantError::DstRangeTooSmall);
1134            }
1135
1136            let value = value as u16;
1137            Ok(Literal::U16(value))
1138        }
1139        Scalar::I32 => {
1140            // https://webidl.spec.whatwg.org/#js-long
1141            if !value.is_finite() {
1142                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1143            }
1144
1145            let value = value.trunc();
1146            if value < f64::from(i32::MIN) || value > f64::from(i32::MAX) {
1147                return Err(PipelineConstantError::DstRangeTooSmall);
1148            }
1149
1150            let value = value as i32;
1151            Ok(Literal::I32(value))
1152        }
1153        Scalar::U32 => {
1154            // https://webidl.spec.whatwg.org/#js-unsigned-long
1155            if !value.is_finite() {
1156                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1157            }
1158
1159            let value = value.trunc();
1160            if value < f64::from(u32::MIN) || value > f64::from(u32::MAX) {
1161                return Err(PipelineConstantError::DstRangeTooSmall);
1162            }
1163
1164            let value = value as u32;
1165            Ok(Literal::U32(value))
1166        }
1167        Scalar::F16 => {
1168            // https://webidl.spec.whatwg.org/#js-float
1169            if !value.is_finite() {
1170                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1171            }
1172
1173            let value = half::f16::from_f64(value);
1174            if !value.is_finite() {
1175                return Err(PipelineConstantError::DstRangeTooSmall);
1176            }
1177
1178            Ok(Literal::F16(value))
1179        }
1180        Scalar::F32 => {
1181            // https://webidl.spec.whatwg.org/#js-float
1182            if !value.is_finite() {
1183                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1184            }
1185
1186            let value = value as f32;
1187            if !value.is_finite() {
1188                return Err(PipelineConstantError::DstRangeTooSmall);
1189            }
1190
1191            Ok(Literal::F32(value))
1192        }
1193        Scalar::F64 => {
1194            // https://webidl.spec.whatwg.org/#js-double
1195            if !value.is_finite() {
1196                return Err(PipelineConstantError::SrcNeedsToBeFinite);
1197            }
1198
1199            Ok(Literal::F64(value))
1200        }
1201        Scalar::ABSTRACT_FLOAT | Scalar::ABSTRACT_INT => {
1202            unreachable!("abstract values should not be validated out of override processing")
1203        }
1204        _ => unreachable!("unrecognized scalar type for override"),
1205    }
1206}
1207
1208#[test]
1209fn test_map_value_to_literal() {
1210    let bool_test_cases = [
1211        (0.0, false),
1212        (-0.0, false),
1213        (f64::NAN, false),
1214        (1.0, true),
1215        (f64::INFINITY, true),
1216        (f64::NEG_INFINITY, true),
1217    ];
1218    for (value, out) in bool_test_cases {
1219        let res = Ok(Literal::Bool(out));
1220        assert_eq!(map_value_to_literal(value, Scalar::BOOL), res);
1221    }
1222
1223    for scalar in [Scalar::I32, Scalar::U32, Scalar::F32, Scalar::F64] {
1224        for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
1225            let res = Err(PipelineConstantError::SrcNeedsToBeFinite);
1226            assert_eq!(map_value_to_literal(value, scalar), res);
1227        }
1228    }
1229
1230    // i32
1231    assert_eq!(
1232        map_value_to_literal(f64::from(i32::MIN), Scalar::I32),
1233        Ok(Literal::I32(i32::MIN))
1234    );
1235    assert_eq!(
1236        map_value_to_literal(f64::from(i32::MAX), Scalar::I32),
1237        Ok(Literal::I32(i32::MAX))
1238    );
1239    assert_eq!(
1240        map_value_to_literal(f64::from(i32::MIN) - 1.0, Scalar::I32),
1241        Err(PipelineConstantError::DstRangeTooSmall)
1242    );
1243    assert_eq!(
1244        map_value_to_literal(f64::from(i32::MAX) + 1.0, Scalar::I32),
1245        Err(PipelineConstantError::DstRangeTooSmall)
1246    );
1247
1248    // u32
1249    assert_eq!(
1250        map_value_to_literal(f64::from(u32::MIN), Scalar::U32),
1251        Ok(Literal::U32(u32::MIN))
1252    );
1253    assert_eq!(
1254        map_value_to_literal(f64::from(u32::MAX), Scalar::U32),
1255        Ok(Literal::U32(u32::MAX))
1256    );
1257    assert_eq!(
1258        map_value_to_literal(f64::from(u32::MIN) - 1.0, Scalar::U32),
1259        Err(PipelineConstantError::DstRangeTooSmall)
1260    );
1261    assert_eq!(
1262        map_value_to_literal(f64::from(u32::MAX) + 1.0, Scalar::U32),
1263        Err(PipelineConstantError::DstRangeTooSmall)
1264    );
1265
1266    // f32
1267    assert_eq!(
1268        map_value_to_literal(f64::from(f32::MIN), Scalar::F32),
1269        Ok(Literal::F32(f32::MIN))
1270    );
1271    assert_eq!(
1272        map_value_to_literal(f64::from(f32::MAX), Scalar::F32),
1273        Ok(Literal::F32(f32::MAX))
1274    );
1275    assert_eq!(
1276        map_value_to_literal(-f64::from_bits(0x47efffffefffffff), Scalar::F32),
1277        Ok(Literal::F32(f32::MIN))
1278    );
1279    assert_eq!(
1280        map_value_to_literal(f64::from_bits(0x47efffffefffffff), Scalar::F32),
1281        Ok(Literal::F32(f32::MAX))
1282    );
1283    assert_eq!(
1284        map_value_to_literal(-f64::from_bits(0x47effffff0000000), Scalar::F32),
1285        Err(PipelineConstantError::DstRangeTooSmall)
1286    );
1287    assert_eq!(
1288        map_value_to_literal(f64::from_bits(0x47effffff0000000), Scalar::F32),
1289        Err(PipelineConstantError::DstRangeTooSmall)
1290    );
1291
1292    // f64
1293    assert_eq!(
1294        map_value_to_literal(f64::MIN, Scalar::F64),
1295        Ok(Literal::F64(f64::MIN))
1296    );
1297    assert_eq!(
1298        map_value_to_literal(f64::MAX, Scalar::F64),
1299        Ok(Literal::F64(f64::MAX))
1300    );
1301}