naga/proc/index.rs
1/*!
2Definitions for index bounds checking.
3*/
4
5use core::iter::{self, zip};
6
7use crate::arena::{Handle, HandleSet, UniqueArena};
8use crate::{valid, FastHashSet};
9
10/// How should code generated by Naga do bounds checks?
11///
12/// When a vector, matrix, or array index is out of bounds—either negative, or
13/// greater than or equal to the number of elements in the type—WGSL requires
14/// that some other index of the implementation's choice that is in bounds is
15/// used instead. (There are no types with zero elements.)
16///
17/// Similarly, when out-of-bounds coordinates, array indices, or sample indices
18/// are presented to the WGSL `textureLoad` and `textureStore` operations, the
19/// operation is redirected to do something safe.
20///
21/// Different users of Naga will prefer different defaults:
22///
23/// - When used as part of a WebGPU implementation, the WGSL specification
24/// requires the `Restrict` behavior for array, vector, and matrix accesses,
25/// and either the `Restrict` or `ReadZeroSkipWrite` behaviors for texture
26/// accesses.
27///
28/// - When used by the `wgpu` crate for native development, `wgpu` selects
29/// `ReadZeroSkipWrite` as its default.
30///
31/// - Naga's own default is `Unchecked`, so that shader translations
32/// are as faithful to the original as possible.
33///
34/// Sometimes the underlying hardware and drivers can perform bounds checks
35/// themselves, in a way that performs better than the checks Naga would inject.
36/// If you're using native checks like this, then having Naga inject its own
37/// checks as well would be redundant, and the `Unchecked` policy is
38/// appropriate.
39#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
40#[cfg_attr(feature = "serialize", derive(serde::Serialize))]
41#[cfg_attr(feature = "deserialize", derive(serde::Deserialize))]
42pub enum BoundsCheckPolicy {
43 /// Replace out-of-bounds indexes with some arbitrary in-bounds index.
44 ///
45 /// (This does not necessarily mean clamping. For example, interpreting the
46 /// index as unsigned and taking the minimum with the largest valid index
47 /// would also be a valid implementation. That would map negative indices to
48 /// the last element, not the first.)
49 Restrict,
50
51 /// Out-of-bounds reads return zero, and writes have no effect.
52 ///
53 /// When applied to a chain of accesses, like `a[i][j].b[k]`, all index
54 /// expressions are evaluated, regardless of whether prior or later index
55 /// expressions were in bounds. But all the accesses per se are skipped
56 /// if any index is out of bounds.
57 ReadZeroSkipWrite,
58
59 /// Naga adds no checks to indexing operations. Generate the fastest code
60 /// possible. This is the default for Naga, as a translator, but consumers
61 /// should consider defaulting to a safer behavior.
62 Unchecked,
63}
64
65/// Policies for injecting bounds checks during code generation.
66#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
67#[cfg_attr(feature = "serialize", derive(serde::Serialize))]
68#[cfg_attr(feature = "deserialize", derive(serde::Deserialize))]
69#[cfg_attr(feature = "deserialize", serde(default))]
70pub struct BoundsCheckPolicies {
71 /// How should the generated code handle array, vector, or matrix indices
72 /// that are out of range?
73 pub index: BoundsCheckPolicy,
74
75 /// How should the generated code handle array, vector, or matrix indices
76 /// that are out of range, when those values live in a [`GlobalVariable`] in
77 /// the [`Storage`] or [`Uniform`] address spaces?
78 ///
79 /// Some graphics hardware provides "robust buffer access", a feature that
80 /// ensures that using a pointer cannot access memory outside the 'buffer'
81 /// that it was derived from. In Naga terms, this means that the hardware
82 /// ensures that pointers computed by applying [`Access`] and
83 /// [`AccessIndex`] expressions to a [`GlobalVariable`] whose [`space`] is
84 /// [`Storage`] or [`Uniform`] will never read or write memory outside that
85 /// global variable.
86 ///
87 /// When hardware offers such a feature, it is probably undesirable to have
88 /// Naga inject bounds checking code for such accesses, since the hardware
89 /// can probably provide the same protection more efficiently. However,
90 /// bounds checks are still needed on accesses to indexable values that do
91 /// not live in buffers, like local variables.
92 ///
93 /// So, this option provides a separate policy that applies only to accesses
94 /// to storage and uniform globals. When depending on hardware bounds
95 /// checking, this policy can be `Unchecked` to avoid unnecessary overhead.
96 ///
97 /// When special hardware support is not available, this should probably be
98 /// the same as `index_bounds_check_policy`.
99 ///
100 /// [`GlobalVariable`]: crate::GlobalVariable
101 /// [`space`]: crate::GlobalVariable::space
102 /// [`Restrict`]: crate::proc::BoundsCheckPolicy::Restrict
103 /// [`ReadZeroSkipWrite`]: crate::proc::BoundsCheckPolicy::ReadZeroSkipWrite
104 /// [`Access`]: crate::Expression::Access
105 /// [`AccessIndex`]: crate::Expression::AccessIndex
106 /// [`Storage`]: crate::AddressSpace::Storage
107 /// [`Uniform`]: crate::AddressSpace::Uniform
108 pub buffer: BoundsCheckPolicy,
109
110 /// How should the generated code handle image texel loads that are out
111 /// of range?
112 ///
113 /// This controls the behavior of [`ImageLoad`] expressions when a coordinate,
114 /// texture array index, level of detail, or multisampled sample number is out of range.
115 ///
116 /// There is no corresponding policy for [`ImageStore`] statements. All the
117 /// platforms we support already discard out-of-bounds image stores,
118 /// effectively implementing the "skip write" part of [`ReadZeroSkipWrite`].
119 ///
120 /// [`ImageLoad`]: crate::Expression::ImageLoad
121 /// [`ImageStore`]: crate::Statement::ImageStore
122 /// [`ReadZeroSkipWrite`]: BoundsCheckPolicy::ReadZeroSkipWrite
123 pub image_load: BoundsCheckPolicy,
124
125 /// How should the generated code handle binding array indexes that are out of bounds.
126 pub binding_array: BoundsCheckPolicy,
127}
128
129/// The default `BoundsCheckPolicy` is `Unchecked`.
130impl Default for BoundsCheckPolicy {
131 fn default() -> Self {
132 BoundsCheckPolicy::Unchecked
133 }
134}
135
136impl BoundsCheckPolicies {
137 /// Determine which policy applies to `base`.
138 ///
139 /// `base` is the "base" expression (the expression being indexed) of a `Access`
140 /// and `AccessIndex` expression. This is either a pointer, a value, being directly
141 /// indexed, or a binding array.
142 ///
143 /// See the documentation for [`BoundsCheckPolicy`] for details about
144 /// when each policy applies.
145 pub fn choose_policy(
146 &self,
147 base: Handle<crate::Expression>,
148 types: &UniqueArena<crate::Type>,
149 info: &valid::FunctionInfo,
150 ) -> BoundsCheckPolicy {
151 let ty = info[base].ty.inner_with(types);
152
153 if let crate::TypeInner::BindingArray { .. } = *ty {
154 return self.binding_array;
155 }
156
157 match ty.pointer_space() {
158 Some(crate::AddressSpace::Storage { access: _ } | crate::AddressSpace::Uniform) => {
159 self.buffer
160 }
161 // This covers other address spaces, but also accessing vectors and
162 // matrices by value, where no pointer is involved.
163 _ => self.index,
164 }
165 }
166
167 /// Return `true` if any of `self`'s policies are `policy`.
168 pub fn contains(&self, policy: BoundsCheckPolicy) -> bool {
169 self.index == policy || self.buffer == policy || self.image_load == policy
170 }
171}
172
173/// An index that may be statically known, or may need to be computed at runtime.
174///
175/// This enum lets us handle both [`Access`] and [`AccessIndex`] expressions
176/// with the same code.
177///
178/// [`Access`]: crate::Expression::Access
179/// [`AccessIndex`]: crate::Expression::AccessIndex
180#[derive(Clone, Copy, Debug)]
181pub enum GuardedIndex {
182 Known(u32),
183 Expression(Handle<crate::Expression>),
184}
185
186/// Build a set of expressions used as indices, to cache in temporary variables when
187/// emitted.
188///
189/// Given the bounds-check policies `policies`, construct a `HandleSet` containing the handle
190/// indices of all the expressions in `function` that are ever used as guarded indices
191/// under the [`ReadZeroSkipWrite`] policy. The `module` argument must be the module to
192/// which `function` belongs, and `info` should be that function's analysis results.
193///
194/// Such index expressions will be used twice in the generated code: first for the
195/// comparison to see if the index is in bounds, and then for the access itself, should
196/// the comparison succeed. To avoid computing the expressions twice, the generated code
197/// should cache them in temporary variables.
198///
199/// Why do we need to build such a set in advance, instead of just processing access
200/// expressions as we encounter them? Whether an expression needs to be cached depends on
201/// whether it appears as something like the [`index`] operand of an [`Access`] expression
202/// or the [`level`] operand of an [`ImageLoad`] expression, and on the index bounds check
203/// policies that apply to those accesses. But [`Emit`] statements just identify a range
204/// of expressions by index; there's no good way to tell what an expression is used
205/// for. The only way to do it is to just iterate over all the expressions looking for
206/// relevant `Access` expressions --- which is what this function does.
207///
208/// Simple expressions like variable loads and constants don't make sense to cache: it's
209/// no better than just re-evaluating them. But constants are not covered by `Emit`
210/// statements, and `Load`s are always cached to ensure they occur at the right time, so
211/// we don't bother filtering them out from this set.
212///
213/// Fortunately, we don't need to deal with [`ImageStore`] statements here. When we emit
214/// code for a statement, the writer isn't in the middle of an expression, so we can just
215/// emit declarations for temporaries, initialized appropriately.
216///
217/// None of these concerns apply for SPIR-V output, since it's easy to just reuse an
218/// instruction ID in two places; that has the same semantics as a temporary variable, and
219/// it's inherent in the design of SPIR-V. This function is more useful for text-based
220/// back ends.
221///
222/// [`ReadZeroSkipWrite`]: BoundsCheckPolicy::ReadZeroSkipWrite
223/// [`index`]: crate::Expression::Access::index
224/// [`Access`]: crate::Expression::Access
225/// [`level`]: crate::Expression::ImageLoad::level
226/// [`ImageLoad`]: crate::Expression::ImageLoad
227/// [`Emit`]: crate::Statement::Emit
228/// [`ImageStore`]: crate::Statement::ImageStore
229pub fn find_checked_indexes(
230 module: &crate::Module,
231 function: &crate::Function,
232 info: &valid::FunctionInfo,
233 policies: BoundsCheckPolicies,
234) -> HandleSet<crate::Expression> {
235 use crate::Expression as Ex;
236
237 let mut guarded_indices = HandleSet::for_arena(&function.expressions);
238
239 // Don't bother scanning if we never need `ReadZeroSkipWrite`.
240 if policies.contains(BoundsCheckPolicy::ReadZeroSkipWrite) {
241 for (_handle, expr) in function.expressions.iter() {
242 // There's no need to handle `AccessIndex` expressions, as their
243 // indices never need to be cached.
244 match *expr {
245 Ex::Access { base, index } => {
246 if policies.choose_policy(base, &module.types, info)
247 == BoundsCheckPolicy::ReadZeroSkipWrite
248 && access_needs_check(
249 base,
250 GuardedIndex::Expression(index),
251 module,
252 &function.expressions,
253 info,
254 )
255 .is_some()
256 {
257 guarded_indices.insert(index);
258 }
259 }
260 Ex::ImageLoad {
261 coordinate,
262 array_index,
263 sample,
264 level,
265 ..
266 } => {
267 if policies.image_load == BoundsCheckPolicy::ReadZeroSkipWrite {
268 guarded_indices.insert(coordinate);
269 if let Some(array_index) = array_index {
270 guarded_indices.insert(array_index);
271 }
272 if let Some(sample) = sample {
273 guarded_indices.insert(sample);
274 }
275 if let Some(level) = level {
276 guarded_indices.insert(level);
277 }
278 }
279 }
280 _ => {}
281 }
282 }
283 }
284
285 guarded_indices
286}
287
288/// Determine whether `index` is statically known to be in bounds for `base`.
289///
290/// If we can't be sure that the index is in bounds, return the limit within
291/// which valid indices must fall.
292///
293/// The return value is one of the following:
294///
295/// - `Some(Known(n))` indicates that `n` is the largest valid index.
296///
297/// - `Some(Computed(global))` indicates that the largest valid index is one
298/// less than the length of the array that is the last member of the
299/// struct held in `global`.
300///
301/// - `None` indicates that the index need not be checked, either because it
302/// is statically known to be in bounds, or because the applicable policy
303/// is `Unchecked`.
304///
305/// This function only handles subscriptable types: arrays, vectors, and
306/// matrices. It does not handle struct member indices; those never require
307/// run-time checks, so it's best to deal with them further up the call
308/// chain.
309///
310/// This function assumes that any relevant overrides have fully-evaluated
311/// constants as their values (as arranged by [`process_overrides`], for
312/// example).
313///
314/// [`process_overrides`]: crate::back::pipeline_constants::process_overrides
315///
316/// # Panics
317///
318/// - If `base` is not an indexable type, panic.
319///
320/// - If `base` is an override-sized array, but the override's value is not a
321/// fully-evaluated constant expression, panic.
322pub fn access_needs_check(
323 base: Handle<crate::Expression>,
324 mut index: GuardedIndex,
325 module: &crate::Module,
326 expressions: &crate::Arena<crate::Expression>,
327 info: &valid::FunctionInfo,
328) -> Option<IndexableLength> {
329 let base_inner = info[base].ty.inner_with(&module.types);
330 // Unwrap safety: `Err` here indicates unindexable base types and invalid
331 // length constants, but `access_needs_check` is only used by back ends, so
332 // validation should have caught those problems.
333 let length = base_inner.indexable_length_resolved(module).unwrap();
334 index.try_resolve_to_constant(expressions, module);
335 if let (&GuardedIndex::Known(index), &IndexableLength::Known(length)) = (&index, &length) {
336 if index < length {
337 // Index is statically known to be in bounds, no check needed.
338 return None;
339 }
340 };
341
342 Some(length)
343}
344
345/// Items returned by the [`bounds_check_iter`] iterator.
346#[cfg_attr(not(feature = "msl-out"), allow(dead_code))]
347pub(crate) struct BoundsCheck {
348 /// The base of the [`Access`] or [`AccessIndex`] expression.
349 ///
350 /// [`Access`]: crate::Expression::Access
351 /// [`AccessIndex`]: crate::Expression::AccessIndex
352 pub base: Handle<crate::Expression>,
353
354 /// The index being accessed.
355 pub index: GuardedIndex,
356
357 /// The length of `base`.
358 pub length: IndexableLength,
359}
360
361/// Returns an iterator of accesses within the chain of `Access` and
362/// `AccessIndex` expressions starting from `chain` that may need to be
363/// bounds-checked at runtime.
364///
365/// Items are yielded as [`BoundsCheck`] instances.
366///
367/// Accesses through a struct are omitted, since you never need a bounds check
368/// for accessing a struct field.
369///
370/// If `chain` isn't an `Access` or `AccessIndex` expression at all, the
371/// iterator is empty.
372pub(crate) fn bounds_check_iter<'a>(
373 mut chain: Handle<crate::Expression>,
374 module: &'a crate::Module,
375 function: &'a crate::Function,
376 info: &'a valid::FunctionInfo,
377) -> impl Iterator<Item = BoundsCheck> + 'a {
378 iter::from_fn(move || {
379 let (next_expr, result) = match function.expressions[chain] {
380 crate::Expression::Access { base, index } => {
381 (base, Some((base, GuardedIndex::Expression(index))))
382 }
383 crate::Expression::AccessIndex { base, index } => {
384 // Don't try to check indices into structs. Validation already took
385 // care of them, and access_needs_check doesn't handle that case.
386 let mut base_inner = info[base].ty.inner_with(&module.types);
387 if let crate::TypeInner::Pointer { base, .. } = *base_inner {
388 base_inner = &module.types[base].inner;
389 }
390 match *base_inner {
391 crate::TypeInner::Struct { .. } => (base, None),
392 _ => (base, Some((base, GuardedIndex::Known(index)))),
393 }
394 }
395 _ => return None,
396 };
397 chain = next_expr;
398 Some(result)
399 })
400 .flatten()
401 .filter_map(|(base, index)| {
402 access_needs_check(base, index, module, &function.expressions, info).map(|length| {
403 BoundsCheck {
404 base,
405 index,
406 length,
407 }
408 })
409 })
410}
411
412/// Returns all the types which we need out-of-bounds locals for; that is,
413/// all of the types which the code might attempt to get an out-of-bounds
414/// pointer to, in which case we yield a pointer to the out-of-bounds local
415/// of the correct type.
416pub fn oob_local_types(
417 module: &crate::Module,
418 function: &crate::Function,
419 info: &valid::FunctionInfo,
420 policies: BoundsCheckPolicies,
421) -> FastHashSet<Handle<crate::Type>> {
422 let mut result = FastHashSet::default();
423
424 if policies.index != BoundsCheckPolicy::ReadZeroSkipWrite {
425 return result;
426 }
427
428 for statement in &function.body {
429 // The only situation in which we end up actually needing to create an
430 // out-of-bounds pointer is when passing one to a function.
431 //
432 // This is because pointers are never baked; they're just inlined everywhere
433 // they're used. That means that loads can just return 0, and stores can just do
434 // nothing; functions are the only case where you actually *have* to produce a
435 // pointer.
436 if let crate::Statement::Call {
437 function: callee,
438 ref arguments,
439 ..
440 } = *statement
441 {
442 // Now go through the arguments of the function looking for pointers which need bounds checks.
443 for (arg_info, &arg) in zip(&module.functions[callee].arguments, arguments) {
444 match module.types[arg_info.ty].inner {
445 crate::TypeInner::ValuePointer { .. } => {
446 // `ValuePointer`s should only ever be used when resolving the types of
447 // expressions, since the arena can no longer be modified at that point; things
448 // in the arena should always use proper `Pointer`s.
449 unreachable!("`ValuePointer` found in arena")
450 }
451 crate::TypeInner::Pointer { base, .. } => {
452 if bounds_check_iter(arg, module, function, info)
453 .next()
454 .is_some()
455 {
456 result.insert(base);
457 }
458 }
459 _ => continue,
460 };
461 }
462 }
463 }
464 result
465}
466
467impl GuardedIndex {
468 /// Make a `GuardedIndex::Known` from a `GuardedIndex::Expression` if possible.
469 ///
470 /// Return values that are already `Known` unchanged.
471 pub(crate) fn try_resolve_to_constant(
472 &mut self,
473 expressions: &crate::Arena<crate::Expression>,
474 module: &crate::Module,
475 ) {
476 if let GuardedIndex::Expression(expr) = *self {
477 *self = GuardedIndex::from_expression(expr, expressions, module);
478 }
479 }
480
481 pub(crate) fn from_expression(
482 expr: Handle<crate::Expression>,
483 expressions: &crate::Arena<crate::Expression>,
484 module: &crate::Module,
485 ) -> Self {
486 match module.to_ctx().eval_expr_to_u32_from(expr, expressions) {
487 Ok(value) => Self::Known(value),
488 Err(_) => Self::Expression(expr),
489 }
490 }
491}
492
493#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq)]
494pub enum IndexableLengthError {
495 #[error("Type is not indexable, and has no length (validation error)")]
496 TypeNotIndexable,
497 #[error(transparent)]
498 ResolveArraySizeError(#[from] super::ResolveArraySizeError),
499 #[error("Array size is still pending")]
500 Pending(crate::ArraySize),
501}
502
503impl crate::TypeInner {
504 /// Return the length of a subscriptable type.
505 ///
506 /// The `self` parameter should be a handle to a vector, matrix, or array
507 /// type, a pointer to one of those, or a value pointer. Arrays may be
508 /// fixed-size, dynamically sized, or sized by a specializable constant.
509 /// This function does not handle struct member references, as with
510 /// `AccessIndex`.
511 ///
512 /// The value returned is appropriate for bounds checks on subscripting.
513 ///
514 /// Return an error if `self` does not describe a subscriptable type at all.
515 pub fn indexable_length(
516 &self,
517 module: &crate::Module,
518 ) -> Result<IndexableLength, IndexableLengthError> {
519 use crate::TypeInner as Ti;
520 let known_length = match *self {
521 Ti::Vector { size, .. } => size as _,
522 Ti::Matrix { columns, .. } => columns as _,
523 Ti::Array { size, .. } | Ti::BindingArray { size, .. } => {
524 return size.to_indexable_length(module);
525 }
526 Ti::ValuePointer {
527 size: Some(size), ..
528 } => size as _,
529 Ti::Pointer { base, .. } => {
530 // When assigning types to expressions, ResolveContext::Resolve
531 // does a separate sub-match here instead of a full recursion,
532 // so we'll do the same.
533 let base_inner = &module.types[base].inner;
534 match *base_inner {
535 Ti::Vector { size, .. } => size as _,
536 Ti::Matrix { columns, .. } => columns as _,
537 Ti::Array { size, .. } | Ti::BindingArray { size, .. } => {
538 return size.to_indexable_length(module)
539 }
540 _ => return Err(IndexableLengthError::TypeNotIndexable),
541 }
542 }
543 _ => return Err(IndexableLengthError::TypeNotIndexable),
544 };
545 Ok(IndexableLength::Known(known_length))
546 }
547
548 /// Return the length of `self`, assuming overrides are yet to be supplied.
549 ///
550 /// Return the number of elements in `self`:
551 ///
552 /// - If `self` is a runtime-sized array, then return
553 /// [`IndexableLength::Dynamic`].
554 ///
555 /// - If `self` is an override-sized array, then assume that override values
556 /// have not yet been supplied, and return [`IndexableLength::Dynamic`].
557 ///
558 /// - Otherwise, the type simply tells us the length of `self`, so return
559 /// [`IndexableLength::Known`].
560 ///
561 /// If `self` is not an indexable type at all, return an error.
562 ///
563 /// The difference between this and `indexable_length_resolved` is that we
564 /// treat override-sized arrays and dynamically-sized arrays both as
565 /// [`Dynamic`], on the assumption that our callers want to treat both cases
566 /// as "not yet possible to check".
567 ///
568 /// [`Dynamic`]: IndexableLength::Dynamic
569 pub fn indexable_length_pending(
570 &self,
571 module: &crate::Module,
572 ) -> Result<IndexableLength, IndexableLengthError> {
573 let length = self.indexable_length(module);
574 if let Err(IndexableLengthError::Pending(_)) = length {
575 return Ok(IndexableLength::Dynamic);
576 }
577 length
578 }
579
580 /// Return the length of `self`, assuming overrides have been resolved.
581 ///
582 /// Return the number of elements in `self`:
583 ///
584 /// - If `self` is a runtime-sized array, then return
585 /// [`IndexableLength::Dynamic`].
586 ///
587 /// - If `self` is an override-sized array, then assume that the override's
588 /// value is a fully-evaluated constant expression, and return
589 /// [`IndexableLength::Known`]. Otherwise, return an error.
590 ///
591 /// - Otherwise, the type simply tells us the length of `self`, so return
592 /// [`IndexableLength::Known`].
593 ///
594 /// If `self` is not an indexable type at all, return an error.
595 ///
596 /// The difference between this and `indexable_length_pending` is
597 /// that if `self` is override-sized, we require the override's
598 /// value to be known.
599 pub fn indexable_length_resolved(
600 &self,
601 module: &crate::Module,
602 ) -> Result<IndexableLength, IndexableLengthError> {
603 let length = self.indexable_length(module);
604
605 // If the length is override-based, then try to compute its value now.
606 if let Err(IndexableLengthError::Pending(size)) = length {
607 if let IndexableLength::Known(computed) = size.resolve(module.to_ctx())? {
608 return Ok(IndexableLength::Known(computed));
609 }
610 }
611 length
612 }
613}
614
615/// The number of elements in an indexable type.
616///
617/// This summarizes the length of vectors, matrices, and arrays in a way that is
618/// convenient for indexing and bounds-checking code.
619#[derive(Debug)]
620pub enum IndexableLength {
621 /// Values of this type always have the given number of elements.
622 Known(u32),
623
624 /// The number of elements is determined at runtime.
625 Dynamic,
626}
627
628impl crate::ArraySize {
629 pub const fn to_indexable_length(
630 self,
631 _module: &crate::Module,
632 ) -> Result<IndexableLength, IndexableLengthError> {
633 match self {
634 Self::Constant(length) => Ok(IndexableLength::Known(length.get())),
635 Self::Pending(_) => Err(IndexableLengthError::Pending(self)),
636 Self::Dynamic => Ok(IndexableLength::Dynamic),
637 }
638 }
639}