1use alloc::vec::Vec;
6
7use arrayvec::ArrayVec;
8use spirv::Word;
9
10use super::{
11 helpers::map_storage_class, index::BoundsCheckResult, selection::Selection, Block,
12 BlockContext, Dimension, Error, IdGenerator, Instruction, LocalType, LookupType, NumericType,
13 ResultMember, WrappedFunction, Writer, WriterFlags,
14};
15use crate::{
16 arena::Handle, back::spv::helpers::is_uniform_matcx2_struct_member_access,
17 proc::index::GuardedIndex, Statement,
18};
19
20fn get_dimension(type_inner: &crate::TypeInner) -> Dimension {
21 match *type_inner {
22 crate::TypeInner::Scalar(_) => Dimension::Scalar,
23 crate::TypeInner::Vector { .. } => Dimension::Vector,
24 crate::TypeInner::Matrix { .. } => Dimension::Matrix,
25 crate::TypeInner::CooperativeMatrix { .. } => Dimension::CooperativeMatrix,
26 _ => unreachable!(),
27 }
28}
29
30#[derive(Copy, Clone)]
42enum AccessTypeAdjustment {
43 None,
52
53 IntroducePointer(spirv::StorageClass),
77
78 UseStd140CompatType,
88}
89
90enum ExpressionPointer {
94 Ready { pointer_id: Word },
97
98 Conditional {
104 condition: Word,
105 access: Instruction,
106 },
107}
108
109enum BlockExit {
111 Return,
113 Branch {
115 target: Word,
117 },
118 BreakIf {
124 condition: Handle<crate::Expression>,
126 preamble_id: Word,
128 },
129}
130
131#[must_use]
142enum BlockExitDisposition {
143 Used,
147
148 Discarded,
153}
154
155#[derive(Clone, Copy, Default)]
156struct LoopContext {
157 continuing_id: Option<Word>,
158 break_id: Option<Word>,
159}
160
161#[derive(Debug)]
162pub(crate) struct DebugInfoInner<'a> {
163 pub source_code: &'a str,
164 pub source_file_id: Word,
165}
166
167impl Writer {
168 fn write_epilogue_position_y_flip(
173 &mut self,
174 position_id: Word,
175 body: &mut Vec<Instruction>,
176 ) -> Result<(), Error> {
177 let float_ptr_type_id = self.get_f32_pointer_type_id(spirv::StorageClass::Output);
178 let index_y_id = self.get_index_constant(1);
179 let access_id = self.id_gen.next();
180 body.push(Instruction::access_chain(
181 float_ptr_type_id,
182 access_id,
183 position_id,
184 &[index_y_id],
185 ));
186
187 let float_type_id = self.get_f32_type_id();
188 let load_id = self.id_gen.next();
189 body.push(Instruction::load(float_type_id, load_id, access_id, None));
190
191 let neg_id = self.id_gen.next();
192 body.push(Instruction::unary(
193 spirv::Op::FNegate,
194 float_type_id,
195 neg_id,
196 load_id,
197 ));
198
199 body.push(Instruction::store(access_id, neg_id, None));
200 Ok(())
201 }
202
203 fn write_epilogue_frag_depth_clamp(
205 &mut self,
206 frag_depth_id: Word,
207 body: &mut Vec<Instruction>,
208 ) -> Result<(), Error> {
209 let float_type_id = self.get_f32_type_id();
210 let zero_scalar_id = self.get_constant_scalar(crate::Literal::F32(0.0));
211 let one_scalar_id = self.get_constant_scalar(crate::Literal::F32(1.0));
212
213 let original_id = self.id_gen.next();
214 body.push(Instruction::load(
215 float_type_id,
216 original_id,
217 frag_depth_id,
218 None,
219 ));
220
221 let clamp_id = self.id_gen.next();
222 body.push(Instruction::ext_inst_gl_op(
223 self.gl450_ext_inst_id,
224 spirv::GlslStd450Op::FClamp,
225 float_type_id,
226 clamp_id,
227 &[original_id, zero_scalar_id, one_scalar_id],
228 ));
229
230 body.push(Instruction::store(frag_depth_id, clamp_id, None));
231 Ok(())
232 }
233
234 fn write_entry_point_return(
235 &mut self,
236 value_id: Word,
237 ir_result: &crate::FunctionResult,
238 result_members: &[ResultMember],
239 body: &mut Vec<Instruction>,
240 ) -> Result<Instruction, Error> {
241 for (index, res_member) in result_members.iter().enumerate() {
242 if res_member.built_in == Some(crate::BuiltIn::MeshTaskSize) {
244 return Ok(Instruction::return_value(value_id));
245 }
246 let member_value_id = match ir_result.binding {
247 Some(_) => value_id,
248 None => {
249 let member_value_id = self.id_gen.next();
250 body.push(Instruction::composite_extract(
251 res_member.type_id,
252 member_value_id,
253 value_id,
254 &[index as u32],
255 ));
256 member_value_id
257 }
258 };
259
260 self.store_io_with_f16_polyfill(body, res_member.id, member_value_id);
261
262 match res_member.built_in {
263 Some(crate::BuiltIn::Position { .. })
264 if self.flags.contains(WriterFlags::ADJUST_COORDINATE_SPACE) =>
265 {
266 self.write_epilogue_position_y_flip(res_member.id, body)?;
267 }
268 Some(crate::BuiltIn::FragDepth)
269 if self.flags.contains(WriterFlags::CLAMP_FRAG_DEPTH) =>
270 {
271 self.write_epilogue_frag_depth_clamp(res_member.id, body)?;
272 }
273 _ => {}
274 }
275 }
276 Ok(Instruction::return_void())
277 }
278}
279
280impl BlockContext<'_> {
281 fn write_force_bounded_loop_instructions(&mut self, mut block: Block, merge_id: Word) -> Block {
294 let uint_type_id = self.writer.get_u32_type_id();
295 let uint2_type_id = self.writer.get_vec2u_type_id();
296 let uint2_ptr_type_id = self
297 .writer
298 .get_vec2u_pointer_type_id(spirv::StorageClass::Function);
299 let bool_type_id = self.writer.get_bool_type_id();
300 let bool2_type_id = self.writer.get_vec2_bool_type_id();
301 let zero_uint_const_id = self.writer.get_constant_scalar(crate::Literal::U32(0));
302 let zero_uint2_const_id = self.writer.get_constant_composite(
303 LookupType::Local(LocalType::Numeric(NumericType::Vector {
304 size: crate::VectorSize::Bi,
305 scalar: crate::Scalar::U32,
306 })),
307 &[zero_uint_const_id, zero_uint_const_id],
308 );
309 let one_uint_const_id = self.writer.get_constant_scalar(crate::Literal::U32(1));
310 let max_uint_const_id = self
311 .writer
312 .get_constant_scalar(crate::Literal::U32(u32::MAX));
313 let max_uint2_const_id = self.writer.get_constant_composite(
314 LookupType::Local(LocalType::Numeric(NumericType::Vector {
315 size: crate::VectorSize::Bi,
316 scalar: crate::Scalar::U32,
317 })),
318 &[max_uint_const_id, max_uint_const_id],
319 );
320
321 let loop_counter_var_id = self.gen_id();
322 if self.writer.flags.contains(WriterFlags::DEBUG) {
323 self.writer
324 .debugs
325 .push(Instruction::name(loop_counter_var_id, "loop_bound"));
326 }
327 let var = super::LocalVariable {
328 id: loop_counter_var_id,
329 instruction: Instruction::variable(
330 uint2_ptr_type_id,
331 loop_counter_var_id,
332 spirv::StorageClass::Function,
333 Some(max_uint2_const_id),
334 ),
335 };
336 self.function.force_loop_bounding_vars.push(var);
337
338 let break_if_block = self.gen_id();
339
340 self.function
341 .consume(block, Instruction::branch(break_if_block));
342 block = Block::new(break_if_block);
343
344 let load_id = self.gen_id();
347 block.body.push(Instruction::load(
348 uint2_type_id,
349 load_id,
350 loop_counter_var_id,
351 None,
352 ));
353
354 let eq_id = self.gen_id();
357 block.body.push(Instruction::binary(
358 spirv::Op::IEqual,
359 bool2_type_id,
360 eq_id,
361 zero_uint2_const_id,
362 load_id,
363 ));
364 let all_eq_id = self.gen_id();
365 block.body.push(Instruction::relational(
366 spirv::Op::All,
367 bool_type_id,
368 all_eq_id,
369 eq_id,
370 ));
371
372 let inc_counter_block_id = self.gen_id();
373 block.body.push(Instruction::selection_merge(
374 inc_counter_block_id,
375 spirv::SelectionControl::empty(),
376 ));
377 self.function.consume(
378 block,
379 Instruction::branch_conditional(all_eq_id, merge_id, inc_counter_block_id),
380 );
381 block = Block::new(inc_counter_block_id);
382
383 let low_id = self.gen_id();
389 block.body.push(Instruction::composite_extract(
390 uint_type_id,
391 low_id,
392 load_id,
393 &[1],
394 ));
395 let low_overflow_id = self.gen_id();
396 block.body.push(Instruction::binary(
397 spirv::Op::IEqual,
398 bool_type_id,
399 low_overflow_id,
400 low_id,
401 zero_uint_const_id,
402 ));
403 let carry_bit_id = self.gen_id();
404 block.body.push(Instruction::select(
405 uint_type_id,
406 carry_bit_id,
407 low_overflow_id,
408 one_uint_const_id,
409 zero_uint_const_id,
410 ));
411 let decrement_id = self.gen_id();
412 block.body.push(Instruction::composite_construct(
413 uint2_type_id,
414 decrement_id,
415 &[carry_bit_id, one_uint_const_id],
416 ));
417 let result_id = self.gen_id();
418 block.body.push(Instruction::binary(
419 spirv::Op::ISub,
420 uint2_type_id,
421 result_id,
422 load_id,
423 decrement_id,
424 ));
425 block
426 .body
427 .push(Instruction::store(loop_counter_var_id, result_id, None));
428
429 block
430 }
431
432 fn maybe_write_immediate_vector_dynamic_access(
459 &mut self,
460 pointer: Handle<crate::Expression>,
461 block: &mut Block,
462 ) -> Result<Option<Word>, Error> {
463 let crate::Expression::Access {
466 base: vector_pointer,
467 index,
468 } = self.ir_function.expressions[pointer]
469 else {
470 return Ok(None);
471 };
472
473 if let GuardedIndex::Known(_) =
476 GuardedIndex::from_expression(index, &self.ir_function.expressions, self.ir_module)
477 {
478 return Ok(None);
479 }
480
481 let vector_pointer_ty = self.fun_info[vector_pointer]
484 .ty
485 .inner_with(&self.ir_module.types);
486 if vector_pointer_ty.pointer_space() != Some(crate::AddressSpace::Immediate) {
487 return Ok(None);
488 }
489 let Some(vector_base_ty) = vector_pointer_ty.pointer_base_type() else {
490 return Ok(None);
491 };
492 let crate::TypeInner::Vector { size, scalar } =
493 *vector_base_ty.inner_with(&self.ir_module.types)
494 else {
495 return Ok(None);
496 };
497
498 let vector_type_id = self.get_numeric_type_id(NumericType::Vector { size, scalar });
499 let component_type_id = self.get_numeric_type_id(NumericType::Scalar(scalar));
500
501 let vector_load_id = self.write_checked_load(
502 vector_pointer,
503 block,
504 AccessTypeAdjustment::None,
505 vector_type_id,
506 )?;
507
508 let result_id = self.write_vector_access(
509 component_type_id,
510 vector_pointer,
511 Some(vector_load_id),
512 GuardedIndex::Expression(index),
513 block,
514 )?;
515
516 Ok(Some(result_id))
517 }
518
519 fn maybe_write_uniform_matcx2_dynamic_access(
536 &mut self,
537 pointer: Handle<crate::Expression>,
538 block: &mut Block,
539 ) -> Result<Option<Word>, Error> {
540 let (column_pointer, component_index) = match self.fun_info[pointer]
546 .ty
547 .inner_with(&self.ir_module.types)
548 .pointer_base_type()
549 {
550 Some(resolution) => match *resolution.inner_with(&self.ir_module.types) {
551 crate::TypeInner::Scalar(_) => match self.ir_function.expressions[pointer] {
552 crate::Expression::Access { base, index } => {
553 (base, Some(GuardedIndex::Expression(index)))
554 }
555 crate::Expression::AccessIndex { base, index } => {
556 (base, Some(GuardedIndex::Known(index)))
557 }
558 _ => return Ok(None),
559 },
560 crate::TypeInner::Vector { .. } => (pointer, None),
561 _ => return Ok(None),
562 },
563 None => return Ok(None),
564 };
565
566 let crate::Expression::Access {
569 base: matrix_pointer,
570 index: column_index,
571 } = self.ir_function.expressions[column_pointer]
572 else {
573 return Ok(None);
574 };
575
576 let crate::TypeInner::Pointer {
578 base: matrix_pointer_base_type,
579 space: crate::AddressSpace::Uniform,
580 } = *self.fun_info[matrix_pointer]
581 .ty
582 .inner_with(&self.ir_module.types)
583 else {
584 return Ok(None);
585 };
586
587 let crate::TypeInner::Matrix {
589 columns,
590 rows: rows @ crate::VectorSize::Bi,
591 scalar,
592 } = self.ir_module.types[matrix_pointer_base_type].inner
593 else {
594 return Ok(None);
595 };
596
597 let matrix_type_id = self.get_numeric_type_id(NumericType::Matrix {
598 columns,
599 rows,
600 scalar,
601 });
602 let column_type_id = self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
603 let component_type_id = self.get_numeric_type_id(NumericType::Scalar(scalar));
604 let get_column_function_id = self.writer.wrapped_functions
605 [&WrappedFunction::MatCx2GetColumn {
606 r#type: matrix_pointer_base_type,
607 }];
608
609 let matrix_load_id = self.write_checked_load(
610 matrix_pointer,
611 block,
612 AccessTypeAdjustment::None,
613 matrix_type_id,
614 )?;
615
616 let column_index_id = match *self.fun_info[column_index]
619 .ty
620 .inner_with(&self.ir_module.types)
621 {
622 crate::TypeInner::Scalar(crate::Scalar {
623 kind: crate::ScalarKind::Uint,
624 ..
625 }) => self.cached[column_index],
626 crate::TypeInner::Scalar(crate::Scalar {
627 kind: crate::ScalarKind::Sint,
628 ..
629 }) => {
630 let cast_id = self.gen_id();
631 let u32_type_id = self.writer.get_u32_type_id();
632 block.body.push(Instruction::unary(
633 spirv::Op::Bitcast,
634 u32_type_id,
635 cast_id,
636 self.cached[column_index],
637 ));
638 cast_id
639 }
640 _ => return Err(Error::Validation("Matrix access index must be u32 or i32")),
641 };
642 let column_id = self.gen_id();
643 block.body.push(Instruction::function_call(
644 column_type_id,
645 column_id,
646 get_column_function_id,
647 &[matrix_load_id, column_index_id],
648 ));
649 let result_id = match component_index {
650 Some(index) => self.write_vector_access(
651 component_type_id,
652 column_pointer,
653 Some(column_id),
654 index,
655 block,
656 )?,
657 None => column_id,
658 };
659
660 Ok(Some(result_id))
661 }
662
663 fn maybe_write_load_uniform_matcx2_struct_member(
675 &mut self,
676 pointer: Handle<crate::Expression>,
677 block: &mut Block,
678 ) -> Result<Option<Word>, Error> {
679 let crate::TypeInner::Pointer {
681 base: matrix_type,
682 space: space @ crate::AddressSpace::Uniform,
683 } = *self.fun_info[pointer].ty.inner_with(&self.ir_module.types)
684 else {
685 return Ok(None);
686 };
687
688 let crate::TypeInner::Matrix {
689 columns,
690 rows: rows @ crate::VectorSize::Bi,
691 scalar,
692 } = self.ir_module.types[matrix_type].inner
693 else {
694 return Ok(None);
695 };
696
697 let crate::Expression::AccessIndex {
700 base: struct_pointer,
701 index: member_index,
702 } = self.ir_function.expressions[pointer]
703 else {
704 return Ok(None);
705 };
706
707 let crate::TypeInner::Pointer {
708 base: struct_type, ..
709 } = *self.fun_info[struct_pointer]
710 .ty
711 .inner_with(&self.ir_module.types)
712 else {
713 return Ok(None);
714 };
715
716 let crate::TypeInner::Struct { .. } = self.ir_module.types[struct_type].inner else {
717 return Ok(None);
718 };
719
720 let matrix_type_id = self.get_numeric_type_id(NumericType::Matrix {
721 columns,
722 rows,
723 scalar,
724 });
725 let column_type_id = self.get_numeric_type_id(NumericType::Vector { size: rows, scalar });
726 let column_pointer_type_id =
727 self.get_pointer_type_id(column_type_id, map_storage_class(space));
728 let column0_index = self.writer.std140_compat_uniform_types[&struct_type].member_indices
729 [member_index as usize];
730 let column_indices = (0..columns as u32)
731 .map(|c| self.get_index_constant(column0_index + c))
732 .collect::<ArrayVec<_, 4>>();
733
734 let load_mat_from_struct =
737 |struct_pointer_id: Word, id_gen: &mut IdGenerator, block: &mut Block| -> Word {
738 let mut column_ids: ArrayVec<Word, 4> = ArrayVec::new();
739 for index in &column_indices {
740 let column_pointer_id = id_gen.next();
741 block.body.push(Instruction::access_chain(
742 column_pointer_type_id,
743 column_pointer_id,
744 struct_pointer_id,
745 &[*index],
746 ));
747 let column_id = id_gen.next();
748 block.body.push(Instruction::load(
749 column_type_id,
750 column_id,
751 column_pointer_id,
752 None,
753 ));
754 column_ids.push(column_id);
755 }
756 let result_id = id_gen.next();
757 block.body.push(Instruction::composite_construct(
758 matrix_type_id,
759 result_id,
760 &column_ids,
761 ));
762 result_id
763 };
764
765 let result_id = match self.write_access_chain(
766 struct_pointer,
767 block,
768 AccessTypeAdjustment::UseStd140CompatType,
769 )? {
770 ExpressionPointer::Ready { pointer_id } => {
771 load_mat_from_struct(pointer_id, &mut self.writer.id_gen, block)
772 }
773 ExpressionPointer::Conditional { condition, access } => self
774 .write_conditional_indexed_load(
775 matrix_type_id,
776 condition,
777 block,
778 |id_gen, block| {
779 let pointer_id = access.result_id.unwrap();
780 block.body.push(access);
781 load_mat_from_struct(pointer_id, id_gen, block)
782 },
783 ),
784 };
785
786 Ok(Some(result_id))
787 }
788
789 pub(super) fn cache_expression_value(
791 &mut self,
792 expr_handle: Handle<crate::Expression>,
793 block: &mut Block,
794 ) -> Result<(), Error> {
795 let is_named_expression = self
796 .ir_function
797 .named_expressions
798 .contains_key(&expr_handle);
799
800 if self.fun_info[expr_handle].ref_count == 0 && !is_named_expression {
801 return Ok(());
802 }
803
804 let result_type_id = self.get_expression_type_id(&self.fun_info[expr_handle].ty);
805 let id = match self.ir_function.expressions[expr_handle] {
806 crate::Expression::Literal(literal) => self.writer.get_constant_scalar(literal),
807 crate::Expression::Constant(handle) => {
808 let init = self.ir_module.constants[handle].init;
809 self.writer.constant_ids[init]
810 }
811 crate::Expression::Override(_) => return Err(Error::Override),
812 crate::Expression::ZeroValue(_) => self.writer.get_constant_null(result_type_id),
813 crate::Expression::Compose { ty, ref components } => {
814 self.temp_list.clear();
815 if self.expression_constness.is_const(expr_handle) {
816 self.temp_list.extend(
817 crate::proc::flatten_compose(
818 ty,
819 components,
820 &self.ir_function.expressions,
821 &self.ir_module.types,
822 )
823 .map(|component| self.cached[component]),
824 );
825 self.writer
826 .get_constant_composite(LookupType::Handle(ty), &self.temp_list)
827 } else {
828 self.temp_list
829 .extend(components.iter().map(|&component| self.cached[component]));
830
831 let id = self.gen_id();
832 block.body.push(Instruction::composite_construct(
833 result_type_id,
834 id,
835 &self.temp_list,
836 ));
837 id
838 }
839 }
840 crate::Expression::Splat { size, value } => {
841 let value_id = self.cached[value];
842 let components = &[value_id; 4][..size as usize];
843
844 if self.expression_constness.is_const(expr_handle) {
845 let ty = self
846 .writer
847 .get_expression_lookup_type(&self.fun_info[expr_handle].ty);
848 self.writer.get_constant_composite(ty, components)
849 } else {
850 let id = self.gen_id();
851 block.body.push(Instruction::composite_construct(
852 result_type_id,
853 id,
854 components,
855 ));
856 id
857 }
858 }
859 crate::Expression::Access { base, index } => {
860 let base_ty_inner = self.fun_info[base].ty.inner_with(&self.ir_module.types);
861 match *base_ty_inner {
862 crate::TypeInner::Pointer { .. } | crate::TypeInner::ValuePointer { .. } => {
863 0
869 }
870 _ if self.function.spilled_accesses.contains(base) => {
871 self.function.spilled_accesses.insert(expr_handle);
879 self.maybe_access_spilled_composite(expr_handle, block, result_type_id)?
880 }
881 crate::TypeInner::Vector { .. } => self.write_vector_access(
882 result_type_id,
883 base,
884 None,
885 GuardedIndex::Expression(index),
886 block,
887 )?,
888 crate::TypeInner::Array { .. } | crate::TypeInner::Matrix { .. } => {
889 match GuardedIndex::from_expression(
891 index,
892 &self.ir_function.expressions,
893 self.ir_module,
894 ) {
895 GuardedIndex::Known(value) => {
896 let id = self.gen_id();
906 let base_id = self.cached[base];
907 block.body.push(Instruction::composite_extract(
908 result_type_id,
909 id,
910 base_id,
911 &[value],
912 ));
913 id
914 }
915 GuardedIndex::Expression(_) => {
916 self.spill_to_internal_variable(base, block);
923
924 self.function.spilled_accesses.insert(expr_handle);
927 self.maybe_access_spilled_composite(
928 expr_handle,
929 block,
930 result_type_id,
931 )?
932 }
933 }
934 }
935 crate::TypeInner::BindingArray {
936 base: binding_type, ..
937 } => {
938 let result_id = match self.write_access_chain(
941 expr_handle,
942 block,
943 AccessTypeAdjustment::IntroducePointer(
944 spirv::StorageClass::UniformConstant,
945 ),
946 )? {
947 ExpressionPointer::Ready { pointer_id } => pointer_id,
948 ExpressionPointer::Conditional { .. } => {
949 return Err(Error::FeatureNotImplemented(
950 "Texture array out-of-bounds handling",
951 ));
952 }
953 };
954
955 let binding_type_id = self.get_handle_type_id(binding_type);
956
957 let load_id = self.gen_id();
958 block.body.push(Instruction::load(
959 binding_type_id,
960 load_id,
961 result_id,
962 None,
963 ));
964
965 if self.fun_info[index].uniformity.non_uniform_result.is_some() {
969 self.writer
970 .decorate_non_uniform_binding_array_access(load_id)?;
971 }
972
973 load_id
974 }
975 ref other => {
976 log::error!(
977 "Unable to access base {:?} of type {:?}",
978 self.ir_function.expressions[base],
979 other
980 );
981 return Err(Error::Validation(
982 "only vectors and arrays may be dynamically indexed by value",
983 ));
984 }
985 }
986 }
987 crate::Expression::AccessIndex { base, index } => {
988 match *self.fun_info[base].ty.inner_with(&self.ir_module.types) {
989 crate::TypeInner::Pointer { .. } | crate::TypeInner::ValuePointer { .. } => {
990 0
996 }
997 _ if self.function.spilled_accesses.contains(base) => {
998 self.function.spilled_accesses.insert(expr_handle);
1006 self.maybe_access_spilled_composite(expr_handle, block, result_type_id)?
1007 }
1008 crate::TypeInner::Vector { .. }
1009 | crate::TypeInner::Matrix { .. }
1010 | crate::TypeInner::Array { .. }
1011 | crate::TypeInner::Struct { .. } => {
1012 let id = self.gen_id();
1017 let base_id = self.cached[base];
1018 block.body.push(Instruction::composite_extract(
1019 result_type_id,
1020 id,
1021 base_id,
1022 &[index],
1023 ));
1024 id
1025 }
1026 crate::TypeInner::BindingArray {
1027 base: binding_type, ..
1028 } => {
1029 let result_id = match self.write_access_chain(
1032 expr_handle,
1033 block,
1034 AccessTypeAdjustment::IntroducePointer(
1035 spirv::StorageClass::UniformConstant,
1036 ),
1037 )? {
1038 ExpressionPointer::Ready { pointer_id } => pointer_id,
1039 ExpressionPointer::Conditional { .. } => {
1040 return Err(Error::FeatureNotImplemented(
1041 "Texture array out-of-bounds handling",
1042 ));
1043 }
1044 };
1045
1046 let binding_type_id = self.get_handle_type_id(binding_type);
1047
1048 let load_id = self.gen_id();
1049 block.body.push(Instruction::load(
1050 binding_type_id,
1051 load_id,
1052 result_id,
1053 None,
1054 ));
1055
1056 load_id
1057 }
1058 ref other => {
1059 log::error!("Unable to access index of {other:?}");
1060 return Err(Error::FeatureNotImplemented("access index for type"));
1061 }
1062 }
1063 }
1064 crate::Expression::GlobalVariable(handle) => {
1065 self.writer.global_variables[handle].access_id
1066 }
1067 crate::Expression::Swizzle {
1068 size,
1069 vector,
1070 pattern,
1071 } => {
1072 let vector_id = self.cached[vector];
1073 self.temp_list.clear();
1074 for &sc in pattern[..size as usize].iter() {
1075 self.temp_list.push(sc as Word);
1076 }
1077 let id = self.gen_id();
1078 block.body.push(Instruction::vector_shuffle(
1079 result_type_id,
1080 id,
1081 vector_id,
1082 vector_id,
1083 &self.temp_list,
1084 ));
1085 id
1086 }
1087 crate::Expression::Unary { op, expr } => {
1088 let id = self.gen_id();
1089 let expr_id = self.cached[expr];
1090 let expr_ty_inner = self.fun_info[expr].ty.inner_with(&self.ir_module.types);
1091
1092 let spirv_op = match op {
1093 crate::UnaryOperator::Negate => match expr_ty_inner.scalar_kind() {
1094 Some(crate::ScalarKind::Float) => spirv::Op::FNegate,
1095 Some(crate::ScalarKind::Sint) => spirv::Op::SNegate,
1096 _ => return Err(Error::Validation("Unexpected kind for negation")),
1097 },
1098 crate::UnaryOperator::LogicalNot => spirv::Op::LogicalNot,
1099 crate::UnaryOperator::BitwiseNot => spirv::Op::Not,
1100 };
1101
1102 block
1103 .body
1104 .push(Instruction::unary(spirv_op, result_type_id, id, expr_id));
1105 id
1106 }
1107 crate::Expression::Binary { op, left, right } => {
1108 let id = self.gen_id();
1109 let left_id = self.cached[left];
1110 let right_id = self.cached[right];
1111 let left_type_id = self.get_expression_type_id(&self.fun_info[left].ty);
1112 let right_type_id = self.get_expression_type_id(&self.fun_info[right].ty);
1113
1114 if let Some(function_id) =
1115 self.writer
1116 .wrapped_functions
1117 .get(&WrappedFunction::BinaryOp {
1118 op,
1119 left_type_id,
1120 right_type_id,
1121 })
1122 {
1123 block.body.push(Instruction::function_call(
1124 result_type_id,
1125 id,
1126 *function_id,
1127 &[left_id, right_id],
1128 ));
1129 } else {
1130 let left_ty_inner = self.fun_info[left].ty.inner_with(&self.ir_module.types);
1131 let right_ty_inner = self.fun_info[right].ty.inner_with(&self.ir_module.types);
1132
1133 let left_dimension = get_dimension(left_ty_inner);
1134 let right_dimension = get_dimension(right_ty_inner);
1135
1136 let mut reverse_operands = false;
1137
1138 let spirv_op = match op {
1139 crate::BinaryOperator::Add => match *left_ty_inner {
1140 crate::TypeInner::Scalar(scalar)
1141 | crate::TypeInner::Vector { scalar, .. } => match scalar.kind {
1142 crate::ScalarKind::Float => spirv::Op::FAdd,
1143 _ => spirv::Op::IAdd,
1144 },
1145 crate::TypeInner::Matrix {
1146 columns,
1147 rows,
1148 scalar,
1149 } => {
1150 self.write_matrix_matrix_column_op(
1152 block,
1153 id,
1154 result_type_id,
1155 left_id,
1156 right_id,
1157 columns,
1158 rows,
1159 scalar.width,
1160 spirv::Op::FAdd,
1161 );
1162
1163 self.cached[expr_handle] = id;
1164 return Ok(());
1165 }
1166 crate::TypeInner::CooperativeMatrix { .. } => spirv::Op::FAdd,
1167 _ => unimplemented!(),
1168 },
1169 crate::BinaryOperator::Subtract => match *left_ty_inner {
1170 crate::TypeInner::Scalar(scalar)
1171 | crate::TypeInner::Vector { scalar, .. } => match scalar.kind {
1172 crate::ScalarKind::Float => spirv::Op::FSub,
1173 _ => spirv::Op::ISub,
1174 },
1175 crate::TypeInner::Matrix {
1176 columns,
1177 rows,
1178 scalar,
1179 } => {
1180 self.write_matrix_matrix_column_op(
1181 block,
1182 id,
1183 result_type_id,
1184 left_id,
1185 right_id,
1186 columns,
1187 rows,
1188 scalar.width,
1189 spirv::Op::FSub,
1190 );
1191
1192 self.cached[expr_handle] = id;
1193 return Ok(());
1194 }
1195 crate::TypeInner::CooperativeMatrix { .. } => spirv::Op::FSub,
1196 _ => unimplemented!(),
1197 },
1198 crate::BinaryOperator::Multiply => {
1199 match (left_dimension, right_dimension) {
1200 (Dimension::Scalar, Dimension::Vector) => {
1201 self.write_vector_scalar_mult(
1202 block,
1203 id,
1204 result_type_id,
1205 right_id,
1206 left_id,
1207 right_ty_inner,
1208 );
1209
1210 self.cached[expr_handle] = id;
1211 return Ok(());
1212 }
1213 (Dimension::Vector, Dimension::Scalar) => {
1214 self.write_vector_scalar_mult(
1215 block,
1216 id,
1217 result_type_id,
1218 left_id,
1219 right_id,
1220 left_ty_inner,
1221 );
1222
1223 self.cached[expr_handle] = id;
1224 return Ok(());
1225 }
1226 (Dimension::Vector, Dimension::Matrix) => {
1227 spirv::Op::VectorTimesMatrix
1228 }
1229 (Dimension::Matrix, Dimension::Scalar)
1230 | (Dimension::CooperativeMatrix, Dimension::Scalar) => {
1231 spirv::Op::MatrixTimesScalar
1232 }
1233 (Dimension::Scalar, Dimension::Matrix)
1234 | (Dimension::Scalar, Dimension::CooperativeMatrix) => {
1235 reverse_operands = true;
1236 spirv::Op::MatrixTimesScalar
1237 }
1238 (Dimension::Matrix, Dimension::Vector) => {
1239 spirv::Op::MatrixTimesVector
1240 }
1241 (Dimension::Matrix, Dimension::Matrix) => {
1242 spirv::Op::MatrixTimesMatrix
1243 }
1244 (Dimension::Vector, Dimension::Vector)
1245 | (Dimension::Scalar, Dimension::Scalar)
1246 if left_ty_inner.scalar_kind()
1247 == Some(crate::ScalarKind::Float) =>
1248 {
1249 spirv::Op::FMul
1250 }
1251 (Dimension::Vector, Dimension::Vector)
1252 | (Dimension::Scalar, Dimension::Scalar) => spirv::Op::IMul,
1253 (Dimension::CooperativeMatrix, Dimension::CooperativeMatrix)
1254 | (Dimension::CooperativeMatrix, _)
1256 | (_, Dimension::CooperativeMatrix) => {
1257 unimplemented!()
1258 }
1259 }
1260 }
1261 crate::BinaryOperator::Divide => match left_ty_inner.scalar_kind() {
1262 Some(crate::ScalarKind::Sint) => spirv::Op::SDiv,
1263 Some(crate::ScalarKind::Uint) => spirv::Op::UDiv,
1264 Some(crate::ScalarKind::Float) => spirv::Op::FDiv,
1265 _ => unimplemented!(),
1266 },
1267 crate::BinaryOperator::Modulo => match left_ty_inner.scalar_kind() {
1268 Some(crate::ScalarKind::Float) => spirv::Op::FRem,
1269 Some(crate::ScalarKind::Sint) => {
1270 unreachable!("signed modulo must be lowered via the wrapped path")
1277 }
1278 Some(crate::ScalarKind::Uint) => {
1279 assert!(!self.writer.emit_int_div_checks);
1280 spirv::Op::UMod
1281 }
1282 _ => unimplemented!(),
1283 },
1284 crate::BinaryOperator::Equal => match left_ty_inner.scalar_kind() {
1285 Some(crate::ScalarKind::Sint | crate::ScalarKind::Uint) => {
1286 spirv::Op::IEqual
1287 }
1288 Some(crate::ScalarKind::Float) => spirv::Op::FOrdEqual,
1289 Some(crate::ScalarKind::Bool) => spirv::Op::LogicalEqual,
1290 _ => unimplemented!(),
1291 },
1292 crate::BinaryOperator::NotEqual => match left_ty_inner.scalar_kind() {
1293 Some(crate::ScalarKind::Sint | crate::ScalarKind::Uint) => {
1294 spirv::Op::INotEqual
1295 }
1296 Some(crate::ScalarKind::Float) => spirv::Op::FOrdNotEqual,
1297 Some(crate::ScalarKind::Bool) => spirv::Op::LogicalNotEqual,
1298 _ => unimplemented!(),
1299 },
1300 crate::BinaryOperator::Less => match left_ty_inner.scalar_kind() {
1301 Some(crate::ScalarKind::Sint) => spirv::Op::SLessThan,
1302 Some(crate::ScalarKind::Uint) => spirv::Op::ULessThan,
1303 Some(crate::ScalarKind::Float) => spirv::Op::FOrdLessThan,
1304 _ => unimplemented!(),
1305 },
1306 crate::BinaryOperator::LessEqual => match left_ty_inner.scalar_kind() {
1307 Some(crate::ScalarKind::Sint) => spirv::Op::SLessThanEqual,
1308 Some(crate::ScalarKind::Uint) => spirv::Op::ULessThanEqual,
1309 Some(crate::ScalarKind::Float) => spirv::Op::FOrdLessThanEqual,
1310 _ => unimplemented!(),
1311 },
1312 crate::BinaryOperator::Greater => match left_ty_inner.scalar_kind() {
1313 Some(crate::ScalarKind::Sint) => spirv::Op::SGreaterThan,
1314 Some(crate::ScalarKind::Uint) => spirv::Op::UGreaterThan,
1315 Some(crate::ScalarKind::Float) => spirv::Op::FOrdGreaterThan,
1316 _ => unimplemented!(),
1317 },
1318 crate::BinaryOperator::GreaterEqual => match left_ty_inner.scalar_kind() {
1319 Some(crate::ScalarKind::Sint) => spirv::Op::SGreaterThanEqual,
1320 Some(crate::ScalarKind::Uint) => spirv::Op::UGreaterThanEqual,
1321 Some(crate::ScalarKind::Float) => spirv::Op::FOrdGreaterThanEqual,
1322 _ => unimplemented!(),
1323 },
1324 crate::BinaryOperator::And => match left_ty_inner.scalar_kind() {
1325 Some(crate::ScalarKind::Bool) => spirv::Op::LogicalAnd,
1326 _ => spirv::Op::BitwiseAnd,
1327 },
1328 crate::BinaryOperator::ExclusiveOr => spirv::Op::BitwiseXor,
1329 crate::BinaryOperator::InclusiveOr => match left_ty_inner.scalar_kind() {
1330 Some(crate::ScalarKind::Bool) => spirv::Op::LogicalOr,
1331 _ => spirv::Op::BitwiseOr,
1332 },
1333 crate::BinaryOperator::LogicalAnd => spirv::Op::LogicalAnd,
1334 crate::BinaryOperator::LogicalOr => spirv::Op::LogicalOr,
1335 crate::BinaryOperator::ShiftLeft => spirv::Op::ShiftLeftLogical,
1336 crate::BinaryOperator::ShiftRight => match left_ty_inner.scalar_kind() {
1337 Some(crate::ScalarKind::Sint) => spirv::Op::ShiftRightArithmetic,
1338 Some(crate::ScalarKind::Uint) => spirv::Op::ShiftRightLogical,
1339 _ => unimplemented!(),
1340 },
1341 };
1342
1343 block.body.push(Instruction::binary(
1344 spirv_op,
1345 result_type_id,
1346 id,
1347 if reverse_operands { right_id } else { left_id },
1348 if reverse_operands { left_id } else { right_id },
1349 ));
1350 }
1351 id
1352 }
1353 crate::Expression::Math {
1354 fun,
1355 arg,
1356 arg1,
1357 arg2,
1358 arg3,
1359 } => {
1360 use crate::MathFunction as Mf;
1361 enum MathOp {
1362 Ext(spirv::GlslStd450Op),
1363 Custom(Instruction),
1364 }
1365
1366 let arg0_id = self.cached[arg];
1367 let arg_ty = self.fun_info[arg].ty.inner_with(&self.ir_module.types);
1368 let arg_scalar_kind = arg_ty.scalar_kind();
1369 let arg1_id = match arg1 {
1370 Some(handle) => self.cached[handle],
1371 None => 0,
1372 };
1373 let arg2_id = match arg2 {
1374 Some(handle) => self.cached[handle],
1375 None => 0,
1376 };
1377 let arg3_id = match arg3 {
1378 Some(handle) => self.cached[handle],
1379 None => 0,
1380 };
1381
1382 let id = self.gen_id();
1383 let math_op = match fun {
1384 Mf::Abs => {
1386 match arg_scalar_kind {
1387 Some(crate::ScalarKind::Float) => {
1388 MathOp::Ext(spirv::GlslStd450Op::FAbs)
1389 }
1390 Some(crate::ScalarKind::Sint) => MathOp::Ext(spirv::GlslStd450Op::SAbs),
1391 Some(crate::ScalarKind::Uint) => {
1392 MathOp::Custom(Instruction::unary(
1393 spirv::Op::CopyObject, result_type_id,
1395 id,
1396 arg0_id,
1397 ))
1398 }
1399 other => unimplemented!("Unexpected abs({:?})", other),
1400 }
1401 }
1402 Mf::Min => MathOp::Ext(match arg_scalar_kind {
1403 Some(crate::ScalarKind::Float) => spirv::GlslStd450Op::FMin,
1404 Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::SMin,
1405 Some(crate::ScalarKind::Uint) => spirv::GlslStd450Op::UMin,
1406 other => unimplemented!("Unexpected min({:?})", other),
1407 }),
1408 Mf::Max => MathOp::Ext(match arg_scalar_kind {
1409 Some(crate::ScalarKind::Float) => spirv::GlslStd450Op::FMax,
1410 Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::SMax,
1411 Some(crate::ScalarKind::Uint) => spirv::GlslStd450Op::UMax,
1412 other => unimplemented!("Unexpected max({:?})", other),
1413 }),
1414 Mf::Clamp => match arg_scalar_kind {
1415 Some(crate::ScalarKind::Float) => MathOp::Ext(spirv::GlslStd450Op::FClamp),
1419 Some(_) => {
1420 let (min_op, max_op) = match arg_scalar_kind {
1421 Some(crate::ScalarKind::Sint) => {
1422 (spirv::GlslStd450Op::SMin, spirv::GlslStd450Op::SMax)
1423 }
1424 Some(crate::ScalarKind::Uint) => {
1425 (spirv::GlslStd450Op::UMin, spirv::GlslStd450Op::UMax)
1426 }
1427 _ => unreachable!(),
1428 };
1429
1430 let max_id = self.gen_id();
1431 block.body.push(Instruction::ext_inst_gl_op(
1432 self.writer.gl450_ext_inst_id,
1433 max_op,
1434 result_type_id,
1435 max_id,
1436 &[arg0_id, arg1_id],
1437 ));
1438
1439 MathOp::Custom(Instruction::ext_inst_gl_op(
1440 self.writer.gl450_ext_inst_id,
1441 min_op,
1442 result_type_id,
1443 id,
1444 &[max_id, arg2_id],
1445 ))
1446 }
1447 other => unimplemented!("Unexpected max({:?})", other),
1448 },
1449 Mf::Saturate => {
1450 let (maybe_size, scalar) = match *arg_ty {
1451 crate::TypeInner::Vector { size, scalar } => (Some(size), scalar),
1452 crate::TypeInner::Scalar(scalar) => (None, scalar),
1453 ref other => unimplemented!("Unexpected saturate({:?})", other),
1454 };
1455 let scalar = crate::Scalar::float(scalar.width);
1456 let mut arg1_id = self.writer.get_constant_scalar_with(0, scalar)?;
1457 let mut arg2_id = self.writer.get_constant_scalar_with(1, scalar)?;
1458
1459 if let Some(size) = maybe_size {
1460 let ty =
1461 LocalType::Numeric(NumericType::Vector { size, scalar }).into();
1462
1463 self.temp_list.clear();
1464 self.temp_list.resize(size as _, arg1_id);
1465
1466 arg1_id = self.writer.get_constant_composite(ty, &self.temp_list);
1467
1468 self.temp_list.fill(arg2_id);
1469
1470 arg2_id = self.writer.get_constant_composite(ty, &self.temp_list);
1471 }
1472
1473 MathOp::Custom(Instruction::ext_inst_gl_op(
1474 self.writer.gl450_ext_inst_id,
1475 spirv::GlslStd450Op::FClamp,
1476 result_type_id,
1477 id,
1478 &[arg0_id, arg1_id, arg2_id],
1479 ))
1480 }
1481 Mf::Sin => MathOp::Ext(spirv::GlslStd450Op::Sin),
1483 Mf::Sinh => MathOp::Ext(spirv::GlslStd450Op::Sinh),
1484 Mf::Asin => MathOp::Ext(spirv::GlslStd450Op::Asin),
1485 Mf::Cos => MathOp::Ext(spirv::GlslStd450Op::Cos),
1486 Mf::Cosh => MathOp::Ext(spirv::GlslStd450Op::Cosh),
1487 Mf::Acos => MathOp::Ext(spirv::GlslStd450Op::Acos),
1488 Mf::Tan => MathOp::Ext(spirv::GlslStd450Op::Tan),
1489 Mf::Tanh => MathOp::Ext(spirv::GlslStd450Op::Tanh),
1490 Mf::Atan => MathOp::Ext(spirv::GlslStd450Op::Atan),
1491 Mf::Atan2 => MathOp::Ext(spirv::GlslStd450Op::Atan2),
1492 Mf::Asinh => MathOp::Ext(spirv::GlslStd450Op::Asinh),
1493 Mf::Acosh => MathOp::Ext(spirv::GlslStd450Op::Acosh),
1494 Mf::Atanh => MathOp::Ext(spirv::GlslStd450Op::Atanh),
1495 Mf::Radians => MathOp::Ext(spirv::GlslStd450Op::Radians),
1496 Mf::Degrees => MathOp::Ext(spirv::GlslStd450Op::Degrees),
1497 Mf::Ceil => MathOp::Ext(spirv::GlslStd450Op::Ceil),
1499 Mf::Round => MathOp::Ext(spirv::GlslStd450Op::RoundEven),
1500 Mf::Floor => MathOp::Ext(spirv::GlslStd450Op::Floor),
1501 Mf::Fract => MathOp::Ext(spirv::GlslStd450Op::Fract),
1502 Mf::Trunc => MathOp::Ext(spirv::GlslStd450Op::Trunc),
1503 Mf::Modf => MathOp::Ext(spirv::GlslStd450Op::ModfStruct),
1504 Mf::Frexp => MathOp::Ext(spirv::GlslStd450Op::FrexpStruct),
1505 Mf::Ldexp => MathOp::Ext(spirv::GlslStd450Op::Ldexp),
1506 Mf::Dot => match *self.fun_info[arg].ty.inner_with(&self.ir_module.types) {
1508 crate::TypeInner::Vector {
1509 scalar:
1510 crate::Scalar {
1511 kind: crate::ScalarKind::Float,
1512 ..
1513 },
1514 ..
1515 } => MathOp::Custom(Instruction::binary(
1516 spirv::Op::Dot,
1517 result_type_id,
1518 id,
1519 arg0_id,
1520 arg1_id,
1521 )),
1522 crate::TypeInner::Vector { size, .. } => {
1524 self.write_dot_product(
1525 id,
1526 result_type_id,
1527 arg0_id,
1528 arg1_id,
1529 size as u32,
1530 block,
1531 |result_id, composite_id, index| {
1532 Instruction::composite_extract(
1533 result_type_id,
1534 result_id,
1535 composite_id,
1536 &[index],
1537 )
1538 },
1539 );
1540 self.cached[expr_handle] = id;
1541 return Ok(());
1542 }
1543 _ => unreachable!(
1544 "Correct TypeInner for dot product should be already validated"
1545 ),
1546 },
1547 fun @ (Mf::Dot4I8Packed | Mf::Dot4U8Packed) => {
1548 if self
1549 .writer
1550 .require_all(&[
1551 spirv::Capability::DotProduct,
1552 spirv::Capability::DotProductInput4x8BitPacked,
1553 ])
1554 .is_ok()
1555 {
1556 if self.writer.lang_version() < (1, 6) {
1558 self.writer.use_extension("SPV_KHR_integer_dot_product");
1562 }
1563
1564 let op = match fun {
1565 Mf::Dot4I8Packed => spirv::Op::SDot,
1566 Mf::Dot4U8Packed => spirv::Op::UDot,
1567 _ => unreachable!(),
1568 };
1569
1570 block.body.push(Instruction::ternary(
1571 op,
1572 result_type_id,
1573 id,
1574 arg0_id,
1575 arg1_id,
1576 spirv::PackedVectorFormat::PackedVectorFormat4x8Bit as Word,
1577 ));
1578 } else {
1579 let (extract_op, arg0_id, arg1_id) = match fun {
1581 Mf::Dot4U8Packed => (spirv::Op::BitFieldUExtract, arg0_id, arg1_id),
1582 Mf::Dot4I8Packed => {
1583 let new_arg0_id = self.gen_id();
1586 block.body.push(Instruction::unary(
1587 spirv::Op::Bitcast,
1588 result_type_id,
1589 new_arg0_id,
1590 arg0_id,
1591 ));
1592
1593 let new_arg1_id = self.gen_id();
1594 block.body.push(Instruction::unary(
1595 spirv::Op::Bitcast,
1596 result_type_id,
1597 new_arg1_id,
1598 arg1_id,
1599 ));
1600
1601 (spirv::Op::BitFieldSExtract, new_arg0_id, new_arg1_id)
1602 }
1603 _ => unreachable!(),
1604 };
1605
1606 let eight = self.writer.get_constant_scalar(crate::Literal::U32(8));
1607
1608 const VEC_LENGTH: u8 = 4;
1609 let bit_shifts: [_; VEC_LENGTH as usize] =
1610 core::array::from_fn(|index| {
1611 self.writer
1612 .get_constant_scalar(crate::Literal::U32(index as u32 * 8))
1613 });
1614
1615 self.write_dot_product(
1616 id,
1617 result_type_id,
1618 arg0_id,
1619 arg1_id,
1620 VEC_LENGTH as Word,
1621 block,
1622 |result_id, composite_id, index| {
1623 Instruction::ternary(
1624 extract_op,
1625 result_type_id,
1626 result_id,
1627 composite_id,
1628 bit_shifts[index as usize],
1629 eight,
1630 )
1631 },
1632 );
1633 }
1634
1635 self.cached[expr_handle] = id;
1636 return Ok(());
1637 }
1638 Mf::Outer => MathOp::Custom(Instruction::binary(
1639 spirv::Op::OuterProduct,
1640 result_type_id,
1641 id,
1642 arg0_id,
1643 arg1_id,
1644 )),
1645 Mf::Cross => MathOp::Ext(spirv::GlslStd450Op::Cross),
1646 Mf::Distance => MathOp::Ext(spirv::GlslStd450Op::Distance),
1647 Mf::Length => MathOp::Ext(spirv::GlslStd450Op::Length),
1648 Mf::Normalize => MathOp::Ext(spirv::GlslStd450Op::Normalize),
1649 Mf::FaceForward => MathOp::Ext(spirv::GlslStd450Op::FaceForward),
1650 Mf::Reflect => MathOp::Ext(spirv::GlslStd450Op::Reflect),
1651 Mf::Refract => MathOp::Ext(spirv::GlslStd450Op::Refract),
1652 Mf::Exp => MathOp::Ext(spirv::GlslStd450Op::Exp),
1654 Mf::Exp2 => MathOp::Ext(spirv::GlslStd450Op::Exp2),
1655 Mf::Log => MathOp::Ext(spirv::GlslStd450Op::Log),
1656 Mf::Log2 => MathOp::Ext(spirv::GlslStd450Op::Log2),
1657 Mf::Pow => MathOp::Ext(spirv::GlslStd450Op::Pow),
1658 Mf::Sign => MathOp::Ext(match arg_scalar_kind {
1660 Some(crate::ScalarKind::Float) => spirv::GlslStd450Op::FSign,
1661 Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::SSign,
1662 other => unimplemented!("Unexpected sign({:?})", other),
1663 }),
1664 Mf::Fma => MathOp::Ext(spirv::GlslStd450Op::Fma),
1665 Mf::Mix => {
1666 let selector = arg2.unwrap();
1667 let selector_ty =
1668 self.fun_info[selector].ty.inner_with(&self.ir_module.types);
1669 match (arg_ty, selector_ty) {
1670 (
1672 &crate::TypeInner::Vector { size, .. },
1673 &crate::TypeInner::Scalar(scalar),
1674 ) => {
1675 let selector_type_id =
1676 self.get_numeric_type_id(NumericType::Vector { size, scalar });
1677 self.temp_list.clear();
1678 self.temp_list.resize(size as usize, arg2_id);
1679
1680 let selector_id = self.gen_id();
1681 block.body.push(Instruction::composite_construct(
1682 selector_type_id,
1683 selector_id,
1684 &self.temp_list,
1685 ));
1686
1687 MathOp::Custom(Instruction::ext_inst_gl_op(
1688 self.writer.gl450_ext_inst_id,
1689 spirv::GlslStd450Op::FMix,
1690 result_type_id,
1691 id,
1692 &[arg0_id, arg1_id, selector_id],
1693 ))
1694 }
1695 _ => MathOp::Ext(spirv::GlslStd450Op::FMix),
1696 }
1697 }
1698 Mf::Step => MathOp::Ext(spirv::GlslStd450Op::Step),
1699 Mf::SmoothStep => MathOp::Ext(spirv::GlslStd450Op::SmoothStep),
1700 Mf::Sqrt => MathOp::Ext(spirv::GlslStd450Op::Sqrt),
1701 Mf::InverseSqrt => MathOp::Ext(spirv::GlslStd450Op::InverseSqrt),
1702 Mf::Inverse => MathOp::Ext(spirv::GlslStd450Op::MatrixInverse),
1703 Mf::Transpose => MathOp::Custom(Instruction::unary(
1704 spirv::Op::Transpose,
1705 result_type_id,
1706 id,
1707 arg0_id,
1708 )),
1709 Mf::Determinant => MathOp::Ext(spirv::GlslStd450Op::Determinant),
1710 Mf::QuantizeToF16 => MathOp::Custom(Instruction::unary(
1711 spirv::Op::QuantizeToF16,
1712 result_type_id,
1713 id,
1714 arg0_id,
1715 )),
1716 Mf::ReverseBits => MathOp::Custom(Instruction::unary(
1717 spirv::Op::BitReverse,
1718 result_type_id,
1719 id,
1720 arg0_id,
1721 )),
1722 Mf::CountTrailingZeros => {
1723 let uint_id = match *arg_ty {
1724 crate::TypeInner::Vector { size, scalar } => {
1725 let ty =
1726 LocalType::Numeric(NumericType::Vector { size, scalar }).into();
1727
1728 self.temp_list.clear();
1729 self.temp_list.resize(
1730 size as _,
1731 self.writer
1732 .get_constant_scalar_with(scalar.width * 8, scalar)?,
1733 );
1734
1735 self.writer.get_constant_composite(ty, &self.temp_list)
1736 }
1737 crate::TypeInner::Scalar(scalar) => self
1738 .writer
1739 .get_constant_scalar_with(scalar.width * 8, scalar)?,
1740 _ => unreachable!(),
1741 };
1742
1743 let lsb_id = self.gen_id();
1744 block.body.push(Instruction::ext_inst_gl_op(
1745 self.writer.gl450_ext_inst_id,
1746 spirv::GlslStd450Op::FindILsb,
1747 result_type_id,
1748 lsb_id,
1749 &[arg0_id],
1750 ));
1751
1752 MathOp::Custom(Instruction::ext_inst_gl_op(
1753 self.writer.gl450_ext_inst_id,
1754 spirv::GlslStd450Op::UMin,
1755 result_type_id,
1756 id,
1757 &[uint_id, lsb_id],
1758 ))
1759 }
1760 Mf::CountLeadingZeros => {
1761 let (int_type_id, int_id, width) = match *arg_ty {
1762 crate::TypeInner::Vector { size, scalar } => {
1763 let ty =
1764 LocalType::Numeric(NumericType::Vector { size, scalar }).into();
1765
1766 self.temp_list.clear();
1767 self.temp_list.resize(
1768 size as _,
1769 self.writer
1770 .get_constant_scalar_with(scalar.width * 8 - 1, scalar)?,
1771 );
1772
1773 (
1774 self.get_type_id(ty),
1775 self.writer.get_constant_composite(ty, &self.temp_list),
1776 scalar.width,
1777 )
1778 }
1779 crate::TypeInner::Scalar(scalar) => (
1780 self.get_numeric_type_id(NumericType::Scalar(scalar)),
1781 self.writer
1782 .get_constant_scalar_with(scalar.width * 8 - 1, scalar)?,
1783 scalar.width,
1784 ),
1785 _ => unreachable!(),
1786 };
1787
1788 if width != 4 {
1789 unreachable!("This is validated out until a polyfill is implemented. https://github.com/gfx-rs/wgpu/issues/5276");
1790 };
1791
1792 let msb_id = self.gen_id();
1793 block.body.push(Instruction::ext_inst_gl_op(
1794 self.writer.gl450_ext_inst_id,
1795 if width != 4 {
1796 spirv::GlslStd450Op::FindILsb
1797 } else {
1798 spirv::GlslStd450Op::FindUMsb
1799 },
1800 int_type_id,
1801 msb_id,
1802 &[arg0_id],
1803 ));
1804
1805 MathOp::Custom(Instruction::binary(
1806 spirv::Op::ISub,
1807 result_type_id,
1808 id,
1809 int_id,
1810 msb_id,
1811 ))
1812 }
1813 Mf::CountOneBits => MathOp::Custom(Instruction::unary(
1814 spirv::Op::BitCount,
1815 result_type_id,
1816 id,
1817 arg0_id,
1818 )),
1819 Mf::ExtractBits => {
1820 let op = match arg_scalar_kind {
1821 Some(crate::ScalarKind::Uint) => spirv::Op::BitFieldUExtract,
1822 Some(crate::ScalarKind::Sint) => spirv::Op::BitFieldSExtract,
1823 other => unimplemented!("Unexpected sign({:?})", other),
1824 };
1825
1826 let bit_width = arg_ty.scalar_width().unwrap() * 8;
1841 let width_constant = self
1842 .writer
1843 .get_constant_scalar(crate::Literal::U32(bit_width as u32));
1844
1845 let u32_type =
1846 self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32));
1847
1848 let offset_id = self.gen_id();
1850 block.body.push(Instruction::ext_inst_gl_op(
1851 self.writer.gl450_ext_inst_id,
1852 spirv::GlslStd450Op::UMin,
1853 u32_type,
1854 offset_id,
1855 &[arg1_id, width_constant],
1856 ));
1857
1858 let max_count_id = self.gen_id();
1860 block.body.push(Instruction::binary(
1861 spirv::Op::ISub,
1862 u32_type,
1863 max_count_id,
1864 width_constant,
1865 offset_id,
1866 ));
1867
1868 let count_id = self.gen_id();
1870 block.body.push(Instruction::ext_inst_gl_op(
1871 self.writer.gl450_ext_inst_id,
1872 spirv::GlslStd450Op::UMin,
1873 u32_type,
1874 count_id,
1875 &[arg2_id, max_count_id],
1876 ));
1877
1878 MathOp::Custom(Instruction::ternary(
1879 op,
1880 result_type_id,
1881 id,
1882 arg0_id,
1883 offset_id,
1884 count_id,
1885 ))
1886 }
1887 Mf::InsertBits => {
1888 let bit_width = arg_ty.scalar_width().unwrap() * 8;
1891 let width_constant = self
1892 .writer
1893 .get_constant_scalar(crate::Literal::U32(bit_width as u32));
1894
1895 let u32_type =
1896 self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32));
1897
1898 let offset_id = self.gen_id();
1900 block.body.push(Instruction::ext_inst_gl_op(
1901 self.writer.gl450_ext_inst_id,
1902 spirv::GlslStd450Op::UMin,
1903 u32_type,
1904 offset_id,
1905 &[arg2_id, width_constant],
1906 ));
1907
1908 let max_count_id = self.gen_id();
1910 block.body.push(Instruction::binary(
1911 spirv::Op::ISub,
1912 u32_type,
1913 max_count_id,
1914 width_constant,
1915 offset_id,
1916 ));
1917
1918 let count_id = self.gen_id();
1920 block.body.push(Instruction::ext_inst_gl_op(
1921 self.writer.gl450_ext_inst_id,
1922 spirv::GlslStd450Op::UMin,
1923 u32_type,
1924 count_id,
1925 &[arg3_id, max_count_id],
1926 ));
1927
1928 MathOp::Custom(Instruction::quaternary(
1929 spirv::Op::BitFieldInsert,
1930 result_type_id,
1931 id,
1932 arg0_id,
1933 arg1_id,
1934 offset_id,
1935 count_id,
1936 ))
1937 }
1938 Mf::FirstTrailingBit => MathOp::Ext(spirv::GlslStd450Op::FindILsb),
1939 Mf::FirstLeadingBit => {
1940 if arg_ty.scalar_width() == Some(4) {
1941 let thing = match arg_scalar_kind {
1942 Some(crate::ScalarKind::Uint) => spirv::GlslStd450Op::FindUMsb,
1943 Some(crate::ScalarKind::Sint) => spirv::GlslStd450Op::FindSMsb,
1944 other => unimplemented!("Unexpected firstLeadingBit({:?})", other),
1945 };
1946 MathOp::Ext(thing)
1947 } else {
1948 unreachable!("This is validated out until a polyfill is implemented. https://github.com/gfx-rs/wgpu/issues/5276");
1949 }
1950 }
1951 Mf::Pack4x8unorm => MathOp::Ext(spirv::GlslStd450Op::PackUnorm4x8),
1952 Mf::Pack4x8snorm => MathOp::Ext(spirv::GlslStd450Op::PackSnorm4x8),
1953 Mf::Pack2x16float => MathOp::Ext(spirv::GlslStd450Op::PackHalf2x16),
1954 Mf::Pack2x16unorm => MathOp::Ext(spirv::GlslStd450Op::PackUnorm2x16),
1955 Mf::Pack2x16snorm => MathOp::Ext(spirv::GlslStd450Op::PackSnorm2x16),
1956 fun @ (Mf::Pack4xI8 | Mf::Pack4xU8 | Mf::Pack4xI8Clamp | Mf::Pack4xU8Clamp) => {
1957 let is_signed = matches!(fun, Mf::Pack4xI8 | Mf::Pack4xI8Clamp);
1958 let should_clamp = matches!(fun, Mf::Pack4xI8Clamp | Mf::Pack4xU8Clamp);
1959
1960 let last_instruction =
1961 if self.writer.require_all(&[spirv::Capability::Int8]).is_ok() {
1962 self.write_pack4x8_optimized(
1963 block,
1964 result_type_id,
1965 arg0_id,
1966 id,
1967 is_signed,
1968 should_clamp,
1969 )
1970 } else {
1971 self.write_pack4x8_polyfill(
1972 block,
1973 result_type_id,
1974 arg0_id,
1975 id,
1976 is_signed,
1977 should_clamp,
1978 )
1979 };
1980
1981 MathOp::Custom(last_instruction)
1982 }
1983 Mf::Unpack4x8unorm => MathOp::Ext(spirv::GlslStd450Op::UnpackUnorm4x8),
1984 Mf::Unpack4x8snorm => MathOp::Ext(spirv::GlslStd450Op::UnpackSnorm4x8),
1985 Mf::Unpack2x16float => MathOp::Ext(spirv::GlslStd450Op::UnpackHalf2x16),
1986 Mf::Unpack2x16unorm => MathOp::Ext(spirv::GlslStd450Op::UnpackUnorm2x16),
1987 Mf::Unpack2x16snorm => MathOp::Ext(spirv::GlslStd450Op::UnpackSnorm2x16),
1988 fun @ (Mf::Unpack4xI8 | Mf::Unpack4xU8) => {
1989 let is_signed = matches!(fun, Mf::Unpack4xI8);
1990
1991 let last_instruction =
1992 if self.writer.require_all(&[spirv::Capability::Int8]).is_ok() {
1993 self.write_unpack4x8_optimized(
1994 block,
1995 result_type_id,
1996 arg0_id,
1997 id,
1998 is_signed,
1999 )
2000 } else {
2001 self.write_unpack4x8_polyfill(
2002 block,
2003 result_type_id,
2004 arg0_id,
2005 id,
2006 is_signed,
2007 )
2008 };
2009
2010 MathOp::Custom(last_instruction)
2011 }
2012 };
2013
2014 block.body.push(match math_op {
2015 MathOp::Ext(op) => Instruction::ext_inst_gl_op(
2016 self.writer.gl450_ext_inst_id,
2017 op,
2018 result_type_id,
2019 id,
2020 &[arg0_id, arg1_id, arg2_id, arg3_id][..fun.argument_count()],
2021 ),
2022 MathOp::Custom(inst) => inst,
2023 });
2024 id
2025 }
2026 crate::Expression::LocalVariable(variable) => {
2027 if let Some(rq_tracker) = self
2028 .function
2029 .ray_query_initialization_tracker_variables
2030 .get(&variable)
2031 {
2032 self.ray_query_tracker_expr.insert(
2033 expr_handle,
2034 super::RayQueryTrackers {
2035 initialized_tracker: rq_tracker.id,
2036 t_max_tracker: self
2037 .function
2038 .ray_query_t_max_tracker_variables
2039 .get(&variable)
2040 .expect("Both trackers are set at the same time.")
2041 .id,
2042 },
2043 );
2044 }
2045 self.function.variables[&variable].id
2046 }
2047 crate::Expression::Load { pointer } => {
2048 self.write_checked_load(pointer, block, AccessTypeAdjustment::None, result_type_id)?
2049 }
2050 crate::Expression::FunctionArgument(index) => self.function.parameter_id(index),
2051 crate::Expression::CallResult(_)
2052 | crate::Expression::AtomicResult { .. }
2053 | crate::Expression::WorkGroupUniformLoadResult { .. }
2054 | crate::Expression::RayQueryProceedResult
2055 | crate::Expression::SubgroupBallotResult
2056 | crate::Expression::SubgroupOperationResult { .. } => self.cached[expr_handle],
2057 crate::Expression::As { expr, target } => {
2058 self.write_as_expression(expr, target, block, result_type_id)?
2059 }
2060 crate::Expression::Bitcast { expr, ty } => {
2061 self.write_bitcast_expression(expr, ty, block, result_type_id)
2062 }
2063 crate::Expression::ImageLoad {
2064 image,
2065 coordinate,
2066 array_index,
2067 sample,
2068 level,
2069 } => self.write_image_load(
2070 result_type_id,
2071 image,
2072 coordinate,
2073 array_index,
2074 level,
2075 sample,
2076 block,
2077 )?,
2078 crate::Expression::ImageSample {
2079 image,
2080 sampler,
2081 gather,
2082 coordinate,
2083 array_index,
2084 offset,
2085 level,
2086 depth_ref,
2087 clamp_to_edge,
2088 } => self.write_image_sample(
2089 result_type_id,
2090 image,
2091 sampler,
2092 gather,
2093 coordinate,
2094 array_index,
2095 offset,
2096 level,
2097 depth_ref,
2098 clamp_to_edge,
2099 block,
2100 )?,
2101 crate::Expression::Select {
2102 condition,
2103 accept,
2104 reject,
2105 } => {
2106 let id = self.gen_id();
2107 let mut condition_id = self.cached[condition];
2108 let accept_id = self.cached[accept];
2109 let reject_id = self.cached[reject];
2110
2111 let condition_ty = self.fun_info[condition]
2112 .ty
2113 .inner_with(&self.ir_module.types);
2114 let object_ty = self.fun_info[accept].ty.inner_with(&self.ir_module.types);
2115
2116 if let (
2117 &crate::TypeInner::Scalar(
2118 condition_scalar @ crate::Scalar {
2119 kind: crate::ScalarKind::Bool,
2120 ..
2121 },
2122 ),
2123 &crate::TypeInner::Vector { size, .. },
2124 ) = (condition_ty, object_ty)
2125 {
2126 self.temp_list.clear();
2127 self.temp_list.resize(size as usize, condition_id);
2128
2129 let bool_vector_type_id = self.get_numeric_type_id(NumericType::Vector {
2130 size,
2131 scalar: condition_scalar,
2132 });
2133
2134 let id = self.gen_id();
2135 block.body.push(Instruction::composite_construct(
2136 bool_vector_type_id,
2137 id,
2138 &self.temp_list,
2139 ));
2140 condition_id = id
2141 }
2142
2143 let instruction =
2144 Instruction::select(result_type_id, id, condition_id, accept_id, reject_id);
2145 block.body.push(instruction);
2146 id
2147 }
2148 crate::Expression::Derivative { axis, ctrl, expr } => {
2149 use crate::{DerivativeAxis as Axis, DerivativeControl as Ctrl};
2150 match ctrl {
2151 Ctrl::Coarse | Ctrl::Fine => {
2152 self.writer.require_any(
2153 "DerivativeControl",
2154 &[spirv::Capability::DerivativeControl],
2155 )?;
2156 }
2157 Ctrl::None => {}
2158 }
2159 let id = self.gen_id();
2160 let expr_id = self.cached[expr];
2161 let op = match (axis, ctrl) {
2162 (Axis::X, Ctrl::Coarse) => spirv::Op::DPdxCoarse,
2163 (Axis::X, Ctrl::Fine) => spirv::Op::DPdxFine,
2164 (Axis::X, Ctrl::None) => spirv::Op::DPdx,
2165 (Axis::Y, Ctrl::Coarse) => spirv::Op::DPdyCoarse,
2166 (Axis::Y, Ctrl::Fine) => spirv::Op::DPdyFine,
2167 (Axis::Y, Ctrl::None) => spirv::Op::DPdy,
2168 (Axis::Width, Ctrl::Coarse) => spirv::Op::FwidthCoarse,
2169 (Axis::Width, Ctrl::Fine) => spirv::Op::FwidthFine,
2170 (Axis::Width, Ctrl::None) => spirv::Op::Fwidth,
2171 };
2172 block
2173 .body
2174 .push(Instruction::derivative(op, result_type_id, id, expr_id));
2175 id
2176 }
2177 crate::Expression::ImageQuery { image, query } => {
2178 self.write_image_query(result_type_id, image, query, block)?
2179 }
2180 crate::Expression::Relational { fun, argument } => {
2181 use crate::RelationalFunction as Rf;
2182 let arg_id = self.cached[argument];
2183 let op = match fun {
2184 Rf::All => spirv::Op::All,
2185 Rf::Any => spirv::Op::Any,
2186 Rf::IsNan => spirv::Op::IsNan,
2187 Rf::IsInf => spirv::Op::IsInf,
2188 };
2189 let id = self.gen_id();
2190 block
2191 .body
2192 .push(Instruction::relational(op, result_type_id, id, arg_id));
2193 id
2194 }
2195 crate::Expression::ArrayLength(expr) => self.write_runtime_array_length(expr, block)?,
2196 crate::Expression::RayQueryGetIntersection { query, committed } => {
2197 let query_id = self.cached[query];
2198 let init_tracker_id = *self
2199 .ray_query_tracker_expr
2200 .get(&query)
2201 .expect("not a cached ray query");
2202 let func_id = self
2203 .writer
2204 .write_ray_query_get_intersection_function(committed, self.ir_module);
2205 let ray_intersection = self.ir_module.special_types.ray_intersection.unwrap();
2206 let intersection_type_id = self.get_handle_type_id(ray_intersection);
2207 let id = self.gen_id();
2208 block.body.push(Instruction::function_call(
2209 intersection_type_id,
2210 id,
2211 func_id,
2212 &[query_id, init_tracker_id.initialized_tracker],
2213 ));
2214 id
2215 }
2216 crate::Expression::HitObjectQuery { hit_object, query } => {
2217 self.write_hit_object_query(hit_object, query, block)?
2218 }
2219 crate::Expression::RayQueryVertexPositions { query, committed } => {
2220 self.writer.require_any(
2221 "RayQueryVertexPositions",
2222 &[spirv::Capability::RayQueryPositionFetchKHR],
2223 )?;
2224 self.write_ray_query_return_vertex_position(query, block, committed)
2225 }
2226 crate::Expression::CooperativeLoad { ref data, .. } => {
2227 self.writer.require_any(
2228 "CooperativeMatrix",
2229 &[spirv::Capability::CooperativeMatrixKHR],
2230 )?;
2231 let layout = if data.row_major {
2232 spirv::CooperativeMatrixLayout::RowMajorKHR
2233 } else {
2234 spirv::CooperativeMatrixLayout::ColumnMajorKHR
2235 };
2236 let layout_id = self.get_index_constant(layout as u32);
2237 let stride_id = self.cached[data.stride];
2238 match self.write_access_chain(data.pointer, block, AccessTypeAdjustment::None)? {
2239 ExpressionPointer::Ready { pointer_id } => {
2240 let id = self.gen_id();
2241 block.body.push(Instruction::coop_load(
2242 result_type_id,
2243 id,
2244 pointer_id,
2245 layout_id,
2246 stride_id,
2247 ));
2248 id
2249 }
2250 ExpressionPointer::Conditional { condition, access } => self
2251 .write_conditional_indexed_load(
2252 result_type_id,
2253 condition,
2254 block,
2255 |id_gen, block| {
2256 let pointer_id = access.result_id.unwrap();
2257 block.body.push(access);
2258 let id = id_gen.next();
2259 block.body.push(Instruction::coop_load(
2260 result_type_id,
2261 id,
2262 pointer_id,
2263 layout_id,
2264 stride_id,
2265 ));
2266 id
2267 },
2268 ),
2269 }
2270 }
2271 crate::Expression::CooperativeMultiplyAdd { a, b, c } => {
2272 self.writer.require_any(
2273 "CooperativeMatrix",
2274 &[spirv::Capability::CooperativeMatrixKHR],
2275 )?;
2276 let a_id = self.cached[a];
2277 let b_id = self.cached[b];
2278 let c_id = self.cached[c];
2279 let id = self.gen_id();
2280 block.body.push(Instruction::coop_mul_add(
2281 result_type_id,
2282 id,
2283 a_id,
2284 b_id,
2285 c_id,
2286 ));
2287 id
2288 }
2289 };
2290
2291 self.cached[expr_handle] = id;
2292 Ok(())
2293 }
2294
2295 fn write_bitcast_expression(
2299 &mut self,
2300 expr: Handle<crate::Expression>,
2301 ty: Handle<crate::Type>,
2302 block: &mut Block,
2303 result_type_id: u32,
2304 ) -> Word {
2305 let expr_id = self.cached[expr];
2306
2307 let source_inner = self.fun_info[expr].ty.inner_with(&self.ir_module.types);
2310 if *source_inner == self.ir_module.types[ty].inner {
2311 return expr_id;
2312 }
2313
2314 let id = self.gen_id();
2315 block.body.push(Instruction::unary(
2316 spirv::Op::Bitcast,
2317 result_type_id,
2318 id,
2319 expr_id,
2320 ));
2321 id
2322 }
2323
2324 fn write_as_expression(
2327 &mut self,
2328 expr: Handle<crate::Expression>,
2329 target: crate::Scalar,
2330 block: &mut Block,
2331 result_type_id: u32,
2332 ) -> Result<u32, Error> {
2333 use crate::ScalarKind as Sk;
2334 let crate::Scalar {
2335 kind,
2336 width: dst_width,
2337 } = target;
2338 let expr_id = self.cached[expr];
2339 let ty = self.fun_info[expr].ty.inner_with(&self.ir_module.types);
2340
2341 if let crate::TypeInner::Matrix {
2346 columns,
2347 rows,
2348 scalar,
2349 } = *ty
2350 {
2351 if dst_width == scalar.width {
2352 return Ok(expr_id);
2354 }
2355
2356 if kind != Sk::Float {
2357 return Err(Error::Validation("Matrices must be floats"));
2359 }
2360
2361 let column_src_ty =
2363 self.get_type_id(LookupType::Local(LocalType::Numeric(NumericType::Vector {
2364 size: rows,
2365 scalar,
2366 })));
2367
2368 let column_dst_ty =
2370 self.get_type_id(LookupType::Local(LocalType::Numeric(NumericType::Vector {
2371 size: rows,
2372 scalar: crate::Scalar {
2373 kind,
2374 width: dst_width,
2375 },
2376 })));
2377
2378 let mut components = ArrayVec::<Word, 4>::new();
2379
2380 for column in 0..columns as usize {
2381 let column_id = self.gen_id();
2382 block.body.push(Instruction::composite_extract(
2383 column_src_ty,
2384 column_id,
2385 expr_id,
2386 &[column as u32],
2387 ));
2388
2389 let column_conv_id = self.gen_id();
2390 block.body.push(Instruction::unary(
2391 spirv::Op::FConvert,
2392 column_dst_ty,
2393 column_conv_id,
2394 column_id,
2395 ));
2396
2397 components.push(column_conv_id);
2398 }
2399
2400 let construct_id = self.gen_id();
2401
2402 block.body.push(Instruction::composite_construct(
2403 result_type_id,
2404 construct_id,
2405 &components,
2406 ));
2407
2408 return Ok(construct_id);
2409 }
2410
2411 let (src_scalar, src_size) = match *ty {
2412 crate::TypeInner::Scalar(scalar) => (scalar, None),
2413 crate::TypeInner::Vector { scalar, size } => (scalar, Some(size)),
2414 ref other => {
2415 log::error!("As source {other:?}");
2416 return Err(Error::Validation("Unexpected Expression::As source"));
2417 }
2418 };
2419
2420 enum Cast {
2421 Identity(Word),
2422 Unary(spirv::Op, Word),
2423 Binary(spirv::Op, Word, Word),
2424 Ternary(spirv::Op, Word, Word, Word),
2425 }
2426 let cast = match (src_scalar.kind, kind, dst_width) {
2427 (src_kind, kind, dst_width) if src_kind == kind && dst_width == src_scalar.width => {
2430 Cast::Identity(expr_id)
2431 }
2432 (Sk::Bool, Sk::Bool, _) => Cast::Unary(spirv::Op::CopyObject, expr_id),
2433 (_, Sk::Bool, _) => {
2435 let op = match src_scalar.kind {
2436 Sk::Sint | Sk::Uint => spirv::Op::INotEqual,
2437 Sk::Float => spirv::Op::FUnordNotEqual,
2438 Sk::Bool | Sk::AbstractInt | Sk::AbstractFloat => unreachable!(),
2439 };
2440 let zero_scalar_id = self.writer.get_constant_scalar_with(0, src_scalar)?;
2441 let zero_id = match src_size {
2442 Some(size) => {
2443 let ty = LocalType::Numeric(NumericType::Vector {
2444 size,
2445 scalar: src_scalar,
2446 })
2447 .into();
2448
2449 self.temp_list.clear();
2450 self.temp_list.resize(size as _, zero_scalar_id);
2451
2452 self.writer.get_constant_composite(ty, &self.temp_list)
2453 }
2454 None => zero_scalar_id,
2455 };
2456
2457 Cast::Binary(op, expr_id, zero_id)
2458 }
2459 (Sk::Bool, _, dst_width) => {
2461 let dst_scalar = crate::Scalar {
2462 kind,
2463 width: dst_width,
2464 };
2465 let zero_scalar_id = self.writer.get_constant_scalar_with(0, dst_scalar)?;
2466 let one_scalar_id = self.writer.get_constant_scalar_with(1, dst_scalar)?;
2467 let (accept_id, reject_id) = match src_size {
2468 Some(size) => {
2469 let ty = LocalType::Numeric(NumericType::Vector {
2470 size,
2471 scalar: dst_scalar,
2472 })
2473 .into();
2474
2475 self.temp_list.clear();
2476 self.temp_list.resize(size as _, zero_scalar_id);
2477
2478 let vec0_id = self.writer.get_constant_composite(ty, &self.temp_list);
2479
2480 self.temp_list.fill(one_scalar_id);
2481
2482 let vec1_id = self.writer.get_constant_composite(ty, &self.temp_list);
2483
2484 (vec1_id, vec0_id)
2485 }
2486 None => (one_scalar_id, zero_scalar_id),
2487 };
2488
2489 Cast::Ternary(spirv::Op::Select, expr_id, accept_id, reject_id)
2490 }
2491 (Sk::Float, Sk::Sint | Sk::Uint, width) => {
2502 let dst_scalar = crate::Scalar { kind, width };
2503 let (min, max) =
2504 crate::proc::min_max_float_representable_by(src_scalar, dst_scalar);
2505 let expr_type_id = self.get_expression_type_id(&self.fun_info[expr].ty);
2506
2507 let maybe_splat_const = |writer: &mut Writer, const_id| match src_size {
2508 None => const_id,
2509 Some(size) => {
2510 let constituent_ids = [const_id; crate::VectorSize::MAX];
2511 writer.get_constant_composite(
2512 LookupType::Local(LocalType::Numeric(NumericType::Vector {
2513 size,
2514 scalar: src_scalar,
2515 })),
2516 &constituent_ids[..size as usize],
2517 )
2518 }
2519 };
2520 let min_const_id = self.writer.get_constant_scalar(min);
2521 let min_const_id = maybe_splat_const(self.writer, min_const_id);
2522 let max_const_id = self.writer.get_constant_scalar(max);
2523 let max_const_id = maybe_splat_const(self.writer, max_const_id);
2524
2525 let clamp_id = self.gen_id();
2526 block.body.push(Instruction::ext_inst_gl_op(
2527 self.writer.gl450_ext_inst_id,
2528 spirv::GlslStd450Op::FClamp,
2529 expr_type_id,
2530 clamp_id,
2531 &[expr_id, min_const_id, max_const_id],
2532 ));
2533
2534 let op = match dst_scalar.kind {
2535 crate::ScalarKind::Sint => spirv::Op::ConvertFToS,
2536 crate::ScalarKind::Uint => spirv::Op::ConvertFToU,
2537 _ => unreachable!(),
2538 };
2539 Cast::Unary(op, clamp_id)
2540 }
2541 (Sk::Float, Sk::Float, dst_width) if src_scalar.width != dst_width => {
2542 Cast::Unary(spirv::Op::FConvert, expr_id)
2543 }
2544 (Sk::Sint, Sk::Float, _) => Cast::Unary(spirv::Op::ConvertSToF, expr_id),
2545 (Sk::Sint, Sk::Sint, dst_width) if src_scalar.width != dst_width => {
2546 Cast::Unary(spirv::Op::SConvert, expr_id)
2547 }
2548 (Sk::Uint, Sk::Float, _) => Cast::Unary(spirv::Op::ConvertUToF, expr_id),
2549 (Sk::Uint, Sk::Uint, dst_width) if src_scalar.width != dst_width => {
2550 Cast::Unary(spirv::Op::UConvert, expr_id)
2551 }
2552 (Sk::Uint, Sk::Sint, dst_width) if src_scalar.width != dst_width => {
2553 Cast::Unary(spirv::Op::SConvert, expr_id)
2554 }
2555 (Sk::Sint, Sk::Uint, dst_width) if src_scalar.width != dst_width => {
2556 Cast::Unary(spirv::Op::UConvert, expr_id)
2557 }
2558 _ => Cast::Unary(spirv::Op::Bitcast, expr_id),
2561 };
2562 Ok(match cast {
2563 Cast::Identity(expr) => expr,
2564 Cast::Unary(op, op1) => {
2565 let id = self.gen_id();
2566 block
2567 .body
2568 .push(Instruction::unary(op, result_type_id, id, op1));
2569 id
2570 }
2571 Cast::Binary(op, op1, op2) => {
2572 let id = self.gen_id();
2573 block
2574 .body
2575 .push(Instruction::binary(op, result_type_id, id, op1, op2));
2576 id
2577 }
2578 Cast::Ternary(op, op1, op2, op3) => {
2579 let id = self.gen_id();
2580 block
2581 .body
2582 .push(Instruction::ternary(op, result_type_id, id, op1, op2, op3));
2583 id
2584 }
2585 })
2586 }
2587
2588 fn write_access_chain(
2599 &mut self,
2600 mut expr_handle: Handle<crate::Expression>,
2601 block: &mut Block,
2602 type_adjustment: AccessTypeAdjustment,
2603 ) -> Result<ExpressionPointer, Error> {
2604 let result_type_id = {
2605 let resolution = &self.fun_info[expr_handle].ty;
2606 match type_adjustment {
2607 AccessTypeAdjustment::None => self.writer.get_expression_type_id(resolution),
2608 AccessTypeAdjustment::IntroducePointer(class) => {
2609 self.writer.get_resolution_pointer_id(resolution, class)
2610 }
2611 AccessTypeAdjustment::UseStd140CompatType => {
2612 match *resolution.inner_with(&self.ir_module.types) {
2613 crate::TypeInner::Pointer {
2614 base,
2615 space: space @ crate::AddressSpace::Uniform,
2616 } => self.writer.get_pointer_type_id(
2617 self.writer.std140_compat_uniform_types[&base].type_id,
2618 map_storage_class(space),
2619 ),
2620 _ => unreachable!(
2621 "`UseStd140CompatType` must only be used with uniform pointer types"
2622 ),
2623 }
2624 }
2625 }
2626 };
2627
2628 let mut accumulated_checks = None;
2632
2633 let mut is_non_uniform_binding_array = false;
2635
2636 let mut prev_decomposed_matrix_index = None;
2642
2643 self.temp_list.clear();
2644 let root_id = loop {
2645 if let Some(spilled) = self.function.spilled_composites.get(&expr_handle) {
2648 break spilled.id;
2651 }
2652
2653 expr_handle = match self.ir_function.expressions[expr_handle] {
2654 crate::Expression::Access { base, index } => {
2655 is_non_uniform_binding_array |=
2656 self.is_nonuniform_binding_array_access(base, index);
2657
2658 let index = GuardedIndex::Expression(index);
2659 let index_id =
2660 self.write_access_chain_index(base, index, &mut accumulated_checks, block)?;
2661 self.temp_list.push(index_id);
2662
2663 base
2664 }
2665 crate::Expression::AccessIndex { base, index } => {
2666 let mut base_ty = self.fun_info[base].ty.inner_with(&self.ir_module.types);
2669 let mut base_ty_handle = self.fun_info[base].ty.handle();
2670 let mut pointer_space = None;
2671 if let crate::TypeInner::Pointer { base, space } = *base_ty {
2672 base_ty = &self.ir_module.types[base].inner;
2673 base_ty_handle = Some(base);
2674 pointer_space = Some(space);
2675 }
2676 match *base_ty {
2677 crate::TypeInner::Struct { .. } => {
2684 let index = match base_ty_handle.and_then(|handle| {
2685 self.writer.std140_compat_uniform_types.get(&handle)
2686 }) {
2687 Some(std140_type_info)
2688 if pointer_space == Some(crate::AddressSpace::Uniform) =>
2689 {
2690 std140_type_info.member_indices[index as usize]
2691 + prev_decomposed_matrix_index.take().unwrap_or(0)
2692 }
2693 _ => index,
2694 };
2695 let index_id = self.get_index_constant(index);
2696 self.temp_list.push(index_id);
2697 }
2698 _ if is_uniform_matcx2_struct_member_access(
2705 self.ir_function,
2706 self.fun_info,
2707 self.ir_module,
2708 base,
2709 ) =>
2710 {
2711 assert!(prev_decomposed_matrix_index.is_none());
2712 prev_decomposed_matrix_index = Some(index);
2713 }
2714 _ => {
2715 let index_id = self.write_access_chain_index(
2722 base,
2723 GuardedIndex::Known(index),
2724 &mut accumulated_checks,
2725 block,
2726 )?;
2727 self.temp_list.push(index_id);
2728 }
2729 }
2730 base
2731 }
2732 crate::Expression::GlobalVariable(handle) => {
2733 let gv = &self.writer.global_variables[handle];
2734 break gv.access_id;
2735 }
2736 crate::Expression::LocalVariable(variable) => {
2737 let local_var = &self.function.variables[&variable];
2738 break local_var.id;
2739 }
2740 crate::Expression::FunctionArgument(index) => {
2741 break self.function.parameter_id(index);
2742 }
2743 ref other => unimplemented!("Unexpected pointer expression {:?}", other),
2744 }
2745 };
2746
2747 let (pointer_id, expr_pointer) = if self.temp_list.is_empty() {
2748 (
2749 root_id,
2750 ExpressionPointer::Ready {
2751 pointer_id: root_id,
2752 },
2753 )
2754 } else {
2755 self.temp_list.reverse();
2756 let pointer_id = self.gen_id();
2757 let access =
2758 Instruction::access_chain(result_type_id, pointer_id, root_id, &self.temp_list);
2759
2760 let expr_pointer = match accumulated_checks {
2765 Some(condition) => ExpressionPointer::Conditional { condition, access },
2766 None => {
2767 block.body.push(access);
2768 ExpressionPointer::Ready { pointer_id }
2769 }
2770 };
2771 (pointer_id, expr_pointer)
2772 };
2773 if is_non_uniform_binding_array {
2777 self.writer
2778 .decorate_non_uniform_binding_array_access(pointer_id)?;
2779 }
2780
2781 Ok(expr_pointer)
2782 }
2783
2784 fn is_nonuniform_binding_array_access(
2785 &mut self,
2786 base: Handle<crate::Expression>,
2787 index: Handle<crate::Expression>,
2788 ) -> bool {
2789 let crate::Expression::GlobalVariable(var_handle) = self.ir_function.expressions[base]
2790 else {
2791 return false;
2792 };
2793
2794 let gvar = &self.ir_module.global_variables[var_handle];
2797 let crate::TypeInner::BindingArray { .. } = self.ir_module.types[gvar.ty].inner else {
2798 return false;
2799 };
2800
2801 self.fun_info[index].uniformity.non_uniform_result.is_some()
2802 }
2803
2804 fn write_access_chain_index(
2814 &mut self,
2815 base: Handle<crate::Expression>,
2816 index: GuardedIndex,
2817 accumulated_checks: &mut Option<Word>,
2818 block: &mut Block,
2819 ) -> Result<Word, Error> {
2820 match self.write_bounds_check(base, index, block)? {
2821 BoundsCheckResult::KnownInBounds(known_index) => {
2822 let scalar = crate::Literal::U32(known_index);
2825 Ok(self.writer.get_constant_scalar(scalar))
2826 }
2827 BoundsCheckResult::Computed(computed_index_id) => Ok(computed_index_id),
2828 BoundsCheckResult::Conditional {
2829 condition_id: condition,
2830 index_id: index,
2831 } => {
2832 self.extend_bounds_check_condition_chain(accumulated_checks, condition, block);
2833
2834 Ok(index)
2836 }
2837 }
2838 }
2839
2840 fn extend_bounds_check_condition_chain(
2859 &mut self,
2860 chain: &mut Option<Word>,
2861 comparison_id: Word,
2862 block: &mut Block,
2863 ) {
2864 match *chain {
2865 Some(ref mut prior_checks) => {
2866 let combined = self.gen_id();
2867 block.body.push(Instruction::binary(
2868 spirv::Op::LogicalAnd,
2869 self.writer.get_bool_type_id(),
2870 combined,
2871 *prior_checks,
2872 comparison_id,
2873 ));
2874 *prior_checks = combined;
2875 }
2876 None => {
2877 *chain = Some(comparison_id);
2879 }
2880 }
2881 }
2882
2883 fn write_checked_load(
2884 &mut self,
2885 pointer: Handle<crate::Expression>,
2886 block: &mut Block,
2887 access_type_adjustment: AccessTypeAdjustment,
2888 result_type_id: Word,
2889 ) -> Result<Word, Error> {
2890 if let Some(result_id) = self.maybe_write_immediate_vector_dynamic_access(pointer, block)? {
2891 Ok(result_id)
2892 } else if let Some(result_id) =
2893 self.maybe_write_uniform_matcx2_dynamic_access(pointer, block)?
2894 {
2895 Ok(result_id)
2896 } else if let Some(result_id) =
2897 self.maybe_write_load_uniform_matcx2_struct_member(pointer, block)?
2898 {
2899 Ok(result_id)
2900 } else {
2901 struct WrappedLoad {
2907 access_type_adjustment: AccessTypeAdjustment,
2908 r#type: Handle<crate::Type>,
2909 }
2910 let mut wrapped_load = None;
2911 if let crate::TypeInner::Pointer {
2912 base: pointer_base_type,
2913 space: crate::AddressSpace::Uniform,
2914 } = *self.fun_info[pointer].ty.inner_with(&self.ir_module.types)
2915 {
2916 if self
2917 .writer
2918 .std140_compat_uniform_types
2919 .contains_key(&pointer_base_type)
2920 {
2921 wrapped_load = Some(WrappedLoad {
2922 access_type_adjustment: AccessTypeAdjustment::UseStd140CompatType,
2923 r#type: pointer_base_type,
2924 });
2925 };
2926 };
2927
2928 let (load_type_id, access_type_adjustment) = match wrapped_load {
2929 Some(ref wrapped_load) => (
2930 self.writer.std140_compat_uniform_types[&wrapped_load.r#type].type_id,
2931 wrapped_load.access_type_adjustment,
2932 ),
2933 None => (result_type_id, access_type_adjustment),
2934 };
2935
2936 let load_id = match self.write_access_chain(pointer, block, access_type_adjustment)? {
2937 ExpressionPointer::Ready { pointer_id } => {
2938 let id = self.gen_id();
2939 let atomic_space =
2940 match *self.fun_info[pointer].ty.inner_with(&self.ir_module.types) {
2941 crate::TypeInner::Pointer { base, space } => {
2942 match self.ir_module.types[base].inner {
2943 crate::TypeInner::Atomic { .. } => Some(space),
2944 _ => None,
2945 }
2946 }
2947 _ => None,
2948 };
2949 let instruction = if let Some(space) = atomic_space {
2950 let (semantics, scope) = space.to_spirv_semantics_and_scope();
2951 let scope_constant_id = self.get_scope_constant(scope as u32);
2952 let semantics_id = self.get_index_constant(semantics.bits());
2953 Instruction::atomic_load(
2954 result_type_id,
2955 id,
2956 pointer_id,
2957 scope_constant_id,
2958 semantics_id,
2959 )
2960 } else {
2961 Instruction::load(load_type_id, id, pointer_id, None)
2962 };
2963 block.body.push(instruction);
2964 id
2965 }
2966 ExpressionPointer::Conditional { condition, access } => {
2967 self.write_conditional_indexed_load(
2969 load_type_id,
2970 condition,
2971 block,
2972 move |id_gen, block| {
2973 let pointer_id = access.result_id.unwrap();
2975 let value_id = id_gen.next();
2976 block.body.push(access);
2977 block.body.push(Instruction::load(
2978 load_type_id,
2979 value_id,
2980 pointer_id,
2981 None,
2982 ));
2983 value_id
2984 },
2985 )
2986 }
2987 };
2988
2989 match wrapped_load {
2990 Some(ref wrapped_load) => {
2991 let result_id = self.gen_id();
2994 let function_id = self.writer.wrapped_functions
2995 [&WrappedFunction::ConvertFromStd140CompatType {
2996 r#type: wrapped_load.r#type,
2997 }];
2998 block.body.push(Instruction::function_call(
2999 result_type_id,
3000 result_id,
3001 function_id,
3002 &[load_id],
3003 ));
3004 Ok(result_id)
3005 }
3006 None => Ok(load_id),
3007 }
3008 }
3009 }
3010
3011 fn spill_to_internal_variable(&mut self, base: Handle<crate::Expression>, block: &mut Block) {
3012 use indexmap::map::Entry;
3013
3014 let spill_variable_id = match self.function.spilled_composites.entry(base) {
3016 Entry::Occupied(preexisting) => preexisting.get().id,
3017 Entry::Vacant(vacant) => {
3018 let pointer_type_id = self.writer.get_resolution_pointer_id(
3021 &self.fun_info[base].ty,
3022 spirv::StorageClass::Function,
3023 );
3024 let id = self.writer.id_gen.next();
3025 vacant.insert(super::LocalVariable {
3026 id,
3027 instruction: Instruction::variable(
3028 pointer_type_id,
3029 id,
3030 spirv::StorageClass::Function,
3031 None,
3032 ),
3033 });
3034 id
3035 }
3036 };
3037
3038 let base_id = self.cached[base];
3063 block
3064 .body
3065 .push(Instruction::store(spill_variable_id, base_id, None));
3066 }
3067
3068 fn maybe_access_spilled_composite(
3085 &mut self,
3086 access: Handle<crate::Expression>,
3087 block: &mut Block,
3088 result_type_id: Word,
3089 ) -> Result<Word, Error> {
3090 let access_uses = self.function.access_uses.get(&access).map_or(0, |r| *r);
3091 if access_uses == self.fun_info[access].ref_count {
3092 Ok(0)
3096 } else {
3097 self.write_checked_load(
3102 access,
3103 block,
3104 AccessTypeAdjustment::IntroducePointer(spirv::StorageClass::Function),
3105 result_type_id,
3106 )
3107 }
3108 }
3109
3110 #[allow(clippy::too_many_arguments)]
3112 fn write_matrix_matrix_column_op(
3113 &mut self,
3114 block: &mut Block,
3115 result_id: Word,
3116 result_type_id: Word,
3117 left_id: Word,
3118 right_id: Word,
3119 columns: crate::VectorSize,
3120 rows: crate::VectorSize,
3121 width: u8,
3122 op: spirv::Op,
3123 ) {
3124 self.temp_list.clear();
3125
3126 let vector_type_id = self.get_numeric_type_id(NumericType::Vector {
3127 size: rows,
3128 scalar: crate::Scalar::float(width),
3129 });
3130
3131 for index in 0..columns as u32 {
3132 let column_id_left = self.gen_id();
3133 let column_id_right = self.gen_id();
3134 let column_id_res = self.gen_id();
3135
3136 block.body.push(Instruction::composite_extract(
3137 vector_type_id,
3138 column_id_left,
3139 left_id,
3140 &[index],
3141 ));
3142 block.body.push(Instruction::composite_extract(
3143 vector_type_id,
3144 column_id_right,
3145 right_id,
3146 &[index],
3147 ));
3148 block.body.push(Instruction::binary(
3149 op,
3150 vector_type_id,
3151 column_id_res,
3152 column_id_left,
3153 column_id_right,
3154 ));
3155
3156 self.temp_list.push(column_id_res);
3157 }
3158
3159 block.body.push(Instruction::composite_construct(
3160 result_type_id,
3161 result_id,
3162 &self.temp_list,
3163 ));
3164 }
3165
3166 fn write_vector_scalar_mult(
3168 &mut self,
3169 block: &mut Block,
3170 result_id: Word,
3171 result_type_id: Word,
3172 vector_id: Word,
3173 scalar_id: Word,
3174 vector: &crate::TypeInner,
3175 ) {
3176 let (size, kind) = match *vector {
3177 crate::TypeInner::Vector {
3178 size,
3179 scalar: crate::Scalar { kind, .. },
3180 } => (size, kind),
3181 _ => unreachable!(),
3182 };
3183
3184 let (op, operand_id) = match kind {
3185 crate::ScalarKind::Float => (spirv::Op::VectorTimesScalar, scalar_id),
3186 _ => {
3187 let operand_id = self.gen_id();
3188 self.temp_list.clear();
3189 self.temp_list.resize(size as usize, scalar_id);
3190 block.body.push(Instruction::composite_construct(
3191 result_type_id,
3192 operand_id,
3193 &self.temp_list,
3194 ));
3195 (spirv::Op::IMul, operand_id)
3196 }
3197 };
3198
3199 block.body.push(Instruction::binary(
3200 op,
3201 result_type_id,
3202 result_id,
3203 vector_id,
3204 operand_id,
3205 ));
3206 }
3207
3208 #[expect(clippy::too_many_arguments)]
3215 fn write_dot_product(
3216 &mut self,
3217 result_id: Word,
3218 result_type_id: Word,
3219 arg0_id: Word,
3220 arg1_id: Word,
3221 size: u32,
3222 block: &mut Block,
3223 extractor: impl Fn(Word, Word, Word) -> Instruction,
3224 ) {
3225 let mut partial_sum = self.writer.get_constant_null(result_type_id);
3226 let last_component = size - 1;
3227 for index in 0..=last_component {
3228 let a_id = self.gen_id();
3230 block.body.push(extractor(a_id, arg0_id, index));
3231 let b_id = self.gen_id();
3232 block.body.push(extractor(b_id, arg1_id, index));
3233 let prod_id = self.gen_id();
3234 block.body.push(Instruction::binary(
3235 spirv::Op::IMul,
3236 result_type_id,
3237 prod_id,
3238 a_id,
3239 b_id,
3240 ));
3241
3242 let id = if index == last_component {
3244 result_id
3245 } else {
3246 self.gen_id()
3247 };
3248
3249 block.body.push(Instruction::binary(
3251 spirv::Op::IAdd,
3252 result_type_id,
3253 id,
3254 partial_sum,
3255 prod_id,
3256 ));
3257 partial_sum = id;
3259 }
3260 }
3261
3262 fn write_pack4x8_optimized(
3264 &mut self,
3265 block: &mut Block,
3266 result_type_id: u32,
3267 arg0_id: u32,
3268 id: u32,
3269 is_signed: bool,
3270 should_clamp: bool,
3271 ) -> Instruction {
3272 let int_type = if is_signed {
3273 crate::ScalarKind::Sint
3274 } else {
3275 crate::ScalarKind::Uint
3276 };
3277 let wide_vector_type = NumericType::Vector {
3278 size: crate::VectorSize::Quad,
3279 scalar: crate::Scalar {
3280 kind: int_type,
3281 width: 4,
3282 },
3283 };
3284 let wide_vector_type_id = self.get_numeric_type_id(wide_vector_type);
3285 let packed_vector_type_id = self.get_numeric_type_id(NumericType::Vector {
3286 size: crate::VectorSize::Quad,
3287 scalar: crate::Scalar {
3288 kind: crate::ScalarKind::Uint,
3289 width: 1,
3290 },
3291 });
3292
3293 let mut wide_vector = arg0_id;
3294 if should_clamp {
3295 let (min, max, clamp_op) = if is_signed {
3296 (
3297 crate::Literal::I32(-128),
3298 crate::Literal::I32(127),
3299 spirv::GlslStd450Op::SClamp,
3300 )
3301 } else {
3302 (
3303 crate::Literal::U32(0),
3304 crate::Literal::U32(255),
3305 spirv::GlslStd450Op::UClamp,
3306 )
3307 };
3308 let [min, max] = [min, max].map(|lit| {
3309 let scalar = self.writer.get_constant_scalar(lit);
3310 self.writer.get_constant_composite(
3311 LookupType::Local(LocalType::Numeric(wide_vector_type)),
3312 &[scalar; 4],
3313 )
3314 });
3315
3316 let clamp_id = self.gen_id();
3317 block.body.push(Instruction::ext_inst_gl_op(
3318 self.writer.gl450_ext_inst_id,
3319 clamp_op,
3320 wide_vector_type_id,
3321 clamp_id,
3322 &[wide_vector, min, max],
3323 ));
3324
3325 wide_vector = clamp_id;
3326 }
3327
3328 let packed_vector = self.gen_id();
3329 block.body.push(Instruction::unary(
3330 spirv::Op::UConvert, packed_vector_type_id,
3332 packed_vector,
3333 wide_vector,
3334 ));
3335
3336 Instruction::unary(spirv::Op::Bitcast, result_type_id, id, packed_vector)
3341 }
3342
3343 fn write_pack4x8_polyfill(
3345 &mut self,
3346 block: &mut Block,
3347 result_type_id: u32,
3348 arg0_id: u32,
3349 id: u32,
3350 is_signed: bool,
3351 should_clamp: bool,
3352 ) -> Instruction {
3353 let int_type = if is_signed {
3354 crate::ScalarKind::Sint
3355 } else {
3356 crate::ScalarKind::Uint
3357 };
3358 let uint_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::U32));
3359 let int_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar {
3360 kind: int_type,
3361 width: 4,
3362 }));
3363
3364 let mut last_instruction = Instruction::new(spirv::Op::Nop);
3365
3366 let zero = self.writer.get_constant_scalar(crate::Literal::U32(0));
3367 let mut preresult = zero;
3368 block
3369 .body
3370 .reserve(usize::from(VEC_LENGTH) * (2 + usize::from(is_signed)));
3371
3372 let eight = self.writer.get_constant_scalar(crate::Literal::U32(8));
3373 const VEC_LENGTH: u8 = 4;
3374 for i in 0..u32::from(VEC_LENGTH) {
3375 let offset = self.writer.get_constant_scalar(crate::Literal::U32(i * 8));
3376 let mut extracted = self.gen_id();
3377 block.body.push(Instruction::binary(
3378 spirv::Op::CompositeExtract,
3379 int_type_id,
3380 extracted,
3381 arg0_id,
3382 i,
3383 ));
3384 if is_signed {
3385 let casted = self.gen_id();
3386 block.body.push(Instruction::unary(
3387 spirv::Op::Bitcast,
3388 uint_type_id,
3389 casted,
3390 extracted,
3391 ));
3392 extracted = casted;
3393 }
3394 if should_clamp {
3395 let (min, max, clamp_op) = if is_signed {
3396 (
3397 crate::Literal::I32(-128),
3398 crate::Literal::I32(127),
3399 spirv::GlslStd450Op::SClamp,
3400 )
3401 } else {
3402 (
3403 crate::Literal::U32(0),
3404 crate::Literal::U32(255),
3405 spirv::GlslStd450Op::UClamp,
3406 )
3407 };
3408 let [min, max] = [min, max].map(|lit| self.writer.get_constant_scalar(lit));
3409
3410 let clamp_id = self.gen_id();
3411 block.body.push(Instruction::ext_inst_gl_op(
3412 self.writer.gl450_ext_inst_id,
3413 clamp_op,
3414 result_type_id,
3415 clamp_id,
3416 &[extracted, min, max],
3417 ));
3418
3419 extracted = clamp_id;
3420 }
3421 let is_last = i == u32::from(VEC_LENGTH - 1);
3422 if is_last {
3423 last_instruction = Instruction::quaternary(
3424 spirv::Op::BitFieldInsert,
3425 result_type_id,
3426 id,
3427 preresult,
3428 extracted,
3429 offset,
3430 eight,
3431 )
3432 } else {
3433 let new_preresult = self.gen_id();
3434 block.body.push(Instruction::quaternary(
3435 spirv::Op::BitFieldInsert,
3436 result_type_id,
3437 new_preresult,
3438 preresult,
3439 extracted,
3440 offset,
3441 eight,
3442 ));
3443 preresult = new_preresult;
3444 }
3445 }
3446 last_instruction
3447 }
3448
3449 fn write_unpack4x8_optimized(
3451 &mut self,
3452 block: &mut Block,
3453 result_type_id: u32,
3454 arg0_id: u32,
3455 id: u32,
3456 is_signed: bool,
3457 ) -> Instruction {
3458 let (int_type, convert_op) = if is_signed {
3459 (crate::ScalarKind::Sint, spirv::Op::SConvert)
3460 } else {
3461 (crate::ScalarKind::Uint, spirv::Op::UConvert)
3462 };
3463
3464 let packed_vector_type_id = self.get_numeric_type_id(NumericType::Vector {
3465 size: crate::VectorSize::Quad,
3466 scalar: crate::Scalar {
3467 kind: int_type,
3468 width: 1,
3469 },
3470 });
3471
3472 let packed_vector = self.gen_id();
3477 block.body.push(Instruction::unary(
3478 spirv::Op::Bitcast,
3479 packed_vector_type_id,
3480 packed_vector,
3481 arg0_id,
3482 ));
3483
3484 Instruction::unary(convert_op, result_type_id, id, packed_vector)
3485 }
3486
3487 fn write_unpack4x8_polyfill(
3489 &mut self,
3490 block: &mut Block,
3491 result_type_id: u32,
3492 arg0_id: u32,
3493 id: u32,
3494 is_signed: bool,
3495 ) -> Instruction {
3496 let (int_type, extract_op) = if is_signed {
3497 (crate::ScalarKind::Sint, spirv::Op::BitFieldSExtract)
3498 } else {
3499 (crate::ScalarKind::Uint, spirv::Op::BitFieldUExtract)
3500 };
3501
3502 let sint_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::I32));
3503
3504 let eight = self.writer.get_constant_scalar(crate::Literal::U32(8));
3505 let int_type_id = self.get_numeric_type_id(NumericType::Scalar(crate::Scalar {
3506 kind: int_type,
3507 width: 4,
3508 }));
3509 block
3510 .body
3511 .reserve(usize::from(VEC_LENGTH) * 2 + usize::from(is_signed));
3512 let arg_id = if is_signed {
3513 let new_arg_id = self.gen_id();
3514 block.body.push(Instruction::unary(
3515 spirv::Op::Bitcast,
3516 sint_type_id,
3517 new_arg_id,
3518 arg0_id,
3519 ));
3520 new_arg_id
3521 } else {
3522 arg0_id
3523 };
3524
3525 const VEC_LENGTH: u8 = 4;
3526 let parts: [_; VEC_LENGTH as usize] = core::array::from_fn(|_| self.gen_id());
3527 for (i, part_id) in parts.into_iter().enumerate() {
3528 let index = self
3529 .writer
3530 .get_constant_scalar(crate::Literal::U32(i as u32 * 8));
3531 block.body.push(Instruction::ternary(
3532 extract_op,
3533 int_type_id,
3534 part_id,
3535 arg_id,
3536 index,
3537 eight,
3538 ));
3539 }
3540
3541 Instruction::composite_construct(result_type_id, id, &parts)
3542 }
3543
3544 fn write_block(
3561 &mut self,
3562 label_id: Word,
3563 naga_block: &crate::Block,
3564 exit: BlockExit,
3565 loop_context: LoopContext,
3566 debug_info: Option<&DebugInfoInner>,
3567 ) -> Result<BlockExitDisposition, Error> {
3568 let mut block = Block::new(label_id);
3569 for (statement, span) in naga_block.span_iter() {
3570 if let (Some(debug_info), false) = (
3571 debug_info,
3572 matches!(
3573 statement,
3574 &(Statement::Block(..)
3575 | Statement::Break
3576 | Statement::Continue
3577 | Statement::Kill
3578 | Statement::Return { .. }
3579 | Statement::Loop { .. })
3580 ),
3581 ) {
3582 let loc: crate::SourceLocation = span.location(debug_info.source_code);
3583 block.body.push(Instruction::line(
3584 debug_info.source_file_id,
3585 loc.line_number,
3586 loc.line_position,
3587 ));
3588 };
3589 match *statement {
3590 Statement::Emit(ref range) => {
3591 for handle in range.clone() {
3592 if !self.expression_constness.is_const(handle) {
3594 self.cache_expression_value(handle, &mut block)?;
3595 }
3596 }
3597 }
3598 Statement::Block(ref block_statements) => {
3599 let scope_id = self.gen_id();
3600 self.function.consume(block, Instruction::branch(scope_id));
3601
3602 let merge_id = self.gen_id();
3603 let merge_used = self.write_block(
3604 scope_id,
3605 block_statements,
3606 BlockExit::Branch { target: merge_id },
3607 loop_context,
3608 debug_info,
3609 )?;
3610
3611 match merge_used {
3612 BlockExitDisposition::Used => {
3613 block = Block::new(merge_id);
3614 }
3615 BlockExitDisposition::Discarded => {
3616 return Ok(BlockExitDisposition::Discarded);
3617 }
3618 }
3619 }
3620 Statement::If {
3621 condition,
3622 ref accept,
3623 ref reject,
3624 } => {
3625 if !(accept.is_empty() && reject.is_empty()) {
3631 let condition_id = self.cached[condition];
3632
3633 let merge_id = self.gen_id();
3634 block.body.push(Instruction::selection_merge(
3635 merge_id,
3636 spirv::SelectionControl::NONE,
3637 ));
3638
3639 let accept_id = if accept.is_empty() {
3640 None
3641 } else {
3642 Some(self.gen_id())
3643 };
3644 let reject_id = if reject.is_empty() {
3645 None
3646 } else {
3647 Some(self.gen_id())
3648 };
3649
3650 self.function.consume(
3651 block,
3652 Instruction::branch_conditional(
3653 condition_id,
3654 accept_id.unwrap_or(merge_id),
3655 reject_id.unwrap_or(merge_id),
3656 ),
3657 );
3658
3659 if let Some(block_id) = accept_id {
3660 let _ = self.write_block(
3665 block_id,
3666 accept,
3667 BlockExit::Branch { target: merge_id },
3668 loop_context,
3669 debug_info,
3670 )?;
3671 }
3672 if let Some(block_id) = reject_id {
3673 let _ = self.write_block(
3678 block_id,
3679 reject,
3680 BlockExit::Branch { target: merge_id },
3681 loop_context,
3682 debug_info,
3683 )?;
3684 }
3685
3686 block = Block::new(merge_id);
3687 }
3688 }
3689 Statement::Switch {
3690 selector,
3691 ref cases,
3692 } => {
3693 let selector_id = self.cached[selector];
3694
3695 let merge_id = self.gen_id();
3696 block.body.push(Instruction::selection_merge(
3697 merge_id,
3698 spirv::SelectionControl::NONE,
3699 ));
3700
3701 let mut default_id = None;
3702 let mut last_id = None;
3704
3705 let mut raw_cases = Vec::with_capacity(cases.len());
3706 let mut case_ids = Vec::with_capacity(cases.len());
3707 for case in cases.iter() {
3708 let label_id = last_id.take().unwrap_or_else(|| self.gen_id());
3710
3711 if case.fall_through && case.body.is_empty() {
3712 last_id = Some(label_id);
3713 }
3714
3715 case_ids.push(label_id);
3716
3717 match case.value {
3718 crate::SwitchValue::I32(value) => {
3719 raw_cases.push(super::instructions::Case {
3720 value: value as Word,
3721 label_id,
3722 });
3723 }
3724 crate::SwitchValue::U32(value) => {
3725 raw_cases.push(super::instructions::Case { value, label_id });
3726 }
3727 crate::SwitchValue::Default => {
3728 default_id = Some(label_id);
3729 }
3730 }
3731 }
3732
3733 let default_id = default_id.unwrap();
3734
3735 self.function.consume(
3736 block,
3737 Instruction::switch(selector_id, default_id, &raw_cases),
3738 );
3739
3740 let inner_context = LoopContext {
3741 break_id: Some(merge_id),
3742 ..loop_context
3743 };
3744
3745 for (i, (case, label_id)) in cases
3746 .iter()
3747 .zip(case_ids.iter())
3748 .filter(|&(case, _)| !(case.fall_through && case.body.is_empty()))
3749 .enumerate()
3750 {
3751 let case_finish_id = if case.fall_through {
3752 case_ids[i + 1]
3753 } else {
3754 merge_id
3755 };
3756 let _ = self.write_block(
3765 *label_id,
3766 &case.body,
3767 BlockExit::Branch {
3768 target: case_finish_id,
3769 },
3770 inner_context,
3771 debug_info,
3772 )?;
3773 }
3774
3775 block = Block::new(merge_id);
3776 }
3777 Statement::Loop {
3778 ref body,
3779 ref continuing,
3780 break_if,
3781 } => {
3782 let preamble_id = self.gen_id();
3783 self.function
3784 .consume(block, Instruction::branch(preamble_id));
3785
3786 let merge_id = self.gen_id();
3787 let body_id = self.gen_id();
3788 let continuing_id = self.gen_id();
3789
3790 block = Block::new(preamble_id);
3793 if let Some(debug_info) = debug_info {
3796 let loc: crate::SourceLocation = span.location(debug_info.source_code);
3797 block.body.push(Instruction::line(
3798 debug_info.source_file_id,
3799 loc.line_number,
3800 loc.line_position,
3801 ))
3802 }
3803 block.body.push(Instruction::loop_merge(
3804 merge_id,
3805 continuing_id,
3806 spirv::SelectionControl::NONE,
3807 ));
3808
3809 if self.force_loop_bounding {
3810 block = self.write_force_bounded_loop_instructions(block, merge_id);
3811 }
3812 self.function.consume(block, Instruction::branch(body_id));
3813
3814 let _ = self.write_block(
3818 body_id,
3819 body,
3820 BlockExit::Branch {
3821 target: continuing_id,
3822 },
3823 LoopContext {
3824 continuing_id: Some(continuing_id),
3825 break_id: Some(merge_id),
3826 },
3827 debug_info,
3828 )?;
3829
3830 let exit = match break_if {
3831 Some(condition) => BlockExit::BreakIf {
3832 condition,
3833 preamble_id,
3834 },
3835 None => BlockExit::Branch {
3836 target: preamble_id,
3837 },
3838 };
3839
3840 let _ = self.write_block(
3844 continuing_id,
3845 continuing,
3846 exit,
3847 LoopContext {
3848 continuing_id: None,
3849 break_id: Some(merge_id),
3850 },
3851 debug_info,
3852 )?;
3853
3854 block = Block::new(merge_id);
3855 }
3856 Statement::Break => {
3857 self.function
3858 .consume(block, Instruction::branch(loop_context.break_id.unwrap()));
3859 return Ok(BlockExitDisposition::Discarded);
3860 }
3861 Statement::Continue => {
3862 self.function.consume(
3863 block,
3864 Instruction::branch(loop_context.continuing_id.unwrap()),
3865 );
3866 return Ok(BlockExitDisposition::Discarded);
3867 }
3868 Statement::Return { value: Some(value) } => {
3869 let value_id = self.cached[value];
3870 let instruction = match self.function.entry_point_context {
3871 Some(ref context) => self.writer.write_entry_point_return(
3874 value_id,
3875 self.ir_function.result.as_ref().unwrap(),
3876 &context.results,
3877 &mut block.body,
3878 )?,
3879 None => Instruction::return_value(value_id),
3880 };
3881 self.function.consume(block, instruction);
3882 return Ok(BlockExitDisposition::Discarded);
3883 }
3884 Statement::Return { value: None } => {
3885 self.function.consume(block, Instruction::return_void());
3886 return Ok(BlockExitDisposition::Discarded);
3887 }
3888 Statement::Kill => {
3889 self.function.consume(block, Instruction::kill());
3890 return Ok(BlockExitDisposition::Discarded);
3891 }
3892 Statement::ControlBarrier(flags) => {
3893 self.writer.write_control_barrier(flags, &mut block.body);
3894 }
3895 Statement::MemoryBarrier(flags) => {
3896 self.writer.write_memory_barrier(flags, &mut block);
3897 }
3898 Statement::Store { pointer, value } => {
3899 let value_id = self.cached[value];
3900 match self.write_access_chain(
3901 pointer,
3902 &mut block,
3903 AccessTypeAdjustment::None,
3904 )? {
3905 ExpressionPointer::Ready { pointer_id } => {
3906 let atomic_space = match *self.fun_info[pointer]
3907 .ty
3908 .inner_with(&self.ir_module.types)
3909 {
3910 crate::TypeInner::Pointer { base, space } => {
3911 match self.ir_module.types[base].inner {
3912 crate::TypeInner::Atomic { .. } => Some(space),
3913 _ => None,
3914 }
3915 }
3916 _ => None,
3917 };
3918 let instruction = if let Some(space) = atomic_space {
3919 let (semantics, scope) = space.to_spirv_semantics_and_scope();
3920 let scope_constant_id = self.get_scope_constant(scope as u32);
3921 let semantics_id = self.get_index_constant(semantics.bits());
3922 Instruction::atomic_store(
3923 pointer_id,
3924 scope_constant_id,
3925 semantics_id,
3926 value_id,
3927 )
3928 } else {
3929 Instruction::store(pointer_id, value_id, None)
3930 };
3931 block.body.push(instruction);
3932 }
3933 ExpressionPointer::Conditional { condition, access } => {
3934 let mut selection = Selection::start(&mut block, ());
3935 selection.if_true(self, condition, ());
3936
3937 let pointer_id = access.result_id.unwrap();
3939 selection.block().body.push(access);
3940 selection
3941 .block()
3942 .body
3943 .push(Instruction::store(pointer_id, value_id, None));
3944
3945 selection.finish(self, ());
3948 }
3949 };
3950 }
3951 Statement::ImageStore {
3952 image,
3953 coordinate,
3954 array_index,
3955 value,
3956 } => self.write_image_store(image, coordinate, array_index, value, &mut block)?,
3957 Statement::Call {
3958 function: local_function,
3959 ref arguments,
3960 result,
3961 } => {
3962 let id = self.gen_id();
3963 self.temp_list.clear();
3964 for &argument in arguments {
3965 self.temp_list.push(self.cached[argument]);
3966 }
3967
3968 let type_id = match result {
3969 Some(expr) => {
3970 self.cached[expr] = id;
3971 self.get_expression_type_id(&self.fun_info[expr].ty)
3972 }
3973 None => self.writer.void_type,
3974 };
3975
3976 block.body.push(Instruction::function_call(
3977 type_id,
3978 id,
3979 self.writer.lookup_function[&local_function],
3980 &self.temp_list,
3981 ));
3982 }
3983 Statement::Atomic {
3984 pointer,
3985 ref fun,
3986 value,
3987 result,
3988 } => {
3989 let id = self.gen_id();
3990 let result_type_id =
3994 self.get_expression_type_id(&self.fun_info[result.unwrap_or(value)].ty);
3995
3996 if let Some(result) = result {
3997 self.cached[result] = id;
3998 }
3999
4000 let pointer_id = match self.write_access_chain(
4001 pointer,
4002 &mut block,
4003 AccessTypeAdjustment::None,
4004 )? {
4005 ExpressionPointer::Ready { pointer_id } => pointer_id,
4006 ExpressionPointer::Conditional { .. } => {
4007 return Err(Error::FeatureNotImplemented(
4008 "Atomics out-of-bounds handling",
4009 ));
4010 }
4011 };
4012
4013 let space = self.fun_info[pointer]
4014 .ty
4015 .inner_with(&self.ir_module.types)
4016 .pointer_space()
4017 .unwrap();
4018 let (semantics, scope) = space.to_spirv_semantics_and_scope();
4019 let scope_constant_id = self.get_scope_constant(scope as u32);
4020 let semantics_id = self.get_index_constant(semantics.bits());
4021 let value_id = self.cached[value];
4022 let value_inner = self.fun_info[value].ty.inner_with(&self.ir_module.types);
4023
4024 let crate::TypeInner::Scalar(scalar) = *value_inner else {
4025 return Err(Error::FeatureNotImplemented(
4026 "Atomics with non-scalar values",
4027 ));
4028 };
4029
4030 let instruction = match *fun {
4031 crate::AtomicFunction::Add => {
4032 let spirv_op = match scalar.kind {
4033 crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4034 spirv::Op::AtomicIAdd
4035 }
4036 crate::ScalarKind::Float => spirv::Op::AtomicFAddEXT,
4037 _ => unimplemented!(),
4038 };
4039 Instruction::atomic_binary(
4040 spirv_op,
4041 result_type_id,
4042 id,
4043 pointer_id,
4044 scope_constant_id,
4045 semantics_id,
4046 value_id,
4047 )
4048 }
4049 crate::AtomicFunction::Subtract => {
4050 let (spirv_op, value_id) = match scalar.kind {
4051 crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4052 (spirv::Op::AtomicISub, value_id)
4053 }
4054 crate::ScalarKind::Float => {
4055 let neg_result_id = self.gen_id();
4058 block.body.push(Instruction::unary(
4059 spirv::Op::FNegate,
4060 result_type_id,
4061 neg_result_id,
4062 value_id,
4063 ));
4064 (spirv::Op::AtomicFAddEXT, neg_result_id)
4065 }
4066 _ => unimplemented!(),
4067 };
4068 Instruction::atomic_binary(
4069 spirv_op,
4070 result_type_id,
4071 id,
4072 pointer_id,
4073 scope_constant_id,
4074 semantics_id,
4075 value_id,
4076 )
4077 }
4078 crate::AtomicFunction::And => {
4079 let spirv_op = match scalar.kind {
4080 crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4081 spirv::Op::AtomicAnd
4082 }
4083 _ => unimplemented!(),
4084 };
4085 Instruction::atomic_binary(
4086 spirv_op,
4087 result_type_id,
4088 id,
4089 pointer_id,
4090 scope_constant_id,
4091 semantics_id,
4092 value_id,
4093 )
4094 }
4095 crate::AtomicFunction::InclusiveOr => {
4096 let spirv_op = match scalar.kind {
4097 crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4098 spirv::Op::AtomicOr
4099 }
4100 _ => unimplemented!(),
4101 };
4102 Instruction::atomic_binary(
4103 spirv_op,
4104 result_type_id,
4105 id,
4106 pointer_id,
4107 scope_constant_id,
4108 semantics_id,
4109 value_id,
4110 )
4111 }
4112 crate::AtomicFunction::ExclusiveOr => {
4113 let spirv_op = match scalar.kind {
4114 crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4115 spirv::Op::AtomicXor
4116 }
4117 _ => unimplemented!(),
4118 };
4119 Instruction::atomic_binary(
4120 spirv_op,
4121 result_type_id,
4122 id,
4123 pointer_id,
4124 scope_constant_id,
4125 semantics_id,
4126 value_id,
4127 )
4128 }
4129 crate::AtomicFunction::Min => {
4130 let spirv_op = match scalar.kind {
4131 crate::ScalarKind::Sint => spirv::Op::AtomicSMin,
4132 crate::ScalarKind::Uint => spirv::Op::AtomicUMin,
4133 _ => unimplemented!(),
4134 };
4135 Instruction::atomic_binary(
4136 spirv_op,
4137 result_type_id,
4138 id,
4139 pointer_id,
4140 scope_constant_id,
4141 semantics_id,
4142 value_id,
4143 )
4144 }
4145 crate::AtomicFunction::Max => {
4146 let spirv_op = match scalar.kind {
4147 crate::ScalarKind::Sint => spirv::Op::AtomicSMax,
4148 crate::ScalarKind::Uint => spirv::Op::AtomicUMax,
4149 _ => unimplemented!(),
4150 };
4151 Instruction::atomic_binary(
4152 spirv_op,
4153 result_type_id,
4154 id,
4155 pointer_id,
4156 scope_constant_id,
4157 semantics_id,
4158 value_id,
4159 )
4160 }
4161 crate::AtomicFunction::Exchange { compare: None } => {
4162 Instruction::atomic_binary(
4163 spirv::Op::AtomicExchange,
4164 result_type_id,
4165 id,
4166 pointer_id,
4167 scope_constant_id,
4168 semantics_id,
4169 value_id,
4170 )
4171 }
4172 crate::AtomicFunction::Exchange { compare: Some(cmp) } => {
4173 let scalar_type_id =
4174 self.get_numeric_type_id(NumericType::Scalar(scalar));
4175 let bool_type_id =
4176 self.get_numeric_type_id(NumericType::Scalar(crate::Scalar::BOOL));
4177
4178 let cas_result_id = self.gen_id();
4179 let equality_result_id = self.gen_id();
4180 let equality_operator = match scalar.kind {
4181 crate::ScalarKind::Sint | crate::ScalarKind::Uint => {
4182 spirv::Op::IEqual
4183 }
4184 _ => unimplemented!(),
4185 };
4186
4187 let mut cas_instr = Instruction::new(spirv::Op::AtomicCompareExchange);
4188 cas_instr.set_type(scalar_type_id);
4189 cas_instr.set_result(cas_result_id);
4190 cas_instr.add_operand(pointer_id);
4191 cas_instr.add_operand(scope_constant_id);
4192 cas_instr.add_operand(semantics_id); cas_instr.add_operand(semantics_id); cas_instr.add_operand(value_id);
4195 cas_instr.add_operand(self.cached[cmp]);
4196 block.body.push(cas_instr);
4197 block.body.push(Instruction::binary(
4198 equality_operator,
4199 bool_type_id,
4200 equality_result_id,
4201 cas_result_id,
4202 self.cached[cmp],
4203 ));
4204 Instruction::composite_construct(
4205 result_type_id,
4206 id,
4207 &[cas_result_id, equality_result_id],
4208 )
4209 }
4210 };
4211
4212 block.body.push(instruction);
4213 }
4214 Statement::ImageAtomic {
4215 image,
4216 coordinate,
4217 array_index,
4218 fun,
4219 value,
4220 } => {
4221 self.write_image_atomic(
4222 image,
4223 coordinate,
4224 array_index,
4225 fun,
4226 value,
4227 &mut block,
4228 )?;
4229 }
4230 Statement::WorkGroupUniformLoad { pointer, result } => {
4231 self.writer
4232 .write_control_barrier(crate::Barrier::WORK_GROUP, &mut block.body);
4233 let result_type_id = self.get_expression_type_id(&self.fun_info[result].ty);
4234 let id = self.write_checked_load(
4237 pointer,
4238 &mut block,
4239 AccessTypeAdjustment::None,
4240 result_type_id,
4241 )?;
4242 self.cached[result] = id;
4243 self.writer
4244 .write_control_barrier(crate::Barrier::WORK_GROUP, &mut block.body);
4245 }
4246 Statement::RayQuery { query, ref fun } => {
4247 self.write_ray_query_function(query, fun, &mut block);
4248 }
4249 Statement::SubgroupBallot {
4250 result,
4251 ref predicate,
4252 } => {
4253 self.write_subgroup_ballot(predicate, result, &mut block)?;
4254 }
4255 Statement::SubgroupCollectiveOperation {
4256 ref op,
4257 ref collective_op,
4258 argument,
4259 result,
4260 } => {
4261 self.write_subgroup_operation(op, collective_op, argument, result, &mut block)?;
4262 }
4263 Statement::SubgroupGather {
4264 ref mode,
4265 argument,
4266 result,
4267 } => {
4268 self.write_subgroup_gather(mode, argument, result, &mut block)?;
4269 }
4270 Statement::SubgroupBallotFindBit {
4271 order,
4272 argument,
4273 result,
4274 } => {
4275 self.write_subgroup_ballot_find_bit(order, argument, result, &mut block)?;
4276 }
4277 Statement::CooperativeStore { target, ref data } => {
4278 let target_id = self.cached[target];
4279 let layout = if data.row_major {
4280 spirv::CooperativeMatrixLayout::RowMajorKHR
4281 } else {
4282 spirv::CooperativeMatrixLayout::ColumnMajorKHR
4283 };
4284 let layout_id = self.get_index_constant(layout as u32);
4285 let stride_id = self.cached[data.stride];
4286 match self.write_access_chain(
4287 data.pointer,
4288 &mut block,
4289 AccessTypeAdjustment::None,
4290 )? {
4291 ExpressionPointer::Ready { pointer_id } => {
4292 block.body.push(Instruction::coop_store(
4293 target_id, pointer_id, layout_id, stride_id,
4294 ));
4295 }
4296 ExpressionPointer::Conditional { condition, access } => {
4297 let mut selection = Selection::start(&mut block, ());
4298 selection.if_true(self, condition, ());
4299
4300 let pointer_id = access.result_id.unwrap();
4302 selection.block().body.push(access);
4303 selection.block().body.push(Instruction::coop_store(
4304 target_id, pointer_id, layout_id, stride_id,
4305 ));
4306
4307 selection.finish(self, ());
4310 }
4311 };
4312 }
4313 Statement::RayPipelineFunction(ref fun) => {
4314 self.write_ray_tracing_pipeline_function(fun, &mut block)?;
4315 }
4316 Statement::HitObject {
4317 hit_object,
4318 ref fun,
4319 } => {
4320 self.write_hit_object_function(hit_object, fun, &mut block)?;
4321 }
4322 Statement::DebugPrintf {
4323 ref format,
4324 ref arguments,
4325 } => {
4326 let mut format_params = Vec::with_capacity(arguments.len());
4327 for &arg in arguments {
4328 let word = self.cached[arg];
4329 format_params.push(word);
4330 }
4331
4332 self.writer
4333 .write_debug_printf(&mut block, format, &format_params);
4334 }
4335 }
4336 }
4337
4338 let termination = match exit {
4339 BlockExit::Return => match self.ir_function.result {
4342 Some(ref result) if self.function.entry_point_context.is_none() => {
4343 let type_id = self.get_handle_type_id(result.ty);
4344 let null_id = self.writer.get_constant_null(type_id);
4345 Instruction::return_value(null_id)
4346 }
4347 _ => Instruction::return_void(),
4348 },
4349 BlockExit::Branch { target } => Instruction::branch(target),
4350 BlockExit::BreakIf {
4351 condition,
4352 preamble_id,
4353 } => {
4354 let condition_id = self.cached[condition];
4355
4356 Instruction::branch_conditional(
4357 condition_id,
4358 loop_context.break_id.unwrap(),
4359 preamble_id,
4360 )
4361 }
4362 };
4363
4364 self.function.consume(block, termination);
4365 Ok(BlockExitDisposition::Used)
4366 }
4367
4368 pub(super) fn write_function_body(
4369 &mut self,
4370 entry_id: Word,
4371 debug_info: Option<&DebugInfoInner>,
4372 ) -> Result<(), Error> {
4373 let _ = self.write_block(
4376 entry_id,
4377 &self.ir_function.body,
4378 BlockExit::Return,
4379 LoopContext::default(),
4380 debug_info,
4381 )?;
4382
4383 Ok(())
4384 }
4385}