Mirrors/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-04-27 18:57:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use layout field of OperandRef in generic_simd_intrinsic

Browse Source
This commit is contained in:
bjorn3
2025-05-30 09:39:11 +00:00
parent 1f717ae778
commit 38a6daeb23
3 changed files with 67 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
-1
View File
@@ -488,7 +488,6 @@ fn codegen_intrinsic_call(
match generic_simd_intrinsic(
self,
name,
callee_ty,
args,
result.layout.ty,
result.layout.gcc_type(self),
compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
+41 -46
View File
@@ -28,7 +28,6 @@
pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
bx: &mut Builder<'a, 'gcc, 'tcx>,
name: Symbol,
callee_ty: Ty<'tcx>,
args: &[OperandRef<'tcx, RValue<'gcc>>],
ret_ty: Ty<'tcx>,
llret_ty: Type<'gcc>,
@@ -54,24 +53,17 @@ macro_rules! require_simd {
};
}
let tcx = bx.tcx();
let sig = tcx.normalize_erasing_late_bound_regions(
ty::TypingEnv::fully_monomorphized(),
callee_ty.fn_sig(tcx),
);
let arg_tys = sig.inputs();
if name == sym::simd_select_bitmask {
require_simd!(
arg_tys[1],
InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
args[1].layout.ty,
InvalidMonomorphization::SimdArgument { span, name, ty: args[1].layout.ty }
);
let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
let (len, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
let expected_int_bits = (len.max(8) - 1).next_power_of_two();
let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
let mask_ty = arg_tys[0];
let mask_ty = args[0].layout.ty;
let mut mask = match *mask_ty.kind() {
ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
@@ -121,8 +113,11 @@ macro_rules! require_simd {
}
// every intrinsic below takes a SIMD vector as its first argument
require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] });
let in_ty = arg_tys[0];
require_simd!(
args[0].layout.ty,
InvalidMonomorphization::SimdInput { span, name, ty: args[0].layout.ty }
);
let in_ty = args[0].layout.ty;
let comparison = match name {
sym::simd_eq => Some(BinOp::Eq),
@@ -134,7 +129,7 @@ macro_rules! require_simd {
_ => None,
};
let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
let (in_len, in_elem) = args[0].layout.ty.simd_size_and_type(bx.tcx());
if let Some(cmp_op) = comparison {
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
@@ -401,13 +396,13 @@ macro_rules! require_simd {
#[cfg(feature = "master")]
if name == sym::simd_insert || name == sym::simd_insert_dyn {
require!(
in_elem == arg_tys[2],
in_elem == args[2].layout.ty,
InvalidMonomorphization::InsertedType {
span,
name,
in_elem,
in_ty,
out_ty: arg_tys[2]
out_ty: args[2].layout.ty
}
);
@@ -439,10 +434,10 @@ macro_rules! require_simd {
let m_elem_ty = in_elem;
let m_len = in_len;
require_simd!(
arg_tys[1],
InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
args[1].layout.ty,
InvalidMonomorphization::SimdArgument { span, name, ty: args[1].layout.ty }
);
let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
let (v_len, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
require!(
m_len == v_len,
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
@@ -911,18 +906,18 @@ fn gather<'a, 'gcc, 'tcx>(
// All types must be simd vector types
require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
require_simd!(
arg_tys[1],
InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
args[1].layout.ty,
InvalidMonomorphization::SimdSecond { span, name, ty: args[1].layout.ty }
);
require_simd!(
arg_tys[2],
InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
args[2].layout.ty,
InvalidMonomorphization::SimdThird { span, name, ty: args[2].layout.ty }
);
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
// Of the same length:
let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
let (out_len, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
let (out_len2, _) = args[2].layout.ty.simd_size_and_type(bx.tcx());
require!(
in_len == out_len,
InvalidMonomorphization::SecondArgumentLength {
@@ -930,7 +925,7 @@ fn gather<'a, 'gcc, 'tcx>(
name,
in_len,
in_ty,
arg_ty: arg_tys[1],
arg_ty: args[1].layout.ty,
out_len
}
);
@@ -941,7 +936,7 @@ fn gather<'a, 'gcc, 'tcx>(
name,
in_len,
in_ty,
arg_ty: arg_tys[2],
arg_ty: args[2].layout.ty,
out_len: out_len2
}
);
@@ -970,8 +965,8 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
// The second argument must be a simd vector with an element type that's a pointer
// to the element type of the first argument
let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
let (_, element_ty0) = args[0].layout.ty.simd_size_and_type(bx.tcx());
let (_, element_ty1) = args[1].layout.ty.simd_size_and_type(bx.tcx());
let (pointer_count, underlying_ty) = match *element_ty1.kind() {
ty::RawPtr(p_ty, _) if p_ty == in_elem => {
(ptr_count(element_ty1), non_ptr(element_ty1))
@@ -983,7 +978,7 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
span,
name,
expected_element: element_ty1,
second_arg: arg_tys[1],
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Not,
@@ -998,7 +993,7 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
// The element type of the third argument must be an integer type of any width:
// TODO: also support unsigned integers.
let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
let (_, element_ty2) = args[2].layout.ty.simd_size_and_type(bx.tcx());
match *element_ty2.kind() {
ty::Int(_) => (),
_ => {
@@ -1030,17 +1025,17 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
// All types must be simd vector types
require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
require_simd!(
arg_tys[1],
InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
args[1].layout.ty,
InvalidMonomorphization::SimdSecond { span, name, ty: args[1].layout.ty }
);
require_simd!(
arg_tys[2],
InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
args[2].layout.ty,
InvalidMonomorphization::SimdThird { span, name, ty: args[2].layout.ty }
);
// Of the same length:
let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
let (element_len1, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
let (element_len2, _) = args[2].layout.ty.simd_size_and_type(bx.tcx());
require!(
in_len == element_len1,
InvalidMonomorphization::SecondArgumentLength {
@@ -1048,7 +1043,7 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
name,
in_len,
in_ty,
arg_ty: arg_tys[1],
arg_ty: args[1].layout.ty,
out_len: element_len1
}
);
@@ -1059,7 +1054,7 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
name,
in_len,
in_ty,
arg_ty: arg_tys[2],
arg_ty: args[2].layout.ty,
out_len: element_len2
}
);
@@ -1082,9 +1077,9 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
// The second argument must be a simd vector with an element type that's a pointer
// to the element type of the first argument
let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
let (_, element_ty0) = args[0].layout.ty.simd_size_and_type(bx.tcx());
let (_, element_ty1) = args[1].layout.ty.simd_size_and_type(bx.tcx());
let (_, element_ty2) = args[2].layout.ty.simd_size_and_type(bx.tcx());
let (pointer_count, underlying_ty) = match *element_ty1.kind() {
ty::RawPtr(p_ty, mutbl) if p_ty == in_elem && mutbl == hir::Mutability::Mut => {
(ptr_count(element_ty1), non_ptr(element_ty1))
@@ -1096,7 +1091,7 @@ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
span,
name,
expected_element: element_ty1,
second_arg: arg_tys[1],
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Mut,
@@ -1194,8 +1189,8 @@ macro_rules! arith_unary {
return_error!(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1,
vector_type: arg_tys[0],
expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
vector_type: args[0].layout.ty,
});
}
};
compiler/rustc_codegen_llvm/src/intrinsic.rs
+26 -32
View File
@@ -593,7 +593,6 @@ fn codegen_intrinsic_call(
match generic_simd_intrinsic(
self,
name,
callee_ty,
fn_args,
&loaded_args,
result.layout.ty,
@@ -1145,7 +1144,6 @@ fn get_rust_try_fn<'a, 'll, 'tcx>(
fn generic_simd_intrinsic<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
name: Symbol,
callee_ty: Ty<'tcx>,
fn_args: GenericArgsRef<'tcx>,
args: &[OperandRef<'tcx, &'ll Value>],
ret_ty: Ty<'tcx>,
@@ -1216,26 +1214,22 @@ fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
}
let tcx = bx.tcx();
let sig = tcx.normalize_erasing_late_bound_regions(bx.typing_env(), callee_ty.fn_sig(tcx));
let arg_tys = sig.inputs();
// Sanity-check: all vector arguments must be immediates.
if cfg!(debug_assertions) {
for (ty, arg) in arg_tys.iter().zip(args) {
if ty.is_simd() {
for arg in args {
if arg.layout.ty.is_simd() {
assert_matches!(arg.val, OperandValue::Immediate(_));
}
}
}
if name == sym::simd_select_bitmask {
let (len, _) = require_simd!(arg_tys[1], SimdArgument);
let (len, _) = require_simd!(args[1].layout.ty, SimdArgument);
let expected_int_bits = len.max(8).next_power_of_two();
let expected_bytes = len.div_ceil(8);
let mask_ty = arg_tys[0];
let mask_ty = args[0].layout.ty;
let mask = match mask_ty.kind() {
ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
@@ -1269,8 +1263,8 @@ fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
}
// every intrinsic below takes a SIMD vector as its first argument
let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
let in_ty = arg_tys[0];
let (in_len, in_elem) = require_simd!(args[0].layout.ty, SimdInput);
let in_ty = args[0].layout.ty;
let comparison = match name {
sym::simd_eq => Some(BinOp::Eq),
@@ -1401,13 +1395,13 @@ fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
if name == sym::simd_insert || name == sym::simd_insert_dyn {
require!(
in_elem == arg_tys[2],
in_elem == args[2].layout.ty,
InvalidMonomorphization::InsertedType {
span,
name,
in_elem,
in_ty,
out_ty: arg_tys[2]
out_ty: args[2].layout.ty
}
);
@@ -1458,7 +1452,7 @@ fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
if name == sym::simd_select {
let m_elem_ty = in_elem;
let m_len = in_len;
let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
let (v_len, _) = require_simd!(args[1].layout.ty, SimdArgument);
require!(
m_len == v_len,
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
@@ -1659,9 +1653,9 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
// The second argument must be a simd vector with an element type that's a pointer
// to the element type of the first argument
let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
let (out_len, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
let (out_len, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
// The element type of the third argument must be a signed integer type of any width:
let (out_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
let (out_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
require_simd!(ret_ty, SimdReturn);
// Of the same length:
@@ -1672,7 +1666,7 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
name,
in_len,
in_ty,
arg_ty: arg_tys[1],
arg_ty: args[1].layout.ty,
out_len
}
);
@@ -1683,7 +1677,7 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
name,
in_len,
in_ty,
arg_ty: arg_tys[2],
arg_ty: args[2].layout.ty,
out_len: out_len2
}
);
@@ -1703,7 +1697,7 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
span,
name,
expected_element: element_ty1,
second_arg: arg_tys[1],
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Not,
@@ -1764,10 +1758,10 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
let (mask_len, mask_elem) = (in_len, in_elem);
// The second argument must be a pointer matching the element type
let pointer_ty = arg_tys[1];
let pointer_ty = args[1].layout.ty;
// The last argument is a passthrough vector providing values for disabled lanes
let values_ty = arg_tys[2];
let values_ty = args[2].layout.ty;
let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
require_simd!(ret_ty, SimdReturn);
@@ -1855,10 +1849,10 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
let (mask_len, mask_elem) = (in_len, in_elem);
// The second argument must be a pointer matching the element type
let pointer_ty = arg_tys[1];
let pointer_ty = args[1].layout.ty;
// The last argument specifies the values to store to memory
let values_ty = arg_tys[2];
let values_ty = args[2].layout.ty;
let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
// Of the same length:
@@ -1938,8 +1932,8 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
// The second argument must be a simd vector with an element type that's a pointer
// to the element type of the first argument
let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
let (element_len1, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
let (element_len1, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
let (element_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
// Of the same length:
require!(
@@ -1949,7 +1943,7 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
name,
in_len,
in_ty,
arg_ty: arg_tys[1],
arg_ty: args[1].layout.ty,
out_len: element_len1
}
);
@@ -1960,7 +1954,7 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
name,
in_len,
in_ty,
arg_ty: arg_tys[2],
arg_ty: args[2].layout.ty,
out_len: element_len2
}
);
@@ -1975,7 +1969,7 @@ fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -
span,
name,
expected_element: element_ty1,
second_arg: arg_tys[1],
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Mut,
@@ -2497,7 +2491,7 @@ macro_rules! arith_unary {
let ptrs = args[0].immediate();
// The second argument must be a ptr-sized integer.
// (We don't care about the signedness, this is wrapping anyway.)
let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx());
let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
span_bug!(
span,
@@ -2521,8 +2515,8 @@ macro_rules! arith_unary {
return_error!(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1,
vector_type: arg_tys[0]
expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
vector_type: args[0].layout.ty
});
}
};