-
Notifications
You must be signed in to change notification settings - Fork 15.2k
[CIR] Upstream codegen for X86 SSE builtins #167584
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Conversation
|
@llvm/pr-subscribers-clangir Author: Thibault Monnier (Thibault-Monnier) ChangesThis PR implements ClangIR codegen for the X86 SSE builtins listed in Full diff: https://github.com/llvm/llvm-project/pull/167584.diff 8 Files Affected:
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 16258513239d9..76ce67fb1ea1d 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -413,6 +413,12 @@ def CIR_ConstantOp : CIR_Op<"const", [
template <typename T>
T getValueAttr() { return mlir::dyn_cast<T>(getValue()); }
+
+ llvm::APInt getIntValue() {
+ if (const auto intAttr = getValueAttr<cir::IntAttr>())
+ return intAttr.getValue();
+ llvm_unreachable("Expected an IntAttr in ConstantOp");
+ }
}];
let hasFolder = 1;
@@ -2579,6 +2585,40 @@ def CIR_FuncOp : CIR_Op<"func", [
}];
}
+//===----------------------------------------------------------------------===//
+// LLVMIntrinsicCallOp
+//===----------------------------------------------------------------------===//
+
+def CIR_LLVMIntrinsicCallOp : CIR_Op<"llvm.intrinsic"> {
+ let summary = "Call to llvm intrinsic functions that is not defined in CIR";
+ let description = [{
+ `cir.llvm.intrinsic` operation represents a call-like expression which has
+ return type and arguments that maps directly to a llvm intrinsic.
+ It only records intrinsic `intrinsic_name`.
+ }];
+
+ let results = (outs Optional<CIR_AnyType>:$result);
+ let arguments = (ins
+ StrAttr:$intrinsic_name, Variadic<CIR_AnyType>:$arg_ops);
+
+ let skipDefaultBuilders = 1;
+
+ let assemblyFormat = [{
+ $intrinsic_name $arg_ops `:` functional-type($arg_ops, $result) attr-dict
+ }];
+
+ let builders = [
+ OpBuilder<(ins "mlir::StringAttr":$intrinsic_name, "mlir::Type":$resType,
+ CArg<"mlir::ValueRange", "{}">:$operands), [{
+ $_state.addAttribute("intrinsic_name", intrinsic_name);
+ $_state.addOperands(operands);
+ if (resType)
+ $_state.addTypes(resType);
+ }]>,
+ ];
+
+}
+
//===----------------------------------------------------------------------===//
// CallOp
//===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 4e6a5ee7ee210..f948a02e79e8c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -625,6 +625,22 @@ CIRGenFunction::emitTargetBuiltinExpr(unsigned builtinID, const CallExpr *e,
getTarget().getTriple().getArch());
}
+mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(
+ const unsigned ICEArguments, const unsigned idx, const CallExpr *e) {
+ mlir::Value arg = {};
+ if ((ICEArguments & (1 << idx)) == 0) {
+ arg = emitScalarExpr(e->getArg(idx));
+ } else {
+ // If this is required to be a constant, constant fold it so that we
+ // know that the generated intrinsic gets a ConstantInt.
+ const std::optional<llvm::APSInt> result =
+ e->getArg(idx)->getIntegerConstantExpr(getContext());
+ assert(result && "Expected argument to be a constant");
+ arg = builder.getConstInt(getLoc(e->getSourceRange()), *result);
+ }
+ return arg;
+}
+
/// Given a builtin id for a function like "__builtin_fabsf", return a Function*
/// for "fabsf".
cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
index 0198a9d4eb192..7cf3e9081a2f3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
@@ -18,9 +18,22 @@
#include "clang/CIR/MissingFeatures.h"
#include "llvm/IR/IntrinsicsX86.h"
+#define UNIMPLEMENTED_BUILTIN() \
+ do { \
+ cgm.errorNYI(e->getSourceRange(), \
+ std::string("unimplemented X86 builtin call: ") + \
+ getContext().BuiltinInfo.getName(builtinID)); \
+ return {}; \
+ } while (0)
+
using namespace clang;
using namespace clang::CIRGen;
+/// Get integer from a mlir::Value that is an int constant or a constant op.
+static int64_t getIntValueFromConstOp(mlir::Value val) {
+ return val.getDefiningOp<cir::ConstantOp>().getIntValue().getSExtValue();
+}
+
mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
const CallExpr *e) {
if (builtinID == Builtin::BI__builtin_cpu_is) {
@@ -43,15 +56,76 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
// Find out if any arguments are required to be integer constant expressions.
assert(!cir::MissingFeatures::handleBuiltinICEArguments());
+ llvm::SmallVector<mlir::Value, 4> ops;
+
+ // Find out if any arguments are required to be integer constant expressions.
+ unsigned ICEArguments = 0;
+ ASTContext::GetBuiltinTypeError error;
+ getContext().GetBuiltinType(builtinID, error, &ICEArguments);
+ assert(error == ASTContext::GE_None && "Should not codegen an error");
+
+ for (auto [idx, _] : llvm::enumerate(e->arguments())) {
+ ops.push_back(emitScalarOrConstFoldImmArg(ICEArguments, idx, e));
+ }
+
switch (builtinID) {
default:
return {};
- case X86::BI_mm_prefetch:
- case X86::BI_mm_clflush:
- case X86::BI_mm_lfence:
- case X86::BI_mm_pause:
- case X86::BI_mm_mfence:
- case X86::BI_mm_sfence:
+ case X86::BI_mm_prefetch: {
+ mlir::Value address = builder.createPtrBitcast(ops[0], voidTy);
+
+ int64_t hint = getIntValueFromConstOp(ops[1]);
+ mlir::Value rw =
+ cir::ConstantOp::create(builder, getLoc(e->getExprLoc()),
+ cir::IntAttr::get(sInt32Ty, (hint >> 2) & 0x1));
+ mlir::Value locality =
+ cir::ConstantOp::create(builder, getLoc(e->getExprLoc()),
+ cir::IntAttr::get(sInt32Ty, hint & 0x3));
+ mlir::Value data = cir::ConstantOp::create(builder, getLoc(e->getExprLoc()),
+ cir::IntAttr::get(sInt32Ty, 1));
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("prefetch"), voidTy,
+ mlir::ValueRange{address, rw, locality, data})
+ .getResult();
+ }
+ case X86::BI_mm_clflush: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.clflush"), voidTy, ops[0])
+ .getResult();
+ }
+ case X86::BI_mm_lfence: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.lfence"), voidTy)
+ .getResult();
+ }
+ case X86::BI_mm_pause: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.pause"), voidTy)
+ .getResult();
+ }
+ case X86::BI_mm_mfence: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.mfence"), voidTy)
+ .getResult();
+ }
+ case X86::BI_mm_sfence: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse.sfence"), voidTy)
+ .getResult();
+ }
case X86::BI__rdtsc:
case X86::BI__builtin_ia32_rdtscp:
case X86::BI__builtin_ia32_lzcnt_u16:
@@ -82,10 +156,27 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
case X86::BI__builtin_ia32_vec_set_v16hi:
case X86::BI__builtin_ia32_vec_set_v8si:
case X86::BI__builtin_ia32_vec_set_v4di:
+ UNIMPLEMENTED_BUILTIN();
case X86::BI_mm_setcsr:
- case X86::BI__builtin_ia32_ldmxcsr:
+ case X86::BI__builtin_ia32_ldmxcsr: {
+ Address tmp =
+ createMemTemp(e->getArg(0)->getType(), getLoc(e->getExprLoc()));
+ builder.createStore(getLoc(e->getExprLoc()), ops[0], tmp);
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse.ldmxcsr"), builder.getVoidTy(),
+ tmp.getPointer())
+ .getResult();
+ }
case X86::BI_mm_getcsr:
- case X86::BI__builtin_ia32_stmxcsr:
+ case X86::BI__builtin_ia32_stmxcsr: {
+ Address tmp = createMemTemp(e->getType(), getLoc(e->getExprLoc()));
+ cir::LLVMIntrinsicCallOp::create(builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse.stmxcsr"),
+ builder.getVoidTy(), tmp.getPointer())
+ .getResult();
+ return builder.createLoad(getLoc(e->getExprLoc()), tmp);
+ }
case X86::BI__builtin_ia32_xsave:
case X86::BI__builtin_ia32_xsave64:
case X86::BI__builtin_ia32_xrstor:
@@ -798,9 +889,6 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:
case X86::BI__builtin_ia32_prefetchi:
- cgm.errorNYI(e->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return {};
+ UNIMPLEMENTED_BUILTIN();
}
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index f879e580989f7..06a4ddd4ba39b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -1699,6 +1699,9 @@ class CIRGenFunction : public CIRGenTypeCache {
void emitScalarInit(const clang::Expr *init, mlir::Location loc,
LValue lvalue, bool capturedByInit = false);
+ mlir::Value emitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned idx,
+ const CallExpr *e);
+
void emitStaticVarDecl(const VarDecl &d, cir::GlobalLinkageKind linkage);
void emitStoreOfComplex(mlir::Location loc, mlir::Value v, LValue dest,
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index b4afed7019417..6415160efa901 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -646,6 +646,29 @@ mlir::LogicalResult CIRToLLVMASinOpLowering::matchAndRewrite(
return mlir::success();
}
+mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(
+ cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,
+ mlir::ConversionPatternRewriter &rewriter) const {
+ mlir::Type llvmResTy =
+ getTypeConverter()->convertType(op->getResultTypes()[0]);
+ if (!llvmResTy)
+ return op.emitError("expected LLVM result type");
+ StringRef name = op.getIntrinsicName();
+ // Some llvm intrinsics require ElementType attribute to be attached to
+ // the argument of pointer type. That prevents us from generating LLVM IR
+ // because from LLVM dialect, we have LLVM IR like the below which fails
+ // LLVM IR verification.
+ // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)
+ // The expected LLVM IR should be like
+ // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)
+ // TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way
+ // to set LLVM IR attribute.
+ assert(!cir::MissingFeatures::llvmIntrinsicElementTypeSupport());
+ replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy,
+ adaptor.getOperands());
+ return mlir::success();
+}
+
mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
cir::AssumeOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
diff --git a/clang/test/CIR/CodeGen/X86/pause.c b/clang/test/CIR/CodeGen/X86/pause.c
new file mode 100644
index 0000000000000..67a0d6770c517
--- /dev/null
+++ b/clang/test/CIR/CodeGen/X86/pause.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature +sse2 -fclangir -emit-cir -o %t.cir %s
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature +sse2 -fclangir -emit-llvm -o %t.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature -sse2 -fclangir -emit-cir -o %t.cir %s
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature -sse2 -fclangir -emit-llvm -o %t.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+// This test mimics clang/test/CodeGen/X86/pause.c, which eventually
+// CIR shall be able to support fully.
+
+#include <x86intrin.h>
+
+void test_mm_pause(void) {
+ // CIR-LABEL: test_mm_pause
+ // LLVM-LABEL: test_mm_pause
+ _mm_pause();
+ // CIR: {{%.*}} = cir.llvm.intrinsic "x86.sse2.pause" : () -> !void
+ // LLVM: call void @llvm.x86.sse2.pause()
+}
diff --git a/clang/test/CIR/CodeGen/X86/sse-builtins.c b/clang/test/CIR/CodeGen/X86/sse-builtins.c
new file mode 100644
index 0000000000000..57f3af18aab7b
--- /dev/null
+++ b/clang/test/CIR/CodeGen/X86/sse-builtins.c
@@ -0,0 +1,52 @@
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// This test mimics clang/test/CodeGen/X86/sse-builtins.c, which eventually
+// CIR shall be able to support fully.
+
+#include <immintrin.h>
+
+void test_mm_prefetch(char const* p) {
+ // CIR-LABEL: test_mm_prefetch
+ // LLVM-LABEL: test_mm_prefetch
+ _mm_prefetch(p, 0);
+ // CIR: cir.prefetch read locality(0) %{{.*}} : !cir.ptr<!void>
+ // LLVM: call void @llvm.prefetch.p0(ptr {{.*}}, i32 0, i32 0, i32 1)
+}
+
+void test_mm_sfence(void) {
+ // CIR-LABEL: test_mm_sfence
+ // LLVM-LABEL: test_mm_sfence
+ _mm_sfence();
+ // CIR: {{%.*}} = cir.llvm.intrinsic "x86.sse.sfence" : () -> !void
+ // LLVM: call void @llvm.x86.sse.sfence()
+}
+
+void test_mm_setcsr(unsigned int A) {
+ // CIR-LABEL: test_mm_setcsr
+ // CIR: cir.store {{.*}}, {{.*}} : !u32i
+ // CIR: cir.llvm.intrinsic "x86.sse.ldmxcsr" {{.*}} : (!cir.ptr<!u32i>) -> !void
+
+ // LLVM-LABEL: test_mm_setcsr
+ // LLVM: store i32
+ // LLVM: call void @llvm.x86.sse.ldmxcsr(ptr {{.*}})
+ _mm_setcsr(A);
+}
+
+unsigned int test_mm_getcsr(void) {
+ // CIR-LABEL: test_mm_getcsr
+ // CIR: cir.llvm.intrinsic "x86.sse.stmxcsr" %{{.*}} : (!cir.ptr<!u32i>) -> !void
+ // CIR: cir.load {{.*}} : !cir.ptr<!u32i>, !u32i
+
+ // LLVM-LABEL: test_mm_getcsr
+ // LLVM: call void @llvm.x86.sse.stmxcsr(ptr %{{.*}})
+ // LLVM: load i32
+ return _mm_getcsr();
+}
diff --git a/clang/test/CIR/CodeGen/X86/sse2-builtins.c b/clang/test/CIR/CodeGen/X86/sse2-builtins.c
new file mode 100644
index 0000000000000..9dffc07caf0fa
--- /dev/null
+++ b/clang/test/CIR/CodeGen/X86/sse2-builtins.c
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR-CHECK --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR-CHECK --input-file=%t.cir %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM-CHECK --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM-CHECK --input-file=%t.ll %s
+
+// This test mimics clang/test/CodeGen/X86/sse2-builtins.c, which eventually
+// CIR shall be able to support fully.
+
+#include <immintrin.h>
+
+void test_mm_clflush(void* A) {
+ // CIR-LABEL: test_mm_clflush
+ // LLVM-LABEL: test_mm_clflush
+ _mm_clflush(A);
+ // CIR-CHECK: {{%.*}} = cir.llvm.intrinsic "x86.sse2.clflush" {{%.*}} : (!cir.ptr<!void>) -> !void
+ // LLVM-CHECK: call void @llvm.x86.sse2.clflush(ptr {{%.*}})
+}
+
+void test_mm_lfence(void) {
+ // CIR-CHECK-LABEL: test_mm_lfence
+ // LLVM-CHECK-LABEL: test_mm_lfence
+ _mm_lfence();
+ // CIR-CHECK: {{%.*}} = cir.llvm.intrinsic "x86.sse2.lfence" : () -> !void
+ // LLVM-CHECK: call void @llvm.x86.sse2.lfence()
+}
+
+void test_mm_mfence(void) {
+ // CIR-CHECK-LABEL: test_mm_mfence
+ // LLVM-CHECK-LABEL: test_mm_mfence
+ _mm_mfence();
+ // CIR-CHECK: {{%.*}} = cir.llvm.intrinsic "x86.sse2.mfence" : () -> !void
+ // LLVM-CHECK: call void @llvm.x86.sse2.mfence()
+}
\ No newline at end of file
|
|
@llvm/pr-subscribers-clang Author: Thibault Monnier (Thibault-Monnier) ChangesThis PR implements ClangIR codegen for the X86 SSE builtins listed in Full diff: https://github.com/llvm/llvm-project/pull/167584.diff 8 Files Affected:
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 16258513239d9..76ce67fb1ea1d 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -413,6 +413,12 @@ def CIR_ConstantOp : CIR_Op<"const", [
template <typename T>
T getValueAttr() { return mlir::dyn_cast<T>(getValue()); }
+
+ llvm::APInt getIntValue() {
+ if (const auto intAttr = getValueAttr<cir::IntAttr>())
+ return intAttr.getValue();
+ llvm_unreachable("Expected an IntAttr in ConstantOp");
+ }
}];
let hasFolder = 1;
@@ -2579,6 +2585,40 @@ def CIR_FuncOp : CIR_Op<"func", [
}];
}
+//===----------------------------------------------------------------------===//
+// LLVMIntrinsicCallOp
+//===----------------------------------------------------------------------===//
+
+def CIR_LLVMIntrinsicCallOp : CIR_Op<"llvm.intrinsic"> {
+ let summary = "Call to llvm intrinsic functions that is not defined in CIR";
+ let description = [{
+ `cir.llvm.intrinsic` operation represents a call-like expression which has
+ return type and arguments that maps directly to a llvm intrinsic.
+ It only records intrinsic `intrinsic_name`.
+ }];
+
+ let results = (outs Optional<CIR_AnyType>:$result);
+ let arguments = (ins
+ StrAttr:$intrinsic_name, Variadic<CIR_AnyType>:$arg_ops);
+
+ let skipDefaultBuilders = 1;
+
+ let assemblyFormat = [{
+ $intrinsic_name $arg_ops `:` functional-type($arg_ops, $result) attr-dict
+ }];
+
+ let builders = [
+ OpBuilder<(ins "mlir::StringAttr":$intrinsic_name, "mlir::Type":$resType,
+ CArg<"mlir::ValueRange", "{}">:$operands), [{
+ $_state.addAttribute("intrinsic_name", intrinsic_name);
+ $_state.addOperands(operands);
+ if (resType)
+ $_state.addTypes(resType);
+ }]>,
+ ];
+
+}
+
//===----------------------------------------------------------------------===//
// CallOp
//===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 4e6a5ee7ee210..f948a02e79e8c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -625,6 +625,22 @@ CIRGenFunction::emitTargetBuiltinExpr(unsigned builtinID, const CallExpr *e,
getTarget().getTriple().getArch());
}
+mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(
+ const unsigned ICEArguments, const unsigned idx, const CallExpr *e) {
+ mlir::Value arg = {};
+ if ((ICEArguments & (1 << idx)) == 0) {
+ arg = emitScalarExpr(e->getArg(idx));
+ } else {
+ // If this is required to be a constant, constant fold it so that we
+ // know that the generated intrinsic gets a ConstantInt.
+ const std::optional<llvm::APSInt> result =
+ e->getArg(idx)->getIntegerConstantExpr(getContext());
+ assert(result && "Expected argument to be a constant");
+ arg = builder.getConstInt(getLoc(e->getSourceRange()), *result);
+ }
+ return arg;
+}
+
/// Given a builtin id for a function like "__builtin_fabsf", return a Function*
/// for "fabsf".
cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
index 0198a9d4eb192..7cf3e9081a2f3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
@@ -18,9 +18,22 @@
#include "clang/CIR/MissingFeatures.h"
#include "llvm/IR/IntrinsicsX86.h"
+#define UNIMPLEMENTED_BUILTIN() \
+ do { \
+ cgm.errorNYI(e->getSourceRange(), \
+ std::string("unimplemented X86 builtin call: ") + \
+ getContext().BuiltinInfo.getName(builtinID)); \
+ return {}; \
+ } while (0)
+
using namespace clang;
using namespace clang::CIRGen;
+/// Get integer from a mlir::Value that is an int constant or a constant op.
+static int64_t getIntValueFromConstOp(mlir::Value val) {
+ return val.getDefiningOp<cir::ConstantOp>().getIntValue().getSExtValue();
+}
+
mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
const CallExpr *e) {
if (builtinID == Builtin::BI__builtin_cpu_is) {
@@ -43,15 +56,76 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
// Find out if any arguments are required to be integer constant expressions.
assert(!cir::MissingFeatures::handleBuiltinICEArguments());
+ llvm::SmallVector<mlir::Value, 4> ops;
+
+ // Find out if any arguments are required to be integer constant expressions.
+ unsigned ICEArguments = 0;
+ ASTContext::GetBuiltinTypeError error;
+ getContext().GetBuiltinType(builtinID, error, &ICEArguments);
+ assert(error == ASTContext::GE_None && "Should not codegen an error");
+
+ for (auto [idx, _] : llvm::enumerate(e->arguments())) {
+ ops.push_back(emitScalarOrConstFoldImmArg(ICEArguments, idx, e));
+ }
+
switch (builtinID) {
default:
return {};
- case X86::BI_mm_prefetch:
- case X86::BI_mm_clflush:
- case X86::BI_mm_lfence:
- case X86::BI_mm_pause:
- case X86::BI_mm_mfence:
- case X86::BI_mm_sfence:
+ case X86::BI_mm_prefetch: {
+ mlir::Value address = builder.createPtrBitcast(ops[0], voidTy);
+
+ int64_t hint = getIntValueFromConstOp(ops[1]);
+ mlir::Value rw =
+ cir::ConstantOp::create(builder, getLoc(e->getExprLoc()),
+ cir::IntAttr::get(sInt32Ty, (hint >> 2) & 0x1));
+ mlir::Value locality =
+ cir::ConstantOp::create(builder, getLoc(e->getExprLoc()),
+ cir::IntAttr::get(sInt32Ty, hint & 0x3));
+ mlir::Value data = cir::ConstantOp::create(builder, getLoc(e->getExprLoc()),
+ cir::IntAttr::get(sInt32Ty, 1));
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("prefetch"), voidTy,
+ mlir::ValueRange{address, rw, locality, data})
+ .getResult();
+ }
+ case X86::BI_mm_clflush: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.clflush"), voidTy, ops[0])
+ .getResult();
+ }
+ case X86::BI_mm_lfence: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.lfence"), voidTy)
+ .getResult();
+ }
+ case X86::BI_mm_pause: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.pause"), voidTy)
+ .getResult();
+ }
+ case X86::BI_mm_mfence: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse2.mfence"), voidTy)
+ .getResult();
+ }
+ case X86::BI_mm_sfence: {
+ mlir::Type voidTy = cir::VoidType::get(&getMLIRContext());
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse.sfence"), voidTy)
+ .getResult();
+ }
case X86::BI__rdtsc:
case X86::BI__builtin_ia32_rdtscp:
case X86::BI__builtin_ia32_lzcnt_u16:
@@ -82,10 +156,27 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
case X86::BI__builtin_ia32_vec_set_v16hi:
case X86::BI__builtin_ia32_vec_set_v8si:
case X86::BI__builtin_ia32_vec_set_v4di:
+ UNIMPLEMENTED_BUILTIN();
case X86::BI_mm_setcsr:
- case X86::BI__builtin_ia32_ldmxcsr:
+ case X86::BI__builtin_ia32_ldmxcsr: {
+ Address tmp =
+ createMemTemp(e->getArg(0)->getType(), getLoc(e->getExprLoc()));
+ builder.createStore(getLoc(e->getExprLoc()), ops[0], tmp);
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse.ldmxcsr"), builder.getVoidTy(),
+ tmp.getPointer())
+ .getResult();
+ }
case X86::BI_mm_getcsr:
- case X86::BI__builtin_ia32_stmxcsr:
+ case X86::BI__builtin_ia32_stmxcsr: {
+ Address tmp = createMemTemp(e->getType(), getLoc(e->getExprLoc()));
+ cir::LLVMIntrinsicCallOp::create(builder, getLoc(e->getExprLoc()),
+ builder.getStringAttr("x86.sse.stmxcsr"),
+ builder.getVoidTy(), tmp.getPointer())
+ .getResult();
+ return builder.createLoad(getLoc(e->getExprLoc()), tmp);
+ }
case X86::BI__builtin_ia32_xsave:
case X86::BI__builtin_ia32_xsave64:
case X86::BI__builtin_ia32_xrstor:
@@ -798,9 +889,6 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:
case X86::BI__builtin_ia32_prefetchi:
- cgm.errorNYI(e->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return {};
+ UNIMPLEMENTED_BUILTIN();
}
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index f879e580989f7..06a4ddd4ba39b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -1699,6 +1699,9 @@ class CIRGenFunction : public CIRGenTypeCache {
void emitScalarInit(const clang::Expr *init, mlir::Location loc,
LValue lvalue, bool capturedByInit = false);
+ mlir::Value emitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned idx,
+ const CallExpr *e);
+
void emitStaticVarDecl(const VarDecl &d, cir::GlobalLinkageKind linkage);
void emitStoreOfComplex(mlir::Location loc, mlir::Value v, LValue dest,
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index b4afed7019417..6415160efa901 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -646,6 +646,29 @@ mlir::LogicalResult CIRToLLVMASinOpLowering::matchAndRewrite(
return mlir::success();
}
+mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(
+ cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,
+ mlir::ConversionPatternRewriter &rewriter) const {
+ mlir::Type llvmResTy =
+ getTypeConverter()->convertType(op->getResultTypes()[0]);
+ if (!llvmResTy)
+ return op.emitError("expected LLVM result type");
+ StringRef name = op.getIntrinsicName();
+ // Some llvm intrinsics require ElementType attribute to be attached to
+ // the argument of pointer type. That prevents us from generating LLVM IR
+ // because from LLVM dialect, we have LLVM IR like the below which fails
+ // LLVM IR verification.
+ // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)
+ // The expected LLVM IR should be like
+ // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)
+ // TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way
+ // to set LLVM IR attribute.
+ assert(!cir::MissingFeatures::llvmIntrinsicElementTypeSupport());
+ replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy,
+ adaptor.getOperands());
+ return mlir::success();
+}
+
mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
cir::AssumeOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
diff --git a/clang/test/CIR/CodeGen/X86/pause.c b/clang/test/CIR/CodeGen/X86/pause.c
new file mode 100644
index 0000000000000..67a0d6770c517
--- /dev/null
+++ b/clang/test/CIR/CodeGen/X86/pause.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature +sse2 -fclangir -emit-cir -o %t.cir %s
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature +sse2 -fclangir -emit-llvm -o %t.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature -sse2 -fclangir -emit-cir -o %t.cir %s
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-linux -Wno-implicit-function-declaration -target-feature -sse2 -fclangir -emit-llvm -o %t.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+// This test mimics clang/test/CodeGen/X86/pause.c, which eventually
+// CIR shall be able to support fully.
+
+#include <x86intrin.h>
+
+void test_mm_pause(void) {
+ // CIR-LABEL: test_mm_pause
+ // LLVM-LABEL: test_mm_pause
+ _mm_pause();
+ // CIR: {{%.*}} = cir.llvm.intrinsic "x86.sse2.pause" : () -> !void
+ // LLVM: call void @llvm.x86.sse2.pause()
+}
diff --git a/clang/test/CIR/CodeGen/X86/sse-builtins.c b/clang/test/CIR/CodeGen/X86/sse-builtins.c
new file mode 100644
index 0000000000000..57f3af18aab7b
--- /dev/null
+++ b/clang/test/CIR/CodeGen/X86/sse-builtins.c
@@ -0,0 +1,52 @@
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// This test mimics clang/test/CodeGen/X86/sse-builtins.c, which eventually
+// CIR shall be able to support fully.
+
+#include <immintrin.h>
+
+void test_mm_prefetch(char const* p) {
+ // CIR-LABEL: test_mm_prefetch
+ // LLVM-LABEL: test_mm_prefetch
+ _mm_prefetch(p, 0);
+ // CIR: cir.prefetch read locality(0) %{{.*}} : !cir.ptr<!void>
+ // LLVM: call void @llvm.prefetch.p0(ptr {{.*}}, i32 0, i32 0, i32 1)
+}
+
+void test_mm_sfence(void) {
+ // CIR-LABEL: test_mm_sfence
+ // LLVM-LABEL: test_mm_sfence
+ _mm_sfence();
+ // CIR: {{%.*}} = cir.llvm.intrinsic "x86.sse.sfence" : () -> !void
+ // LLVM: call void @llvm.x86.sse.sfence()
+}
+
+void test_mm_setcsr(unsigned int A) {
+ // CIR-LABEL: test_mm_setcsr
+ // CIR: cir.store {{.*}}, {{.*}} : !u32i
+ // CIR: cir.llvm.intrinsic "x86.sse.ldmxcsr" {{.*}} : (!cir.ptr<!u32i>) -> !void
+
+ // LLVM-LABEL: test_mm_setcsr
+ // LLVM: store i32
+ // LLVM: call void @llvm.x86.sse.ldmxcsr(ptr {{.*}})
+ _mm_setcsr(A);
+}
+
+unsigned int test_mm_getcsr(void) {
+ // CIR-LABEL: test_mm_getcsr
+ // CIR: cir.llvm.intrinsic "x86.sse.stmxcsr" %{{.*}} : (!cir.ptr<!u32i>) -> !void
+ // CIR: cir.load {{.*}} : !cir.ptr<!u32i>, !u32i
+
+ // LLVM-LABEL: test_mm_getcsr
+ // LLVM: call void @llvm.x86.sse.stmxcsr(ptr %{{.*}})
+ // LLVM: load i32
+ return _mm_getcsr();
+}
diff --git a/clang/test/CIR/CodeGen/X86/sse2-builtins.c b/clang/test/CIR/CodeGen/X86/sse2-builtins.c
new file mode 100644
index 0000000000000..9dffc07caf0fa
--- /dev/null
+++ b/clang/test/CIR/CodeGen/X86/sse2-builtins.c
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR-CHECK --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR-CHECK --input-file=%t.cir %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM-CHECK --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM-CHECK --input-file=%t.ll %s
+
+// This test mimics clang/test/CodeGen/X86/sse2-builtins.c, which eventually
+// CIR shall be able to support fully.
+
+#include <immintrin.h>
+
+void test_mm_clflush(void* A) {
+ // CIR-LABEL: test_mm_clflush
+ // LLVM-LABEL: test_mm_clflush
+ _mm_clflush(A);
+ // CIR-CHECK: {{%.*}} = cir.llvm.intrinsic "x86.sse2.clflush" {{%.*}} : (!cir.ptr<!void>) -> !void
+ // LLVM-CHECK: call void @llvm.x86.sse2.clflush(ptr {{%.*}})
+}
+
+void test_mm_lfence(void) {
+ // CIR-CHECK-LABEL: test_mm_lfence
+ // LLVM-CHECK-LABEL: test_mm_lfence
+ _mm_lfence();
+ // CIR-CHECK: {{%.*}} = cir.llvm.intrinsic "x86.sse2.lfence" : () -> !void
+ // LLVM-CHECK: call void @llvm.x86.sse2.lfence()
+}
+
+void test_mm_mfence(void) {
+ // CIR-CHECK-LABEL: test_mm_mfence
+ // LLVM-CHECK-LABEL: test_mm_mfence
+ _mm_mfence();
+ // CIR-CHECK: {{%.*}} = cir.llvm.intrinsic "x86.sse2.mfence" : () -> !void
+ // LLVM-CHECK: call void @llvm.x86.sse2.mfence()
+}
\ No newline at end of file
|
|
This may have a merge conflict with #167125. |
|
Hi, this is mostly a duplicate of #167401 unfortunately |
|
Right, should we close this one? |
|
Ok! Could you please tell me what sse builtins are missing? It seems to me that there are no others in the switch case... |
|
You can upstream the missing ones from the incubator repo here for example |
andykaylor
left a comment
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We've got a lot of overlapping work going on right now. I love the enthusiasm from so many new contributors, but we've got to get this under control.
I'd like to see them land in this order:
That's no judgment on the quality of the PRs. It's based on which ones I'm seeing most in external testing, and a bit of layering that seems logical to me.
|
@andykaylor I've applied your suggestions and upstreamed all the other builtins that were in |
@Thibault-Monnier With the additional builtins added this is getting too large to effectively review. I'd prefer to have it broken up in accordance with the groupings in the table on #167752. Multiple PRs will be easier to review and will make for a better Git commit history. I realize this will slow down the review process overall, but it will tend towards a better result. |
|
Ok. I'll work on it as soon as I can. Should we close this PR? |
That's not necessary. Just trim it back to a single group of builtins at the granularity of the table in #167752. |
bfccdb1 to
5586dbb
Compare
This PR implements ClangIR codegen for the X86 SSE builtins listed in
CIRGenBuiltinX86.cppand the associated tests. Everything was pulled in from the incubator.