Move float control management to a separate pass

Functions with CMFloatControl attribute have to save float controls on
entry and restore before return. To do it we need a register, so cr0
read/writes need to be created before regalloc

Author: mshelego

IGC/VectorCompiler/lib/GenXCodeGen/CMakeLists.txt

@@ -1,6 +1,6 @@
 #=========================== begin_copyright_notice ============================
 #
-# Copyright (C) 2017-2023 Intel Corporation
+# Copyright (C) 2017-2024 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -43,6 +43,7 @@ set(CODEGEN_SOURCES
   GenXEmulate.cpp
   GenXExtractVectorizer.cpp
   GenXFixInvalidFuncName.cpp
+  GenXFloatControl.cpp
   GenXLegalizeGVLoadUses.cpp
   GenXGASCastAnalyzer.cpp
   GenXGASDynamicResolution.cpp

IGC/VectorCompiler/lib/GenXCodeGen/GenX.h

@@ -1,6 +1,6 @@
 /*========================== begin_copyright_notice ============================
 
-Copyright (C) 2017-2023 Intel Corporation
+Copyright (C) 2017-2024 Intel Corporation
 
 SPDX-License-Identifier: MIT
 
@@ -156,6 +156,7 @@ FunctionPass *createGenXGlobalUniformAnalysisPass();
 ModulePass *createGenXBuiltinFunctionsPass(BuiltinFunctionKind Kind);
 FunctionPass *createGenXLegacyToLscTranslatorPass();
 ModulePass *createGenXSLMResolution();
+ModulePass *createGenXFloatControlWrapperPass();
 
 namespace genx {
 

IGC/VectorCompiler/lib/GenXCodeGen/GenXCisaBuilder.cpp

@@ -386,22 +386,6 @@ class GenXKernelBuilder {
   std::map<Function *, VISA_GenVar *> FPMap;
   SmallVector<InsertValueInst *, 10> RetvInserts;
 
-  // The default float control from kernel attribute. Each subroutine may
-  // overrride this control mask, but it should revert back to the default float
-  // control mask before exiting from the subroutine.
-  uint32_t DefaultFloatControl = 0;
-
-  enum CRBits {
-    SinglePrecisionMode = 1,
-    RoundingMode = 3 << 4,
-    DoublePrecisionDenorm = 1 << 6,
-    SinglePrecisionDenorm = 1 << 7,
-    HalfPrecisionDenorm = 1 << 10,
-    SystolicDenorm = 1 << 30,
-  };
-
-  uint32_t CRMask = 0;
-
   // normally false, set to true if there is any SIMD CF in the func or this is
   // (indirectly) called inside any SIMD CF.
   bool NoMask = false;
@@ -434,7 +418,6 @@ class GenXKernelBuilder {
   bool buildInstruction(Instruction *Inst);
   bool buildMainInst(Instruction *Inst, genx::BaleInfo BI, unsigned Mod,
                      const DstOpndDesc &DstDesc);
-  void buildControlRegUpdate(unsigned Mask, bool Clear);
   void buildJoin(CallInst *Join, BranchInst *Branch);
   bool buildBranch(BranchInst *Branch);
   void buildIndirectBr(IndirectBrInst *Br);
@@ -1126,12 +1109,6 @@ bool GenXKernelBuilder::run() {
   GrfByteSize = Subtarget->getGRFByteSize();
   StackSurf = Subtarget->stackSurface();
 
-  CRMask = CRBits::RoundingMode | CRBits::DoublePrecisionDenorm |
-           CRBits::SinglePrecisionDenorm | CRBits::HalfPrecisionDenorm;
-
-  if (Subtarget->hasSystolicDenormControl())
-    CRMask |= CRBits::SystolicDenorm;
-
   StackCallExecSize =
       getExecSizeFromValue(BackendConfig->getInteropSubgroupSize());
 
@@ -1324,28 +1301,6 @@ void GenXKernelBuilder::buildInstructions() {
     beginFunctionLight(Func);
     CurrentPadding = 0;
 
-    // If a float control is specified, emit code to make that happen.
-    // Float control contains rounding mode, denorm behaviour and single
-    // precision float mode (ALT or IEEE) Relevant bits are already set as
-    // defined for VISA control reg in header definition on enums
-    if (Func->hasFnAttribute(genx::FunctionMD::CMFloatControl)) {
-      uint32_t FloatControl = 0;
-      Func->getFnAttribute(genx::FunctionMD::CMFloatControl)
-          .getValueAsString()
-          .getAsInteger(0, FloatControl);
-
-      // Clear current float control bits to known zero state
-      buildControlRegUpdate(CRMask, true);
-
-      // Set rounding mode to required state if that isn't zero
-      FloatControl &= CRMask;
-      if (FloatControl) {
-        if (FG->getHead() == Func)
-          DefaultFloatControl = FloatControl;
-        buildControlRegUpdate(FloatControl, false);
-      }
-    }
-
     // Only output a label for the initial basic block if it is used from
     // somewhere else.
     bool NeedsLabel = !Func->front().use_empty();
@@ -4015,39 +3970,6 @@ void GenXKernelBuilder::buildIntrinsic(CallInst *CI, unsigned IntrinID,
 #include "GenXIntrinsicsBuildMap.inc"
 }
 
-/**************************************************************************************************
- * buildControlRegUpdate : generate an instruction to apply a mask to
- *                         the control register (V14).
- *
- * Enter:   Mask = the mask to apply
- *          Clear = false if bits set in Mask should be set in V14,
- *                  true if bits set in Mask should be cleared in V14.
- */
-void GenXKernelBuilder::buildControlRegUpdate(unsigned Mask, bool Clear) {
-  ISA_Opcode Opcode;
-  // write opcode
-  if (Clear) {
-    Opcode = ISA_AND;
-    Mask = ~Mask;
-  } else
-    Opcode = ISA_OR;
-
-  Region Single = Region(1, 4);
-
-  VISA_GenVar *Decl = nullptr;
-  CISA_CALL(Kernel->GetPredefinedVar(Decl, PREDEFINED_CR0));
-  VISA_VectorOpnd *dst =
-      createRegionOperand(&Single, Decl, DONTCARESIGNED, 0, true);
-  VISA_VectorOpnd *src0 =
-      createRegionOperand(&Single, Decl, DONTCARESIGNED, 0, false);
-
-  VISA_VectorOpnd *src1 = nullptr;
-  CISA_CALL(Kernel->CreateVISAImmediate(src1, &Mask, ISA_TYPE_UD));
-
-  appendVISALogicOrShiftInst(Opcode, nullptr, false, vISA_EMASK_M1, EXEC_SIZE_1,
-                             dst, src0, src1);
-}
-
 /***********************************************************************
  * buildBranch : build a conditional or unconditional branch
  *
@@ -5447,17 +5369,6 @@ void GenXKernelBuilder::buildCall(CallInst *CI, const DstOpndDesc &DstDesc) {
 }
 
 void GenXKernelBuilder::buildRet(ReturnInst *RI) {
-  uint32_t FloatControl = 0;
-  auto F = RI->getFunction();
-  F->getFnAttribute(genx::FunctionMD::CMFloatControl)
-      .getValueAsString()
-      .getAsInteger(0, FloatControl);
-  FloatControl &= CRMask;
-  if (FloatControl != DefaultFloatControl) {
-    buildControlRegUpdate(CRMask, true);
-    if (DefaultFloatControl)
-      buildControlRegUpdate(DefaultFloatControl, false);
-  }
   if (vc::requiresStackCall(Func)) {
     appendVISACFFunctionRetInst(nullptr, vISA_EMASK_M1, StackCallExecSize);
   } else {

IGC/VectorCompiler/lib/GenXCodeGen/GenXFloatControl.cpp

@@ -0,0 +1,142 @@
+/*========================== begin_copyright_notice ============================
+
+Copyright (C) 2024 Intel Corporation
+
+SPDX-License-Identifier: MIT
+
+============================= end_copyright_notice ===========================*/
+
+#include "FunctionGroup.h"
+#include "GenX.h"
+#include "GenXTargetMachine.h"
+#include "GenXUtil.h"
+
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/InitializePasses.h"
+
+#define DEBUG_TYPE "GENX_FLOAT_CONTROL"
+
+namespace llvm {
+
+class GenXFloatControl : public FGPassImplInterface, IDMixin<GenXFloatControl> {
+
+  // Must be up to date with CR0 layout
+  enum Bits {
+    SinglePrecisionMode = 1,
+    RoundingMode = 3 << 4,
+    DoublePrecisionDenorm = 1 << 6,
+    SinglePrecisionDenorm = 1 << 7,
+    HalfPrecisionDenorm = 1 << 10,
+    FPToIntRoundingMode = 1 << 12,
+    SystolicDenorm = 1 << 30,
+  };
+
+  uint32_t DefaultFloatControl = 0;
+  uint32_t Mask = 0;
+
+  bool getFloatControl(Function *F, uint32_t *Val);
+
+  Value *buildCr0Update(Value* V, Instruction *InsertBefore);
+
+public:
+  void releaseMemory() {}
+  void print(raw_ostream &OS, const FunctionGroup *FG) const {}
+  bool runOnFunctionGroup(FunctionGroup &FG);
+  void verifyAnalysis() const {}
+
+  static void getAnalysisUsage(AnalysisUsage &AU) {
+    AU.addRequired<TargetPassConfig>();
+    AU.setPreservesAll();
+  }
+  static StringRef getPassName() { return "GenXFloatControl"; }
+};
+
+void initializeGenXFloatControlWrapperPass(PassRegistry &);
+
+using GenXFloatControlWrapper = FunctionGroupWrapperPass<GenXFloatControl>;
+
+} // namespace llvm
+
+using namespace llvm;
+using namespace genx;
+
+INITIALIZE_PASS_BEGIN(GenXFloatControlWrapper, "GenXFloatControlWrapper",
+                      "GenXFloatControlWrapper", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(GenXFloatControlWrapper, "GenXFloatControlWrapper",
+                    "GenXFloatControlWrapper", false, false)
+
+ModulePass *llvm::createGenXFloatControlWrapperPass() {
+  initializeGenXFloatControlWrapperPass(*PassRegistry::getPassRegistry());
+  return new GenXFloatControlWrapper();
+}
+
+bool GenXFloatControl::runOnFunctionGroup(FunctionGroup &FG) {
+  // By default allow to specify with the attribute only rounding and denorm
+  // modes
+  Mask = Bits::RoundingMode | Bits::DoublePrecisionDenorm |
+             Bits::SinglePrecisionDenorm | Bits::HalfPrecisionDenorm;
+  const auto *Subtarget = &getAnalysis<TargetPassConfig>()
+                               .getTM<GenXTargetMachine>()
+                               .getGenXSubtarget();
+  if (Subtarget->hasSystolicDenormControl()) {
+    Mask |= Bits::SystolicDenorm;
+  }
+
+  for (auto It = FG.begin(); It != FG.end(); It++) {
+    Function *F = *It;
+    uint32_t FloatControl = DefaultFloatControl;
+    if (!getFloatControl(F, &FloatControl) && !fg::isHead(*F))
+      continue;
+    // Kernels, stackcalls and subroutines with attribute set float control on
+    // entry - provided by the attribute or the default one
+    auto *V = ConstantInt::get(Type::getInt32Ty(F->getContext()), FloatControl);
+    auto *OldV = buildCr0Update(V, F->getEntryBlock().getFirstNonPHI());
+    if (fg::isGroupHead(*F))
+      continue;
+    // Stackcalls and subroutines with attribute must save caller's float
+    // control on entry and restore it before return
+    for (auto &BB : *F)
+      if (auto *RI = dyn_cast<ReturnInst>(BB.getTerminator()))
+        buildCr0Update(OldV, RI);
+  }
+
+  return true;
+}
+
+bool GenXFloatControl::getFloatControl(Function *F, uint32_t *Val) {
+  if (!F->hasFnAttribute(FunctionMD::CMFloatControl))
+    return false;
+  F->getFnAttribute(FunctionMD::CMFloatControl)
+      .getValueAsString()
+      .getAsInteger(0, *Val);
+  return true;
+}
+
+Value *GenXFloatControl::buildCr0Update(Value *V, Instruction *InsertBefore) {
+  IRBuilder<> B(InsertBefore);
+  auto &DL = InsertBefore->getDebugLoc();
+  auto *M = InsertBefore->getModule();
+
+  auto *Ty = B.getInt32Ty();
+  auto *VTy = IGCLLVM::FixedVectorType::get(Ty, 4);
+  auto *Id = B.getInt32(PreDefined_Vars::PREDEFINED_CR0);
+
+  auto *Cr0 = B.CreateCall(
+      vc::getAnyDeclaration(M, GenXIntrinsic::genx_read_predef_reg, {VTy, VTy}),
+      {Id, UndefValue::get(VTy)});
+
+  Region R(Ty);
+
+  auto *RdR = R.createRdRegion(Cr0, "cr0.0", InsertBefore, DL, true);
+  auto *AndOld = B.CreateAnd(RdR, ~Mask);
+  auto *AndNew = B.CreateAnd(V, Mask);
+  auto *Or = B.CreateOr(AndOld, AndNew);
+  auto *WrR = R.createWrRegion(Cr0, Or, "cr0.0.updated", InsertBefore, DL);
+
+  B.CreateCall(
+      vc::getAnyDeclaration(M, GenXIntrinsic::genx_write_predef_reg, {VTy, VTy}),
+      {Id, WrR});
+
+  return RdR;
+}

IGC/VectorCompiler/lib/GenXCodeGen/GenXTargetMachine.cpp

@@ -191,6 +191,7 @@ void initializeGenXPasses(PassRegistry &registry) {
   initializeGenXLegacyToLscTranslatorPass(registry);
   initializeGenXSLMResolutionPass(registry);
   initializeGenXTypeLegalizationPass(registry);
+  initializeGenXFloatControlWrapperPass(registry);
   // WRITE HERE MORE PASSES IF IT'S NEEDED;
 }
 
@@ -709,6 +710,7 @@ bool GenXTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
   vc::addPass(PM, createGlobalDCEPass());
   /// .. include:: GenXModule.h
   vc::addPass(PM, createGenXModulePass());
+  vc::addPass(PM, createGenXFloatControlWrapperPass());
   /// .. include:: GenXLiveness.h
   vc::addPass(PM, createGenXGroupBalingWrapperPass(BalingKind::BK_Analysis,
                                                    &Subtarget));

IGC/VectorCompiler/lib/GenXCodeGen/GenXTargetMachine.h

@@ -271,6 +271,8 @@ void initializeGenXInitBiFConstantsPass(PassRegistry &);
 void initializeGenXFinalizerPass(PassRegistry &);
 void initializeGenXBuiltinFunctionsPass(PassRegistry &);
 void initializeGenXSLMResolutionPass(PassRegistry &);
+void initializeGenXFloatControlWrapperPass(PassRegistry &);
+
 } // End llvm namespace
 
 #endif

IGC/VectorCompiler/lib/GenXCodeGen/GenXVisaRegAlloc.cpp

@@ -1,6 +1,6 @@
 /*========================== begin_copyright_notice ============================
 
-Copyright (C) 2017-2023 Intel Corporation
+Copyright (C) 2017-2024 Intel Corporation
 
 SPDX-License-Identifier: MIT
 

IGC/VectorCompiler/lib/GenXCodeGen/GenXVisaRegAlloc.h

@@ -1,6 +1,6 @@
 /*========================== begin_copyright_notice ============================
 
-Copyright (C) 2017-2023 Intel Corporation
+Copyright (C) 2017-2024 Intel Corporation
 
 SPDX-License-Identifier: MIT
 

IGC/VectorCompiler/test/CisaBuilder/debugtrap.ll

@@ -1,6 +1,6 @@
 ;=========================== begin_copyright_notice ============================
 ;
-; Copyright (C) 2023 Intel Corporation
+; Copyright (C) 2023-2024 Intel Corporation
 ;
 ; SPDX-License-Identifier: MIT
 ;

IGC/VectorCompiler/test/DebugInfo/basic_var_locs.ll

@@ -1,6 +1,6 @@
 ;=========================== begin_copyright_notice ============================
 ;
-; Copyright (C) 2020-2021 Intel Corporation
+; Copyright (C) 2020-2024 Intel Corporation
 ;
 ; SPDX-License-Identifier: MIT
 ;
@@ -112,8 +112,6 @@ attributes #1 = { nounwind readnone speculatable willreturn }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readonly }
 attributes #4 = { nounwind }
-attributes #5 = { noinline nounwind "CMFloatControl"="0" }
-attributes #6 = { noinline nounwind "CMFloatControl"="48" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.dbg.cu = !{!2}