Add option to split evaluate messages

Author: davidjwoo

IGC/Compiler/CISACodeGen/CISABuilder.cpp

@@ -3803,6 +3803,11 @@ namespace IGC
             SaveOption(vISA_cloneSampleInst, true);
         }
 
+        if (m_program->m_Platform->getWATable().Wa_22011142311 && IGC_IS_FLAG_ENABLED(EnableEvaluateSamplerSplit))
+        {
+            SaveOption(vISA_cloneEvaluateSampleInst, true);
+        }
+
         if (IGC_IS_FLAG_ENABLED(ForceFFIDOverwrite)/*|| m_program->m_Platform->WaOverwriteFFID()*/)
         {
             unsigned int ffid[unsigned(ShaderType::END)] = {

IGC/common/igc_flags.def

@@ -425,6 +425,8 @@ DECLARE_IGC_REGKEY(bool, UseNewRegEncoding, true, "Use new location encoding for
 DECLARE_IGC_REGKEY(bool, EmitOffsetInDbgLoc, false, "Emit offset of private memory in DW_AT_location when available", false)
 DECLARE_IGC_REGKEY(bool, EnableA64WA, true, "Guarantee A64 load/store addres-hi is uniform", false)
 DECLARE_IGC_REGKEY(bool, EnableSamplerSplit, true, "Split SIMD8 Sampler message to 2 subspans and SIMD16 to odd and even", false)
+DECLARE_IGC_REGKEY(bool, EnableEvaluateSamplerSplit, false, "Split evaluate messages to sampler into either SIMD8 or SIMD1 messages", false)
+
 DECLARE_IGC_REGKEY(bool, EnableZEBinary, false,  "Enable output in ZE binary format", true)
 DECLARE_IGC_REGKEY(DWORD, OverrideOCLMaxParamSize, 0,  "Override the value imposed on the kernel by CL_DEVICE_MAX_PARAMETER_SIZE. Value in bytes, if value==0 no override happens.", true)
 

visa/Gen4_IR.cpp

@@ -468,6 +468,12 @@ void G4_SendMsgDescriptor::setBindingTableIdx(unsigned idx)
     desc.value |= idx;
 }
 
+uint32_t G4_SendMsgDescriptor::getSamplerMessageType() const
+{
+    MUST_BE_TRUE(isSampler(), "wrong descriptor type for method");
+    return (getFuncCtrl() >> 12) & 0x1f;
+}
+
 bool G4_SendMsgDescriptor::is16BitInput() const
 {
     return desc.layout.simdMode2 == 1;

visa/Gen4_IR.hpp

@@ -469,6 +469,7 @@ class G4_SendMsgDescriptor
     // for sampler mesasges only
     bool isSampler() const { return getFuncId() == SFID::SAMPLER; }
     bool isCPSEnabled() const { return extDesc.layout.cps != 0; }
+    uint32_t getSamplerMessageType() const;
     bool is16BitInput() const;
     bool is16BitReturn() const;
 

visa/Optimizer.cpp

@@ -426,8 +426,9 @@ void Optimizer::insertHashMovs()
 //
 void Optimizer::cloneSampleInst()
 {
-
-    if (!builder.getOption(vISA_cloneSampleInst))
+    bool cloneSample = builder.getOption(vISA_cloneSampleInst);
+    bool cloneEvaluateSample = builder.getOption(vISA_cloneEvaluateSampleInst);
+    if (!cloneSample && !cloneEvaluateSample)
     {
         return;
     }
@@ -443,18 +444,46 @@ void Optimizer::cloneSampleInst()
             auto inst = *I;
             if (inst->isSend() && inst->asSendInst()->getMsgDesc()->isSampler() && inst->getExecSize() >= builder.getNativeExecSize())
             {
-                if (!hasSample)
+                G4_InstSend* sendInst = inst->asSendInst();
+                bool isEval = sendInst->getMsgDesc()->ResponseLength() == 0;
+                uint32_t messageType = sendInst->getMsgDesc()->getSamplerMessageType();
+                assert(!inst->getPredicate() && "do not handle predicated sampler inst for now");
+                if (!isEval && cloneSample)
                 {
-                    hasSample = true;
-                    auto flagInit = builder.createMov(g4::SIMD1, builder.createDst(tmpFlag->getRegVar(), isSIMD32 ? Type_UD : Type_UW),
-                        builder.createImm(isSIMD32 ? 0x0F0F0F0F : 0x0F0F, isSIMD32 ? Type_UD : Type_UW), InstOpt_WriteEnable, false);
+                    if (!hasSample)
+                    {
+                        hasSample = true;
+                        auto flagInit = builder.createMov(g4::SIMD1, builder.createDst(tmpFlag->getRegVar(), isSIMD32 ? Type_UD : Type_UW),
+                            builder.createImm(isSIMD32 ? 0x0F0F0F0F : 0x0F0F, isSIMD32 ? Type_UD : Type_UW), InstOpt_WriteEnable, false);
+                        bb->insertBefore(I, flagInit);
+                    }
+                    auto newInst = inst->cloneInst();
+                    inst->setPredicate(builder.createPredicate(PredState_Plus, tmpFlag->getRegVar(), 0));
+                    newInst->setPredicate(builder.createPredicate(PredState_Minus, tmpFlag->getRegVar(), 0));
+                    bb->insertAfter(I, newInst);
+                }
+                else if(isEval && cloneEvaluateSample && messageType != 0x1F)
+                {
+                    // 0x1F is the opcode for sampler cache flush
+                    uint32_t newExecSize = (messageType == VISA_3D_SAMPLE_L || messageType == VISA_3D_LD) ? 8 : 1;
+                    uint32_t mask = (1 << newExecSize) - 1;
+                    auto evalTmpFlag = builder.createTempFlag(isSIMD32 ? 2 : 1);
+                    auto flagInit = builder.createMov(g4::SIMD1, builder.createDst(evalTmpFlag->getRegVar(), isSIMD32 ? Type_UD : Type_UW),
+                        builder.createImm(mask, isSIMD32 ? Type_UD : Type_UW), InstOpt_WriteEnable, false);
                     bb->insertBefore(I, flagInit);
+                    inst->setPredicate(builder.createPredicate(PredState_Plus, evalTmpFlag->getRegVar(), 0));
+                    unsigned numInsts = kernel.getSimdSize() / newExecSize;
+                    for (unsigned int i = 1; i < numInsts; i++)
+                    {
+                        auto newInst = inst->cloneInst();
+                        bb->insertAfter(I, newInst);
+                        evalTmpFlag = builder.createTempFlag(isSIMD32 ? 2 : 1);
+                        flagInit = builder.createMov(g4::SIMD1, builder.createDst(evalTmpFlag->getRegVar(), isSIMD32 ? Type_UD : Type_UW),
+                            builder.createImm(mask << (i * newExecSize), isSIMD32 ? Type_UD : Type_UW), InstOpt_WriteEnable, false);
+                        newInst->setPredicate(builder.createPredicate(PredState_Plus, evalTmpFlag->getRegVar(), 0));
+                        bb->insertAfter(I, flagInit);
+                    }
                 }
-                assert(!inst->getPredicate() && "do not handle predicated sampler inst for now");
-                auto newInst = inst->cloneInst();
-                inst->setPredicate(builder.createPredicate(PredState_Plus, tmpFlag->getRegVar(), 0));
-                newInst->setPredicate(builder.createPredicate(PredState_Minus, tmpFlag->getRegVar(), 0));
-                bb->insertAfter(I, newInst);
             }
             I = Next;
         }

visa/include/VISAOptions.def

@@ -234,6 +234,7 @@ DEF_VISA_OPTION(vISA_forceNoMaskWA,                 ET_BOOL,  "-forceNoMaskWA",
 DEF_VISA_OPTION(vISA_DstSrcOverlapWA,               ET_BOOL,  "-dstSrcOverlapWA",  UNUSED, true)
 DEF_VISA_OPTION(vISA_noSendSrcDstOverlap,           ET_BOOL,  "-noSendSrcDstOverlap", UNUSED, false)
 DEF_VISA_OPTION(vISA_cloneSampleInst,               ET_BOOL,  "-cloneSampleInst", UNUSED, false)
+DEF_VISA_OPTION(vISA_cloneEvaluateSampleInst,       ET_BOOL,  "-cloneEvaluateSampleInst", UNUSED, false)
 
 //=== HW debugging options ===
 DEF_VISA_OPTION(vISA_GenerateDebugInfo,   ET_BOOL,  "-generateDebugInfo", UNUSED, false)