Back-port of some patches from llvm16 version of Translator. (Khronos…

…Group#2371)

* Backport from llvm16 branch: Support FPGA function !spirv.Decoration metadata

Supporting the new FPGA kernel attribute pipelined needs the translator
to handle !spirv.Decoration metadata on functions as well as on global
variables. If the MD is found on the function, call the existing
translation of metadata.

* Backport from llvm16 branch: Implement RegisterMapInterfaceINTEL execution mode

This PR adds the new execution mode RegisterMapInterfaceINTEL, see the Khronos
SPIRV spec here: KhronosGroup/SPIRV-Registry#176
This execution mode allows specifying a 'register' based interface for FPGA kernels.

* Backport from llvm16 branch: Implement SPIR-V support for max_reinvocation_delay attribute

The attribute generates the llvm.loop.intel.max_reinvocation_delay.count metadata in LLVM IR.
There is one positive integer literal operand denoting the maximum number of cycles allowed
between loop invocations.

* Backport from llvm18 branch: Support sycl_ext_oneapi_prefetch

Handle new properties and decorate prefetch's arg.

lib/SPIRV/PreprocessMetadata.cpp

@@ -224,18 +224,38 @@ void PreprocessMetadataBase::visit(Module *M) {
     if (MDNode *Interface =
             Kernel.getMetadata(kSPIR2MD::IntelFPGAIPInterface)) {
       std::set<std::string> InterfaceStrSet;
-      // Default mode is 'csr' aka !ip_interface !N
-      //                           !N = !{!”csr”}
-      // don't emit any particular SPIR-V for it
+      for (size_t I = 0; I != Interface->getNumOperands(); ++I)
+        InterfaceStrSet.insert(getMDOperandAsString(Interface, I).str());
+
+      // ip_interface metadata will either have Register Map metadata or
+      // Streaming metadata.
+      //
+      // Register Map mode metadata:
+      // Not 'WaitForDoneWrite' mode (to be mapped on '0' literal)
+      // !ip_interface !N
+      // !N = !{!"csr"}
+      // 'WaitForDoneWrite' mode (to be mapped on '1' literal)
+      // !ip_interface !N
+      // !N = !{!"csr", !"wait_for_done_write"}
+      if (InterfaceStrSet.find("csr") != InterfaceStrSet.end()) {
+        int32_t InterfaceMode = 0;
+        if (InterfaceStrSet.find("wait_for_done_write") !=
+            InterfaceStrSet.end())
+          InterfaceMode = 1;
+        EM.addOp()
+            .add(&Kernel)
+            .add(spv::ExecutionModeRegisterMapInterfaceINTEL)
+            .add(InterfaceMode)
+            .done();
+      }
+
       // Streaming mode metadata be like:
       // Not 'stall free' mode (to be mapped on '0' literal)
       // !ip_interface !N
       // !N = !{!"streaming"}
       // 'stall free' mode (to be mapped on '1' literal)
       // !ip_interface !N
       // !N = !{!"streaming", !"stall_free_return"}
-      for (size_t I = 0; I != Interface->getNumOperands(); ++I)
-        InterfaceStrSet.insert(getMDOperandAsString(Interface, I).str());
       if (InterfaceStrSet.find("streaming") != InterfaceStrSet.end()) {
         int32_t InterfaceMode = 0;
         if (InterfaceStrSet.find("stall_free_return") != InterfaceStrSet.end())
@@ -319,17 +339,16 @@ void PreprocessMetadataBase::preprocessVectorComputeMetadata(Module *M,
           FPRoundingModeExecModeMap::map(getFPRoundingMode(Mode));
       spv::ExecutionMode ExecFloatMode =
           FPOperationModeExecModeMap::map(getFPOperationMode(Mode));
-      VCFloatTypeSizeMap::foreach (
-          [&](VCFloatType FloatType, unsigned TargetWidth) {
-            EM.addOp().add(&F).add(ExecRoundMode).add(TargetWidth).done();
-            EM.addOp().add(&F).add(ExecFloatMode).add(TargetWidth).done();
-            EM.addOp()
-                .add(&F)
-                .add(FPDenormModeExecModeMap::map(
-                    getFPDenormMode(Mode, FloatType)))
-                .add(TargetWidth)
-                .done();
-          });
+      VCFloatTypeSizeMap::foreach ([&](VCFloatType FloatType,
+                                       unsigned TargetWidth) {
+        EM.addOp().add(&F).add(ExecRoundMode).add(TargetWidth).done();
+        EM.addOp().add(&F).add(ExecFloatMode).add(TargetWidth).done();
+        EM.addOp()
+            .add(&F)
+            .add(FPDenormModeExecModeMap::map(getFPDenormMode(Mode, FloatType)))
+            .add(TargetWidth)
+            .done();
+      });
     }
     if (Attrs.hasFnAttr(kVCMetadata::VCSLMSize)) {
       SPIRVWord SLMSize = 0;

lib/SPIRV/SPIRVReader.cpp

@@ -932,6 +932,14 @@ void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
           "llvm.loop.intel.loopcount_avg", static_cast<int64_t>(LoopCountAvg)));
     }
   }
+  if (LC & spv::internal::LoopControlMaxReinvocationDelayINTELMask) {
+    Metadata.push_back(llvm::MDNode::get(
+        *Context, getMetadataFromNameAndParameter(
+                      "llvm.loop.intel.max_reinvocation_delay.count",
+                      LoopControlParameters[NumParam++])));
+    assert(NumParam <= LoopControlParameters.size() &&
+           "Missing loop control parameter!");
+  }
   llvm::MDNode *Node = llvm::MDNode::get(*Context, Metadata);
 
   // Set the first operand to refer itself
@@ -4091,6 +4099,27 @@ bool SPIRVToLLVM::transMetadata() {
       F->setMetadata(kSPIR2MD::FmaxMhz,
                      getMDNodeStringIntVec(Context, EM->getLiterals()));
     }
+    // Generate metadata for Intel FPGA register map interface
+    if (auto *EM =
+            BF->getExecutionMode(ExecutionModeRegisterMapInterfaceINTEL)) {
+      std::vector<uint32_t> InterfaceVec = EM->getLiterals();
+      assert(InterfaceVec.size() == 1 &&
+             "Expected RegisterMapInterfaceINTEL to have exactly 1 literal");
+      std::vector<Metadata *> InterfaceMDVec =
+          [&]() -> std::vector<Metadata *> {
+        switch (InterfaceVec[0]) {
+        case 0:
+          return {MDString::get(*Context, "csr")};
+        case 1:
+          return {MDString::get(*Context, "csr"),
+                  MDString::get(*Context, "wait_for_done_write")};
+        default:
+          llvm_unreachable("Invalid register map interface mode");
+        }
+      }();
+      F->setMetadata(kSPIR2MD::IntelFPGAIPInterface,
+                     MDNode::get(*Context, InterfaceMDVec));
+    }
     // Generate metadata for Intel FPGA streaming interface
     if (auto *EM = BF->getExecutionMode(
             internal::ExecutionModeStreamingInterfaceINTEL)) {

lib/SPIRV/SPIRVWriter.cpp

@@ -973,6 +973,8 @@ void LLVMToSPIRVBase::transVectorComputeMetadata(Function *F) {
   }
 }
 
+static void transMetadataDecorations(Metadata *MD, SPIRVEntry *Target);
+
 void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,
                                                 Function *F) {
   if (MDNode *StallEnable = F->getMetadata(kSPIR2MD::StallEnable)) {
@@ -1021,6 +1023,10 @@ void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,
       BF->addDecorate(new SPIRVDecoratePipelineEnableINTEL(BF, !Disable));
     }
   }
+
+  // In addition, process the decorations on the function
+  if (auto *FDecoMD = F->getMetadata(SPIRV_MD_DECORATIONS))
+    transMetadataDecorations(FDecoMD, BF);
 }
 
 void LLVMToSPIRVBase::transAuxDataInst(SPIRVFunction *BF, Function *F) {
@@ -1563,6 +1569,14 @@ LLVMToSPIRVBase::getLoopControl(const BranchInst *Branch,
           BM->addCapability(CapabilityFPGALoopControlsINTEL);
           LoopCount.Avg = getMDOperandAsInt(Node, 1);
           LoopControl |= spv::internal::LoopControlLoopCountINTELMask;
+        } else if (S == "llvm.loop.intel.max_reinvocation_delay.count") {
+          BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls);
+          BM->addCapability(CapabilityFPGALoopControlsINTEL);
+          size_t I = getMDOperandAsInt(Node, 1);
+          ParametersToSort.emplace_back(
+              spv::internal::LoopControlMaxReinvocationDelayINTELMask, I);
+          LoopControl |=
+              spv::internal::LoopControlMaxReinvocationDelayINTELMask;
         }
       }
     }
@@ -2820,6 +2834,12 @@ using DecorationsInfoVec =
 struct AnnotationDecorations {
   DecorationsInfoVec MemoryAttributesVec;
   DecorationsInfoVec MemoryAccessesVec;
+  DecorationsInfoVec CacheControlVec;
+
+  bool empty() {
+    return (MemoryAttributesVec.empty() && MemoryAccessesVec.empty() &&
+            CacheControlVec.empty());
+  }
 };
 
 struct IntelLSUControlsInfo {
@@ -2996,6 +3016,8 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
       BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_memory_accesses);
   const bool AllowFPGAMemAttr = BM->isAllowedToUseExtension(
       ExtensionID::SPV_INTEL_fpga_memory_attributes);
+  const bool AllowCacheControls =
+      BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_cache_controls);
 
   bool ValidDecorationFound = false;
   DecorationsInfoVec DecorationsVec;
@@ -3014,8 +3036,15 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
       std::vector<std::string> DecValues;
       if (tryParseAnnotationDecoValues(ValueStr, DecValues)) {
         ValidDecorationFound = true;
-        DecorationsVec.emplace_back(static_cast<Decoration>(DecorationKind),
-                                    std::move(DecValues));
+
+        if (AllowCacheControls &&
+            DecorationKind == internal::DecorationCacheControlLoadINTEL) {
+          Decorates.CacheControlVec.emplace_back(
+              static_cast<Decoration>(DecorationKind), std::move(DecValues));
+        } else {
+          DecorationsVec.emplace_back(static_cast<Decoration>(DecorationKind),
+                                      std::move(DecValues));
+        }
       }
       continue;
     }
@@ -3117,7 +3146,7 @@ void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
       continue;
     }
 
-    switch (I.first) {
+    switch (static_cast<size_t>(I.first)) {
     case DecorationUserSemantic:
       M->getErrorLog().checkError(I.second.size() == 1,
                                   SPIRVEC_InvalidLlvmModule,
@@ -3201,6 +3230,21 @@ void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
         E->addDecorate(I.first, Result);
       }
     } break;
+    case spv::internal::DecorationCacheControlLoadINTEL: {
+      if (M->isAllowedToUseExtension(ExtensionID::SPV_INTEL_cache_controls)) {
+        M->getErrorLog().checkError(
+            I.second.size() == 2, SPIRVEC_InvalidLlvmModule,
+            "CacheControlLoadINTEL requires exactly 2 extra operands");
+        SPIRVWord CacheLevel = 0;
+        SPIRVWord CacheControl = 0;
+        StringRef(I.second[0]).getAsInteger(10, CacheLevel);
+        StringRef(I.second[1]).getAsInteger(10, CacheControl);
+        E->addDecorate(new SPIRVDecorateCacheControlLoadINTEL(
+            E, CacheLevel,
+            static_cast<internal::LoadCacheControlINTEL>(CacheControl)));
+      }
+    }
+
     default:
       // Other decorations are either not supported by the translator or
       // handled in other places.
@@ -3931,15 +3975,17 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
       } else {
         // Memory accesses to a standalone pointer variable
         auto *DecSubj = transValue(II->getArgOperand(0), BB);
-        if (Decorations.MemoryAccessesVec.empty())
+        if (Decorations.empty())
           DecSubj->addDecorate(new SPIRVDecorateUserSemanticAttr(
               DecSubj, AnnotationString.c_str()));
-        else
+        else {
           // Apply the LSU parameter decoration to the pointer result of an
           // instruction. Note it's the address to the accessed memory that's
           // loaded from the original pointer variable, and not the value
           // accessed by the latter.
           addAnnotationDecorations(DecSubj, Decorations.MemoryAccessesVec);
+          addAnnotationDecorations(DecSubj, Decorations.CacheControlVec);
+        }
         II->replaceAllUsesWith(II->getOperand(0));
       }
     }
@@ -4762,6 +4808,19 @@ bool LLVMToSPIRVBase::transExecutionMode() {
       case spv::ExecutionModeNumSIMDWorkitemsINTEL:
       case spv::ExecutionModeSchedulerTargetFmaxMhzINTEL:
       case spv::ExecutionModeMaxWorkDimINTEL:
+      case spv::ExecutionModeRegisterMapInterfaceINTEL: {
+        if (!BM->isAllowedToUseExtension(
+                ExtensionID::SPV_INTEL_kernel_attributes))
+          break;
+        AddSingleArgExecutionMode(static_cast<ExecutionMode>(EMode));
+        BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes);
+        BM->addCapability(CapabilityFPGAKernelAttributesINTEL);
+        // RegisterMapInterfaceINTEL mode is defined by the
+        // CapabilityFPGAKernelAttributesv2INTEL capability and that
+        // capability implicitly defines CapabilityFPGAKernelAttributesINTEL
+        if (EMode == spv::ExecutionModeRegisterMapInterfaceINTEL)
+          BM->addCapability(CapabilityFPGAKernelAttributesv2INTEL);
+      } break;
       case spv::internal::ExecutionModeStreamingInterfaceINTEL: {
         if (!BM->isAllowedToUseExtension(
                 ExtensionID::SPV_INTEL_kernel_attributes))

lib/SPIRV/libSPIRV/SPIRVEntry.cpp

@@ -626,6 +626,7 @@ void SPIRVExecutionMode::decode(std::istream &I) {
   case ExecutionModeMaxWorkDimINTEL:
   case ExecutionModeNumSIMDWorkitemsINTEL:
   case ExecutionModeSchedulerTargetFmaxMhzINTEL:
+  case ExecutionModeRegisterMapInterfaceINTEL:
   case internal::ExecutionModeStreamingInterfaceINTEL:
     WordLiterals.resize(1);
     break;

lib/SPIRV/libSPIRV/SPIRVEnum.h

@@ -273,6 +273,8 @@ template <> inline void SPIRVMap<SPIRVExecutionModeKind, SPIRVCapVec>::init() {
                {CapabilityVectorComputeINTEL});
   ADD_VEC_INIT(internal::ExecutionModeFastCompositeKernelINTEL,
                {internal::CapabilityFastCompositeINTEL});
+  ADD_VEC_INIT(ExecutionModeRegisterMapInterfaceINTEL,
+               {CapabilityFPGAKernelAttributesv2INTEL});
   ADD_VEC_INIT(internal::ExecutionModeStreamingInterfaceINTEL,
                {CapabilityFPGAKernelAttributesINTEL});
   ADD_VEC_INIT(ExecutionModeNamedBarrierCountINTEL,

lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h

@@ -71,6 +71,7 @@ inline bool isValid(spv::ExecutionModel V) {
   case ExecutionModelClosestHitKHR:
   case ExecutionModelMissKHR:
   case ExecutionModelCallableKHR:
+  case ExecutionModeRegisterMapInterfaceINTEL:
   case internal::ExecutionModeStreamingInterfaceINTEL:
     return true;
   default:

lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h

@@ -568,6 +568,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilityFPGALoopControlsINTEL, "FPGALoopControlsINTEL");
   add(CapabilityKernelAttributesINTEL, "KernelAttributesINTEL");
   add(CapabilityFPGAKernelAttributesINTEL, "FPGAKernelAttributesINTEL");
+  add(CapabilityFPGAKernelAttributesv2INTEL, "FPGAKernelAttributesv2INTEL");
   add(CapabilityFPGAMemoryAccessesINTEL, "FPGAMemoryAccessesINTEL");
   add(CapabilityFPGAClusterAttributesINTEL, "FPGAClusterAttributesINTEL");
   add(CapabilityLoopFuseINTEL, "LoopFuseINTEL");

lib/SPIRV/libSPIRV/spirv_internal.hpp

@@ -123,7 +123,10 @@ enum InternalExecutionMode {
   IExecModeStreamingInterfaceINTEL = 6154
 };
 
-enum InternalLoopControlMask { ILoopControlLoopCountINTELMask = 0x1000000 };
+enum InternalLoopControlMask {
+  ILoopControlLoopCountINTELMask = 0x1000000,
+  ILoopControlMaxReinvocationDelayINTELMask = 0x2000000
+};
 
 constexpr LinkageType LinkageTypeInternal =
     static_cast<LinkageType>(ILTInternal);
@@ -293,6 +296,8 @@ constexpr ExecutionMode ExecutionModeStreamingInterfaceINTEL =
 
 constexpr LoopControlMask LoopControlLoopCountINTELMask =
     static_cast<LoopControlMask>(ILoopControlLoopCountINTELMask);
+static const LoopControlMask LoopControlMaxReinvocationDelayINTELMask =
+    static_cast<LoopControlMask>(ILoopControlMaxReinvocationDelayINTELMask);
 
 } // namespace internal
 } // namespace spv