Adapation for Merging Devel

Author: runwangdl

Deeploy/Targets/PULPOpen/Templates/FloatGELUTemplate.py

@@ -28,4 +28,14 @@
 referenceTemplate = NodeTemplate("""
 // GELU (Name: ${nodeName}, Op: ${nodeOp})
 PULP_GELU_fp${data_in_type.referencedType.typeWidth}_fp${data_out_type.referencedType.typeWidth}(${data_in}, ${data_out}, ${size});
+""")
+
+referenceGradTemplate = NodeTemplate("""
+// GELU Parallel (Name: ${nodeName}, Op: ${nodeOp})
+int8_t ${nodeName}_core_id = pi_core_id();
+int8_t ${nodeName}_log2Core = log2(NUM_CORES);
+int16_t ${nodeName}_chunk = (${size} >> ${nodeName}_log2Core) + ((${size} & (NUM_CORES-1))!=0);
+int16_t ${nodeName}_chunk_start = MIN(${nodeName}_chunk*${nodeName}_core_id, ${size});
+int16_t ${nodeName}_chunk_stop = MIN(${nodeName}_chunk_start + ${nodeName}_chunk, ${size});
+GELU_fp${data_in_type.referencedType.typeWidth}_fp${grad_out_type.referencedType.typeWidth}_sigmoid_grad_chunk(${grad_in}, ${data_in}, ${grad_out}, ${nodeName}_chunk_start, ${nodeName}_chunk_stop);
 """)

Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py

@@ -22,16 +22,38 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the Licens
-from Deeploy.DeeployTypes import NodeTemplate
+from Deeploy.DeeployTypes import NodeTemplate, NetworkContext, OperatorRepresentation
+from Deeploy.AbstractDataTypes import float32_tPtr
+from typing import Tuple, Dict, List
 
-referenceTemplate = NodeTemplate("""
+class PULPFloatGEMMTemplate(NodeTemplate):
+    
+    def __init__(self, templateStr):
+        super().__init__(templateStr)
+    
+    def alignToContext(self, ctxt: NetworkContext,
+                      operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]:
+    
+        if 'C' not in operatorRepresentation or operatorRepresentation['C'] is None:
+            # No bias case - set C to NULL and provide a default type
+            operatorRepresentation['C'] = None
+            operatorRepresentation['C_type'] = float32_tPtr  # Default to fp32 type
+        
+        return ctxt, operatorRepresentation, []
+
+referenceTemplate = PULPFloatGEMMTemplate("""
 // GEMM (Name: ${nodeName}, Op: ${nodeOp})
 ${A_type.typeName} ref_${data_out}_${A} = ${A};
 ${B_type.typeName} ref_${data_out}_${B} = ${B};
+% if C is not None:
 ${C_type.typeName} ref_${data_out}_${C} = ${C};
+% else:
+${C_type.typeName} ref_${data_out}_C = NULL;
+% endif
 ${data_out_type.typeName} ref_${data_out}_${data_out} = ${data_out};
 
 for(uint32_t i=0; i<${batch}; i++){
+    % if C is not None:
     PULP_Gemm_fp${A_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_fp${C_type.referencedType.typeWidth}_fp${data_out_type.referencedType.typeWidth}(
         ref_${data_out}_${A},
         ref_${data_out}_${B},
@@ -43,10 +65,25 @@
         ${transA},
         ${transB}
     );
+    % else:
+    PULP_Gemm_fp${A_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_fp${C_type.referencedType.typeWidth}_fp${data_out_type.referencedType.typeWidth}(
+        ref_${data_out}_${A},
+        ref_${data_out}_${B},
+        NULL,
+        ref_${data_out}_${data_out},
+        ${M},
+        ${N},
+        ${O},
+        ${transA},
+        ${transB}
+    );
+    % endif
     
     ref_${data_out}_${A} += ${M} * ${N};
     ref_${data_out}_${B} += ${N} * ${O};
+    % if C is not None:
     ref_${data_out}_${C} += ${M} * ${O};
+    % endif
     ref_${data_out}_${data_out} += ${M} * ${O};
 }
 """)

Deeploy/Targets/PULPOpen/Templates/FloatLayernormTemplate.py

@@ -36,4 +36,38 @@
     ${size}, 
     ${lastDimLength}
 );
+""")
+
+referenceGradTemplate = NodeTemplate("""
+// FloatLayernormGrad Parallel (Name: ${nodeName}, Op: ${nodeOp})
+
+int8_t ${nodeName}_core_id = pi_core_id();
+int8_t ${nodeName}_log2Core = log2(NUM_CORES);
+
+int32_t ${nodeName}_seq_length = ${size} / ${lastDimLength};
+int32_t ${nodeName}_chunk = (${nodeName}_seq_length >> ${nodeName}_log2Core) + 
+                          ((${nodeName}_seq_length & (NUM_CORES-1)) != 0);
+int32_t ${nodeName}_start = MIN(${nodeName}_chunk * ${nodeName}_core_id, ${nodeName}_seq_length);
+int32_t ${nodeName}_end = MIN(${nodeName}_start + ${nodeName}_chunk, ${nodeName}_seq_length);
+ 
+int32_t ${nodeName}_elem_start = ${nodeName}_start * ${lastDimLength};
+int32_t ${nodeName}_elem_end = ${nodeName}_end * ${lastDimLength};
+int32_t ${nodeName}_elem_count = ${nodeName}_elem_end - ${nodeName}_elem_start;
+ 
+const float* ${nodeName}_grad_in_ptr = ${grad_in} + ${nodeName}_elem_start;
+const float* ${nodeName}_data_in_ptr = ${data_in} + ${nodeName}_elem_start;
+float* ${nodeName}_grad_out_ptr = ${grad_out} + ${nodeName}_elem_start;
+ 
+if (${nodeName}_elem_count > 0) {
+  LayernormGrad_fp${grad_in_type.referencedType.typeWidth}_fp${grad_out_type.referencedType.typeWidth}(
+      ${nodeName}_grad_in_ptr,     // Upstream gradient (dy)
+      ${nodeName}_data_in_ptr,     // Original input (x)
+      ${nodeName}_grad_out_ptr,    // Output gradient (dx)
+      ${weight},                   // Input Scale parameter
+      ${bias},                     // Input Bias parameter
+      ${epsilon},                  // Epsilon for numerical stability
+      ${nodeName}_elem_count,      // Number of elements to process
+      ${lastDimLength}             // Size of the feature dimension
+  );
+}
 """)

Deeploy/Targets/Redmule/Templates/GEMMTemplate.py

@@ -22,9 +22,26 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the Licens
-from Deeploy.DeeployTypes import NodeTemplate
+from Deeploy.DeeployTypes import NodeTemplate, NetworkContext, OperatorRepresentation
+from Deeploy.AbstractDataTypes import float32_tPtr
+from typing import Tuple, Dict, List
 
-referenceTemplate = NodeTemplate("""
+class RedMuleGEMMTemplate(NodeTemplate):
+    
+    def __init__(self, templateStr):
+        super().__init__(templateStr)
+    
+    def alignToContext(self, ctxt: NetworkContext,
+                      operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]:
+    
+        if 'C' not in operatorRepresentation or operatorRepresentation['C'] is None:
+            # No bias case - set C to NULL and provide a default type
+            operatorRepresentation['C'] = None
+            operatorRepresentation['C_type'] = float32_tPtr  # Default to fp32 type
+        
+        return ctxt, operatorRepresentation, []
+
+referenceTemplate = RedMuleGEMMTemplate("""
 // GEMM using RedMule hardware accelerator (Name: ${nodeName}, Op: ${nodeOp})
 
 int8_t ${nodeName}_core_id = pi_core_id();
@@ -33,10 +50,13 @@
     for(uint32_t b=0; b<${batch}; b++) {
         ${A_type.typeName} batch_A = ${A} + b * ${M} * ${N};
         ${B_type.typeName} batch_B = ${B} + b * ${N} * ${O};
+        % if C is not None:
         ${C_type.typeName} batch_C = ${C} + b * ${M} * ${O};
+        % endif
         ${data_out_type.typeName} batch_out = ${data_out} + b * ${M} * ${O};
         
-        % if beta == 0:
+        % if C is None or beta == 0:
+        // No bias or beta=0: use MatMul
         MatMul_fp${A_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_Redmule(
             (const float32_t *) batch_A,
             (const float32_t *) batch_B,
@@ -46,6 +66,7 @@
             ${O}
         );
         % else:
+        // With bias and beta!=0: use Gemm
         Gemm_fp${A_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_Redmule(
             (const float32_t *) batch_A,
             (const float32_t *) batch_B,
@@ -58,5 +79,4 @@
         % endif
     }
 }
-"""
-)
+""")

Deeploy/Targets/Redmule/TileConstraints/GEMMTileConstraint.py

@@ -1,4 +1,3 @@
-
 # ----------------------------------------------------------------------
 #
 # File: GEMMTileConstraint.py
@@ -8,8 +7,7 @@
 # Copyright (C) 2023, ETH Zurich and University of Bologna.
 #
 # Author:
-# - Victor Jung, [email protected], ETH Zurich
-# - Moritz Scherer, [email protected], ETH Zurich
+# - Run Wang, ETH Zurich
 #
 # ----------------------------------------------------------------------
 # SPDX-License-Identifier: Apache-2.0
@@ -46,16 +44,23 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw
         # Get to-be-tiled tensor's buffers
         bufferA = ctxt.lookup(name = parseDict['A'])
         bufferB = ctxt.lookup(name = parseDict['B'])
-        bufferC = ctxt.lookup(name = parseDict['C'])
         outputBuffer = ctxt.lookup(name = parseDict['data_out'])
 
+        # Check if bias exists
+        has_bias = 'C' in parseDict and parseDict['C'] is not None
+        if has_bias:
+            bufferC = ctxt.lookup(name = parseDict['C'])
+
         # Add I/O dimensions to the model as variables
-        for bufferName in [bufferA.name, bufferB.name, bufferC.name, outputBuffer.name]:
+        tensor_names = [bufferA.name, bufferB.name, outputBuffer.name]
+        if has_bias:
+            tensor_names.append(bufferC.name)
+            
+        for bufferName in tensor_names:
             tilerModel.addTensorDimToModel(ctxt, bufferName)
 
         dimOffsetA = len(bufferA.shape) - 2
         dimOffsetB = len(bufferB.shape) - 2
-        dimOffsetC = len(bufferC.shape) - 2
         dimOffsetOut = len(outputBuffer.shape) - 2
 
         AFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, dimIdx = dimOffsetA + parseDict['transA'])
@@ -74,10 +79,13 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw
         # Add GEMM Geometrical constraints
         tilerModel.addConstraint(ASecondDimVar == BFirstDimVar)
 
-        addDimVar_1 = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = dimOffsetC)
-        addDimVar_2 = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = dimOffsetC + 1)
-        tilerModel.addConstraint(outputFirstDimVar == addDimVar_1)
-        tilerModel.addConstraint(outputSecondDimVar == addDimVar_2)
+        # Add bias constraints only if bias exists
+        if has_bias:
+            dimOffsetC = len(bufferC.shape) - 2
+            addDimVar_1 = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = dimOffsetC)
+            addDimVar_2 = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = dimOffsetC + 1)
+            tilerModel.addConstraint(outputFirstDimVar == addDimVar_1)
+            tilerModel.addConstraint(outputSecondDimVar == addDimVar_2)
 
         return tilerModel
 
@@ -114,7 +122,15 @@ def serializeTilingSolution(
             operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]:
         outputCubes = [cube.rectangle for cube in absoluteOutputCubes]
 
-        addrNames = ['A', 'B', 'C', 'data_out']
+        # Check if bias exists
+        has_bias = 'C' in operatorRepresentation and operatorRepresentation['C'] is not None
+        
+        # Adjust address names based on bias existence
+        if has_bias:
+            addrNames = ['A', 'B', 'C', 'data_out']
+        else:
+            addrNames = ['A', 'B', 'data_out']
+            
         inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel,
                                                                   operatorRepresentation, addrNames)
 
@@ -169,11 +185,13 @@ def serializeTilingSolution(
             else:
                 BCube = HyperRectangle((BatchOffset, BOffset, OOffset, NOffset), (BatchSize, BSize, OSize, NSize))
 
-            CCube = HyperRectangle(cube.offset, cube.dims)
-
             inputACubes.append(ACube)
             inputBCubes.append(BCube)
-            inputAddCubes.append(CCube)
+            
+            # Only add bias cubes if bias exists
+            if has_bias:
+                CCube = HyperRectangle(cube.offset, cube.dims)
+                inputAddCubes.append(CCube)
 
         inputLoadSchedule = []
         outputLoadSchedule = []
@@ -187,12 +205,17 @@ def serializeTilingSolution(
             "batch": PointerClass(uint8_t)
         }
 
-        for a, b, c in zip(inputACubes, inputBCubes, inputAddCubes):
-            inputLoadSchedule.append({"A": a, "B": b, "C": c})
+        # Create input load schedule based on bias existence
+        if has_bias:
+            for a, b, c in zip(inputACubes, inputBCubes, inputAddCubes):
+                inputLoadSchedule.append({"A": a, "B": b, "C": c})
+        else:
+            for a, b in zip(inputACubes, inputBCubes):
+                inputLoadSchedule.append({"A": a, "B": b})
 
         for out in outputCubes:
             outputLoadSchedule.append({"data_out": out})
 
         schedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule)
 
-        return VariableReplacementScheme(replacements, replacementTypes), schedule
+        return VariableReplacementScheme(replacements, replacementTypes), schedule

Deeploy/TilingExtension/CodeTransformationPasses/TilingPrototypes.py

@@ -42,11 +42,11 @@ class TilingMetaInfo:
 _CodeSegmentType = List[CodeSnippet]
 
 _measureCycles = NodeTemplate("""
-${nodeName}_${measurementName}_measurements[${tileIdx}] = getCycles();
+${nodeName}_${measurementName}_t[${tileIdx}] = getCycles();
 """)
 
 _measurementArrayDeclaration = NodeTemplate("""
-uint32_t ${nodeName}_${measurementName}_measurements[${numTiles}];
+static uint32_t ${nodeName}_${measurementName}_t[${numTiles}];
 """)
 
 _printPrefixAndSufixDeclaration = NodeTemplate("""
@@ -74,7 +74,7 @@ class TilingMetaInfo:
 """)
 _printCycleDifference = NodeTemplate(r"""
 printf("%s%u] %s%u%s", ${nodeName}_prefix,${tileIdx},"${flavorStr}", \
-${nodeName}_${endMeasurementName}_measurements[${tileIdx}] - ${nodeName}_${startMeasurementName}_measurements[${tileIdx}],${nodeName}_suffix);
+${nodeName}_${endMeasurementName}_t[${tileIdx}] - ${nodeName}_${startMeasurementName}_t[${tileIdx}],${nodeName}_suffix);
 """)
 
 _printLoopTeardown = NodeTemplate("""