Skip to content

Optimize RVV backend for Deconvolution and QuantizedAdd #4547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jxgxxx wants to merge 1 commit into
base: master
Choose a base branch
from
+148 −10
Open

Optimize RVV backend for Deconvolution and QuantizedAdd #4547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 22 additions & 10 deletions source/backend/cpu/CPUDeconvolution.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@
//#define MNN_OPEN_TIME_TRACE
#include <MNN/AutoTime.hpp>

#if defined(__riscv) && defined(MNN_USE_RVV)
extern void MNNDeconvTransformWeightC4_RVV(const float* src, float* dst, int outputCount, int srcCount, int area);
#endif

namespace MNN {

CPUDeconvolutionBasic::CPUDeconvolutionBasic(int inputChannel, const Op* convOp, Backend* b)
Expand All @@ -42,17 +46,25 @@ ErrorCode CPUDeconvolutionBasic::onResize(const std::vector<Tensor*>& inputs, co
static void _transformWeight(const uint8_t* tempWeight, uint8_t* dest, int outputCount, int srcCount, int fh, int fw,
uint8_t* cache, const CoreFunctions* core) {
auto outputC4 = UP_DIV(outputCount, core->pack);
int offset[] = {
(int)(fw * fh),
(int)(fw * fh),
};
// c, n, h, w-> c, n/4 * 4, h, w
for (int c=0; c<srcCount; ++c) {
auto dst = cache + c * outputC4 * fw * fh * core->pack * core->bytes;
auto src = tempWeight + c * outputCount * fw * fh * core->bytes;
core->MNNPackCUnit((float*)dst, (const float*)src, fw*fh, outputCount, offset);
#if defined(__riscv) && defined(MNN_USE_RVV)
if (core->supportRVV && core->bytes == 4 && core->pack == 4) {
MNNDeconvTransformWeightC4_RVV((const float*)tempWeight, (float*)cache, outputCount, srcCount, fh * fw);
} else
#endif
{
int offset[] = {
(int)(fw * fh),
(int)(fw * fh),
};
// c, n, h, w-> c, n/4 * 4, h, w
for (int c = 0; c < srcCount; ++c) {
auto dst = cache + c * outputC4 * fw * fh * core->pack * core->bytes;
auto src = tempWeight + c * outputCount * fw * fh * core->bytes;
core->MNNPackCUnit((float*)dst, (const float*)src, fw * fh, outputCount, offset);
}
}
//printf("%d - %d - %d - %d\n", outputCount, srcCount, fh, fw);

// printf("%d - %d - %d - %d\n", outputCount, srcCount, fh, fw);
core->MNNPackForMatMul_B((float*)dest, (const float*)cache, outputC4 * fw * fh * core->pack, 1, srcCount, false);
}
std::shared_ptr<DeconvolutionResource> CPUDeconvolution::makeResource(int srcCount, const Op *convOp, Backend* backend, bool dynamic) {
Expand Down
28 changes: 28 additions & 0 deletions source/backend/cpu/CPUQuantizedAdd.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@
#include "core/Concurrency.h"
#include "core/Macro.h"

#if defined(__riscv) && defined(MNN_USE_RVV)
extern void CPUQuantizedAdd_RVV(const uint8_t* input1Data, const uint8_t* input2Data, uint8_t* outputData, size_t size,
int32_t input1Offset, int32_t input2Offset, int32_t outputOffset,
int32_t leftShiftResult1, int32_t leftShiftResult2, int32_t input1Multiplier,
int32_t input2Multiplier, int32_t rightShift1, int32_t rightShift2,
int32_t leftShiftOut, int32_t outputMultiplier, int32_t rightShiftOut,
int32_t outputActivationMin, int32_t outputActivationMax);
#endif

namespace MNN {

CPUQuantizedAdd::CPUQuantizedAdd(Backend *backend, const Op *op) : Execution(backend) {
Expand Down Expand Up @@ -88,6 +97,25 @@ ErrorCode CPUQuantizedAdd::onExecute(const std::vector<MNN::Tensor *> &inputs,
int threadNumber = std::max(((CPUBackend *)backend())->threadNumber(), 1);
int countUnit = UP_DIV(size, threadNumber);

#if defined(__riscv) && defined(MNN_USE_RVV)
if (((CPUBackend*)backend())->functions()->supportRVV) {
MNN_CONCURRENCY_BEGIN(tId, threadNumber) {
int realDstCount = (int)ALIMIN(size - tId * countUnit, countUnit);
uint8_t* curInput1Data = input1Data + tId * countUnit;
uint8_t* curInput2Data = input2Data + tId * countUnit;
uint8_t* curOutputData = outputData + tId * countUnit;
if (realDstCount > 0) {
CPUQuantizedAdd_RVV(curInput1Data, curInput2Data, curOutputData, (size_t)realDstCount, mInput1Offset,
mInput2Offset, mOutputOffset, mLeftShiftResult1, mLeftShiftResult2,
mInput1Multiplier, mInput2Multiplier, mRightShift1, mRightShift2, mLeftShiftOut,
mOutputMultiplier, mRightShiftOut, mOutputActivationMin, mOutputActivationMax);
}
}
MNN_CONCURRENCY_END();
return NO_ERROR;
}
#endif

MNN_CONCURRENCY_BEGIN(tId, threadNumber) {
int realDstCount = (int)ALIMIN(size - tId * countUnit, countUnit);
uint8_t *curInput1Data = input1Data + tId * countUnit;
Expand Down
62 changes: 62 additions & 0 deletions source/backend/cpu/riscv/rvv/CPUQuantizedAddOnExecute.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#include <riscv_vector.h>
#include <stddef.h>
#include <stdint.h>

#ifndef __RISCV_VXRM_RNU
#define __RISCV_VXRM_RNU 0
#endif

void CPUQuantizedAdd_RVV(const uint8_t* input1Data, const uint8_t* input2Data, uint8_t* outputData, size_t size,
int32_t input1Offset, int32_t input2Offset, int32_t outputOffset,
int32_t leftShiftResult1, int32_t leftShiftResult2, int32_t input1Multiplier,
int32_t input2Multiplier, int32_t rightShift1, int32_t rightShift2, int32_t leftShiftOut,
int32_t outputMultiplier, int32_t rightShiftOut, int32_t outputActivationMin,
int32_t outputActivationMax) {
while (size > 0) {
size_t vl = __riscv_vsetvl_e32m8(size);

vuint8m2_t input1U8 = __riscv_vle8_v_u8m2(input1Data, vl);
vuint16m4_t input1U16 = __riscv_vwaddu_vx_u16m4(input1U8, 0, vl);
vuint32m8_t input1U32 = __riscv_vwaddu_vx_u32m8(input1U16, 0, vl);
vint32m8_t input1 = __riscv_vreinterpret_v_u32m8_i32m8(input1U32);

vuint8m2_t input2U8 = __riscv_vle8_v_u8m2(input2Data, vl);
vuint16m4_t input2U16 = __riscv_vwaddu_vx_u16m4(input2U8, 0, vl);
vuint32m8_t input2U32 = __riscv_vwaddu_vx_u32m8(input2U16, 0, vl);
vint32m8_t input2 = __riscv_vreinterpret_v_u32m8_i32m8(input2U32);

input1 = __riscv_vadd_vx_i32m8(input1, input1Offset, vl);
input1 = __riscv_vmul_vx_i32m8(input1, leftShiftResult1, vl);
input1 = __riscv_vsmul_vx_i32m8(input1, input1Multiplier, __RISCV_VXRM_RNU, vl);
if (rightShift1 > 0) {
input1 = __riscv_vssra_vx_i32m8(input1, rightShift1, __RISCV_VXRM_RNU, vl);
}

input2 = __riscv_vadd_vx_i32m8(input2, input2Offset, vl);
input2 = __riscv_vmul_vx_i32m8(input2, leftShiftResult2, vl);
input2 = __riscv_vsmul_vx_i32m8(input2, input2Multiplier, __RISCV_VXRM_RNU, vl);
if (rightShift2 > 0) {
input2 = __riscv_vssra_vx_i32m8(input2, rightShift2, __RISCV_VXRM_RNU, vl);
}

vint32m8_t sum = __riscv_vadd_vv_i32m8(input1, input2, vl);
sum = __riscv_vmul_vx_i32m8(sum, 1 << leftShiftOut, vl);
sum = __riscv_vsmul_vx_i32m8(sum, outputMultiplier, __RISCV_VXRM_RNU, vl);
if (rightShiftOut > 0) {
sum = __riscv_vssra_vx_i32m8(sum, rightShiftOut, __RISCV_VXRM_RNU, vl);
}
sum = __riscv_vadd_vx_i32m8(sum, outputOffset, vl);
sum = __riscv_vmax_vx_i32m8(sum, outputActivationMin, vl);
sum = __riscv_vmin_vx_i32m8(sum, outputActivationMax, vl);

vuint32m8_t outputU32 = __riscv_vreinterpret_v_i32m8_u32m8(sum);
vuint16m4_t outputU16 = __riscv_vncvt_x_x_w_u16m4(outputU32, vl);
vuint8m2_t outputU8 = __riscv_vncvt_x_x_w_u8m2(outputU16, vl);
__riscv_vse8_v_u8m2(outputData, outputU8, vl);

input1Data += vl;
input2Data += vl;
outputData += vl;
size -= vl;
}
}
36 changes: 36 additions & 0 deletions source/backend/cpu/riscv/rvv/MNNDeconvTransformWeightC4.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#include <riscv_vector.h>

#include "core/Macro.h"

void MNNDeconvTransformWeightC4_RVV(const float* src, float* dst, int outputCount, int srcCount, int area) {
constexpr int pack = 4;
int outputC4 = UP_DIV(outputCount, pack);
int fullOutputC4 = outputCount / pack;
int tail = outputCount % pack;
size_t vl4 = __riscv_vsetvl_e32m1(pack);

for (int c = 0; c < srcCount; ++c) {
for (int oc = 0; oc < fullOutputC4; ++oc) {
for (int a = 0; a < area; ++a) {
int dstIndex = c * outputC4 * area * pack + oc * area * pack + a * pack;
const float* srcPtr = src + c * outputCount * area + oc * pack * area + a;
vfloat32m1_t value = __riscv_vlse32_v_f32m1(srcPtr, area * sizeof(float), vl4);
__riscv_vse32_v_f32m1(dst + dstIndex, value, vl4);
}
}

if (tail > 0) {
int oc = fullOutputC4;
size_t vl = __riscv_vsetvl_e32m1(tail);
for (int a = 0; a < area; ++a) {
int dstIndex = c * outputC4 * area * pack + oc * area * pack + a * pack;
const float* srcPtr = src + c * outputCount * area + oc * pack * area + a;
vfloat32m1_t value = __riscv_vlse32_v_f32m1(srcPtr, area * sizeof(float), vl);
__riscv_vse32_v_f32m1(dst + dstIndex, value, vl);
for (int p = tail; p < pack; ++p) {
dst[dstIndex + p] = 0.0f;
}
}
}
}
}