Skip to content

Commit 0aba6f5

Browse files
committed
dnn(test): add filters for ONNX conformance tests
1 parent 58a8e45 commit 0aba6f5

6 files changed

+3754
-35
lines changed

modules/dnn/test/test_common.hpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
#define INF_ENGINE_VER_MAJOR_LE(ver) (((INF_ENGINE_RELEASE) / 10000) <= ((ver) / 10000))
1919
#define INF_ENGINE_VER_MAJOR_EQ(ver) (((INF_ENGINE_RELEASE) / 10000) == ((ver) / 10000))
2020

21-
#define CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE "dnn_skip_onnx_conformance"
22-
21+
#define CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND "dnn_skip_opencv_backend"
2322
#define CV_TEST_TAG_DNN_SKIP_HALIDE "dnn_skip_halide"
23+
#define CV_TEST_TAG_DNN_SKIP_CPU "dnn_skip_cpu"
2424
#define CV_TEST_TAG_DNN_SKIP_OPENCL "dnn_skip_ocl"
2525
#define CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 "dnn_skip_ocl_fp16"
2626
#define CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER "dnn_skip_ie_nn_builder"
@@ -39,6 +39,9 @@
3939
#define CV_TEST_TAG_DNN_SKIP_IE_MYRIAD CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2, CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X
4040
#define CV_TEST_TAG_DNN_SKIP_IE_ARM_CPU "dnn_skip_ie_arm_cpu"
4141

42+
#define CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE "dnn_skip_onnx_conformance"
43+
#define CV_TEST_TAG_DNN_SKIP_PARSER "dnn_skip_parser"
44+
4245

4346
#ifdef HAVE_INF_ENGINE
4447
#if INF_ENGINE_VER_MAJOR_EQ(2018050000)

modules/dnn/test/test_common.impl.hpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -361,10 +361,15 @@ void initDNNTests()
361361
cvtest::addDataSearchPath(extraTestDataPath);
362362

363363
registerGlobalSkipTag(
364-
CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE,
365-
CV_TEST_TAG_DNN_SKIP_HALIDE,
364+
CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND,
365+
CV_TEST_TAG_DNN_SKIP_CPU,
366366
CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16
367367
);
368+
#if defined(HAVE_HALIDE)
369+
registerGlobalSkipTag(
370+
CV_TEST_TAG_DNN_SKIP_HALIDE
371+
)
372+
#endif
368373
#if defined(INF_ENGINE_RELEASE)
369374
registerGlobalSkipTag(
370375
CV_TEST_TAG_DNN_SKIP_IE,
@@ -393,6 +398,10 @@ void initDNNTests()
393398
CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16
394399
);
395400
#endif
401+
registerGlobalSkipTag(
402+
CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE,
403+
CV_TEST_TAG_DNN_SKIP_PARSER
404+
);
396405
}
397406

398407
} // namespace

modules/dnn/test/test_onnx_conformance.cpp

Lines changed: 141 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,18 @@
66
#include "test_precomp.hpp"
77
#include "npy_blob.hpp"
88
#include <opencv2/dnn/shape_utils.hpp>
9-
#include <opencv2/core/utils/filesystem.hpp>
109

11-
#include <tuple>
10+
#if defined(_MSC_VER) // workaround for 32-bit MSVC compiler
11+
#pragma optimize("", off)
12+
#endif
13+
14+
15+
#define CV_TEST_TAG_DNN_ERROR_PARSER "dnn_error_parser"
16+
#define CV_TEST_TAG_DNN_ERROR_NET_SETUP "dnn_error_net_setup"
17+
#define CV_TEST_TAG_DNN_ERROR_FORWARD "dnn_error_forward"
18+
#define CV_TEST_TAG_DNN_LAYER_FALLBACK "dnn_layer_fallback"
19+
#define CV_TEST_TAG_DNN_NO_ACCURACY_CHECK "dnn_no_accuracy_check"
20+
1221

1322
namespace opencv_test {
1423

@@ -968,7 +977,7 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
968977
backend = get<0>(get<1>(GetParam()));
969978
target = get<1>(get<1>(GetParam()));
970979

971-
if (target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
980+
if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
972981
{
973982
default_l1 = 4e-3;
974983
default_lInf = 2e-2;
@@ -980,14 +989,14 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
980989
}
981990
}
982991

983-
void expectNoFallbacks(Net& net) const
992+
bool checkFallbacks(Net& net) const
984993
{
985994
// Check if all the layers are supported with current backend and target.
986995
// Some layers might be fused so their timings equal to zero.
987996
std::vector<double> timings;
988997
net.getPerfProfile(timings);
989998
std::vector<String> names = net.getLayerNames();
990-
ASSERT_EQ(names.size(), timings.size());
999+
CV_CheckEQ(names.size(), timings.size(), "DNN critical error");
9911000

9921001
bool hasFallbacks = false;
9931002
for (int i = 0; i < names.size(); ++i)
@@ -1000,58 +1009,160 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
10001009
std::cout << "FALLBACK: Layer [" << l->type << "]:[" << l->name << "] is expected to has backend implementation" << endl;
10011010
}
10021011
}
1003-
ASSERT_FALSE(hasFallbacks) << "Implementation fallbacks are not expected in this test";
1012+
return hasFallbacks;
10041013
}
10051014
};
10061015

10071016
TEST_P(Test_ONNX_conformance, Layer_Test)
10081017
{
1009-
applyTestTag(CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1018+
std::string name = test_case.name;
1019+
//Backend backend = ...;
1020+
//Target target = ...;
1021+
1022+
bool checkLayersFallbacks = true;
1023+
bool checkAccuracy = true;
1024+
1025+
#include "test_onnx_conformance_layer_filter_parser.inl.hpp"
1026+
if (backend == DNN_BACKEND_OPENCV)
1027+
{
1028+
#include "test_onnx_conformance_layer_filter__opencv.inl.hpp"
1029+
}
1030+
#if 0 //def HAVE_HALIDE
1031+
else if (backend == DNN_BACKEND_HALIDE)
1032+
{
1033+
#include "test_onnx_conformance_layer_filter__halide.inl.hpp"
1034+
}
1035+
#endif
1036+
#if 0 //def HAVE_INF_ENGINE
1037+
else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
1038+
{
1039+
#include "test_onnx_conformance_layer_filter__ngraph.inl.hpp"
1040+
}
1041+
#endif
1042+
#if 0 //def HAVE_VULKAN
1043+
else if (backend == DNN_BACKEND_VKCOM)
1044+
{
1045+
#include "test_onnx_conformance_layer_filter__vulkan.inl.hpp"
1046+
}
1047+
#endif
1048+
#if 0 //def HAVE_CUDA
1049+
else if (backend == DNN_BACKEND_CUDA)
1050+
{
1051+
#include "test_onnx_conformance_layer_filter__cuda.inl.hpp"
1052+
}
1053+
#endif
1054+
else
1055+
{
1056+
std::ostringstream ss;
1057+
ss << "No test filter available for backend ";
1058+
PrintTo(backend, &ss);
1059+
ss << ". Run test by default";
1060+
std::cout << ss.str() << std::endl;
1061+
}
10101062

10111063
std::vector<Mat> inputs;
10121064
std::vector<Mat> ref_outputs;
10131065

1014-
std::transform(test_case.input_paths.begin(), test_case.input_paths.end(),
1015-
std::back_inserter(inputs), readTensorFromONNX);
1066+
Net net;
1067+
try
1068+
{
1069+
//cout << "Read ONNX inputs..." << endl;
1070+
std::transform(test_case.input_paths.begin(), test_case.input_paths.end(),
1071+
std::back_inserter(inputs), readTensorFromONNX);
10161072

1017-
std::transform(test_case.output_paths.begin(), test_case.output_paths.end(),
1018-
std::back_inserter(ref_outputs), readTensorFromONNX);
1073+
//cout << "Read ONNX reference outputs..." << endl;
1074+
std::transform(test_case.output_paths.begin(), test_case.output_paths.end(),
1075+
std::back_inserter(ref_outputs), readTensorFromONNX);
10191076

1020-
Net net = readNetFromONNX(test_case.model_path);
1077+
//cout << "Parse model..." << endl;
1078+
net = readNetFromONNX(test_case.model_path);
1079+
if (net.empty())
1080+
{
1081+
applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER);
1082+
}
1083+
}
1084+
catch (...)
1085+
{
1086+
cout << "Exception during ONNX model parse / loading input / loading reference data!" << endl;
1087+
applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER);
1088+
throw;
1089+
}
10211090
ASSERT_FALSE(net.empty());
10221091

1023-
net.setPreferableBackend(backend);
1024-
net.setPreferableTarget(target);
1025-
10261092
std::vector<String> inputNames;
10271093
for (int i = 0; i < inputs.size(); ++i)
1028-
inputNames.push_back(format("%d", i));
1094+
inputNames.push_back(cv::format("%d", i));
10291095
net.setInputsNames(inputNames);
10301096

1031-
for (int i = 0; i < inputs.size(); ++i)
1032-
net.setInput(inputs[i], inputNames[i]);
1097+
try
1098+
{
1099+
net.setPreferableBackend(backend);
1100+
net.setPreferableTarget(target);
1101+
1102+
for (int i = 0; i < inputs.size(); ++i)
1103+
net.setInput(inputs[i], inputNames[i]);
1104+
}
1105+
catch (...)
1106+
{
1107+
cout << "Exception during network configuration!" << endl;
1108+
applyTestTag(CV_TEST_TAG_DNN_ERROR_NET_SETUP);
1109+
throw;
1110+
}
10331111

1034-
std::vector<std::string> layerNames = net.getUnconnectedOutLayersNames();
1035-
std::vector<std::vector<Mat>> outputs_;
1036-
net.forward(outputs_, layerNames);
1037-
ASSERT_EQ(outputs_.size(), 1);
1112+
std::vector<String> layerNames = net.getUnconnectedOutLayersNames();
1113+
std::vector< std::vector<Mat> > outputs_;
1114+
try
1115+
{
1116+
net.forward(outputs_, layerNames);
1117+
}
1118+
catch (...)
1119+
{
1120+
cout << "Exception during net.forward() call!" << endl;
1121+
applyTestTag(CV_TEST_TAG_DNN_ERROR_FORWARD);
1122+
throw;
1123+
}
1124+
ASSERT_GE(outputs_.size(), 1);
10381125
const std::vector<Mat>& outputs = outputs_[0];
10391126

1040-
if (ref_outputs.size() == 1)
1127+
if (checkLayersFallbacks)
10411128
{
1042-
// probably we found random unconnected layers.
1043-
normAssert(ref_outputs[0], outputs[0], "", default_l1, default_lInf);
1129+
if (checkFallbacks(net))
1130+
{
1131+
applyTestTag(CV_TEST_TAG_DNN_LAYER_FALLBACK);
1132+
}
10441133
}
1045-
else
1134+
1135+
if (checkAccuracy)
10461136
{
1047-
ASSERT_EQ(outputs.size(), ref_outputs.size());
1048-
for (size_t i = 0; i < ref_outputs.size(); ++i)
1137+
try
1138+
{
1139+
if (ref_outputs.size() == 1)
1140+
{
1141+
// probably we found random unconnected layers.
1142+
normAssert(ref_outputs[0], outputs[0], "", default_l1, default_lInf);
1143+
}
1144+
else
1145+
{
1146+
ASSERT_EQ(outputs.size(), ref_outputs.size());
1147+
for (size_t i = 0; i < ref_outputs.size(); ++i)
1148+
{
1149+
normAssert(ref_outputs[i], outputs[i], "", default_l1, default_lInf);
1150+
}
1151+
}
1152+
}
1153+
catch (...)
10491154
{
1050-
normAssert(ref_outputs[i], outputs[i], "", default_l1, default_lInf);
1155+
cout << "Exception during accuracy check!" << endl;
1156+
throw;
10511157
}
10521158
}
1159+
else
1160+
{
1161+
applyTestTag(CV_TEST_TAG_DNN_NO_ACCURACY_CHECK);
1162+
}
10531163

1054-
expectNoFallbacks(net);
1164+
if (!HasFailure())
1165+
cout << "Test passed!" << endl;
10551166
}
10561167

10571168
INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance,

0 commit comments

Comments
 (0)