6
6
#include " test_precomp.hpp"
7
7
#include " npy_blob.hpp"
8
8
#include < opencv2/dnn/shape_utils.hpp>
9
- #include < opencv2/core/utils/filesystem.hpp>
10
9
11
- #include < tuple>
10
+ #if defined(_MSC_VER) // workaround for 32-bit MSVC compiler
11
+ #pragma optimize("", off)
12
+ #endif
13
+
14
+
15
+ #define CV_TEST_TAG_DNN_ERROR_PARSER " dnn_error_parser"
16
+ #define CV_TEST_TAG_DNN_ERROR_NET_SETUP " dnn_error_net_setup"
17
+ #define CV_TEST_TAG_DNN_ERROR_FORWARD " dnn_error_forward"
18
+ #define CV_TEST_TAG_DNN_LAYER_FALLBACK " dnn_layer_fallback"
19
+ #define CV_TEST_TAG_DNN_NO_ACCURACY_CHECK " dnn_no_accuracy_check"
20
+
12
21
13
22
namespace opencv_test {
14
23
@@ -968,7 +977,7 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
968
977
backend = get<0 >(get<1 >(GetParam ()));
969
978
target = get<1 >(get<1 >(GetParam ()));
970
979
971
- if (target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
980
+ if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
972
981
{
973
982
default_l1 = 4e-3 ;
974
983
default_lInf = 2e-2 ;
@@ -980,14 +989,14 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
980
989
}
981
990
}
982
991
983
- void expectNoFallbacks (Net& net) const
992
+ bool checkFallbacks (Net& net) const
984
993
{
985
994
// Check if all the layers are supported with current backend and target.
986
995
// Some layers might be fused so their timings equal to zero.
987
996
std::vector<double > timings;
988
997
net.getPerfProfile (timings);
989
998
std::vector<String> names = net.getLayerNames ();
990
- ASSERT_EQ (names.size (), timings.size ());
999
+ CV_CheckEQ (names.size (), timings.size (), " DNN critical error " );
991
1000
992
1001
bool hasFallbacks = false ;
993
1002
for (int i = 0 ; i < names.size (); ++i)
@@ -1000,58 +1009,160 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
1000
1009
std::cout << " FALLBACK: Layer [" << l->type << " ]:[" << l->name << " ] is expected to has backend implementation" << endl;
1001
1010
}
1002
1011
}
1003
- ASSERT_FALSE (hasFallbacks) << " Implementation fallbacks are not expected in this test " ;
1012
+ return hasFallbacks ;
1004
1013
}
1005
1014
};
1006
1015
1007
1016
TEST_P (Test_ONNX_conformance, Layer_Test)
1008
1017
{
1009
- applyTestTag (CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1018
+ std::string name = test_case.name ;
1019
+ // Backend backend = ...;
1020
+ // Target target = ...;
1021
+
1022
+ bool checkLayersFallbacks = true ;
1023
+ bool checkAccuracy = true ;
1024
+
1025
+ #include " test_onnx_conformance_layer_filter_parser.inl.hpp"
1026
+ if (backend == DNN_BACKEND_OPENCV)
1027
+ {
1028
+ #include " test_onnx_conformance_layer_filter__opencv.inl.hpp"
1029
+ }
1030
+ #if 0 //def HAVE_HALIDE
1031
+ else if (backend == DNN_BACKEND_HALIDE)
1032
+ {
1033
+ #include "test_onnx_conformance_layer_filter__halide.inl.hpp"
1034
+ }
1035
+ #endif
1036
+ #if 0 //def HAVE_INF_ENGINE
1037
+ else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
1038
+ {
1039
+ #include "test_onnx_conformance_layer_filter__ngraph.inl.hpp"
1040
+ }
1041
+ #endif
1042
+ #if 0 //def HAVE_VULKAN
1043
+ else if (backend == DNN_BACKEND_VKCOM)
1044
+ {
1045
+ #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp"
1046
+ }
1047
+ #endif
1048
+ #if 0 //def HAVE_CUDA
1049
+ else if (backend == DNN_BACKEND_CUDA)
1050
+ {
1051
+ #include "test_onnx_conformance_layer_filter__cuda.inl.hpp"
1052
+ }
1053
+ #endif
1054
+ else
1055
+ {
1056
+ std::ostringstream ss;
1057
+ ss << " No test filter available for backend " ;
1058
+ PrintTo (backend, &ss);
1059
+ ss << " . Run test by default" ;
1060
+ std::cout << ss.str () << std::endl;
1061
+ }
1010
1062
1011
1063
std::vector<Mat> inputs;
1012
1064
std::vector<Mat> ref_outputs;
1013
1065
1014
- std::transform (test_case.input_paths .begin (), test_case.input_paths .end (),
1015
- std::back_inserter (inputs), readTensorFromONNX);
1066
+ Net net;
1067
+ try
1068
+ {
1069
+ // cout << "Read ONNX inputs..." << endl;
1070
+ std::transform (test_case.input_paths .begin (), test_case.input_paths .end (),
1071
+ std::back_inserter (inputs), readTensorFromONNX);
1016
1072
1017
- std::transform (test_case.output_paths .begin (), test_case.output_paths .end (),
1018
- std::back_inserter (ref_outputs), readTensorFromONNX);
1073
+ // cout << "Read ONNX reference outputs..." << endl;
1074
+ std::transform (test_case.output_paths .begin (), test_case.output_paths .end (),
1075
+ std::back_inserter (ref_outputs), readTensorFromONNX);
1019
1076
1020
- Net net = readNetFromONNX (test_case.model_path );
1077
+ // cout << "Parse model..." << endl;
1078
+ net = readNetFromONNX (test_case.model_path );
1079
+ if (net.empty ())
1080
+ {
1081
+ applyTestTag (CV_TEST_TAG_DNN_ERROR_PARSER);
1082
+ }
1083
+ }
1084
+ catch (...)
1085
+ {
1086
+ cout << " Exception during ONNX model parse / loading input / loading reference data!" << endl;
1087
+ applyTestTag (CV_TEST_TAG_DNN_ERROR_PARSER);
1088
+ throw ;
1089
+ }
1021
1090
ASSERT_FALSE (net.empty ());
1022
1091
1023
- net.setPreferableBackend (backend);
1024
- net.setPreferableTarget (target);
1025
-
1026
1092
std::vector<String> inputNames;
1027
1093
for (int i = 0 ; i < inputs.size (); ++i)
1028
- inputNames.push_back (format (" %d" , i));
1094
+ inputNames.push_back (cv:: format (" %d" , i));
1029
1095
net.setInputsNames (inputNames);
1030
1096
1031
- for (int i = 0 ; i < inputs.size (); ++i)
1032
- net.setInput (inputs[i], inputNames[i]);
1097
+ try
1098
+ {
1099
+ net.setPreferableBackend (backend);
1100
+ net.setPreferableTarget (target);
1101
+
1102
+ for (int i = 0 ; i < inputs.size (); ++i)
1103
+ net.setInput (inputs[i], inputNames[i]);
1104
+ }
1105
+ catch (...)
1106
+ {
1107
+ cout << " Exception during network configuration!" << endl;
1108
+ applyTestTag (CV_TEST_TAG_DNN_ERROR_NET_SETUP);
1109
+ throw ;
1110
+ }
1033
1111
1034
- std::vector<std::string> layerNames = net.getUnconnectedOutLayersNames ();
1035
- std::vector<std::vector<Mat>> outputs_;
1036
- net.forward (outputs_, layerNames);
1037
- ASSERT_EQ (outputs_.size (), 1 );
1112
+ std::vector<String> layerNames = net.getUnconnectedOutLayersNames ();
1113
+ std::vector< std::vector<Mat> > outputs_;
1114
+ try
1115
+ {
1116
+ net.forward (outputs_, layerNames);
1117
+ }
1118
+ catch (...)
1119
+ {
1120
+ cout << " Exception during net.forward() call!" << endl;
1121
+ applyTestTag (CV_TEST_TAG_DNN_ERROR_FORWARD);
1122
+ throw ;
1123
+ }
1124
+ ASSERT_GE (outputs_.size (), 1 );
1038
1125
const std::vector<Mat>& outputs = outputs_[0 ];
1039
1126
1040
- if (ref_outputs. size () == 1 )
1127
+ if (checkLayersFallbacks )
1041
1128
{
1042
- // probably we found random unconnected layers.
1043
- normAssert (ref_outputs[0 ], outputs[0 ], " " , default_l1, default_lInf);
1129
+ if (checkFallbacks (net))
1130
+ {
1131
+ applyTestTag (CV_TEST_TAG_DNN_LAYER_FALLBACK);
1132
+ }
1044
1133
}
1045
- else
1134
+
1135
+ if (checkAccuracy)
1046
1136
{
1047
- ASSERT_EQ (outputs.size (), ref_outputs.size ());
1048
- for (size_t i = 0 ; i < ref_outputs.size (); ++i)
1137
+ try
1138
+ {
1139
+ if (ref_outputs.size () == 1 )
1140
+ {
1141
+ // probably we found random unconnected layers.
1142
+ normAssert (ref_outputs[0 ], outputs[0 ], " " , default_l1, default_lInf);
1143
+ }
1144
+ else
1145
+ {
1146
+ ASSERT_EQ (outputs.size (), ref_outputs.size ());
1147
+ for (size_t i = 0 ; i < ref_outputs.size (); ++i)
1148
+ {
1149
+ normAssert (ref_outputs[i], outputs[i], " " , default_l1, default_lInf);
1150
+ }
1151
+ }
1152
+ }
1153
+ catch (...)
1049
1154
{
1050
- normAssert (ref_outputs[i], outputs[i], " " , default_l1, default_lInf);
1155
+ cout << " Exception during accuracy check!" << endl;
1156
+ throw ;
1051
1157
}
1052
1158
}
1159
+ else
1160
+ {
1161
+ applyTestTag (CV_TEST_TAG_DNN_NO_ACCURACY_CHECK);
1162
+ }
1053
1163
1054
- expectNoFallbacks (net);
1164
+ if (!HasFailure ())
1165
+ cout << " Test passed!" << endl;
1055
1166
}
1056
1167
1057
1168
INSTANTIATE_TEST_CASE_P (/* */ , Test_ONNX_conformance,
0 commit comments