Add model file collector, fix some bugs and add some features (#123)

add model path searcher and remove hardwritten file paths from benchmark configs; pack ppresnet, mobilenet & crnn with labels; fix palm det data; add flags to enable models of different precision separately (#123)

Author: fengyuentau

benchmark/README.md

@@ -1,41 +1,39 @@
 # OpenCV Zoo Benchmark
 
-Benchmarking different models in the zoo.
+Benchmarking the speed of OpenCV DNN inferring different models in the zoo. Result of each model includes the time of its preprocessing, inference and postprocessing stages.
 
 Data for benchmarking will be downloaded and loaded in [data](./data) based on given config.
 
-Time is measured from data preprocess (resize is excluded), to a forward pass of a network, and postprocess to get final results. The final time data presented is averaged from a 100-time run.
-
 ## Preparation
 
 1. Install `python >= 3.6`.
 2. Install dependencies: `pip install -r requirements.txt`.
 3. Download data for benchmarking.
     1. Download all data: `python download_data.py`
     2. Download one or more specified data: `python download_data.py face text`. Available names can be found in `download_data.py`.
-    3. If download fails, you can download all data from https://pan.baidu.com/s/18sV8D4vXUb2xC9EG45k7bg (code: pvrw). Please place and extract data packages under [./data](./data).
+    3. You can also download all data from https://pan.baidu.com/s/18sV8D4vXUb2xC9EG45k7bg (code: pvrw). Please place and extract data packages under [./data](./data).
 
 ## Benchmarking
 
-Run the following command to benchmark on a given config:
+**Linux**:
 
 ```shell
 export PYTHONPATH=$PYTHONPATH:.. 
 python benchmark.py --cfg ./config/face_detection_yunet.yaml
 ```
 
-If you are a Windows user and wants to run in CMD/PowerShell, use this command instead:
+**Windows**:
 - CMD
-```shell
-set PYTHONPATH=%PYTHONPATH%;..
-python benchmark.py --cfg ./config/face_detection_yunet.yaml
-```
+    ```shell
+    set PYTHONPATH=%PYTHONPATH%;..
+    python benchmark.py --cfg ./config/face_detection_yunet.yaml
+    ```
 
 - PowerShell
-```shell
-$env:PYTHONPATH=$env:PYTHONPATH+";.."
-python benchmark.py --cfg ./config/face_detection_yunet.yaml
-```
+    ```shell
+    $env:PYTHONPATH=$env:PYTHONPATH+";.."
+    python benchmark.py --cfg ./config/face_detection_yunet.yaml
+    ```
 <!--
 Omit `--cfg` if you want to benchmark all included models:
 ```shell

benchmark/benchmark.py

@@ -11,6 +11,9 @@
 parser = argparse.ArgumentParser("Benchmarks for OpenCV Zoo.")
 parser.add_argument('--cfg', '-c', type=str,
                     help='Benchmarking on the given config.')
+parser.add_argument("--fp32", action="store_true", help="Runs models of float32 precision only.")
+parser.add_argument("--fp16", action="store_true", help="Runs models of float16 precision only.")
+parser.add_argument("--int8", action="store_true", help="Runs models of int8 precision only.")
 args = parser.parse_args()
 
 def build_from_cfg(cfg, registery, key=None, name=None):
@@ -24,14 +27,6 @@ def build_from_cfg(cfg, registery, key=None, name=None):
     else:
         raise NotImplementedError()
 
-def prepend_pythonpath(cfg):
-    for k, v in cfg.items():
-        if isinstance(v, dict):
-            prepend_pythonpath(v)
-        else:
-            if 'path' in k.lower():
-                cfg[k] = os.path.join(os.environ['PYTHONPATH'].split(os.pathsep)[-1], v)
-
 class Benchmark:
     def __init__(self, **kwargs):
         self._type = kwargs.pop('type', None)
@@ -115,16 +110,30 @@ def printResults(self):
     with open(args.cfg, 'r') as f:
         cfg = yaml.safe_load(f)
 
-    # prepend PYTHONPATH to each path
-    prepend_pythonpath(cfg)
-
-    # Instantiate benchmarking
+    # Instantiate benchmark
     benchmark = Benchmark(**cfg['Benchmark'])
 
     # Instantiate model
-    model = build_from_cfg(cfg=cfg['Model'], registery=MODELS, key='name')
-
-    # Run benchmarking
-    print('Benchmarking {}:'.format(model.name))
-    benchmark.run(model)
-    benchmark.printResults()
+    model_config = cfg['Model']
+    model_handler, model_paths = MODELS.get(model_config.pop('name'))
+
+    _model_paths = []
+    if args.fp32 or args.fp16 or args.int8:
+        if args.fp32:
+            _model_paths += model_paths['fp32']
+        if args.fp16:
+            _model_paths += model_paths['fp16']
+        if args.int8:
+            _model_paths += model_paths['int8']
+    else:
+        _model_paths = model_paths['fp32'] + model_paths['fp16'] + model_paths['int8']
+
+    for model_path in _model_paths:
+        model = model_handler(*model_path, **model_config)
+        # Format model_path
+        for i in range(len(model_path)):
+            model_path[i] = model_path[i].split('/')[-1]
+        print('Benchmarking {} with {}'.format(model.name, model_path))
+        # Run benchmark
+        benchmark.run(model)
+        benchmark.printResults()

benchmark/config/face_detection_yunet.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Face Detection Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/face_detection"
+    path: "data/face_detection"
     files: ["group.jpg", "concerts.jpg", "dance.jpg"]
     sizes: # [[w1, h1], ...], Omit to run at original scale
       - [160, 120]
@@ -16,8 +16,6 @@ Benchmark:
 
 Model:
   name: "YuNet"
-  modelPath: "models/face_detection_yunet/face_detection_yunet_2022mar.onnx"
   confThreshold: 0.6
   nmsThreshold: 0.3
   topK: 5000
-

benchmark/config/face_recognition_sface.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Face Recognition Benchmark"
   type: "Recognition"
   data:
-    path: "benchmark/data/face_recognition"
+    path: "data/face_recognition"
     files: ["Aaron_Tippin_0001.jpg", "Alvaro_Uribe_0028.jpg", "Alvaro_Uribe_0029.jpg", "Jose_Luis_Rodriguez_Zapatero_0001.jpg"]
   metric: # 'sizes' is omitted since this model requires input of fixed size
     warmup: 30
@@ -13,4 +13,3 @@ Benchmark:
 
 Model:
   name: "SFace"
-  modelPath: "models/face_recognition_sface/face_recognition_sface_2021dec.onnx"

benchmark/config/facial_expression_recognition.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Facial Expression Recognition Benchmark"
   type: "Recognition"
   data:
-    path: "benchmark/data/facial_expression_recognition/fer_evaluation"
+    path: "data/facial_expression_recognition/fer_evaluation"
     files: ["RAF_test_0_61.jpg", "RAF_test_0_30.jpg", "RAF_test_6_25.jpg"]
   metric: # 'sizes' is omitted since this model requires input of fixed size
     warmup: 30
@@ -13,4 +13,3 @@ Benchmark:
 
 Model:
   name: "FacialExpressionRecog"
-  modelPath: "models/facial_expression_recognition/facial_expression_recognition_mobilefacenet_2022july.onnx"

benchmark/config/handpose_estimation_mediapipe.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Hand Pose Estimation Benchmark"
   type: "Recognition"
   data:
-    path: "benchmark/data/palm_detection"
+    path: "data/palm_detection_20230125"
     files: ["palm1.jpg", "palm2.jpg", "palm3.jpg"]
     sizes: # [[w1, h1], ...], Omit to run at original scale
       - [256, 256]
@@ -15,5 +15,4 @@ Benchmark:
 
 Model:
   name: "MPHandPose"
-  modelPath: "models/handpose_estimation_mediapipe/handpose_estimation_mediapipe_2022may.onnx"
   confThreshold: 0.9

benchmark/config/human_segmentation_pphumanseg.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Human Segmentation Benchmark"
   type: "Base"
   data:
-    path: "benchmark/data/human_segmentation"
+    path: "data/human_segmentation"
     files: ["messi5.jpg", "100040721_1.jpg", "detect.jpg"]
     sizes: [[192, 192]]
     toRGB: True
@@ -15,4 +15,3 @@ Benchmark:
 
 Model:
   name: "PPHumanSeg"
-  modelPath: "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct.onnx"

benchmark/config/image_classification_mobilenetv1.yaml renamed to benchmark/config/image_classification_mobilenet.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Image Classification Benchmark"
   type: "Classification"
   data:
-    path: "benchmark/data/image_classification"
+    path: "data/image_classification"
     files: ["coffee_mug.jpg", "umbrella.jpg", "wall_clock.jpg"]
     sizes: [[256, 256]]
     toRGB: True
@@ -15,6 +15,4 @@ Benchmark:
   target: "cpu"
 
 Model:
-  name: "MobileNetV1"
-  modelPath: "models/image_classification_mobilenet/image_classification_mobilenetv1_2022apr.onnx"
-
+  name: "MobileNet"

benchmark/config/image_classification_mobilenetv2.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Image Classification Benchmark"
   type: "Classification"
   data:
-    path: "benchmark/data/image_classification"
+    path: "data/image_classification"
     files: ["coffee_mug.jpg", "umbrella.jpg", "wall_clock.jpg"]
     sizes: [[256, 256]]
     toRGB: True
@@ -16,5 +16,3 @@ Benchmark:
 
 Model:
   name: "PPResNet"
-  modelPath: "models/image_classification_ppresnet/image_classification_ppresnet50_2022jan.onnx"
-

benchmark/config/image_classification_ppresnet.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "License Plate Detection Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/license_plate_detection"
+    path: "data/license_plate_detection"
     files: ["1.jpg", "2.jpg", "3.jpg", "4.jpg"]
     sizes: # [[w1, h1], ...], Omit to run at original scale
       - [320, 240]
@@ -15,7 +15,6 @@ Benchmark:
 
 Model:
   name: "LPD_YuNet"
-  modelPath: "models/license_plate_detection_yunet/license_plate_detection_lpd_yunet_2022may.onnx"
   confThreshold: 0.8
   nmsThreshold: 0.3
   topK: 5000

benchmark/config/license_plate_detection_yunet.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Object Detection Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/object_detection"
+    path: "data/object_detection"
     files: ["1.png", "2.png", "3.png"]
     sizes:
       - [416, 416]
@@ -15,7 +15,5 @@ Benchmark:
 
 Model:
   name: "NanoDet"
-  modelPath: "models/object_detection_nanodet/object_detection_nanodet_2022nov.onnx"
   prob_threshold: 0.35
   iou_threshold: 0.6
-

benchmark/config/object_detection_nanodet.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Object Detection Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/object_detection"
+    path: "data/object_detection"
     files: ["1.png", "2.png", "3.png"]
     sizes:
       - [640, 640]
@@ -15,8 +15,6 @@ Benchmark:
 
 Model:
   name: "YoloX"
-  modelPath: "models/object_detection_yolox/object_detection_yolox_2022nov.onnx"
   confThreshold: 0.35
   nmsThreshold: 0.5
   objThreshold: 0.5
-

benchmark/config/object_detection_yolox.yaml

@@ -3,7 +3,7 @@ Benchmark:
   type: "Tracking"
   data:
     type: "TrackingVideoLoader"
-    path: "benchmark/data/object_tracking"
+    path: "data/object_tracking"
     files: ["throw_cup.mp4"]
   metric:
     type: "Tracking"
@@ -13,6 +13,3 @@ Benchmark:
 
 Model:
   name: "DaSiamRPN"
-  model_path: "models/object_tracking_dasiamrpn/object_tracking_dasiamrpn_model_2021nov.onnx"
-  kernel_cls1_path: "models/object_tracking_dasiamrpn/object_tracking_dasiamrpn_kernel_cls1_2021nov.onnx"
-  kernel_r1_path: "models/object_tracking_dasiamrpn/object_tracking_dasiamrpn_kernel_r1_2021nov.onnx"

benchmark/config/object_tracking_dasiamrpn.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Palm Detection Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/palm_detection"
+    path: "data/palm_detection_20230125"
     files: ["palm1.jpg", "palm2.jpg", "palm3.jpg"]
     sizes: # [[w1, h1], ...], Omit to run at original scale
       - [256, 256]
@@ -15,8 +15,6 @@ Benchmark:
 
 Model:
   name: "MPPalmDet"
-  modelPath: "models/palm_detection_mediapipe/palm_detection_mediapipe_2022may.onnx"
   scoreThreshold: 0.5
   nmsThreshold: 0.3
   topK: 1
-

benchmark/config/palm_detection_mediapipe.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Person ReID Benchmark"
   type: "Base"
   data:
-    path: "benchmark/data/person_reid"
+    path: "data/person_reid"
     files: ["0030_c1_f0056923.jpg", "0042_c5_f0068994.jpg", "0056_c8_f0017063.jpg"]
     sizes: [[128, 256]]
   metric:
@@ -14,4 +14,3 @@ Benchmark:
 
 Model:
   name: "YoutuReID"
-  modelPath: "models/person_reid_youtureid/person_reid_youtu_2021nov.onnx"

benchmark/config/person_reid_youtureid.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "QRCode Detection and Decoding Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/qrcode"
+    path: "data/qrcode"
     files: ["opencv.png", "opencv_zoo.png"]
     sizes:
       - [100, 100]
@@ -16,7 +16,3 @@ Benchmark:
 
 Model:
   name: "WeChatQRCode"
-  detect_prototxt_path: "models/qrcode_wechatqrcode/detect_2021nov.prototxt"
-  detect_model_path: "models/qrcode_wechatqrcode/detect_2021nov.caffemodel"
-  sr_prototxt_path: "models/qrcode_wechatqrcode/sr_2021nov.prototxt"
-  sr_model_path: "models/qrcode_wechatqrcode/sr_2021nov.caffemodel"

benchmark/config/qrcode_wechatqrcode.yaml

@@ -2,7 +2,7 @@ Benchmark:
   name: "Text Detection Benchmark"
   type: "Detection"
   data:
-    path: "benchmark/data/text"
+    path: "data/text"
     files: ["1.jpg", "2.jpg", "3.jpg"]
     sizes: # [[w1, h1], ...], Omit to run at original scale
       - [640, 480]
@@ -15,8 +15,7 @@ Benchmark:
 
 Model:
   name: "DB"
-  modelPath: "models/text_detection_db/text_detection_DB_TD500_resnet18_2021sep.onnx"
   binaryThreshold: 0.3
   polygonThreshold: 0.5
   maxCandidates: 200
-  unclipRatio: 2.0
+  unclipRatio: 2.0