sparseinst support export onnx now.

Your Name · Your Name · commit 5d5e23bd47ba · 2022-04-15T11:14:00.000+08:00
diff --git a/.gitignore b/.gitignore
@@ -138,15 +138,13 @@ dmypy.json
 .pyre/
 
 .idea/
-weights/
 *.pth
 *.onnx
 
 .vscode/
 output/
 datasets/
-weights/
 vendor/vendor/
 vendor/
-log2.md
-vendor/
+log2.md
+vendor/
diff --git a/export_onnx.py b/export_onnx.py
@@ -182,11 +182,11 @@ def change_detr_onnx(onnx_path):
     print(f"[INFO] onnx修改完成, 保存在{onnx_path + '_changed.onnx'}.")
 
 
-def load_test_image(f, h, w):
+def load_test_image(f, h, w, bs=1):
     a = cv2.imread(f)
     a = cv2.resize(a, (w, h))
-    # a_t = torch.tensor(a.astype(np.float32)).to(device).unsqueeze(0)
-    a_t = torch.tensor(a.astype(np.float32)).to(device)
+    a_t = torch.tensor(a.astype(np.float32)).to(device).unsqueeze(0).repeat(bs, 1, 1, 1)
+    # a_t = torch.tensor(a.astype(np.float32)).to(device)
     return a_t, a
 
 
@@ -238,6 +238,15 @@ def vis_res_fast(res, img, colors):
     return img
 
 
+def get_output_names_from_config_file(config_file):
+    if 'sparse_inst' in config_file:
+        return ['masks', 'scores', 'labels']
+    elif 'detr' in config_file:
+        return ['boxes', 'scores', 'labels']
+    else:
+        return ['outs']
+
+
 if __name__ == "__main__":
     mp.set_start_method("spawn", force=True)
     args = get_parser().parse_args()
@@ -255,14 +264,14 @@ def vis_res_fast(res, img, colors):
     metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0])
     predictor = DefaultPredictor(cfg)
 
-    h = 1056
-    w = 1920
-    # h = 640
-    # w = 640
+    # h = 1056
+    # w = 1920
+    h = 640
+    w = 640
     inp, ori_img = load_test_image(args.input, h, w)
     # TODO: remove hard coded for detr
     # inp, ori_img = load_test_image_detr(args.input, h, w)
-    print("input shape: ", inp.shape)
+    logger.info(f"input shape: {inp.shape}")
 
     model = predictor.model
     model = model.float()
@@ -273,9 +282,9 @@ def vis_res_fast(res, img, colors):
     )
     torch.onnx.export(
         model,
-        [inp],
+        inp,
         onnx_f,
-        output_names={"out"},
+        output_names=get_output_names_from_config_file(args.config_file),
         opset_version=12,
         do_constant_folding=True,
         verbose=args.verbose,
diff --git a/readme.md b/readme.md
@@ -93,7 +93,6 @@ Here are some tasks need to be claimed:
 
 
 
-
 ## 💁‍♂️ Results
 
 | YOLOv7 Instance             |  Face & Detection |
@@ -103,35 +102,9 @@ Here are some tasks need to be claimed:
 ![](https://s1.ax1x.com/2022/03/25/qN5zp6.png)  |  ![](https://s2.loli.net/2022/03/25/MBwq9YT7zC5Sd1A.png)
 
 
-## 🤔 Features
-
-Some highlights of YOLOv7 are:
-
-- A simple and standard training framework for any detection && instance segmentation tasks, based on detectron2;
-- Supports DETR and many transformer based detection framework out-of-box;
-- Supports easy to deploy pipeline thought onnx.
-- **This is the only framework support YOLOv4 + InstanceSegmentation** in single stage style;
-- Easily plugin into transformers based detector;
-
-We are strongly recommend you send PR if you have any further development on this project, **the only reason for opensource it is just for using community power to make it stronger and further**. It's very welcome for anyone contribute on any features!
-
-
-
-## 😎 Rules
-
-There are some rules you must follow to if you want train on your own dataset:
-
-- Rule No.1: Always set your own anchors on your dataset, using `tools/compute_anchors.py`, this applys to any other anchor-based detection methods as well (EfficientDet etc.);
-- Rule No.2: Keep a faith on your loss will goes down eventually, if not, dig deeper to find out why (but do not post issues repeated caused I might don't know either.).
-- Rule No.3: No one will tells u but it's real: *do not change backbone easily, whole params coupled with your backbone, dont think its simple as you think it should be*, also a Deeplearning engineer **is not an easy work as you think**, the whole knowledge like an ocean, and your knowledge is just a tiny drop of water...
-- Rule No.4: **must** using pretrain weights for **transoformer based backbone**, otherwise your loss will bump;
-
-Make sure you have read **rules** before ask me any questions.
-
-
-
 ## 🆕 News!
 
+- **2022.04.15**: Now, we support the `SparseInst` onnx expport!
 - **2022.03.25**: New instance seg supported! 40 FPS @ 37 mAP!! Which is fast;
 - **2021.09.16**: First transformer based DETR model added, will explore more DETR series models;
 - **2021.08.02**: **YOLOX** arch added, you can train YOLOX as well in this repo;
@@ -145,21 +118,32 @@ Make sure you have read **rules** before ask me any questions.
 
 - See [docs/install.md](docs/install.md)
 
-## 😎 Train
 
-For training, quite simple, same as detectron2:
 
-```
-python train_net.py --config-file configs/coco/darknet53.yaml --num-gpus 8
-```
+## 🤔 Features
 
-If you want train YOLOX, you can using config file `configs/coco/yolox_s.yaml`. All support arch are:
+Some highlights of YOLOv7 are:
 
-- **YOLOX**: anchor free yolo;
-- **YOLOv7**: traditional yolo with some explorations, mainly focus on loss experiments;
-- **YOLOv7P**: traditional yolo merged with decent arch from YOLOX;
-- **YOLOMask**: arch do detection and segmentation at the same time (tbd);
-- **YOLOInsSeg**: instance segmentation based on YOLO detection (tbd);
+- A simple and standard training framework for any detection && instance segmentation tasks, based on detectron2;
+- Supports DETR and many transformer based detection framework out-of-box;
+- Supports easy to deploy pipeline thought onnx.
+- **This is the only framework support YOLOv4 + InstanceSegmentation** in single stage style;
+- Easily plugin into transformers based detector;
+
+We are strongly recommend you send PR if you have any further development on this project, **the only reason for opensource it is just for using community power to make it stronger and further**. It's very welcome for anyone contribute on any features!
+
+## 🧙‍♂️ Pretrained Models
+
+| model | backbone | input | aug | AP<sup>val</sup> |  AP  | FPS | weights |
+| :---- | :------  | :---: | :-: |:--------------: | :--: | :-: | :-----: |
+| [SparseInst](configs/sparse_inst_r50_base.yaml) | [R-50]() | 640 | &#x2718; | 32.8 | - | 44.3 | [model](https://drive.google.com/file/d/12RQLHD5EZKIOvlqW3avUCeYjFG1NPKDy/view?usp=sharing) |
+| [SparseInst](sparse_inst_r50vd_base.yaml) | [R-50-vd](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 640 | &#x2718; | 34.1 | - | 42.6 | [model]()|
+| [SparseInst (G-IAM)](configs/sparse_inst_r50_giam.yaml) | [R-50]() | 608 | &#x2718; | 33.4 | - | 44.6 | [model](https://drive.google.com/file/d/1pXU7Dsa1L7nUiLU9ULG2F6Pl5m5NEguL/view?usp=sharing) |
+| [SparseInst (G-IAM)](configs/sparse_inst_r50_giam_aug.yaml) | [R-50]() | 608 | &#10003; | 34.2 | 34.7 | 44.6 | [model](https://drive.google.com/file/d/1MK8rO3qtA7vN9KVSBdp0VvZHCNq8-bvz/view?usp=sharing) |
+| [SparseInst (G-IAM)](configs/sparse_inst_r50_dcn_giam_aug.yaml) | [R-50-DCN]() | 608 | &#10003;| 36.4 | 36.8 | 41.6 | [model](https://drive.google.com/file/d/1qxdLRRHbIWEwRYn-NPPeCCk6fhBjc946/view?usp=sharing) |
+| [SparseInst (G-IAM)](configs/sparse_inst_r50vd_giam_aug.yaml) | [R-50-vd](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 608 | &#10003;| 35.6 | 36.1 | 42.8| [model](https://drive.google.com/file/d/1dlamg7ych_BdWpPUCuiBXbwE0SXpsfGx/view?usp=sharing) |
+| [SparseInst (G-IAM)](configs/sparse_inst_r50vd_dcn_giam_aug.yaml) | [R-50-vd-DCN](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 608 | &#10003; | 37.4 | 37.9 | 40.0  | [model](https://drive.google.com/file/d/1clYPdCNrDNZLbmlAEJ7wjsrOLn1igOpT/view?usp=sharing)|
+| [SparseInst (G-IAM)](sparse_inst_r50vd_dcn_giam_aug.yaml) | [R-50-vd-DCN](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 640 | &#10003; | 37.7 | 38.1 | 39.3 |  [model](https://drive.google.com/file/d/1clYPdCNrDNZLbmlAEJ7wjsrOLn1igOpT/view?usp=sharing)| 
 
 
 ## 🥰 Demo
@@ -182,27 +166,41 @@ python demo.py --config-file configs/coco/sparseinst/sparse_inst_r50vd_giam_aug.
 python3 demo_lazyconfig.py --config-file configs/new_baselines/panoptic_fpn_regnetx_0.4g.py --opts train.init_checkpoint=output/model_0004999.pth
 ```
 
+## 😎 Train
 
-## 🧙‍♂️ Pretrained Models
+For training, quite simple, same as detectron2:
+
+```
+python train_net.py --config-file configs/coco/darknet53.yaml --num-gpus 8
+```
+
+If you want train YOLOX, you can using config file `configs/coco/yolox_s.yaml`. All support arch are:
+
+- **YOLOX**: anchor free yolo;
+- **YOLOv7**: traditional yolo with some explorations, mainly focus on loss experiments;
+- **YOLOv7P**: traditional yolo merged with decent arch from YOLOX;
+- **YOLOMask**: arch do detection and segmentation at the same time (tbd);
+- **YOLOInsSeg**: instance segmentation based on YOLO detection (tbd);
+
+
+## 😎 Rules
+
+There are some rules you must follow to if you want train on your own dataset:
+
+- Rule No.1: Always set your own anchors on your dataset, using `tools/compute_anchors.py`, this applys to any other anchor-based detection methods as well (EfficientDet etc.);
+- Rule No.2: Keep a faith on your loss will goes down eventually, if not, dig deeper to find out why (but do not post issues repeated caused I might don't know either.).
+- Rule No.3: No one will tells u but it's real: *do not change backbone easily, whole params coupled with your backbone, dont think its simple as you think it should be*, also a Deeplearning engineer **is not an easy work as you think**, the whole knowledge like an ocean, and your knowledge is just a tiny drop of water...
+- Rule No.4: **must** using pretrain weights for **transoformer based backbone**, otherwise your loss will bump;
+
+Make sure you have read **rules** before ask me any questions.
 
-| model | backbone | input | aug | AP<sup>val</sup> |  AP  | FPS | weights |
-| :---- | :------  | :---: | :-: |:--------------: | :--: | :-: | :-----: |
-| [SparseInst](configs/sparse_inst_r50_base.yaml) | [R-50]() | 640 | &#x2718; | 32.8 | - | 44.3 | [model](https://drive.google.com/file/d/12RQLHD5EZKIOvlqW3avUCeYjFG1NPKDy/view?usp=sharing) |
-| [SparseInst](sparse_inst_r50vd_base.yaml) | [R-50-vd](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 640 | &#x2718; | 34.1 | - | 42.6 | [model]()|
-| [SparseInst (G-IAM)](configs/sparse_inst_r50_giam.yaml) | [R-50]() | 608 | &#x2718; | 33.4 | - | 44.6 | [model](https://drive.google.com/file/d/1pXU7Dsa1L7nUiLU9ULG2F6Pl5m5NEguL/view?usp=sharing) |
-| [SparseInst (G-IAM)](configs/sparse_inst_r50_giam_aug.yaml) | [R-50]() | 608 | &#10003; | 34.2 | 34.7 | 44.6 | [model](https://drive.google.com/file/d/1MK8rO3qtA7vN9KVSBdp0VvZHCNq8-bvz/view?usp=sharing) |
-| [SparseInst (G-IAM)](configs/sparse_inst_r50_dcn_giam_aug.yaml) | [R-50-DCN]() | 608 | &#10003;| 36.4 | 36.8 | 41.6 | [model](https://drive.google.com/file/d/1qxdLRRHbIWEwRYn-NPPeCCk6fhBjc946/view?usp=sharing) |
-| [SparseInst (G-IAM)](configs/sparse_inst_r50vd_giam_aug.yaml) | [R-50-vd](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 608 | &#10003;| 35.6 | 36.1 | 42.8| [model](https://drive.google.com/file/d/1dlamg7ych_BdWpPUCuiBXbwE0SXpsfGx/view?usp=sharing) |
-| [SparseInst (G-IAM)](configs/sparse_inst_r50vd_dcn_giam_aug.yaml) | [R-50-vd-DCN](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 608 | &#10003; | 37.4 | 37.9 | 40.0  | [model](https://drive.google.com/file/d/1clYPdCNrDNZLbmlAEJ7wjsrOLn1igOpT/view?usp=sharing)|
-| [SparseInst (G-IAM)](sparse_inst_r50vd_dcn_giam_aug.yaml) | [R-50-vd-DCN](https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet50d_ra2-464e36ba.pth) | 640 | &#10003; | 37.7 | 38.1 | 39.3 |  [model](https://drive.google.com/file/d/1clYPdCNrDNZLbmlAEJ7wjsrOLn1igOpT/view?usp=sharing)| 
 
 ## 🔨 Export ONNX && TensorRTT && TVM
 
 1. `detr`:
 
   ```
   python export_onnx.py --config-file detr/config/file
-  
   ```
 
   this works has been done, inference script included inside `tools`.
@@ -211,7 +209,8 @@ python3 demo_lazyconfig.py --config-file configs/new_baselines/panoptic_fpn_regn
 
   anchorDETR also supported training and exporting to ONNX.
 
-
+3. `SparseInst`:
+   Sparsinst already supported exporting to onnx!!
 
 
 
diff --git a/weights/.gitignore b/weights/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!get_models.sh
diff --git a/weights/get_models.sh b/weights/get_models.sh
@@ -0,0 +1 @@
+gdown https://drive.google.com/file/d/1MK8rO3qtA7vN9KVSBdp0VvZHCNq8-bvz/view\?usp\=sharing --fuzzy
diff --git a/yolov7/modeling/meta_arch/sparseinst.py b/yolov7/modeling/meta_arch/sparseinst.py
@@ -259,5 +259,8 @@ def inference_onnx(self, output, batched_inputs, max_shape, image_sizes):
 
         all_scores = torch.stack(all_scores)
         all_labels = torch.stack(all_labels)
-        all_masks = torch.stack(all_masks)
+        all_masks = torch.stack(all_masks).to(torch.int64)
+        logger.info(f'all_scores: {all_scores.shape}')
+        logger.info(f'all_labels: {all_labels.shape}')
+        logger.info(f'all_masks: {all_masks.shape}')
         return all_masks, all_scores, all_labels
diff --git a/yolov7/modeling/transcoders/encoder_sparseinst.py b/yolov7/modeling/transcoders/encoder_sparseinst.py
@@ -9,11 +9,33 @@
 
 from detectron2.utils.registry import Registry
 from detectron2.layers import Conv2d
+from alfred.utils.log import logger
 
 SPARSE_INST_ENCODER_REGISTRY = Registry("SPARSE_INST_ENCODER")
 SPARSE_INST_ENCODER_REGISTRY.__doc__ = "registry for SparseInst decoder"
 
 
+class MyAdaptiveAvgPool2d(nn.Module):
+    def __init__(self, sz=None):
+        super().__init__()
+        self.sz = sz
+
+    def forward(self, x):
+        inp_size = x.size()
+        kernel_width, kernel_height = inp_size[2], inp_size[3]
+        if self.sz is not None:
+            if isinstance(self.sz, int):
+                kernel_width = math.ceil(inp_size[2] / self.sz)
+                kernel_height = math.ceil(inp_size[3] / self.sz)
+        elif isinstance(self.sz, list) or isinstance(self.sz, tuple):
+            assert len(self.sz) == 2
+            kernel_width = math.ceil(inp_size[2] / self.sz[0])
+            kernel_height = math.ceil(inp_size[3] / self.sz[1])
+        return F.avg_pool2d(
+            input=x, ceil_mode=False, kernel_size=(kernel_width, kernel_height)
+        )
+
+
 class PyramidPoolingModule(nn.Module):
     def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6)):
         super().__init__()
@@ -24,7 +46,11 @@ def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6)):
         self.bottleneck = Conv2d(in_channels + len(sizes) * channels, in_channels, 1)
 
     def _make_stage(self, features, out_features, size):
-        prior = nn.AdaptiveAvgPool2d(output_size=(size, size))
+        if torch.onnx.is_in_onnx_export:
+            logger.warning(f'Replace nn.AdaptiveAvgPool2d for onnx export, size: {size}x{size}')
+            prior = MyAdaptiveAvgPool2d((size, size))
+        else:
+            prior = nn.AdaptiveAvgPool2d(output_size=(size, size))
         conv = Conv2d(features, out_features, 1)
         return nn.Sequential(prior, conv)
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+*`
	`2`	`+!.gitignore`
	`3`	`+!get_models.sh`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+gdown https://drive.google.com/file/d/1MK8rO3qtA7vN9KVSBdp0VvZHCNq8-bvz/view\?usp\=sharing --fuzzy`