Add CE test for dygraph qat (#748) (#749)

Author: juncaipeng

ce_tests/dygraph/qat/readme.md

@@ -0,0 +1,41 @@
+1. 准备
+
+安装需要测试的Paddle版本和PaddleSlim版本。
+
+准备ImageNet数据集，假定解压到`/dataset/ILSVRC2012`文件夹，该文件夹下有`train文件夹、val文件夹、train_list.txt和val_list.txt文件`。
+
+2. 产出量化模型
+
+在`run_train.sh`文件中设置`data_path`为上述ImageNet数据集的路径`/dataset/ILSVRC2012`。
+
+根据实际情况，在`run_train.sh`文件中设置使用GPU的id等参数。
+
+执行`sh run_train.sh` 会对几个分类模型使用动态图量化训练功能进行量化，其中只执行一个epoch。
+执行完后，在`output_models/quant_dygraph`目录下有产出的量化模型。
+
+3. 转换量化模型
+
+在Intel CPU上部署量化模型，需要使用`test/save_quant_model.py`脚本进行模型转换。
+
+如下是对`mobilenet_v1`模型进行转换的示例。
+```
+python src/save_quant_model.py --load_model_path output_models/quant_dygraph/mobilenet_v1 --save_model_path int8_models/mobilenet_v1
+```
+
+4. 测试量化模型
+
+在`run_test.sh`脚本中设置`data_path`为上述ImageNet数据集的路径`/dataset/ILSVRC2012`。
+
+根据实际情况，在`run_test.sh`文件中设置使用GPU的id等参数。
+
+使用`run_test.sh`脚本测试转换前和转换后的量化模型精度。
+
+比如：
+```
+sh run_test.sh output_models/quant_dygraph/mobilenet_v1
+sh run_test.sh int8_models/mobilenet_v1
+```
+
+5. 测试目标
+
+使用动态图量化训练功能，产出`mobilenet_v1`,`mobilenet_v2`,`resnet50`,`vgg16`量化模型，测试转换前后量化模型精度在1%误差范围内。

ce_tests/dygraph/qat/run_test.sh

@@ -0,0 +1,11 @@
+model_path=$1
+test_samples=1000  # if set as -1, use all test samples
+data_path='/dataset/ILSVRC2012/'
+batch_size=16
+
+echo "--------eval model: ${model_name}-------------"
+python ./src/eval.py \
+   --model_path=$model_path \
+   --data_dir=${data_path} \
+   --test_samples=${test_samples} \
+   --batch_size=${batch_size}

ce_tests/dygraph/qat/run_train.sh

@@ -0,0 +1,22 @@
+export CUDA_VISIBLE_DEVICES=5
+
+data_path="/dataset/ILSVRC2012"
+epoch=1
+lr=0.0001
+batch_size=32
+num_workers=3
+output_dir=$PWD/output_models
+
+for model in mobilenet_v1 mobilenet_v2 resnet50 vgg16 
+do
+    python ./src/qat.py \
+        --arch=${model} \
+        --data=${data_path} \
+        --epoch=${epoch} \
+        --batch_size=${batch_size} \
+        --num_workers=${num_workers} \
+        --lr=${lr} \
+        --output_dir=${output_dir} \
+        --enable_quant
+        #--use_pact
+done

ce_tests/dygraph/qat/src/eval.py

@@ -0,0 +1,100 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import numpy as np
+import time
+import sys
+import argparse
+import functools
+import math
+
+import paddle
+import paddle.inference as paddle_infer
+from utility import add_arguments, print_arguments
+import imagenet_dataset as dataset
+
+
+def eval(args):
+    model_file = os.path.join(args.model_path, args.model_filename)
+    params_file = os.path.join(args.model_path, args.params_filename)
+    config = paddle_infer.Config(model_file, params_file)
+    config.enable_mkldnn()
+
+    predictor = paddle_infer.create_predictor(config)
+
+    input_names = predictor.get_input_names()
+    input_handle = predictor.get_input_handle(input_names[0])
+    output_names = predictor.get_output_names()
+    output_handle = predictor.get_output_handle(output_names[0])
+
+    val_dataset = dataset.ImageNetDataset(data_dir=args.data_dir, mode='val')
+    eval_loader = paddle.io.DataLoader(
+        val_dataset, batch_size=args.batch_size, drop_last=True)
+
+    cost_time = 0.
+    total_num = 0.
+    correct_1_num = 0
+    correct_5_num = 0
+    for batch_id, data in enumerate(eval_loader()):
+        img_np = np.array([tensor.numpy() for tensor in data[0]])
+        label_np = np.array([tensor.numpy() for tensor in data[1]])
+
+        input_handle.reshape(img_np.shape)
+        input_handle.copy_from_cpu(img_np)
+
+        t1 = time.time()
+        predictor.run()
+        t2 = time.time()
+        cost_time += (t2 - t1)
+
+        output_data = output_handle.copy_to_cpu()
+
+        for i in range(len(label_np)):
+            label = label_np[i][0]
+            result = output_data[i, :]
+            index = result.argsort()
+            total_num += 1
+            if index[-1] == label:
+                correct_1_num += 1
+            if label in index[-5:]:
+                correct_5_num += 1
+
+        if batch_id % 10 == 0:
+            acc1 = correct_1_num / total_num
+            acc5 = correct_5_num / total_num
+            avg_time = cost_time / total_num
+            print(
+                "batch_id {}, acc1 {:.3f}, acc5 {:.3f}, avg time {:.5f} sec/img".
+                format(batch_id, acc1, acc5, avg_time))
+
+        if args.test_samples > 0 and \
+            (batch_id + 1)* args.batch_size >= args.test_samples:
+            break
+
+    acc1 = correct_1_num / total_num
+    acc5 = correct_5_num / total_num
+    print("End test: test_acc1 {:.3f}, test_acc5 {:.5f}".format(acc1, acc5))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    add_arg = functools.partial(add_arguments, argparser=parser)
+    add_arg('model_path', str, "", "The inference model path.")
+    add_arg('model_filename', str, "int8_infer.pdmodel", "model filename")
+    add_arg('params_filename', str, "int8_infer.pdiparams", "params filename")
+    add_arg('data_dir', str, "/dataset/ILSVRC2012/",
+            "The ImageNet dataset root dir.")
+    add_arg('test_samples', int, -1,
+            "Test samples. If set -1, use all test samples")
+    add_arg('batch_size', int, 16, "Batch size.")
+
+    args = parser.parse_args()
+    print_arguments(args)
+
+    eval(args)
+
+
+if __name__ == '__main__':
+    main()