add data pattern

Author: Jin-Ying

dwf/common/exception.py

@@ -47,3 +47,8 @@ def __init__(self, status, msg):
 Exceptions for Development
 '''
 NOT_IMPLEMENTED = DWFException(5001, '方法未实现')
+
+'''
+Exceptions for Data Pattern
+'''
+DATA_PATTERN_MISMATCH = DWFException(6001, '数据模式不匹配')

dwf/datapattern/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['metadata']

dwf/datapattern/metadata/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['base_pattern', '2imageFolder4Detection_txt']

dwf/datapattern/metadata/base_pattern.py

@@ -0,0 +1,18 @@
+import json
+
+class BasePattern(object):
+
+    def __init__(self):
+        pass
+
+    def dumps(self):
+        return json.dumps(self, default=lambda o: o.__dict__)
+
+    def load(self, json_str):
+        self.__dict__ = json.loads(json_str)
+
+    def check(self, folder_path):
+        raise NotImplementedError
+
+    def generate(self, folder_path):
+        raise NotImplementedError

dwf/datapattern/metadata/image_folder4_classification.py

@@ -0,0 +1,62 @@
+from dwf.datapattern.metadata.base_pattern import BasePattern
+from dwf.common.exception import DATA_PATTERN_MISMATCH
+from torchvision.datasets.folder import ImageFolder
+import os
+import cv2
+from PIL import Image
+
+class ImageFolder4Classfication(BasePattern):
+    def __init__(self):
+        super(ImageFolder4Classfication, self).__init__()
+        self.data_type = 'image'
+        self.organization = 'imageFolder4Classfication'
+        self.algos = 'resnet'
+        self.organization_parameter_width = None
+        self.organization_parameter_height = None
+        self.organization_parameter_channel = None
+        self.organization_parameter_preprocess_resize_need = True
+        self.organization_parameter_preprocess_resize_size = 224
+        self.organization_parameter_preprocess_crop_need = True
+        self.organization_parameter_preprocess_crop_need = 256
+        self.organization_parameter_preprocess_shuffle_need = True
+        self.organization_parameter_preprocess_normalization_need = True
+        self.organization_parameter_preprocess_normalization_mean = [0.5, 0.5, 0.5]
+        self.organization_parameter_preprocess_normalization_std = [0.5, 0.5, 0.5]
+
+        self.semantic = '10'
+
+    def check(self, folder_path):
+        try:
+            dataset_pending = ImageFolder(root=folder_path)
+        except:
+            raise DATA_PATTERN_MISMATCH
+        return True
+
+    def generate(self, folder_path):
+        max_width = 0
+        min_width = 10000
+        max_height = 0
+        min_height = 10000
+        channel_num = None
+        for sub_dir in os.listdir(folder_path):
+            for file in os.listdir(os.path.join(folder_path, sub_dir)):
+                im = cv2.imread(os.path.join(folder_path, sub_dir,file))
+                width, height, channel = im.shape[0],im.shape[1],im.shape[2]
+                if channel_num is None:
+                    channel_num = channel
+                if channel != channel_num:
+                    raise DATA_PATTERN_MISMATCH
+                max_width = max(max_width, width)
+                min_width = min(min_width, width)
+                max_height = max(max_height, height)
+                min_height = min(min_height, height)
+
+        self.organization_parameter_width = [min_width, max_width]
+        self.organization_parameter_height = [min_height, max_height]
+        self.organization_parameter_channel = channel_num
+        return self.dumps()
+
+# pattern = ImageFolder4Classfication()
+# print(pattern.generate('/Users/sherry/Desktop/0920crop'))
+
+

dwf/datapattern/metadata/two_image_folder4_detection_txt.py

@@ -0,0 +1,69 @@
+from dwf.datapattern.metadata.base_pattern import BasePattern
+from dwf.common.exception import DATA_PATTERN_MISMATCH
+import os
+import cv2
+from PIL import Image
+
+class TwoImageFolder4Detection_txt(BasePattern):
+    def __init__(self):
+        super(TwoImageFolder4Detection_txt, self).__init__()
+        self.data_type = 'image'
+        self.organization = 'twoImageFolder4Detection@txt'
+        self.algos = 'ssd'
+        self.organization_parameter_width = None
+        self.organization_parameter_height = None
+        self.organization_parameter_channel = None
+        self.organization_parameter_preprocess_resize_need = True
+        self.organization_parameter_preprocess_resize_size = 224
+        self.organization_parameter_preprocess_crop_need = True
+        self.organization_parameter_preprocess_crop_need = 256
+        self.organization_parameter_preprocess_shuffle_need = True
+        self.organization_parameter_preprocess_normalization_need = True
+        self.organization_parameter_preprocess_normalization_mean = [0.5, 0.5, 0.5]
+        self.organization_parameter_preprocess_normalization_std = [0.5, 0.5, 0.5]
+
+        self.semantic = '01'
+
+    def check(self, folder_path):
+        image_path = os.path.join(folder_path, 'Images')
+        label_path = os.path.join(folder_path, 'Labels')
+
+        if not os.path.exists(image_path):
+            raise DATA_PATTERN_MISMATCH
+
+        if not os.path.exists(label_path):
+            raise DATA_PATTERN_MISMATCH
+
+        image_list = os.listdir(image_path)
+        label_list = os.listdir(label_path)
+
+        if len(image_list) != len(label_list):
+            raise DATA_PATTERN_MISMATCH
+        return True
+
+    def generate(self, folder_path):
+        max_width = 0
+        min_width = 10000
+        max_height = 0
+        min_height = 10000
+        channel_num = None
+        for file in os.listdir(os.path.join(folder_path, 'Images')):
+            im = cv2.imread(os.path.join(folder_path, 'Images',file))
+            width, height, channel = im.shape[0],im.shape[1],im.shape[2]
+            if channel_num is None:
+                channel_num = channel
+            if channel != channel_num:
+                raise DATA_PATTERN_MISMATCH
+            max_width = max(max_width, width)
+            min_width = min(min_width, width)
+            max_height = max(max_height, height)
+            min_height = min(min_height, height)
+        self.organization_parameter_width = [min_width, max_width]
+        self.organization_parameter_height = [min_height, max_height]
+        self.organization_parameter_channel = channel_num
+        return self.dumps()
+
+# pattern = TwoImageFolder4Detection_txt()
+# print(pattern.generate('/Users/sherry/Desktop/xlearn_data/1228'))
+
+