第一次提交

Author: hhaAndroid

.gitignore

@@ -14,8 +14,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
 parts/
 sdist/
 var/

lib/__init__.py

@@ -0,0 +1,5 @@
+from .dataset import Dataset
+from .sampler import Sampler, BatchSampler, SequentialSampler, RandomSampler
+from .dataloader import DataLoader
+
+__all__ = ['Dataset', 'Sampler', 'BatchSampler', 'SequentialSampler', 'RandomSampler', 'DataLoader']

lib/dataloader.py

@@ -0,0 +1,50 @@
+from torch.utils.data._utils.collate import default_collate
+
+from .sampler import BatchSampler, SequentialSampler, RandomSampler
+
+
+class DataLoader(object):
+    def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None,
+                 batch_sampler=None, collate_fn=None, drop_last=False):
+        self.dataset = dataset
+
+        # 因为这两个功能是冲突的，假设shuffle=True,但是sampler里面是SequentialSampler，那么就违背设计思想了
+        if sampler is not None and shuffle:
+            raise ValueError('sampler option is mutually exclusive with '
+                             'shuffle')
+
+        if batch_sampler is not None:
+            # 一旦设置了batch_sampler，那么batch_size、shuffle、sampler和drop_last四个参数就不能传入
+            # 因为这4个参数功能和batch_sampler功能冲突了
+            if batch_size != 1 or shuffle or sampler is not None or drop_last:
+                raise ValueError('batch_sampler option is mutually exclusive '
+                                 'with batch_size, shuffle, sampler, and '
+                                 'drop_last')
+            batch_size = None
+            drop_last = False
+
+        if sampler is None:
+            if shuffle:
+                sampler = RandomSampler(dataset)
+            else:
+                sampler = SequentialSampler(dataset)
+
+        # 也就是说batch_sampler必须要存在，你如果没有设置，那么采用默认类
+        if batch_sampler is None:
+            batch_sampler = BatchSampler(sampler, batch_size, drop_last)
+
+        self.batch_size = batch_size
+        self.drop_last = drop_last
+        self.sampler = sampler
+        self.batch_sampler = batch_sampler
+
+        if collate_fn is None:
+            collate_fn = default_collate
+
+        self.collate_fn = collate_fn
+
+    @property
+    def _auto_collation(self):
+        return self.batch_sampler is not None
+
+

lib/dataset.py

@@ -0,0 +1,7 @@
+class Dataset(object):
+    def __getitem__(self, index):
+        raise NotImplementedError
+
+    def __len__(self):
+        raise NotImplementedError
+

lib/sampler.py

@@ -0,0 +1,83 @@
+import torch
+
+
+class Sampler(object):
+    def __init__(self, data_source):
+        pass
+
+    def __iter__(self):
+        raise NotImplementedError
+
+    def __len__(self):
+        raise NotImplementedError
+
+
+class BatchSampler(Sampler):
+    def __init__(self, sampler, batch_size, drop_last, data_source):
+        super(BatchSampler, self).__init__(data_source)
+        self.sampler = sampler
+        self.batch_size = batch_size
+        self.drop_last = drop_last
+
+    def __iter__(self):
+        batch = []
+        for idx in self.sampler:
+            batch.append(idx)
+            if len(batch) == self.batch_size:
+                yield batch
+                batch = []
+        if len(batch) > 0 and not self.drop_last:
+            yield batch
+
+    def __len__(self):
+        if self.drop_last:
+            return len(self.sampler) // self.batch_size
+        else:
+            return (len(self.sampler) + self.batch_size - 1) // self.batch_size
+
+
+class SequentialSampler(Sampler):
+
+    def __init__(self, data_source):
+        super(SequentialSampler, self).__init__(data_source)
+        self.data_source = data_source
+
+    def __iter__(self):
+        return iter(range(len(self.data_source)))
+
+    def __len__(self):
+        return len(self.data_source)
+
+
+class RandomSampler(Sampler):
+    def __init__(self, data_source, replacement=False, num_samples=None):
+        super(RandomSampler, self).__init__(data_source)
+        self.data_source = data_source
+        self.replacement = replacement
+        self._num_samples = num_samples
+
+        if self._num_samples is not None and not replacement:
+            raise ValueError("With replacement=False, num_samples should not be specified, "
+                             "since a random permute will be performed.")
+
+        if not isinstance(self.num_samples, int) or self.num_samples <= 0:
+            raise ValueError("num_samples should be a positive integer "
+                             "value, but got num_samples={}".format(self.num_samples))
+
+    @property
+    def num_samples(self) -> int:
+        if self._num_samples is None:
+            return len(self.data_source)
+        return self._num_samples
+
+    def __iter__(self):
+        n = len(self.data_source)
+        if self.replacement:
+            for _ in range(self.num_samples // 32):
+                yield from torch.randint(high=n, size=(32,), dtype=torch.int64).tolist()
+            yield from torch.randint(high=n, size=(self.num_samples % 32,), dtype=torch.int64).tolist()
+        else:
+            yield from torch.randperm(n).tolist()
+
+    def __len__(self):
+        return self.num_samples

main.py

@@ -0,0 +1,16 @@
+# This is a sample Python script.
+
+# Press Shift+F10 to execute it or replace it with your code.
+# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
+
+import torch.utils.data
+def print_hi(name):
+    # Use a breakpoint in the code line below to debug your script.
+    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.
+
+
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    print_hi('PyCharm')
+
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/