Improve ease of use of datasets, and remove some unfinished code

Author: bjerga

datasets.py

@@ -0,0 +1,83 @@
+import keras.datasets as _kds
+from keras.utils.np_utils import to_categorical
+
+
+class _Dataset:
+    def __init__(self, name, input_shape, load_fn, use_1d=False):
+        if use_1d:
+            name += '_1d'
+
+            input_shape_1d = 1
+            for dim in input_shape:
+                input_shape_1d *= dim
+            input_shape = input_shape_1d,
+
+        self.name = name
+        self.input_shape = input_shape
+        self._load_fn = load_fn
+
+    def load_train_data(self):
+        (train_data, train_targets), _ = self._load_fn()
+        return self._process(train_data, train_targets)
+
+    def load_test_data(self):
+        _, (test_data, test_targets) = self._load_fn()
+        return self._process(test_data, test_targets)
+
+    def _process(self, data, targets):
+        # reshape data (to fit input layer)
+        data = data.reshape((data.shape[0],) + self.input_shape)
+
+        # normalize data to [0.0, 1.0]
+        data = data.astype('float32')
+        data /= 255.0
+
+        # make targets categorical (to fit output layer)
+        targets = to_categorical(targets, 10)
+
+        return data, targets
+
+
+class _CIFAR10(_Dataset):
+    def __init__(self, use_1d=False):
+        name = 'cifar10'
+        input_shape = (32, 32, 3)
+        load_fn = _kds.cifar10.load_data
+        super(_CIFAR10, self).__init__(name, input_shape, load_fn, use_1d)
+
+
+class _CIFAR100(_Dataset):
+    def __init__(self, use_1d=False):
+        name = 'cifar100'
+        input_shape = (32, 32, 3)
+        load_fn = _kds.cifar100.load_data
+        super(_CIFAR100, self).__init__(name, input_shape, load_fn, use_1d)
+
+
+class _FASHION_MNIST(_Dataset):
+    def __init__(self, use_1d=False):
+        name = 'fashion_mnist'
+        input_shape = (28, 28, 1)
+        load_fn = _kds.fashion_mnist.load_data
+        super(_FASHION_MNIST, self).__init__(name, input_shape, load_fn, use_1d)
+
+
+class _MNIST(_Dataset):
+    def __init__(self, use_1d=False):
+        name = 'mnist'
+        input_shape = (28, 28, 1)
+        load_fn = _kds.mnist.load_data
+        super(_MNIST, self).__init__(name, input_shape, load_fn, use_1d)
+
+
+cifar10 = _CIFAR10()
+cifar10_1d = _CIFAR10(use_1d=True)
+
+cifar100 = _CIFAR100()
+cifar100_1d = _CIFAR100(use_1d=True)
+
+fashion_mnist = _FASHION_MNIST()
+fashion_mnist_1d = _FASHION_MNIST(use_1d=True)
+
+mnist = _MNIST()
+mnist_1d = _MNIST(use_1d=True)

loader.py

@@ -1,24 +1,20 @@
-from loader import get_input_shape, mnist_train_data, mnist_test_data
-
 import numpy as np
 import tensorflow as tf
-from os import mkdir
-from os.path import join
 from time import time
 
-from keras.models import Sequential, load_model
+from keras.models import Sequential
 from keras.layers import Dense
 from keras.activations import relu, softmax
 from keras.losses import categorical_crossentropy
 from keras.optimizers import Adam
 from keras.metrics import categorical_accuracy
 
+# NOTE: change import to use different dataset
+from datasets import mnist_1d as dataset
+
 # removes pestering tensorflow warnings
 tf.logging.set_verbosity(tf.logging.ERROR)
 
-MODELS_FOLDER = 'models'
-MODEL_EXT = '.h5'
-
 
 def main():
     start_time = time()
@@ -33,10 +29,8 @@ def main():
 
 
 def create():
-    input_shape = get_input_shape()
-
     model = Sequential([
-        Dense(units=1, activation=relu, input_shape=input_shape),
+        Dense(units=1, activation=relu, input_shape=dataset.input_shape),
         Dense(units=10, activation=softmax)
     ])
 
@@ -53,7 +47,7 @@ def create():
 def train(model, epochs=10):
     print('\nCommence model training\n')
 
-    train_images, train_targets = mnist_train_data()
+    train_images, train_targets = dataset.load_train_data()
 
     # https://keras.io/models/sequential/#fit
     model.fit(
@@ -74,7 +68,7 @@ def train(model, epochs=10):
 def test(model, verbose=False):
     print('\nCommence model testing\n')
 
-    test_images, test_targets = mnist_test_data()
+    test_images, test_targets = dataset.load_test_data()
 
     # test for all indices and count correctly classified
     correctly_classified = 0
@@ -104,7 +98,7 @@ def test(model, verbose=False):
 def test_with_evaluate(model, verbose=True):
     print('\nCommence model testing\n')
 
-    test_images, test_targets = mnist_test_data()
+    test_images, test_targets = dataset.load_test_data()
 
     # https://keras.io/models/sequential/#evaluate
     loss_and_metrics = model.evaluate(
@@ -117,21 +111,4 @@ def test_with_evaluate(model, verbose=True):
     print('\nCompleted model testing')
 
 
-def save(model, filename):
-    # make path, if not exists
-    try:
-        mkdir(MODELS_FOLDER)
-    except FileExistsError:
-        # file exists, which is what we want
-        pass
-
-    path = join(MODELS_FOLDER, filename + MODEL_EXT)
-    model.save(path)
-
-
-def load(filename):
-    path = join(MODELS_FOLDER, filename + MODEL_EXT)
-    return load_model(path)
-
-
 main()