Training errors? #40
Description
Hi, and thank you for making this code available. Is it fast enough for real time use? How long does the function take to return an alpha from the trimap/colour input?
I am trying to train some data, and am getting the following error:
python train.py Using TensorFlow backend. 2019-04-25 16:09:02.261034: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 WARNING:tensorflow:From C:\Users\bsharp\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. WARNING:tensorflow:From C:\Python37\lib\site-packages\keras\backend\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use rateinstead ofkeep_prob. Rate should be set to rate = 1 - keep_prob`.
Layer (type) Output Shape Param # Connected to
input_1 (InputLayer) (None, 320, 320, 4) 0
zero_padding2d_1 (ZeroPadding2D (None, 322, 322, 4) 0 input_1[0][0]
conv1_1 (Conv2D) (None, 320, 320, 64) 2368 zero_padding2d_1[0][0]
zero_padding2d_2 (ZeroPadding2D (None, 322, 322, 64) 0 conv1_1[0][0]
conv1_2 (Conv2D) (None, 320, 320, 64) 36928 zero_padding2d_2[0][0]
max_pooling2d_1 (MaxPooling2D) (None, 160, 160, 64) 0 conv1_2[0][0]
zero_padding2d_3 (ZeroPadding2D (None, 162, 162, 64) 0 max_pooling2d_1[0][0]
conv2_1 (Conv2D) (None, 160, 160, 128 73856 zero_padding2d_3[0][0]
zero_padding2d_4 (ZeroPadding2D (None, 162, 162, 128 0 conv2_1[0][0]
conv2_2 (Conv2D) (None, 160, 160, 128 147584 zero_padding2d_4[0][0]
max_pooling2d_2 (MaxPooling2D) (None, 80, 80, 128) 0 conv2_2[0][0]
zero_padding2d_5 (ZeroPadding2D (None, 82, 82, 128) 0 max_pooling2d_2[0][0]
conv3_1 (Conv2D) (None, 80, 80, 256) 295168 zero_padding2d_5[0][0]
zero_padding2d_6 (ZeroPadding2D (None, 82, 82, 256) 0 conv3_1[0][0]
conv3_2 (Conv2D) (None, 80, 80, 256) 590080 zero_padding2d_6[0][0]
zero_padding2d_7 (ZeroPadding2D (None, 82, 82, 256) 0 conv3_2[0][0]
conv3_3 (Conv2D) (None, 80, 80, 256) 590080 zero_padding2d_7[0][0]
max_pooling2d_3 (MaxPooling2D) (None, 40, 40, 256) 0 conv3_3[0][0]
zero_padding2d_8 (ZeroPadding2D (None, 42, 42, 256) 0 max_pooling2d_3[0][0]
conv4_1 (Conv2D) (None, 40, 40, 512) 1180160 zero_padding2d_8[0][0]
zero_padding2d_9 (ZeroPadding2D (None, 42, 42, 512) 0 conv4_1[0][0]
conv4_2 (Conv2D) (None, 40, 40, 512) 2359808 zero_padding2d_9[0][0]
zero_padding2d_10 (ZeroPadding2 (None, 42, 42, 512) 0 conv4_2[0][0]
conv4_3 (Conv2D) (None, 40, 40, 512) 2359808 zero_padding2d_10[0][0]
max_pooling2d_4 (MaxPooling2D) (None, 20, 20, 512) 0 conv4_3[0][0]
zero_padding2d_11 (ZeroPadding2 (None, 22, 22, 512) 0 max_pooling2d_4[0][0]
conv5_1 (Conv2D) (None, 20, 20, 512) 2359808 zero_padding2d_11[0][0]
zero_padding2d_12 (ZeroPadding2 (None, 22, 22, 512) 0 conv5_1[0][0]
conv5_2 (Conv2D) (None, 20, 20, 512) 2359808 zero_padding2d_12[0][0]
zero_padding2d_13 (ZeroPadding2 (None, 22, 22, 512) 0 conv5_2[0][0]
conv5_3 (Conv2D) (None, 20, 20, 512) 2359808 zero_padding2d_13[0][0]
max_pooling2d_5 (MaxPooling2D) (None, 10, 10, 512) 0 conv5_3[0][0]
up_sampling2d_1 (UpSampling2D) (None, 20, 20, 512) 0 max_pooling2d_5[0][0]
reshape_1 (Reshape) (None, 1, 20, 20, 51 0 conv5_3[0][0]
reshape_2 (Reshape) (None, 1, 20, 20, 51 0 up_sampling2d_1[0][0]
concatenate_1 (Concatenate) (None, 2, 20, 20, 51 0 reshape_1[0][0]
reshape_2[0][0]
unpooling_1 (Unpooling) (None, 20, 20, 512) 0 concatenate_1[0][0]
deconv5_1 (Conv2D) (None, 20, 20, 512) 2359808 unpooling_1[0][0]
batch_normalization_1 (BatchNor (None, 20, 20, 512) 2048 deconv5_1[0][0]
deconv5_2 (Conv2D) (None, 20, 20, 512) 2359808 batch_normalization_1[0][0]
batch_normalization_2 (BatchNor (None, 20, 20, 512) 2048 deconv5_2[0][0]
deconv5_3 (Conv2D) (None, 20, 20, 512) 2359808 batch_normalization_2[0][0]
batch_normalization_3 (BatchNor (None, 20, 20, 512) 2048 deconv5_3[0][0]
up_sampling2d_2 (UpSampling2D) (None, 40, 40, 512) 0 batch_normalization_3[0][0]
reshape_3 (Reshape) (None, 1, 40, 40, 51 0 conv4_3[0][0]
reshape_4 (Reshape) (None, 1, 40, 40, 51 0 up_sampling2d_2[0][0]
concatenate_2 (Concatenate) (None, 2, 40, 40, 51 0 reshape_3[0][0]
reshape_4[0][0]
unpooling_2 (Unpooling) (None, 40, 40, 512) 0 concatenate_2[0][0]
deconv4_1 (Conv2D) (None, 40, 40, 256) 1179904 unpooling_2[0][0]
batch_normalization_4 (BatchNor (None, 40, 40, 256) 1024 deconv4_1[0][0]
deconv4_2 (Conv2D) (None, 40, 40, 256) 590080 batch_normalization_4[0][0]
batch_normalization_5 (BatchNor (None, 40, 40, 256) 1024 deconv4_2[0][0]
deconv4_3 (Conv2D) (None, 40, 40, 256) 590080 batch_normalization_5[0][0]
batch_normalization_6 (BatchNor (None, 40, 40, 256) 1024 deconv4_3[0][0]
up_sampling2d_3 (UpSampling2D) (None, 80, 80, 256) 0 batch_normalization_6[0][0]
reshape_5 (Reshape) (None, 1, 80, 80, 25 0 conv3_3[0][0]
reshape_6 (Reshape) (None, 1, 80, 80, 25 0 up_sampling2d_3[0][0]
concatenate_3 (Concatenate) (None, 2, 80, 80, 25 0 reshape_5[0][0]
reshape_6[0][0]
unpooling_3 (Unpooling) (None, 80, 80, 256) 0 concatenate_3[0][0]
deconv3_1 (Conv2D) (None, 80, 80, 128) 295040 unpooling_3[0][0]
batch_normalization_7 (BatchNor (None, 80, 80, 128) 512 deconv3_1[0][0]
deconv3_2 (Conv2D) (None, 80, 80, 128) 147584 batch_normalization_7[0][0]
batch_normalization_8 (BatchNor (None, 80, 80, 128) 512 deconv3_2[0][0]
deconv3_3 (Conv2D) (None, 80, 80, 128) 147584 batch_normalization_8[0][0]
batch_normalization_9 (BatchNor (None, 80, 80, 128) 512 deconv3_3[0][0]
up_sampling2d_4 (UpSampling2D) (None, 160, 160, 128 0 batch_normalization_9[0][0]
reshape_7 (Reshape) (None, 1, 160, 160, 0 conv2_2[0][0]
reshape_8 (Reshape) (None, 1, 160, 160, 0 up_sampling2d_4[0][0]
concatenate_4 (Concatenate) (None, 2, 160, 160, 0 reshape_7[0][0]
reshape_8[0][0]
unpooling_4 (Unpooling) (None, 160, 160, 128 0 concatenate_4[0][0]
deconv2_1 (Conv2D) (None, 160, 160, 64) 73792 unpooling_4[0][0]
batch_normalization_10 (BatchNo (None, 160, 160, 64) 256 deconv2_1[0][0]
deconv2_2 (Conv2D) (None, 160, 160, 64) 36928 batch_normalization_10[0][0]
batch_normalization_11 (BatchNo (None, 160, 160, 64) 256 deconv2_2[0][0]
up_sampling2d_5 (UpSampling2D) (None, 320, 320, 64) 0 batch_normalization_11[0][0]
reshape_9 (Reshape) (None, 1, 320, 320, 0 conv1_2[0][0]
reshape_10 (Reshape) (None, 1, 320, 320, 0 up_sampling2d_5[0][0]
concatenate_5 (Concatenate) (None, 2, 320, 320, 0 reshape_9[0][0]
reshape_10[0][0]
unpooling_5 (Unpooling) (None, 320, 320, 64) 0 concatenate_5[0][0]
deconv1_1 (Conv2D) (None, 320, 320, 64) 36928 unpooling_5[0][0]
batch_normalization_12 (BatchNo (None, 320, 320, 64) 256 deconv1_1[0][0]
deconv1_2 (Conv2D) (None, 320, 320, 64) 36928 batch_normalization_12[0][0]
batch_normalization_13 (BatchNo (None, 320, 320, 64) 256 deconv1_2[0][0]
lambda_11 (Lambda) (None, 320, 320, 3) 0 input_1[0][0]
pred (Conv2D) (None, 320, 320, 1) 577 batch_normalization_13[0][0]
concatenate_6 (Concatenate) (None, 320, 320, 4) 0 lambda_11[0][0]
pred[0][0]
conv2d_1 (Conv2D) (None, 320, 320, 64) 2368 concatenate_6[0][0]
batch_normalization_14 (BatchNo (None, 320, 320, 64) 256 conv2d_1[0][0]
conv2d_2 (Conv2D) (None, 320, 320, 64) 36928 batch_normalization_14[0][0]
batch_normalization_15 (BatchNo (None, 320, 320, 64) 256 conv2d_2[0][0]
conv2d_3 (Conv2D) (None, 320, 320, 64) 36928 batch_normalization_15[0][0]
batch_normalization_16 (BatchNo (None, 320, 320, 64) 256 conv2d_3[0][0]
refinement_pred (Conv2D) (None, 320, 320, 1) 577 batch_normalization_16[0][0]
Total params: 25,019,458
Trainable params: 25,013,186
Non-trainable params: 6,272
None
Epoch 1/1000
Using TensorFlow backend.
Using TensorFlow backend.
Using TensorFlow backend.
Using TensorFlow backend.
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Python37\lib\multiprocessing\pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "C:\Python37\lib\site-packages\keras\utils\data_utils.py", line 401, in get_index
return _SHARED_SEQUENCES[uid][i]
File "X:\dev\sandbox_bsharp\ML\Deep-Image-Matting\data_generator.py", line 123, in getitem
im_name = fg_files[fcount]
IndexError: list index out of range
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "train.py", line 79, in
workers=2
File "C:\Python37\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Python37\lib\site-packages\keras\engine\training.py", line 1418, in fit_generator
initial_epoch=initial_epoch)
File "C:\Python37\lib\site-packages\keras\engine\training_generator.py", line 181, in fit_generator
generator_output = next(output_generator)
File "C:\Python37\lib\site-packages\keras\utils\data_utils.py", line 601, in get
six.reraise(*sys.exc_info())
File "C:\Python37\lib\site-packages\six.py", line 693, in reraise
raise value
File "C:\Python37\lib\site-packages\keras\utils\data_utils.py", line 595, in get
inputs = self.queue.get(block=True).get()
File "C:\Python37\lib\multiprocessing\pool.py", line 657, in get
raise self._value
IndexError: list index out of range
`
What am i doing wrong?
Thanks!