CUDA error: initialization error in training

[lokeaichirou]

RuntimeError Traceback (most recent call last)
in ()
1 if name == "main":
----> 2 main(args)

5 frames
in main(args)
69
70 epoch_loss = train(train_epoch, model, optimizer, train_loader,
---> 71 vis, vis_window, args)
72 all_training_losses.append(epoch_loss)
73

in train(epoch, model, optimizer, train_loader, vis, vis_window, args)
7
8 sample_prob = min(args.sample_prob, int(epoch / 5) * 0.05)
----> 9 for train_iter, data in enumerate(train_loader):
10 (img_batch, tempo_seg_pos, tempo_seg_neg, sentence_batch) = data
11 img_batch = Variable(img_batch)

/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py in next(self)
515 if self._sampler_iter is None:
516 self._reset()
--> 517 data = self._next_data()
518 self._num_yielded += 1
519 if self._dataset_kind == _DatasetKind.Iterable and \

/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py in _next_data(self)
1197 else:
1198 del self._task_info[idx]
-> 1199 return self._process_data(data)
1200
1201 def _try_put_index(self):

/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py in _process_data(self, data)
1223 self._try_put_index()
1224 if isinstance(data, ExceptionWrapper):
-> 1225 data.reraise()
1226 return data
1227

/usr/local/lib/python3.7/dist-packages/torch/_utils.py in reraise(self)
427 # have message field
428 raise self.exc_type(message=msg)
--> 429 raise self.exc_type(msg)
430
431

RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/worker.py", line 202, in _worker_loop
data = fetcher.fetch(index)
File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
return self.collate_fn(data)
File "/content/anet_dataset.py", line 514, in anet_collate_fn
sentence_batch[batch_idx] = sentence.data
RuntimeError: CUDA error: initialization error