Validation loop outputs are the same for every input

[avKostanov]

Hello everyone, I can't figure out why my model is performing on train loop, but for validation, outputs the same stuff for every sample in the whole epoch. Snippet is below. As you can see, I've checked, that dataloaders are correct.

learning_rate: 0.0005
weight_decay: 0.005
batch_size: 2

train_dataloader = torch.utils.data.DataLoader(
            train_dataset, batch_size=2, num_workers=4)
valid_dataloader = torch.utils.data.DataLoader(
            valid_dataset, batch_size=2, num_workers=4)
            
lass CNNModel(pl.LightningModule):

    def __init__(
        self,
        learning_rate,
        weight_decay,
        batch_size
    ):
        super().__init__()
        self.learning_rate = learning_rate
        self.weight_decay = weight_decay
        self.batch_size = batch_size
        self.hemm_types = ['IPH', 'IVH', 'SDH', 'EDH', 'SAH']

        self.model = EfficientNetBN(
            model_name='efficientnet-b2',
            pretrained=False,
            spatial_dims=3,
            in_channels=1,
            num_classes=5
        )
        self.criterion = torch.nn.BCEWithLogitsLoss()


    def forward(self, x):
        return self.model(x)

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(
            self.parameters(),
            lr=self.learning_rate,
            weight_decay=self.weight_decay
        )
        return optimizer

    def training_step(self, batch, batch_idx):
        x, y = batch['image'][tio.DATA], batch['label']
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch['image'][tio.DATA], batch['label']
        y_hat = self(x)
        print(y_hat)
        print('\n y', y)
        loss = self.criterion(y_hat, y)
        return loss

model = CNNModel(
    learning_rate=config['training']['learning_rate'],
    weight_decay=config['training']['weight_decay'],
    batch_size=config['training']['batch_size']
)

wandb_logger = WandbLogger(
    project="hemorrage",
    log_model='all',
    config=config
)
wandb_logger.watch(model, 'all')
trainer = Trainer(
    precision=16,
    accelerator="gpu",
    devices=1,
    logger=wandb_logger,
    track_grad_norm=2,
    max_epochs=3,
    log_every_n_steps=1,
    limit_train_batches=3,
    limit_val_batches=3
)

model.cuda()
with torch.no_grad():
    for i, batch in enumerate(valid_dataloader):
        x, y =  batch['image'][tio.DATA].cuda(), batch['label'].cuda()
        print(model(x))
        print('y', y)
        if i > 4:
            break

tensor([[-2.4252, -1.1539,  0.6247, -2.5878, -1.1578],
        [-6.6326, -3.3592,  2.1962,  4.7116,  1.1905]], device='cuda:0')
y tensor([[1., 0., 1., 0., 1.],
        [1., 0., 0., 0., 0.]], device='cuda:0')
tensor([[-1.0112,  0.3689, -2.6230, -1.2685, -1.2422],
        [-2.6487, -1.8573,  4.3041,  4.7194,  0.3905]], device='cuda:0')
y tensor([[0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 1.]], device='cuda:0')
tensor([[-3.1488, -2.0642,  1.0020,  1.2713,  0.3937],
        [-3.2816, -0.9960,  0.8024,  1.1730, -0.5235]], device='cuda:0')
y tensor([[1., 1., 0., 0., 0.],
        [0., 1., 0., 0., 1.]], device='cuda:0')
tensor([[-0.5105, -1.0014,  0.1273, -0.3777, -0.3808],
        [-4.7996, -4.8799,  0.2024,  3.6530,  0.8399]], device='cuda:0')
y tensor([[1., 0., 1., 1., 1.],
        [0., 0., 0., 0., 0.]], device='cuda:0')
tensor([[-1.9868, -1.6867,  0.2548,  2.2171,  1.7830],
        [-1.0199,  0.3248,  0.8840, -0.3296, -0.7767]], device='cuda:0')
y tensor([[1., 1., 0., 0., 0.],
        [0., 0., 0., 0., 1.]], device='cuda:0')
tensor([[-0.8025, -0.2954,  0.9403, -1.0009, -1.6953],
        [-3.1871, -2.8466,  2.3447,  1.5574,  3.4682]], device='cuda:0')
y tensor([[1., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0.]], device='cuda:0')

trainer.fit(
    model, 
    train_dataloader,
    valid_dataloader
)
# only validation loop
Epoch 0:  50%|█████     | 3/6 [00:15<00:15,  5.04s/it, loss=0.949, v_num=2qi6]tensor([[-0.0470, -0.0829, -0.0731, -0.0068, -0.0091],
        [-0.0470, -0.0829, -0.0732, -0.0068, -0.0091]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[1., 0., 1., 0., 1.],
        [1., 0., 0., 0., 0.]], device='cuda:0')
Epoch 0:  67%|██████▋   | 4/6 [00:30<00:15,  7.73s/it, loss=0.949, v_num=2qi6]tensor([[-0.0470, -0.0829, -0.0732, -0.0068, -0.0091],
        [-0.0470, -0.0829, -0.0732, -0.0068, -0.0091]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 1.]], device='cuda:0')
Epoch 0:  83%|████████▎ | 5/6 [00:34<00:06,  6.90s/it, loss=0.949, v_num=2qi6]tensor([[-0.0470, -0.0829, -0.0732, -0.0068, -0.0091],
        [-0.0470, -0.0829, -0.0732, -0.0068, -0.0091]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[1., 1., 0., 0., 0.],
        [0., 1., 0., 0., 1.]], device='cuda:0')
Epoch 1:  50%|█████     | 3/6 [00:50<00:50, 16.69s/it, loss=1.38, v_num=2qi6] tensor([[-0.1346, -0.1453, -0.1445,  0.0238, -0.0494],
        [-0.1346, -0.1453, -0.1445,  0.0238, -0.0494]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[1., 0., 1., 0., 1.],
        [1., 0., 0., 0., 0.]], device='cuda:0')
Epoch 1:  67%|██████▋   | 4/6 [01:06<00:33, 16.73s/it, loss=1.38, v_num=2qi6]tensor([[-0.1346, -0.1453, -0.1445,  0.0238, -0.0494],
        [-0.1346, -0.1453, -0.1445,  0.0238, -0.0494]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 1.]], device='cuda:0')
Epoch 1:  83%|████████▎ | 5/6 [01:11<00:14, 14.23s/it, loss=1.38, v_num=2qi6]tensor([[-0.1346, -0.1453, -0.1445,  0.0238, -0.0494],
        [-0.1346, -0.1453, -0.1445,  0.0238, -0.0494]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[1., 1., 0., 0., 0.],
        [0., 1., 0., 0., 1.]], device='cuda:0')
Epoch 2:  50%|█████     | 3/6 [01:26<01:26, 28.92s/it, loss=1.29, v_num=2qi6]tensor([[-0.1641, -0.2607, -0.2827, -0.0202, -0.1065],
        [-0.1641, -0.2607, -0.2827, -0.0202, -0.1065]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[1., 0., 1., 0., 1.],
        [1., 0., 0., 0., 0.]], device='cuda:0')
Epoch 2:  67%|██████▋   | 4/6 [01:42<00:51, 25.63s/it, loss=1.29, v_num=2qi6]tensor([[-0.1641, -0.2607, -0.2827, -0.0202, -0.1065],
        [-0.1641, -0.2607, -0.2827, -0.0202, -0.1065]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 1.]], device='cuda:0')
Epoch 2:  83%|████████▎ | 5/6 [01:46<00:21, 21.29s/it, loss=1.29, v_num=2qi6]tensor([[-0.1641, -0.2607, -0.2827, -0.0202, -0.1065],
        [-0.1641, -0.2607, -0.2827, -0.0202, -0.1065]], device='cuda:0',
       dtype=torch.float16)

 y tensor([[1., 1., 0., 0., 0.],
        [0., 1., 0., 0., 1.]], device='cuda:0')
Epoch 2: 100%|██████████| 6/6 [01:48<00:00, 18.04s/it, loss=1.29, v_num=2qi6]

Also, weights became 0

model._conv_stem.weight tensor(0.)
model._bn0.weight tensor(0.)
model._bn0.bias tensor(0.)
model._blocks.0.0._depthwise_conv.weight tensor(0.)
model._blocks.0.0._bn1.weight tensor(0.)
model._blocks.0.0._bn1.bias tensor(0.)
model._blocks.0.0._se_reduce.weight tensor(0.)
model._blocks.0.0._se_reduce.bias tensor(0.)
model._blocks.0.0._se_expand.weight tensor(0.)
model._blocks.0.0._se_expand.bias tensor(0.)
model._blocks.0.0._project_conv.weight tensor(0.)
model._blocks.0.0._bn2.weight tensor(0.)
model._blocks.0.0._bn2.bias tensor(0.)
model._blocks.0.1._depthwise_conv.weight tensor(0.)
model._blocks.0.1._bn1.weight tensor(0.)
model._blocks.0.1._bn1.bias tensor(0.)
model._blocks.0.1._se_reduce.weight tensor(0.)
model._blocks.0.1._se_reduce.bias tensor(0.)
model._blocks.0.1._se_expand.weight tensor(0.)
model._blocks.0.1._se_expand.bias tensor(0.)
model._blocks.0.1._project_conv.weight tensor(0.)
model._blocks.0.1._bn2.weight tensor(0.)
model._blocks.0.1._bn2.bias tensor(0.)
model._blocks.1.2._expand_conv.weight tensor(0.)
model._blocks.1.2._bn0.weight tensor(0.)
...
model._bn1.weight tensor(0.)
model._bn1.bias tensor(0.)
model._fc.weight tensor(0.)
model._fc.bias tensor(0.)

Thank you in advance!!

[rohitgr7]

try checking the value of x inside validation_step and see if they are different or not

[avKostanov]

Thank you, I've checked.

UPDATE: running model.train() makes everything ok, however after model.eval(), outputs become the same every epoch. I assume it's not lightning problem already.