Hey everyone,
this is my second pytorch implementation so far, for my first implementation the same happend; the model does not learn anything and outputs the same loss and accuracy for every epoch and even for each batch with an epoch. My personal guess is that something with the way I feed the data to the model is not correctly implemented.
I try to follow the basic tutorial for implementing a pytorch model: Optimizing Model Parameters — PyTorch Tutorials 1.8.1+cu102 documentation 1
I read similar topics on thepytorch forum, e.g. Same values in every epoch when training 6
I’m using nn.BCEWithLogitsLoss() and already tried to overfit the model on a training sample of size 600.
It’s rather a simple model, so I’m a bit confused that it’s not working at all.
Find the code below:
Load the data
df = pd.read_csv(Path()/ "cleaned_data.csv", nrows=1000)
#delete id column
del df["Unnamed: 0"]
train, validate, test = np.split(df.sample(frac=1), [int(.6*len(df)), int(.8*len(df))])

X_train = train.iloc[:,1:]
Y_train = train.iloc[:,1]

X_validate = validate.iloc[:,1:]
Y_validate = validate.iloc[:,1]

X_test = validate.iloc[:,1:]
Y_test = validate.iloc[:,1]

x_train_pt = torch.from_numpy(X_train.values)
y_train_pt = torch.from_numpy(Y_train.values.reshape((-1,1)))

x_val_pt = torch.from_numpy(X_validate.values)
y_val_pt = torch.from_numpy(Y_validate.values.reshape((-1,1)))

x_test_pt = torch.from_numpy(X_test.values)
y_test_pt = torch.from_numpy(Y_test.values.reshape((-1,1)))
print(x_train_pt.shape) #torch.Size([600, 22])
print(y_train_pt.shape) #torch.Size([600, 1])

batch_size=100
train_dataset = TensorDataset(x_train_pt,y_train_pt) # create your datset
trainloader = DataLoader(train_dataset, batch_size=batch_size) # create your dataloader

val_dataset = TensorDataset(x_val_pt,y_val_pt)
valloader = DataLoader(val_dataset, batch_size=batch_size)

test_dataset = TensorDataset(x_test_pt,y_test_pt)
testloader = DataLoader(test_dataset, batch_size=batch_size)

The Model
class LendingClub(nn.Module):
    def __init__(self):
        super(LendingClub, self).__init__()
        self.linear = nn.Sequential(
            nn.Linear(22, 100),
            nn.ReLU(),
            nn.Linear(100, 64),
            nn.ReLU(),
            nn.Linear(64, 32),
            nn.ReLU(),
            nn.Linear(32, 16),
            nn.ReLU(),
            nn.Linear(16, 8),
            nn.ReLU(),
            nn.Linear(8, 1),
            nn.Sigmoid(),
        )

    def forward(self, x):
        return self.linear(x)
    
model = LendingClub()

def weights_init(m):
  if isinstance(m, nn.Linear):
      nn.init.uniform_(m.weight.data, -1,1)
      nn.init.zeros_(m.bias.data)

model.apply(weights_init)

Training
loss_fn = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

def train_loop(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    for batch, (X, y) in enumerate(dataloader):
        # Compute prediction and loss
        pred = model(X.float())
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        #if batch % 100 == 0:
        loss, current = loss.item(), batch * len(X)
        print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")


def test_loop(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    test_loss, correct = 0, 0

    with torch.no_grad():
        for X, y in dataloader:
            pred = model(X.float())
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()

    test_loss /= size
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

model.train()
start = time.time()
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train_loop(trainloader, model, loss_fn, optimizer)
    test_loop(valloader, model, loss_fn)
end = time.time()
print(f"total training time in minutes: {(end-start)/60}")

Output
Epoch 1
-------------------------------
loss: 0.693147  [    0/  600]
loss: 0.693147  [  100/  600]
loss: 0.693147  [  200/  600]
loss: 0.693147  [  300/  600]
loss: 0.693147  [  400/  600]
loss: 0.693147  [  500/  600]
Epoch 2
-------------------------------
loss: 0.693147  [    0/  600]
loss: 0.693147  [  100/  600]
loss: 0.693147  [  200/  600]
loss: 0.693147  [  300/  600]
loss: 0.693147  [  400/  600]
loss: 0.693147  [  500/  600]
Epoch 3
-------------------------------
loss: 0.693147  [    0/  600]
loss: 0.693147  [  100/  600]
loss: 0.693147  [  200/  600]
loss: 0.693147  [  300/  600]
loss: 0.693147  [  400/  600]
loss: 0.693147  [  500/  600]
Epoch 4
-------------------------------
loss: 0.693147  [    0/  600]
loss: 0.693147  [  100/  600]
loss: 0.693147  [  200/  600]
loss: 0.693147  [  300/  600]
loss: 0.693147  [  400/  600]
loss: 0.693147  [  500/  600]
Epoch 5
-------------------------------
loss: 0.693147  [    0/  600]
loss: 0.693147  [  100/  600]
loss: 0.693147  [  200/  600]
loss: 0.693147  [  300/  600]
loss: 0.693147  [  400/  600]
loss: 0.693147  [  500/  600]
total training time in minutes: 0.011684314409891764

The model does not always output exactly the same loss, but the fact that for every epoch the losses are the same does not change.
So the output from another run looked as following:
Epoch 1
-------------------------------
loss: 1.026536  [    0/  600]
loss: 0.980619  [  100/  600]
loss: 0.964926  [  200/  600]
loss: 1.120505  [  300/  600]
loss: 1.092735  [  400/  600]
loss: 0.910104  [  500/  600]
Test Error: 
 Accuracy: 0.0%, Avg loss: 0.010968 

Epoch 2
-------------------------------
loss: 1.026536  [    0/  600]
loss: 0.980619  [  100/  600]
loss: 0.964926  [  200/  600]
loss: 1.120505  [  300/  600]
loss: 1.092735  [  400/  600]
loss: 0.910104  [  500/  600]
Test Error: 
 Accuracy: 0.0%, Avg loss: 0.010968 

Epoch 3
-------------------------------
loss: 1.026536  [    0/  600]
loss: 0.980619  [  100/  600]
loss: 0.964926  [  200/  600]
loss: 1.120505  [  300/  600]
loss: 1.092735  [  400/  600]
loss: 0.910104  [  500/  600]
Test Error: 
 Accuracy: 0.0%, Avg loss: 0.010968

Thanks for your replies, any help is very much appreciated