Alright, totally didnt see that, thanks a lot.
I changed some other details and got it running now, but the acccuracy of the results is varying a lot. Is this due to an insufficient amount of training data (144 sets for training, 16 for testing)  or is there some other error in my code I am not seeing ?
import os
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader, TensorDataset
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch import Tensor
from natsort import os_sorted

# Device configuration
device = torch.device('cpu')

# Hyper-parameters 
input_size = 16
hidden_size = 32
num_classes = 2
num_epochs = 10
batch_size = 1
learning_rate = 0.001
Train_Val_Pct = 0.1 #distribution of learning and testing data in percent

class Features(Dataset):
    good_data = 'Preprocessed_Data/Balanced_Data/Good'
    bad_data = 'Preprocessed_Data/Balanced_Data/Bad'
    labels = {good_data: 0, bad_data: 1}
    samples = []
    
    goodcount = 0
    badcount = 0
    
    def __init__(self):
        for label in self.labels:
            print(label)
            for run in os_sorted(os.listdir(label)): #os_sorted needed to go through data as it is listed by windows (14,101,165,...) otherwise (101,14,165,...)
                path = os.path.join(label, run)
                print(path)
                data = pd.read_csv(path,header = None)
                self.samples.append(np.array(data), np.eye(2)[self.labels[label]])
                
                if label == self.good_data:
                    self.goodcount += 1
                elif label == self.bad_data:
                    self.badcount += 1
        #np.random.shuffle(self.samples)
        np.save("samples.npy", self.samples)        
    
    def __len__(self):
        return len(self.samples)
    
    def __getitem__(self, idx):
        return self.samples[idx]
    
class Net(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes):
        super().__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.relu1 = nn.ReLU() #activation function
        self.fc2 = nn.Linear(hidden_size, hidden_size)
        self.relu2 = nn.ReLU()
        self.fc3 = nn.Linear(hidden_size, hidden_size)
        self.relu3 = nn.ReLU() #activation function
        self.fc4 = nn.Linear(hidden_size, hidden_size)
        self.relu4 = nn.ReLU() #activation function
        self.fc5 = nn.Linear(hidden_size, num_classes)

    def forward(self, x): #x is a batch of inputs
        out = self.fc1(x)
        out = self.relu1(out)
        out = self.fc2(out)
        out = self.relu2(out)
        out = self.fc3(out)
        out = self.relu3(out)
        out = self.fc4(out)
        out = self.relu4(out)
        out = self.fc5(out)     #out is the number of classes
        return out
        #return F.softmax(out, dim=1)  
    
net = Net(input_size, hidden_size, num_classes).to(device)

samples = np.load('samples.npy',allow_pickle=True)

X = torch.Tensor([i[0] for i in samples])
y = torch.Tensor([i[1] for i in samples])

size = int(Train_Val_Pct*len(X))

train_X = X[:-size].squeeze(-1)
train_y = y[:-size].squeeze(-1)

test_X = X[-size:].squeeze(-1)
test_y = y[-size:].squeeze(-1)

trainset = TensorDataset(Tensor(train_X), Tensor(train_y))
testset = TensorDataset(Tensor(test_X), Tensor(test_y))

trainloader = DataLoader(trainset,batch_size=batch_size,shuffle=True)
testloader = DataLoader(testset,batch_size=batch_size,shuffle=False)

criterion = nn.MSELoss()
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)  
#Train the model
total_step = len(trainloader)
for epoch in range(num_epochs):
    for i, (data, labels) in enumerate(trainloader):  
        # Move tensors to the configured device
        data = data.to(device)
        labels = labels.to(device)
        
        # Forward pass
        outputs = net(data)
        #print('Outputs Train: ',outputs)
        loss = criterion(outputs, labels)
        # Backward and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    print(loss)
    

# Test the model
# In test phase, we don't need to compute gradients (for memory efficiency)
with torch.no_grad():
    correct = 0
    total = 0
    for data, labels in testloader:
        data = data.to(device)
        labels = torch.argmax(labels).to(device)  #real class
        #real = torch.max(labels).to(device)
        outputs = torch.argmax(net(data))
        #print('Outputs Test: ',outputs)
        #print('Real output: ',real)
        #print('Labels: ',labels)
        predicted = outputs
        if torch.eq(predicted, labels):
            correct += 1
        total += 1

    print("Accuracy:", round(correct/total,3))

# Save the model checkpoint
torch.save(net.state_dict(), 'net.ckpt')