HI folks. I’m just having an unreasonable amount of difficulty shaping outputs to use in  graph. Is anyone available to just talk about data types etc?
I’m going to include a bunch of code below, and I’d like to know how best to save my predictions and the associated labels (given that I’m shuffling between epochs). In particular I’d like to make a confusion matrix, but can’t seem to get my outputs into a single structure (list/array/tensor). I’m also interested in if there is a good way to include the datum id in here somewhere, so I can compare accuracy within some of the other fields I’m not using in training my model. I’d also be happy to know what I’m making harder than it needs to be and whats just bad code.
If this sort of code dump and request for general help is inappropriate for the forum, please let me know. I’m just so frustrated with how messy this project has been… I’m about to finish a masters degree and this shouldn’t be this hard…
Many many thanks for your help.
params = {'batch_size': 200,
          'shuffle': True}
batch_size=200
learning_rate = 0.001
num_epochs = 5
...
train_pd #pandasDF with  
   # index='id'= 5 char string, 
   # additional data not used in training
   # label= int in range (0,8), 
   # size, indices, values = sparse vector information
   # val_pd=same

# Prep for Data Loader
partition={'train': train_pd.index,'val': val_pd.index}

labels_train=train_pd['label'].to_dict()
labels_val=val_pd['label'].to_dict()

i_train=train_pd['indices'].to_dict()
i_val=val_pd['indices'].to_dict()

v_train=train_pd['values'].to_dict()
v_val=val_pd['values'].to_dict()

inputSize=train_pd['size'][0]

class Dataset(data.Dataset):
  'Characterizes a dataset for PyTorch'
  def __init__(self, list_IDs, labels, indices,values,vecSize):
        'Initialization'
        self.labels = labels
        self.list_IDs = list_IDs
        self.i= indices
        self.v =values
        self.vecSize=vecSize

  def __len__(self):
        'Denotes the total number of samples'
        return len(self.list_IDs)

  def __getitem__(self, index):
        'Generates one sample of data'
        # Select sample
        ID = self.list_IDs[index]
        try:
            # Load data and get label
            X = torch.sparse.FloatTensor(
                torch.LongTensor([self.i[ID]]),
                torch.FloatTensor(self.v[ID]), 
                torch.Size([self.vecSize])
            ).to_dense()
        except RuntimeError:
            print(ID)
            
        i = self.labels[ID]
        y=np.zeros(8)
        y[i]=1

        return X, torch.LongTensor([i])
    
training_set = Dataset(partition['train'], labels_train,i_train,v_train,inputSize)
training_generator = data.DataLoader(training_set, **params)

validation_set = Dataset(partition['val'], labels_val,i_val,v_val,inputSize)
validation_generator = data.DataLoader(validation_set, **params)

# Model
model = nn.Sequential(
                    nn.Linear(inputSize,16),
                    nn.ReLU(),
                    nn.Linear(16,8)#,
)

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)  

# Train the model
losses=[]
accuracies=[]
comparisons=[]

total_step = len(training_generator)
for epoch in range(3):
    i=0
    total=0
    correct=0
    for local_batch, local_labels in training_generator:
        i+=1
        local_batch, local_labels = local_batch.to(device), local_labels.to(device)

        optimizer.zero_grad()
        # Forward pass
        outputs = model(local_batch)

        total += local_labels.size(0)
        labs=local_labels.flatten()
        pred=np.argmax(outputs.detach().numpy(), axis=1)
        correct += (pred == labs).sum().item()
  
        acc=correct/total

        loss = criterion(outputs, local_labels.squeeze())
        losses.append(loss.item())
        accuracies.append(acc)
        
        if epoch==num_epochs-1:
            comparisons.append([pred,labs])

        # Backward and optimize
        loss.backward()
        optimizer.step()
        
        
        if i%100==0:
            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Accuracy:{:,.4f}' 
                   .format(epoch+1, num_epochs,i, total_step, loss.item(), acc))

# Validate the model
Vcomparisons=[]

total_step = len(validation_generator)
total=0
correct=0
Vaccuracies=[]
alllab=[]
allpred=[]
for local_batch, local_labels in validation_generator:
    i+=1
    local_batch, local_labels = local_batch.to(device), local_labels.to(device)

    # Forward pass
    outputs = model(local_batch)

    total += local_labels.size(0)
    labs=local_labels.flatten()
    pred=np.argmax(outputs.detach().numpy(), axis=1)
    correct += (pred == labs).sum().item()
    
    alllab.append(labs.numpy())
    allpred.append(pred)

    acc=correct/total

    Vaccuracies.append(acc)

    Vcomparisons.append([pred,labs])
print ('Accuracy:{:,.4f}'  .format( acc))

labs=alllab[0]
preds=allpred[0]
for i in range(1,len(alllab)-1):
    print(labs, preds)
    print(alllab[i],allpred[i])
    labs=np.concatenate(labs,alllab[i], axis=1) #errors
    preds=np.concatenate(preds,allpred[i],axis=1) #errors

#Eventually sklearn confusion matrix here