I am using this model:
  
      

      GitHub
  

  
    

kylemin/S3D 1

  Release of the pretrained S3D Network in PyTorch (ECCV 2018) - kylemin/S3D


  

  
    
    
  

  


I should train the model to recognize a series of (volcanic) videos which can belong to 8 different classes (all or only some of them) For example:
-video 1 belongs to classes 1,4,6
-video 2 belongs to class 3
-video 3 belongs to classes 2,5
etc…
Each class has a binary label 1 (belongs) or 0 (does not belong).
I thought of calculating the loss for each of these 8 classes by specifying in each of these the weight of that specific class.
This is my class_weight = [5.56, 51.5, 3.67, 1.39, 5.36, 2.82, 5.18, 1.76]
def binary_loss_fn(outputs, targets):
      t1, t2, t3, t4, t5, t6, t7, t8 = targets
      o1, o2, o3, o4, o5, o6, o7, o8 = outputs
      
      l1 = nn.BCEWithLogitsLoss(pos_weight=class_weight[0], reduction='none')(o1, t1)
      l2 = nn.BCEWithLogitsLoss(pos_weight=class_weight[1], reduction='none')(o2, t2)
      l3 = nn.BCEWithLogitsLoss(pos_weight=class_weight[2], reduction='none')(o3, t3)
      l4 = nn.BCEWithLogitsLoss(pos_weight=class_weight[3], reduction='none')(o4, t4)
      l5 = nn.BCEWithLogitsLoss(pos_weight=class_weight[4], reduction='none')(o5, t5)
      l6 = nn.BCEWithLogitsLoss(pos_weight=class_weight[5], reduction='none')(o6, t6)
      l7 = nn.BCEWithLogitsLoss(pos_weight=class_weight[6], reduction='none')(o7, t7)
      l8 = nn.BCEWithLogitsLoss(pos_weight=class_weight[7], reduction='none')(o8, t8)
      return (l1 + l2 + l3 + l4 + l5 + l6 + l7 + l8) / 8

My dataset is structured as follows:
210 videos of 300 frames each, of these I took one every five frames and combined in 29 different combinations each of 32 frames:
frames 1,2,3,4,5,6 … 300 → frames 1,6,11,16,21 … (total 60 frames):

first combination: first 32 frames (1-32)
second combination: second 32 frames shifted by 1 (2-33)
third combination: third 32 frames shifted by 1 (3-34)
etc…

total array 210x29x32
The input of the s3d model are these 32 frames concatenated with pytorch and resized:
torch.Size ([2, 3, 32, 224, 224])
associated with a list of 8 targets each relating to a class (1 or 0)
def fit(model, train_dataloader):
    print('Training')
    model.train()
    train_running_loss = 0.0
    train_running_correct = 0
    for i, data in tqdm(enumerate(train_dataloader), total=int(len(train_data)/train_dataloader.batch_size)):
        #data, target, path = data[0].to(device), data[1].to(device), data[2]
        # extract the features and labels
        video = data['video'].to(device)
        target1 = data['label1'].to(device)
        target2 = data['label2'].to(device)
        target3 = data['label3'].to(device)
        target4 = data['label4'].to(device)
        target5 = data['label5'].to(device)
        target6 = data['label6'].to(device)
        target7 = data['label7'].to(device)
        target8 = data['label8'].to(device)
        path = data['path']


        optimizer.zero_grad()
        outputs = model(video)
        targets = (target1, target2, target3, target4, target5, target6, target7, target8)
        outputs = torch.sigmoid(outputs)
        loss = binary_loss_fn(outputs, targets)
          
        train_running_loss += loss.item()

        preds = torch.tensor([[1 if i >= 0.5 else 0 for i in j] for j in outputs]).to(device)
        train_running_correct = (preds == target).sum().item()

        loss.backward()
        optimizer.step()
        
    train_loss = train_running_loss/len(train_dataloader.dataset)
    train_accuracy = 100. * train_running_correct/len(train_dataloader.dataset)
    return train_loss, train_accuracy

This is my fit function where I am going to pass my targets and my model’s output inside binary_loss_fn. I specified the number of classes when I initialize the model
model = S3D (8) .to (device)
But i can’t understand what kind of output I get on these s3d because i can’t calculate individually my 8 loss functions.
This is a type model’s output with batch equal to 2:
torch.tensor ([
[0.4150, 0.6833, 0.4165, 0.4718, 0.4183, 0.5799, 0.4773, 0.3026],
[0.4923, 0.5274, 0.4894, 0.5042, 0.4440, 0.4746, 0.4580, 0.4449]
])

If it can be useful this is my dataset function
class ImageDataset(Dataset):
    def __init__(self, images, labels=None):
        self.X = images
        self.y = labels
         
    def __len__(self):
        rows = len(self.X)
        columns = len(self.X[0])
        total_length = rows * columns
        return (total_length)

    def __getitem__(self, i):
        col = len(self.X[0])
        a = i//col
        b = (i%col)-1
        image = self.X[a][b]
        tensor_sequence = []
        for idx in range(0, len(image)):  
          path_img = image[idx]
          tensor_img = tensor[path_img]
          tensor_sequence.append(tensor_img)
        video = torch.cat(tensor_sequence, 1)
        x = path_img.split("data/videos/")
        path = x[1].split(".avi/EMOT")
        path=path[0]
        col = len(self.y[0])
        a1 = i//col
        b1 = (i%col)-1
        label = self.y[a1][b1]
        target = [int(i) for i in label[0].split(",")]
        
        label1 = torch.tensor(target[0], dtype=torch.float32)
        label2 = torch.tensor(target[1], dtype=torch.float32)
        label3 = torch.tensor(target[2], dtype=torch.float32)
        label4 = torch.tensor(target[3], dtype=torch.float32)
        label5 = torch.tensor(target[4], dtype=torch.float32)
        label6 = torch.tensor(target[5], dtype=torch.float32)
        label7 = torch.tensor(target[6], dtype=torch.float32)
        label8 = torch.tensor(target[7], dtype=torch.float32)
        return {
            'video': video,
            'label1': label1,
            'label2': label2,
            'label3': label3,
            'label4': label4,
            'label5': label5,
            'label6': label6,
            'label7': label7,
            'label8': label8,
            'path': path
        }

#Defining the Training and Validation Data Loaders
train_data = ImageDataset(xtrain, ytrain)
val_data = ImageDataset(xval, yval)
test_data = ImageDataset(xtest, ytest)

# dataloaders
trainloader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
valloader = DataLoader(val_data, batch_size=batch_size, shuffle=False)
testloader = DataLoader(test_data, batch_size=batch_size, shuffle=False)

This is my csv file where for each image path I have its associated binarized target:

How could I fix this? Thank you