sure @ptrblck,
I’m trying to classify 6 classes for video classification problem. I tried running end-2-end model (i3d) but it gave me bad results (42% accuracy) so I decided to solve 6 different classification problem (binary instead of multi).
1 model for class A tells me 1 if the it’s class A or 0 if it’s not class A, same for class B i trained a model that predicts 1 if the video belongs to class B or 0 if it’s not etc for all of my 6 classes.
Each of these 6 models has their own mean and std (calculated on the training set) but i chose to do it a little differently… I decided for each model to calculate only the mean and std for the specific class - for example for model 1 that predicts class A i divided my training set into positive and negative samples (positive - 1 - all the videos belonging to class A and 0 - all the videos that are not classified as A) - so that’s how i calculated my mean and std for each of my classes.
I wrote a wrapper Module that takes all these 6 models and inference each one individually and concatenating the results into a Linear layer with 6 output channels (for the 6 classification i want to predict) when running my model in training it takes a lot of time to do the inference and a lot of memory for some reason.
i’m attaching my code for that:
import os,sys,inspect

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

import torch
from torch import nn
from Utils.Transforms import ToTensor
from torch.autograd import Variable


class NetEnsemble(nn.Module):
	"""docstring for NetEnsemble"""
	def __init__(self, batch_size, use_half, ensemble_models=None, normalizations=[]):
		super(ClarityNetEnsemble, self).__init__()
		to_tensor = ToTensor()

		self.ensemble_models = nn.ModuleList(ensemble_models)
		self.log_softmax = nn.LogSoftmax(1)
		self.fc = nn.Linear(12, 6)

		self.register_buffer('net1_mean', to_tensor(normalizations[0][0]).unsqueeze(0))
		self.register_buffer('net1_std', to_tensor(normalizations[0][1]).unsqueeze(0))
		self.register_buffer('net2_mean', to_tensor(normalizations[1][0]).unsqueeze(0))
		self.register_buffer('net2_std', to_tensor(normalizations[1][1]).unsqueeze(0))
		self.register_buffer('net3_mean', to_tensor(normalizations[2][0]).unsqueeze(0))
		self.register_buffer('net3_std', to_tensor(normalizations[2][1]).unsqueeze(0))
		self.register_buffer('net4_mean', to_tensor(normalizations[3][0]).unsqueeze(0))
		self.register_buffer('net4_std', to_tensor(normalizations[3][1]).unsqueeze(0))
		self.register_buffer('net5_mean', to_tensor(normalizations[4][0]).unsqueeze(0))
		self.register_buffer('net5_std', to_tensor(normalizations[4][1]).unsqueeze(0))
		self.register_buffer('net6_mean', to_tensor(normalizations[5][0]).unsqueeze(0))
		self.register_buffer('net6_std', to_tensor(normalizations[5][1]).unsqueeze(0))

	def forward(self, x):
		log_softmaxs = []
		softmaxs = []

		#normalize x according to the model mean and std
		x_normalized = (x - Variable(self.net1_mean, volatile=True))/Variable(self.net1_std, volatile=True)
		#do inference on the normalized input
		log_softmax, softmax = self.ensemble_models[0](x_normalized)
		log_softmaxs.append(log_softmax)
		softmaxs.append(softmax)

		#normalize x according to the model mean and std
		x_normalized = (x - Variable(self.net2_mean, volatile=True))/Variable(self.net2_std, volatile=True)
		#do inference on the normalized input
		log_softmax, softmax = self.ensemble_models[1](x_normalized)
		log_softmaxs.append(log_softmax)
		softmaxs.append(softmax)

		#normalize x according to the model mean and std
		x_normalized = (x - Variable(self.net3_mean, volatile=True))/Variable(self.net3_std, volatile=True)
		#do inference on the normalized input
		log_softmax, softmax = self.ensemble_models[2](x_normalized)
		log_softmaxs.append(log_softmax)
		softmaxs.append(softmax)

		#normalize x according to the model mean and std
		x_normalized = (x - Variable(self.net4_mean, volatile=True))/Variable(self.net4_std, volatile=True)
		#do inference on the normalized input
		log_softmax, softmax = self.ensemble_models[3](x_normalized)
		log_softmaxs.append(log_softmax)
		softmaxs.append(softmax)

		#normalize x according to the model mean and std
		x_normalized = (x - Variable(self.net5_mean, volatile=True))/Variable(self.net5_std, volatile=True)
		#do inference on the normalized input
		log_softmax, softmax = self.ensemble_models[4](x_normalized)
		log_softmaxs.append(log_softmax)
		softmaxs.append(softmax)

		#normalize x according to the model mean and std
		x_normalized = (x - Variable(self.net6_mean, volatile=True))/Variable(self.net6_std, volatile=True)
		#do inference on the normalized input
		log_softmax, softmax = self.ensemble_models[5](x_normalized)
		log_softmaxs.append(log_softmax)
		softmaxs.append(softmax)

		out = torch.cat(log_softmaxs, 1)
		out = out.view(out.size(0), -1) #flatten outputs
		out = self.fc(out)
		
		out.volatile = False
		out.requires_grad = True
		
		return self.log_softmax(out)