Hi,
I’m having trouble getting a classifier working. It’s based on a tensorflow application (unfortunately I can’t release the source for examination) which predicts a binary class label for multiple targets.
The problem lies in that the input data has a high proportion of missing values. These are characterised by their own label, which is supposed to be ignored in the loss function. I’ve tried nllloss with the built-in ability to perform this, and also BCE by multiplying the output of the network by zeros for missing values - neither seems to work.
I suspect I’m making a mistake somewhere, but I’m fairly new to pytorch and can’t seem to figure it out. The network doesn’t appear to be treating each task independently - average ROCAUC across all tasks from the original program is ~0.7-0.8, whereas this doesn’t go above 0.55ish. Below is the (relevant, fairly verbose) code:
train_features=torch.from_numpy(np.array(train_features))
train_labels=torch.from_numpy(np.array(train_labels))
test_features=torch.from_numpy(np.array(test_features))
test_labels=torch.from_numpy(np.array(test_labels))

test_labels += 1 #Targets in range {-1,0,1} where 0 is unknown datapoint
train_labels += 1 #Change targets range from {-1,0,1} to {0,1,2}

train = torch.utils.data.TensorDataset(train_features.float(), train_labels.long())
train_loader = torch.utils.data.DataLoader(train, batch_size=batch, shuffle=True)
train_loader_2 = torch.utils.data.DataLoader(train, batch_size=train_features.shape[0], shuffle=False)
test = torch.utils.data.TensorDataset(test_features.float(), test_labels.long())
test_loader = torch.utils.data.DataLoader(test, batch_size=batch, shuffle=False)
#Build test, train, and AUROC dataloaders
#for rocauc, convert index to one-hot vector
rocauc_labels=torch.from_numpy(np.array([[i==j for i in [0,1,2]] for j in np.array(test_labels)], np.int32))
rocauc_labels = torch.transpose(rocauc_labels, 1, 2)
rocauc_set = torch.utils.data.TensorDataset(test_features.float(), rocauc_labels.long())
rocauc_loader = torch.utils.data.DataLoader(rocauc_set, batch_size=test_features.shape[0], shuffle=False)
#init network from library
net = Network.FC_FF_RELU_NN([train_features.shape[1],1024,1024,1024,train_labels.shape[1]])

criterion = F.nll_loss #using negative-log loss - this requires a log_softmax before use (is in final layer)
optimizer = torch.optim.SGD(net.parameters(),lr=learning_rate, momentum=momentum, weight_decay=L2loss)#, nesterov=True)

listofloss = []
listofclasslls = []
listofclassrocaucs = []
listofaverageclassll = []
listofaveragerocauc = []
listofactives = []

for i, (features, labels) in enumerate(train_loader_2):
    print(labels.shape)
    for k in range(len(labels[0,:])): #for each task
        numactives = 0
        for j in range(len(labels[:,k])): #for each record
            if labels[j,k] == 2:
                numactives += 1
        listofactives.append(numactives)
print(listofactives)

for epoch in range(num_epochs):
    epochloss = 0.
    for i, (features, labels) in enumerate(train_loader):
        features, labels = Variable(features, requires_grad=True), Variable(labels, requires_grad=True)
        optimizer.zero_grad() #zero gradients
        outputs = net(features, droprate) #calculate outputs from features for batch
        loss = 0. #zero loss
        for k in range(len(outputs[0,:,0])): #output dims are: [record,task,ln(classprob)]
            #weight = torch.Tensor([1.0,0,1.0]) #init class weighting - prefer to correctly classify actives
            #weight *= min(listofactives) / listofactives[k]
            loss += criterion(outputs[:,k,:], labels[:,k], ignore_index=1)#, weight=weight)
            #ignore unknown: for each target ignore class=1(unknown) and calculate loss over known for all mols
        loss /= len(outputs.data[0,:,0]) #average loss over all tasks
        epochloss += loss.data[0] #increment epoch loss over all batches
        loss.backward()
        optimizer.step()
    epochloss /= i #average epoch loss over all batches
    if epoch % 2 == 0:
        listofloss.append([epoch, epochloss])
        print('Epoch = {}, average epoch loss = {}'.format(epoch, epochloss))
        for tsfeatures, tslabels in (rocauc_loader):
            tsfeatures = Variable(tsfeatures)
            tsoutputs = net(tsfeatures, 0).data.numpy()
        classlls = [epoch] #add epoch no. to stats lists
        classrocaucs = [epoch] # ""      ""
        for i in range(len(tslabels[0,:,0])): #for each task
            knownpreds = []
            knowntars = []
            for j in range(len(tslabels[:,i,0])): #for each record in task i
                if not tslabels[j,i,1] == 1: #if the value is not unknown
                    knownpreds.append(math.exp(tsoutputs[j,i,2])) #add to list
                    knowntars.append(tslabels[j,i,2]) #add to list
            classrocauc = roc_auc_score((knowntars), (knownpreds))
            classrocaucs.append(classrocauc)
        listofclassrocaucs.append(classrocaucs)
        listofaveragerocauc.append([epoch,sum(classrocaucs[1:]) / (len(classrocaucs)-1)])
        print("Average AUROC: {}".format(sum(classrocaucs[1:]) / (len(classrocaucs)-1)))

NN code:
class FC_FF_RELU_NN(nn.Module):
    def __init__(self, ListOfLayers): #define the init params of the Network Architecture
        super(FC_FF_RELU_NN, self).__init__()
        self.nb_layers = len(ListOfLayers) #Accept ListOfLayers as an array of ints, specifying input, hidden, and output layer widths
        fc = [] #Init fully connected network
        for i in range(self.nb_layers-2):
            fc.append(nn.Linear(ListOfLayers[i],ListOfLayers[i+1])) #Append layers to list
        self.fc = nn.ModuleList(fc) #Convert to ModuleList
        self.active = nn.Linear(ListOfLayers[-2],ListOfLayers[-1])
        self.inactive = nn.Linear(ListOfLayers[-2],ListOfLayers[-1])
        self.unknown = nn.Linear(ListOfLayers[-2],ListOfLayers[-1])
        #active, inactive, and unknown may not actually correspond to their respective indices, but are placeholder names
    def forward(self, x, droprate): #Define forward function
        for i in range(self.nb_layers-2):
            x = F.relu(self.fc[i](x)) #ReLu
            x = F.dropout(x, p=droprate)
        active = self.active(x)
        inactive = self.inactive(x)
        unknown = self.unknown(x)
        output = torch.stack([active,unknown,inactive],dim=-1)
        return F.log_softmax(output, dim=-1) #LogSoftMax for last layers for NLL_Loss

Any help would be greatly appreciated, this has been driving me nuts for a couple of weeks now.