Adaptive threshold possible?
I am trying to threshold Variables using the adaptive value as below. <SCODE>import torch
from torch.autograd import Variable
import torch.nn as nn

x = Variable(torch.rand(10))
maxx = torch.max(x)
m = nn.Threshold(0.6*maxx,0)
y = m(x)
<ECODE> Thanks in advance for your help.
Seungyoung_Park
Hi, I think you want to use directly the nn functional version of threshold: <SCODE>import torch
from torch.autograd import Variable
import torch.nn as nn

x = Variable(torch.rand(10))
maxx = torch.max(x)
y = nn.functional.threshold(x, 0.6*maxx.data[0], 0)
<ECODE>
albanD
Seungyoung_Park
Hi,
albanD
Thanks for your reply. It works as I want.
Seungyoung_Park
When I use this snippet in my code, I got an error as <SCODE>RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation
<ECODE> My test code is as follows. <SCODE>import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable

class AdaReLU(nn.Module):
def __init__(self):
    super(AdaReLU, self).__init__()
    self.relu = nn.ReLU()
    
def forward(self, x):
    y = x
    for i in range(x.size(0)):
        for ii in range(x.size(1)):
            maxx = torch.max(x[i,ii,0,:])
            y[i,ii,0,:] = F.threshold(x[i,ii,0,:], config.max_th*maxx.data[0], 0)

    x = self.relu(y)
    return x

class TestNet(nn.Module):
def __init__(self):
    super(TestNet, self).__init__()
    self.conv_1 = nn.Conv2d(len(config.alphabet), config.basic_frames, kernel_size = (1,3), stride = (1,2), padding = (0,0), bias = False)
    self.batchnorm_1 = nn.BatchNorm2d(config.basic_frames)
    self.conv_2 = nn.ConvTranspose2d(config.basic_frames, len(config.alphabet), kernel_size = (1,3), stride = (1,2), padding = (0,0), bias = False)
    self.batchnorm_2 = nn.BatchNorm2d(len(config.alphabet))
    self.adarelu = AdaReLU()

def forward(self, x):
    x = self.adarelu(self.batchnorm_1(self.conv_1(x)))
    x = F.sigmoid(self.batchnorm_2(self.conv_2(x)))
    return x

x = Variable(torch.rand(32,68,1,2283))
model = TestNet()
optimizer = optim.SGD(model.parameters(), lr =0.01, momentum=0.9)
optimizer.zero_grad()
output = model(data)
loss = F.binary_cross_entropy(output, data)
loss.backward()
optimizer.step()<ECODE>
Seungyoung_Park
albanD
Seungyoung_Park
Hi, I think you can replace the loop by: <SCODE>y = x * (1 - x.lt(maxx.expand_as(x))).float()
<ECODE>
albanD
The whole function can be simplified to this: <SCODE>def AdaReLU(x):                    
    max_val, _ = torch.max(x, 3)   
    mask = x < max_val.expand_as(x)
    mask.detach_()                 
    x = x.clone()                  
    x[mask] = 0                    
    return F.relu(x)               
<ECODE>
apaszke
Seungyoung_Park