@ptrblck There is something also i found out , i am building a 1D conv over the top of ALBERT model. I tried different combinations while freezing ALBERT. I tried :
1-
There is something also i found out , i am build a 1D conv over the top of ALBERT model. I tried different combinations while freezing ALBERT. I tried :
1-

class Grammer_Classification(nn.Module):
def init(self,Albert):
super(Grammer_Classification,self).init()
self.attn_1 = Self_Attention(2048,2048)
self.conv_1 = Conv1D_Layer(2048,4,1024)
self.attn_2 = Self_Attention(1024,1024)
self.conv_2 = Conv1D_Layer(1024,2,512)
self.FFN1 = nn.Linear(2560,512)
self.Layer_norm = nn.LayerNorm(512,eps=1e-12)
self.FFN2 = nn.Linear(512,2)
def forward(self,X,mask):
X= Albert(X)
#X = X[0].view(X[0].shape[0],-1)
X = self.attn_1(X[0])
X = self.conv_1(X)
X = self.attn_2(X)
X = self.conv_2(X)
X = X.reshape(X.shape[0],X.shape[1]*X.shape[2])
X = F.leaky_relu(self.FFN1(X))
X = self.Layer_norm(X)
X = self.FFN2(X)
return X

2-

class Grammer_Classification(nn.Module):
def init(self,Albert):
super(Grammer_Classification,self).init()
self.attn_1 = Self_Attention(1024,1024)
self.conv_1 = Conv1D_Layer(2048,16,1024)
self.attn_2 = Self_Attention(512,512)
self.conv_2 = Conv1D_Layer(1024,4,512)
self.FFN1 = nn.Linear(94208,1024)
self.Layer_norm = nn.LayerNorm(1024,eps=1e-12)
self.FFN2 = nn.Linear(1024,2)
def forward(self,X,mask):
X= Albert(X)
X = X[0].view(X[0].shape[0],-1)
X = F.leaky_relu(self.FFN1(X))
X = self.Layer_norm(X)
X = self.FFN2(X)
return X

and only conv1D but i deleted the code.
And the accuracy never changes while the loss scaling. Any idea how can i debug this further more ? I printed all the matrices and there are no NaNs except after 30-40 steps