I’m trying to implement text classification with a transformer with sentence pairs as the data.
class IronyClassifier(nn.Module):
    def __init__(self, batch_size, n_tokens, d_model, n_heads, n_hid, n_layers, dropout_p=0.5):
        super(IronyClassifier, self).__init__()

        self.batch_size = batch_size
        self.d_model = d_model

        self.word_embedding = self.load_word_embedding(trainable=True)
        print('word_embedding loaded')
        self.positional_encoder = PositionalEncoding(d_model, dropout_p)
        self.segment_encoding = SegmentEncoding(d_model=d_model)

        # encoder definition
        encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=n_heads, dim_feedforward=n_hid,
                                                   dropout=dropout_p, activation='gelu')
        self.transformer_encoder = nn.TransformerEncoder(encoder_layer=encoder_layer, num_layers=n_layers)

        self.classifier = nn.Linear(in_features=(d_model), out_features=1)

    def load_word_embedding(self, trainable=True):
        ...

        return word_embedding

    def generate_src_mask(self, sentence_lens: tuple, first_sentence_lens) -> torch.Tensor:
        max_len = max(sentence_lens)
        max_len_last = max(first_sentence_lens)

        src_mask = []

        for current_len, first_current_len in zip(sentence_lens, first_sentence_lens):
            src_mask.append([True] + ([False] * (first_sentence_len - 1)) + ([True] * (max_len_first - first_current_len)) + [False] + ([False] * ((current_len - 3))) + ([True] * ((max_len) - ((current_len)))) + [False])

        src_mask = torch.BoolTensor(src_mask).to(next(self.parameters()).device)

        return src_mask

    def forward(self, src: torch.Tensor, sentence_lens: tuple, first: bool, first_sentence_embedding: Optional[torch.Tensor] = torch.zeros((10, 20, 200)).to(torch.device('cuda')), last_sentence_lens: Optional[tuple] = None, chain_training: Optional[bool] = True):

        src = self.word_embedding(src.long()) * math.sqrt(self.d_model)


        src_0 = src[1:]
        src = torch.cat((src[0].unsqueeze(0), first_sentence_embedding, src_0), dim=0)        # 'CLS', 'Sentence_0', 'SEP', 'Sentence_1', 'SEP'
        src = self.positional_encoder(src)
        src += self.segment_encoding(sentence_lens=sentence_lens, first_sentence_lens=first_sentence_lens).unsqueeze(1).repeat(1, self.batch_size, 1)

        src_mask = self.generate_src_mask(sentence_lens=sentence_lens, first_sentence_lens=first_sentence_lens)
        out = self.transformer_encoder(src, src_key_padding_mask=src_mask)
        out = self.classifier(out[0])

        return out, word_embedding

With
d_model = 300 (Word Embedding)
n_hids = 512 (Feed Forward Input - / Output - Size)
n_heads = 6
n_layers = 7
dropout_p = 0.25
.
.
.

I’ve implemented all the encodings (positional encoding and segment / sentence encoding). I use glove pretrained word embeddings and the SARC - Dataset, which should not be the problem. I think, in theory it should work, but for a reason, I haven’t found for more than a week, my model decides not to train for getting the BCEWithLogitsLoss at least lower than about 0.63, as that is the loss value it currently reaches, until it starts to overfit badly.
I really appreciate any help.
Regards,
Unity