Here is essentially the training code, I removed a few non-essential things for clarity.
class Trainer:
    def __init__(self, model, train_ds, val_ds,
                 train_bs, val_bs, ignore_idx, dest_path):
        self.model = model
       
 # optimizer and scheduler are initialized in the train() method
        self.opti = None
        self.scheduler = None

        self.ignore_idx = ignore_idx
        self.loss = nn.CrossEntropyLoss(ignore_index=self.ignore_idx)
        self.train_dl = DataLoader(train_ds, train_bs)
        self.val_dl = DataLoader(val_ds, val_bs)

    def train(self, epochs: int, start_lr: float, lr_scheduler='no', **kwargs):

        possibleOptimizers = {
         "Adam" : optim.Adam(self.model.parameters(), start_lr),
         "SGD" : optim.SGD(self.model.parameters(), lr=start_lr, momentum=0.9),
         "Adadelta" : optim.Adadelta(self.model.parameters(), lr=start_lr),
         "Adagrad" : optim.Adagrad(self.model.parameters(), lr=start_lr),
         "Adamax": optim.Adamax(self.model.parameters(), lr=start_lr),
         "ASGD" : optim.ASGD(self.model.parameters(), lr=start_lr),
         "LBFGS" : optim.LBFGS(self.model.parameters(), lr=start_lr),
         "RMSprop" : optim.RMSprop(self.model.parameters(), lr=start_lr)
        }

        self.opti = possibleOptimizers.get(kwargs.get("optimizer", "Adam"))


        #initialize scheduler
        lr_factor = kwargs.get('lr_factor', 0.5)
        patience = kwargs.get('patience', 5)
        reduce_every = kwargs.get('reduce_every', 30)
        step_size_up = kwargs.get('step_size_up', 400)
        self.init_scheduler(lr_scheduler, lr_factor=lr_factor, patience=patience, n_epochs=epochs,
                            steps_per_epoch=len(self.train_dl), base_lr = start_lr*0.5, max_lr=start_lr*15, step_size_up = step_size_up, reduce_at_ep=reduce_every)

        per_epoch_train_loss
        
# Drawing one random sample and keeping it for rest of training.
        test_sample = iter(self.train_dl).next()
        for epoch in range(epochs):
            self.model.train()
            for i, train_batch in enumerate(self.train_dl):
                #only one sample for testing
                midis, chordseqs = test_sample['midi'], test_sample['chords']
                #shift tgt sequences so that inps start with BOS token
                chordseqs_tgt, chordseqs_inp = chordseqs[:, 1:], chordseqs[:, :chordseqs.size()[1]-1]
                preds = self.model(midis, chordseqs_inp) 
                train_loss = self.loss(preds.view(-1, preds.shape[2]),
                                        chordseqs_tgt.reshape(-1))
                
                train_loss.backward()
                #torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5)
                self.opti.step()
                self.opti.zero_grad()
                
                if lr_scheduler == 'one_cycle' or lr_scheduler == 'cyclic':
                    self.scheduler.step()

                per_epoch_train_loss += train_loss.item()
            print(per_epoch_train_loss/len(self.train_dl))

All the things you mention are essentially there. I also tried with something as dumb as predicting b from a with:
a = torch.Tensor([[[1],[2],[3],[1],[2],[3]]])
b = torch.Tensor([[2],[3],[1],[2],[3]]).transpose(0,1)
b = b.long()

After adapting the dimensions of the model. Even there, the loss doesn’t decrease to 0, so there is definitely something very wrong here, but for the life of me I can’t figure out what.