I just made this change - zeroing grad right after optimizer.step() and this works. I cannot figure out why, but this works.
          for j in tqdm(range(self.train_feed.num_batch), desc='Trainer.{}'.format(self.name())):
                self.optimizer.zero_grad()
                input_ = self.train_feed.next_batch()
                _, loss, __ = self.train_one_batch(input_)

                print('loss: ', loss.item())
                losses.append(loss)
                loss.backward()

                self.optimizer.step()


          for j in tqdm(range(self.train_feed.num_batch), desc='Trainer.{}'.format(self.name())):
                input_ = self.train_feed.next_batch()
                _, loss, __ = self.train_one_batch(input_)

                print('loss: ', loss.item())
                losses.append(loss)
                loss.backward()

                self.optimizer.step()
                self.optimizer.zero_grad()