Yes I did put it on GPU.
def load_model(args, model_class, config):

    model = BERTplusAoA(config, args)
    

    if args.local_rank == 0:
        # Make sure only the first process in distributed training will download model & vocab
        torch.distributed.barrier()

    model.to(args.device)
...