I created a gist that should help reproduce this, it generally occurs within the 10th epoch.

  
      gist.github.com
  
  
    https://gist.github.com/anonymous/a5b4282a46be7e2971baf690e8cde054 11
invalid_argument_error.py
import os
import torch
import torch.distributed as dist
from torch import multiprocessing, nn, randperm
from torch.nn import functional as F
from torch.utils.data import DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
from torch.autograd import Variable
from torch.optim import SGD
from torchvision import datasets, transforms
This file has been truncated. show original