Sure, thanks for that. This is the most basic code based implementatoin that would reproduce the error on my system without the NCCL_SOCKET_IFNAME tweaks.

import torch
from torch.utils.data.distributed import DistributedSampler
from torch.utils.data import DataLoader
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import argparse
import os
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("–local_rank", type=int, help=“Local rank. Necessary for using the torch.distributed.launch utility.”)
arg = parser.parse_args()
local_rank = arg.local_rank   
torch.cuda.set_device(arg.local_rank)

print(os.environ["MASTER_PORT"], os.environ["MASTER_ADDR"], arg.local_rank, os.environ["LOCAL_RANK"], os.environ["RANK"], os.environ["WORLD_SIZE"])

torch.distributed.init_process_group(backend='nccl', init_method='env://') #, world_size=int(os.environ["WORLD_SIZE"]), rank=int(os.environ["RANK"]))
model = torchvision.models.resnet18(pretrained=False)

model = model.cuda()

ddp_model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[local_rank], output_device=local_rank)
  
transform = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

train_set = torchvision.datasets.CIFAR10(root="data", train=True, download=False, transform=transform) 
test_set = torchvision.datasets.CIFAR10(root="data", train=False, download=False, transform=transform)

train_sampler = DistributedSampler(dataset=train_set)

train_loader = DataLoader(dataset=train_set, batch_size=256, sampler=train_sampler, num_workers=8)
test_loader = DataLoader(dataset=test_set, batch_size=128, shuffle=False, num_workers=8)

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(ddp_model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-5)

for epoch in range(5):
    ddp_model.train()

    for data in train_loader:
        inputs, labels = data[0].cuda(), data[1].cuda() #data[0].to(device), data[1].to(device)
        optimizer.zero_grad()
        outputs = ddp_model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

if name == “main”:
main()

Thanks for your offer of assistance (and to @aivanou for the comments on elastic - I will have to read more into elastic to better understand it).