As = [...]  # list of tensors
Bs = [...]  # list of tensors

#copy each tensor to it a GPU
for i in range(4):
 As[i] = As[i].cuda(device=i)
 Bs[i] = Bs[i].cuda(device=i)

# solve linear system
outputs = []
for i in range(4):
   outputs.append(torch.gesv(Bs[i], As[i])

for i in range(4):
  with torch.cuda.device(i):
    torch.cuda.synchronize()

I don’t know why you’d want to explicitly synchronize, though. Copying to the CPU will also synchronize.