I haven’t tried this tutorial but I think you’ve just got the call to super wrong.
Try:
class DistributedModel(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.embedding = torch.nn.Embedding(1000, 10)
        self.rnn = torch.nn.Linear(10, 10).cuda(0)

    def forward(self, x):
        # Compute embedding on CPU
        x = self.embedding(x)

        # Transfer to GPU
        x = x.cuda(0)

        # Compute RNN on GPU
        x = self.rnn(x)
        return x