Until this is fixed I was thinking of using a custom Embedding layer.
The following has very limited functionality but enough for my use case. I don’t know if it’s fully correct though:-
class myEmbedding(nn.Module):
    def __init__(self, inp, out):
        super().__init__()

        self.inp = inp
        self.out = out

        self.embeddings = nn.Parameter(torch.randn(inp, out))
        self.dim_bias = nn.Parameter(torch.randn(out))
        self.bias = nn.Parameter(torch.randn(1))

        self.params = nn.ParameterList([self.embeddings, self.dim_bias, self.bias])

    def forward(self, ind):
        # inp = (*)
        ind_shape = ind.size()
        ind = ind.view(-1)  # (<*>)
        emb = self.embeddings[ind] + self.dim_bias[None, :] + self.bias[None, :]  # (<*>, out dim)
        emb = emb.view(*ind_shape, self.out)  # (*, out dim)
        return emb