I updated to the nightly version. Using this code for testing:
import torch
import numpy as np
from torch.quantization import QuantStub, DeQuantStub, float_qparams_weight_only_qconfig, default_qconfig


class EmbeddingWithLinear(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.emb = torch.nn.Embedding(num_embeddings=10, embedding_dim=12)
        self.fc = torch.nn.Linear(5, 5)
        self.emb.qconfig = float_qparams_weight_only_qconfig
        self.qconfig = default_qconfig

    def forward(self, indices, linear_in):
        return self.emb(indices), self.fc(linear_in)


# create a model instance
model_fp32 = EmbeddingWithLinear()

indices_fp32 = torch.tensor(np.array([1, 3, 4, 5])).long()
input_fp32 = torch.randn(5, 5)

model_fp32.eval()
res = model_fp32(indices_fp32, input_fp32)
print(res)

model_fp32_prepared = torch.quantization.prepare(model_fp32)

model_fp32_prepared(indices_fp32, input_fp32)


model_int8 = torch.quantization.convert(model_fp32_prepared)
res = model_int8(indices_fp32, input_fp32)
print(res)


Getting this error:
RuntimeError: Could not run 'quantized::linear' with arguments from the 'CPU' backend. ...

Isn’t that exactly your example code?