It is possible to set the qconfig of Embeddings to None if you wish to skip quantizing them. For example
class EmbeddingWithLinear(torch.nn.Module):
            def __init__(self):
                super().__init__()
                self.emb = torch.nn.Embedding(num_embeddings=10, embedding_dim=12)
                self.fc = torch.nn.Linear(5, 5)
                self.emb.qconfig = None
                self.qconfig = default_qconfig

            def forward(self, indices, linear_in):
                return self.emb(indices), self.fc(linear_in)