Yeah, I thought about this. But in our particular case this would make model distribution much more complicated (or it would negate the effects of factorization).
But I fould an elegant solution. I forgot that bias=True is not mandatory in PyTorch. You can just do this and it solves the problem. You can pass the above class to the below class and it will just quantize:
class FactorizedQLinear(nn.Module):
    def __init__(self,
                 f_linear):
        super().__init__()
        self.in_features = f_linear.in_features
        self.out_features = f_linear.out_features
        self.dim_ratio = f_linear.dim_ratio
        self.u_linear = nn.Linear(in_features=f_linear.u.data.size(0),
                                  out_features=f_linear.u.data.size(1),
                                  bias=True)
        self.vh_linear = nn.Linear(in_features=f_linear.vh.data.size(0),
                                   out_features=f_linear.vh.data.size(1),
                                   bias=False)
        self.u_linear.weight.data = f_linear.u.data
        self.u_linear.bias.data = f_linear.bias
        self.vh_linear.weight.data = f_linear.vh.data

    def extra_repr(self) -> str:
        return (f'in_features={self.in_features}, '
                f'out_features={self.out_features}, '
                f'bias=True, dim_ratio={self.dim_ratio}')

    def forward(self, x):
        return self.u_linear(self.vh_linear(x))