My final choice, A little bit slightly complicated…
class MirrorLinear(nn.Module):
    def __init__(self, in_features, out_features, weight, bias=True):
        super(MirrorLinear, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.register_parameter('weight', weight)
        if bias:
            self.bias = nn.Parameter(torch.Tensor(out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_bias()

        def reset_bias(self):
            stdv = 1. / math.sqrt(self.weight.size(1))
            if self.bias is not None:
                self.bias.data.uniform_(-stdv, stdv)

        def forward(self, input):
            if self.bias is None:
                return self._backend.Linear()(input, self.weight.t())
            else:
                return self._backend.Linear()(input, self.weight.t(), self.bias)

        def __repr__(self):
            return self.__class__.__name__ + ' (' \
                + str(self.in_features) + ' -> ' \
                + str(self.out_features) + ')'