Hey, I figured out how to do it with autograd.Function after reading through the pages. Let me know if it makes sense.
class TwoOpGradientController(torch.autograd.Function):
    @staticmethod
    def forward(ctx, *args):
        ctx.args_0_shape = args[0].shape
        return args[0] @ args[1]
    
    @staticmethod
    def backward(ctx, grad_output):
        lhs_zero_grad_input = torch.zeros_like(ctx.args_0_shape)
        rhs_grad_output = # do weird stuff to rhs gradients
        
        return lhs_zero_grad_input, rhs_grad_output