How are you initializing ht_1?
Since you are not updating it, I assume it’s a standard tensor?
This code snippet seems to work:
whh = torch.randn(10,10,requires_grad=True)
wxt = torch.randn(10,1,requires_grad=True)
why = torch.randn(1,10,requires_grad=True)
ht_1 = torch.randn(10,1)
lr = 1e-3

for i in range(10):
    inp=torch.randn(1,1)
    y=torch.randn(1,1)
        
    ht=torch.tanh(whh.mm(ht_1)+wxt.mm(inp.float()))
    y_pred=why.mm(ht)

    loss=(y_pred-y.float()).pow(2).sum()
    loss.backward()
    
    with torch.no_grad():
        whh-=lr*whh.grad
        wxt-=lr*wxt.grad
        why-=lr*why.grad
        whh.grad.zero_()
        wxt.grad.zero_()
        why.grad.zero_()
        ht_1=ht.detach()
    print(loss)