Thanks!!  That was correct for that one.  But I still don’t understand the ‘with torch.no_grad()’ function works.  On line 64 in the code below I do an inplace operation under the no_grad() function an it increments Weights_HiddenToOutput._version from 0 to 1.  I thought it wasn’t supposed to do that under the no_grad() function.  Why does that happen??
(Sorry about the strange code formatting.  I’m new at this and I can’t figure out how to put all the code in one block quote.)
import torch
torch.autograd.set_detect_anomaly(True)
def sigmoid(x):
    return 1/(1+torch.exp(-x))

dtype = torch.float
device = torch.device("cpu")
Time, N, D_in, H, D_out = 7, 3, 10, 5, 2
X = torch.randn(Time, N, D_in, device=device, dtype=dtype, requires_grad=False)
Y = torch.randn(Time, D_out, D_in, device=device, dtype=dtype, requires_grad=False)
yHat = torch.randn(Time, D_out, D_in, device=device, dtype=dtype, requires_grad=False)
Hidden = 5
WeightsForget = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsForget = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsUpdate = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsCell = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsOutput = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
BiasForget = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
BiasUpdate = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
BiasCell = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
BiasOutput = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
Weights_HiddenToOutput = torch.randn(D_out, H, device=device, dtype=dtype, requires_grad=True)
Bias_HiddenToOutput = torch.randn(D_out, 1, device=device, dtype=dtype, requires_grad=True)
Hidden = torch.zeros(Time, H, D_in, device=device, dtype=dtype, requires_grad=False)
Cell = torch.zeros(Time, H, D_in, device=device, dtype=dtype, requires_grad=False)
hLast = torch.zeros(H, D_in, device=device, dtype=dtype, requires_grad=False)
Hidden[-1] = hLast
cLast = torch.zeros(hLast.shape)
Cell[-1] = cLast
learning_rate = 1e-6

for e in range(100):
    for t in range(len(X)):
        forget = sigmoid(torch.mm(WeightsForget, torch.cat((X[t,:,:],hLast),0)) + BiasForget)
        update = sigmoid(torch.mm(WeightsUpdate, torch.cat((X[t,:,:],hLast),0)) + BiasUpdate)
        candCell = torch.tanh(torch.mm(WeightsCell, torch.cat((X[t,:,:],hLast),0)) + BiasCell)
        cNext = forget * cLast + update * candCell
        output = sigmoid(torch.mm(WeightsOutput, torch.cat((X[t,:,:],hLast),0)) + BiasOutput)
        hNext = output * torch.tanh(cNext)

        prediction = torch.softmax((torch.mm(Weights_HiddenToOutput, hNext) + Bias_HiddenToOutput),0)

        Hidden[t] = hNext
        hLast = hNext.clone()
        Cell[t] = cNext
        cLast = cNext.clone()
        yHat[t] = prediction

    loss = (yHat - Y).pow(2).sum()
    loss.backward(retain_graph=True)

    with torch.no_grad():
        # Update Weights
        WeightsCell.data -= learning_rate * WeightsCell.grad
        WeightsOutput.data -= learning_rate * WeightsOutput.grad
        print('WeightsUpdate._version = ', WeightsUpdate._version)
        WeightsUpdate.data -= learning_rate * WeightsUpdate.grad
        print('WeightsUpdate._version = ', WeightsUpdate._version)
        WeightsForget.data -= learning_rate * WeightsForget.grad
        print('Weights_HiddenToOutput.shape = ', Weights_HiddenToOutput.shape)
        print('Weights_HiddenToOutput._version = ', Weights_HiddenToOutput._version)
        Weights_HiddenToOutput -= learning_rate * Weights_HiddenToOutput.grad
        print('Weights_HiddenToOutput._version = ', Weights_HiddenToOutput._version)
        
        # Update Bias
        BiasCell -= learning_rate * BiasCell.grad
        BiasForget -= learning_rate * BiasForget.grad
        BiasOutput -= learning_rate * BiasOutput.grad
        BiasUpdate -= learning_rate * BiasUpdate.grad
        Bias_HiddenToOutput -= learning_rate * Bias_HiddenToOutput.grad
        
        # Zero out the gradients
        WeightsCell.grad.zero_()
        WeightsForget.grad.zero_()
        WeightsOutput.grad.zero_()
        WeightsUpdate.grad.zero_()
        Weights_HiddenToOutput.grad.zero_()
        BiasCell.grad.zero_()
        BiasForget.grad.zero_()
        BiasOutput.grad.zero_()
        BiasUpdate.grad.zero_()
        Bias_HiddenToOutput.grad.zero_()