Oh thanks. That was a clumsy mistake
I added loss.backward
Better or worse??
Epoch 1 Accuracy 49.0854 Loss 1.098612
Epoch 2 Accuracy 49.1870 Loss 1.098612
Epoch 3 Accuracy 49.2378 Loss 1.098612
Epoch 4 Accuracy 49.1362 Loss 1.098612
Epoch 5 Accuracy 49.1870 Loss 1.098612
Epoch 6 Accuracy 49.2124 Loss 1.098612
Epoch 7 Accuracy 49.2124 Loss 1.098612
Epoch 8 Accuracy 49.1870 Loss 1.098612
Epoch 9 Accuracy 49.1616 Loss 1.098612
Epoch 10 Accuracy 49.1870 Loss 1.098612
Epoch 11 Accuracy 49.2124 Loss 1.098612
Epoch 12 Accuracy 49.3394 Loss 1.098612
Epoch 13 Accuracy 49.2632 Loss 1.098612
Epoch 14 Accuracy 49.2378 Loss 1.098612
Epoch 15 Accuracy 49.1616 Loss 1.098612
Epoch 16 Accuracy 49.2632 Loss 1.098612
Epoch 17 Accuracy 49.3394 Loss 1.098612
Epoch 18 Accuracy 49.3648 Loss 1.098612
Epoch 19 Accuracy 49.1616 Loss 1.098612
Epoch 20 Accuracy 49.1870 Loss 1.098612
Epoch 21 Accuracy 49.1108 Loss 1.098612
Epoch 22 Accuracy 49.2632 Loss 1.098612
Epoch 23 Accuracy 49.1870 Loss 1.098612
Epoch 24 Accuracy 49.2632 Loss 1.098612
Epoch 25 Accuracy 49.2632 Loss 1.098612
Epoch 26 Accuracy 49.2886 Loss 1.098612
Epoch 27 Accuracy 49.1870 Loss 1.098612

The loss is now constant and the accuracy doesnt increase or reduce
Network Architecture
class Network(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1,64,4,2,1)
        self.conv2 = nn.Conv2d(64,64,2,1,1)

        self.fc1 = nn.Linear(64*5*5,1024)
        self.fc2 = nn.Linear(1024,3)

    def forward(self,x):
        x = x.view(-1,1,8,8)
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))

        x = x.view(-1,64*25)

        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        return x

    def load(f):
        self.state_dict = torch.load(f).state_dict()
        self.eval()

Train loop
trainloader = DataLoader(train,batch_size=128,shuffle=True)
testloader = DataLoader(test,batch_size=32,shuffle=True)
print(f'Name {" "*10}|len\nTest {" "*10}|{len(test)}\nTrin {" "*10}|{len(train)}')

epochs = 500
lr = 0.001

loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(),lr=lr)
pacc = 0
for e in range(epochs):
    net.train()
    for x,y in trainloader:
        y=torch.argmax(y,dim=1)
        optimizer.zero_grad()
        p = net(x)
        loss = loss_fn(p,y)
        loss.backward()
        optimizer.step()
    net.eval()
    acc = []
    with torch.no_grad():
        for x,y in testloader:
            x = x.unsqueeze(1)
            p = net(x)
            a = accuracy(p,torch.argmax(y,dim=1))
            acc.append(a)
        acc = torch.mean(torch.Tensor(acc))

    print(f'Epoch {e+1} Accuracy {acc:.4f} Loss {loss:5f}')
    if acc > pacc:
        torch.save(net,f'acc_{acc:.4f}')
        pacc = acc