Thanks for your reply. Yes, I know which part should block it.
I tried with the following code, but when I check the weights, all of them does not have the gradient.
class simm(nn.Module):
    def __init__(self, input_channels=1, output_channels=32):
        super(simm, self).__init__()
        self.layer1 =  nn.Conv3d(in_channels=input_channels, out_channels=output_channels, kernel_size=(3, 3, 3), padding=1)
        self.layer2 = nn.Conv3d(in_channels=output_channels, out_channels=2*output_channels, kernel_size=(3, 3, 3), padding=1)
        self.layer3 = nn.Conv3d(in_channels=2 * output_channels + 1, out_channels=1, kernel_size=(3, 3, 3), padding=1)
        self.layer4 =  nn.ReLU(inplace=False)

    def forward(self, x):
        o1 = self.layer1(x)
        o2 = self.layer2(o1)
        x1 = o2.detach()
        x2 = torch.cat((x1,x),1)
        o3 = self.layer3(x2)
        return self.layer4(o3)

ceriterion = nn.MSELoss()

model = simm().to(device)
w1 = model.layer1.weight.data
w2 = model.layer2.weight.data
w3 = model.layer3.weight.data

optimizer = optim.Adam(model.parameters(), lr=0.1)
    
data = torch.rand(5,1,10,12,13).to(device)

model.train()

output_b = model(data)

lablel = 10*torch.ones_like(output_b)
loss = ceriterion(output_b,lablel)
optimizer.zero_grad()
loss.backward()
optimizer.step() 
print(torch.mean(abs(model.layer1.weight.data - w1)))
print(torch.mean(abs(model.layer2.weight.data - w2)))
print(torch.mean(abs(model.layer3.weight.data - w3)))

tensor(0., device='cuda:0')
tensor(0., device='cuda:0')
tensor(0., device='cuda:0')

I think the weights at the last layer should change, am I right?