Hello, thank for attention I ran into the following problem, the weights before and after did not change, here is the code:
class fastRCNN(torch.nn.Module):
def __init__(self, params):

    super(fastRCNN, self).__init__()

    # roi pooling size

    self.rsize = params['roi']['output_size']

    ###########################   model architecture

    ###########################   CNN BLOCK

    self.cnn_block = layer.Sequential(

        layer.Conv2d(**params['cnn11']),

        layer.Conv2d(**params['cnn12']),

        layer.AvgPool2d(**params['pool1']),

        ##################################

        layer.Conv2d(**params['cnn21']),

        layer.Conv2d(**params['cnn22']),

        layer.AvgPool2d(**params['pool2']),

        ##################################

        layer.Conv2d(**params['cnn31']),

        layer.Conv2d(**params['cnn32']),

        layer.Conv2d(**params['cnn33']),

        layer.AvgPool2d(**params['pool3']),

        ##################################

        layer.Conv2d(**params['cnn41']),

        layer.Conv2d(**params['cnn42']),

        layer.Conv2d(**params['cnn43']),

        layer.AvgPool2d(**params['pool4']),

    )

    ########### **ROI LAYER** ##########

    ###########################   FC

    self.fc_block = layer.Sequential(

        layer.Linear(**params['dense11']),

        layer.Linear(**params['dense12'])

    )

    ###########################   CLASSIFIER OUTPUT

    self.classifier_output = layer.Sequential(

        layer.Linear(**params['denseclf'])

        #layer.Softmax()

    )

    ###########################   REGRESSOR OUTPUT

    self.regressor_output = layer.Linear(**params['densereg'])

def forward(self, Ximg, Xroi):

    cnn_inp = self.cnn_block(Ximg)

    ###########################

    roi_inp = roi_pool(cnn_inp, Xroi, self.rsize)

    ###########################

    inp = roi_inp.view(383, 512*7*7)

    inp = self.fc_block(inp)

    ###########################

    classif_out = self.classifier_output(inp)

    ###########################

    regress_out = self.regressor_output(inp)

    return torch.cat((classif_out, regress_out), dim=1)

params = {
            "cnn11": {"in_channels":3, "out_channels":64, "kernel_size":3},

            "cnn12": {"in_channels": 64, "out_channels":64,"kernel_size": 3},

            "pool1": {"kernel_size": 3},

            "cnn21": {"in_channels": 64, "out_channels":128, "kernel_size": 3},

            "cnn22": {"in_channels": 128, "out_channels":128, "kernel_size": 3},

            "pool2": {"kernel_size": 3},

            "cnn31": {"in_channels": 128, "out_channels":256, "kernel_size": 3},

            "cnn32": {"in_channels": 256, "out_channels":256 ,"kernel_size": 3},

            "cnn33": {"in_channels": 256, "out_channels":256, "kernel_size": 3},

            "pool3": {"kernel_size": 3},

            "cnn41": {"in_channels": 256, "out_channels":512, "kernel_size": 3},

            "cnn42": {"in_channels": 512, "out_channels":512, "kernel_size": 3},

            "cnn43": {"in_channels": 512, "out_channels":512, "kernel_size": 3},

            "pool4": {"kernel_size": 2},

            "roi": {"output_size": [7, 7]},

            "dense11": {"in_features":512*7*7, "out_features":4096},

            "dense12": {"in_features":4096, "out_features":4096},

            "denseclf": {"in_features":4096, "out_features": 1},

            "densereg": {"in_features":4096, "out_features": 4}

        }

model = fastRCNN(params)
Code for training model:
def lossf(tPred, tTrue):
p = torch.autograd.Variable(tPred[0].data, requires_grad=True)

print("test", p)

b = torch.autograd.Variable(tPred[1:].data, requires_grad=True)

a = torch.autograd.Variable(tTrue[1:].data, requires_grad=True) 

print("test", a)

return torch.sum(torch.abs(b-a)) - torch.log(p)

optimizer.zero_grad()
labels = torch.from_numpy(out[0])
outputs = model(torch.from_numpy(rgb_matr), torch.from_numpy(np.insert(one_img[1], 0, np.arange(one_img[1].shape[0]), axis=1)))
for ypred, ytrue in zip(outputs, labels):
loss = lossf(ypred, ytrue)

loss.backward(retain_graph=True)

optimizer.step()