Hi,
I am currently fiddling around with a PPO implementation where the loss depends on the entropy of a multivariate normal distribution. The entropy is calculated as follows
class Actor(nn.Module):
    def __init__(self, state_dim, action_dim, config):
        super(Actor, self).__init__()

        self.net = nn.Sequential(
                        nn.Linear(state_dim, 64),
                        nn.ReLU(),
                        nn.Linear(64, 64),
                        nn.ReLU(),
                        nn.Linear(64, action_dim),
                        nn.Tanh())

        self.action_std = torch.nn.Parameter(torch.ones(action_dim)*action_std).to(device)

    def evaluate(self, state, action):
        action_mean = self.net(state)
        dist = torch.distributions.normal.Normal(action_mean, self.action_std)
        action_logprobs = torch.sum(dist.log_prob(action), dim=1)
        dist_entropy = torch.sum(dist.entropy(), dim=1)
        return action_logprobs, dist_entropy

and the loss also depends on the return value “dist_entropy”. However during the backward pass the parameter “self.action_std” does not change at all. Any clue about the reason?