Hi, can I use the same dropout object for multiple drop-out layers? And the same ReLU object? Or do these need to be created individually for each separate use in a layer?
e.g. this:
class Model(nn.Module):
  def __init__(self):
    super().__init__()
    self.dropout = nn.Dropout(0.5)
    self.relu = nn.ReLU()
    self.lin1 = nn.Linear(4096, 4096)
    self.lin2 = nn.Linear(4096, 4096)
    self.lin3 = nn.Linear(4096, 4096)
  def forward(self, x):
    output = self.dropout(self.relu(self.lin1(x)))
    output = self.dropout(self.relu(self.lin2(output)))
    output = self.dropout(self.relu(self.lin3(output)))
    ....

vs:
class Model(nn.Module):
  def __init__(self):
    super().__init__()
    self.dropout1 = nn.Dropout(0.5)
    self.dropout2 = nn.Dropout(0.5)
    self.dropout3 = nn.Dropout(0.5)
    self.relu1 = nn.ReLU()
    self.relu2 = nn.ReLU()
    self.relu3 = nn.ReLU()
    self.lin1 = nn.Linear(4096, 4096)
    self.lin2 = nn.Linear(4096, 4096)
    self.lin3 = nn.Linear(4096, 4096)
  def forward(self, x):
    output = self.dropout1(self.relu1(self.lin1(x)))
    output = self.dropout2(self.relu2(self.lin2(output)))
    output = self.dropout3(self.relu3(self.lin3(output)))
    ....

I am pretty sure it wouldn’t make a different for the ReLU, but I am not so convinced about the dropout?