Hi,
I’m trying to train an LSTM model for a time series problem, and I’m getting an error on the backward step. I’m using a custom dataset to load the data from a *.csv file and a dataloader.
This is my code:
LSTM Model:
class LSTM_RUL_Estimator(nn.Module):

    def __init__(self, n_features, hidden_dim, seq_length, num_layers=2, output_dim=1):
        super(LSTM_RUL_Estimator, self).__init__()
        self.hidden_dim = hidden_dim
        self.seq_length = seq_length
        self.num_layers = num_layers

        # Define the LSTM layers
        self.lstm = nn.LSTM(
            input_size=n_features,
            hidden_size=self.hidden_dim,
            num_layers=self.num_layers,
            batch_first=True,
            dropout=0.2
        )
        self.linear = nn.Linear(in_features=self.hidden_dim, out_features=output_dim)

    def forward(self, input):
        # Initialize hidden state with zeros
        h0 = torch.zeros(self.num_layers, input.size(0), self.hidden_dim).requires_grad_().to(device)
        # Initialize cell state
        c0 = torch.zeros(self.num_layers, input.size(0), self.hidden_dim).requires_grad_().to(device)

        lstm_out, (hn, cn) = self.lstm(input.float(), (h0.detach(), c0.detach()))
        pred = self.linear(lstm_out[:, -1, :])
        return pred

this is my “train” function:
# Training Function
def train_model(model, loss_function, optimizer, num_epochs=25):
    since = time.time()

    train_loss = []
    validation_loss = []

    for epoch in range(1, num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs))
        print('-' * 10)

        # Each epoch has a training phase and a validation phase at every 10 epochs
        for phase in ['train', 'test']:
            # Set model to training or evaluation mode
            model.train() if phase == 'train' else model.eval()

            # Iterate over data.
            for idx, (inputs, labels) in tqdm(enumerate(dataloaders[phase]),
                                              leave=True,
                                              total=len(dataloaders[phase])):
                inputs = inputs.to(device)
                labels = labels.to(device)
                labels = labels.float()

                # Pytorch accumulates gradients, we need to clear them out before each instance.
                model.zero_grad()

                if phase == 'train':
                    outputs = model(inputs)
                    loss = loss_function(outputs, labels)
                    train_loss.append(loss.item())
                    if epoch % 10 == 0:
                        print(f'Epoch {epoch} train loss: {loss.item()}')

                if phase == 'test':
                    with torch.no_grad():
                        outputs_val = model(inputs)
                        loss_val = loss_function(outputs_val, labels)
                    validation_loss.append(loss_val.item())
                    if epoch % 10 == 0:
                        print(f'Epoch {epoch} train loss: {loss.item()} val loss: {loss_val.item()}')

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
    return model, train_loss, validation_loss

when I call the train function I get the error:
loss.backward()
  File "C:\Miniconda3\envs\PyTorch\lib\site-packages\torch\_tensor.py", line 255, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "C:\Miniconda3\envs\PyTorch\lib\site-packages\torch\autograd\__init__.py", line 147, in backward
    Variable._execution_engine.run_backward(
RuntimeError: Trying to backward through the graph a second time (or directly access saved variables after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved variables after calling backward.

and if I use: loss.backward(retain_graph=True) then I get the error:
loss.backward(retain_graph=True)
  File "C:\Miniconda3\envs\PyTorch\lib\site-packages\torch\_tensor.py", line 255, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "C:\Miniconda3\envs\PyTorch\lib\site-packages\torch\autograd\__init__.py", line 147, in backward
    Variable._execution_engine.run_backward(
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [100, 1]], which is output 0 of TBackward, is at version 646; expected version 645 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

any help would be appreciated.