Hi,
I am quite new to PyTorch forecasting and I am trying to build a TimeSeriesDataSet (number of infections per country).
dateRep                            object
day                              category
month                            category
year                             category
cases                               int64
deaths                              int64
countriesAndTerritories            object
geoId                              object
countryterritoryCode               object
popData2020                         int64
continentExp                       object
NEWTIME                    datetime64[ns]
TIME_IDX                            int64
weekday                          category
dtype: object

Made a TimeSeriesDataset out of it:
{'time_idx': 'TIME_IDX',
 'target': ['cases'],
 'group_ids': ['countriesAndTerritories', 'geoId', 'countryterritoryCode'],
 'weight': None,
 'max_encoder_length': 14,
 'min_encoder_length': 7,
 'min_prediction_idx': 0,
 'min_prediction_length': 1,
 'max_prediction_length': 7,
 'static_categoricals': ['countriesAndTerritories',
  'geoId',
  'countryterritoryCode'],
 'static_reals': ['popData2020', 'encoder_length'],
 'time_varying_known_categoricals': ['day', 'month', 'year', 'weekday'],
 'time_varying_known_reals': ['TIME_IDX'],
 'time_varying_unknown_categoricals': [],
 'time_varying_unknown_reals': ['cases'],
 'variable_groups': {},
 'constant_fill_strategy': {},
 'allow_missing_timesteps': True,
 'lags': {},
 'add_relative_time_idx': False,
 'add_target_scales': False,
 'add_encoder_length': True,
 'target_normalizer': MultiNormalizer(normalizers=[NaNLabelEncoder()]),
 'categorical_encoders': {'data': NaNLabelEncoder(add_nan=True),
  '__group_id__countriesAndTerritories': NaNLabelEncoder(),
  '__group_id__geoId': NaNLabelEncoder(),
  '__group_id__countryterritoryCode': NaNLabelEncoder(),
  'countriesAndTerritories': NaNLabelEncoder(),
  'geoId': NaNLabelEncoder(),
  'countryterritoryCode': NaNLabelEncoder(),
  'day': NaNLabelEncoder(),
  'month': NaNLabelEncoder(),
  'year': NaNLabelEncoder(),
  'weekday': NaNLabelEncoder()},
 'scalers': {'popData2020': StandardScaler(),
  'encoder_length': StandardScaler(),
  'TIME_IDX': StandardScaler()},
 'randomize_length': None,
 'predict_mode': False}

Everything works fine until trying to find the correct learning rate. I get an index error for the validation dataset.
# find optimal learning rate
res = trainer.tuner.lr_find(
    tft,
    train_dataloader=train_dataloader,
    val_dataloaders=val_dataloader,
    max_lr=10.0,
    min_lr=1e-6,
)

print(f"suggested learning rate: {res.suggestion()}")
fig = res.plot(show=True, suggest=True)
fig.show()

Error:
/Users/quirly/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:817: LightningDeprecationWarning: `trainer.tune(train_dataloader)` is deprecated in v1.4 and will be removed in v1.6. Use `trainer.tune(train_dataloaders)` instead. HINT: added 's'
  rank_zero_deprecation(

   | Name                               | Type                            | Params
----------------------------------------------------------------------------------------
0  | loss                               | QuantileLoss                    | 0     
1  | logging_metrics                    | ModuleList                      | 0     
2  | input_embeddings                   | MultiEmbedding                  | 1.4 K 
3  | prescalers                         | ModuleDict                      | 64    
4  | static_variable_selection          | VariableSelectionNetwork        | 1.6 K 
5  | encoder_variable_selection         | VariableSelectionNetwork        | 1.7 K 
6  | decoder_variable_selection         | VariableSelectionNetwork        | 1.1 K 
7  | static_context_variable_selection  | GatedResidualNetwork            | 1.1 K 
8  | static_context_initial_hidden_lstm | GatedResidualNetwork            | 1.1 K 
9  | static_context_initial_cell_lstm   | GatedResidualNetwork            | 1.1 K 
10 | static_context_enrichment          | GatedResidualNetwork            | 1.1 K 
11 | lstm_encoder                       | LSTM                            | 2.2 K 
12 | lstm_decoder                       | LSTM                            | 2.2 K 
13 | post_lstm_gate_encoder             | GatedLinearUnit                 | 544   
14 | post_lstm_add_norm_encoder         | AddNorm                         | 32    
15 | static_enrichment                  | GatedResidualNetwork            | 1.4 K 
16 | multihead_attn                     | InterpretableMultiHeadAttention | 1.1 K 
17 | post_attn_gate_norm                | GateAddNorm                     | 576   
18 | pos_wise_ff                        | GatedResidualNetwork            | 1.1 K 
19 | pre_output_gate_norm               | GateAddNorm                     | 576   
20 | output_layer                       | Linear                          | 119   
----------------------------------------------------------------------------------------
20.1 K    Trainable params
0         Non-trainable params
20.1 K    Total params
0.080     Total estimated model params size (MB)
/Users/quirly/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/data_loading.py:105: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  rank_zero_warn(
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
/var/folders/vd/h0c811n10fq4tk4gw79gg5jw0000gn/T/ipykernel_2661/1514959711.py in <module>
      1 # find optimal learning rate
----> 2 res = trainer.tuner.lr_find(
      3     tft,
      4     train_dataloader=train_dataloader,
      5     val_dataloaders=val_dataloader,

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/tuner/tuning.py in lr_find(self, model, train_dataloaders, val_dataloaders, datamodule, min_lr, max_lr, num_training, mode, early_stop_threshold, update_attr, train_dataloader)
    186         """
    187         self.trainer.auto_lr_find = True
--> 188         result = self.trainer.tune(
    189             model,
    190             train_dataloaders=train_dataloaders,

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in tune(self, model, train_dataloaders, val_dataloaders, datamodule, scale_batch_size_kwargs, lr_find_kwargs, train_dataloader)
    835         )
    836 
--> 837         result = self.tuner._tune(model, scale_batch_size_kwargs=scale_batch_size_kwargs, lr_find_kwargs=lr_find_kwargs)
    838 
    839         assert self.state.stopped

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/tuner/tuning.py in _tune(self, model, scale_batch_size_kwargs, lr_find_kwargs)
     51         if self.trainer.auto_lr_find:
     52             lr_find_kwargs.setdefault("update_attr", True)
---> 53             result["lr_find"] = lr_find(self.trainer, model, **lr_find_kwargs)
     54 
     55         self.trainer.state.status = TrainerStatus.FINISHED

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/tuner/lr_finder.py in lr_find(trainer, model, min_lr, max_lr, num_training, mode, early_stop_threshold, update_attr)
    246 
    247     # Fit, lr & loss logged in callback
--> 248     trainer.tuner._run(model)
    249 
    250     # Prompt if we stopped early

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/tuner/tuning.py in _run(self, *args, **kwargs)
     61         self.trainer.state.status = TrainerStatus.RUNNING  # last `_run` call might have set it to `FINISHED`
     62         self.trainer.training = True
---> 63         self.trainer._run(*args, **kwargs)
     64         self.trainer.tuning = True
     65 

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in _run(self, model)
    920 
    921         # dispatch `start_training` or `start_evaluating` or `start_predicting`
--> 922         self._dispatch()
    923 
    924         # plugin will finalized fitting (e.g. ddp_spawn will load trained model)

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in _dispatch(self)
    988             self.accelerator.start_predicting(self)
    989         else:
--> 990             self.accelerator.start_training(self)
    991 
    992     def run_stage(self):

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in start_training(self, trainer)
     90 
     91     def start_training(self, trainer: "pl.Trainer") -> None:
---> 92         self.training_type_plugin.start_training(trainer)
     93 
     94     def start_evaluating(self, trainer: "pl.Trainer") -> None:

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer)
    159     def start_training(self, trainer: "pl.Trainer") -> None:
    160         # double dispatch to initiate the training loop
--> 161         self._results = trainer.run_stage()
    162 
    163     def start_evaluating(self, trainer: "pl.Trainer") -> None:

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in run_stage(self)
    998         if self.predicting:
    999             return self._run_predict()
-> 1000         return self._run_train()
   1001 
   1002     def _pre_training_routine(self):

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in _run_train(self)
   1033             self.progress_bar_callback.disable()
   1034 
-> 1035         self._run_sanity_check(self.lightning_module)
   1036 
   1037         # enable train mode

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in _run_sanity_check(self, ref_model)
   1120             # run eval step
   1121             with torch.no_grad():
-> 1122                 self._evaluation_loop.run()
   1123 
   1124             self.on_sanity_check_end()

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/loops/base.py in run(self, *args, **kwargs)
    109             try:
    110                 self.on_advance_start(*args, **kwargs)
--> 111                 self.advance(*args, **kwargs)
    112                 self.on_advance_end()
    113                 self.iteration_count += 1

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py in advance(self, *args, **kwargs)
    108         dl_max_batches = self._max_batches[self.current_dataloader_idx]
    109 
--> 110         dl_outputs = self.epoch_loop.run(
    111             dataloader_iter, self.current_dataloader_idx, dl_max_batches, self.num_dataloaders
    112         )

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/loops/base.py in run(self, *args, **kwargs)
    109             try:
    110                 self.on_advance_start(*args, **kwargs)
--> 111                 self.advance(*args, **kwargs)
    112                 self.on_advance_end()
    113                 self.iteration_count += 1

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py in advance(self, dataloader_iter, dataloader_idx, dl_max_batches, num_dataloaders)
    109         # lightning module methods
    110         with self.trainer.profiler.profile("evaluation_step_and_end"):
--> 111             output = self.evaluation_step(batch, batch_idx, dataloader_idx)
    112             output = self.evaluation_step_end(output)
    113 

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py in evaluation_step(self, batch, batch_idx, dataloader_idx)
    156             self.trainer.lightning_module._current_fx_name = "validation_step"
    157             with self.trainer.profiler.profile("validation_step"):
--> 158                 output = self.trainer.accelerator.validation_step(step_kwargs)
    159 
    160         return output

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in validation_step(self, step_kwargs)
    209         """
    210         with self.precision_plugin.val_step_context(), self.training_type_plugin.val_step_context():
--> 211             return self.training_type_plugin.validation_step(*step_kwargs.values())
    212 
    213     def test_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[STEP_OUTPUT]:

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in validation_step(self, *args, **kwargs)
    176 
    177     def validation_step(self, *args, **kwargs):
--> 178         return self.model.validation_step(*args, **kwargs)
    179 
    180     def test_step(self, *args, **kwargs):

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_forecasting/models/base_model.py in validation_step(self, batch, batch_idx)
    368     def validation_step(self, batch, batch_idx):
    369         x, y = batch
--> 370         log, out = self.step(x, y, batch_idx)
    371         log.update(self.create_log(x, y, out, batch_idx))
    372         return log

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_forecasting/models/base_model.py in step(self, x, y, batch_idx, **kwargs)
    490             loss = loss * (1 + monotinicity_loss)
    491         else:
--> 492             out = self(x, **kwargs)
    493 
    494             # calculate loss

~/opt/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_forecasting/models/temporal_fusion_transformer/__init__.py in forward(self, x)
    400         timesteps = x_cont.size(1)  # encode + decode length
    401         max_encoder_length = int(encoder_lengths.max())
--> 402         input_vectors = self.input_embeddings(x_cat)
    403         input_vectors.update(
    404             {

~/opt/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

~/opt/anaconda3/lib/python3.8/site-packages/pytorch_forecasting/models/nn/embeddings.py in forward(self, x)
     95                 )
     96             else:
---> 97                 input_vectors[name] = emb(x[..., self.x_categoricals.index(name)])
     98         return input_vectors

~/opt/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

~/opt/anaconda3/lib/python3.8/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    156 
    157     def forward(self, input: Tensor) -> Tensor:
--> 158         return F.embedding(
    159             input, self.weight, self.padding_idx, self.max_norm,
    160             self.norm_type, self.scale_grad_by_freq, self.sparse)

~/opt/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2042         # remove once script supports set_grad_enabled
   2043         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2044     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   2045 
   2046 

IndexError: index out of range in self

Something seems to be wrong with the val_dataloader size. Can you help?
Here is my code:

  
      

      GitHub
  

  
    

GitHub - Quirly/PyTorch_Forecasting_TFT_0 1

  Contribute to Quirly/PyTorch_Forecasting_TFT_0 development by creating an account on GitHub.


  

  
    
    
  

  


Thank you very much in advance!!