The parameter update step is defined here (a vanilla gradient descent step):

  
      github.com
  
  
    pytorch/examples/blob/master/word_language_model/main.py#L141 11

model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(eval_batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
    data, targets = get_batch(data_source, i, evaluation=True)
    output, hidden = model(data, hidden)
    output_flat = output.view(-1, ntokens)
    total_loss += len(data) * criterion(output_flat, targets).data
    hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)




def train():
# Turn on training mode which enables dropout.
model.train()
total_loss = 0
start_time = time.time()
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(args.batch_size)
for batch, i in enumerate(range(0, train_data.size(0) - 1, args.bptt)):



  
  
    
    
  
  


The learning rate is made to decay in the train loop, defined here:

  
      github.com
  
  
    pytorch/examples/blob/master/word_language_model/main.py#L177



    if batch % args.log_interval == 0 and batch > 0:
        cur_loss = total_loss[0] / args.log_interval
        elapsed = time.time() - start_time
        print('| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | '
                'loss {:5.2f} | ppl {:8.2f}'.format(
            epoch, batch, len(train_data) // args.bptt, lr,
            elapsed * 1000 / args.log_interval, cur_loss, math.exp(cur_loss)))
        total_loss = 0
        start_time = time.time()


# Loop over epochs.
lr = args.lr
best_val_loss = None


# At any point you can hit Ctrl + C to break out of training early.
try:
for epoch in range(1, args.epochs+1):
    epoch_start_time = time.time()
    train()
    val_loss = evaluate(val_data)