Hello folks.
I create an dqn implement according the tutorial reinforcement_q_learning, with the following changes.

Use gym observation as state
Use an MLP instead of the DQN class in the tutorial

The model diverged if loss = F.smooth_l1_loss{ loss_fn = nn.SmoothL1Loss()} ,
If loss_fn = nn.MSELoss(), the model seems to work (much slower than the tutorial)
What did I do wrong? Any ideas?
import math
import random
import numpy as np
from collections import namedtuple
import matplotlib.pyplot as plt
from itertools import count
from time import sleep
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
import gym

use_cuda = False
FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor
ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor
Tensor = FloatTensor

env = gym.make('CartPole-v0')
 
Transition = namedtuple('Transition', ('state', 'action', 'next_state', 'reward'))

class ReplayMemory(object):

    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, *args):
        if len(self.memory) < self.capacity:
            self.memory.append(None)
        self.memory[self.position] = Transition(*args)
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

BATCH_SIZE = 128
GAMMA = 0.999
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 200

model = nn.Sequential(
          nn.Linear(4,1024), 
          nn.ReLU(), 
          nn.Linear(1024,1024), 
          nn.ReLU(),
          nn.Linear(1024,2)
        )

optimizer = optim.Adam(model.parameters(),  lr=1e-4)
loss_fn = nn.MSELoss()
# loss_fn = nn.SmoothL1Loss()
memory = ReplayMemory(10000)

steps_done = 0

def select_action(state):
    global steps_done
    sample = random.random()
    eps_threshold = EPS_END + (EPS_START - EPS_END) * math.exp(-1. * steps_done / EPS_DECAY)
    steps_done += 1
    if sample > eps_threshold:
        output = model(Variable(state.view(-1, 4), volatile=True))
        output = output.data
        _, index = output.max(1)
        print(f'State By Model {output}')
        print(f'State By Model {index[0]}')
        return index.view(1, 1)
    else:
        index = random.randrange(2)
        # print(f'State By Random {index}')
        return LongTensor([[index]])


episode_durations = []

def plot_durations():
    plt.figure(2)
    plt.clf()
    durations_t = torch.FloatTensor(episode_durations)
    plt.title('Training...')
    plt.xlabel('Episode')
    plt.ylabel('Duration')
    plt.plot(durations_t.numpy())
    # Take 100 episode averages and plot them too
    if len(durations_t) >= 100:
        means = durations_t.unfold(0, 100, 1).mean(1).view(-1)
        means = torch.cat((torch.zeros(99), means))
        plt.plot(means.numpy())

    plt.pause(0.1)  # pause a bit so that plots are updated

last_sync = 0

def optimize_model():
    global last_sync
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for
    # detailed explanation).
    batch = Transition(*zip(*transitions))

    state_batch = [s.view(1,-1) for s in batch.state]
    state_batch = torch.cat(state_batch, 0)
    state_batch = Variable(state_batch)

    action_batch = torch.cat(batch.action)
    action_batch = Variable(action_batch) 
    
    reward_batch = torch.cat(batch.reward)
    reward_batch = Variable(reward_batch)

    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
    # columns of actions taken
    model_outputs = model(state_batch)
    state_action_values = model_outputs.gather(1, action_batch)


    non_final_mask = map(lambda s: s is not None, batch.next_state)
    non_final_mask = tuple(non_final_mask)
    # Compute a mask of non-final states and concatenate the batch elements
    non_final_mask = ByteTensor(non_final_mask)

    # We don't want to backprop through the expected action values and volatile
    # will save us on temporarily changing the model parameters'
    # requires_grad to False!
    non_final_next_states = [s.view(1, -1) for s in batch.next_state if s is not None]
    non_final_next_states = torch.cat(non_final_next_states, 0)
    non_final_next_states = Variable(non_final_next_states, volatile=True)
    # Compute V(s_{t+1}) for all next states.
    next_state_values = Variable(torch.zeros(BATCH_SIZE).type(Tensor))
    next_model_outputs = model(non_final_next_states)
    max_next_state_values, _ = next_model_outputs.max(1)
    next_state_values[non_final_mask] = max_next_state_values
    # Now, we don't want to mess up the loss with a volatile flag, so let's
    # clear it. After this, we'll just end up with a Variable that has
    # requires_grad=False
    next_state_values.volatile = False
    # Compute the expected Q values
    expected_state_action_values = (next_state_values * GAMMA) + reward_batch

    # Compute Huber loss
    loss = loss_fn(state_action_values, expected_state_action_values)

    print(f'Loss={loss.data[0]}');
    # Optimize the model
    optimizer.zero_grad()
    loss.backward()
    # for param in model.parameters(): param.grad.data.clamp_(-1, 1)
    optimizer.step()

num_episodes = 1000
for i_episode in range(num_episodes):
    # Initialize the environment and state
    observation = env.reset()
    state = FloatTensor(observation)  
    for t in count():
        env.render()
#        print(f'State:{state.tolist()}')
        action = select_action(state)
        step_action = action[0, 0]
        observation, reward, done, info = env.step(step_action)
#        print(f'observation={observation}')
#        print(f'reward={reward}')
#        print(f'done={done}') 
        reward = Tensor([reward])
        
        # Observe new state
        next_state = FloatTensor(observation) if not done else None

        # Store the transition in memory
        assert state is not None
        memory.push(state, action, next_state, reward) 

        # if next_state is not None : print(f'NextState:{next_state.tolist()}')

        # Move to the next state
        state = next_state
        
        # Perform one step of the optimization (on the target network)
        optimize_model()
        if done:
            episode_durations.append(t + 1)
            plot_durations()
            break