I have done more test for the memory and you are right, the tensor gets clear from the memory when there is no more reference. I am trying to figure out in the script that I have shared where references are kept. Using clone where I could helped reducing the footprint, but there is still a leakage. Here is the updated code.
import resource

import argparse
import gym
import numpy as np
from itertools import count
from collections import namedtuple
import os 

import torch
import random
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.autograd as autograd
from torch.autograd import Variable
import torchvision.transforms as T
import cv2
import pickle
import glob
import time
import subprocess
from collections import namedtuple

# Class
class ReplayMemory(object):
    '''
    A simple class to wrap around the concept of memory
    this helps for managing how much data is used. 
    '''
    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0
        
    def push(self, *args):
        """Saves a transition."""
        if len(self.memory) < self.capacity:
            self.memory.append(None) 
        self.memory[self.position] = Transition(*args)
        self.position = (self.position + 1) % self.capacity
        
    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)
    
    def __len__(self):
        return len(self.memory)

# Functions
def ProcessState(state,torchOutput=True):
    img = cv2.cvtColor(state, cv2.COLOR_BGR2GRAY)
    img = cv2.resize(img, (imageShape[1],imageShape[0])).astype('float32')
    if torchOutput:
        img = torch.from_numpy(img)
    img /= 255
    img -= 0.5 
    img *= 2
    return img

# Variables
Transition = namedtuple('Transition', ('state', 'action', 'next_state', 'reward', 'done'))

imageShape = (110,80)
env = gym.make('PongDeterministic-v3')
action = 0 
memory = []
reward = 0
done = False 
# Example with pytorch
for i_episode in range(25):
    # break
    print ('Pytorch: Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
    obser = env.reset()
    obser = ProcessState(obser).clone()

    state = torch.ones((3,imageShape[0],imageShape[1])).clone()    
    state = torch.cat((state.clone(),obser.view(1,imageShape[0],imageShape[1])),0).clone()

    for t in range(10000): 
        obser, reward, done, _ = env.step(0)

        #this is new observation getting process
        obser = ProcessState(obser).clone()

        state = torch.cat((state.clone(),obser.view(1,imageShape[0],imageShape[1])),0).clone()
        
        memory.append({'state':state[:-1].clone(), 'action': action, 'state1':state[1:].clone(), 
            'reward':reward, 'done':done})
        if len(memory) > 32:
            memory = memory[1:]

        state = state[1:].clone()

        if done:
            break
# quit()
# memory = ReplayMemory(32)
# Numpy
for i_episode in range(25):
    print ('Numpy: Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
    obser = env.reset()
    obser = ProcessState(obser,False)

    state = np.zeros((3,imageShape[0],imageShape[1]))
    state = np.concatenate([state, obser.reshape((1,imageShape[0],imageShape[1]))])

    for t in range(10000): 
        obser, reward, done, _ = env.step(0)

        #this is new observation getting process
        obser = ProcessState(obser,False)

        # state = torch.cat((state,obser.view(1,imageShape[0],imageShape[1])),0)
        state = np.concatenate([state, obser.reshape((1,imageShape[0],imageShape[1]))])
        
        memory.append({'state':state[:-1], 'action': action, 'state1':state[1:], 
            'reward':reward, 'done':done})
        if len(memory) > 32:
            memory = memory[1:]
        state = state[1:]

        if done:
            break

Here is my output:
(py3) jtremblay@office:~/code/Personal-git/dqn$ python memory_issue.py 
[2017-03-07 12:19:41,554] Making new env: PongDeterministic-v3
Pytorch: Memory usage: 115628 (kb)
Pytorch: Memory usage: 131180 (kb)
Pytorch: Memory usage: 132500 (kb)
Pytorch: Memory usage: 133556 (kb)
Pytorch: Memory usage: 135932 (kb)
Pytorch: Memory usage: 137252 (kb)
Pytorch: Memory usage: 137780 (kb)
Pytorch: Memory usage: 138308 (kb)
Pytorch: Memory usage: 139364 (kb)
Pytorch: Memory usage: 139892 (kb)
Pytorch: Memory usage: 140420 (kb)
Pytorch: Memory usage: 140684 (kb)
Pytorch: Memory usage: 141476 (kb)
Pytorch: Memory usage: 141476 (kb)
Pytorch: Memory usage: 142004 (kb)
Pytorch: Memory usage: 142268 (kb)
Pytorch: Memory usage: 143060 (kb)
Pytorch: Memory usage: 143588 (kb)
Pytorch: Memory usage: 143852 (kb)
Pytorch: Memory usage: 143852 (kb)
Pytorch: Memory usage: 144116 (kb)
Pytorch: Memory usage: 144380 (kb)
Pytorch: Memory usage: 144380 (kb)
Pytorch: Memory usage: 144644 (kb)
Pytorch: Memory usage: 144644 (kb)
Numpy: Memory usage: 144908 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)
Numpy: Memory usage: 154932 (kb)

I have also try to create a simple code that would replicate the behaviour:
import torch
import resource
import numpy as np

import argparse
parser = argparse.ArgumentParser(description='Memory issue')
parser.add_argument('--numpy',    action='store_true')
args = parser.parse_args()


a = [None for _ in range(10)]
# print (a)
j = 0

if args.numpy:
    state = np.ones((3,200,200))
    state = np.concatenate([state,np.random.rand(1,200,200)],0)
else:
    state = torch.ones((3,200,200))
    state = torch.cat((state,torch.rand(1,200,200)),0)

for i in range(5000):
    if i % 400 is 0:
        if args.numpy:
            print ('Numpy Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
        else:
            print ('Torch Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
    if j > 8:
        j = -1
    j+=1

    if args.numpy:
        state = np.concatenate([state,np.random.rand(1,200,200)],0)
    else:
        state = torch.cat((state,torch.rand(1,200,200)),0)
    a[j] = state[0:3]
    state = state[1:]

Here is the output:
(py3) jtremblay@office:~/code/Personal-git/dqn$ python memory_simple.py
Torch Memory usage: 82428 (kb)
Torch Memory usage: 95536 (kb)
Torch Memory usage: 96024 (kb)
Torch Memory usage: 96024 (kb)
Torch Memory usage: 96024 (kb)
Torch Memory usage: 96088 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96660 (kb)
Torch Memory usage: 96808 (kb)
Torch Memory usage: 96808 (kb)
Torch Memory usage: 96808 (kb)
Torch Memory usage: 96808 (kb)
Torch Memory usage: 96808 (kb)
Torch Memory usage: 96808 (kb)
(py3) jtremblay@office:~/code/Personal-git/dqn$ python memory_simple.py --numpy
Numpy Memory usage: 81132 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)
Numpy Memory usage: 96064 (kb)

Really I am not sure how this helps or not. As for the time taken storing numpy arrays and then translate them into torch tensor, I will run more experiments and report them here.