I check nvidia-smi output during MNIST training but up to 25% (not fully allocated).
And I found following things,
1)MNIST learning with 2epochs works fine under bottleneck profiler.
2)MNIST learning with 3epochs does outputs following error under bottleneck profiler.
(I execute it two times, the trace of out of memory is same.)
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/bottleneck/__main__.py", line 149, in run_prof
    exec(code, globs, None)
  File "main.py", line 109, in <module>
    main()
  File "main.py", line 104, in main
    train(args, model, device, train_loader, optimizer, epoch)
  File "main.py", line 29, in train
    for batch_idx, (data, target) in enumerate(train_loader):
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 264, in __next__
    batch = self.collate_fn([self.dataset[i] for i in indices])
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 264, in <listcomp>
    batch = self.collate_fn([self.dataset[i] for i in indices])
  File "/opt/conda/lib/python3.6/site-packages/torchvision/datasets/mnist.py", line 77, in __getitem__
    img = self.transform(img)
  File "/opt/conda/lib/python3.6/site-packages/torchvision/transforms/transforms.py", line 49, in __call__
    img = t(img)
  File "/opt/conda/lib/python3.6/site-packages/torchvision/transforms/transforms.py", line 143, in __call__
    return F.normalize(tensor, self.mean, self.std)
  File "/opt/conda/lib/python3.6/site-packages/torchvision/transforms/functional.py", line 167, in normalize
    for t, m, s in zip(tensor, mean, std):
  File "/opt/conda/lib/python3.6/site-packages/torch/tensor.py", line 361, in <lambda>
    return iter(imap(lambda i: self[i], range(self.size(0))))
RuntimeError: /pytorch/torch/csrc/autograd/profiler.h:53: out of memory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/bottleneck/__main__.py", line 280, in <module>
    main()
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/bottleneck/__main__.py", line 261, in main
    autograd_prof_cpu, autograd_prof_cuda = run_autograd_prof(code, globs)
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/bottleneck/__main__.py", line 155, in run_autograd_prof
    result.append(run_prof(use_cuda=True))
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/bottleneck/__main__.py", line 149, in run_prof
    exec(code, globs, None)
  File "/opt/conda/lib/python3.6/site-packages/torch/autograd/profiler.py", line 191, in __exit__
    records = torch.autograd._disable_profiler()
RuntimeError: /pytorch/torch/csrc/autograd/profiler.h:53: out of memory