I tried the nightly b. (https://download.pytorch.org/whl/nightly/cu100/torch_nightly-1.2.0.dev20190616-cp37-cp37m-linux_x86_64.whl)

prng state issue is fixed: within the same run, the prng state does not change when the seed is fixed.
Now, it is possible to fix the same prng state over different runs. (by using a fixed seed, or a stored prng state on disc, for instance).

Code:
import copy
import os


import torch
import numpy as np


seed = 1
# reset the seed of Numpy to control its initial rng state.
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)

use_cuda = True

state_cpu = torch.get_rng_state()
state_cuda = torch.cuda.get_rng_state(device=0)

l_cuda = (torch.tensor(state_cuda).type(torch.uint8))
l_cpu = (torch.tensor(state_cpu).type(torch.uint8))

LOAD = True
filecuda = "prng/cuda.npy"
filecpu = "prng/cpu.npy"
if LOAD:
    if not os.path.exists(filecuda) and not os.path.exists(filecpu):
        l_cuda = (torch.cuda.get_rng_state(device=0))
        torch.save(l_cuda, filecuda)
        l_cpu = (torch.get_rng_state())
        torch.save(l_cpu, filecpu)
    else:
        arrcuda = torch.load(filecuda)
        l_cuda = (torch.tensor(arrcuda).type(torch.uint8))
        arrcpu = torch.load(filecpu)
        l_cpu = (torch.tensor(arrcpu).type(torch.uint8))

for i in range(10):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    # force the cuda rng state to be equal to `state`.
    torch.cuda.set_rng_state(l_cuda, device=0)
    torch.set_rng_state(l_cpu)
    lastlcuda = copy.deepcopy(l_cuda)
    lastlcpu = copy.deepcopy(l_cpu)
    l_cuda = (torch.cuda.get_rng_state(device=0))
    st = bool((l_cuda == lastlcuda).all())
    print("Iter. {}: current (rngs.sum(){}) == previous (rngs.sum(){})? {} CUDA".format(
        i, l_cuda.sum(), lastlcuda.sum(), st))
    st = bool((l_cpu == lastlcpu).all())
    print("Iter. {}: current (rngs.sum(){}) == previous (rngs.sum(){})? {} CPU".format(
        i, l_cpu.sum(), lastlcpu.sum(), st))
    a = torch.rand(1000, device="cuda:0")  # torch.bernoulli(torch.full((300, 300), 0.5, device='cuda:0'))
    print(a.sum())

run 1: (store rng state on disc)
Iter. 0: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 0: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 1: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 1: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 2: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 2: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 3: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 3: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 4: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 4: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 5: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 5: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 6: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 6: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 7: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 7: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 8: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 8: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 9: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 9: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')

run 2: use the stored prng state. (same results when using the same seed without storing anything.)
Iter. 0: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? **True** CUDA
Iter. 0: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 1: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 1: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 2: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 2: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 3: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 3: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 4: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 4: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 5: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 5: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 6: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 6: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 7: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 7: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 8: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 8: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')
Iter. 9: current (rngs.sum()210120001) == previous (rngs.sum()210120001)? True CUDA
Iter. 9: current (rngs.sum()316421) == previous (rngs.sum()316421)? True CPU
tensor(488.7892, device='cuda:0')

Thanks!