I have implemented Siamese network for my project as the following:
import torchvision.models as models
class Siamese(nn.Module):
    def __init__(self, num_output=1):
        super(Siamese, self).__init__()
        self.head_model = models.__dict__['resnet18'](num_classes=1024)
        self.tail_model = nn.Sequential(
            nn.Linear(1024, 1024),
            nn.BatchNorm1d(1024),
            nn.ReLU(),
            nn.Linear(1024, num_output)
        )
        for m in self.modules():
            if isinstance(m, nn.ReLU):
                m.inplace = False

    def forward(self, input1, input2):
        feat1 = self.head_model(input1)
        feat2 = self.head_model(input2)
        feat_diff = (feat1 - feat2) ** 2
        output = self.tail_model(feat_diff)
        return output

[resolved, see updates below] When I train it with single gpu, it crashes with the following output. (Similar to How to implement siamese network?)
*** Error in `python': free(): invalid pointer: 0x00007f6b83d25ac0 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7f6bbff5b7e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8037a)[0x7f6bbff6437a]
/lib/x86_64-linux-gnu/libc.so.6(cfree+0x4c)[0x7f6bbff6853c]
/usr/lib/x86_64-linux-gnu/libstdc++.so.6(_ZNSt15basic_stringbufIcSt11char_traitsIcESaIcEE8overflowEi+0x181)[0x7f6bbc523fa1]
/usr/lib/x86_64-linux-gnu/libstdc++.so.6(_ZNSt15basic_streambufIcSt11char_traitsIcEE6xsputnEPKcl+0x89)[0x7f6bbc57ae79]
/home/yuluo/.local/lib/python2.7/site-packages/torch/lib/libshm.so(_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l+0x1c5)[0x7f6b83a9a235]
/home/yuluo/.local/lib/python2.7/site-packages/torch/_C.so(+0x5d2742)[0x7f6b842fb742]
/home/yuluo/.local/lib/python2.7/site-packages/torch/_C.so(+0x5d33ae)[0x7f6b842fc3ae]
/home/yuluo/.local/lib/python2.7/site-packages/torch/_C.so(_ZN5torch2nn33SpatialConvolutionMM_updateOutputEPN4thpp6TensorES3_S3_S3_S3_S3_iiiiii+0xb3)[0x7f6b843100a3]
/home/yuluo/.local/lib/python2.7/site-packages/torch/_C.so(+0x5cae27)[0x7f6b842f3e27]
/home/yuluo/.local/lib/python2.7/site-packages/torch/_C.so(_ZN5torch8autograd11ConvForward5applyERKSt6vectorISt10shared_ptrINS0_8VariableEESaIS5_EE+0x17bf)[0x7f6b842f855f]
/home/yuluo/.local/lib/python2.7/site-packages/torch/_C.so(+0x5c181b)[0x7f6b842ea81b]
python(PyObject_Call+0x43)[0x4b0cb3]
python(PyEval_EvalFrameEx+0x5faf)[0x4c9faf]
python(PyEval_EvalCodeEx+0x255)[0x4c2765]
python(PyEval_EvalFrameEx+0x6099)[0x4ca099]
python(PyEval_EvalCodeEx+0x255)[0x4c2765]
python[0x4de8b8]
python(PyObject_Call+0x43)[0x4b0cb3]
python(PyEval_EvalFrameEx+0x2ad1)[0x4c6ad1]
python(PyEval_EvalCodeEx+0x255)[0x4c2765]
python[0x4de6fe]
python(PyObject_Call+0x43)[0x4b0cb3]
python[0x4f492e]
python(PyObject_Call+0x43)[0x4b0cb3]
python[0x553187]
python(PyObject_Call+0x43)[0x4b0cb3]
python(PyEval_EvalFrameEx+0x5faf)[0x4c9faf]
python(PyEval_EvalCodeEx+0x255)[0x4c2765]
python[0x4de8b8]
python(PyObject_Call+0x43)[0x4b0cb3]
python(PyEval_EvalFrameEx+0x2ad1)[0x4c6ad1]
python(PyEval_EvalCodeEx+0x255)[0x4c2765]
python[0x4de6fe]
python(PyObject_Call+0x43)[0x4b0cb3]
python[0x4f492e]
python(PyObject_Call+0x43)[0x4b0cb3]
python[0x553187]
python(PyObject_Call+0x43)[0x4b0cb3]
python(PyEval_EvalFrameEx+0x5faf)[0x4c9faf]
python(PyEval_EvalFrameEx+0x5d8f)[0x4c9d8f]
python(PyEval_EvalFrameEx+0x5d8f)[0x4c9d8f]
python(PyEval_EvalFrameEx+0x5d8f)[0x4c9d8f]
python(PyEval_EvalCodeEx+0x255)[0x4c2765]
python(PyEval_EvalCode+0x19)[0x4c2509]
python[0x4f1def]
python(PyRun_FileExFlags+0x82)[0x4ec652]
python(PyRun_SimpleFileExFlags+0x191)[0x4eae31]
python(Py_Main+0x68a)[0x49e14a]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7f6bbff04830]
python(_start+0x29)[0x49d9d9]
======= Memory map: ========
00400000-006ea000 r-xp 00000000 08:02 7472346                            /usr/bin/python2.7
008e9000-008eb000 r--p 002e9000 08:02 7472346                            /usr/bin/python2.7
008eb000-00962000 rw-p 002eb000 08:02 7472346                            /usr/bin/python2.7
00962000-00985000 rw-p 00000000 00:00 0 
00d77000-1fffe5000 rw-p 00000000 00:00 0                                 [heap]
200000000-200200000 rw-s 70e29b000 00:06 493                             /dev/nvidiactl
200200000-200400000 ---p 00000000 00:00 0 
200400000-200404000 rw-s 696146000 00:06 493                             /dev/nvidiactl
200404000-200600000 ---p 00000000 00:00 0 
200600000-200a00000 rw-s 122f97000 00:06 493                             /dev/nvidiactl
200a00000-201800000 ---p 00000000 00:00 0 
201800000-201804000 rw-s 5ac93a000 00:06 493                             /dev/nvidiactl
201804000-201a00000 ---p 00000000 00:00 0 
201a00000-201e00000 rw-s 14847b000 00:06 493                             /dev/nvidiactl
201e00000-202c00000 ---p 00000000 00:00 0 
202c00000-202c04000 rw-s 82be5e000 00:06 493                             /dev/nvidiactl
202c04000-202e00000 ---p 00000000 00:00 0 
202e00000-203200000 rw-s 5ca437000 00:06 493                             /dev/nvidiactl
203200000-204000000 ---p 00000000 00:00 0 
204000000-204004000 rw-s 559932000 00:06 493                             /dev/nvidiactl
204004000-204200000 ---p 00000000 00:00 0 
204200000-204600000 rw-s 711697000 00:06 493                             /dev/nvidiactl
204600000-205400000 ---p 00000000 00:00 0 
205400000-205404000 rw-s 5bc316000 00:06 493                             /dev/nvidiactl
205404000-205600000 ---p 00000000 00:00 0 
205600000-205a00000 rw-s 5d46b7000 00:06 493                             /dev/nvidiactl
205a00000-206800000 ---p 00000000 00:00 0 
206800000-206804000 rw-s 83998e000 00:06 493                             /dev/nvidiactl
206804000-206a00000 ---p 00000000 00:00 0 
206a00000-206e00000 rw-s 14af77000 00:06 493                             /dev/nvidiactl
206e00000-207c00000 ---p 00000000 00:00 0 
207c00000-207c04000 rw-s 6bb1ee000 00:06 493                             /dev/nvidiactl
207c04000-207e00000 ---p 00000000 00:00 0 
207e00000-208200000 rw-s 78418b000 00:06 493                             /dev/nvidiactl
208200000-209000000 ---p 00000000 00:00 0 
209000000-209004000 rw-s 10aeae000 00:06 493                             /dev/nvidiactl
209004000-209200000 ---p 00000000 00:00 0 
209200000-209600000 rw-s 152ec7000 00:06 493                             /dev/nvidiactl
209600000-20a400000 ---p 00000000 00:00 0 
20a400000-20a404000 rw-s 15ef63000 00:06 493                             /dev/nvidiactl
20a404000-20a600000 ---p 00000000 00:00 0 
20a600000-20aa00000 rw-s 15f0ec000 00:06 493                             /dev/nvidiactl
20aa00000-20aa04000 rw-s 11b073000 00:06 493                             /dev/nvidiactl
20aa04000-20ac00000 ---p 00000000 00:00 0 
20ac00000-20b000000 rw-s 6c7b00000 00:06 493                             /dev/nvidiactl
20b000000-20b004000 rw-s 130997000 00:06 493                             /dev/nvidiactl
20b004000-20b200000 ---p 00000000 00:00 0 
20b200000-20b600000 rw-s 7786ec000 00:06 493                             /dev/nvidiactl
20b600000-20b604000 rw-s 156043000 00:06 493                             /dev/nvidiactl
20b604000-20b800000 ---p 00000000 00:00 0 
20b800000-20bc00000 rw-s 6c4ef0000 00:06 493                             /dev/nvidiactl
20bc00000-20bc04000 rw-s 6595af000 00:06 493                             /dev/nvidiactl
20bc04000-20be00000 ---p 00000000 00:00 0 
20be00000-20c200000 rw-s 711444000 00:06 493                             /dev/nvidiactl
20c200000-20c204000 rw-s 775ed7000 00:06 493                             /dev/nvidiactl
20c204000-20c400000 ---p 00000000 00:00 0 
20c400000-20c800000 rw-s 521f0c000 00:06 493                             /dev/nvidiactl
20c800000-20c804000 rw-s 6c4f3f000 00:06 493                             /dev/nvidiactl
20c804000-20ca00000 ---p 00000000 00:00 0 
20ca00000-20ce00000 rw-s 7402cd000 00:06 493                             /dev/nvidiactl
20ce00000-20ce04000 rw-s 135948000 00:06 493                             /dev/nvidiactl
20ce04000-20d000000 ---p 00000000 00:00 0 
20d000000-20d400000 rw-s 7b1e05000 00:06 493                             /dev/nvidiactl
20d400000-400200000 ---p 00000000 00:00 0 
10000000000-10304200000 ---p 00000000 00:00 0 
10304200000-10304400000 rw-s 6f26a4000 00:06 493                         /dev/nvidiactl
10304400000-10304600000 rw-s 13530f000 00:06 493                         /dev/nvidiactl
10304600000-10304800000 rw-s 43c0b4000 00:06 493                         /dev/nvidiactl
10304800000-10304ad6000 rw-s 73e694000 00:06 493                         /dev/nvidiactl
10304ad6000-1030e200000 ---p 00000000 00:00 0 
1030e200000-1030e400000 rw-s 00000000 00:05 3431410                      /dev/zero (deleted)
1030e400000-10315a00000 ---p 00000000 00:00 0 
10315a00000-1031a380000 rw-s 00000000 00:05 3437954                      /dev/zero (deleted)
1031a380000-1031a400000 ---p 00000000 00:00 0 
1031a400000-1031ed80000 rw-s 00000000 00:05 3437955                      /dev/zero (deleted)
1031ed80000-1031ee00000 ---p 00000000 00:00 0 
1031ee00000-1031f000000 rw-s 00000000 00:05 3437956                      /dev/zero (deleted)
1031f000000-10323980000 rw-s 00000000 00:05 3437961                      /dev/zero (deleted)
10323980000-10323a00000 ---p 00000000 00:00 0 
10323a00000-10328380000 rw-s 00000000 00:05 3437962                      /dev/zero (deleted)
10328380000-10328400000 ---p 00000000 00:00 0 
10328400000-1032cd80000 rw-s 00000000 00:05 3437967                      /dev/zero (deleted)
1032cd80000-1032ce00000 ---p 00000000 00:00 0 
1032ce00000-10331780000 rw-s 00000000 00:05 3437968                      /dev/zero (deleted)
10331780000-10331800000 ---p 00000000 00:00 0 
10331800000-10336180000 rw-s 00000000 00:05 3427061                      /dev/zero (deleted)
10336180000-10336200000 ---p 00000000 00:00 0 
10336200000-1033ab80000 rw-s 00000000 00:05 3427062                      /dev/zero (deleted)
1033ab80000-1033ac00000 ---p 00000000 00:00 0 
1033ac00000-1033f580000 rw-s 00000000 00:05 3427067                      /dev/zero (deleted)
1033f580000-1033f600000 ---p 00000000 00:00 0 
1033f600000-10343f80000 rw-s 00000000 00:05 3427068                      /dev/zero (deleted)
10343f80000-10344000000 ---p 00000000 00:00 0 
7f68e4183000-7f6a2b902000 rw-p 00000000 00:00 0 
7f6af0000000-7f6af0021000 rw-p 00000000 00:00 0 
7f6af0021000-7f6af4000000 ---p 00000000 00:00 0 
7f6af6d00000-7f6afb680000 rw-s 00000000 00:16 147                        /dev/shm/torch_11743_4183124175 (deleted)
7f6afb680000-7f6b00000000 rw-s 00000000 00:16 112                        /dev/shm/torch_11743_869236225 (deleted)
7f6b00000000-7f6b00021000 rw-p 00000000 00:00 0 
7f6b00021000-7f6b04000000 ---p 00000000 00:00 0 
7f6b077ff000-7f6b07800000 ---p 00000000 00:00 0 
7f6b07800000-7f6b08000000 rw-p 00000000 00:00 0 
7f6b08000000-7f6b08021000 rw-p 00000000 00:00 0 
7f6b08021000-7f6b0c000000 ---p 00000000 00:00 0 
7f6b0c000000-7f6b0c021000 rw-p 00000000 00:00 0 
7f6b0c021000-7f6b10000000 ---p 00000000 00:00 0 
7f6b10000000-7f6b10021000 rw-p 00000000 00:00 0 
7f6b10021000-7f6b14000000 ---p 00000000 00:00 0 
7f6b140fb000-7f6b141fb000 rw-p 00000000 00:00 0 
7f6b141fb000-7f6b1a1fb000 ---p 00000000 00:00 0 
7f6b1a1fb000-7f6b26567000 rw-p 00000000 00:00 0 
7f6b265fd000-7f6b290fd000 rw-p 00000000 00:00 0 
7f6b290fd000-7f6b29101000 r-xp 00000000 08:02 7605181                    /usr/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so
7f6b29101000-7f6b29300000 ---p 00004000 08:02 7605181                    /usr/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so
7f6b29300000-7f6b29301000 r--p 00003000 08:02 7605181                    /usr/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so
7f6b29301000-7f6b29303000 rw-p 00004000 08:02 7605181                    /usr/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so
7f6b29303000-7f6b2930e000 r-xp 00000000 08:02 3015825                    /lib/x86_64-linux-gnu/libnss_files-2.23.so
7f6b2930e000-7f6b2950d000 ---p 0000b000 08:02 3015825                    /lib/x86_64-linux-gnu/libnss_files-2.23.so
7f6b2950d000-7f6b2950e000 r--p 0000a000 08:02 3015825                    /lib/x86_64-linux-gnu/libnss_files-2.23.so
7f6b2950e000-7f6b2950f000 rw-p 0000b000 08:02 3015825                    /lib/x86_64-linux-gnu/libnss_files-2.23.so
7f6b2950f000-7f6b2a455000 rw-p 00000000 00:00 0 
7f6b2a460000-7f6b2a560000 rw-p 00000000 00:00 0 
7f6b2a560000-7f6b2a5a2000 r-xp 00000000 08:02 7478123                    /usr/lib/nvidia-375/libnvidia-fatbinaryloader.so.375.66
7f6b2a5a2000-7f6b2a7a1000 ---p 00042000 08:02 7478123                    /usr/lib/nvidia-375/libnvidia-fatbinaryloader.so.375.66
7f6b2a7a1000-7f6b2a7ab000 rw-p 00041000 08:02 7478123                    /usr/lib/nvidia-375/libnvidia-fatbinaryloader.so.375.66
7f6b2a7ab000-7f6b2a7ac000 rw-p 00000000 00:00 0 
7f6b2a7ac000-7f6b2ae6e000 r-xp 00000000 08:02 7482960                    /usr/lib/x86_64-linux-gnu/libcuda.so.375.66
7f6b2ae6e000-7f6b2b06d000 ---p 006c2000 08:02 7482960                    /usr/lib/x86_64-linux-gnu/libcuda.so.375.66
7f6b2b06d000-7f6b2b188000 rw-p 006c1000 08:02 7482960                    /usr/lib/x86_64-linux-gnu/libcuda.so.375.66
7f6b2b188000-7f6b2b195000 rw-p 00000000 00:00 0 
7f6b2b195000-7f6b2b264000 r-xp 00000000 08:02 7480311                    /usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6
7f6b2b264000-7f6b2b464000 ---p 000cf000 08:02 7480311                    /usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6
7f6b2b464000-7f6b2b467000 r--p 000cf000 08:02 7480311                    /usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6
7f6b2b467000-7f6b2b469000 rw-p 000d2000 08:02 7480311                    /usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6
7f6b2b469000-7f6b2b46a000 rw-p 00000000 00:00 0 
7f6b2b46a000-7f6b2b47c000 r-xp 00000000 08:02 7605164                    /usr/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so
7f6b2b47c000-7f6b2b67b000 ---p 00012000 08:02 7605164                    /usr/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so
7f6b2b67b000-7f6b2b67c000 r--p 00011000 08:02 7605164                    /usr/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so
7f6b2b67c000-7f6b2b67e000 rw-p 00012000 08:02 7605164                    /usr/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so
7f6b2b67e000-7f6b4dd81000 rw-p 00000000 00:00 0 
7f6b4dd81000-7f6b4dd91000 r-xp 00000000 08:02 2102527                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5l.so
7f6b4dd91000-7f6b4df90000 ---p 00010000 08:02 2102527                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5l.so
7f6b4df90000-7f6b4df93000 rw-p 0000f000 08:02 2102527                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5l.so
7f6b4df93000-7f6b4df95000 rw-p 0004c000 08:02 2102527                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5l.so
7f6b4df95000-7f6b4dfa7000 r-xp 00000000 08:02 2102510                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5o.so
7f6b4dfa7000-7f6b4e1a6000 ---p 00012000 08:02 2102510                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5o.so
7f6b4e1a6000-7f6b4e1aa000 rw-p 00011000 08:02 2102510                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5o.so
7f6b4e1aa000-7f6b4e1ac000 rw-p 00057000 08:02 2102510                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5o.so
7f6b4e1ac000-7f6b4e1b0000 r-xp 00000000 08:02 2102506                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5fd.so
7f6b4e1b0000-7f6b4e3b0000 ---p 00004000 08:02 2102506                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5fd.so
7f6b4e3b0000-7f6b4e3b1000 rw-p 00004000 08:02 2102506                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5fd.so
7f6b4e3b1000-7f6b4e3b3000 rw-p 00014000 08:02 2102506                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5fd.so
7f6b4e3b3000-7f6b4e3be000 r-xp 00000000 08:02 2102514                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5i.so
7f6b4e3be000-7f6b4e5be000 ---p 0000b000 08:02 2102514                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5i.so
7f6b4e5be000-7f6b4e5c0000 rw-p 0000b000 08:02 2102514                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5i.so
7f6b4e5c0000-7f6b4e5c2000 rw-p 00036000 08:02 2102514                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5i.so
7f6b4e5c2000-7f6b4e5df000 r-xp 00000000 08:02 2102517                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5g.so
7f6b4e5df000-7f6b4e7de000 ---p 0001d000 08:02 2102517                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5g.so
7f6b4e7de000-7f6b4e7e1000 rw-p 0001c000 08:02 2102517                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5g.so
7f6b4e7e1000-7f6b4e7e2000 rw-p 00000000 00:00 0 
7f6b4e7e2000-7f6b4e7e4000 rw-p 00087000 08:02 2102517                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5g.so
7f6b4e7e4000-7f6b4e7fb000 r-xp 00000000 08:02 2102508                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5f.so
7f6b4e7fb000-7f6b4e9fa000 ---p 00017000 08:02 2102508                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5f.so
7f6b4e9fa000-7f6b4e9fe000 rw-p 00016000 08:02 2102508                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5f.so
7f6b4e9fe000-7f6b4ea00000 rw-p 00068000 08:02 2102508                    /home/yuluo/.local/lib/python2.7/site-packages/h5py/h5f.so
7f6b4ea00000-7f6b4ea40000 rw-p 00000000 00:00 0 
7f6b4ea80000-7f6b4eb80000 rw-p 00000000 00:00 0 
7f6b4eb80000-7f6b4eba9000 r-xp 00000000 08:02 2491420                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_hausdorff.so
7f6b4eba9000-7f6b4eda8000 ---p 00029000 08:02 2491420                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_hausdorff.so
7f6b4eda8000-7f6b4edab000 rw-p 00028000 08:02 2491420                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_hausdorff.so
7f6b4edab000-7f6b4edac000 rw-p 00000000 00:00 0 
7f6b4edac000-7f6b4edc2000 r-xp 00000000 08:02 2491411                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_distance_wrap.so
7f6b4edc2000-7f6b4efc2000 ---p 00016000 08:02 2491411                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_distance_wrap.so
7f6b4efc2000-7f6b4efc3000 rw-p 00016000 08:02 2491411                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_distance_wrap.so
7f6b4efc3000-7f6b4efec000 r-xp 00000000 08:02 2491417                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_voronoi.so
7f6b4efec000-7f6b4f1ec000 ---p 00029000 08:02 2491417                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_voronoi.so
7f6b4f1ec000-7f6b4f1ef000 rw-p 00029000 08:02 2491417                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/_voronoi.so
7f6b4f1ef000-7f6b4f4f0000 rw-p 00000000 00:00 0 
7f6b4f4f0000-7f6b4f5cf000 r-xp 00000000 08:02 2491408                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/qhull.so
7f6b4f5cf000-7f6b4f7ce000 ---p 000df000 08:02 2491408                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/qhull.so
7f6b4f7ce000-7f6b4f7d7000 rw-p 000de000 08:02 2491408                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/qhull.so
7f6b4f7d7000-7f6b4f7d9000 rw-p 00000000 00:00 0 
7f6b4f7d9000-7f6b4f7e2000 rw-p 003bd000 08:02 2491408                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/qhull.so
7f6b4f7e2000-7f6b4f879000 r-xp 00000000 08:02 2491415                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/ckdtree.so
7f6b4f879000-7f6b4fa79000 ---p 00097000 08:02 2491415                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/ckdtree.so
7f6b4fa79000-7f6b4fa81000 rw-p 00097000 08:02 2491415                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/spatial/ckdtree.so
7f6b4fa81000-7f6b4facf000 r-xp 00000000 08:02 2491814                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/interpnd.so
7f6b4facf000-7f6b4fcce000 ---p 0004e000 08:02 2491814                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/interpnd.so
7f6b4fcce000-7f6b4fcd3000 rw-p 0004d000 08:02 2491814                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/interpnd.so
7f6b4fcd3000-7f6b4fcd4000 rw-p 00000000 00:00 0 
7f6b4fcd4000-7f6b4fd24000 r-xp 00000000 08:02 2491821                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_ppoly.so
7f6b4fd24000-7f6b4ff23000 ---p 00050000 08:02 2491821                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_ppoly.so
7f6b4ff23000-7f6b4ff29000 rw-p 0004f000 08:02 2491821                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_ppoly.so
7f6b4ff29000-7f6b4ff2a000 rw-p 00000000 00:00 0 
7f6b4ff2a000-7f6b4ff2d000 rw-p 00181000 08:02 2491821                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_ppoly.so
7f6b4ff2d000-7f6b4ff69000 r-xp 00000000 08:02 2491817                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_bspl.so
7f6b4ff69000-7f6b50169000 ---p 0003c000 08:02 2491817                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_bspl.so
7f6b50169000-7f6b5016e000 rw-p 0003c000 08:02 2491817                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_bspl.so
7f6b5016e000-7f6b5016f000 rw-p 00000000 00:00 0 
7f6b5016f000-7f6b50171000 rw-p 00124000 08:02 2491817                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_bspl.so
7f6b50171000-7f6b501b1000 rw-p 00000000 00:00 0 
7f6b501b1000-7f6b50211000 r-xp 00000000 08:02 2491823                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/dfitpack.so
7f6b50211000-7f6b50411000 ---p 00060000 08:02 2491823                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/dfitpack.so
7f6b50411000-7f6b50418000 rw-p 00060000 08:02 2491823                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/dfitpack.so
7f6b50418000-7f6b5041a000 rw-p 000b1000 08:02 2491823                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/dfitpack.so
7f6b5041a000-7f6b5044f000 r-xp 00000000 08:02 2491809                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_fitpack.so
7f6b5044f000-7f6b5064f000 ---p 00035000 08:02 2491809                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_fitpack.so
7f6b5064f000-7f6b50650000 rw-p 00035000 08:02 2491809                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_fitpack.so
7f6b50650000-7f6b50652000 rw-p 0004e000 08:02 2491809                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/interpolate/_fitpack.so
7f6b50652000-7f6b5065c000 r-xp 00000000 08:02 2491256                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_nnls.so
7f6b5065c000-7f6b5085c000 ---p 0000a000 08:02 2491256                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_nnls.so
7f6b5085c000-7f6b5085d000 rw-p 0000a000 08:02 2491256                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_nnls.so
7f6b5085d000-7f6b5085f000 rw-p 0001c000 08:02 2491256                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_nnls.so
7f6b5085f000-7f6b50862000 r-xp 00000000 08:02 2491290                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_zeros.so
7f6b50862000-7f6b50a61000 ---p 00003000 08:02 2491290                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_zeros.so
7f6b50a61000-7f6b50a62000 rw-p 00002000 08:02 2491290                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_zeros.so
7f6b50a62000-7f6b50aa2000 rw-p 00000000 00:00 0 
7f6b50aa2000-7f6b50ac4000 r-xp 00000000 08:02 2491344                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lsq/givens_elimination.so
7f6b50ac4000-7f6b50cc4000 ---p 00022000 08:02 2491344                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lsq/givens_elimination.so
7f6b50cc4000-7f6b50cc7000 rw-p 00022000 08:02 2491344                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lsq/givens_elimination.so
7f6b50cc7000-7f6b50cf1000 r-xp 00000000 08:02 2491277                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_group_columns.so
7f6b50cf1000-7f6b50ef1000 ---p 0002a000 08:02 2491277                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_group_columns.so
7f6b50ef1000-7f6b50ef4000 rw-p 0002a000 08:02 2491277                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_group_columns.so
7f6b50ef4000-7f6b50f13000 r-xp 00000000 08:02 2491254                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_minpack.so
7f6b50f13000-7f6b51113000 ---p 0001f000 08:02 2491254                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_minpack.so
7f6b51113000-7f6b51114000 rw-p 0001f000 08:02 2491254                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_minpack.so
7f6b51114000-7f6b51116000 rw-p 00031000 08:02 2491254                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_minpack.so
7f6b51116000-7f6b5112d000 r-xp 00000000 08:02 2491275                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_slsqp.so
7f6b5112d000-7f6b5132d000 ---p 00017000 08:02 2491275                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_slsqp.so
7f6b5132d000-7f6b5132e000 rw-p 00017000 08:02 2491275                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_slsqp.so
7f6b5132e000-7f6b51330000 rw-p 0002c000 08:02 2491275                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_slsqp.so
7f6b51330000-7f6b5134e000 r-xp 00000000 08:02 2491253                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_cobyla.so
7f6b5134e000-7f6b5154e000 ---p 0001e000 08:02 2491253                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_cobyla.so
7f6b5154e000-7f6b5154f000 rw-p 0001e000 08:02 2491253                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_cobyla.so
7f6b5154f000-7f6b51551000 rw-p 00035000 08:02 2491253                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_cobyla.so
7f6b51551000-7f6b5155d000 r-xp 00000000 08:02 2491287                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/moduleTNC.so
7f6b5155d000-7f6b5175c000 ---p 0000c000 08:02 2491287                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/moduleTNC.so
7f6b5175c000-7f6b5175d000 rw-p 0000b000 08:02 2491287                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/moduleTNC.so
7f6b5175d000-7f6b5177b000 r-xp 00000000 08:02 2491248                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lbfgsb.so
7f6b5177b000-7f6b5197b000 ---p 0001e000 08:02 2491248                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lbfgsb.so
7f6b5197b000-7f6b5197c000 rw-p 0001e000 08:02 2491248                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lbfgsb.so
7f6b5197c000-7f6b5197f000 rw-p 00035000 08:02 2491248                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/_lbfgsb.so
7f6b5197f000-7f6b51988000 r-xp 00000000 08:02 2491259                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/minpack2.so
7f6b51988000-7f6b51b88000 ---p 00009000 08:02 2491259                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/minpack2.so
7f6b51b88000-7f6b51b89000 rw-p 00009000 08:02 2491259                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/minpack2.so
7f6b51b89000-7f6b51b8b000 rw-p 0001d000 08:02 2491259                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/optimize/minpack2.so
7f6b51b8b000-7f6b51bcb000 rw-p 00000000 00:00 0 
7f6b51bcb000-7f6b51c57000 r-xp 00000000 08:02 2492214                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/eigen/arpack/_arpack.so
7f6b51c57000-7f6b51e56000 ---p 0008c000 08:02 2492214                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/eigen/arpack/_arpack.so
7f6b51e56000-7f6b51e62000 rw-p 0008b000 08:02 2492214                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/eigen/arpack/_arpack.so
7f6b51e62000-7f6b51e63000 rw-p 00000000 00:00 0 
7f6b51e63000-7f6b51e67000 rw-p 000d3000 08:02 2492214                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/eigen/arpack/_arpack.so
7f6b51e67000-7f6b51eba000 r-xp 00000000 08:02 2492241                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/dsolve/_superlu.so
7f6b51eba000-7f6b520ba000 ---p 00053000 08:02 2492241                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/dsolve/_superlu.so
7f6b520ba000-7f6b520bc000 rw-p 00053000 08:02 2492241                    /home/yuluo/.local/lib/python2.7/site-packages/scipy/sparse/linalg/dsolve/_superlu.so


When I use DataParallel, it continuously allocate cpu memory until swap memory is full (and system crashes then). But there is no above invalid pointer error.
Things I have tried:

Memory leak is not from dataloader since when I comment all forward and backward, it behaves well.
The problem happens in forward stage since when I comment back propogagtion, it crashes.

Any ideas how to debug on this? It looks like a memory leak of dataloader for me. As for the single gpu, there seems to be some problems.
Thanks in advance.
[Update]

By import torch after scipy, the crash on single GPU is resolved. However, dataparellel still has memory leak.