I used pytorch to train a cnn+ctc OCR model whose input images with increasing width size. Every time at 95-99% of first epoch, the system crashed with little information (Segmentation fault). I am sure the GPU and CPU memory were enough. I used gdb to debug, and infos show below. Can anyone has the same issue? I always think it’s the problem with torch.utils.data.DataLoader.  Wired things: If I reduce the size of training data from 3000000 to 50000 with changing size, it works well, only sometimes output
connectionrefusederror: [errno 111] connection refused.
newest: i just try my dataloader, it outputs the same issue!!! If i reduce the size of samples from 3000000 to 20000, it works well
my pytorch version is
python 3.6
torch           0.4.1
torchvision     0.2.1
warpctc-pytorch 0.1
my debug code is :
class resizeNormalize(object):

    def __init__(self, size, interpolation=Image.ANTIALIAS):
        self.size = size
        self.interpolation = interpolation
        self.toTensor = transforms.ToTensor()

    def __call__(self, img):
        img = img.resize(self.size, self.interpolation)
        img = self.toTensor(img)
        img.sub_(0.5).div_(0.5)
        return img

class resizePasteNormalize(object):

    def __init__(self, imgH=32, img_bg=None, is_trainning=True, interpolation=Image.ANTIALIAS):
        self.imgH = imgH
        self.img_bg = img_bg
        self.is_trainning = is_trainning
        self.interpolation = interpolation
        self.toTensor = transforms.ToTensor()

    def __call__(self, img):
        w, h = img.size
        img = img.resize((int(self.imgH * w / h), self.imgH), self.interpolation)
        if self.is_trainning:
            new_img_bg = self.img_bg.copy()
            new_img_bg.paste(img, (0, 0))
            img = new_img_bg
        img = self.toTensor(img)
        img.sub_(0.5).div_(0.5)
        return img

class alignCollate(object):

    def __init__(self, imgH=32, imgW=128, keep_ratio=False, min_ratio=1):
        self.imgH = imgH
        self.imgW = imgW
        self.keep_ratio = keep_ratio
        self.min_ratio = min_ratio

    def __call__(self, batch):
        images, labels = zip(*batch)

        imgH = self.imgH
        imgW = self.imgW
        if self.keep_ratio:
            ratios = []
            for image in images:
                w, h = image.size
                ratios.append(w / float(h))
            ratios.sort()
            num_ratios = len(ratios)
            choose_ratio = ratios[int(2 * num_ratios / 3)]
            imgW = int(np.floor(choose_ratio * imgH))
            imgW = max(imgH * self.min_ratio, imgW)  # ensure imgW >= imgH

        transform_1 = resizeNormalize((imgW, imgH))
        new_img = Image.new('RGB', (imgW, imgH), (0, 0, 0))
        transform_2 = resizePasteNormalize(imgH, new_img)

        ratios = []
        new_images = []
        for image in images:
            w = image.size[0]
            if w >= imgW:
                ratios.append(1.0)
                new_images += [transform_1(image)]
            else:
                ratios.append(w / imgW)
                new_images += [transform_2(image)]

        images = torch.cat([t.unsqueeze(0) for t in new_images], 0)
        return ratios, images, labels


class randomSubsetSampler(sampler.Sampler):

    def __init__(self, data_source):
        self.word_len_list = data_source.word_len_list
        self.num_samples = len(data_source)

    def __iter__(self):
        np_idx = []
        prev_count = 0
        for word_len in self.word_len_list:
            subidx = list(range(word_len))
            random.shuffle(subidx)
            np_idx += [i + prev_count for i in subidx]
            prev_count += word_len
        index = torch.IntTensor(np_idx)

        return iter(index)

    def __len__(self):
        return self.num_samples

class seqDataset(Dataset):

    def __init__(self, root=None, sort_seq=False, min_chars=1, max_chars=60, transform=None, target_transform=None):

        if not os.path.exists(root):
            raise Exception("root not exist")
        table_list = os.listdir(root)

        self.img_path = []  # len(self.img_path)=0
        self.text = []
        self.word_len_list = []

        for table_name in table_list:
            if 'txt' not in table_name:
                continue
            table_path = os.path.join(root, table_name)
            with open(table_path, 'r', encoding='utf8') as f:
                lines = list(f)  # len(lines)=300
                for line in lines:
                    img_path, text = line.split(' ', 1)
                    img_path = img_path.strip()
                    text = text.strip()
                    if len(text) > max_chars or len(text) < min_chars:
                        continue
                    self.img_path.append(img_path)
                    self.text.append(text)

        if sort_seq == True:
            combine = zip(self.img_path, self.text)
            self.img_path, self.text = zip(*(sorted(combine, key=lambda x: len(x[1]))))

        assert len(self.img_path) == len(self.text)

        self.nSamples = len(self.text)

        len_dict = {}
        for text in self.text:
            len_dict[len(text)] = len_dict.get(len(text), 0) + 1
        self.word_len_list = list(len_dict.values())

        assert self.nSamples == sum(self.word_len_list)

        self.transform = transform
        self.target_transform = target_transform

    def __len__(self):
        return self.nSamples

    def __getitem__(self, index):
        assert index <= len(self), 'index range error'
        try:
            img = Image.open(self.img_path[index]).convert('RGB')
        except IOError:
            print('Corrupted image for' + self.img_path[index])
            return self[index + 1]

        if self.transform is not None:
            img = self.transform(img)

        label = self.text[index]

        if self.target_transform is not None:
            label = self.target_transform(label)

        return (img, label)


    shuffle = False
    sampler = exp_dataset.randomSubsetSampler(train_dataset)

if __name__ == '__main__':
    root = './tables'
    reader = seqDataset(root, sort_seq=True)
    train_loader = torch.utils.data.DataLoader(
        reader, batch_size=128,
        shuffle=False, sampler=randomSubsetSampler(reader),
        num_workers=16, drop_last=True,
        collate_fn=alignCollate(imgH=32, keep_ratio=True))
    print('dataset size:', len(reader))

    train_iter = iter(train_loader)
    i = 0
    print(len(train_loader))
    while i < len(train_loader):
        print(i)
        ratios, image, label = train_iter.next()
        print(ratios)
        i += 1

    train_iter = iter(train_loader)
    i = 0
    print(len(train_loader))
    while i < len(train_loader):
        print(i)
        ratios, image, label = train_iter.next()
        i += 1


Output
    c = Client(address, authkey=process.current_process().authkey)
  File "/usr/local/python3/lib/python3.6/multiprocessing/connection.py", line 379, in _recv
    chunk = read(handle, remaining)
  File "/usr/local/python3/lib/python3.6/multiprocessing/connection.py", line 487, in Client
    c = SocketClient(address)
  File "/usr/local/python3/lib/python3.6/multiprocessing/connection.py", line 614, in SocketClient
    s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
  File "/usr/local/python3/lib/python3.6/multiprocessing/connection.py", line 614, in SocketClient
    s.connect(address)
ConnectionResetError: [Errno 104] Connection reset by peer
ConnectionRefusedError: [Errno 111] Connection refused
ConnectionRefusedError: [Errno 111] Connection refused
Segmentation fault

and gdb debug infos:
[0/1000][93154/93750] Train loss: 1.980 , accuray: 0.723 , cost time: 0.218
[0/1000][93248/93750] Train loss: 1.930 , accuray: 0.733 , cost time: 0.229
[0/1000][93342/93750] Train loss: 2.021 , accuray: 0.740 , cost time: 0.219
[0/1000][93436/93750] Train loss: 1.804 , accuray: 0.737 , cost time: 0.223
[0/1000][93530/93750] Train loss: 1.948 , accuray: 0.746 , cost time: 0.234
[0/1000][93624/93750] Train loss: 2.106 , accuray: 0.714 , cost time: 0.235
[0/1000][93718/93750] Train loss: 2.136 , accuray: 0.738 , cost time: 0.237

Program received signal SIGSEGV, Segmentation fault.
lookdict_unicode (mp=0x7ffff6384240, key=0x7ffff31725e0,
    hash=-6490995071133369473, value_addr=0x7fffff7ff060, hashpos=0x0)
    at Objects/dictobject.c:804
804	{
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-13.el7.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.13.2-12.el7_2.x86_64 libcom_err-1.42.9-7.el7.x86_64 libgcc-4.8.5-4.el7.x86_64 libselinux-2.2.2-6.el7.x86_64 libstdc++-4.8.5-4.el7.x86_64 libuuid-2.23.2-26.el7_2.3.x86_64 openssl-libs-1.0.1e-51.el7_2.7.x86_64 pcre-8.32-15.el7_2.1.x86_64 xz-libs-5.1.2-12alpha.el7.x86_64 zlib-1.2.7-15.el7.x86_64

#0  lookdict_unicode (mp=0x7ffff6384240, key=0x7ffff31725e0,
    hash=-6490995071133369473, value_addr=0x7fffff7ff060, hashpos=0x0)
    at Objects/dictobject.c:804
#1  0x000000000049be26 in PyDict_GetItem (op=op@entry=0x7ffff6384240,
    key=key@entry=0x7ffff31725e0) at Objects/dictobject.c:1439
#2  0x00000000004ad303 in _PyObject_GenericGetAttrWithDict (
    dict=0x7ffff6384240, name=0x7ffff31725e0,
    name@entry=0x88bf00 <PyModule_Type>, obj=0x7ffff63859f8, obj@entry=0x0)
    at Objects/object.c:1089
#3  PyObject_GenericGetAttr (obj=obj@entry=0x7ffff63859f8,
    name=name@entry=0x7ffff31725e0) at Objects/object.c:1121
#4  0x00000000004a96bf in module_getattro (m=0x7ffff63859f8,
    name=0x7ffff31725e0) at Objects/moduleobject.c:663
#5  0x0000000000544f37 in _PyEval_EvalFrameDefault (f=<optimized out>,
    throwflag=<optimized out>) at Python/ceval.c:2832
#6  0x000000000053eef1 in PyEval_EvalFrameEx (throwflag=0, f=0x7fff03883708)
    at Python/ceval.c:718
#7  _PyFunction_FastCall (co=co@entry=0x7fffe581bdb0, args=<optimized out>,
    args@entry=0x7fffff7ff3c0, nargs=nargs@entry=1,
    globals=globals@entry=0x7fffe5820bd0) at Python/ceval.c:4891
#8  0x0000000000548c96 in _PyFunction_FastCallDict (
    func=func@entry=0x7fffe5822c80, args=args@entry=0x7fffff7ff3c0,
    nargs=nargs@entry=1, kwargs=kwargs@entry=0x0) at Python/ceval.c:4993
#9  0x000000000045135f in _PyObject_FastCallDict (func=0x7fffe5822c80,
   args=0x7fffff7ff3c0, nargs=1, kwargs=0x0) at Objects/abstract.c:2295
#10 0x000000000045145b in _PyObject_Call_Prepend (func=0x7fffe5822c80,
    obj=0x7ffe95b3e3c8, args=0x7ffff7fa5048, kwargs=0x0)
    at Objects/abstract.c:2358
#11 0x0000000000451212 in _PyObject_FastCallDict (func=0x7fffd90dcfc8,
    args=0x0, nargs=nargs@entry=0, kwargs=kwargs@entry=0x0)
    at Objects/abstract.c:2316
#12 0x0000000000540e6e in PyEval_CallObjectWithKeywords (
    func=func@entry=0x7fffd90dcfc8, args=args@entry=0x0,
    kwargs=kwargs@entry=0x0) at Python/ceval.c:4726
#13 0x00000000004c71e1 in slot_tp_finalize (self=0x7ffe95b3e3c8)
    at Objects/typeobject.c:6450
#14 0x00000000004ab3a1 in PyObject_CallFinalizer (self=0x7ffe95b3e3c8)
    at Objects/object.c:297
#15 PyObject_CallFinalizerFromDealloc (self=self@entry=0x7ffe95b3e3c8)
    at Objects/object.c:314
#16 0x00000000004bae4a in subtype_dealloc (self=0x7ffe95b3e3c8)
    at Objects/typeobject.c:1151
#17 0x00007fff73284f62 in operator= (new_ptr=0x0, this=0x7ffed4a2b0d8)
    at /pytorch/torch/csrc/utils/object_ptr.h:17
#18 torch::PyObjectFinalizer::~PyObjectFinalizer (this=0x7ffed4a2b0d0,
    __in_chrg=<optimized out>) at /pytorch/torch/csrc/finalizer.h:26
#19 0x00007fff73285084 in ~PyObjectFinalizer (this=0x7ffed4a2b0d0,
    __in_chrg=<optimized out>) at /pytorch/torch/csrc/finalizer.h:27
#20 operator() (this=<optimized out>, __ptr=0x7ffed4a2b0d0)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:76
#21 ~unique_ptr (this=0x7ffed4a2b100, __in_chrg=<optimized out>)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:236
#22 ~PyObjectFinalizer (this=0x7ffed4a2b0f0, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:22
#23 ~PyObjectFinalizer (this=0x7ffed4a2b0f0, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:27
#24 operator() (this=<optimized out>, __ptr=0x7ffed4a2b0f0)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:76
#25 ~unique_ptr (this=0x7ffed4a2b100, __in_chrg=<optimized out>)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:236
#26 ~PyObjectFinalizer (this=0x7ffed4a2b110, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:22
#27 ~PyObjectFinalizer (this=0x7ffed4a2b110, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:27
#28 operator() (this=<optimized out>, __ptr=0x7ffed4a2b110)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:76
#29 ~unique_ptr (this=0x7ffed4a2b100, __in_chrg=<optimized out>)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:236
#30 ~PyObjectFinalizer (this=0x7ffed4a2b130, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:22
#31 ~PyObjectFinalizer (this=0x7ffed4a2b130, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:27
#32 operator() (this=<optimized out>, __ptr=0x7ffed4a2b130)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:76
#33 ~unique_ptr (this=0x7ffed4a2b100, __in_chrg=<optimized out>)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:236
#34 torch::PyObjectFinalizer::~PyObjectFinalizer (this=0x7ffed4a2b150,
    __in_chrg=<optimized out>) at /pytorch/torch/csrc/finalizer.h:22
#35 0x00007fff73285084 in ~PyObjectFinalizer (this=0x7ffed4a2b150,
    __in_chrg=<optimized out>) at /pytorch/torch/csrc/finalizer.h:27
#36 operator() (this=<optimized out>, __ptr=0x7ffed4a2b150)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:76
#37 ~unique_ptr (this=0x7ffed4a2b180, __in_chrg=<optimized out>)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:236
#38 ~PyObjectFinalizer (this=0x7ffed4a2b170, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:22
#39 ~PyObjectFinalizer (this=0x7ffed4a2b170, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:27
#40 operator() (this=<optimized out>, __ptr=0x7ffed4a2b170)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:76
#41 ~unique_ptr (this=0x7ffed4a2b180, __in_chrg=<optimized out>)
    at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unique_ptr.h:236
#42 ~PyObjectFinalizer (this=0x7ffed4a2b190, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:22
#43 ~PyObjectFinalizer (this=0x7ffed4a2b190, __in_chrg=<optimized out>)
    at /pytorch/torch/csrc/finalizer.h:27