I found that it is also slow when I process in SSD…
This is the code to process

  
      github.com
  
  
    OpenNMT/OpenNMT-py/blob/master/preprocess.py 4
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import os
import glob
import sys

import torch

import onmt.io
import opts


def check_existing_pt_files(opt):
    # We will use glob.glob() to find sharded {train|valid}.[0-9]*.pt
    # when training, so check to avoid tampering with existing pt files
    # or mixing them up.
    for t in ['train', 'valid', 'vocab']:
        pattern = opt.save_data + '.' + t + '*.pt'


  This file has been truncated. show original