Hi, it’s an own implementation:
Here is the code:
lass DataLoader(object):
def init(self, vocab, fname):
self.src_data = []
self.tgt_data = []
self.scores =[]
self.vocab = vocab
with codecs.open(fname,‘r’,‘utf-8’) as f:
for line in f.readlines():
info  = line.strip().split(’\t’)
assert len(info) == 6, line
src = info[0].split()
tgt = info[1].split()
scores = [ float(x) for x in info[2:]]
self.src_data.append(src)
self.tgt_data.append(tgt)
self.scores.append(scores)
def get_batches(self, batch_size, shuffle = True):
    idx = list(range(len(self.src_data)))
    if shuffle:
        np.random.shuffle(idx)
    cur_size = 0
    input, target, score = [], [], []
    for _id in sorted(idx, key = lambda x: len(self.src_data[x])):
        cur_size += len(self.src_data[_id])
        input.append(self.src_data[_id])
        target.append(self.tgt_data[_id])
        score.append(self.scores[_id])
        if cur_size  >= batch_size:
            cur_size  = 0
            seq_len = max(len(t) for t in input)
            input = [ self.vocab.src2id(t)+ [0]*(seq_len - len(t)) for t in input ]
            seq_len = max(len(t) for t in target)
            target = [ self.vocab.tgt2id(t) + [0]*(seq_len - len(t)) for t in target ]
            yield input, target, score