#!/usr/bin/env python
# coding: utf-8

# In[1]:


get_ipython().run_line_magic('matplotlib', 'inline')


# In[2]:


from __future__ import unicode_literals, print_function, division
from io import open
import unicodedata
import string
import re
import random

import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")


# In[3]:


SOS_token = 0
EOS_token = 1


class Lang:
    def __init__(self, name):
        self.name = name
        self.word2index = {}
        self.word2count = {}
        self.index2word = {0: "SOS", 1: "EOS"}
        self.n_words = 2  # Count SOS and EOS

    def addSentence(self, sentence):
        for word in sentence.split(' '):
            self.addWord(word)

    def addWord(self, word):
        
        if word not in self.word2index:
#             print(word)
            self.word2index[word] = self.n_words
            self.word2count[word] = 1
            self.index2word[self.n_words] = word
            self.n_words += 1
        else:
            self.word2count[word] += 1


# In[4]:


# Turn a Unicode string to plain ASCII, thanks to
# http://stackoverflow.com/a/518232/2809427
def unicodeToAscii(s):
    return ''.join(
        c for c in unicodedata.normalize('NFD', s)
        if unicodedata.category(c) != 'Mn'
    )

# Lowercase, trim, and remove non-letter characters


def normalizeString(s):
    s = unicodeToAscii(s.lower().strip())
    s = re.sub(r"([.!?])", r" \1", s)
    s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
    return s


# In[5]:


def reading_bigfiles(lang1, lang2, reverse=False):
    print("Reading lines...")
    
    input_lines = open("../data/english_rough_changed_updated", encoding='utf-8').read().strip().split('\n')
    output_lines = open("../data/french_rough_changed_updated", encoding='utf-8').read().strip().split('\n')
    print(len(input_lines),len(output_lines))
    lines = []
    for i,j in zip(input_lines,output_lines):
        lines.append(i+"\t"+j)
    print(random.choice(lines))
    # Read the file and split into lines
#     lines = open('data/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\
#         read().strip().split('\n')

#     # Split every line into pairs and normalize
    pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
    print(len(pairs),"pairs")
    
    # Reverse pairs, make Lang instances
    if reverse:
        pairs = [list(reversed(p)) for p in pairs]
        input_lang = Lang(lang2)
        output_lang = Lang(lang1)
    else:
        input_lang = Lang(lang1)
        output_lang = Lang(lang2)

    return input_lang, output_lang, pairs


# In[6]:


# def readLangs(lang1, lang2, reverse=False):
#     print("Reading lines...")

#     # Read the file and split into lines
#     lines = open('../sorted_entofrdata/combined_data', encoding='utf-8').\
#         read().strip().split('\n')
#     print(lines[10],"the lines value")
#     # Split every line into pairs and normalize
#     pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
#     print(len(pairs),"pairs")
    
#     # Reverse pairs, make Lang instances
#     if reverse:
#         pairs = [list(reversed(p)) for p in pairs]
#         input_lang = Lang(lang2)
#         output_lang = Lang(lang1)
#     else:
#         input_lang = Lang(lang1)
#         output_lang = Lang(lang2)

#     return input_lang, output_lang, pairs


# In[7]:


# dummy = "B4-0012/98 - O-0009/98 et B4-0013/98 - O-0010/98 des députés Maij-Weggen, Moorhouse et Oomen-Ruijten, au nom du groupe du parti populaire européen, au Conseil et à la Commission, sur l'attitude de l'Union européenne à l'égard du Nigéria; -B4-0020/98 - O-0028/98 et B4-0122/98 - O-0029/98 de M. Bertens, au nom du groupe du parti européen des libéraux démocrates et réformateurs, au Conseil et à la Commission, sur l'attitude de l'Union européenne à l'égard du Nigéria; -B4-0123/98 - O-0030/98 et B4-0124/98 - O-0031/98 des députés Müller, Aelvoet, Telkämper et McKenna, au nom du groupe des verts au Parlement européen, au Conseil et à la Commission, sur l'attitude de l'Union européenne à l'égard du Nigéria; -B4-0127/98 - O-0035/98 des députés Carnero González et Pettinari, au nom du groupe confédéral de la gauche unitaire européenne - gauche verte nordique, au Conseil, sur les relations entre l'Union européenne et le Nigéria; -B4-0128/98 - O-0036/98 et B4-0129/98 - O-0037/98 des députés Macartney et Hory, au nom du groupe de l'alliance radicale européenne, au Conseil et à la Commission, sur l'attitude de l'Union européenne vis-à-vis du Nigéria; -B4-0134/98 - O-0042/98 et B4-0135/98 - O-0043/98 des députés Andrews et Girão Pereira, au nom du groupe Union pour l'Europe, au Conseil et à la Commission, sur l'attitude de l'Union européenne vis-à-vis du Nigéria; -B4-0137/98 - O-0046/98 et B4-0169/98 - O-0051/98 des députés Kinnock et Vecchi, au nom du groupe du parti des socialistes européens, au Conseil et à la Commission, sur la situation au Nigéria; -B4-0014/98 - O-0011/98 et B4-0015/98 - O-0012/98 des députés Maij-Weggen et Oomen-Ruijten, au nom du groupe du parti populaire européen, au Conseil et à la Commission, sur les violations des droits de l'homme en Birmanie; -B4-0018/98 - O-0026/98 et B4-0019/98 - O-0027/98 dude M. Bertens, au nom du groupe du parti européen des libéraux démocrates et réformateurs, au Conseil et à la Commission, sur les violations des droits de l'homme en Birmanie; -B4-0126/98 - O-0034/98 des députés Pettinari et Vinci, au nom du groupe confédéral de la gauche unitaire européenne - gauche verte nordique, au Conseil, sur les relations entre l'Union européenne et la Birmanie; -B4-0131/98 - O-0039/98 de M. Dupuis, au nom du groupe de l'alliance radicale européenne, à la Commission, sur la violation des droits de l'homme en Birmanie; -B4-0133/98 - O-0041/98 des députés Telkämper, Aelvoet, Kreissl-Dörfler et Lannoye, au nom du groupe des verts au Parlement européen, à la Commission, sur les violations des droits de l'homme en Birmanie; -B4-0136/98 - O-0045/98 et B4-0141/98 - O-0050/98 des députés Kinnock et Vecchi, au nom du groupe du parti des socialistes européens, au Conseil et à la Commission, sur la situation en Birmanie; -B4-0139/98 - O-0048/98) de Mme van Bladel, au nom du groupe Union pour l'Europe, au Conseil, sur la situation des droits de l'homme en Birmanie; -B4-0011/98 - O-0182/98 de M. Bertens, au nom du groupe du parti européen des libéraux démocrates et réformateurs, au Conseil, sur la position de l'Union européenne lors de la 54e Assemblée des Nations unies sur les droits de l'homme à Genève; -B4-0125/98 - O-0033/98 des députés Carnero González, Manisco et Alavanos, au nom du groupe confédéral de la gauche unitaire européenne - gauche verte nordique, au Conseil, sur la Commission des Droits de l'homme des Nations unies; -B4-0130/98 - O-0038/98 de M. Dupuis, au nom du groupe de l'alliance radicale européenne, au Conseil, sur la position de l'Union européenne lors de la 54e session de la Commission des Droits de l'homme des Nations unies à Genève; -B4-0132/98 - O-0040/98 des députés Aglietta, Müller, Kreissl-Dörfler et Ripa Di Meana, au nom du groupe des verts au Parlement européen, au Conseil, sur la position de l'Union européenne lors de la 54e session de la Commission des Droits de l'homme des Nations unies à Genève; -B4-0138/98 - O-0047/98 de Mme van Bladel, au nom du groupe Union pour l'Europe, au Conseil, sur la 54e session des Nations unies sur les droits de l'homme; -B4-0140/98 - O-0049/98 de M. Barros Moura, au nom du groupe du parti des socialistes européens, au Conseil, sur la position de l'Union européenne lors de la 54e Assemblée des Nations unies sur les droits de l'homme à Genève."
# print(len(dummy.split(" ")))


# In[8]:


MAX_LENGTH = 700

eng_prefixes = (
    "i am ", "i m ",
    "he is", "he s ",
    "she is", "she s",
    "you are", "you re ",
    "we are", "we re ",
    "they are", "they re "
)


def filterPair(p):
    return len(p[0].split(' ')) < MAX_LENGTH and         len(p[1].split(' ')) < MAX_LENGTH 


def filterPairs(pairs):
    return [pair for pair in pairs if filterPair(pair)]


# In[9]:


def prepareData(lang1, lang2, reverse=False):
    input_lang, output_lang, pairs = reading_bigfiles(lang1, lang2, reverse)
    
    print("Read %s sentence pairs" % len(pairs))
    pairs = filterPairs(pairs)
    print("Trimmed to %s sentence pairs" % len(pairs))
    print("Counting words...")
    for pair in pairs:
        input_lang.addSentence(pair[0])
        output_lang.addSentence(pair[1])
    print("Counted words:")
    print(input_lang.name, input_lang.n_words)
    print(output_lang.name, output_lang.n_words)
    return input_lang, output_lang, pairs


input_lang, output_lang, pairs = prepareData('eng', 'fra', True)
print(random.choice(pairs))
print(len(pairs),"after completion")


# In[10]:


print(output_lang.n_words,"number of words")


# In[11]:


import json
with open("../processed/input_index2word",'w') as f:
    f.write(json.dumps(input_lang.index2word))
with open("../processed/input_word2index",'w') as f:
    f.write(json.dumps(input_lang.word2index))

# add_text_to_files = open('./data/technical_text.txt', encoding='UTF-8').read().strip().split('\n')

with open("../processed/output_index2word",'w') as f:
    f.write(json.dumps(output_lang.index2word))
with open("../processed/output_word2index",'w') as f:
    f.write(json.dumps(output_lang.word2index))


# In[12]:


class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(EncoderRNN, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)

    def forward(self, input, hidden):
        embedded = self.embedding(input).view(1, 1, -1)
        output = embedded
        output, hidden = self.gru(output, hidden)
        return output, hidden

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)


# In[13]:


class DecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size):
        super(DecoderRNN, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, input, hidden):
        output = self.embedding(input).view(1, 1, -1)
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
        output = self.softmax(self.out(output[0]))
        return output, hidden

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)


# In[14]:


class AttnDecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
        super(AttnDecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout_p = dropout_p
        self.max_length = max_length

        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
        self.dropout = nn.Dropout(self.dropout_p)
        self.gru = nn.GRU(self.hidden_size, self.hidden_size)
        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, encoder_outputs):
        embedded = self.embedding(input).view(1, 1, -1)
        embedded = self.dropout(embedded)

        attn_weights = F.softmax(
            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
        attn_applied = torch.bmm(attn_weights.unsqueeze(0),
                                 encoder_outputs.unsqueeze(0))

        output = torch.cat((embedded[0], attn_applied[0]), 1)
        output = self.attn_combine(output).unsqueeze(0)

        output = F.relu(output)
        output, hidden = self.gru(output, hidden)

        output = F.log_softmax(self.out(output[0]), dim=1)
        return output, hidden, attn_weights

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)


# In[15]:


def indexesFromSentence(lang, sentence):
    return [lang.word2index[word] for word in sentence.split(' ')]


def tensorFromSentence(lang, sentence):
    indexes = indexesFromSentence(lang, sentence)
    indexes.append(EOS_token)
    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)


def tensorsFromPair(pair):
    input_tensor = tensorFromSentence(input_lang, pair[0])
    target_tensor = tensorFromSentence(output_lang, pair[1])
    return (input_tensor, target_tensor)


# In[16]:


teacher_forcing_ratio = 0.5


def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):
    encoder_hidden = encoder.module.initHidden()

    encoder_optimizer.zero_grad()
    decoder_optimizer.zero_grad()

    input_length = input_tensor.size(0)
    target_length = target_tensor.size(0)

    encoder_outputs = torch.zeros(max_length, encoder.module.hidden_size, device=device)

    loss = 0

    for ei in range(input_length):
        encoder_output, encoder_hidden = encoder(
            input_tensor[ei], encoder_hidden)
        encoder_outputs[ei] = encoder_output[0, 0]

    decoder_input = torch.tensor([[SOS_token]], device=device)

    decoder_hidden = encoder_hidden

    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

    if use_teacher_forcing:
        # Teacher forcing: Feed the target as the next input
        for di in range(target_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            loss += criterion(decoder_output, target_tensor[di])
            decoder_input = target_tensor[di]  # Teacher forcing

    else:
        # Without teacher forcing: use its own predictions as the next input
        for di in range(target_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            topv, topi = decoder_output.topk(1)
            decoder_input = topi.squeeze().detach()  # detach from history as input

            loss += criterion(decoder_output, target_tensor[di])
            if decoder_input.item() == EOS_token:
                break

    loss.backward()

    encoder_optimizer.step()
    decoder_optimizer.step()

    return loss.item() / target_length


# In[17]:


import time
import math


def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)


def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / (percent)
    rs = es - s
    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))


# In[18]:


def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):
    start = time.time()
    plot_losses = []
    print_loss_total = 0  # Reset every print_every
    plot_loss_total = 0  # Reset every plot_every

    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
    training_pairs = [tensorsFromPair(random.choice(pairs))
                      for i in range(n_iters)]
    criterion = nn.NLLLoss()

    for iter in range(1, n_iters + 1):
        training_pair = training_pairs[iter - 1]
        input_tensor = training_pair[0]
        target_tensor = training_pair[1]

        loss = train(input_tensor, target_tensor, encoder,
                     decoder, encoder_optimizer, decoder_optimizer, criterion)
        print_loss_total += loss
        plot_loss_total += loss

        if iter % print_every == 0:
            print_loss_avg = print_loss_total / print_every
            print_loss_total = 0
            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),
                                         iter, iter / n_iters * 100, print_loss_avg))

        if iter % plot_every == 0:
            plot_loss_avg = plot_loss_total / plot_every
            plot_losses.append(plot_loss_avg)
            plot_loss_total = 0

    showPlot(plot_losses)


# In[19]:


import matplotlib.pyplot as plt
plt.switch_backend('agg')
import matplotlib.ticker as ticker
import numpy as np


def showPlot(points):
    plt.figure()
    fig, ax = plt.subplots()
    # this locator puts ticks at regular intervals
    loc = ticker.MultipleLocator(base=0.2)
    ax.yaxis.set_major_locator(loc)
    plt.plot(points)


# In[20]:


def evaluate(encoder, decoder, sentence, max_length=MAX_LENGTH):
    with torch.no_grad():
        input_tensor = tensorFromSentence(input_lang, sentence)
        input_length = input_tensor.size()[0]
        encoder_hidden = encoder.initHidden()

        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)

        for ei in range(input_length):
            encoder_output, encoder_hidden = encoder(input_tensor[ei],
                                                     encoder_hidden)
            encoder_outputs[ei] += encoder_output[0, 0]

        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS

        decoder_hidden = encoder_hidden

        decoded_words = []
        decoder_attentions = torch.zeros(max_length, max_length)

        for di in range(max_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            decoder_attentions[di] = decoder_attention.data
            topv, topi = decoder_output.data.topk(1)
            if topi.item() == EOS_token:
                decoded_words.append('<EOS>')
                break
            else:
                decoded_words.append(output_lang.index2word[topi.item()])

            decoder_input = topi.squeeze().detach()

        return decoded_words, decoder_attentions[:di + 1]


# In[21]:


def evaluateRandomly(encoder, decoder, n=10):
    for i in range(n):
        pair = random.choice(pairs)
        print('>', pair[0])
        print('=', pair[1])
        output_words, attentions = evaluate(encoder, decoder, pair[0])
        output_sentence = ' '.join(output_words)
        print('<', output_sentence)
        print('')


# In[22]:


hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)


# In[23]:


encoder1.parameters()


# In[24]:


if torch.cuda.device_count() > 1:
  print("Let's use", torch.cuda.device_count(), "GPUs!")
  # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
  encoder1 = nn.DataParallel(encoder1,device_ids=[0,1,2,3])
  attn_decoder1 = nn.DataParallel(attn_decoder1)
encoder1.train()


# In[25]:


attn_decoder1.train()


# In[26]:


trainIters(encoder1, attn_decoder1, 80000, print_every=5000)


# In[ ]:


import os
model_path = './model/translate.pt'


# In[ ]:


if not os.path.exists('./model'):
    os.makedirs('model')
torch.save({'encoder':encoder1.state_dict(),
            'attentondecoder':attn_decoder1.state_dict()},model_path)


# In[ ]:


model_loaded = torch.load(model_path)
encoder1.load_state_dict(model_loaded['encoder'])
attn_decoder1.load_state_dict(model_loaded['attentondecoder'])
encoder1.eval()


# In[ ]:


attn_decoder1.eval()


# In[ ]:


evaluateRandomly(encoder1, attn_decoder1)


# In[ ]:


output_words, attentions = evaluate(
    encoder1, attn_decoder1, "je suis trop froid .")
plt.matshow(attentions.numpy())


# In[ ]:


len(output_lang.index2word),len(output_lang.word2index),len(input_lang.index2word),len(input_lang.word2index)


# In[ ]:


add_text_pairs = [[ens for ens in s.split('\t')] for s in add_text_to_files]
# english_extra = [[output_lang.word2index[ens] for ens in end.split(' ')] for end,frd in add_text_pairs]
for end,frd in add_text_pairs:
    output_lang.addSentence(end)
    input_lang.addSentence(frd)
len(output_lang.index2word),len(output_lang.word2index),len(input_lang.index2word),len(input_lang.word2index)


# In[ ]:


def showAttention(input_sentence, output_words, attentions):
    # Set up figure with colorbar
    fig = plt.figure()
    ax = fig.add_subplot(111)
    cax = ax.matshow(attentions.numpy(), cmap='bone')
    fig.colorbar(cax)

    # Set up axes
    ax.set_xticklabels([''] + input_sentence.split(' ') +
                       ['<EOS>'], rotation=90)
    ax.set_yticklabels([''] + output_words)

    # Show label at every tick
    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))

    plt.show()


def evaluateAndShowAttention(input_sentence):
    output_words, attentions = evaluate(
        encoder1, attn_decoder1, input_sentence)
    print('input =', input_sentence)
    print('output =', ' '.join(output_words))
    showAttention(input_sentence, output_words, attentions)


evaluateAndShowAttention("elle a cinq ans de moins que moi .")

evaluateAndShowAttention("elle est trop petit .")

evaluateAndShowAttention("je ne crains pas de mourir .")

evaluateAndShowAttention("salut comment allez-vous")


# In[ ]:


# evaluateAndShowAttention("")


# In[ ]:





In this i used nn.dataparallel() to make the model to work with mult-gpu after changing the code i executed the program and its showing the error
RuntimeError: invalid argument 6: wrong matrix size at /pytorch/aten/src/THC/generic/THCTensorMathBlas.cu:492
I think that error is maybe of different lengths of text so using dataparallel api it doesn’t able to split them into equal parts maybe that was the error i am not sure.
could anybody help me to solve this error.