add all files

.gitattributes

@@ -25,3 +25,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+30_tokenizer_ass.pickle filter=lfs diff=lfs merge=lfs -text
+30_tokenizer_eng.pickle filter=lfs diff=lfs merge=lfs -text

Data.py

@@ -0,0 +1,62 @@
+import pandas as pd
+import re
+import tensorflow as tf
+from tensorflow.keras.layers import Embedding, LSTM, Dense,Bidirectional
+from tensorflow.keras.models import Model
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+import numpy as np
+import string
+from string import digits
+from sklearn.utils import shuffle
+from sklearn.model_selection import train_test_split
+import nltk
+from nltk.tokenize import word_tokenize
+from tqdm import tqdm
+
+class Dataset:
+    def __init__(self, data, tknizer_ass, tknizer_eng, max_len):
+        self.encoder_inps = data['ass'].values
+        self.decoder_inps = data['eng_inp'].values
+        self.decoder_outs = data['eng_out'].values
+        self.tknizer_eng = tknizer_eng
+        self.tknizer_ass = tknizer_ass
+        self.max_len = max_len
+
+    def __getitem__(self, i):
+        self.encoder_seq = self.tknizer_ass.texts_to_sequences([self.encoder_inps[i]]) # need to pass list of values
+        self.decoder_inp_seq = self.tknizer_eng.texts_to_sequences([self.decoder_inps[i]])
+        self.decoder_out_seq = self.tknizer_eng.texts_to_sequences([self.decoder_outs[i]])
+
+        self.encoder_seq = pad_sequences(self.encoder_seq, maxlen=self.max_len, dtype='int32', padding='post')
+        self.decoder_inp_seq = pad_sequences(self.decoder_inp_seq, maxlen=self.max_len, dtype='int32', padding='post')
+        self.decoder_out_seq = pad_sequences(self.decoder_out_seq, maxlen=self.max_len, dtype='int32', padding='post')
+        return self.encoder_seq, self.decoder_inp_seq, self.decoder_out_seq
+
+    def __len__(self): # your model.fit_gen requires this function
+        return len(self.encoder_inps)
+
+    
+class Dataloder(tf.keras.utils.Sequence):    
+    def __init__(self, dataset, batch_size=1):
+        self.dataset = dataset
+        self.batch_size = batch_size
+        self.indexes = np.arange(len(self.dataset.encoder_inps))
+
+
+    def __getitem__(self, i):
+        start = i * self.batch_size
+        stop = (i + 1) * self.batch_size
+        data = []
+        for j in range(start, stop):
+            data.append(self.dataset[j])
+
+        batch = [np.squeeze(np.stack(samples, axis=1), axis=0) for samples in zip(*data)]
+        # we are creating data like ([italian, english_inp], english_out) these are already converted into seq
+        return tuple([[batch[0],batch[1]],batch[2]])
+
+    def __len__(self):  # your model.fit_gen requires this function
+        return len(self.indexes) // self.batch_size
+
+    def on_epoch_end(self):
+        self.indexes = np.random.permutation(self.indexes)

README.md

@@ -1,6 +1,6 @@
 ---
 title: Attention_based_Assamese_English_NMT_BiLSTM
-emoji: 🦀
+emoji: 📚
 colorFrom: gray
 colorTo: indigo
 sdk: streamlit

app.py

@@ -0,0 +1,82 @@
+import streamlit as st
+import pandas as pd
+import datetime
+import numpy as np
+import datetime
+import model
+import inference
+
+
+# Global params
+if 'model' not in st.session_state:
+    loaded_model,tokenizer_eng,tokenizer_ass,in_input_length = model.main()
+    st.session_state['model'] = loaded_model
+    st.session_state['tokenizer_eng'] = tokenizer_eng
+    st.session_state['tokenizer_ass'] = tokenizer_ass
+    st.session_state['in_input_length'] = in_input_length
+    
+# st.write(st.session_state)
+
+# def model_loading():
+#     return model.main()
+
+def show_information():
+    # Show Information about the selected Stock
+    st.header('Now translate everything into English!')
+    # st.caption("Analyzing data from 2015 to 2021")
+    # st.text("1) There is a 60% chance of gap up opening in any random trade in Reliance 😮 ")
+    # st.text("2) 1% of the gap up is more than Rs:15.00 i.e more quantity == more profit😇")
+    # st.text("3) Median, Q3 or 75th percentile have increased from 2015(1.8) to 2021(11.55)💰")
+
+def select_text():
+    # Select the Suggested Assamese Text
+    option = st.selectbox(
+     'Select these suggested Assamese Sentences',
+     ('সমগ্ৰ দেশজুৰি ব্যাপক চৰ্চা হৈছিল উক্ত ঘটনাৰ ',
+      'দৃষ্টান্ত ব্যৱহাৰ কৰাৰ সম্পৰ্কে আমি যীচুৰ পৰা কি শিকিব পাৰোঁ ',
+      'তেওঁ যি ইচ্ছা তাকে কৰিব নোৱাৰে '))
+
+    st.write('You have selected suggested text')
+    
+    title = st.text_input('Assamese Text Input', option)
+    # st.write('Your Assamese Text', title)
+    
+    return title
+    # return selected_date
+
+# @st.cache
+# def prepare_data_for_selected_date():
+#     df = pd.read_csv("dataset/reliance_30min.csv")
+#     df = helper.format_date(df)
+#     df = helper.replace_vol(df)
+#     df = helper.feature_main(df)
+#     df.to_csv('dataset/processed_reliance30m.csv')
+    
+#     return df
+
+# @st.cache
+# def show_result(sentence):
+#   pass   
+
+
+# def show_prediction_result(prepared_data):
+#     model = all_model.load_model()
+#     result = all_model.prediction(model,prepared_data)
+    
+#     return result
+    
+
+
+
+def main():
+    st.title('📚Assamese to English Translator🤓')
+    show_information()
+    text = select_text()
+    if st.button('Translate'):
+        result = inference.infer(st.session_state['model'],text,st.session_state['tokenizer_ass'],
+                                 st.session_state['tokenizer_eng'],st.session_state['in_input_length'])
+        st.caption('Your Assamese translated text')
+        st.text(result[:-6])
+      
+if __name__ == "__main__":
+    main()

dataset/30_length/train.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cfb065d12104363eb94913a59d8610568e64ee3c1c9a77a14bf898900e0b756
+size 29548368

dataset/30_length/validation.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3662a9bcb3bc6d91999c035944c0e913c8bdb2edd47b4757a1a69f18cb2b630
+size 7384761

inference.py

@@ -0,0 +1,26 @@
+from tensorflow import argmax,expand_dims,convert_to_tensor
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+# BRUTE FORCE
+def infer(model,sentence,tokenizer_ass,tokenizer_eng,in_input_length):
+  encoder_seq = tokenizer_ass.texts_to_sequences([sentence]) # need to pass list of values
+  encoder_seq = pad_sequences(encoder_seq, maxlen=in_input_length, dtype='int32', padding='post')
+  encoder_seq = convert_to_tensor(encoder_seq)
+  initial_state = model.layers[0].initialize_states_bidirectional(batch_size=1)
+  encoder_outputs, f_encoder_hidden, f_encoder_cell,b_encoder_hidden, b_encoder_cell = model.layers[0](encoder_seq,initial_state)
+  dec_input = expand_dims([tokenizer_eng.word_index['<start>']],0)
+
+  result = ''
+  for t in range(30):
+    Output, dec_h,dec_c,attention_w,context_vec = model.layers[1].onestep_decoder(dec_input,encoder_outputs,f_encoder_hidden, f_encoder_cell,b_encoder_hidden, b_encoder_cell)
+    # result_beam_list = beam_search(Output,k=1)
+    # result_beam = result_beam_list[0][0]
+    # attention_weights = tf.reshape(attention_w,(-1,))
+    predict_id = argmax(Output[0]).numpy()
+    result += tokenizer_eng.index_word[predict_id]+' '
+    if tokenizer_eng.index_word[predict_id] == '<end>':
+      break
+    dec_input = expand_dims([predict_id],0)
+
+  
+  print(result)
+  return result

model.py

@@ -0,0 +1,314 @@
+# import matplotlib.pyplot as plt
+# %matplotlib inline
+# import seaborn as sns
+import pickle
+import pandas as pd
+import re
+import os
+os.add_dll_directory("C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2/bin")
+import tensorflow as tf
+from tensorflow.keras.layers import Embedding, LSTM, Dense,Bidirectional
+from tensorflow.keras.models import Model
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from tensorflow.keras import backend as K
+import numpy as np
+import string
+from string import digits
+from sklearn.utils import shuffle
+from sklearn.model_selection import train_test_split
+import nltk
+from nltk.tokenize import word_tokenize
+from tqdm import tqdm
+from Data import Dataset,Dataloder
+
+
+
+"""########################################------MODEL------########################################
+"""
+
+########################################------Encoder model------########################################
+class Encoder(tf.keras.Model):
+
+
+    def __init__(self,inp_vocab_size,embedding_size,lstm_size,input_length):
+        super().__init__()
+
+        self.inp_vocab_size = inp_vocab_size
+        self.embedding_size = embedding_size
+        self.lstm_size = lstm_size
+        self.input_length = input_length
+        #Initialize Embedding layer
+
+    def build(self,input_shape):
+        self.embedding = Embedding(input_dim=self.inp_vocab_size, output_dim=self.embedding_size,
+                                  input_length=self.input_length,trainable=True,name="encoder_embed")
+        #Intialize Encoder LSTM layer
+        self.bilstm = tf.keras.layers.Bidirectional(LSTM(units = self.lstm_size,return_sequences=True,return_state=True),merge_mode='sum')
+
+    def call(self,input_sequence,initial_state):
+        '''
+          Input:Input_sequence[batch_size,input_length]
+                Initial_state 4x[batch_size,encoder_units]
+          
+          Output: lstm_enc_output [batch_size,input_length,encoder_units]
+                  forward_h/c & backward_h/c [batch_size,encoder_units]
+        '''
+        # print("initial_state",len(initial_state))
+        input_embd = self.embedding(input_sequence)
+        lstm_enc_output, forward_h, forward_c, backward_h, backward_c = self.bilstm(input_embd,initial_state)
+        return lstm_enc_output, forward_h, forward_c, backward_h, backward_c
+        # return lstm_enc_output, forward_h, forward_c
+
+    
+    def initialize_states(self,batch_size):
+      '''
+      Given a batch size it will return intial hidden state and intial cell state.
+      If batch size is 32- Hidden state is zeros of size [32,lstm_units], cell state zeros is of size [32,lstm_units]
+      '''
+      self.lstm_state_h = tf.random.uniform(shape=[batch_size,self.lstm_size],dtype=tf.float32)
+      self.lstm_state_c = tf.random.uniform(shape=[batch_size,self.lstm_size],dtype=tf.float32)
+      return self.lstm_state_h,self.lstm_state_c
+
+    def initialize_states_bidirectional(self,batch_size):
+      states = [tf.zeros((batch_size, self.lstm_size)) for i in range(4)]
+      return states
+
+
+
+########################################------Attention model------########################################
+class Attention(tf.keras.layers.Layer):
+    def __init__(self,scoring_function, att_units):
+        super().__init__()
+        self.att_units = att_units
+        self.scoring_function = scoring_function
+        # self.batch_size = batch_size
+        # Please go through the reference notebook and research paper to complete the scoring functions
+
+        if self.scoring_function=='dot':
+            pass
+        
+        elif scoring_function == 'general':
+            self.dense = Dense(self.att_units)
+        
+        elif scoring_function == 'concat':
+            self.dense = tf.keras.layers.Dense(att_units, activation='tanh')
+            self.dense1 = tf.keras.layers.Dense(1)
+  
+  
+    def call(self,decoder_hidden_state,encoder_output):
+
+
+    
+        if self.scoring_function == 'dot':
+            decoder_hidden_state = tf.expand_dims(decoder_hidden_state,axis=2)
+            similarity = tf.matmul(encoder_output,decoder_hidden_state)
+            weights    = tf.nn.softmax(similarity,axis=1)
+            context_vector = tf.matmul(weights,encoder_output,transpose_a=True)
+            context_vector = tf.squeeze(context_vector, axis=1)
+            return context_vector,weights
+
+        elif self.scoring_function == 'general':
+            decoder_hidden_state=tf.expand_dims(decoder_hidden_state, 1)
+            score = tf.matmul(decoder_hidden_state, self.dense(
+                    encoder_output), transpose_b=True)
+            attention_weights = tf.keras.activations.softmax(score, axis=-1) 
+            context_vector = tf.matmul(attention_weights, encoder_output)
+            context_vector=tf.reduce_sum(context_vector, axis=1)
+            attention_weights=tf.reduce_sum(attention_weights, axis=1)
+            attention_weights=tf.expand_dims(attention_weights, 2)
+            return context_vector,attention_weights
+
+        elif self.scoring_function == 'concat':
+            decoder_hidden_state=tf.expand_dims(decoder_hidden_state, 1)
+            decoder_hidden_state = tf.tile(
+                        decoder_hidden_state, [1,30, 1])
+            score = self.dense1(
+                        self.dense(tf.concat((decoder_hidden_state, encoder_output), axis=-1)))
+            score = tf.transpose(score, [0, 2, 1])
+            attention_weights = tf.keras.activations.softmax(score, axis=-1) 
+            context_vector = tf.matmul(attention_weights, encoder_output)
+            context_vector=tf.reduce_sum(context_vector, axis=1)
+            attention_weights=tf.reduce_sum(attention_weights, axis=1)
+            attention_weights=tf.expand_dims(attention_weights, 2)
+            
+            return context_vector,attention_weights
+    
+
+########################################------OneStepDecoder model------########################################
+class OneStepDecoder(tf.keras.Model):
+    def __init__(self,tar_vocab_size, embedding_dim, input_length, dec_units ,score_fun ,att_units):
+
+      # Initialize decoder embedding layer, LSTM and any other objects needed
+      super().__init__()
+      self.tar_vocab_size = tar_vocab_size
+      self.embedding_dim = embedding_dim
+      self.input_length = input_length
+      self.dec_units = dec_units
+      self.score_fun = score_fun
+      self.att_units = att_units
+
+    def build(self,input_shape):
+      self.attention = Attention('concat', self.att_units)
+      self.embedding = Embedding(input_dim=self.tar_vocab_size,output_dim=self.embedding_dim,
+                                 input_length=self.input_length,mask_zero=True,trainable=True,name="Decoder_Embed")
+      self.bilstm = tf.keras.layers.Bidirectional(LSTM(units = self.dec_units,return_sequences=True,return_state=True),merge_mode='sum')
+      self.dense = Dense(self.tar_vocab_size)
+      
+
+
+    def call(self,input_to_decoder, encoder_output, f_state_h,f_state_c,b_state_h,b_state_c):
+        dec_embd = self.embedding(input_to_decoder)
+        context_vectors,attention_weights = self.attention(f_state_h,encoder_output)
+        context_vectors_ = tf.expand_dims(context_vectors,axis=1)
+        concat_vector = tf.concat([dec_embd,context_vectors_],axis=2)
+        states = [f_state_h,f_state_c,b_state_h,b_state_c]
+        decoder_outputs,dec_f_state_h,dec_f_state_c,dec_b_state_h,dec_b_state_c = self.bilstm(concat_vector,states)
+        decoder_outputs = tf.squeeze(decoder_outputs,axis=1)
+        dense_output = self.dense(decoder_outputs)
+        
+        return dense_output,dec_f_state_h,dec_f_state_c,attention_weights,context_vectors
+    
+    
+########################################------Decoder model------########################################
+class Decoder(tf.keras.Model):
+    def __init__(self,out_vocab_size, embedding_dim, input_length, dec_units ,score_fun ,att_units):
+      #Intialize necessary variables and create an object from the class onestepdecoder
+      super().__init__()
+      self.out_vocab_size = out_vocab_size
+      self.embedding_dim = embedding_dim
+      self.input_length = input_length
+      self.dec_units = dec_units
+      self.score_fun = score_fun
+      self.att_units = att_units
+
+    def build(self,input_shape):
+      self.onestep_decoder = OneStepDecoder(self.out_vocab_size, self.embedding_dim, self.input_length, self.dec_units ,self.score_fun ,
+                                            self.att_units)
+
+    def call(self, input_to_decoder,encoder_output,f_decoder_hidden_state,f_decoder_cell_state,b_decoder_hidden_state,b_decoder_cell_state ):
+
+      all_outputs = tf.TensorArray(tf.float32, size=self.input_length,name="output_array")
+      
+      for timestep in range(self.input_length):
+        output,state_h,state_c,attention_weights,context_vector = self.onestep_decoder(input_to_decoder[:,timestep:timestep+1],encoder_output,
+                                                                                       f_decoder_hidden_state,f_decoder_cell_state,b_decoder_hidden_state,b_decoder_cell_state)
+        all_outputs = all_outputs.write(timestep,output)
+
+      all_outputs = tf.transpose(all_outputs.stack(),[1,0,2])
+      
+      return all_outputs
+  
+########################################------encoder_decoder model------########################################
+class encoder_decoder(tf.keras.Model):
+    def __init__(self,out_vocab_size,inp_vocab_size,embedding_dim,embedding_size,in_input_length,tar_input_length,dec_units,lstm_size,att_units,batch_size):
+        super().__init__()
+        #Intialize objects from encoder decoder
+        self.out_vocab_size = out_vocab_size
+        self.inp_vocab_size = inp_vocab_size
+
+        self.embedding_dim_target = embedding_dim
+        self.embedding_dim_input = embedding_size
+        self.in_input_length = in_input_length
+        self.tar_input_length = tar_input_length
+
+        self.dec_lstm_size = dec_units 
+        self.enc_lstm_size = lstm_size
+
+        self.att_units = att_units
+        self.batch_size = batch_size
+
+    def build(self,input_shape):
+        self.encoder = Encoder(self.inp_vocab_size,self.embedding_dim_input,self.enc_lstm_size,self.in_input_length)
+        self.decoder = Decoder(self.out_vocab_size,self.embedding_dim_target, self.tar_input_length, self.dec_lstm_size ,'general' ,self.att_units)
+    
+    def call(self,data):
+        input_sequence, target_sequence = data[0],data[1]
+        # print(input_sequence.shape)
+        encoder_initial_state = self.encoder.initialize_states_bidirectional(self.batch_size)
+        # print(len(encoder_initial_state))
+        encoder_output,f_encoder_state_h,f_encoder_state_c,b_encoder_state_h,b_encoder_state_c = self.encoder(input_sequence,encoder_initial_state)
+        decoder_output = self.decoder(target_sequence,encoder_output,f_encoder_state_h,f_encoder_state_c,b_encoder_state_h,b_encoder_state_c)
+        return decoder_output
+
+
+def loss_function(real, pred):
+    loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
+    from_logits=True)
+    mask = tf.math.logical_not(tf.math.equal(real, 0))
+    loss_ = loss_object(real, pred)
+    mask = tf.cast(mask, dtype=loss_.dtype)
+    loss_ *= mask
+
+    return tf.reduce_mean(loss_)
+
+def accuracy(real,pred):
+    pred_val = K.cast(K.argmax(pred,axis=-1),dtype='float32')
+    real_val = K.cast(K.equal(real,pred_val),dtype='float32')
+
+    mask = K.cast(K.greater(real,0),dtype='float32')
+    n_correct = K.sum(mask*real_val)
+    n_total = K.sum(mask)
+
+    return n_correct/n_total
+
+def load_weights():
+    """======================================================LOADING======================================================"""
+    # Dataset
+    with open('dataset/30_length/train.pickle', 'rb') as handle:
+        train = pickle.load(handle)
+
+    with open('dataset/30_length/validation.pickle', 'rb') as handle:
+        validation = pickle.load(handle)
+
+    # Tokenizer
+    with open('tokenizer/30_tokenizer_eng.pickle', 'rb') as handle:
+        tokenizer_eng = pickle.load(handle)
+
+    with open('tokenizer/30_tokenizer_ass.pickle', 'rb') as handle:
+        tokenizer_ass = pickle.load(handle)
+
+    # Vocab Size
+    vocab_size_ass = len(tokenizer_ass.word_index.keys())
+    vocab_size_eng = len(tokenizer_eng.word_index.keys())
+    
+    return train,validation,tokenizer_eng,tokenizer_ass,vocab_size_ass,vocab_size_eng
+
+def main():
+    train,validation,tokenizer_eng,tokenizer_ass,vocab_size_ass,vocab_size_eng = load_weights()
+    in_input_length = 30
+    tar_input_length = 30
+    inp_vocab_size = vocab_size_ass
+    out_vocab_size = vocab_size_eng
+
+    dec_units = 128
+    lstm_size = 128
+    att_units = 256
+    batch_size = 32
+    embedding_dim = 300
+    embedding_size = 300
+
+    train_dataset = Dataset(train, tokenizer_ass, tokenizer_eng, in_input_length)
+    test_dataset  = Dataset(validation, tokenizer_ass, tokenizer_eng, in_input_length)
+
+    train_dataloader = Dataloder(train_dataset, batch_size)
+    test_dataloader = Dataloder(test_dataset, batch_size)
+
+
+    print(train_dataloader[0][0][0].shape, train_dataloader[0][0][1].shape, train_dataloader[0][1].shape)
+    
+    model = encoder_decoder(out_vocab_size,inp_vocab_size,embedding_dim,embedding_size,in_input_length,tar_input_length,dec_units,lstm_size,att_units,batch_size)
+    optimizer = tf.keras.optimizers.Adam()
+    model.compile(optimizer=optimizer,loss=loss_function,metrics=[accuracy])
+    
+    # train_steps=train.shape[0]//32
+    # valid_steps=validation.shape[0]//32
+    model.fit(train_dataloader, steps_per_epoch=10, epochs=1,verbose=1, validation_data=train_dataloader, validation_steps=1)
+    
+    model.load_weights('models/bi_directional_concat_256_batch_160_epoch_30_length_ass_eng_nmt_weights.h5')
+    model.fit(train_dataloader, steps_per_epoch=10, epochs=1,verbose=1, validation_data=train_dataloader, validation_steps=1)
+    model.summary()
+    
+    return model,tokenizer_eng,tokenizer_ass,in_input_length
+# if __name__=="__main__":
+#     main()

models/bi_directional_concat_256_batch_160_epoch_30_length_ass_eng_nmt_weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87b2c0bc456cb3feb5577eea4d62bdba08db30086c5a342491030569b0a700c4
+size 130891904

tokenizer/30_tokenizer_ass.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73a1b81f54315f32ca37eebbee823eec031dbe364aa2c3088e9bb8a8cbdda90d
+size 3461824

tokenizer/30_tokenizer_eng.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:768f98cc461fe1f96b868c92d4bbcdaef5bbbe05daf85ad9e922125aa640b4a3
+size 1209912