first commit

app.py

@@ -0,0 +1,148 @@
+import torch
+import torch.onnx
+from transformer import Transformer
+import torch
+from huggingface_hub import hf_hub_download
+import torch
+import numpy as np
+import gradio as gr
+
+
+# Generated this by filtering Appendix code
+START_TOKEN = '<START>'
+PADDING_TOKEN = '<PADDING>'
+END_TOKEN = '<END>'
+
+
+english_vocabulary = [START_TOKEN, ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', 
+                        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+                        ':', '<', '=', '>', '?', '@',
+                        '[', '\\', ']', '^', '_', '`', 
+                        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
+                        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 
+                        'y', 'z', 
+                        '{', '|', '}', '~', PADDING_TOKEN, END_TOKEN]
+
+
+gujarati_vocabulary = [
+    START_TOKEN, ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', 
+    '૦', '૧', '૨', '૩', '૪', '૫', '૬', '૭', '૮', '૯',
+    ':', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 
+    'અ', 'આ', 'ઇ', 'ઈ', 'ઉ', 'ઊ', 'ઋ', 'એ', 'ઐ', 'ઓ', 'ઔ', 
+    'ક', 'ખ', 'ગ', 'ઘ', 'ઙ', 'ચ', 'છ', 'જ', 'ઝ', 'ઞ', 
+    'ટ', 'ઠ', 'ડ', 'ઢ', 'ણ', 'ત', 'થ', 'દ', 'ધ', 'ન', 
+    'પ', 'ફ', 'બ', 'ભ', 'મ', 'ય', 'ર', 'લ', 'વ', 'શ', 
+    'ષ', 'સ', 'હ', 'ળ', 'ક્ષ', 'જ્ઞ', 'ં', 'ઃ', 'ઁ', 'ા', 
+    'િ', 'ી', 'ુ', 'ૂ', 'ે', 'ૈ', 'ો', 'ૌ', '્', 'ૐ', 
+    '{', '|', '}', '~', PADDING_TOKEN, END_TOKEN
+]
+
+index_to_gujarati = {k:v for k,v in enumerate(gujarati_vocabulary)}
+gujarati_to_index = {v:k for k,v in enumerate(gujarati_vocabulary)}
+index_to_english = {k:v for k,v in enumerate(english_vocabulary)}
+english_to_index = {v:k for k,v in enumerate(english_vocabulary)}
+
+d_model = 512 
+# batch_size = 64
+ffn_hidden = 2048
+num_heads = 8
+drop_prob = 0.1
+num_layers = 6
+max_sequence_length = 200
+kn_vocab_size = len(gujarati_vocabulary)
+device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+
+transformer = Transformer(d_model, 
+                          ffn_hidden,
+                          num_heads, 
+                          drop_prob, 
+                          num_layers, 
+                          max_sequence_length,
+                          kn_vocab_size,
+                          english_to_index,
+                          gujarati_to_index,
+                          START_TOKEN, 
+                          END_TOKEN, 
+                          PADDING_TOKEN)
+
+model_file = hf_hub_download(repo_id="yashAI007/English_to_Gujarati_Translation", filename="model.pth")
+model = torch.load(model_file)
+transformer.load_state_dict(model['model_state_dict'])
+transformer.to(device)
+transformer.eval()
+
+
+NEG_INFTY = -1e9
+
+def create_masks(eng_batch, kn_batch):
+    num_sentences = len(eng_batch)
+    look_ahead_mask = torch.full([max_sequence_length, max_sequence_length] , True)
+    look_ahead_mask = torch.triu(look_ahead_mask, diagonal=1)
+    encoder_padding_mask = torch.full([num_sentences, max_sequence_length, max_sequence_length] , False)
+    decoder_padding_mask_self_attention = torch.full([num_sentences, max_sequence_length, max_sequence_length] , False)
+    decoder_padding_mask_cross_attention = torch.full([num_sentences, max_sequence_length, max_sequence_length] , False)
+
+    for idx in range(num_sentences):
+      eng_sentence_length, kn_sentence_length = len(eng_batch[idx]), len(kn_batch[idx])
+      eng_chars_to_padding_mask = np.arange(eng_sentence_length + 1, max_sequence_length)
+      kn_chars_to_padding_mask = np.arange(kn_sentence_length + 1, max_sequence_length)
+      encoder_padding_mask[idx, :, eng_chars_to_padding_mask] = True
+      encoder_padding_mask[idx, eng_chars_to_padding_mask, :] = True
+      decoder_padding_mask_self_attention[idx, :, kn_chars_to_padding_mask] = True
+      decoder_padding_mask_self_attention[idx, kn_chars_to_padding_mask, :] = True
+      decoder_padding_mask_cross_attention[idx, :, eng_chars_to_padding_mask] = True
+      decoder_padding_mask_cross_attention[idx, kn_chars_to_padding_mask, :] = True
+
+    encoder_self_attention_mask = torch.where(encoder_padding_mask, NEG_INFTY, 0)
+    decoder_self_attention_mask =  torch.where(look_ahead_mask + decoder_padding_mask_self_attention, NEG_INFTY, 0)
+    decoder_cross_attention_mask = torch.where(decoder_padding_mask_cross_attention, NEG_INFTY, 0)
+    return encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask
+
+transformer.eval()
+def translate(eng_sentence):
+  eng_sentence = (eng_sentence.lower(),)
+  kn_sentence = ("",)
+  for word_counter in range(max_sequence_length):
+    encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask= create_masks(eng_sentence, kn_sentence)
+    predictions = transformer(eng_sentence,
+                              kn_sentence,
+                              encoder_self_attention_mask.to(device), 
+                              decoder_self_attention_mask.to(device), 
+                              decoder_cross_attention_mask.to(device),
+                              enc_start_token=False,
+                              enc_end_token=False,
+                              dec_start_token=True,
+                              dec_end_token=False)
+    next_token_prob_distribution = predictions[0][word_counter]
+    next_token_index = torch.argmax(next_token_prob_distribution).item()
+    next_token = index_to_gujarati[next_token_index]
+    kn_sentence = (kn_sentence[0] + next_token, )
+    if next_token == END_TOKEN:
+      break
+  return kn_sentence[0][:-5]
+
+examples = [
+    ["Hello, how are you?"],
+    ["What is your name?"],
+    ["I love programming."],
+    ["This is a beautiful day."],
+    ["Can you help me with this?"],
+    ["What time is it?"],
+    ["I am learning data science."],
+    ["Where is the nearest bus stop?"],
+    ["I enjoy reading books."],
+    ["Thank you for your help."]
+]
+
+description = "This tool translates English sentences into Gujarati. Please enter your text above to get started!"
+
+iface = gr.Interface(fn=translate, 
+                     inputs="text",
+                     outputs="text",
+                     title="English to Gujarati Translation",
+                     examples=examples,
+                     description=description,
+                     )
+
+if __name__ == "__main__":
+    iface.launch()

requirements.txt

@@ -0,0 +1,93 @@
+aiofiles==23.2.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+attrs==23.2.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.1
+cycler==0.12.1
+dnspython==2.6.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
+fastapi==0.111.1
+fastapi-cli==0.0.4
+ffmpy==0.3.2
+filelock==3.15.4
+fonttools==4.53.1
+fsspec==2024.6.1
+gradio==4.38.1
+gradio_client==1.1.0
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.5
+idna==3.7
+importlib_resources==6.4.0
+Jinja2==3.1.4
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.9.1
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+orjson==3.10.6
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.35.1
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.19.0
+ruff==0.5.2
+safetensors==0.4.3
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.37.2
+sympy==1.13.0
+tokenizers==0.19.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.3.1
+tqdm==4.66.4
+transformers==4.42.4
+triton==2.3.1
+typer==0.12.3
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.30.1
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==11.0.3

transformer.py

@@ -0,0 +1,305 @@
+import numpy as np
+import torch
+import math
+from torch import nn
+import torch.nn.functional as F
+
+def get_device():
+    return torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+    # return torch.device('cpu')
+
+def scaled_dot_product(q, k, v, mask=None):
+    d_k = q.size()[-1]
+    scaled = torch.matmul(q, k.transpose(-1, -2)) / math.sqrt(d_k)
+    if mask is not None:
+        scaled = scaled.permute(1, 0, 2, 3) + mask
+        scaled = scaled.permute(1, 0, 2, 3)
+    attention = F.softmax(scaled, dim=-1)
+    values = torch.matmul(attention, v)
+    return values, attention
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, max_sequence_length):
+        super().__init__()
+        self.max_sequence_length = max_sequence_length
+        self.d_model = d_model
+
+    def forward(self):
+        even_i = torch.arange(0, self.d_model, 2).float()
+        denominator = torch.pow(10000, even_i/self.d_model)
+        position = (torch.arange(self.max_sequence_length)
+                          .reshape(self.max_sequence_length, 1))
+        even_PE = torch.sin(position / denominator)
+        odd_PE = torch.cos(position / denominator)
+        stacked = torch.stack([even_PE, odd_PE], dim=2)
+        PE = torch.flatten(stacked, start_dim=1, end_dim=2)
+        return PE
+
+class SentenceEmbedding(nn.Module):
+    "For a given sentence, create an embedding"
+    def __init__(self, max_sequence_length, d_model, language_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN):
+        super().__init__()
+        self.vocab_size = len(language_to_index)
+        self.max_sequence_length = max_sequence_length
+        self.embedding = nn.Embedding(self.vocab_size, d_model)
+        self.language_to_index = language_to_index
+        self.position_encoder = PositionalEncoding(d_model, max_sequence_length)
+        self.dropout = nn.Dropout(p=0.1)
+        self.START_TOKEN = START_TOKEN
+        self.END_TOKEN = END_TOKEN
+        self.PADDING_TOKEN = PADDING_TOKEN
+    
+    def batch_tokenize(self, batch, start_token, end_token):
+
+        def tokenize(sentence, start_token, end_token):
+            sentence_word_indicies = [self.language_to_index[token] for token in list(sentence)]
+            if start_token:
+                sentence_word_indicies.insert(0, self.language_to_index[self.START_TOKEN])
+            if end_token:
+                sentence_word_indicies.append(self.language_to_index[self.END_TOKEN])
+            for _ in range(len(sentence_word_indicies), self.max_sequence_length):
+                sentence_word_indicies.append(self.language_to_index[self.PADDING_TOKEN])
+            return torch.tensor(sentence_word_indicies)
+
+        tokenized = []
+        for sentence_num in range(len(batch)):
+           tokenized.append( tokenize(batch[sentence_num], start_token, end_token) )
+        tokenized = torch.stack(tokenized)
+        return tokenized.to(get_device())
+    
+    def forward(self, x, start_token, end_token): # sentence
+        x = self.batch_tokenize(x, start_token, end_token)
+        x = self.embedding(x)
+        pos = self.position_encoder().to(get_device())
+        x = self.dropout(x + pos)
+        return x
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, d_model, num_heads):
+        super().__init__()
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.head_dim = d_model // num_heads
+        self.qkv_layer = nn.Linear(d_model , 3 * d_model)
+        self.linear_layer = nn.Linear(d_model, d_model)
+    
+    def forward(self, x, mask):
+        batch_size, sequence_length, d_model = x.size()
+        qkv = self.qkv_layer(x)
+        qkv = qkv.reshape(batch_size, sequence_length, self.num_heads, 3 * self.head_dim)
+        qkv = qkv.permute(0, 2, 1, 3)
+        q, k, v = qkv.chunk(3, dim=-1)
+        values, attention = scaled_dot_product(q, k, v, mask)
+        values = values.permute(0, 2, 1, 3).reshape(batch_size, sequence_length, self.num_heads * self.head_dim)
+        out = self.linear_layer(values)
+        return out
+
+
+class LayerNormalization(nn.Module):
+    def __init__(self, parameters_shape, eps=1e-5):
+        super().__init__()
+        self.parameters_shape=parameters_shape
+        self.eps=eps
+        self.gamma = nn.Parameter(torch.ones(parameters_shape))
+        self.beta =  nn.Parameter(torch.zeros(parameters_shape))
+
+    def forward(self, inputs):
+        dims = [-(i + 1) for i in range(len(self.parameters_shape))]
+        mean = inputs.mean(dim=dims, keepdim=True)
+        var = ((inputs - mean) ** 2).mean(dim=dims, keepdim=True)
+        std = (var + self.eps).sqrt()
+        y = (inputs - mean) / std
+        out = self.gamma * y + self.beta
+        return out
+
+  
+class PositionwiseFeedForward(nn.Module):
+    def __init__(self, d_model, hidden, drop_prob=0.1):
+        super(PositionwiseFeedForward, self).__init__()
+        self.linear1 = nn.Linear(d_model, hidden)
+        self.linear2 = nn.Linear(hidden, d_model)
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(p=drop_prob)
+
+    def forward(self, x):
+        x = self.linear1(x)
+        x = self.relu(x)
+        x = self.dropout(x)
+        x = self.linear2(x)
+        return x
+
+
+class EncoderLayer(nn.Module):
+    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):
+        super(EncoderLayer, self).__init__()
+        self.attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)
+        self.norm1 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout1 = nn.Dropout(p=drop_prob)
+        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)
+        self.norm2 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout2 = nn.Dropout(p=drop_prob)
+
+    def forward(self, x, self_attention_mask):
+        residual_x = x.clone()
+        x = self.attention(x, mask=self_attention_mask)
+        x = self.dropout1(x)
+        x = self.norm1(x + residual_x)
+        residual_x = x.clone()
+        x = self.ffn(x)
+        x = self.dropout2(x)
+        x = self.norm2(x + residual_x)
+        return x
+    
+class SequentialEncoder(nn.Sequential):
+    def forward(self, *inputs):
+        x, self_attention_mask  = inputs
+        for module in self._modules.values():
+            x = module(x, self_attention_mask)
+        return x
+
+class Encoder(nn.Module):
+    def __init__(self, 
+                 d_model, 
+                 ffn_hidden, 
+                 num_heads, 
+                 drop_prob, 
+                 num_layers,
+                 max_sequence_length,
+                 language_to_index,
+                 START_TOKEN,
+                 END_TOKEN, 
+                 PADDING_TOKEN):
+        super().__init__()
+        self.sentence_embedding = SentenceEmbedding(max_sequence_length, d_model, language_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN)
+        self.layers = SequentialEncoder(*[EncoderLayer(d_model, ffn_hidden, num_heads, drop_prob)
+                                      for _ in range(num_layers)])
+
+    def forward(self, x, self_attention_mask, start_token, end_token):
+        x = self.sentence_embedding(x, start_token, end_token)
+        x = self.layers(x, self_attention_mask)
+        return x
+
+
+class MultiHeadCrossAttention(nn.Module):
+    def __init__(self, d_model, num_heads):
+        super().__init__()
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.head_dim = d_model // num_heads
+        self.kv_layer = nn.Linear(d_model , 2 * d_model)
+        self.q_layer = nn.Linear(d_model , d_model)
+        self.linear_layer = nn.Linear(d_model, d_model)
+    
+    def forward(self, x, y, mask):
+        batch_size, sequence_length, d_model = x.size() # in practice, this is the same for both languages...so we can technically combine with normal attention
+        kv = self.kv_layer(x)
+        q = self.q_layer(y)
+        kv = kv.reshape(batch_size, sequence_length, self.num_heads, 2 * self.head_dim)
+        q = q.reshape(batch_size, sequence_length, self.num_heads, self.head_dim)
+        kv = kv.permute(0, 2, 1, 3)
+        q = q.permute(0, 2, 1, 3)
+        k, v = kv.chunk(2, dim=-1)
+        values, attention = scaled_dot_product(q, k, v, mask) # We don't need the mask for cross attention, removing in outer function!
+        values = values.permute(0, 2, 1, 3).reshape(batch_size, sequence_length, d_model)
+        out = self.linear_layer(values)
+        return out
+
+
+class DecoderLayer(nn.Module):
+    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):
+        super(DecoderLayer, self).__init__()
+        self.self_attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)
+        self.layer_norm1 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout1 = nn.Dropout(p=drop_prob)
+
+        self.encoder_decoder_attention = MultiHeadCrossAttention(d_model=d_model, num_heads=num_heads)
+        self.layer_norm2 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout2 = nn.Dropout(p=drop_prob)
+
+        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)
+        self.layer_norm3 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout3 = nn.Dropout(p=drop_prob)
+
+    def forward(self, x, y, self_attention_mask, cross_attention_mask):
+        _y = y.clone()
+        y = self.self_attention(y, mask=self_attention_mask)
+        y = self.dropout1(y)
+        y = self.layer_norm1(y + _y)
+
+        _y = y.clone()
+        y = self.encoder_decoder_attention(x, y, mask=cross_attention_mask)
+        y = self.dropout2(y)
+        y = self.layer_norm2(y + _y)
+
+        _y = y.clone()
+        y = self.ffn(y)
+        y = self.dropout3(y)
+        y = self.layer_norm3(y + _y)
+        return y
+
+
+class SequentialDecoder(nn.Sequential):
+    def forward(self, *inputs):
+        x, y, self_attention_mask, cross_attention_mask = inputs
+        for module in self._modules.values():
+            y = module(x, y, self_attention_mask, cross_attention_mask)
+        return y
+
+class Decoder(nn.Module):
+    def __init__(self, 
+                 d_model, 
+                 ffn_hidden, 
+                 num_heads, 
+                 drop_prob, 
+                 num_layers,
+                 max_sequence_length,
+                 language_to_index,
+                 START_TOKEN,
+                 END_TOKEN, 
+                 PADDING_TOKEN):
+        super().__init__()
+        self.sentence_embedding = SentenceEmbedding(max_sequence_length, d_model, language_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN)
+        self.layers = SequentialDecoder(*[DecoderLayer(d_model, ffn_hidden, num_heads, drop_prob) for _ in range(num_layers)])
+
+    def forward(self, x, y, self_attention_mask, cross_attention_mask, start_token, end_token):
+        y = self.sentence_embedding(y, start_token, end_token)
+        y = self.layers(x, y, self_attention_mask, cross_attention_mask)
+        return y
+
+
+class Transformer(nn.Module):
+    def __init__(self, 
+                d_model, 
+                ffn_hidden, 
+                num_heads, 
+                drop_prob, 
+                num_layers,
+                max_sequence_length, 
+                kn_vocab_size,
+                english_to_index,
+                kannada_to_index,
+                START_TOKEN, 
+                END_TOKEN, 
+                PADDING_TOKEN
+                ):
+        super().__init__()
+        self.encoder = Encoder(d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, english_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN)
+        self.decoder = Decoder(d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, kannada_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN)
+        self.linear = nn.Linear(d_model, kn_vocab_size)
+        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+
+    def forward(self, 
+                x, 
+                y, 
+                encoder_self_attention_mask=None, 
+                decoder_self_attention_mask=None, 
+                decoder_cross_attention_mask=None,
+                enc_start_token=False,
+                enc_end_token=False,
+                dec_start_token=False, 
+                dec_end_token=False): 
+        x = self.encoder(x, encoder_self_attention_mask, start_token=enc_start_token, end_token=enc_end_token)
+        out = self.decoder(x, y, decoder_self_attention_mask, decoder_cross_attention_mask, start_token=dec_start_token, end_token=dec_end_token)
+        out = self.linear(out)
+        return out