vocab corpus increased - 300000

app.py

@@ -13,7 +13,7 @@ class DecodeRequest(BaseModel):
     tokens: str
 
 # Initialize the tokenizer
-tokenizer = BPEGujaratiTokenizer(corpus_path="gu_corpus.txt", max_vocab_size=5000, sample_size=50000)
+tokenizer = BPEGujaratiTokenizer(corpus_path="gu_corpus.txt", max_vocab_size=5000, sample_size=300000)
 
 app = FastAPI()
 

encoder.py

@@ -42,6 +42,7 @@ class BPEGujaratiTokenizer:
 
     def train_bpe(self, corpus, max_vocab_size, sample_size=None):
         self.vocab = {idx: bytes([idx]) for idx in range(256)}
+        print(f"Before training: vocab length: {len(self.vocab)}")
         if sample_size :
             corpus = corpus[:sample_size]
         num_merges = max_vocab_size - len(self.vocab)
@@ -66,6 +67,7 @@ class BPEGujaratiTokenizer:
         print(f"After training: tokens length: {len(tokens)}")
         print("After training: merges length: ", len(self.merges))
         print(f"compression ratio: {len(tokens) / len(ids):.2f}X")
+        print(f"After training: vocab length: {len(self.vocab)}")
         return self.vocab, self.merges
 
     def encode(self, text):
@@ -88,7 +90,7 @@ class BPEGujaratiTokenizer:
 import time
 if __name__ == "__main__":
     start_time = time.time()
-    tokenizer = BPEGujaratiTokenizer(corpus_path="gu_corpus.txt", max_vocab_size=5000, sample_size=20000)
+    tokenizer = BPEGujaratiTokenizer(corpus_path="gu_corpus.txt", max_vocab_size=5000, sample_size=300000)
     end_time = time.time()
     print(f"Time taken to train: {end_time - start_time} seconds")
     print("--------------------------------")