READ ME update

app.py

@@ -13,7 +13,7 @@ class DecodeRequest(BaseModel):
     tokens: str
 
 # Initialize the tokenizer
-tokenizer = BPEGujaratiTokenizer(corpus_path="gu_corpus.txt", max_vocab_size=5000, sample_size=150000)
+tokenizer = BPEGujaratiTokenizer(corpus_path="gu_corpus.txt", max_vocab_size=5000, sample_size=300000)
 
 app = FastAPI()
 

indi-lang.ipynb

@@ -109,7 +109,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -118,15 +118,15 @@
      "text": [
       "Before Training Vocab length 256\n",
       "num_merges required 4744\n",
-      "Before training: ids length: 379218\n",
-      "Before training: tokens length: 379218\n",
+      "Before training: ids length: 755940\n",
+      "Before training: tokens length: 755940\n",
       "Before training: merges length:  0\n",
-      "After training: ids length: 36290\n",
-      "After training: tokens length: 379218\n",
+      "After training: ids length: 76306\n",
+      "After training: tokens length: 755940\n",
       "After training: merges length:  4744\n",
       "After Training Vocab length 5000\n",
-      "compression ratio: 10.45X\n",
-      "Time taken to train: 96.17453122138977 seconds\n",
+      "compression ratio: 9.91X\n",
+      "Time taken to train: 199.02717900276184 seconds\n",
       "--------------------------------\n"
      ]
     }
@@ -135,7 +135,7 @@
     "import time\n",
     "\n",
     "start_time = time.time()\n",
-    "tokenizer = BPEGujaratiTokenizer(corpus_path=\"gu_corpus.txt\", max_vocab_size=5000, sample_size=150000)\n",
+    "tokenizer = BPEGujaratiTokenizer(corpus_path=\"gu_corpus.txt\", max_vocab_size=5000, sample_size=300000)\n",
     "end_time = time.time()\n",
     "print(f\"Time taken to train: {end_time - start_time} seconds\")\n",
     "print(\"--------------------------------\")\n"
@@ -143,64 +143,64 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[292, 310, 164, 290, 363, 329, 325, 310, 155, 600]\n",
-      "Time taken to encode: 0.0006651878356933594 seconds\n",
+      "[294, 307, 164, 292, 431, 325, 317, 3229, 444]\n",
+      "Time taken to encode: 0.0007619857788085938 seconds\n",
       "--------------------------------\n",
       "હું તને પ્રેમ કરું છું\n",
-      "Time taken to decode: 0.0004611015319824219 seconds\n",
+      "Time taken to decode: 0.0004019737243652344 seconds\n",
       "--------------------------------\n",
       "original:  હું આજે ખૂબ ખુશ છું.\n",
-      "encoded:  [292, 310, 1987, 150, 314, 172, 1804, 503, 600, 46]\n",
+      "encoded:  [294, 307, 1414, 853, 928, 1793, 482, 444, 46]\n",
       "decoded:  હું આજે ખૂબ ખુશ છું.\n",
       "True\n",
       "original:  તું શું કરે છે? \n",
-      "encoded:  [279, 1700, 310, 412, 267, 155, 260, 63, 32]\n",
+      "encoded:  [3519, 182, 307, 391, 4339, 63, 32]\n",
       "decoded:  તું શું કરે છે? \n",
       "True\n",
       "original:  મને ચા પીવી છે. \n",
-      "encoded:  [274, 290, 154, 553, 549, 269, 155, 260, 46, 32]\n",
+      "encoded:  [274, 292, 154, 758, 519, 269, 296, 46, 32]\n",
       "decoded:  મને ચા પીવી છે. \n",
       "True\n",
       "original:  એ બધું સરસ છે. \n",
-      "encoded:  [479, 334, 343, 310, 184, 1538, 503, 260, 46, 32]\n",
+      "encoded:  [512, 4222, 3997, 2296, 3648, 46, 32]\n",
       "decoded:  એ બધું સરસ છે. \n",
       "True\n",
       "original:  આ પુસ્તક ખૂબ રસપ્રદ છે. \n",
-      "encoded:  [256, 134, 298, 280, 437, 294, 1990, 172, 316, 326, 1308, 361, 503, 260, 46, 32]\n",
+      "encoded:  [256, 4844, 2469, 290, 3227, 311, 4738, 345, 3648, 46, 32]\n",
       "decoded:  આ પુસ્તક ખૂબ રસપ્રદ છે. \n",
       "True\n",
       "original:  તારે ક્યારે આવવું છે? \n",
-      "encoded:  [279, 344, 149, 482, 347, 1691, 155, 260, 63, 32]\n",
+      "encoded:  [2460, 335, 484, 340, 793, 296, 63, 32]\n",
       "decoded:  તારે ક્યારે આવવું છે? \n",
       "True\n",
       "original:  આ મારો મિત્ર છે. \n",
-      "encoded:  [256, 134, 1803, 283, 174, 366, 288, 503, 260, 46, 32]\n",
+      "encoded:  [256, 134, 309, 763, 4071, 3648, 46, 32]\n",
       "decoded:  આ મારો મિત્ર છે. \n",
       "True\n",
       "original:  હું શાકભાજી લઈ આવ્યો છું. \n",
-      "encoded:  [292, 1700, 621, 418, 429, 1527, 388, 788, 413, 155, 600, 46, 32]\n",
+      "encoded:  [294, 307, 182, 533, 455, 397, 666, 451, 655, 2301, 444, 46, 32]\n",
       "decoded:  હું શાકભાજી લઈ આવ્યો છું. \n",
       "True\n",
       "original:  આકાશ માં વાદળ છે. \n",
-      "encoded:  [256, 134, 294, 1089, 307, 285, 181, 405, 345, 503, 260, 46, 32]\n",
+      "encoded:  [256, 134, 290, 676, 1546, 181, 390, 343, 3648, 46, 32]\n",
       "decoded:  આકાશ માં વાદળ છે. \n",
       "True\n",
       "original:  શાળા ક્યારે ��રૂ થશે? \n",
-      "encoded:  [330, 888, 391, 482, 182, 268, 1248, 165, 330, 260, 63, 32]\n",
+      "encoded:  [332, 547, 581, 484, 3680, 165, 1168, 63, 32]\n",
       "decoded:  શાળા ક્યારે શરૂ થશે? \n",
       "True\n",
       "original:  આ પુસ્તક ખૂબ રસપ્રદ છે.\n",
-      "encoded:  [256, 134, 298, 280, 437, 294, 1990, 172, 316, 326, 1308, 361, 503, 260, 46]\n",
+      "encoded:  [256, 4844, 2469, 290, 3227, 311, 4738, 345, 3648, 46]\n",
       "decoded:  આ પુસ્તક ખૂબ રસપ્રદ છે.\n",
       "True\n",
-      "Time taken to decode: 0.009427070617675781 seconds\n",
+      "Time taken to decode: 0.009686946868896484 seconds\n",
       "--------------------------------\n"
      ]
     }