add some initial vocab files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+RWKV/ggml-vocab-rwkv-7-world.gguf filter=lfs diff=lfs merge=lfs -text
+SPM/ggml-vocab-gemma-3.gguf filter=lfs diff=lfs merge=lfs -text
+WPM/ggml-vocab-jina-v2-en.gguf filter=lfs diff=lfs merge=lfs -text

RWKV/ggml-vocab-rwkv-7-world.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46104c6e917c7c954b3348f865b992d56e012e8799c57d69968e435239b7d86
+size 1365451

RWKV/ggml-vocab-rwkv-7-world.gguf.inp

@@ -0,0 +1,112 @@
+ied 4 ½ months
+__ggml_vocab_test__
+Äpfel
+__ggml_vocab_test__
+
+__ggml_vocab_test__
+ 
+__ggml_vocab_test__
+  
+__ggml_vocab_test__
+   
+__ggml_vocab_test__
+	
+__ggml_vocab_test__
+
+
+__ggml_vocab_test__
+
+
+
+__ggml_vocab_test__
+
+
+
+
+__ggml_vocab_test__
+	
+
+__ggml_vocab_test__
+Hello world
+__ggml_vocab_test__
+ Hello world
+__ggml_vocab_test__
+Hello World
+__ggml_vocab_test__
+ Hello World
+__ggml_vocab_test__
+ Hello World!
+__ggml_vocab_test__
+Hello, world!
+__ggml_vocab_test__
+ Hello, world!
+__ggml_vocab_test__
+ this is 🦙.cpp
+__ggml_vocab_test__
+w048 7tuijk dsdfhu
+__ggml_vocab_test__
+нещо на Български
+__ggml_vocab_test__
+កាន់តែពិសេសអាចខលចេញ
+__ggml_vocab_test__
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)
+__ggml_vocab_test__
+Hello
+__ggml_vocab_test__
+ Hello
+__ggml_vocab_test__
+  Hello
+__ggml_vocab_test__
+   Hello
+__ggml_vocab_test__
+    Hello
+__ggml_vocab_test__
+    Hello
+    Hello
+__ggml_vocab_test__
+ (
+__ggml_vocab_test__
+
+ =
+__ggml_vocab_test__
+' era
+__ggml_vocab_test__
+Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
+__ggml_vocab_test__
+!!!!!!
+__ggml_vocab_test__
+3
+__ggml_vocab_test__
+33
+__ggml_vocab_test__
+333
+__ggml_vocab_test__
+3333
+__ggml_vocab_test__
+33333
+__ggml_vocab_test__
+333333
+__ggml_vocab_test__
+3333333
+__ggml_vocab_test__
+33333333
+__ggml_vocab_test__
+333333333
+__ggml_vocab_test__
+Cửa Việt
+__ggml_vocab_test__
+ discards
+__ggml_vocab_test__
+
+ 
+
+ 
+
+
+ 	 		 	
+  
+   
+    
+     
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL
+__ggml_vocab_test__

RWKV/ggml-vocab-rwkv-7-world.gguf.out

@@ -0,0 +1,46 @@
+ 7779 287 4935 46596
+ 2466 2116 1851
+
+ 33
+ 267
+ 3340
+ 10
+ 11
+ 65530
+ 65530 11
+ 258
+ 33155 40213
+ 36786 40213
+ 33155 37576
+ 36786 37576
+ 36786 37576 34
+ 33155 45 40213 34
+ 36786 45 40213 34
+ 32234 4600 33 3319 167 154 47 7282
+ 120 624 57 290 2220 7819 4495 1819 1933
+ 28007 28136 32739 5063 61263 43081
+ 9704 9708 9705 3000 140 2999 144 3000 131 2999 151 2999 184 9707 3000 130 9707 2999 163 9708 2999 134 2999 130 2999 156 2999 134 3000 130 2999 138
+ 3319 155 129 275 41894 42 33 3319 153 183 9810 3319 141 172 19103 275 54156 38362 116 64299 42 33 10056 275 26476 38362 32227 21800 21902 22251 39974 42
+ 33155
+ 36786
+ 267 33155
+ 3340 33155
+ 19250 33155
+ 19250 33155 28352 33155
+ 275
+ 262 62
+ 40 21626
+ 33155 45 356 40 6984 34 20063 21286 22799 33 3319 153 130 298 12605 12509 11454 33839 12137 10460 633 634 635 50 11639 19219
+ 23248 362
+ 52
+ 653
+ 653 52
+ 653 653
+ 653 653 52
+ 653 653 653
+ 653 653 653 52
+ 653 653 653 653
+ 653 653 653 653 52
+ 68 28299 45034
+ 51480 116
+ 262 65530 33 65530 262 259 3322 258 36215 54806 3339 3319 155 129 275 41894 42 33 3319 153 183 9810 3319 141 172 19103 275 54156 38362 116 64299 42 33 10056 33 3319 167 154 3319 167 154 286 3506 3506 52 3506 653 3506 653 52 3506 653 653 3506 653 653 52 3506 653 653 653 286 47 52 286 585 52 286 5362 52 33 9704 9708 9705 3000 140 2999 144 3000 131 2999 151 2999 184 9707 3000 130 9707 2999 163 9708 2999 134 3319 153 130 298 12605 12509 11454 33839 12137 10460 633 634 635 50 11639 19219 43915 48344 32740 28136 32739 5063 61263 43081 19279 5205 6884 6884 97 23252 40269 23248 362 23349 807 308 5231 30259 274 8826 101 4569 460 39934 45 274 1397 22799 32157 64 274 78 22187 32157 308 5229 31358 4601 45 274 69 22799 31296 32074 22575 64 4331 40 1560 332 40 109 77

SPM/ggml-vocab-gemma-3.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d854494379ec3133f08de356ae37c7ad2c8a0cdcd23ded05e90eeee2c79924
+size 6512800

SPM/ggml-vocab-gemma-3.gguf.inp

@@ -0,0 +1,112 @@
+ied 4 ½ months
+__ggml_vocab_test__
+Äpfel
+__ggml_vocab_test__
+
+__ggml_vocab_test__
+ 
+__ggml_vocab_test__
+  
+__ggml_vocab_test__
+   
+__ggml_vocab_test__
+	
+__ggml_vocab_test__
+
+
+__ggml_vocab_test__
+
+
+
+__ggml_vocab_test__
+
+
+
+
+__ggml_vocab_test__
+	
+
+__ggml_vocab_test__
+Hello world
+__ggml_vocab_test__
+ Hello world
+__ggml_vocab_test__
+Hello World
+__ggml_vocab_test__
+ Hello World
+__ggml_vocab_test__
+ Hello World!
+__ggml_vocab_test__
+Hello, world!
+__ggml_vocab_test__
+ Hello, world!
+__ggml_vocab_test__
+ this is 🦙.cpp
+__ggml_vocab_test__
+w048 7tuijk dsdfhu
+__ggml_vocab_test__
+нещо на Български
+__ggml_vocab_test__
+កាន់តែពិសេសអាចខលចេញ
+__ggml_vocab_test__
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)
+__ggml_vocab_test__
+Hello
+__ggml_vocab_test__
+ Hello
+__ggml_vocab_test__
+  Hello
+__ggml_vocab_test__
+   Hello
+__ggml_vocab_test__
+    Hello
+__ggml_vocab_test__
+    Hello
+    Hello
+__ggml_vocab_test__
+ (
+__ggml_vocab_test__
+
+ =
+__ggml_vocab_test__
+' era
+__ggml_vocab_test__
+Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
+__ggml_vocab_test__
+!!!!!!
+__ggml_vocab_test__
+3
+__ggml_vocab_test__
+33
+__ggml_vocab_test__
+333
+__ggml_vocab_test__
+3333
+__ggml_vocab_test__
+33333
+__ggml_vocab_test__
+333333
+__ggml_vocab_test__
+3333333
+__ggml_vocab_test__
+33333333
+__ggml_vocab_test__
+333333333
+__ggml_vocab_test__
+Cửa Việt
+__ggml_vocab_test__
+ discards
+__ggml_vocab_test__
+
+ 
+
+ 
+
+
+ 	 		 	
+  
+   
+    
+     
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL
+__ggml_vocab_test__

SPM/ggml-vocab-gemma-3.gguf.out

@@ -0,0 +1,46 @@
+ 1178 236743 236812 47041 3794
+ 239122 22744 535
+
+ 236743
+ 138
+ 139
+ 255968
+ 107
+ 108
+ 109
+ 255968 107
+ 9259 1902
+ 26352 1902
+ 9259 4109
+ 26352 4109
+ 26352 4109 236888
+ 9259 236764 1902 236888
+ 26352 236764 1902 236888
+ 672 563 236743 478 397 404 391 236761 12362
+ 236765 236771 236812 236828 236743 236832 11372 12065 31806 3405 9360
+ 1337 12515 1333 4632 165543 3830
+ 234889 63031 219876 66212 239077 237907 144494
+ 242015 568 7382 236768 236743 247717 237243 248989 238178 568 43819 111730 150567 236768 113452 568 8960 64334 600 815 1061 1852 8369 236768
+ 9259
+ 26352
+ 138 9259
+ 139 9259
+ 140 9259
+ 140 9259 107 140 9259
+ 568
+ 107 578
+ 236789 6933
+ 9259 236764 570 236789 712 236888 2088 659 611 170124 2360 62133 237075 17641 11700 236770 236800 236770 236812 236770 236810 236770 237471 238352
+ 123947
+ 236800
+ 236800 236800
+ 236800 236800 236800
+ 236800 236800 236800 236800
+ 236800 236800 236800 236800 236800
+ 236800 236800 236800 236800 236800 236800
+ 236800 236800 236800 236800 236800 236800 236800
+ 236800 236800 236800 236800 236800 236800 236800 236800
+ 236800 236800 236800 236800 236800 236800 236800 236800 236800
+ 236780 29719 33154
+ 2243 2206
+ 107 236743 108 236743 109 236743 255968 236743 255969 236743 255968 107 138 107 139 107 140 107 141 107 242015 568 7382 236768 236743 247717 237243 248989 238178 568 43819 111730 150567 236768 113452 236743 478 397 404 391 478 397 404 391 236743 236800 236743 236800 236800 236743 236800 236800 236800 236743 236800 236800 236800 236800 236743 236800 236800 236800 236800 236800 236743 236800 236800 236800 236800 236800 236800 236743 236800 236800 236800 236800 236800 236800 236800 236743 236800 236800 236800 236800 236800 236800 236800 236800 236743 236800 236761 236800 236743 236800 856 236800 236743 236800 1390 236800 90986 92814 63031 219876 66212 241702 2360 62133 237075 17641 11700 236770 236800 236770 236812 236770 236810 236770 237471 238352 80448 120697 210119 1333 4632 165543 3830 9451 159561 2629 2629 2717 84491 19938 123947 38950 10371 564 236789 560 1010 756 151812 668 236789 236751 993 236764 756 1357 611 2889 236881 756 236792 711 2889 564 236789 859 1386 625 236764 756 236796 611 1133 1070 11115 236881 1191 236789 32541 496 236789 95635

WPM/ggml-vocab-jina-v2-en.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5124151a3c25b256a56d1bf112dbe3ca878ee99b1649ae270bbbbc52b1bd5f98
+size 627966

WPM/ggml-vocab-jina-v2-en.gguf.inp

@@ -0,0 +1,112 @@
+ied 4 ½ months
+__ggml_vocab_test__
+Äpfel
+__ggml_vocab_test__
+
+__ggml_vocab_test__
+ 
+__ggml_vocab_test__
+  
+__ggml_vocab_test__
+   
+__ggml_vocab_test__
+	
+__ggml_vocab_test__
+
+
+__ggml_vocab_test__
+
+
+
+__ggml_vocab_test__
+
+
+
+
+__ggml_vocab_test__
+	
+
+__ggml_vocab_test__
+Hello world
+__ggml_vocab_test__
+ Hello world
+__ggml_vocab_test__
+Hello World
+__ggml_vocab_test__
+ Hello World
+__ggml_vocab_test__
+ Hello World!
+__ggml_vocab_test__
+Hello, world!
+__ggml_vocab_test__
+ Hello, world!
+__ggml_vocab_test__
+ this is 🦙.cpp
+__ggml_vocab_test__
+w048 7tuijk dsdfhu
+__ggml_vocab_test__
+нещо на Български
+__ggml_vocab_test__
+កាន់តែពិសេសអាចខលចេញ
+__ggml_vocab_test__
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)
+__ggml_vocab_test__
+Hello
+__ggml_vocab_test__
+ Hello
+__ggml_vocab_test__
+  Hello
+__ggml_vocab_test__
+   Hello
+__ggml_vocab_test__
+    Hello
+__ggml_vocab_test__
+    Hello
+    Hello
+__ggml_vocab_test__
+ (
+__ggml_vocab_test__
+
+ =
+__ggml_vocab_test__
+' era
+__ggml_vocab_test__
+Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
+__ggml_vocab_test__
+!!!!!!
+__ggml_vocab_test__
+3
+__ggml_vocab_test__
+33
+__ggml_vocab_test__
+333
+__ggml_vocab_test__
+3333
+__ggml_vocab_test__
+33333
+__ggml_vocab_test__
+333333
+__ggml_vocab_test__
+3333333
+__ggml_vocab_test__
+33333333
+__ggml_vocab_test__
+333333333
+__ggml_vocab_test__
+Cửa Việt
+__ggml_vocab_test__
+ discards
+__ggml_vocab_test__
+
+ 
+
+ 
+
+
+ 	 		 	
+  
+   
+    
+     
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL
+__ggml_vocab_test__

WPM/ggml-vocab-jina-v2-en.gguf.out

@@ -0,0 +1,46 @@
+ 29464 2094 1018 1092 2706
+ 9706 7959 2140
+
+
+
+
+
+
+
+
+
+ 7592 2088
+ 7592 2088
+ 7592 2088
+ 7592 2088
+ 7592 2088 999
+ 7592 1010 2088 999
+ 7592 1010 2088 999
+ 2023 2003 100 1012 18133 2361
+ 1059 2692 18139 1021 8525 28418 2243 16233 20952 6979
+ 1192 15290 29754 14150 1192 10260 1181 29755 29436 29741 10260 16856 29747 23925 10325
+ 100
+ 100 1006 3671 1007 100 1006 3674 7861 29147 2483 9530 16280 23854 1007 100 1006 2069 7861 29147 2072 2008 2038 2049 2219 19204 1007
+ 7592
+ 7592
+ 7592
+ 7592
+ 7592
+ 7592 7592
+ 1006
+ 1027
+ 1005 3690
+ 7592 1010 1061 1005 2035 999 2129 2024 2017 100 1029 1855 100 100 6207 100 100 14677 23632 22203 1811 1995
+ 999 999 999 999 999 999
+ 1017
+ 3943
+ 21211
+ 21211 2509
+ 21211 22394
+ 21211 22394 2509
+ 21211 22394 22394
+ 21211 22394 22394 2509
+ 21211 22394 22394 22394
+ 12731 2050 19710
+ 5860 18117
+ 100 1006 3671 1007 100 1006 3674 7861 29147 2483 9530 16280 23854 1007 100 100 1017 3943 21211 21211 2509 21211 22394 21211 22394 2509 21211 22394 22394 21211 22394 22394 2509 1017 1012 1017 1017 1012 1012 1017 1017 1012 1012 1012 1017 100 1029 1855 100 100 6207 100 100 14677 23632 22203 1811 1995 1011 1011 1011 1011 1011 1011 1027 1027 1027 1027 1027 1027 1027 1192 15290 29754 14150 1192 10260 1181 29755 29436 29741 10260 16856 29747 23925 10325 1005 1005 1005 1005 1005 1005 1036 1036 1036 1036 1036 1036 1036 1000 1000 1000 1000 1012 1012 1012 1012 1012 1012 999 999 999 999 999 999 1029 1029 1029 1029 1029 1029 1045 1005 2310 2042 1005 2409 2002 1005 1055 2045 1010 1005 2128 2017 2469 1029 1005 1049 2025 2469 1045 1005 2222 2191 2009 1010 1005 1040 2017 2066 2070 5572 1029 2057 1005 2310 1037 1005 2222