HuyenNguyen commited on
Commit
84eaad5
·
1 Parent(s): 3648845

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -2
  2. tokenizer_config.json +5 -5
  3. vocab.json +32 -86
added_tokens.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "</s>": 92,
3
- "<s>": 91
4
  }
 
1
  {
2
+ "</s>": 38,
3
+ "<s>": 37
4
  }
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "added_tokens_decoder": {
3
- "89": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
@@ -8,7 +8,7 @@
8
  "single_word": false,
9
  "special": false
10
  },
11
- "90": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
@@ -16,7 +16,7 @@
16
  "single_word": false,
17
  "special": false
18
  },
19
- "91": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": true,
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "92": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": true,
@@ -40,7 +40,7 @@
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
42
  "replace_word_delimiter_char": " ",
43
- "target_lang": "vi",
44
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
45
  "unk_token": "[UNK]",
46
  "word_delimiter_token": "|"
 
1
  {
2
  "added_tokens_decoder": {
3
+ "35": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
 
8
  "single_word": false,
9
  "special": false
10
  },
11
+ "36": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
 
16
  "single_word": false,
17
  "special": false
18
  },
19
+ "37": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": true,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "38": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": true,
 
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
42
  "replace_word_delimiter_char": " ",
43
+ "target_lang": "tur",
44
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
45
  "unk_token": "[UNK]",
46
  "word_delimiter_token": "|"
vocab.json CHANGED
@@ -1,95 +1,41 @@
1
  {
2
- "vi": {
3
- "[PAD]": 90,
4
- "[UNK]": 89,
5
  "a": 1,
6
  "b": 2,
7
  "c": 3,
8
  "d": 4,
9
  "e": 5,
10
- "g": 6,
11
- "h": 7,
12
- "i": 8,
13
- "k": 9,
14
- "l": 10,
15
- "m": 11,
16
- "n": 12,
17
- "o": 13,
18
- "p": 14,
19
- "q": 15,
20
- "r": 16,
21
- "s": 17,
22
- "t": 18,
23
- "u": 19,
24
- "v": 20,
25
- "x": 21,
26
- "y": 22,
 
 
 
 
27
  "|": 0,
28
- "à": 23,
29
- "á": 24,
30
- "â": 25,
31
- "ã": 26,
32
- "è": 27,
33
- "é": 28,
34
- "ê": 29,
35
- "ì": 30,
36
- "í": 31,
37
- "ò": 32,
38
- "ó": 33,
39
- "ô": 34,
40
- "õ": 35,
41
- "ù": 36,
42
- "ú": 37,
43
- "ý": 38,
44
- "ă": 39,
45
- "đ": 40,
46
- "ĩ": 41,
47
- "ũ": 42,
48
- "ơ": 43,
49
- "ư": 44,
50
- "ạ": 45,
51
- "ả": 46,
52
- "ấ": 47,
53
- "ầ": 48,
54
- "ẩ": 49,
55
- "ẫ": 50,
56
- "ậ": 51,
57
- "ắ": 52,
58
- "ằ": 53,
59
- "ẳ": 54,
60
- "ẵ": 55,
61
- "ặ": 56,
62
- "ẹ": 57,
63
- "ẻ": 58,
64
- "ẽ": 59,
65
- "ế": 60,
66
- "ề": 61,
67
- "ể": 62,
68
- "ễ": 63,
69
- "ệ": 64,
70
- "ỉ": 65,
71
- "ị": 66,
72
- "ọ": 67,
73
- "ỏ": 68,
74
- "ố": 69,
75
- "ồ": 70,
76
- "ổ": 71,
77
- "ỗ": 72,
78
- "ộ": 73,
79
- "ớ": 74,
80
- "ờ": 75,
81
- "ở": 76,
82
- "ỡ": 77,
83
- "ợ": 78,
84
- "ụ": 79,
85
- "ủ": 80,
86
- "ứ": 81,
87
- "ừ": 82,
88
- "ử": 83,
89
- "ữ": 84,
90
- "ự": 85,
91
- "ỳ": 86,
92
- "ỷ": 87,
93
- "ỹ": 88
94
  }
95
  }
 
1
  {
2
+ "tur": {
3
+ "[PAD]": 36,
4
+ "[UNK]": 35,
5
  "a": 1,
6
  "b": 2,
7
  "c": 3,
8
  "d": 4,
9
  "e": 5,
10
+ "f": 6,
11
+ "g": 7,
12
+ "h": 8,
13
+ "i": 9,
14
+ "j": 10,
15
+ "k": 11,
16
+ "l": 12,
17
+ "m": 13,
18
+ "n": 14,
19
+ "o": 15,
20
+ "p": 16,
21
+ "q": 17,
22
+ "r": 18,
23
+ "s": 19,
24
+ "t": 20,
25
+ "u": 21,
26
+ "v": 22,
27
+ "w": 23,
28
+ "x": 24,
29
+ "y": 25,
30
+ "z": 26,
31
  "|": 0,
32
+ "ç": 27,
33
+ "ë": 28,
34
+ "ö": 29,
35
+ "ü": 30,
36
+ "ğ": 31,
37
+ "ı": 32,
38
+ "ş": 33,
39
+ "̇": 34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  }
41
  }