Hugging Face's logo

karina-zadorozhny
/
amino_acid_tokenizer

Model card Files Community
karina-zadorozhny commited on
Commit
97967c0
·
verified ·
1 Parent(s): 937dcfb

Delete tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +0 -176
tokenizer.json DELETED
@@ -1,176 +0,0 @@
1
- {
2
- "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
5
- "added_tokens": [
6
- {
7
- "id": 0,
8
- "content": "<cls>",
9
- "single_word": false,
10
- "lstrip": false,
11
- "rstrip": false,
12
- "normalized": false,
13
- "special": true
14
- },
15
- {
16
- "id": 1,
17
- "content": "<pad>",
18
- "single_word": false,
19
- "lstrip": false,
20
- "rstrip": false,
21
- "normalized": false,
22
- "special": true
23
- },
24
- {
25
- "id": 2,
26
- "content": "<eos>",
27
- "single_word": false,
28
- "lstrip": false,
29
- "rstrip": false,
30
- "normalized": false,
31
- "special": true
32
- },
33
- {
34
- "id": 3,
35
- "content": "<unk>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 32,
44
- "content": "<mask>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- }
51
- ],
52
- "normalizer": null,
53
- "pre_tokenizer": null,
54
- "post_processor": {
55
- "type": "TemplateProcessing",
56
- "single": [
57
- {
58
- "SpecialToken": {
59
- "id": "<cls>",
60
- "type_id": 0
61
- }
62
- },
63
- {
64
- "Sequence": {
65
- "id": "A",
66
- "type_id": 0
67
- }
68
- },
69
- {
70
- "SpecialToken": {
71
- "id": "<eos>",
72
- "type_id": 0
73
- }
74
- }
75
- ],
76
- "pair": [
77
- {
78
- "SpecialToken": {
79
- "id": "<cls>",
80
- "type_id": 0
81
- }
82
- },
83
- {
84
- "Sequence": {
85
- "id": "A",
86
- "type_id": 0
87
- }
88
- },
89
- {
90
- "SpecialToken": {
91
- "id": "<eos>",
92
- "type_id": 0
93
- }
94
- },
95
- {
96
- "Sequence": {
97
- "id": "B",
98
- "type_id": 1
99
- }
100
- },
101
- {
102
- "SpecialToken": {
103
- "id": "<eos>",
104
- "type_id": 1
105
- }
106
- }
107
- ],
108
- "special_tokens": {
109
- "<cls>": {
110
- "id": "<cls>",
111
- "ids": [
112
- 0
113
- ],
114
- "tokens": [
115
- "<cls>"
116
- ]
117
- },
118
- "<eos>": {
119
- "id": "<eos>",
120
- "ids": [
121
- 2
122
- ],
123
- "tokens": [
124
- "<eos>"
125
- ]
126
- }
127
- }
128
- },
129
- "decoder": null,
130
- "model": {
131
- "type": "BPE",
132
- "dropout": null,
133
- "unk_token": "<unk>",
134
- "continuing_subword_prefix": null,
135
- "end_of_word_suffix": null,
136
- "fuse_unk": false,
137
- "byte_fallback": false,
138
- "ignore_merges": true,
139
- "vocab": {
140
- "<cls>": 0,
141
- "<pad>": 1,
142
- "<eos>": 2,
143
- "<unk>": 3,
144
- "L": 4,
145
- "A": 5,
146
- "G": 6,
147
- "V": 7,
148
- "S": 8,
149
- "E": 9,
150
- "R": 10,
151
- "T": 11,
152
- "I": 12,
153
- "D": 13,
154
- "P": 14,
155
- "K": 15,
156
- "Q": 16,
157
- "N": 17,
158
- "F": 18,
159
- "Y": 19,
160
- "M": 20,
161
- "H": 21,
162
- "W": 22,
163
- "C": 23,
164
- "X": 24,
165
- "B": 25,
166
- "U": 26,
167
- "Z": 27,
168
- "O": 28,
169
- ".": 29,
170
- "-": 30,
171
- "<null_1>": 31,
172
- "<mask>": 32
173
- },
174
- "merges": []
175
- }
176
- }