Hugging Face's logo

stanpony
/
tinylm33M-stella-2sent_7clust-2025-04-05-02-06_full

Text Generation
Transformers
Safetensors
gpt_neo
Model card Files Community
stanpony commited on
Commit
829ea9f
·
verified ·
1 Parent(s): 5ea2258

Upload tokenizer

Browse files
Files changed (6) hide show
  1. added_tokens.json +9 -0
  2. merges.txt +0 -0
  3. special_tokens_map.json +24 -0
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +81 -0
  6. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cluster_0>": 50259,
3
+ "<cluster_1>": 50263,
4
+ "<cluster_2>": 50262,
5
+ "<cluster_3>": 50257,
6
+ "<cluster_4>": 50260,
7
+ "<cluster_5>": 50258,
8
+ "<cluster_6>": 50261
9
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<|endoftext|>",
17
+ "unk_token": {
18
+ "content": "<|endoftext|>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": true,
4
+ "add_prefix_space": false,
5
+ "added_tokens_decoder": {
6
+ "50256": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": true,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "50257": {
15
+ "content": "<cluster_3>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": false
21
+ },
22
+ "50258": {
23
+ "content": "<cluster_5>",
24
+ "lstrip": false,
25
+ "normalized": true,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": false
29
+ },
30
+ "50259": {
31
+ "content": "<cluster_0>",
32
+ "lstrip": false,
33
+ "normalized": true,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": false
37
+ },
38
+ "50260": {
39
+ "content": "<cluster_4>",
40
+ "lstrip": false,
41
+ "normalized": true,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": false
45
+ },
46
+ "50261": {
47
+ "content": "<cluster_6>",
48
+ "lstrip": false,
49
+ "normalized": true,
50
+ "rstrip": false,
51
+ "single_word": false,
52
+ "special": false
53
+ },
54
+ "50262": {
55
+ "content": "<cluster_2>",
56
+ "lstrip": false,
57
+ "normalized": true,
58
+ "rstrip": false,
59
+ "single_word": false,
60
+ "special": false
61
+ },
62
+ "50263": {
63
+ "content": "<cluster_1>",
64
+ "lstrip": false,
65
+ "normalized": true,
66
+ "rstrip": false,
67
+ "single_word": false,
68
+ "special": false
69
+ }
70
+ },
71
+ "bos_token": "<|endoftext|>",
72
+ "clean_up_tokenization_spaces": false,
73
+ "eos_token": "<|endoftext|>",
74
+ "errors": "replace",
75
+ "extra_special_tokens": {},
76
+ "model_max_length": 2048,
77
+ "pad_token": "<|endoftext|>",
78
+ "padding_side": "right",
79
+ "tokenizer_class": "GPT2Tokenizer",
80
+ "unk_token": "<|endoftext|>"
81
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff