Chenghao-Qiu commited on
Commit
3ee2f31
·
verified ·
1 Parent(s): 70de2fc

Upload folder using huggingface_hub

Browse files
config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "pGenomeOcean/GenomeOcean-100M",
3
+ "architectures": [
4
+ "MistralForSequenceClassification"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "pGenomeOcean/GenomeOcean-100M--configuration_mistral.MistralConfig",
9
+ "AutoModel": "pGenomeOcean/GenomeOcean-100M--modeling_mistral.MistralModel",
10
+ "AutoModelForCausalLM": "pGenomeOcean/GenomeOcean-100M--modeling_mistral.MistralForCausalLM",
11
+ "AutoModelForMaskedLM": "pGenomeOcean/GenomeOcean-100M--modeling_mistral.MistralForMaskedLM",
12
+ "AutoModelForSequenceClassification": "pGenomeOcean/GenomeOcean-100M--modeling_mistral.MistralForSequenceClassification"
13
+ },
14
+ "bos_token_id": 1,
15
+ "classifier_dropout": 0.1,
16
+ "eos_token_id": 2,
17
+ "hidden_act": "silu",
18
+ "hidden_size": 768,
19
+ "initializer_range": 0.02,
20
+ "intermediate_size": 3072,
21
+ "is_causal": true,
22
+ "max_position_embeddings": 32768,
23
+ "model_type": "mistral",
24
+ "num_attention_heads": 8,
25
+ "num_hidden_layers": 12,
26
+ "num_key_value_heads": 8,
27
+ "output_router_logits": false,
28
+ "pad_token_id": 3,
29
+ "problem_type": "single_label_classification",
30
+ "rms_norm_eps": 1e-05,
31
+ "rope_theta": 1000000.0,
32
+ "router_aux_loss_coef": 0.02,
33
+ "sliding_window": null,
34
+ "tie_word_embeddings": false,
35
+ "torch_dtype": "float32",
36
+ "transformers_version": "4.45.2",
37
+ "use_cache": true,
38
+ "vocab_size": 4096
39
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a2a513c24d9eb874f36600423a45fe0feb9e4f7877f5e097fe449f8f3470cb0
3
+ size 465662960
optimizer_state_dict.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33e81476fc21e91ac44ee8d6dcef26aaac377e0e51083154daac7fea0cd3be5e
3
+ size 931398901
special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": {
3
+ "content": "[CLS]",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "mask_token": {
10
+ "content": "[MASK]",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "[SEP]",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "[UNK]",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[UNK]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "[CLS]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "[SEP]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "[PAD]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "clean_up_tokenization_spaces": true,
45
+ "cls_token": "[CLS]",
46
+ "mask_token": "[MASK]",
47
+ "model_max_length": 256,
48
+ "pad_token": "[PAD]",
49
+ "padding_side": "right",
50
+ "sep_token": "[SEP]",
51
+ "tokenizer_class": "PreTrainedTokenizerFast",
52
+ "unk_token": "[UNK]"
53
+ }
trainer_state.json ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.11086247861385345,
3
+ "best_model_checkpoint": "output_pipe/prom_300_notata/origin/checkpoint-1000",
4
+ "epoch": 4.0,
5
+ "eval_steps": 200,
6
+ "global_step": 2656,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.15060240963855423,
13
+ "grad_norm": 22.862253189086914,
14
+ "learning_rate": 2.9470452801227934e-05,
15
+ "loss": 0.3165,
16
+ "step": 100
17
+ },
18
+ {
19
+ "epoch": 0.30120481927710846,
20
+ "grad_norm": 15.997178077697754,
21
+ "learning_rate": 2.831926323867997e-05,
22
+ "loss": 0.174,
23
+ "step": 200
24
+ },
25
+ {
26
+ "epoch": 0.30120481927710846,
27
+ "eval_accuracy": 0.9440361786319955,
28
+ "eval_f1": 0.9440350340667568,
29
+ "eval_loss": 0.13831785321235657,
30
+ "eval_matthews_correlation": 0.8884885605641855,
31
+ "eval_precision": 0.9443041112978641,
32
+ "eval_recall": 0.9441844573233038,
33
+ "eval_runtime": 1.5317,
34
+ "eval_samples_per_second": 3464.713,
35
+ "eval_steps_per_second": 54.187,
36
+ "step": 200
37
+ },
38
+ {
39
+ "epoch": 0.45180722891566266,
40
+ "grad_norm": 9.916468620300293,
41
+ "learning_rate": 2.7168073676132003e-05,
42
+ "loss": 0.1727,
43
+ "step": 300
44
+ },
45
+ {
46
+ "epoch": 0.6024096385542169,
47
+ "grad_norm": 8.223430633544922,
48
+ "learning_rate": 2.6016884113584036e-05,
49
+ "loss": 0.1306,
50
+ "step": 400
51
+ },
52
+ {
53
+ "epoch": 0.6024096385542169,
54
+ "eval_accuracy": 0.9470510646316186,
55
+ "eval_f1": 0.9469116079894582,
56
+ "eval_loss": 0.1320875585079193,
57
+ "eval_matthews_correlation": 0.8970111970995432,
58
+ "eval_precision": 0.9503975774077025,
59
+ "eval_recall": 0.946621567301194,
60
+ "eval_runtime": 1.5336,
61
+ "eval_samples_per_second": 3460.585,
62
+ "eval_steps_per_second": 54.123,
63
+ "step": 400
64
+ },
65
+ {
66
+ "epoch": 0.7530120481927711,
67
+ "grad_norm": 8.465655326843262,
68
+ "learning_rate": 2.4865694551036073e-05,
69
+ "loss": 0.1449,
70
+ "step": 500
71
+ },
72
+ {
73
+ "epoch": 0.9036144578313253,
74
+ "grad_norm": 8.037919998168945,
75
+ "learning_rate": 2.3714504988488106e-05,
76
+ "loss": 0.1268,
77
+ "step": 600
78
+ },
79
+ {
80
+ "epoch": 0.9036144578313253,
81
+ "eval_accuracy": 0.9502543810062182,
82
+ "eval_f1": 0.9502384353255368,
83
+ "eval_loss": 0.1223142221570015,
84
+ "eval_matthews_correlation": 0.9019972975694879,
85
+ "eval_precision": 0.9514557669338246,
86
+ "eval_recall": 0.9505419934872148,
87
+ "eval_runtime": 1.5394,
88
+ "eval_samples_per_second": 3447.515,
89
+ "eval_steps_per_second": 53.918,
90
+ "step": 600
91
+ },
92
+ {
93
+ "epoch": 1.0542168674698795,
94
+ "grad_norm": 5.252899646759033,
95
+ "learning_rate": 2.256331542594014e-05,
96
+ "loss": 0.0967,
97
+ "step": 700
98
+ },
99
+ {
100
+ "epoch": 1.2048192771084336,
101
+ "grad_norm": 2.8755605220794678,
102
+ "learning_rate": 2.1412125863392172e-05,
103
+ "loss": 0.0793,
104
+ "step": 800
105
+ },
106
+ {
107
+ "epoch": 1.2048192771084336,
108
+ "eval_accuracy": 0.9534576973808178,
109
+ "eval_f1": 0.9534544978603143,
110
+ "eval_loss": 0.12600094079971313,
111
+ "eval_matthews_correlation": 0.9075729094229542,
112
+ "eval_precision": 0.9539269039717451,
113
+ "eval_recall": 0.9536460489075334,
114
+ "eval_runtime": 1.546,
115
+ "eval_samples_per_second": 3432.697,
116
+ "eval_steps_per_second": 53.686,
117
+ "step": 800
118
+ },
119
+ {
120
+ "epoch": 1.355421686746988,
121
+ "grad_norm": 15.925186157226562,
122
+ "learning_rate": 2.026093630084421e-05,
123
+ "loss": 0.0632,
124
+ "step": 900
125
+ },
126
+ {
127
+ "epoch": 1.5060240963855422,
128
+ "grad_norm": 5.883889198303223,
129
+ "learning_rate": 1.9109746738296238e-05,
130
+ "loss": 0.0664,
131
+ "step": 1000
132
+ },
133
+ {
134
+ "epoch": 1.5060240963855422,
135
+ "eval_accuracy": 0.9634445072545694,
136
+ "eval_f1": 0.9634296411014179,
137
+ "eval_loss": 0.11086247861385345,
138
+ "eval_matthews_correlation": 0.9270444420166869,
139
+ "eval_precision": 0.9637054919711341,
140
+ "eval_recall": 0.9633390224799887,
141
+ "eval_runtime": 1.5519,
142
+ "eval_samples_per_second": 3419.769,
143
+ "eval_steps_per_second": 53.484,
144
+ "step": 1000
145
+ },
146
+ {
147
+ "epoch": 1.6566265060240963,
148
+ "grad_norm": 9.317992210388184,
149
+ "learning_rate": 1.7958557175748274e-05,
150
+ "loss": 0.0651,
151
+ "step": 1100
152
+ },
153
+ {
154
+ "epoch": 1.8072289156626506,
155
+ "grad_norm": 7.534521102905273,
156
+ "learning_rate": 1.6807367613200308e-05,
157
+ "loss": 0.0644,
158
+ "step": 1200
159
+ },
160
+ {
161
+ "epoch": 1.8072289156626506,
162
+ "eval_accuracy": 0.9570378745053703,
163
+ "eval_f1": 0.9570372017874856,
164
+ "eval_loss": 0.14527226984500885,
165
+ "eval_matthews_correlation": 0.9144670396513508,
166
+ "eval_precision": 0.9572853548448508,
167
+ "eval_recall": 0.9571816906821948,
168
+ "eval_runtime": 1.5434,
169
+ "eval_samples_per_second": 3438.416,
170
+ "eval_steps_per_second": 53.776,
171
+ "step": 1200
172
+ },
173
+ {
174
+ "epoch": 1.9578313253012047,
175
+ "grad_norm": 7.254329681396484,
176
+ "learning_rate": 1.565617805065234e-05,
177
+ "loss": 0.0635,
178
+ "step": 1300
179
+ },
180
+ {
181
+ "epoch": 2.108433734939759,
182
+ "grad_norm": 6.914297580718994,
183
+ "learning_rate": 1.4504988488104375e-05,
184
+ "loss": 0.0374,
185
+ "step": 1400
186
+ },
187
+ {
188
+ "epoch": 2.108433734939759,
189
+ "eval_accuracy": 0.9628792161296401,
190
+ "eval_f1": 0.9628401830979684,
191
+ "eval_loss": 0.18788941204547882,
192
+ "eval_matthews_correlation": 0.9266187165348982,
193
+ "eval_precision": 0.9639729425818305,
194
+ "eval_recall": 0.9626467230259594,
195
+ "eval_runtime": 1.5439,
196
+ "eval_samples_per_second": 3437.375,
197
+ "eval_steps_per_second": 53.76,
198
+ "step": 1400
199
+ },
200
+ {
201
+ "epoch": 2.2590361445783134,
202
+ "grad_norm": 0.7644485235214233,
203
+ "learning_rate": 1.3353798925556408e-05,
204
+ "loss": 0.0279,
205
+ "step": 1500
206
+ },
207
+ {
208
+ "epoch": 2.4096385542168672,
209
+ "grad_norm": 8.384675979614258,
210
+ "learning_rate": 1.2202609363008443e-05,
211
+ "loss": 0.0164,
212
+ "step": 1600
213
+ },
214
+ {
215
+ "epoch": 2.4096385542168672,
216
+ "eval_accuracy": 0.9657056717542868,
217
+ "eval_f1": 0.9656756314944064,
218
+ "eval_loss": 0.16530963778495789,
219
+ "eval_matthews_correlation": 0.9320662060488338,
220
+ "eval_precision": 0.9665640323608415,
221
+ "eval_recall": 0.9655027778622123,
222
+ "eval_runtime": 1.5424,
223
+ "eval_samples_per_second": 3440.766,
224
+ "eval_steps_per_second": 53.813,
225
+ "step": 1600
226
+ },
227
+ {
228
+ "epoch": 2.5602409638554215,
229
+ "grad_norm": 18.41074562072754,
230
+ "learning_rate": 1.1051419800460476e-05,
231
+ "loss": 0.0179,
232
+ "step": 1700
233
+ },
234
+ {
235
+ "epoch": 2.710843373493976,
236
+ "grad_norm": 0.21532420814037323,
237
+ "learning_rate": 9.90023023791251e-06,
238
+ "loss": 0.0251,
239
+ "step": 1800
240
+ },
241
+ {
242
+ "epoch": 2.710843373493976,
243
+ "eval_accuracy": 0.9591106086301112,
244
+ "eval_f1": 0.95910930195127,
245
+ "eval_loss": 0.22635453939437866,
246
+ "eval_matthews_correlation": 0.9187108433086283,
247
+ "eval_precision": 0.9594386335644487,
248
+ "eval_recall": 0.9592722248152282,
249
+ "eval_runtime": 1.5425,
250
+ "eval_samples_per_second": 3440.442,
251
+ "eval_steps_per_second": 53.808,
252
+ "step": 1800
253
+ },
254
+ {
255
+ "epoch": 2.86144578313253,
256
+ "grad_norm": 5.3087263107299805,
257
+ "learning_rate": 8.749040675364544e-06,
258
+ "loss": 0.0224,
259
+ "step": 1900
260
+ },
261
+ {
262
+ "epoch": 3.0120481927710845,
263
+ "grad_norm": 0.07524475455284119,
264
+ "learning_rate": 7.597851112816577e-06,
265
+ "loss": 0.0179,
266
+ "step": 2000
267
+ },
268
+ {
269
+ "epoch": 3.0120481927710845,
270
+ "eval_accuracy": 0.9692858488788393,
271
+ "eval_f1": 0.9692716697192839,
272
+ "eval_loss": 0.17637819051742554,
273
+ "eval_matthews_correlation": 0.9387831838919006,
274
+ "eval_precision": 0.9696159708142937,
275
+ "eval_recall": 0.9691673202840899,
276
+ "eval_runtime": 1.5414,
277
+ "eval_samples_per_second": 3442.9,
278
+ "eval_steps_per_second": 53.846,
279
+ "step": 2000
280
+ },
281
+ {
282
+ "epoch": 3.1626506024096384,
283
+ "grad_norm": 0.002716244664043188,
284
+ "learning_rate": 6.446661550268611e-06,
285
+ "loss": 0.0032,
286
+ "step": 2100
287
+ },
288
+ {
289
+ "epoch": 3.3132530120481927,
290
+ "grad_norm": 0.0046457210555672646,
291
+ "learning_rate": 5.295471987720644e-06,
292
+ "loss": 0.0037,
293
+ "step": 2200
294
+ },
295
+ {
296
+ "epoch": 3.3132530120481927,
297
+ "eval_accuracy": 0.9677784058790277,
298
+ "eval_f1": 0.9677761908263713,
299
+ "eval_loss": 0.21041572093963623,
300
+ "eval_matthews_correlation": 0.9355606983847059,
301
+ "eval_precision": 0.9677642707707517,
302
+ "eval_recall": 0.9677964281666167,
303
+ "eval_runtime": 1.5409,
304
+ "eval_samples_per_second": 3444.045,
305
+ "eval_steps_per_second": 53.864,
306
+ "step": 2200
307
+ },
308
+ {
309
+ "epoch": 3.463855421686747,
310
+ "grad_norm": 7.352254390716553,
311
+ "learning_rate": 4.144282425172678e-06,
312
+ "loss": 0.0021,
313
+ "step": 2300
314
+ },
315
+ {
316
+ "epoch": 3.6144578313253013,
317
+ "grad_norm": 0.1295173019170761,
318
+ "learning_rate": 2.993092862624712e-06,
319
+ "loss": 0.0014,
320
+ "step": 2400
321
+ },
322
+ {
323
+ "epoch": 3.6144578313253013,
324
+ "eval_accuracy": 0.9683436970039571,
325
+ "eval_f1": 0.9683377061336159,
326
+ "eval_loss": 0.22055356204509735,
327
+ "eval_matthews_correlation": 0.936697700228757,
328
+ "eval_precision": 0.9683925624578065,
329
+ "eval_recall": 0.9683051418503684,
330
+ "eval_runtime": 1.5412,
331
+ "eval_samples_per_second": 3443.434,
332
+ "eval_steps_per_second": 53.854,
333
+ "step": 2400
334
+ },
335
+ {
336
+ "epoch": 3.765060240963855,
337
+ "grad_norm": 0.0010891439160332084,
338
+ "learning_rate": 1.8419033000767458e-06,
339
+ "loss": 0.0035,
340
+ "step": 2500
341
+ },
342
+ {
343
+ "epoch": 3.9156626506024095,
344
+ "grad_norm": 0.010914706625044346,
345
+ "learning_rate": 6.907137375287798e-07,
346
+ "loss": 0.0026,
347
+ "step": 2600
348
+ },
349
+ {
350
+ "epoch": 3.9156626506024095,
351
+ "eval_accuracy": 0.9690974185038628,
352
+ "eval_f1": 0.9690902179184598,
353
+ "eval_loss": 0.21992520987987518,
354
+ "eval_matthews_correlation": 0.938226978614608,
355
+ "eval_precision": 0.9691836691199455,
356
+ "eval_recall": 0.9690433199920572,
357
+ "eval_runtime": 1.5378,
358
+ "eval_samples_per_second": 3450.927,
359
+ "eval_steps_per_second": 53.972,
360
+ "step": 2600
361
+ },
362
+ {
363
+ "epoch": 4.0,
364
+ "step": 2656,
365
+ "total_flos": 8308929553956864.0,
366
+ "train_loss": 0.06583031266366682,
367
+ "train_runtime": 193.8482,
368
+ "train_samples_per_second": 875.984,
369
+ "train_steps_per_second": 13.701
370
+ }
371
+ ],
372
+ "logging_steps": 100,
373
+ "max_steps": 2656,
374
+ "num_input_tokens_seen": 0,
375
+ "num_train_epochs": 4,
376
+ "save_steps": 200,
377
+ "stateful_callbacks": {
378
+ "TrainerControl": {
379
+ "args": {
380
+ "should_epoch_stop": false,
381
+ "should_evaluate": false,
382
+ "should_log": false,
383
+ "should_save": true,
384
+ "should_training_stop": true
385
+ },
386
+ "attributes": {}
387
+ }
388
+ },
389
+ "total_flos": 8308929553956864.0,
390
+ "train_batch_size": 64,
391
+ "trial_name": null,
392
+ "trial_params": null
393
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:532e6dbf3707e11b6466d45804268d083767dcd336699ed2b97d1d6f4fc9c6b6
3
+ size 5368