akera commited on
Commit
64af42b
·
verified ·
1 Parent(s): cc74676

Training in progress, step 100

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{ bos_token }}
2
+ {%- if messages[0]['role'] == 'system' -%}
3
+ {%- if messages[0]['content'] is string -%}
4
+ {%- set first_user_prefix = messages[0]['content'] + '
5
+
6
+ ' -%}
7
+ {%- else -%}
8
+ {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
9
+
10
+ ' -%}
11
+ {%- endif -%}
12
+ {%- set loop_messages = messages[1:] -%}
13
+ {%- else -%}
14
+ {%- set first_user_prefix = "" -%}
15
+ {%- set loop_messages = messages -%}
16
+ {%- endif -%}
17
+ {%- for message in loop_messages -%}
18
+ {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
19
+ {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
20
+ {%- endif -%}
21
+ {%- if (message['role'] == 'assistant') -%}
22
+ {%- set role = "model" -%}
23
+ {%- else -%}
24
+ {%- set role = message['role'] -%}
25
+ {%- endif -%}
26
+ {{ '<start_of_turn>' + role + '
27
+ ' + (first_user_prefix if loop.first else "") }}
28
+ {%- if message['content'] is string -%}
29
+ {{ message['content'] | trim }}
30
+ {%- elif message['content'] is iterable -%}
31
+ {%- for item in message['content'] -%}
32
+ {%- if item['type'] == 'image' -%}
33
+ {{ '<start_of_image>' }}
34
+ {%- elif item['type'] == 'text' -%}
35
+ {{ item['text'] | trim }}
36
+ {%- endif -%}
37
+ {%- endfor -%}
38
+ {%- else -%}
39
+ {{ raise_exception("Invalid content type") }}
40
+ {%- endif -%}
41
+ {{ '<end_of_turn>
42
+ ' }}
43
+ {%- endfor -%}
44
+ {%- if add_generation_prompt -%}
45
+ {{'<start_of_turn>model
46
+ '}}
47
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,422 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "UltravoxModel"
4
+ ],
5
+ "audio_config": {
6
+ "_name_or_path": "Sunbird/asr-whisper-large-v3-salt",
7
+ "activation_dropout": 0.0,
8
+ "activation_function": "gelu",
9
+ "add_cross_attention": false,
10
+ "apply_spec_augment": false,
11
+ "architectures": [
12
+ "WhisperForConditionalGeneration"
13
+ ],
14
+ "attention_dropout": 0.0,
15
+ "bad_words_ids": null,
16
+ "begin_suppress_tokens": null,
17
+ "bos_token_id": 50257,
18
+ "chunk_size_feed_forward": 0,
19
+ "classifier_proj_size": 256,
20
+ "cross_attention_hidden_size": null,
21
+ "d_model": 1280,
22
+ "decoder_attention_heads": 20,
23
+ "decoder_ffn_dim": 5120,
24
+ "decoder_layerdrop": 0.0,
25
+ "decoder_layers": 32,
26
+ "decoder_start_token_id": 50258,
27
+ "diversity_penalty": 0.0,
28
+ "do_sample": false,
29
+ "dropout": 0.0,
30
+ "early_stopping": false,
31
+ "encoder_attention_heads": 20,
32
+ "encoder_ffn_dim": 5120,
33
+ "encoder_layerdrop": 0.0,
34
+ "encoder_layers": 32,
35
+ "encoder_no_repeat_ngram_size": 0,
36
+ "eos_token_id": 50257,
37
+ "exponential_decay_length_penalty": null,
38
+ "finetuning_task": null,
39
+ "forced_bos_token_id": null,
40
+ "forced_decoder_ids": null,
41
+ "forced_eos_token_id": null,
42
+ "id2label": {
43
+ "0": "LABEL_0",
44
+ "1": "LABEL_1"
45
+ },
46
+ "init_std": 0.02,
47
+ "is_decoder": false,
48
+ "is_encoder_decoder": true,
49
+ "label2id": {
50
+ "LABEL_0": 0,
51
+ "LABEL_1": 1
52
+ },
53
+ "length_penalty": 1.0,
54
+ "mask_feature_length": 10,
55
+ "mask_feature_min_masks": 0,
56
+ "mask_feature_prob": 0.0,
57
+ "mask_time_length": 10,
58
+ "mask_time_min_masks": 2,
59
+ "mask_time_prob": 0.05,
60
+ "max_length": null,
61
+ "max_source_positions": 1500,
62
+ "max_target_positions": 448,
63
+ "median_filter_width": 7,
64
+ "min_length": 0,
65
+ "model_type": "whisper",
66
+ "no_repeat_ngram_size": 0,
67
+ "num_beam_groups": 1,
68
+ "num_beams": 1,
69
+ "num_hidden_layers": 32,
70
+ "num_mel_bins": 128,
71
+ "num_return_sequences": 1,
72
+ "output_attentions": false,
73
+ "output_hidden_states": false,
74
+ "output_scores": false,
75
+ "pad_token_id": 50256,
76
+ "prefix": null,
77
+ "problem_type": null,
78
+ "pruned_heads": {},
79
+ "remove_invalid_values": false,
80
+ "repetition_penalty": 1.0,
81
+ "return_dict": true,
82
+ "return_dict_in_generate": false,
83
+ "scale_embedding": false,
84
+ "sep_token_id": null,
85
+ "suppress_tokens": null,
86
+ "task_specific_params": null,
87
+ "temperature": 1.0,
88
+ "tf_legacy_loss": false,
89
+ "tie_encoder_decoder": false,
90
+ "tie_word_embeddings": true,
91
+ "tokenizer_class": null,
92
+ "top_k": 50,
93
+ "top_p": 1.0,
94
+ "torch_dtype": "float32",
95
+ "torchscript": false,
96
+ "typical_p": 1.0,
97
+ "use_bfloat16": false,
98
+ "use_cache": true,
99
+ "use_weighted_layer_sum": false,
100
+ "vocab_size": 51866
101
+ },
102
+ "audio_model_id": "Sunbird/asr-whisper-large-v3-salt",
103
+ "audio_model_lora_config": {
104
+ "lora_alpha": 64,
105
+ "lora_dropout": 0.05,
106
+ "r": 32,
107
+ "target_modules": [
108
+ "q_proj",
109
+ "k_proj",
110
+ "v_proj",
111
+ "o_proj"
112
+ ]
113
+ },
114
+ "hidden_size": 3840,
115
+ "ignore_index": -100,
116
+ "model_type": "ultravox",
117
+ "norm_init": 0.4,
118
+ "projector_act": "swiglu",
119
+ "projector_ln_mid": true,
120
+ "stack_factor": 8,
121
+ "text_config": {
122
+ "_name_or_path": "jq/gemma3-12b-ug40-merged",
123
+ "add_cross_attention": false,
124
+ "architectures": [
125
+ "Gemma3ForConditionalGeneration"
126
+ ],
127
+ "bad_words_ids": null,
128
+ "begin_suppress_tokens": null,
129
+ "boi_token_index": 255999,
130
+ "bos_token_id": null,
131
+ "chunk_size_feed_forward": 0,
132
+ "cross_attention_hidden_size": null,
133
+ "decoder_start_token_id": null,
134
+ "diversity_penalty": 0.0,
135
+ "do_sample": false,
136
+ "early_stopping": false,
137
+ "encoder_no_repeat_ngram_size": 0,
138
+ "eoi_token_index": 256000,
139
+ "eos_token_id": 1,
140
+ "exponential_decay_length_penalty": null,
141
+ "finetuning_task": null,
142
+ "forced_bos_token_id": null,
143
+ "forced_eos_token_id": null,
144
+ "id2label": {
145
+ "0": "LABEL_0",
146
+ "1": "LABEL_1"
147
+ },
148
+ "image_token_index": 262144,
149
+ "initializer_range": 0.02,
150
+ "is_decoder": false,
151
+ "is_encoder_decoder": false,
152
+ "label2id": {
153
+ "LABEL_0": 0,
154
+ "LABEL_1": 1
155
+ },
156
+ "length_penalty": 1.0,
157
+ "max_length": 20,
158
+ "min_length": 0,
159
+ "mm_tokens_per_image": 256,
160
+ "model_type": "gemma3",
161
+ "no_repeat_ngram_size": 0,
162
+ "num_beam_groups": 1,
163
+ "num_beams": 1,
164
+ "num_return_sequences": 1,
165
+ "output_attentions": false,
166
+ "output_hidden_states": false,
167
+ "output_scores": false,
168
+ "pad_token_id": null,
169
+ "prefix": null,
170
+ "problem_type": null,
171
+ "pruned_heads": {},
172
+ "remove_invalid_values": false,
173
+ "repetition_penalty": 1.0,
174
+ "return_dict": true,
175
+ "return_dict_in_generate": false,
176
+ "sep_token_id": null,
177
+ "suppress_tokens": null,
178
+ "task_specific_params": null,
179
+ "temperature": 1.0,
180
+ "text_config": {
181
+ "_name_or_path": "",
182
+ "_sliding_window_pattern": 6,
183
+ "add_cross_attention": false,
184
+ "architectures": null,
185
+ "attention_bias": false,
186
+ "attention_dropout": 0.0,
187
+ "attn_logit_softcapping": null,
188
+ "bad_words_ids": null,
189
+ "begin_suppress_tokens": null,
190
+ "bos_token_id": 2,
191
+ "cache_implementation": "hybrid",
192
+ "chunk_size_feed_forward": 0,
193
+ "cross_attention_hidden_size": null,
194
+ "decoder_start_token_id": null,
195
+ "diversity_penalty": 0.0,
196
+ "do_sample": false,
197
+ "early_stopping": false,
198
+ "encoder_no_repeat_ngram_size": 0,
199
+ "eos_token_id": 1,
200
+ "exponential_decay_length_penalty": null,
201
+ "final_logit_softcapping": null,
202
+ "finetuning_task": null,
203
+ "forced_bos_token_id": null,
204
+ "forced_eos_token_id": null,
205
+ "head_dim": 256,
206
+ "hidden_activation": "gelu_pytorch_tanh",
207
+ "hidden_size": 3840,
208
+ "id2label": {
209
+ "0": "LABEL_0",
210
+ "1": "LABEL_1"
211
+ },
212
+ "initializer_range": 0.02,
213
+ "intermediate_size": 15360,
214
+ "is_decoder": false,
215
+ "is_encoder_decoder": false,
216
+ "label2id": {
217
+ "LABEL_0": 0,
218
+ "LABEL_1": 1
219
+ },
220
+ "layer_types": [
221
+ "sliding_attention",
222
+ "sliding_attention",
223
+ "sliding_attention",
224
+ "sliding_attention",
225
+ "sliding_attention",
226
+ "full_attention",
227
+ "sliding_attention",
228
+ "sliding_attention",
229
+ "sliding_attention",
230
+ "sliding_attention",
231
+ "sliding_attention",
232
+ "full_attention",
233
+ "sliding_attention",
234
+ "sliding_attention",
235
+ "sliding_attention",
236
+ "sliding_attention",
237
+ "sliding_attention",
238
+ "full_attention",
239
+ "sliding_attention",
240
+ "sliding_attention",
241
+ "sliding_attention",
242
+ "sliding_attention",
243
+ "sliding_attention",
244
+ "full_attention",
245
+ "sliding_attention",
246
+ "sliding_attention",
247
+ "sliding_attention",
248
+ "sliding_attention",
249
+ "sliding_attention",
250
+ "full_attention",
251
+ "sliding_attention",
252
+ "sliding_attention",
253
+ "sliding_attention",
254
+ "sliding_attention",
255
+ "sliding_attention",
256
+ "full_attention",
257
+ "sliding_attention",
258
+ "sliding_attention",
259
+ "sliding_attention",
260
+ "sliding_attention",
261
+ "sliding_attention",
262
+ "full_attention",
263
+ "sliding_attention",
264
+ "sliding_attention",
265
+ "sliding_attention",
266
+ "sliding_attention",
267
+ "sliding_attention",
268
+ "full_attention"
269
+ ],
270
+ "length_penalty": 1.0,
271
+ "max_length": 20,
272
+ "max_position_embeddings": 131072,
273
+ "min_length": 0,
274
+ "model_type": "gemma3_text",
275
+ "no_repeat_ngram_size": 0,
276
+ "num_attention_heads": 16,
277
+ "num_beam_groups": 1,
278
+ "num_beams": 1,
279
+ "num_hidden_layers": 48,
280
+ "num_key_value_heads": 8,
281
+ "num_return_sequences": 1,
282
+ "output_attentions": false,
283
+ "output_hidden_states": false,
284
+ "output_scores": false,
285
+ "pad_token_id": 0,
286
+ "prefix": null,
287
+ "problem_type": null,
288
+ "pruned_heads": {},
289
+ "query_pre_attn_scalar": 256,
290
+ "remove_invalid_values": false,
291
+ "repetition_penalty": 1.0,
292
+ "return_dict": true,
293
+ "return_dict_in_generate": false,
294
+ "rms_norm_eps": 1e-06,
295
+ "rope_local_base_freq": 10000.0,
296
+ "rope_scaling": {
297
+ "factor": 8.0,
298
+ "rope_type": "linear"
299
+ },
300
+ "rope_theta": 1000000.0,
301
+ "sep_token_id": null,
302
+ "sliding_window": 1024,
303
+ "suppress_tokens": null,
304
+ "task_specific_params": null,
305
+ "temperature": 1.0,
306
+ "tf_legacy_loss": false,
307
+ "tie_encoder_decoder": false,
308
+ "tie_word_embeddings": true,
309
+ "tokenizer_class": null,
310
+ "top_k": 50,
311
+ "top_p": 1.0,
312
+ "torch_dtype": "bfloat16",
313
+ "torchscript": false,
314
+ "typical_p": 1.0,
315
+ "use_bfloat16": false,
316
+ "use_cache": false,
317
+ "vocab_size": 262208
318
+ },
319
+ "tf_legacy_loss": false,
320
+ "tie_encoder_decoder": false,
321
+ "tie_word_embeddings": true,
322
+ "tokenizer_class": null,
323
+ "top_k": 50,
324
+ "top_p": 1.0,
325
+ "torch_dtype": "bfloat16",
326
+ "torchscript": false,
327
+ "typical_p": 1.0,
328
+ "use_bfloat16": false,
329
+ "vision_config": {
330
+ "_name_or_path": "",
331
+ "add_cross_attention": false,
332
+ "architectures": null,
333
+ "attention_dropout": 0.0,
334
+ "bad_words_ids": null,
335
+ "begin_suppress_tokens": null,
336
+ "bos_token_id": null,
337
+ "chunk_size_feed_forward": 0,
338
+ "cross_attention_hidden_size": null,
339
+ "decoder_start_token_id": null,
340
+ "diversity_penalty": 0.0,
341
+ "do_sample": false,
342
+ "early_stopping": false,
343
+ "encoder_no_repeat_ngram_size": 0,
344
+ "eos_token_id": null,
345
+ "exponential_decay_length_penalty": null,
346
+ "finetuning_task": null,
347
+ "forced_bos_token_id": null,
348
+ "forced_eos_token_id": null,
349
+ "hidden_act": "gelu_pytorch_tanh",
350
+ "hidden_size": 1152,
351
+ "id2label": {
352
+ "0": "LABEL_0",
353
+ "1": "LABEL_1"
354
+ },
355
+ "image_size": 896,
356
+ "intermediate_size": 4304,
357
+ "is_decoder": false,
358
+ "is_encoder_decoder": false,
359
+ "label2id": {
360
+ "LABEL_0": 0,
361
+ "LABEL_1": 1
362
+ },
363
+ "layer_norm_eps": 1e-06,
364
+ "length_penalty": 1.0,
365
+ "max_length": 20,
366
+ "min_length": 0,
367
+ "model_type": "siglip_vision_model",
368
+ "no_repeat_ngram_size": 0,
369
+ "num_attention_heads": 16,
370
+ "num_beam_groups": 1,
371
+ "num_beams": 1,
372
+ "num_channels": 3,
373
+ "num_hidden_layers": 27,
374
+ "num_return_sequences": 1,
375
+ "output_attentions": false,
376
+ "output_hidden_states": false,
377
+ "output_scores": false,
378
+ "pad_token_id": null,
379
+ "patch_size": 14,
380
+ "prefix": null,
381
+ "problem_type": null,
382
+ "pruned_heads": {},
383
+ "remove_invalid_values": false,
384
+ "repetition_penalty": 1.0,
385
+ "return_dict": true,
386
+ "return_dict_in_generate": false,
387
+ "sep_token_id": null,
388
+ "suppress_tokens": null,
389
+ "task_specific_params": null,
390
+ "temperature": 1.0,
391
+ "tf_legacy_loss": false,
392
+ "tie_encoder_decoder": false,
393
+ "tie_word_embeddings": true,
394
+ "tokenizer_class": null,
395
+ "top_k": 50,
396
+ "top_p": 1.0,
397
+ "torch_dtype": "bfloat16",
398
+ "torchscript": false,
399
+ "typical_p": 1.0,
400
+ "use_bfloat16": false,
401
+ "vision_use_head": false
402
+ }
403
+ },
404
+ "text_model_id": "jq/gemma3-12b-ug40-merged",
405
+ "text_model_lora_config": {
406
+ "lora_alpha": 128,
407
+ "lora_dropout": 0.05,
408
+ "r": 64,
409
+ "target_modules": [
410
+ "q_proj",
411
+ "k_proj",
412
+ "v_proj",
413
+ "o_proj",
414
+ "gate_proj",
415
+ "up_proj",
416
+ "down_proj"
417
+ ]
418
+ },
419
+ "torch_dtype": "bfloat16",
420
+ "transformers_version": "4.54.0",
421
+ "vocab_size": 50257
422
+ }
model-00001-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a51158e195b69d0e4fb8e52939a142682867f81deb17248c38433d402c800620
3
+ size 4904174928
model-00002-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42527c8a9c98183661ddac05fca6f50cfa6316a0e8b1e9e1caab18eaf9c8921f
3
+ size 4997779080
model-00003-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe87aab2a66712dc8b75fd4126a25593065eaefb3c4c8a7eb50bc91124bcb975
3
+ size 4997749272
model-00004-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b92b5781b78d8fde7b3c862001519ca3e1b8f0c25c4fbd670462738a16074ecd
3
+ size 4925626056
model-00005-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd17aa2aa35e5d1424d7c09fb4a573e932002161c6fb26fcb1393501a37596e1
3
+ size 4947022016
model-00006-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e8d6015ee9eba2eda6ad3a1ba250faa3b8b05b788c8b69f1a1fb75572487cef
3
+ size 2190250720
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "boi_token": "<start_of_image>",
3
+ "bos_token": {
4
+ "content": "<bos>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ "eoi_token": "<end_of_image>",
11
+ "eos_token": {
12
+ "content": "<eos>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false
17
+ },
18
+ "image_token": "<image_soft_token>",
19
+ "pad_token": {
20
+ "content": "<pad>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false
25
+ },
26
+ "unk_token": {
27
+ "content": "<unk>",
28
+ "lstrip": false,
29
+ "normalized": false,
30
+ "rstrip": false,
31
+ "single_word": false
32
+ }
33
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
3
+ size 33384568
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17cd4df694a3c06dc85d41661055cdaff81afe715f68c43f1b031fea03a4d911
3
+ size 5777