|
{ |
|
"architectures": [ |
|
"OmniSpeechExaoneForCausalLM" |
|
], |
|
"attention_bias": false, |
|
"attention_dropout": 0.0, |
|
"bos_token_id": 1, |
|
"deepspeed_config": "./scripts/zero2.json", |
|
"eos_token_id": 361, |
|
"fullduplex_dropout": 0.1, |
|
"fullduplex_nhead": 8, |
|
"fullduplex_num_classes": 3, |
|
"head_dim": 64, |
|
"hidden_act": "silu", |
|
"hidden_size": 2048, |
|
"initializer_range": 0.02, |
|
"intermediate_size": 4096, |
|
"layer_types": [ |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention", |
|
"full_attention" |
|
], |
|
"max_position_embeddings": 65536, |
|
"mlp_bias": false, |
|
"mm_tunable_parts": "speech_projector,backbone", |
|
"model_type": "omni_speech_exaone", |
|
"num_attention_heads": 32, |
|
"num_hidden_layers": 30, |
|
"num_key_value_heads": 8, |
|
"pad_token_id": 0, |
|
"pretraining_tp": 1, |
|
"rms_norm_eps": 1e-05, |
|
"rope_scaling": { |
|
"factor": 16.0, |
|
"high_freq_factor": 4.0, |
|
"low_freq_factor": 1.0, |
|
"original_max_position_embeddings": 8192, |
|
"rope_type": "llama3" |
|
}, |
|
"rope_theta": 1000000.0, |
|
"sliding_window": null, |
|
"sliding_window_pattern": null, |
|
"speech_encoder": "./models/speech_encoder/whisper-large-v3", |
|
"speech_encoder_ds_rate": 5, |
|
"speech_encoder_hidden_size": 1280, |
|
"speech_encoder_type": "whisper", |
|
"speech_projector_lr": null, |
|
"speech_projector_type": "linear", |
|
"tie_word_embeddings": true, |
|
"tokenizer_model_max_length": 2048, |
|
"tokenizer_padding_side": "right", |
|
"torch_dtype": "float16", |
|
"transformers_version": "4.54.0", |
|
"use_cache": true, |
|
"use_duplex": false, |
|
"vocab_size": 102400 |
|
} |
|
|