Hugging Face's logo

yzhuang
/
llava-jamba-1.5-Mini-linear-anyres-7by7-rope

Safetensors
jamba
Model card Files Community
llava-jamba-1.5-Mini-linear-anyres-7by7-rope / config.json
yzhuang's picture
yzhuang
Training in progress, step 600
af248ca verified
raw
history blame contribute delete
8.99 kB
{
"_name_or_path": "yzhuang/llava-jamba-1.5-Mini-linear-anyres-merge",
"architectures": [
"LlavaJambaForCausalLM"
],
"attention_dropout": 0.0,
"attn_layer_offset": 4,
"attn_layer_period": 8,
"bos_token_id": 1,
"eos_token_id": [
2,
518
],
"expert_layer_offset": 1,
"expert_layer_period": 2,
"freeze_mm_mlp_adapter": false,
"hidden_act": "silu",
"hidden_size": 4096,
"image_aspect_ratio": "anyres",
"image_grid_pinpoints": [
[
336,
336
],
[
336,
672
],
[
336,
1008
],
[
336,
1344
],
[
336,
1680
],
[
336,
2016
],
[
336,
2352
],
[
336,
2688
],
[
336,
3024
],
[
336,
3360
],
[
336,
3696
],
[
336,
4032
],
[
336,
4368
],
[
336,
4704
],
[
336,
5040
],
[
336,
5376
],
[
336,
5712
],
[
336,
6048
],
[
336,
6384
],
[
336,
6720
],
[
336,
7056
],
[
336,
7392
],
[
336,
7728
],
[
336,
8064
],
[
336,
8400
],
[
336,
8736
],
[
336,
9072
],
[
336,
9408
],
[
336,
9744
],
[
336,
10080
],
[
336,
10416
],
[
336,
10752
],
[
336,
11088
],
[
336,
11424
],
[
336,
11760
],
[
336,
12096
],
[
336,
12432
],
[
336,
12768
],
[
336,
13104
],
[
336,
13440
],
[
336,
13776
],
[
336,
14112
],
[
336,
14448
],
[
336,
14784
],
[
336,
15120
],
[
336,
15456
],
[
336,
15792
],
[
336,
16128
],
[
672,
336
],
[
672,
672
],
[
672,
1008
],
[
672,
1344
],
[
672,
1680
],
[
672,
2016
],
[
672,
2352
],
[
672,
2688
],
[
672,
3024
],
[
672,
3360
],
[
672,
3696
],
[
672,
4032
],
[
672,
4368
],
[
672,
4704
],
[
672,
5040
],
[
672,
5376
],
[
672,
5712
],
[
672,
6048
],
[
672,
6384
],
[
672,
6720
],
[
672,
7056
],
[
672,
7392
],
[
672,
7728
],
[
672,
8064
],
[
1008,
336
],
[
1008,
672
],
[
1008,
1008
],
[
1008,
1344
],
[
1008,
1680
],
[
1008,
2016
],
[
1008,
2352
],
[
1008,
2688
],
[
1008,
3024
],
[
1008,
3360
],
[
1008,
3696
],
[
1008,
4032
],
[
1008,
4368
],
[
1008,
4704
],
[
1008,
5040
],
[
1008,
5376
],
[
1344,
336
],
[
1344,
672
],
[
1344,
1008
],
[
1344,
1344
],
[
1344,
1680
],
[
1344,
2016
],
[
1344,
2352
],
[
1344,
2688
],
[
1344,
3024
],
[
1344,
3360
],
[
1344,
3696
],
[
1344,
4032
],
[
1680,
336
],
[
1680,
672
],
[
1680,
1008
],
[
1680,
1344
],
[
1680,
1680
],
[
1680,
2016
],
[
1680,
2352
],
[
1680,
2688
],
[
1680,
3024
],
[
2016,
336
],
[
2016,
672
],
[
2016,
1008
],
[
2016,
1344
],
[
2016,
1680
],
[
2016,
2016
],
[
2016,
2352
],
[
2016,
2688
],
[
2352,
336
],
[
2352,
672
],
[
2352,
1008
],
[
2352,
1344
],
[
2352,
1680
],
[
2352,
2016
],
[
2352,
2352
],
[
2688,
336
],
[
2688,
672
],
[
2688,
1008
],
[
2688,
1344
],
[
2688,
1680
],
[
2688,
2016
],
[
3024,
336
],
[
3024,
672
],
[
3024,
1008
],
[
3024,
1344
],
[
3024,
1680
],
[
3360,
336
],
[
3360,
672
],
[
3360,
1008
],
[
3360,
1344
],
[
3696,
336
],
[
3696,
672
],
[
3696,
1008
],
[
3696,
1344
],
[
4032,
336
],
[
4032,
672
],
[
4032,
1008
],
[
4032,
1344
],
[
4368,
336
],
[
4368,
672
],
[
4368,
1008
],
[
4704,
336
],
[
4704,
672
],
[
4704,
1008
],
[
5040,
336
],
[
5040,
672
],
[
5040,
1008
],
[
5376,
336
],
[
5376,
672
],
[
5376,
1008
],
[
5712,
336
],
[
5712,
672
],
[
6048,
336
],
[
6048,
672
],
[
6384,
336
],
[
6384,
672
],
[
6720,
336
],
[
6720,
672
],
[
7056,
336
],
[
7056,
672
],
[
7392,
336
],
[
7392,
672
],
[
7728,
336
],
[
7728,
672
],
[
8064,
336
],
[
8064,
672
],
[
8400,
336
],
[
8736,
336
],
[
9072,
336
],
[
9408,
336
],
[
9744,
336
],
[
10080,
336
],
[
10416,
336
],
[
10752,
336
],
[
11088,
336
],
[
11424,
336
],
[
11760,
336
],
[
12096,
336
],
[
12432,
336
],
[
12768,
336
],
[
13104,
336
],
[
13440,
336
],
[
13776,
336
],
[
14112,
336
],
[
14448,
336
],
[
14784,
336
],
[
15120,
336
],
[
15456,
336
],
[
15792,
336
],
[
16128,
336
]
],
"initializer_range": 0.02,
"intermediate_size": 14336,
"mamba_conv_bias": true,
"mamba_d_conv": 4,
"mamba_d_state": 16,
"mamba_dt_rank": 256,
"mamba_expand": 2,
"mamba_proj_bias": false,
"max_position_embeddings": 262144,
"mm_hidden_size": 1024,
"mm_patch_merge_type": "spatial_unpad",
"mm_projector_lr": null,
"mm_projector_type": "linear",
"mm_use_im_patch_token": false,
"mm_use_im_start_end": false,
"mm_vision_select_feature": "patch",
"mm_vision_select_layer": -2,
"mm_vision_tower": "openai/clip-vit-large-patch14-336",
"mm_vision_tower_lr": 2e-06,
"model_type": "jamba",
"num_attention_heads": 32,
"num_experts": 16,
"num_experts_per_tok": 2,
"num_hidden_layers": 32,
"num_key_value_heads": 8,
"num_logits_to_keep": 1,
"output_router_logits": false,
"pad_token_id": 0,
"rms_norm_eps": 1e-06,
"rope_scaling": {
"factor": 8.0,
"high_freq_factor": 4.0,
"low_freq_factor": 1.0,
"original_max_position_embeddings": 8192,
"rope_type": "llama3"
},
"rope_theta": 500000.0,
"router_aux_loss_coef": 0.001,
"sliding_window": null,
"tie_word_embeddings": false,
"tokenizer_model_max_length": 32896,
"tokenizer_padding_side": "right",
"torch_dtype": "bfloat16",
"transformers_version": "4.45.0.dev0",
"tune_mm_mlp_adapter": false,
"unfreeze_mm_vision_tower": false,
"use_cache": false,
"use_mamba_kernels": true,
"use_mm_proj": true,
"vocab_size": 65536
}