Training in progress, step 200

config.json

@@ -0,0 +1,851 @@
+{
+  "_name_or_path": "yzhuang/llava-falcon-mamba-7B-linear-anyres",
+  "architectures": [
+    "LlavaFalconMambaForCausalLM"
+  ],
+  "bos_token_id": 8,
+  "conv_kernel": 4,
+  "eos_token_id": 11,
+  "expand": 16,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_aspect_ratio": "anyres",
+  "image_grid_pinpoints": [
+    [
+      336,
+      336
+    ],
+    [
+      336,
+      672
+    ],
+    [
+      336,
+      1008
+    ],
+    [
+      336,
+      1344
+    ],
+    [
+      336,
+      1680
+    ],
+    [
+      336,
+      2016
+    ],
+    [
+      336,
+      2352
+    ],
+    [
+      336,
+      2688
+    ],
+    [
+      336,
+      3024
+    ],
+    [
+      336,
+      3360
+    ],
+    [
+      336,
+      3696
+    ],
+    [
+      336,
+      4032
+    ],
+    [
+      336,
+      4368
+    ],
+    [
+      336,
+      4704
+    ],
+    [
+      336,
+      5040
+    ],
+    [
+      336,
+      5376
+    ],
+    [
+      336,
+      5712
+    ],
+    [
+      336,
+      6048
+    ],
+    [
+      336,
+      6384
+    ],
+    [
+      336,
+      6720
+    ],
+    [
+      336,
+      7056
+    ],
+    [
+      336,
+      7392
+    ],
+    [
+      336,
+      7728
+    ],
+    [
+      336,
+      8064
+    ],
+    [
+      336,
+      8400
+    ],
+    [
+      336,
+      8736
+    ],
+    [
+      336,
+      9072
+    ],
+    [
+      336,
+      9408
+    ],
+    [
+      336,
+      9744
+    ],
+    [
+      336,
+      10080
+    ],
+    [
+      336,
+      10416
+    ],
+    [
+      336,
+      10752
+    ],
+    [
+      336,
+      11088
+    ],
+    [
+      336,
+      11424
+    ],
+    [
+      336,
+      11760
+    ],
+    [
+      336,
+      12096
+    ],
+    [
+      336,
+      12432
+    ],
+    [
+      336,
+      12768
+    ],
+    [
+      336,
+      13104
+    ],
+    [
+      336,
+      13440
+    ],
+    [
+      336,
+      13776
+    ],
+    [
+      336,
+      14112
+    ],
+    [
+      336,
+      14448
+    ],
+    [
+      336,
+      14784
+    ],
+    [
+      336,
+      15120
+    ],
+    [
+      336,
+      15456
+    ],
+    [
+      336,
+      15792
+    ],
+    [
+      336,
+      16128
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      672,
+      1008
+    ],
+    [
+      672,
+      1344
+    ],
+    [
+      672,
+      1680
+    ],
+    [
+      672,
+      2016
+    ],
+    [
+      672,
+      2352
+    ],
+    [
+      672,
+      2688
+    ],
+    [
+      672,
+      3024
+    ],
+    [
+      672,
+      3360
+    ],
+    [
+      672,
+      3696
+    ],
+    [
+      672,
+      4032
+    ],
+    [
+      672,
+      4368
+    ],
+    [
+      672,
+      4704
+    ],
+    [
+      672,
+      5040
+    ],
+    [
+      672,
+      5376
+    ],
+    [
+      672,
+      5712
+    ],
+    [
+      672,
+      6048
+    ],
+    [
+      672,
+      6384
+    ],
+    [
+      672,
+      6720
+    ],
+    [
+      672,
+      7056
+    ],
+    [
+      672,
+      7392
+    ],
+    [
+      672,
+      7728
+    ],
+    [
+      672,
+      8064
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      1008,
+      672
+    ],
+    [
+      1008,
+      1008
+    ],
+    [
+      1008,
+      1344
+    ],
+    [
+      1008,
+      1680
+    ],
+    [
+      1008,
+      2016
+    ],
+    [
+      1008,
+      2352
+    ],
+    [
+      1008,
+      2688
+    ],
+    [
+      1008,
+      3024
+    ],
+    [
+      1008,
+      3360
+    ],
+    [
+      1008,
+      3696
+    ],
+    [
+      1008,
+      4032
+    ],
+    [
+      1008,
+      4368
+    ],
+    [
+      1008,
+      4704
+    ],
+    [
+      1008,
+      5040
+    ],
+    [
+      1008,
+      5376
+    ],
+    [
+      1344,
+      336
+    ],
+    [
+      1344,
+      672
+    ],
+    [
+      1344,
+      1008
+    ],
+    [
+      1344,
+      1344
+    ],
+    [
+      1344,
+      1680
+    ],
+    [
+      1344,
+      2016
+    ],
+    [
+      1344,
+      2352
+    ],
+    [
+      1344,
+      2688
+    ],
+    [
+      1344,
+      3024
+    ],
+    [
+      1344,
+      3360
+    ],
+    [
+      1344,
+      3696
+    ],
+    [
+      1344,
+      4032
+    ],
+    [
+      1680,
+      336
+    ],
+    [
+      1680,
+      672
+    ],
+    [
+      1680,
+      1008
+    ],
+    [
+      1680,
+      1344
+    ],
+    [
+      1680,
+      1680
+    ],
+    [
+      1680,
+      2016
+    ],
+    [
+      1680,
+      2352
+    ],
+    [
+      1680,
+      2688
+    ],
+    [
+      1680,
+      3024
+    ],
+    [
+      2016,
+      336
+    ],
+    [
+      2016,
+      672
+    ],
+    [
+      2016,
+      1008
+    ],
+    [
+      2016,
+      1344
+    ],
+    [
+      2016,
+      1680
+    ],
+    [
+      2016,
+      2016
+    ],
+    [
+      2016,
+      2352
+    ],
+    [
+      2016,
+      2688
+    ],
+    [
+      2352,
+      336
+    ],
+    [
+      2352,
+      672
+    ],
+    [
+      2352,
+      1008
+    ],
+    [
+      2352,
+      1344
+    ],
+    [
+      2352,
+      1680
+    ],
+    [
+      2352,
+      2016
+    ],
+    [
+      2352,
+      2352
+    ],
+    [
+      2688,
+      336
+    ],
+    [
+      2688,
+      672
+    ],
+    [
+      2688,
+      1008
+    ],
+    [
+      2688,
+      1344
+    ],
+    [
+      2688,
+      1680
+    ],
+    [
+      2688,
+      2016
+    ],
+    [
+      3024,
+      336
+    ],
+    [
+      3024,
+      672
+    ],
+    [
+      3024,
+      1008
+    ],
+    [
+      3024,
+      1344
+    ],
+    [
+      3024,
+      1680
+    ],
+    [
+      3360,
+      336
+    ],
+    [
+      3360,
+      672
+    ],
+    [
+      3360,
+      1008
+    ],
+    [
+      3360,
+      1344
+    ],
+    [
+      3696,
+      336
+    ],
+    [
+      3696,
+      672
+    ],
+    [
+      3696,
+      1008
+    ],
+    [
+      3696,
+      1344
+    ],
+    [
+      4032,
+      336
+    ],
+    [
+      4032,
+      672
+    ],
+    [
+      4032,
+      1008
+    ],
+    [
+      4032,
+      1344
+    ],
+    [
+      4368,
+      336
+    ],
+    [
+      4368,
+      672
+    ],
+    [
+      4368,
+      1008
+    ],
+    [
+      4704,
+      336
+    ],
+    [
+      4704,
+      672
+    ],
+    [
+      4704,
+      1008
+    ],
+    [
+      5040,
+      336
+    ],
+    [
+      5040,
+      672
+    ],
+    [
+      5040,
+      1008
+    ],
+    [
+      5376,
+      336
+    ],
+    [
+      5376,
+      672
+    ],
+    [
+      5376,
+      1008
+    ],
+    [
+      5712,
+      336
+    ],
+    [
+      5712,
+      672
+    ],
+    [
+      6048,
+      336
+    ],
+    [
+      6048,
+      672
+    ],
+    [
+      6384,
+      336
+    ],
+    [
+      6384,
+      672
+    ],
+    [
+      6720,
+      336
+    ],
+    [
+      6720,
+      672
+    ],
+    [
+      7056,
+      336
+    ],
+    [
+      7056,
+      672
+    ],
+    [
+      7392,
+      336
+    ],
+    [
+      7392,
+      672
+    ],
+    [
+      7728,
+      336
+    ],
+    [
+      7728,
+      672
+    ],
+    [
+      8064,
+      336
+    ],
+    [
+      8064,
+      672
+    ],
+    [
+      8400,
+      336
+    ],
+    [
+      8736,
+      336
+    ],
+    [
+      9072,
+      336
+    ],
+    [
+      9408,
+      336
+    ],
+    [
+      9744,
+      336
+    ],
+    [
+      10080,
+      336
+    ],
+    [
+      10416,
+      336
+    ],
+    [
+      10752,
+      336
+    ],
+    [
+      11088,
+      336
+    ],
+    [
+      11424,
+      336
+    ],
+    [
+      11760,
+      336
+    ],
+    [
+      12096,
+      336
+    ],
+    [
+      12432,
+      336
+    ],
+    [
+      12768,
+      336
+    ],
+    [
+      13104,
+      336
+    ],
+    [
+      13440,
+      336
+    ],
+    [
+      13776,
+      336
+    ],
+    [
+      14112,
+      336
+    ],
+    [
+      14448,
+      336
+    ],
+    [
+      14784,
+      336
+    ],
+    [
+      15120,
+      336
+    ],
+    [
+      15456,
+      336
+    ],
+    [
+      15792,
+      336
+    ],
+    [
+      16128,
+      336
+    ]
+  ],
+  "initializer_range": 0.1,
+  "intermediate_size": 8192,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 1000000000000000019884624838656,
+  "mm_hidden_size": 1024,
+  "mm_patch_merge_type": "spatial_unpad",
+  "mm_projector_lr": 2e-05,
+  "mm_projector_type": "linear",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "openai/clip-vit-large-patch14-336",
+  "mm_vision_tower_lr": 2e-06,
+  "model_type": "falcon_mamba",
+  "num_hidden_layers": 64,
+  "pad_token_id": 0,
+  "rescale_prenorm_residual": false,
+  "residual_in_fp32": true,
+  "state_size": 16,
+  "tie_word_embeddings": false,
+  "time_step_floor": 0.0001,
+  "time_step_init_scheme": "random",
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 256,
+  "time_step_scale": 1.0,
+  "tokenizer_model_max_length": 32896,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "tune_mm_mlp_adapter": false,
+  "unfreeze_mm_vision_tower": true,
+  "use_bias": false,
+  "use_cache": false,
+  "use_conv_bias": true,
+  "use_mambapy": false,
+  "use_mm_proj": true,
+  "vocab_size": 65024
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aaccec1a74afadb5d6ff20f9058f8d3e51de57a15e2e6c1ec0eafc120288944
+size 4956192312

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa82bbf5611a20f5730d1e98b96f4a7617b12db2de2aa5f4aedfe6e487fdeaef
+size 4987536920

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c917ebf03079fea8ff87d8d5fbfdde01f12957f4df5ddf55c1328785f2c3cc33
+size 4684478664

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10fdf037b3f4a45376b1dc85840b448ef433ad6cfa3a01da1acf1f60e4a2bf14
+size 532676736

special_tokens_map.json

@@ -0,0 +1,36 @@
+{
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    "assistant",
+    "<|begin_of_text|>",
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": ">>TITLE<<",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,129 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": ">>TITLE<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": ">>ABSTRACT<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": ">>INTRODUCTION<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": ">>SUMMARY<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": ">>COMMENT<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": ">>ANSWER<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": ">>QUESTION<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "assistant",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    "assistant",
+    "<|begin_of_text|>",
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "max_length": null,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 32896,
+  "pad_to_multiple_of": null,
+  "pad_token": ">>TITLE<<",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f3aa1a7744f55e0b995c5d995d21c3cbe39820b9d533547b6672503f8e2b9c
+size 7544