Upload folder using huggingface_hub

README.md

@@ -1,36 +1,21 @@
 ---
-license: mit
-language:
-- sw
+library_name: peft
 ---
-TigerBot-7B LAPT + Heuristics Swahili
-===
+## Training procedure
 
-## How to use
-```python
-from peft import AutoPeftModelForCausalLM
-from transformers import AutoTokenizer
 
-model = AutoPeftModelForCausalLM.from_pretrained(
-  "atsuki-yamaguchi/tigerbot-7b-base-heuristics-sw"
-)
-tokenizer = AutoTokenizer.from_pretrained(
-  "atsuki-yamaguchi/tigerbot-7b-base-heuristics-sw"
-)
-```
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
 
-## Citation
-```
-@article{yamaguchi2024empirical,
-  title={An Empirical Study on Cross-lingual Vocabulary Adaptation for Efficient Generative {LLM} Inference}, 
-  author={Atsuki Yamaguchi and Aline Villavicencio and Nikolaos Aletras},
-  journal={ArXiv},
-  year={2024},
-  volume={abs/2402.10712},
-  url={https://arxiv.org/abs/2402.10712}
-}
-```
-
-## Link
-For more details, please visit https://github.com/gucci-j/llm-cva
 
+- PEFT 0.5.0

adapter_config.json

@@ -1,29 +1 @@
-{
-  "auto_mapping": null,
-  "base_model_name_or_path": "atsuki-yamaguchi/tigerbot-7b-base-heuristics-sw",
-  "bias": "none",
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "lora_alpha": 32,
-  "lora_dropout": 0.05,
-  "modules_to_save": [
-    "lm_head",
-    "embed_tokens"
-  ],
-  "peft_type": "LORA",
-  "r": 8,
-  "revision": null,
-  "target_modules": [
-    "q_proj",
-    "v_proj",
-    "k_proj",
-    "o_proj",
-    "gate_proj",
-    "down_proj",
-    "up_proj"
-  ],
-  "task_type": "CAUSAL_LM"
-}
+{"auto_mapping": null, "base_model_name_or_path": "atsuki-yamaguchi/tigerbot-7b-base-heuristics-sw", "bias": "none", "fan_in_fan_out": false, "inference_mode": true, "init_lora_weights": true, "layers_pattern": null, "layers_to_transform": null, "lora_alpha": 32, "lora_dropout": 0.05, "modules_to_save": ["lm_head", "embed_tokens"], "peft_type": "LORA", "r": 8, "revision": null, "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "down_proj", "up_proj"], "task_type": "CAUSAL_LM"}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/tigerbot-7b-base-sw-heuristics",
+  "_name_or_path": "TigerResearch/tigerbot-7b-base",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -21,7 +21,7 @@
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
+  "torch_dtype": "float64",
   "transformers_version": "4.35.0.dev0",
   "use_cache": true,
   "vocab_size": 50264

model-00001-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2a2ffad4267c12c8d32c5b4cb643c8e6a57b2b79f2065acc7a1d3b571226935
-size 4938306856
+oid sha256:532be22851bb77b3d0dcf25562f0646535ff14737af6538cf659da2b78cdabcb
+size 4952297760

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c0f990f310101840de528cc3529a2dfb820d8cf33ba93c3e24ffd75206d7533
-size 4991424816
+oid sha256:a9c37a245ee5ad4472f4260ab2adc127ec752a1ef2ffe0a61148affecc49caaf
+size 4991424808

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:636f81aa82b462ecc0c32ca267e10bbd405f978003f00255317c4e416a90a263
+oid sha256:8348f7bd98b1fc9689bb7b369d2ca4207d31f52c2743fdd218f9993d4eb0c613
 size 4924315880

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6e5060c0336d0f5ec66cc007a3428542f6946f32027a15b888844006b4c4158
+oid sha256:18f0fdc5a923b1f8ea222154381f364870ab577211eb72fc909d5e23c054b108
 size 4857206904

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48de40589456afa1eb370c85d74d127f9526ab536facc7e03a18805607113fcc
+oid sha256:6af6f4bc0f17b80add0bd36b79310b37014cad2539a442ed43c7ca140f67eb77
 size 4857206904

model-00006-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e28b2e0f832c68d560ca60e48f537fef849bc80a4cd9ef7a940c0a3355d2abb0
-size 2983709496
+oid sha256:be87d1220ad8ec0e251cbe952af8b735b007bab21c646732421e6687c3a8ac8b
+size 3793243976

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 27552137216
+    "total_size": 28375662592
   },
   "weight_map": {
     "lm_head.weight": "model-00006-of-00006.safetensors",
@@ -23,13 +23,13 @@
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
     "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00003-of-00006.safetensors",
@@ -37,10 +37,10 @@
     "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
@@ -77,11 +77,11 @@
     "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
@@ -91,10 +91,10 @@
     "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.18.input_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
@@ -140,11 +140,11 @@
     "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
     "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
@@ -154,10 +154,10 @@
     "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00005-of-00006.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
@@ -194,11 +194,11 @@
     "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
     "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
@@ -208,10 +208,10 @@
     "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
-    "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
     "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
     "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
@@ -239,15 +239,15 @@
     "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
@@ -255,7 +255,7 @@
     "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af0ff80a7465b87b4c4dca60afbf1360d89ee87bb0c56e18b7b6499655878709
+size 865706972

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a8f3b29e0bd2cc11dc0bd5c8d6115a7d91eafaa4a330189e237f189325c2c38
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d403cd2fa0f47b387bc486ed7a0a0523b4d4d12118efa47d66edfd3fa514ef0
+size 1064

special_tokens_map.json

@@ -1,23 +1,5 @@
 {
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
 }

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b244ab5c166e93da6b6a49e8c27efbe55a805da74152d96651e3f7f526154e26
+size 4664