internlm2 best results

llm_toolkit/eval_logical_reasoning.py

@@ -91,10 +91,10 @@ print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
 print(f"{start_gpu_memory} GB of memory reserved.")
 
 if adapter_name_or_path is not None:
-    model_name += "_" + adapter_name_or_path.split("/")[-1]
+    model_name += "/" + adapter_name_or_path.split("/")[-1]
 
 save_results(
-    model_name,
+    f"{model_name}_{dtype}{'_4bit' if load_in_4bit else ''}",
     results_path,
     datasets["test"],
     predictions,

scripts/eval-mgtv-best.sh

@@ -17,14 +17,30 @@ grep MemTotal /proc/meminfo
 #cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes] && cd $BASEDIR
 #pip install transformers==4.41.2
 
-export USING_LLAMA_FACTORY=true
+export USING_LLAMA_FACTORY=false
+export LOAD_IN_4BIT=false
 
 export MODEL_NAME=internlm/internlm2_5-7b-chat-1m
-export ADAPTER_NAME_OR_PATH=llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88
-#export ADAPTER_NAME_OR_PATH=inflaton-ai/InternLM_2_5-7b_LoRA-Adapter
+export ADAPTER_NAME_OR_PATH=inflaton-ai/InternLM_2_5-7b_LoRA-Adapter
 export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
 export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_internlm_best.csv
+
 export USE_FLOAT32_FOR_INFERENCE=true
+echo "Eval $MODEL_NAME with $ADAPTER_NAME_OR_PATH"
+python llm_toolkit/eval_logical_reasoning.py
+
+export USE_FLOAT32_FOR_INFERENCE=false
+export USE_BF16_FOR_INFERENCE=true
+echo "Eval $MODEL_NAME with $ADAPTER_NAME_OR_PATH"
+python llm_toolkit/eval_logical_reasoning.py
+
+export USE_FLOAT32_FOR_INFERENCE=false
+export USE_BF16_FOR_INFERENCE=false
+echo "Eval $MODEL_NAME with $ADAPTER_NAME_OR_PATH"
+python llm_toolkit/eval_logical_reasoning.py
 
+export LOAD_IN_4BIT=true
+export USE_FLOAT32_FOR_INFERENCE=false
+export USE_BF16_FOR_INFERENCE=true
 echo "Eval $MODEL_NAME with $ADAPTER_NAME_OR_PATH"
 python llm_toolkit/eval_logical_reasoning.py

scripts/eval-mgtv.sh

@@ -1 +1 @@
-eval-mgtv-internlm.sh
+eval-mgtv-best.sh