re-run qwen2.5 smaller models

llm_toolkit/eval_shots.py

@@ -30,6 +30,7 @@ batch_size = int(os.getenv("BATCH_SIZE", 1))
 using_llama_factory = os.getenv("USING_LLAMA_FACTORY") == "true"
 max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", 2048))
 start_num_shots = int(os.getenv("START_NUM_SHOTS", 0))
+end_num_shots = int(os.getenv("END_NUM_SHOTS", 50)
 
 print(
     model_name,
@@ -99,6 +100,7 @@ def evaluate_model_with_num_shots(
     model_name,
     data_path,
     start_num_shots=0,
+    end_num_shots=50,
     range_num_shots=[0, 5, 10, 20, 30, 40, 50],
     batch_size=1,
     max_new_tokens=2048,
@@ -109,6 +111,9 @@ def evaluate_model_with_num_shots(
     for num_shots in range_num_shots:
         if num_shots < start_num_shots:
             continue
+            
+        if num_shots > end_num_shots:
+            break
 
         print(f"*** Evaluating with num_shots: {num_shots}")
 
@@ -157,6 +162,7 @@ evaluate_model_with_num_shots(
     max_new_tokens=max_new_tokens,
     device=device,
     start_num_shots=start_num_shots,
+    end_num_shots=end_num_shots,
 )
 
 if is_cuda:

scripts/eval-mgtv-qwen2.5_3b.sh

@@ -13,18 +13,18 @@ cat /etc/os-release
 lscpu
 grep MemTotal /proc/meminfo
 
-# $BASEDIR/scripts/eval-shots.sh Qwen Qwen2.5-3B-Instruct
+export USING_LLAMA_FACTORY=true
+export LOAD_IN_4BIT=false
 
-# $BASEDIR/scripts/eval-shots.sh Qwen Qwen2.5-1.5B-Instruct
+$BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-3B-Instruct
 
-# $BASEDIR/scripts/eval-shots.sh Qwen Qwen2.5-0.5B-Instruct
+$BASEDIR/scripts/eval-shots.sh Qwen Qwen2.5-3B-Instruct
 
-# $BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-3B-Instruct
+$BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-1.5B-Instruct
 
-# $BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-1.5B-Instruct
+$BASEDIR/scripts/eval-shots.sh Qwen Qwen2.5-1.5B-Instruct
 
-# $BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-0.5B-Instruct
+$BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-0.5B-Instruct
 
-export START_NUM_SHOTS=40
 $BASEDIR/scripts/eval-shots.sh Qwen Qwen2.5-0.5B-Instruct
 

scripts/eval-mgtv.sh

@@ -1 +1 @@
-eval-mgtv-qwen2.5_4bit.sh
+eval-mgtv-qwen2.5_3b.sh