Upload folder using huggingface_hub

README.md

@@ -3,16 +3,16 @@
 This is a fine-tuned version of Qwen2.5-7B-Instruct optimized for agent tasks.
 
 ## Dataset Information
-- Train Dataset Size: 387 examples
-- Test Dataset Size: 96 examples
+- Train Dataset Size: 380 examples
+- Test Dataset Size: 94 examples
 
 ## Model Performance
-- Test Accuracy: 0.7983
-- Train Accuracy: 0.9606
+- Test Accuracy: 0.8811
+- Train Accuracy: 0.9912
 
 ## Training Configuration
 - Base Model: Qwen/Qwen2.5-VL-7B-Instruct
-- Checkpoint: checkpoints_27feb/run_20250228_004641/checkpoint-2425
+- Checkpoint: checkpoints_27feb/run_20250302_224224/checkpoint-5035
 - Dataset: AgentEvalDatapointDataset
 - Training Script: [train_transformer.py](train_transformer.py)
 - DeepSpeed Config: [deepspeed_config.json](deepspeed_config.json)

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:912a6e8beb806d1e74da4525dba6232b70ca5699d6fcd6e749191ac4b9350f61
+oid sha256:7e66be497d125879d760904220a2e7e9f170d93a532bd96120dead07347bb114
 size 4968243304

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d49fa25a0488877719f473f2a7e6952db490ee0076949367251e0b0bd1e8810e
+oid sha256:2fcb74ba81b4109788c6135e4ce8ba8044856585e9a0527d8ce857ac66b00859
 size 4991495816

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b1a7d618ae2a83635f0a56196c916e0e602cb68bae41468b7348614641195d8
+oid sha256:cbd6e723e4e3fb285f040dfe214dc318924853da918061fea20b6e8631b19aee
 size 4932751040

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d032ca82caa25a287aed4c0214ba117526479c3e54f27a753244575b4dd8577
+oid sha256:f191a1a83a9b7b616d0a85e5073b264fe89088fc00d5e442d47c0ba835d32c62
 size 1691924384

train_transformer.py

@@ -1,5 +1,7 @@
 import torch
 import gc
+import numpy as np
+import json
 
 torch.cuda.empty_cache()
 import torch.distributed
@@ -221,6 +223,21 @@ class CustomTrainingCallback(TrainerCallback):
     def __init__(self, trainer, eval_epoch_interval=2):
         self.trainer = trainer
         self.eval_epoch_interval = eval_epoch_interval
+        self.best_test_accuracy = 0.0
+        self.best_test_epoch = 0
+        self.best_metrics = {
+            'test_accuracy': 0.0,
+            'train_accuracy': 0.0,
+            'epoch': 0,
+            'global_step': 0
+        }
+
+    def save_best_metrics(self, output_dir):
+        """Save best metrics to a file in the checkpoint directory"""
+        metrics_file = os.path.join(output_dir, 'best_metrics.json')
+        with open(metrics_file, 'w') as f:
+            json.dump(self.best_metrics, f, indent=4)
+        print(f"Saved best metrics to {metrics_file}")
 
     def on_log(self, args, state, control, logs=None, **kwargs):
         """Log metrics at each logging step"""
@@ -257,10 +274,31 @@ class CustomTrainingCallback(TrainerCallback):
         self.trainer.model.eval()
 
         if (state.epoch + 1) % self.eval_epoch_interval == 0 and state.epoch > 4:
-            self.trainer.evaluate_step(dataset=self.trainer.eval_dataset, split="test")
-            self.trainer.evaluate_step(
-                dataset=self.trainer.train_dataset_eval, split="train"
-            )
+            # Get test accuracy
+            test_accuracy = self.trainer.evaluate_step(dataset=self.trainer.eval_dataset, split="test")
+            train_accuracy = self.trainer.evaluate_step(dataset=self.trainer.train_dataset_eval, split="train")
+            
+            print(f"Test accuracy: {test_accuracy:.4f}, Train accuracy: {train_accuracy:.4f}")
+         
+            # Update best test accuracy if current is better
+            if test_accuracy > self.best_test_accuracy:
+                self.best_test_accuracy = test_accuracy
+                self.best_test_epoch = state.epoch + 1
+                
+                # Update best metrics dictionary
+                self.best_metrics.update({
+                    'best_test_accuracy': float(test_accuracy),
+                    'train_accuracy': float(train_accuracy),
+                    'epoch': int(state.epoch + 1),
+                    'global_step': int(state.global_step)
+                })
+                
+                # Save best metrics to file
+                self.save_best_metrics(args.output_dir)
+                
+                # Log to wandb
+                
+                print(f"\nNew best test accuracy: {self.best_test_accuracy:.4f} at epoch {self.best_test_epoch}")
 
         if was_training:
             for_training(self.trainer.model)
@@ -327,7 +365,7 @@ class CustomSFTTrainer(SFTTrainer):
                 }
             )
 
-            # Don't finish wandb here to avoid conflicts with the training process
+            return accuracy  # Return the accuracy value
 
         except Exception as e:
             logger.error(f"Error evaluating: {e}")
@@ -430,7 +468,7 @@ def load_model(MODEL_ID: str, USE_QLORA: bool, training_args):
     return model, processor
 
 
-def main(args):
+def train(args):
     # Set CUDA device explicitly based on local_rank
     if args.local_rank != -1:
         torch.cuda.set_device(args.local_rank)
@@ -539,10 +577,10 @@ def main(args):
     # Pass training args to load_model function
     model, processor = load_model(args.model_id, args.use_qlora, training_args)
     # Train dataset
-    train_dataset = AgentDatapointDataset(split="train")
+    train_dataset = AgentDatapointDataset(split="train", num_samples=args.train_size)
     # Eval datasets
-    test_dataset = AgentEvalDatapointDataset(split="test")
-    train_dataset_eval = AgentEvalDatapointDataset(split="train")
+    test_dataset = AgentEvalDatapointDataset(split="test", num_samples=args.test_size)
+    train_dataset_eval = AgentEvalDatapointDataset(split="train", num_samples=args.train_size)
     for_training(model)
 
     trainer = CustomSFTTrainer(
@@ -615,5 +653,11 @@ if __name__ == "__main__":
     parser.add_argument(
         "--local_rank", type=int, default=-1, help="Local rank for distributed training"
     )
+    parser.add_argument(
+        "--train_size", type=int, default=10000000, help="Number of training samples"
+    )
+    parser.add_argument(
+        "--test_size", type=int, default=10000000, help="Number of test samples"
+    )
     args = parser.parse_args()
-    main(args)
+    train(args)