Update evaluate.py

evaluate.py

@@ -1,32 +1,19 @@
-from transformers import AutoTokenizer, AutoModel
-import torch
 from datasets import load_dataset
-from sklearn.metrics.pairwise import cosine_similarity
-import numpy as np
+from sentence_transformers import SentenceTransformer, util
 
 def evaluate_model(model_name):
     try:
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModel.from_pretrained(model_name)
-    except:
-        return None
-
-    dataset = load_dataset("persiannlp/STS-pairs", split="test[:100]")
-    embeddings1, embeddings2 = [], []
-
-    for item in dataset:
-        inputs1 = tokenizer(item["sentence1"], return_tensors="pt", truncation=True, padding=True)
-        inputs2 = tokenizer(item["sentence2"], return_tensors="pt", truncation=True, padding=True)
+        model = SentenceTransformer(model_name)
+        dataset = load_dataset("arshiaafshani/persian-natural-fluently", split="train[:200]")
 
-        with torch.no_grad():
-            embed1 = model(**inputs1).last_hidden_state[:, 0, :]
-            embed2 = model(**inputs2).last_hidden_state[:, 0, :]
+        scores = []
+        for row in dataset:
+            emb1 = model.encode(row["instruction"], convert_to_tensor=True)
+            emb2 = model.encode(row["output"], convert_to_tensor=True)
+            sim_score = float(util.cos_sim(emb1, emb2)[0])
+            scores.append(sim_score)
 
-        embeddings1.append(embed1.squeeze().numpy())
-        embeddings2.append(embed2.squeeze().numpy())
-
-    sims = [cosine_similarity([e1], [e2])[0][0] for e1, e2 in zip(embeddings1, embeddings2)]
-    labels = [item["similarity_score"] for item in dataset]
-
-    corr = np.corrcoef(sims, labels)[0, 1]
-    return float(corr)
+        return sum(scores) / len(scores)
+    except Exception as e:
+        print(f"Evaluation failed: {e}")
+        return None