Create evaluate.py

evaluate.py

@@ -0,0 +1,32 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+from datasets import load_dataset
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+def evaluate_model(model_name):
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModel.from_pretrained(model_name)
+    except:
+        return None
+
+    dataset = load_dataset("persiannlp/STS-pairs", split="test[:100]")
+    embeddings1, embeddings2 = [], []
+
+    for item in dataset:
+        inputs1 = tokenizer(item["sentence1"], return_tensors="pt", truncation=True, padding=True)
+        inputs2 = tokenizer(item["sentence2"], return_tensors="pt", truncation=True, padding=True)
+
+        with torch.no_grad():
+            embed1 = model(**inputs1).last_hidden_state[:, 0, :]
+            embed2 = model(**inputs2).last_hidden_state[:, 0, :]
+
+        embeddings1.append(embed1.squeeze().numpy())
+        embeddings2.append(embed2.squeeze().numpy())
+
+    sims = [cosine_similarity([e1], [e2])[0][0] for e1, e2 in zip(embeddings1, embeddings2)]
+    labels = [item["similarity_score"] for item in dataset]
+
+    corr = np.corrcoef(sims, labels)[0, 1]
+    return float(corr)