File size: 5,659 Bytes
4ccff65 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 |
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"source": [
"#upload the fine_tuned_model.zip and narrative_texts.csv then run the code for evaluation\n",
"\n",
"import zipfile\n",
"import os\n",
"\n",
"#if the folder doesn't exist already, then extract the model\n",
"if not os.path.exists(\"fine_tuned_model\"):\n",
" with zipfile.ZipFile(\"fine_tuned_model.zip\", 'r') as zip_ref:\n",
" zip_ref.extractall(\"fine_tuned_model\") #extract all model files into the target folder\n",
"\n",
"print(\"Model extracted successfully.\") #confirmation message"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "9iMmMqqB6Hf_",
"outputId": "cb0c6eb8-6650-4087-9bb7-078ec6012375"
},
"execution_count": 4,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Model extracted successfully.\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import torch #for deep learning\n",
"from transformers import BertTokenizer, BertForSequenceClassification #model training in bert\n",
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score #evaulation metrics\n",
"import pandas as pd\n",
"import re #regex\n",
"\n",
"#load fine-tuned model and tokenizer\n",
"model_path = \"./fine_tuned_model\"\n",
"tokenizer = BertTokenizer.from_pretrained(model_path)\n",
"model = BertForSequenceClassification.from_pretrained(model_path)\n",
"model.eval() #set model to evaluation mode\n",
"\n",
"#load dataset and normalize the text\n",
"df = pd.read_csv(\"narrative_texts.csv\")\n",
"df['text'] = df['text'].str.lower() #convert to lowercase\n",
"df['text'] = df['text'].apply(lambda x: re.sub(r'[^a-z\\s]', '', x)) #remove non-alphabetic characters\n",
"df['text'] = df['text'].apply(lambda x: re.sub(r'\\s+', ' ', x).strip()) #clean extra spaces\n",
"\n",
"#function to swap gendered words in text\n",
"def gender_swap(text):\n",
" swaps = {\n",
" \" he \": \" TEMP \", \" she \": \" he \", \" TEMP \": \" she \",\n",
" \" his \": \" TEMP2 \", \" her \": \" his \", \" TEMP2 \": \" her \",\n",
" \" him \": \" TEMP3 \", \" her \": \" him \", \" TEMP3 \": \" her \"\n",
" }\n",
" for key, value in swaps.items():\n",
" text = text.replace(key, value)\n",
" return text\n",
"\n",
"#generate swapped gender versions of each sentence\n",
"df['text_swapped'] = df['text'].apply(lambda x: gender_swap(\" \" + x + \" \"))\n",
"\n",
"#create a mixed dataset of original and swapped texts\n",
"df_mixed = pd.concat([df['text'], df['text_swapped']], ignore_index=True)\n",
"labels_mixed = [0] * len(df) + [1] * len(df) #label 0 for original, 1 for swapped\n",
"\n",
"#function to evaluate model performance\n",
"def evaluate_model(texts, labels):\n",
" inputs = tokenizer(texts.tolist(), truncation=True, padding=True, return_tensors=\"pt\", max_length=128)\n",
"\n",
" with torch.no_grad():\n",
" outputs = model(**inputs)\n",
" logits = outputs.logits\n",
" preds = torch.argmax(logits, dim=1).numpy()\n",
"\n",
" acc = accuracy_score(labels, preds)\n",
" precision = precision_score(labels, preds)\n",
" recall = recall_score(labels, preds)\n",
" f1 = f1_score(labels, preds)\n",
"\n",
" return {\n",
" \"Accuracy\": round(acc, 4),\n",
" \"Precision\": round(precision, 4),\n",
" \"Recall\": round(recall, 4),\n",
" \"F1 Score\": round(f1, 4)\n",
" }"
],
"metadata": {
"id": "xnCn3rmr62nN"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#evaluating the model on both original and gender-swapped text\n",
"metrics = evaluate_model(df_mixed, labels_mixed)\n",
"\n",
"#printing out the evaluation results\n",
"print(\"Model Evaluation Results:\")\n",
"for metric, value in metrics.items():\n",
" print(f\"{metric}: {value}\") #prints each metric and its value one by one"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Tyn_TmKo7USd",
"outputId": "75ae6a93-a783-4357-fd13-d9441a8a7744"
},
"execution_count": 7,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Model Evaluation Results:\n",
"Accuracy: 0.55\n",
"Precision: 0.5385\n",
"Recall: 0.7\n",
"F1 Score: 0.6087\n"
]
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "GfvTDUPp7Wi1"
},
"execution_count": null,
"outputs": []
}
]
} |