NOTEBOOK_FINE-TUNING_XLM-ROBERTA-BASE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PS0DUxR1S-bl"
   },
   "source": [
    "# CODE FOR FINE-TUNING XLM-ROBERTA-BASE 💗🧸\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "csEDxO3zNuYH"
   },
   "source": [
    "The purpose of sharing this code is to show how to fine-tune an XLM-Roberta-Base model. Personally, I used Google Colab because of the GPU."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z-SNvmJSTbqL"
   },
   "source": [
    "If you want to make inferences with my model I have carried out pre-processing or data-cleaning operations, I share them below.\n",
    "\n",
    "\n",
    "**REMEMBER TO DO SAME CLEANING OPERATION THAT ARE PERFORMED IN DF USED FOR FINE-TUNING ALSO IN TEXT WHEN YOU DO INFERENCE AND RENAME YOUR DF COLUMNS IN \"labels\" AND \"text\" ❤**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Vz3uYDCWTm1U"
   },
   "source": [
    "\n",
    "\n",
    "```\n",
    "\n",
    "# Function for removing personal information in ticket's text\n",
    "def identify_names(word:str, name:list=name, surname:list=surname) -> bool:\n",
    "    \"\"\"\n",
    "    :param word: string with one word\n",
    "    :param name: list that contains a sequence of italian names\n",
    "    :param surname: list that contains a sequence of italian surnames\n",
    "    :return boolean true if it's a name or a surname, false otherwise\n",
    "    Verify if a word it's name or a surname\n",
    "    \"\"\"\n",
    "    return word in name or word in surname\n",
    "\n",
    "# Function for removing cities from ticket's text\n",
    "def identify_location(ticket:str, location:list=location) -> str:\n",
    "    \"\"\"\n",
    "    :param ticket: string with ticket description\n",
    "    :param location: list that contains a sequence of italian cities\n",
    "    :return cleaning string without location\n",
    "    Verify if a word it's name or a surname\n",
    "    \"\"\"\n",
    "    pattern = r'\\b(?:' + '|'.join(map(re.escape, location)) + r')\\b'\n",
    "    ticket = re.sub(pattern, '', ticket, flags=re.IGNORECASE)\n",
    "    return ticket.strip()\n",
    "\n",
    "# Function for filter text from noises\n",
    "def extract_text_from_email(text:str)-> str:\n",
    "    \"\"\"\n",
    "    :param text: string with the text of one ticket\n",
    "    :return string the input text cleaning from noises and personal information\n",
    "    Verify if a word it's name or a surname\n",
    "    \"\"\"\n",
    "    if isinstance(text, str):\n",
    "      pattern = pattern = r\"={2,}(.*?)(grazie\\s*[,!.\\s]*|cordiali saluti\\s*[,!.\\s]*|buona giornata\\s*[,!.\\s]*|a presto\\s*[,!.\\s]*|Grazie\\s*[,!.\\s]*|Cordiali Saluti\\s*[,!.\\s]*|Buona Giornata\\s*[,!.\\s]*|A Presto\\s*[,!.\\s]*)\"\n",
    "      match = re.search(pattern, text, re.DOTALL)\n",
    "      if match:\n",
    "          extract_text = match.group(1).strip()\n",
    "          return extract_text\n",
    "      else:\n",
    "          return text\n",
    "    else:\n",
    "      return text\n",
    "\n",
    "# Function for cleaning text by apply identify_names, extract_text_from_email functions and more cleaning operations\n",
    "def clean_text(text: str, customers:list=customers) -> str:\n",
    "    \"\"\"\n",
    "    :param text: string with the text of one ticket\n",
    "    :return string the input text cleaning\n",
    "    Cleaning text\n",
    "    \"\"\"\n",
    "    # 1. Remove link\n",
    "    text = extract_text_from_email(text)\n",
    "    text = re.sub(r'http[s]?://\\S+', '', text)\n",
    "    text = re.sub(r'http[s]?://\\S+|www\\.\\S+', '', text)\n",
    "    # 2. Remove email\n",
    "    text = re.sub(r'\\S+@\\S+\\.\\S+', '', text)\n",
    "    # 3. Remove telephone number\n",
    "    text = re.sub(r'\\+?\\d{1,3}[-.\\s]?\\(?\\d{1,4}\\)?[-.\\s]?\\d{1,4}[-.\\s]?\\d{1,9}', '', text)\n",
    "    # 4. Remove special characters\n",
    "    text = re.sub(r'[=*/\\-+[\\]{}(),:;<>]', '', text)\n",
    "    text = re.sub(r'\\b\\w*_\\w*\\b', '', text)\n",
    "    # 5. Remove interruption roe\n",
    "    text = text.replace('\\n', ' ').replace('\\r', ' ')\n",
    "    # 6. Remove multiple white space\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "    # 7. Remove names\n",
    "    words = text.split()\n",
    "    clean_text = [word for word in words if not identify_names(word)]\n",
    "    noise_location = ['sede','via','città','location']\n",
    "    clean_text = [word for word in clean_text if word not in noise_location]\n",
    "    clean_text = ' '.join(clean_text)\n",
    "    clean_text = identify_location(clean_text)\n",
    "    for i in customers:\n",
    "       if i in clean_text:\n",
    "          clean_text = clean_text.replace(i, '').strip()\n",
    "          break\n",
    "    # Remove images\n",
    "    pattern = r\"\\b\\w+\\.(?:png|jpe?g|gif|bmp|tiff|webp)\\b\"\n",
    "    clean_text = re.sub(pattern, '', clean_text, flags=re.IGNORECASE)  # Case insensitive matching\n",
    "    clean_text = clean_text.strip()\n",
    "    return clean_text\n",
    "```\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hqZaY8WoOA5y"
   },
   "source": [
    "## INSTALL REQUIREMENTS AND IMPORT LIBRARIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lGwh00k_jPGl"
   },
   "outputs": [],
   "source": [
    "! pip install datasets transformers==4.44 sentencepiece evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dwg67VFnOI5r"
   },
   "source": [
    "The correct version of Numpy is very important to avoid conflicts with the transformers library, it requires a version lower than 2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lov-VsxMbyH4"
   },
   "outputs": [],
   "source": [
    "pip install \"numpy<2.0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "V6NsUqwzJYma"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import shutil\n",
    "import datasets\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, pipeline,DataCollatorWithPadding\n",
    "from datasets import Dataset, DatasetDict\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "import re\n",
    "import evaluate\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# THIS IS NOT MANDATORY\n",
    "import wandb\n",
    "wandb.init(mode=\"disabled\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Tr-0jUbeFPYV"
   },
   "source": [
    "# LABEL DEFINITION, MAPPING AND FUNCTIONS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WV6ITe2iPKAf"
   },
   "source": [
    "Mapping is critical because it is used within the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "O2SnJ8F0JBmu"
   },
   "outputs": [],
   "source": [
    "label2id = {\n",
    "    \"Database-DB2\": 4,\n",
    "    \"Database-MS SQL Server\": 15,\n",
    "    \"Database-Oracle\": 5,\n",
    "    \"Hardware-CPU\": 1,\n",
    "    \"Hardware-Disk\": 6,\n",
    "    \"Hardware-Keyboard\": 3,\n",
    "    \"Hardware-Memory\": 13,\n",
    "    \"Hardware-Monitor\": 8,\n",
    "    \"Hardware-Mouse\": 17,\n",
    "    \"Inquiry/Help-Antivirus\": 11,\n",
    "    \"Inquiry/Help-Internal Application\": 7,\n",
    "    \"Network-DHCP\": 14,\n",
    "    \"Network-DNS\": 12,\n",
    "    \"Network-IP Address\": 0,\n",
    "    \"Network-VPN\": 10,\n",
    "    \"Network-Wireless\": 9,\n",
    "    \"Software-Email\": 16,\n",
    "    \"Software-Operating System\": 2\n",
    "  }\n",
    "\n",
    "id2label= {\n",
    "    \"0\": \"Network-IP Address\",\n",
    "    \"1\": \"Hardware-CPU\",\n",
    "    \"2\": \"Software-Operating System\",\n",
    "    \"3\": \"Hardware-Keyboard\",\n",
    "    \"4\": \"Database-DB2\",\n",
    "    \"5\": \"Database-Oracle\",\n",
    "    \"6\": \"Hardware-Disk\",\n",
    "    \"7\": \"Inquiry/Help-Internal Application\",\n",
    "    \"8\": \"Hardware-Monitor\",\n",
    "    \"9\": \"Network-Wireless\",\n",
    "    \"10\": \"Network-VPN\",\n",
    "    \"11\": \"Inquiry/Help-Antivirus\",\n",
    "    \"12\": \"Network-DNS\",\n",
    "    \"13\": \"Hardware-Memory\",\n",
    "    \"14\": \"Network-DHCP\",\n",
    "    \"15\": \"Database-MS SQL Server\",\n",
    "    \"16\": \"Software-Email\",\n",
    "    \"17\": \"Hardware-Mouse\"\n",
    "  }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oYrIxDytPFOu"
   },
   "outputs": [],
   "source": [
    "accuracy = evaluate.load(\"accuracy\")\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return accuracy.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Lgyt6owmTc27"
   },
   "source": [
    "##  IMPORTANT CHECK\n",
    "Please check that you have balanced and proportionate classes, otherwise below you will find 2 versions:\n",
    "- Fine-tuning for balanced classes\n",
    "- Fine-tuning for unbalanced classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QeZAYpTkuCoA"
   },
   "outputs": [],
   "source": [
    "df = pd.read_excel('your_file.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lGD4baiputPJ"
   },
   "outputs": [],
   "source": [
    "check_balanced_class = df.groupby('label').size().reset_index(name='count').sort_values(by='count')\n",
    "check_balanced_class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "chKa0cwtJu6o"
   },
   "source": [
    "# Fine Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZB3pJT4QJxWl"
   },
   "outputs": [],
   "source": [
    "model_checkpoint = \"FacebookAI/xlm-roberta-base\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
    "def preprocess_function(examples):\n",
    "    return tokenizer(examples[\"text\"], truncation=True)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XT6d52KqFdG6"
   },
   "source": [
    "The tokenization process outputs 2 things:\n",
    "- input_ids is text encoding\n",
    "- attention_mask tokens to ignore for the transformer's self-attention mechanism\n",
    "\n",
    "\n",
    "**data_collator** it's usefull becaus It is used to create batches of sequences of variable lengths efficiently by adding dynamic padding to even out the length of the sequences within the batch. Padding is necessary because deep learning models require inputs of constant length in order to be processed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p6KorhLmQmmc"
   },
   "source": [
    "Remember to map to your classes and switch from string type to integer numeric type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KoDXWXaMQmHw"
   },
   "outputs": [],
   "source": [
    "df['label'] = df['label'].map(label2id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hQ_ybPUyQtsT"
   },
   "source": [
    "We remanipulate the shape of the dataset so that we have the desired shape for training with DatasetDict function. Important part is to choose to split your dataset in a stratify way, with: **stratify=df['label']**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VFMn5eBSchZ3"
   },
   "outputs": [],
   "source": [
    "# spli df in training and test with stratift way based on your labels\n",
    "train_set, test_set = train_test_split(df, test_size=0.2, stratify=df['label'], random_state=42)\n",
    "\n",
    "train_set = Dataset.from_dict(train_set)\n",
    "test_set = Dataset.from_dict(test_set)\n",
    "\n",
    "# Manipulate shape of df\n",
    "df_dict = datasets.DatasetDict({\"train\":train_set,\"test\":test_set})\n",
    "\n",
    "# Tokenization and padding\n",
    "tokenized= df_dict.map(preprocess_function, batched=True)\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nsTiJf23RUaF"
   },
   "source": [
    "## FINE TUNING WITH BALANCED CLASS\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hcS29WR6QDGy",
    "outputId": "da21f2dc-1220-40c9-a785-8c45f4dda727"
   },
   "outputs": [],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=len(id2label), id2label=id2label, label2id=label2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6uBDXlUSWQfd"
   },
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"\", #SPECIFY YOUR OUTPUT DIRECTORY\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    num_train_epochs=7,\n",
    "    weight_decay=0.01,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    load_best_model_at_end=True,\n",
    "    #push_to_hub=False,\n",
    ")\n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized[\"train\"],\n",
    "    eval_dataset=tokenized[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BS1cT7xOZEfd"
   },
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hp2CoEQQKxQM"
   },
   "outputs": [],
   "source": [
    "trainer.save_model(\"\") #SPECIFY WHERE SAVE MODEL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8duf4N5fRtz-"
   },
   "source": [
    "If you want to save a local version for download you can zip the template, select the last checkpoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8xrL_XZqO2xQ"
   },
   "outputs": [],
   "source": [
    "shutil.make_archive('path where save the zip', 'zip', 'path to last checkpoint that contains the model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VIf0qmmOWNtA"
   },
   "source": [
    "## FINE TUNING WITH UNBALANCED CLASS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OxckCVGvSW9h"
   },
   "source": [
    "## IMPORT ADDITIONAL LIBRARIES AND FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7J0SBDE_tokB"
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# Count the labels of each class in the training dataset\n",
    "label_counts = Counter(tokenized[\"train\"][\"label\"])\n",
    "\n",
    "# Calculate weights inversely proportional to frequency\n",
    "total_samples = len(tokenized[\"train\"][\"label\"])\n",
    "class_weights = {label: total_samples / count for label, count in label_counts.items()}\n",
    "\n",
    "# Convert to a tensor to pass to PyTorch\n",
    "class_weights_tensor = torch.tensor([class_weights[i] for i in range(len(class_weights))], dtype=torch.float)\n",
    "\n",
    "# Define a custom loss function\n",
    "class WeightedTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.get(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.get(\"logits\")\n",
    "\n",
    "        if labels is None:\n",
    "            return outputs if return_outputs else None\n",
    "        loss_fct = nn.CrossEntropyLoss(weight=class_weights_tensor.to(logits.device))\n",
    "        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6Ts1wkt9T99-"
   },
   "source": [
    "## Fine-tuning Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-_wNVfq6T8TF"
   },
   "outputs": [],
   "source": [
    "model_checkpoint = \"FacebookAI/xlm-roberta-base\"\n",
    "batch_size = 16\n",
    "num_train_epochs = 8\n",
    "logging_steps = len(tokenized[\"train\"]) // (batch_size * num_train_epochs)\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=len(id2label), id2label=id2label, label2id=label2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lugCcXL8R3iT"
   },
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"\",#SPECIFY YOUR OUTPUT DIRECTORY\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    save_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=num_train_epochs,\n",
    "    weight_decay=0.01,\n",
    "    logging_steps=logging_steps,\n",
    "    report_to=\"none\",\n",
    ")\n",
    "\n",
    "trainer = WeightedTrainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=tokenized[\"train\"],\n",
    "    eval_dataset=tokenized[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bcQvM_aHSo_V"
   },
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VFc9RfrcUUIJ"
   },
   "outputs": [],
   "source": [
    "trainer.save_model(\"\") #SPECIFY WHERE SAVE MODEL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zE1LdtalUc3Q"
   },
   "source": [
    "If you want to save a local version for download you can zip the template, select the last checkpoint.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8ISI1LGqUbZy"
   },
   "outputs": [],
   "source": [
    "shutil.make_archive('path where save the zip', 'zip', 'path to last checkpoint that contains the model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "g645l1jfUC_m"
   },
   "source": [
    "## INFERENCE WITH MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "W7ejTyHhTDTl"
   },
   "outputs": [],
   "source": [
    "finetuned_checkpoint = \"\" # PATH OF YOUR LAST CHECKPOINT OF FINE-TUNED MODEL\n",
    "classifier = pipeline(\"text-classification\", model=finetuned_checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RvigAskV_eA2"
   },
   "outputs": [],
   "source": [
    "text = clean_text(\"\"\"vpn not working\"\"\")\n",
    "classifier(text)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}