File size: 4,513 Bytes
dea079c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"source": [
"# Install necessary libraries\n",
"!pip install datasets\n",
"\n",
"# Importing required libraries for dataset, model, tokenizer, training, and evaluation\n",
"from datasets import load_dataset\n",
"from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments\n",
"from sklearn.model_selection import train_test_split\n",
"import torch\n",
"from IPython import get_ipython\n",
"from IPython.display import display"
],
"metadata": {
"id": "HrXBpUDmLbKH"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# 1. Load Dataset\n",
"# Load the Bengali transliteration dataset\n",
"raw_dataset = load_dataset(\"SKNahin/bengali-transliteration-data\")\n",
"\n",
"# Split dataset into training and validation sets (90% training, 10% validation)\n",
"train_data = raw_dataset['train'].train_test_split(test_size=0.1, seed=42)['train'] # Added seed for reproducibility\n",
"val_data = raw_dataset['train'].train_test_split(test_size=0.1, seed=42)['test'] # Added seed for reproducibility\n",
"\n",
"# 2. Preprocessing\n",
"# Define model name for T5 and load its tokenizer\n",
"model_name = \"google/mt5-small\"\n",
"tokenizer = T5Tokenizer.from_pretrained(model_name)\n",
"\n",
"# Tokenize function for preprocessing the data\n",
"def preprocess_data(examples):\n",
" # Tokenize input and target sequences with padding and truncation to fixed length\n",
" # Access the correct columns based on the dataset structure\n",
" inputs = tokenizer(examples['bn'], padding=\"max_length\", truncation=True, max_length=128)\n",
" targets = tokenizer(examples['rm'], padding=\"max_length\", truncation=True, max_length=128)\n",
"\n",
" # Assign the tokenized target as labels for the model\n",
" inputs['labels'] = targets['input_ids']\n",
" return inputs\n",
"\n",
"# Apply the preprocessing function to the training and validation datasets\n",
"train_dataset = train_data.map(preprocess_data, batched=True)\n",
"val_dataset = val_data.map(preprocess_data, batched=True)\n",
"\n",
"# 3. Model Selection\n",
"# Load the T5 model for conditional generation\n",
"model = T5ForConditionalGeneration.from_pretrained(model_name)\n",
"\n",
"# 4. Training\n",
"# Define training arguments like learning rate, batch size, number of epochs, etc.\n",
"training_args = TrainingArguments(\n",
" output_dir=\"./results\", # Directory to store the results\n",
" evaluation_strategy=\"epoch\", # Evaluate model after each epoch\n",
" learning_rate=5e-5, # Learning rate\n",
" per_device_train_batch_size=16, # Batch size for training\n",
" per_device_eval_batch_size=16, # Batch size for evaluation\n",
" num_train_epochs=5, # Number of training epochs\n",
" weight_decay=0.01, # Weight decay for regularization\n",
" save_steps=1000, # Save model every 1000 steps\n",
" save_total_limit=2, # Keep only 2 most recent checkpoints\n",
" logging_dir=\"./logs\", # Directory to store logs\n",
" logging_steps=500, # Log every 500 steps\n",
")\n"
],
"metadata": {
"id": "kxHZMLRKPr6L"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# Initialize the Trainer with the model, arguments, and datasets\n",
"trainer = Trainer(\n",
" model=model,\n",
" args=training_args,\n",
" train_dataset=train_dataset,\n",
" eval_dataset=val_dataset\n",
")\n",
"\n",
"# Start the training process\n",
"trainer.train()"
],
"metadata": {
"id": "9_9U0GCCoTqZ"
},
"execution_count": null,
"outputs": []
}
]
} |