text_plain_output_1.png
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler

from transformers import AdamW, get_linear_schedule_with_warmup

from transformers import AutoTokenizer, AutoModel

import numpy as np

import random

import re

import time

import torch

import torch

import torch.nn as nn

import unicodedata

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

tokenizer = AutoTokenizer.from_pretrained('asafaya/bert-mini-arabic')

def text_preprocessing(text):
    text = unicodedata.normalize('NFC', text)
    text = re.sub('(@.*?)[\\s]', ' ', text)
    text = re.sub('&amp', '&', text)
    text = re.sub('\\s+', ' ', text).strip()
    text = re.sub('^https?:\\/\\/.*[\\r\\n]*', '<URL>', text)
    return text

import emoji
import unicodedata
def preprocessing_for_bert(data, version='mini', text_preprocessing_fn=text_preprocessing):
    input_ids = []
    attention_masks = []
    tokenizer = AutoTokenizer.from_pretrained('asafaya/bert-mini-arabic') if version == 'mini' else AutoTokenizer.from_pretrained('asafaya/bert-base-arabic')
    for i, sent in enumerate(data):
        encoded_sent = tokenizer.encode_plus(text=text_preprocessing_fn(sent), add_special_tokens=True, max_length=MAX_LEN, padding='max_length', return_attention_mask=True, truncation=True)
        input_ids.append(encoded_sent.get('input_ids'))
        attention_masks.append(encoded_sent.get('attention_mask'))
    input_ids = torch.tensor(input_ids)
    attention_masks = torch.tensor(attention_masks)
    return (input_ids, attention_masks)

from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
train_labels = torch.tensor(y_train)
val_labels = torch.tensor(y_val)
batch_size = 16

train_data = TensorDataset(train_inputs, train_masks, train_labels)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
val_data = TensorDataset(val_inputs, val_masks, val_labels)
val_sampler = RandomSampler(val_data)
val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)

from transformers import AdamW, get_linear_schedule_with_warmup
from torch.optim import SparseAdam, Adam
def initialize_model(epochs=4, version='mini'):
    bert_classifier = BertClassifier(freeze_bert=False, version=version)
    bert_classifier.to(device)
    optimizer = AdamW(params=list(bert_classifier.parameters()), lr=5e-05, eps=1e-08)
    total_steps = len(train_dataloader) * epochs
    schedular = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)
    return (bert_classifier, optimizer, schedular)

import random
import time
loss_fn = nn.CrossEntropyLoss()
def set_seed(seed_value=42):
    random.seed(seed_value)
    np.random.seed(seed_value)
    torch.manual_seed(seed_value)
    torch.cuda.manual_seed_all(seed_value)
def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):
    for epoch_i in range(epochs):
        t0_epoch, t0_batch = (time.time(), time.time())
        total_loss, batch_loss, batch_counts = (0, 0, 0)
        model.train()
        for step, batch in enumerate(train_dataloader):
            batch_counts += 1
            b_input_ids, b_attn_mask, b_labels = tuple((t.to(device) for t in batch))
            model.zero_grad()
            logits = model(b_input_ids, b_attn_mask)
            loss = loss_fn(logits, b_labels)
            batch_loss += loss.item()
            total_loss += loss.item()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            scheduler.step()
            if step % 100 == 0 and step != 0 or step == len(train_dataloader) - 1:
                time_elapsed = time.time() - t0_batch
                batch_loss, batch_counts = (0, 0)
                t0_batch = time.time()
        avg_train_loss = total_loss / len(train_dataloader)
        if evaluation == True:
            val_loss, val_accuracy = evaluate(model, val_dataloader)
            time_elapsed = time.time() - t0_epoch

set_seed(42)
bert_classifier, optimizer, scheduler = initialize_model(epochs=2)
train(bert_classifier, train_dataloader, val_dataloader, epochs=2, evaluation=True)