text_plain_output_1.png
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler

from transformers import AutoTokenizer, AutoModel

import re

import torch

import torch

import unicodedata

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

tokenizer = AutoTokenizer.from_pretrained('asafaya/bert-mini-arabic')

def text_preprocessing(text):
    text = unicodedata.normalize('NFC', text)
    text = re.sub('(@.*?)[\\s]', ' ', text)
    text = re.sub('&amp', '&', text)
    text = re.sub('\\s+', ' ', text).strip()
    text = re.sub('^https?:\\/\\/.*[\\r\\n]*', '<URL>', text)
    return text

import emoji
import unicodedata
def preprocessing_for_bert(data, version='mini', text_preprocessing_fn=text_preprocessing):
    input_ids = []
    attention_masks = []
    tokenizer = AutoTokenizer.from_pretrained('asafaya/bert-mini-arabic') if version == 'mini' else AutoTokenizer.from_pretrained('asafaya/bert-base-arabic')
    for i, sent in enumerate(data):
        encoded_sent = tokenizer.encode_plus(text=text_preprocessing_fn(sent), add_special_tokens=True, max_length=MAX_LEN, padding='max_length', return_attention_mask=True, truncation=True)
        input_ids.append(encoded_sent.get('input_ids'))
        attention_masks.append(encoded_sent.get('attention_mask'))
    input_ids = torch.tensor(input_ids)
    attention_masks = torch.tensor(attention_masks)
    return (input_ids, attention_masks)

print('Tokenizing data...')
test_inputs, test_masks = preprocessing_for_bert(X_test)
test_dataset = TensorDataset(test_inputs, test_masks)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)