text_html_output_1.png
from sklearn.model_selection import train_test_split

from transformers import AutoTokenizer, AutoModel

import pandas as pd

import re

import torch

import unicodedata

df_reviews = pd.read_csv('/kaggle/input/arabic-100k-reviews/ar_reviews_100k.tsv', delimiter='\t')

label_mapping = {'Positive': 1, 'Negative': 0}
df_reviews = df_reviews[df_reviews.label != 'Mixed']
df_reviews.label = df_reviews.label.map(label_mapping)
df_reviews.label.value_counts()

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

tokenizer = AutoTokenizer.from_pretrained('asafaya/bert-mini-arabic')

def text_preprocessing(text):
    text = unicodedata.normalize('NFC', text)
    text = re.sub('(@.*?)[\\s]', ' ', text)
    text = re.sub('&amp', '&', text)
    text = re.sub('\\s+', ' ', text).strip()
    text = re.sub('^https?:\\/\\/.*[\\r\\n]*', '<URL>', text)
    return text

import emoji
import unicodedata
def preprocessing_for_bert(data, version='mini', text_preprocessing_fn=text_preprocessing):
    input_ids = []
    attention_masks = []
    tokenizer = AutoTokenizer.from_pretrained('asafaya/bert-mini-arabic') if version == 'mini' else AutoTokenizer.from_pretrained('asafaya/bert-base-arabic')
    for i, sent in enumerate(data):
        encoded_sent = tokenizer.encode_plus(text=text_preprocessing_fn(sent), add_special_tokens=True, max_length=MAX_LEN, padding='max_length', return_attention_mask=True, truncation=True)
        input_ids.append(encoded_sent.get('input_ids'))
        attention_masks.append(encoded_sent.get('attention_mask'))
    input_ids = torch.tensor(input_ids)
    attention_masks = torch.tensor(attention_masks)
    return (input_ids, attention_masks)

from sklearn.model_selection import train_test_split
X = df_reviews.text.values
y = df_reviews.label.values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_test, X_val, y_test, y_val = train_test_split(X_test, y_test, test_size=0.5, random_state=42)

MAX_LEN = 280
token_ids = list(preprocessing_for_bert([X[0]])[0].squeeze().numpy())
print(f'Original: {X[0]}')
print(f'Token IDs: {token_ids}')
print('Tokenizing data...')
train_inputs, train_masks = preprocessing_for_bert(X_train)
val_inputs, val_masks = preprocessing_for_bert(X_val)