text_plain_output_1.png
from transformers import XLMRobertaTokenizer, TFAutoModel, TFAutoModelForSequenceClassification, TFXLMRobertaModel, XLMRobertaModel

import contractions

import matplotlib.pyplot as plt

import matplotlib.pyplot as plt

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')

train_df.shape

nan_count = train_df.isna().sum()
nan_count

train_df.drop(['location'], axis=1)

train_df.duplicated(['text', 'target']).sum()

train_df = train_df.drop_duplicates(['text', 'target'])

train_df.shape

def remove_mentions(data_df):
    mentions_removed = re.sub('@[A-Za-z0-9_]+', '', data_df)
    return mentions_removed

def remove_hashtags(data_df):
    hashtags_removed = re.sub('#[A-Za-z0-9_]+', '', data_df)
    return hashtags_removed

def remove_urls(data_df):
    hashtags_removed = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[\n]|[$-_@.&+\\]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '', data_df)
    return hashtags_removed

def convert_contractions(data_df):
    contractions_converted = contractions.fix(data_df)
    return contractions_converted

train_df['text'] = train_df['text'].astype(str)
train_df['mentions_removed'] = train_df['text'].apply(remove_mentions).tolist()
train_df['hashtags_removed'] = train_df['mentions_removed'].apply(remove_hashtags).tolist()
train_df['url_removed'] = train_df['hashtags_removed'].apply(remove_urls).tolist()
train_df['lower_cased'] = train_df['url_removed'].apply(lambda x: x.lower())
train_df['contractions_converted'] = train_df['lower_cased'].apply(convert_contractions).tolist()

MODEL_TYPE = 'xlm-roberta-large'
xlm_tokenizer = XLMRobertaTokenizer.from_pretrained(MODEL_TYPE)
xlm_model = TFAutoModel.from_pretrained(MODEL_TYPE)

tokenized_feature_raw = xlm_tokenizer.batch_encode_plus(train_df['contractions_converted'], add_special_tokens=True)
token_sentence_length = [len(x) for x in tokenized_feature_raw['input_ids']]
print('max: ', max(token_sentence_length))
print('min: ', min(token_sentence_length))
avg_length = sum(token_sentence_length) / train_df.shape[0]
MAX_LEN = max(token_sentence_length)
import matplotlib.pyplot as plt
plt.figure(figsize=(20, 8))
plt.hist(token_sentence_length, rwidth=0.9)
plt.xlabel('Sequence Length', fontsize=18)
plt.ylabel('No of Samples', fontsize=18)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)