text_plain_output_1.png
import numpy as np

import pandas as pd

import seaborn as sns

import numpy as np
import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
pd.set_option('display.max_colwidth', None)
import os
import random
import re
import unicodedata
import gc
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')
import torch
import torch.nn as nn
import torch.optim as optim
from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer
from datasets import Dataset, DatasetDict

data_path = '/kaggle/input/nlp-getting-started/'

train_df = pd.read_csv(data_path + 'train.csv')
train_df.shape

train_df['target'] = np.float64(train_df.target)

keyword_target = train_df.groupby(['keyword', 'target']).size().unstack(fill_value=0)
keyword_target['disaster_ratio'] = keyword_target[1] / (keyword_target[0] + keyword_target[1])
keyword_target = keyword_target.sort_values(by='disaster_ratio', ascending=False)

train_df.drop_duplicates(['text', 'target'], inplace=True)

train_df.drop_duplicates(['text'], inplace=True)
print('count of tweets with duplicate text:', train_df.duplicated(['text']).sum())