text_plain_output_1.png
import pandas as pd

import re

train_data = pd.read_csv('../input/train.csv')
train_data.sample(5)

train_data['target'].value_counts()[0] / train_data.shape[0] * 100

train_data.isna().sum()

train_data.drop(['id'], axis=1, inplace=True)

"""Adding additional informative columns, as most toxic tweets contain 
   exclamations, capitalized words that can serve as important markers"""
regex = re.compile('[@_!#$%^&*()<>?/\\|}{~:]')
train_data['capitals'] = train_data['comment_text'].apply(lambda x: sum((1 for c in x if c.isupper())))
train_data['exclamation_points'] = train_data['comment_text'].apply(lambda x: len(regex.findall(x)))
train_data['total_length'] = train_data['comment_text'].apply(len)

features_added = ('capitals', 'exclamation_points', 'total_length')
features_existing = ('target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat', 'funny', 'wow', 'sad', 'likes', 'disagree', 'sexual_explicit', 'identity_annotator_count', 'toxicity_annotator_count')
rows = [{c: train_data[f].corr(train_data[c]) for c in features_existing} for f in features_added]
train_correlations = pd.DataFrame(rows, index=features_added)

test_data = pd.read_csv('../input/test.csv')
test_data.head(5)