text_plain_output_1.png
from nltk import tokenize

import os

import pandas as pd

import string

base_path = '/kaggle'
if os.path.exists(base_path):
    input_path = os.path.join(base_path, 'input', 'tweet-sentiment-extraction')
    output_path = os.path.join(base_path, 'working')
else:
    base_path = 'data'
    input_path = base_path
    output_path = os.path.join(base_path, 'submissions')
train_file = os.path.join(input_path, 'train.csv')
test_file = os.path.join(input_path, 'test.csv')

train_df = pd.read_csv(train_file)
test_df = pd.read_csv(test_file)

train_df.sample(n=1)['text']

train_df.dtypes

test_df.dtypes

f"The 3 sentiments are '{', '.join(train_df.sentiment.unique().tolist())}'"

train_df.dropna(inplace=True)

train_df.columns

full_df = pd.concat([train_df, test_df])

full_df.loc[:, 'text_lower'] = full_df.text.str.lower()

def count_punctuations(text):
    text_punctuations = []
    punctuation_list = list(string.punctuation)
    for token in tokenize.word_tokenize(text):
        if not token.isalpha():
            if len(token) > 1:
                sub_tokens = list(token)
                for sub_token in sub_tokens:
                    if sub_token in punctuation_list:
                        text_punctuations.append(sub_token)
            else:
                text_punctuations.append(token)
    return pd.Series(text_punctuations).value_counts()
count_punctuation_df = full_df.text.apply(count_punctuations)