text_plain_output_1.png
from nltk.corpus import stopwords

import nltk

import pandas as pd

train_data = pd.read_csv('../input/spooky-author-identification/train.csv')
test_data = pd.read_csv('../input/spooky-author-identification/test.csv')

def preprocess_text(text, remove_list):
    """
    tokens = nltk.pos_tag(nltk.word_tokenize(text))
    print(tokens)
    good_words = [w for w, wtype in tokens if wtype not in remove_list]
    print(good_words)
    """
    def clean_word(word):
        word = word.lower()
        if len(word) > 1 and word[0] == "'":
            return word[1:]
        return word
    tokens = nltk.pos_tag(nltk.word_tokenize(text))
    tokens = [(clean_word(word), pos) for word, pos in tokens]
    return [(word, pos) for word, pos in tokens if word not in remove_list]
def preprocess_corpus(corpus):
    stop_words = set(stopwords.words('english'))
    stop_words.update(['.', ',', '"', "'", '?', '!', ':', ';', '(', ')', '[', ']', '{', '}'])
    return [preprocess_text(text, stop_words) for text in corpus]
preprocessed_train_corpus = preprocess_corpus(train_data['text'])
preprocessed_test_corpus = preprocess_corpus(test_data['text'])
print(preprocessed_train_corpus[0])