text_plain_output_1.png
import nltk

import pandas as pd

import re

null_text = X.comment_text[2]

import re
import nltk
stop_words = set(nltk.corpus.stopwords.words('english'))
def preprocess_input(t):
    t = t.strip()
    z = re.findall('[A-Za-z]+', t)
    z = [a for a in z if len(a) > 3]
    wnlemma = nltk.stem.WordNetLemmatizer()
    z = [wnlemma.lemmatize(a) for a in z]
    z = [a for a in z if not a in stop_words]
    t = ' '.join(z)
    return t

test = pd.read_csv('../input/test.csv')
test.fillna(value=null_text, inplace=True)

t_id = test['id']
test.drop(['id'], axis=1, inplace=True)
test.comment_text = test.comment_text.apply(lambda z: preprocess_input(z))

test.head()