text_plain_output_1.png
from nltk import pos_tag

from nltk import pos_tag

from nltk.corpus import stopwords

from nltk.corpus import wordnet

from nltk.stem import WordNetLemmatizer

from nltk.stem import WordNetLemmatizer

from pyspark.sql import SparkSession

import nltk

import string

spark = SparkSession.builder.appName('TestCSV').getOrCreate()

train_df = spark.read.csv('../input/train.csv', header=True)
test_df = spark.read.csv('../input/test.csv', header=True)

train_df.take(5)

train_df.columns

out_cols = [i for i in train_df.columns if i not in ['id', 'comment_text']]

train_df = train_df.drop(*out_cols)

train_df.take(5)

test_df.take(5)

all_data = train_df.union(test_df)

all_data.take(5)

from nltk import pos_tag
from nltk.corpus import wordnet
def get_wordnet_pos(pos_tag):
    if pos_tag.startswith('J'):
        return wordnet.ADJ
    elif pos_tag.startswith('V'):
        return wordnet.VERB
    elif pos_tag.startswith('N'):
        return wordnet.NOUN
    elif pos_tag.startswith('R'):
        return wordnet.ADV
    else:
        return wordnet.NOUN

import string
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.tokenize import WhitespaceTokenizer
from nltk.stem import WordNetLemmatizer
def clean_text(text):
    text = text.lower()
    text = [word.strip(string.punctuation) for word in text.split(' ')]
    text = [word for word in text if not any((c.isdigit() for c in word))]
    stop = stopwords.words('english')
    text = [x for x in text if x not in stop]
    text = [t for t in text if len(t) > 0]
    pos_tags = pos_tag(text)
    text = [WordNetLemmatizer().lemmatize(t[0], get_wordnet_pos(t[1])) for t in pos_tags]
    text = [t for t in text if len(t) > 1]
    text = ' '.join(text)
    return text

all_rdd = all_data.select('comment_text').rdd.flatMap(lambda x: x)
all_rdd.take(5)

all_rdd = all_rdd.filter(lambda x: x is not None).filter(lambda x: x != '')

all_rdd = all_rdd.map(lambda x: x.lower())
all_rdd.take(5)

all_rdd = all_rdd.map(lambda x: [word.strip(string.punctuation) for word in x.split(' ')])

all_rdd = all_rdd.map(lambda text: [word for word in text if not any((c.isdigit() for c in word))])
all_rdd.take(5)

stop = stopwords.words('english')
remove_stop = lambda text: [x for x in text if x not in stop]

all_rdd = all_rdd.map(remove_stop)
all_rdd.take(5)

remove_empty = lambda text: [t for t in text if len(t) > 0]
all_rdd = all_rdd.map(remove_empty)
all_rdd.take(5)

import nltk
nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
def lemma(x):
    return lemmatizer.lemmatize(x)
lemmatize = lambda x: [lemma(i) for i in x]
all_rdd = all_rdd.map(lemmatize)

all_rdd.take(5)