text_plain_output_1.png image_output_1.png
from nltk.corpus import stopwords

import nltk

import re

import re
import nltk
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
english_stemmer = nltk.stem.SnowballStemmer('english')
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer

def description_to_wordlist(description, remove_stopwords=True):
    description_text = re.sub('[^a-zA-Z]', ' ', description)
    words = review_text.lower().split()
    if remove_stopwords:
        stops = set(stopwords.words('english'))
        words = [w for w in words if not w in stops]
    b = []
    stemmer = english_stemmer
    for word in words:
        b.append(stemmer.stem(word))
    return b

description_low = []
for description in X_train_low['description']:
    description_low.append(' '.join(description_to_wordlist(review)))
description_med = []
for description in X_train_med['description']:
    description_med.append(' '.join(description_to_wordlist(review)))
description_high = []
for description in X_train_high['description']:
    description_high.append(' '.join(description_to_wordlist(review)))