text_html_output_1.png
from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from sklearn.feature_extraction.text import CountVectorizer

from wordcloud import WordCloud

import json

import matplotlib.pyplot as plt

import pandas as pd

import re

import string

n_common_words = 50
wnl = WordNetLemmatizer()
engstopwords = stopwords.words('english')
engstopwordsV2 = re.sub('[' + re.escape(string.punctuation) + ']', '', ' '.join(engstopwords)).split()
with open('../input/mbtiadditionalwords/additional-stopwords.json') as file:
    additional_stopwords = json.load(file)
engstopwords.extend(additional_stopwords)
engstopwords = set(engstopwords).union(set(engstopwordsV2))
str_punc = string.punctuation
filename = '../input/mbti-type/mbti_1.csv'
cv = CountVectorizer(stop_words='english')
plt.rcParams['figure.dpi'] = 300
wc = WordCloud(stopwords=engstopwords, background_color='white', colormap='Dark2', max_font_size=150, random_state=42, width=800, height=400)

dataset = pd.read_csv(filename)
dataset = dataset.groupby(['type'])['posts'].apply(lambda x: '. '.join(x)).reset_index()

def lemmatize_all_types(word):
    word = wnl.lemmatize(word, 'a')
    word = wnl.lemmatize(word, 'v')
    word = wnl.lemmatize(word, 'n')
    return word
def clean(text):
    text = re.sub('http.*?([ ]|\\|\\|\\||$)', '', text).lower()
    text = re.sub('(:|;).', '', text)
    text = re.sub('[' + re.escape(str_punc) + ']', '', text)
    text = re.sub('(\\[|\\()*\\d+(\\]|\\))*', '', text)
    text = re.sub('[’‘“\\.”…–]', '', text)
    text = list(map(lemmatize_all_types, text.split()))
    text = [word for word in text if word not in engstopwords]
    text = ' '.join(text)
    return text

dataset.posts = pd.Series(dataset.posts.apply(clean))
dataset.index = dataset['type']

posts_vectorized = cv.fit_transform(dataset.posts)
dtm = pd.DataFrame(data=posts_vectorized.toarray(), columns=cv.get_feature_names())
dtm.index = dataset['type']

dtm.iloc[:, -20000:-20016:-1]

dtmT = dtm.transpose()

dtmT.iloc[-20000:-20016:-1, :]

cloud = []
for personality in dtmT.columns:
    topWords = list(dtmT[personality].sort_values(ascending=False).head(n_common_words).index)
    cloud.append({personality: topWords})

for index, personality in enumerate(dtmT.columns):
    wc.generate(dataset.posts[personality])
    plt.imshow(wc, interpolation='bilinear')
    plt.title(personality)
    plt.savefig(personality + '.png')