image_output_1.png
from wordcloud import WordCloud

import bs4

import matplotlib.pyplot as plt

import pandas as pd

import requests

import sklearn.feature_extraction.text as skt

r = requests.get('https://www.washingtonpost.com/politics/2019/07/31/transcript-first-night-second-democratic-debate')
r.status_code

soup = bs4.BeautifulSoup(r.content)
graphs = soup.find_all('p')
utterances = [x.get_text() for x in graphs if 'data-elm-loc' in x.attrs.keys()]

utterances = utterances[2:]
seq = 0
data = []
for i in utterances:
    graph = i.split()
    if graph[0][-1] == ':':
        text = ' '.join(graph[1:])
        num_words = len(graph) - 1
        name = graph[0][:-1]
        seq += 1
    elif len(graph) > 1 and graph[1] == '(?):':
        text = ' '.join(graph[2:])
        num_words = len(graph) - 2
        name = graph[0]
        seq += 1
    else:
        text = ' '.join(graph)
    data.append({'name': name, 'graph': text, 'seq': seq, 'num_words': num_words})

df = pd.DataFrame(data)
df.name.unique()

df = df[df.name != '(UNKNOWN)']
df['name'] = df['name'].apply(lambda x: ''.join([char for char in x if char.isalpha()]))

def processing(x):
    return pd.Series({'graph': f"{' '.join(x['graph'])}", 'name': x['name'].iloc[0]})
df = df.groupby('seq').apply(processing)

import numpy as np
import sklearn.feature_extraction.text as skt
from wordcloud import WordCloud
def topwords_candidate(candidate_name, n):
    lectures = ['this is some food', 'this is some drink']
    vectorizer = skt.TfidfVectorizer(stop_words='english')
    X = vectorizer.fit_transform(df[df['name'] == candidate_name]['graph'])
    feature_names = vectorizer.get_feature_names()
    doc = 0
    feature_index = X[doc, :].nonzero()[1]
    tfidf_scores = zip(feature_index, [X[doc, x] for x in feature_index])
    scored_features = sorted([(feature_names[i], s) for i, s in tfidf_scores], key=lambda x: x[1])
    data = scored_features[-n:]
    wordcloud = WordCloud().generate(' '.join([x[0] for x in data][::-1]))
    import matplotlib.pyplot as plt
    plt.axis('off')
    return (data, wordcloud)

topwords_candidate('SANDERS', 10)