image_output_1.png
import bs4

import pandas as pd

import requests

r = requests.get('https://www.washingtonpost.com/politics/2019/07/31/transcript-first-night-second-democratic-debate')
r.status_code

soup = bs4.BeautifulSoup(r.content)
graphs = soup.find_all('p')
utterances = [x.get_text() for x in graphs if 'data-elm-loc' in x.attrs.keys()]

utterances = utterances[2:]
seq = 0
data = []
for i in utterances:
    graph = i.split()
    if graph[0][-1] == ':':
        text = ' '.join(graph[1:])
        num_words = len(graph) - 1
        name = graph[0][:-1]
        seq += 1
    elif len(graph) > 1 and graph[1] == '(?):':
        text = ' '.join(graph[2:])
        num_words = len(graph) - 2
        name = graph[0]
        seq += 1
    else:
        text = ' '.join(graph)
    data.append({'name': name, 'graph': text, 'seq': seq, 'num_words': num_words})

df = pd.DataFrame(data)
df.name.unique()