text_plain_output_1.png
import bs4

import pandas as pd

import requests

r1 = requests.get('https://www.washingtonpost.com/politics/2019/07/31/transcript-first-night-second-democratic-debate')
r2 = requests.get('https://www.washingtonpost.com/politics/2019/08/01/transcript-night-second-democratic-debate/')

def parse_requests(r, night=None):
    soup = bs4.BeautifulSoup(r.content)
    graphs = soup.find_all('p')
    utterances = [x.get_text() for x in graphs if 'data-elm-loc' in x.attrs.keys()]
    utterances = utterances[2:]
    seq = 0
    data = []
    for i in utterances:
        i = i.replace('DE BLASIO:', 'DEBLASIO:')
        graph = i.split()
        if graph[0][-1] == ':':
            text = ' '.join(graph[1:])
            num_words = len(graph) - 1
            name = graph[0][:-1]
            seq += 1
        elif len(graph) > 1 and graph[1] == '(?):':
            text = ' '.join(graph[2:])
            num_words = len(graph) - 2
            name = graph[0]
            seq += 1
        else:
            text = ' '.join(graph)
        data.append({'name': name, 'graph': text, 'seq': seq, 'num_words': num_words, 'night': night})
    return data
data = parse_requests(r1, night=0) + parse_requests(r2, night=1)

df = pd.DataFrame(data)
df.name.unique()