image_output_1.png
from collections import Counter

import matplotlib.pyplot as plt

import nltk

import os

import pandas as pd

path = '../input/state-of-the-union-corpus-1989-2017'
dirs = os.listdir(path)

def count_words(word, filename):
    file = open(path + '/' + filename, encoding='utf8')
    text = file.read().lower()
    words = nltk.word_tokenize(text)
    word_counter = Counter(words)
    word_count = word_counter[word]
    return word_count

file_dict_list = []
for filename in dirs:
    file_dict = {}
    job_word_count = count_words('job', filename) + count_words('jobs', filename)
    file_dict['year'] = int(filename[-8:-4])
    file_dict['job_word_count'] = job_word_count
    file_dict_list.append(file_dict)

df = pd.DataFrame(file_dict_list)
df

df.set_index('year', inplace=True)
df

years = df.index
job_word_count = df['job_word_count'].values

jobless_rate = pd.read_csv('../input/usa-unemployment-rate-from-1989-to-2017/unemployment_rate.csv', sep=',')
jobless_rate.set_index('Year', inplace=True)
jobless_rate['Annual'] = jobless_rate.mean(axis=1)
jobless_rate

years = jobless_rate.index
joblessness = jobless_rate['Annual'].values

final_df = pd.merge(jobless_rate, df, left_index=True, right_index=True)
final_df

fig, ax1 = plt.subplots(figsize=(8, 5))
final_df['job_word_count'].plot(kind='bar', stacked=False, ax=ax1, label='No. of times "job" is mentioned')
ax2 = ax1.twinx()
ax2.plot(ax1.get_xticks(), final_df['Annual'].values, linestyle='-', marker='o', color='k', linewidth=1.0, label='Unemployment Rate (%)')
lines, labels = ax1.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax1.legend(lines + lines2, labels + labels2, loc='best')
ax1.set_title('How State of the Union Addresses Reflect the Reality', fontweight='bold', size=15)
ax1.set_ylabel('"Job" Word Count', fontsize=12)
ax1.set_xlabel('Year', fontsize=12)
ax2.set_ylabel('Unemployment Rate (%)', fontsize=12)
plt.show()