text_plain_output_1.png
from sklearn.feature_extraction.text import CountVectorizer

import matplotlib.pyplot as plt

import numpy as np

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

papers = pd.read_csv('/kaggle/input/nips-papers/papers.csv')

papers_2010 = papers.loc[papers['year'] == 2010].copy(deep=False)
papers_2017 = papers.loc[papers['year'] == 2017].copy(deep=False)

import re
papers_2010['title_processed'] = papers_2010['title'].map(lambda x: re.sub('[,\\.!?]', '', x))
papers_2017['title_processed'] = papers_2017['title'].map(lambda x: re.sub('[,\\.!?]', '', x))
papers_2010['title_processed'] = papers_2010['title_processed'].map(lambda x: x.lower())
papers_2017['title_processed'] = papers_2017['title_processed'].map(lambda x: x.lower())

from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
def plot_10_most_common_words(count_data, count_vectorizer):
    import matplotlib.pyplot as plt
    words = count_vectorizer.get_feature_names()
    total_counts = np.zeros(len(words))
    for t in count_data:
        total_counts += t.toarray()[0]
    count_dict = zip(words, total_counts)
    count_dict = sorted(count_dict, key=lambda x: x[1], reverse=True)[1:51]
    words = [w[0] for w in count_dict]
    counts = [w[1] for w in count_dict]
    x_pos = np.arange(len(words))
    plt.bar(x_pos, counts, align='center')
    plt.xticks(x_pos, words, rotation=90)
    plt.xlabel('words')
    plt.ylabel('counts')
    plt.title('20 most common words')
    plt.show()
count_vectorizer = CountVectorizer(stop_words='english')
count_data = count_vectorizer.fit_transform(papers_2010['title_processed'])
plot_10_most_common_words(count_data, count_vectorizer)
dict = [count_data, count_vectorizer]