<repo_name>annazhangg/text-mining<file_sep>/movie_rating/extractimdb.py
from imdbpie import Imdb
import pickle


def review_extracter(movie):
    """input movie name, outputs txt file with reviews on the front page of imdb"""
    ### extracts content from movie page ###
    imdb = Imdb()
    url = imdb.search_for_title(movie)[0]
    reviews = imdb.get_title_user_reviews(url["imdb_id"])
    ### creates list of all reviews present on the front page###
    imdb = Imdb()
    url = imdb.search_for_title(movie)[0]
    reviews = imdb.get_title_user_reviews(url["imdb_id"])
    ### creates txt file for all reviews present on the front page###
    review_list = []
    for i in range(len(reviews["reviews"])):
        line = reviews["reviews"][i]["reviewText"]
        review_list.append(line)
    ###pickles list###
    with open(f"{movie}_imdb.pickle", "wb") as f:
        pickle.dump(review_list, f)
    # with open(f"{movie}.pickle", "rb") as input_file:
    #     copy = pickle.load(input_file)
    # print(copy)
def main():
    review_extracter("Mulan")
if __name__ == "__main__":
    main()<file_sep>/movie_rating/review.py
import extractimdb
import extracttwitter
import analyze_nltk
import analyze_text
import os
import matplotlib.pyplot as plt
import scatterplot

#This is a master function for all of our analysis tools.
 
def analyze_movie(movie):
    """shows stats for movie reviews"""
    source1 = "imdb"
    source2 = "twitter"
    # if imdb pickle does not exist, extracts pickle
    if not os.path.isfile(f"{movie}_imdb.pickle"):
        extractimdb.review_extracter(movie)
    # if twitter pickle does not exist, extracts pickle
    if not os.path.isfile(f"{movie}_twitter.pickle"):
        extracttwitter.twitter_extracter(movie)
 
    # process and analyze data
    for source in [source1, source2]:
        hist = analyze_text.process_file(movie, source)
        t = analyze_text.most_common(hist, excluding_stopwords=True)
        print()
        print(f"The most common words in {source} reviews:")
        for freq, word in t[0:20]:
            print(f"{word:<10} {freq}")
        print()
        print(f"Histogram of top 10 words in {source} reviews")
        analyze_text.print_most_common(hist, num=10)
        print()
        average, sentiment_list = analyze_nltk.sentiment_analysis(movie, source)
        print(f"The average sentiment on {source} is {average}.")
        print("Individual scores are:")
        print(sentiment_list)
    # create sentiment scatter plot
    mulan1 = scatterplot.sentiment_scatter("Parasite", source1)
    mulan2 = scatterplot.sentiment_scatter("Parasite", source2)
    scatterplot.scatter(movie, source1, source2, mulan1, mulan2)
 
 
def main():
    movie = "Parasite"
    analyze_movie(movie)
 
 
if __name__ == "__main__":
    main()
 

<file_sep>/movie_rating/extracttwitter.py
from twython import Twython
import pickle
def twitter_extracter(movie, tweet_number=50):
    """extracts twitter posts related to input: movie, and # of tweets. outputs: pickle of tweets"""
    ###enters API keys###
    APP_KEY = "9iNbqBABwgvjucc9dE2pW7mTm"
    APP_SECRET = "<KEY>"
    twitter = Twython(APP_KEY, APP_SECRET)
    ###searches###
    data = twitter.search(q=f"{movie} review", count=tweet_number)
    ###creates list of tweets###
    twitter_list = []
    for status in data["statuses"]:
        twitter_list.append(status["text"])
    ###pickles tweets###
    with open(f"{movie}_twitter.pickle", "wb") as f:
        pickle.dump(twitter_list, f)
def main():
    twitter_extracter("Mulan")
if __name__ == "__main__":
    main()
<file_sep>/movie_rating/scatterplot.py
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import pickle
import matplotlib.pyplot as plt
 
 
# python -m pip install -U pip
# python -m pip install -U matplotlib
 
 
def sentiment_scatter(movie, source):
    """senitment analysis, input str movie and source, output sentiment analysis"""
    # opens pickle
    with open(f"{movie}_{source}.pickle", "rb") as input_file:
        copy = pickle.load(input_file)
    # stores sentiment score in list
    sentiment_pos = []
    sentiment_neg = []
    for line in copy:
        score = SentimentIntensityAnalyzer().polarity_scores(line)
        sentiment_pos.append(score["pos"])
        sentiment_neg.append(score["neg"])
    return sentiment_pos, sentiment_neg
 
 
def scatter(movie, source1, source2, source1_list, source2_list):
    """prints scatter plot when 2 data sources are inputted"""
    source1_pos, source1_neg = source1_list
    source2_pos, source2_neg = source2_list
    one = plt.scatter(source1_pos, source1_neg, color="r", marker="x")
    two = plt.scatter(source2_pos, source2_neg, color="b")
    plt.xlabel("Positive Words")
    plt.ylabel("Negative Words")
    plt.title(f"Postive vs. Negative words in {movie} reviews.")
    plt.legend((one, two), (source1, source2))
    plt.show()
 
 
def main():
    movie = "Parasite"
    source1 = "twitter"
    source2 = "imdb"
    mulan1 = sentiment_scatter(movie, source1)
    mulan2 = sentiment_scatter(movie, source2)
    scatter(movie, source1, source2, mulan1, mulan2)
 
 
if __name__ == "__main__":
    main()
<file_sep>/movie_rating/analyze_nltk.py
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import pickle
def sentiment_analysis(movie, source):
    """senitment analysis, input str movie and source, output sentiment analysis"""
    # opens pickle
    with open(f"{movie}_{source}.pickle", "rb") as input_file:
        copy = pickle.load(input_file)
    # stores sentiment score in list
    sentiment_list = []
    for line in copy:
        score = SentimentIntensityAnalyzer().polarity_scores(line)
        sentiment_list.append(score["compound"])
    # finds average of sentiment score
    average = sum(sentiment_list) / len(sentiment_list)
    # returns average and list
    return average, sentiment_list
def main():
    average, sentiment_list = sentiment_analysis("Mulan", "imdb")
    print(f"The average sentiment is {average}.")
    print("Individual scores are:")
    print(sentiment_list)
if __name__ == "__main__":
    main()<file_sep>/movie_rating/analyze_text.py
import string
import pickle
 
 
def process_file(movie, source):
    """Makes a histogram that contains the words from a file.
 
    movie: string
    source: string
 
    returns: map from each word to the number of times it appears.
    """
    hist = {}
    # opens pickle
    with open(f"{movie}_{source}.pickle", "rb") as input_file:
        fp = pickle.load(input_file)
 
    # removes extra chracters and stores into dictionary by frequency
    strippables = string.punctuation + string.whitespace
 
    for line in fp:
        line = line.replace("-", " ")
        line = line.replace('"', " ")
 
        for word in line.split():
            word = word.strip(strippables)
            word = word.lower()
 
            hist[word] = hist.get(word, 0) + 1
    if "" in hist:
        del hist[""]
 
    return hist
 
 
def most_common(hist, excluding_stopwords=False):
    """Makes a list of word-freq pairs in descending order of frequency.
 
    hist: map from word to frequency
 
    returns: list of (frequency, word) pairs
    """
    # imports stopword.txt and creates lists
    common_words = []
    stopwords = open("stopword.txt")
    stopwords_list = []
    # turns stopwords.txt into list
    for line in stopwords:
        lin = line.strip()
        stopwords_list.append(lin)
    # sorts dictionary by frequency and returns list of tupples
    for word, freq in hist.items():
        if not excluding_stopwords or not word in stopwords_list:
            common_words.append((freq, word))
    common_words.sort(reverse=True)
    return common_words
 
 
def print_most_common(hist, num=10):
    """Prints the most commons words in a histgram and their frequencies.
 
    hist: histogram (map from word to frequency)
    num: number of words to print
    """
    t = most_common(hist, excluding_stopwords=True)
    for freq, word in t[0:num]:
        print(f"{word:<10}", "*" * int((freq)))
 
 
def main():
    movie = "Mulan"
    hist = process_file(movie, "imdb")
    t = most_common(hist, excluding_stopwords=True)
    print("The most common words are:")
    for freq, word in t[0:20]:
        print(f"{word:<10} {freq}")
    print_most_common(hist, num=10)
 
 
if __name__ == "__main__":
    main()

