import praw
import json
import time
import datetime
import pandas as pd
import numpy as np
import sys

# ***********
#  Methods:
# ***********

def get_author(author):
    if author == None:
        return author
    
    return author.name

def get_submission(submission, interval):
    return {
        'approved_at_utc'         : getattr(submission, 'approved_at_utc', None),
        'approved_by'             : getattr(submission, 'approved_by', None),
        'archived'                : getattr(submission, 'archived', None),
        'author_flair_css_class'  : getattr(submission, 'author_flair_css_class', None),
        'author_flair_text'       : getattr(submission, 'author_flair_text', None),
        'banned_at_utc'           : getattr(submission, 'banned_at_utc', None),
        'banned_by'               : getattr(submission, 'banned_by', None),
        'brand_safe'              : getattr(submission, 'brand_safe', None),
        'can_gild'                : getattr(submission, 'can_gild', None),
        'can_mod_post'            : getattr(submission, 'can_mod_post', None),
        'clicked'                 : getattr(submission, 'clicked', None),
        'comment_limit'           : getattr(submission, 'comment_limit', None),
        'comment_sort'            : getattr(submission, 'comment_sort', None),
        'contest_mode'            : getattr(submission, 'contest_mode', None),
        'created'                 : getattr(submission, 'created', None),
        'created_utc'             : getattr(submission, 'created_utc', None),
        'distinguished'           : getattr(submission, 'distinguished', None),
        'domain'                  : getattr(submission, 'domain', None),
        'downs'                   : getattr(submission, 'downs', None),
        'edited'                  : getattr(submission, 'edited', None),
        'gilded'                  : getattr(submission, 'gilded', None),
        'hidden'                  : getattr(submission, 'hidden', None),
        'hide_score'              : getattr(submission, 'hide_score', None),
        'id'                      : getattr(submission, 'id', None),
        'is_crosspostable'        : getattr(submission, 'is_crosspostable', None),
        'is_reddit_media_domain'  : getattr(submission, 'is_reddit_media_domain', None),
        'is_self'                 : getattr(submission, 'is_self', None),
        'is_video'                : getattr(submission, 'is_video', None),
        'likes'                   : getattr(submission, 'likes', None),
        'link_flair_css_class'    : getattr(submission, 'link_flair_css_class', None),
        'link_flair_text'         : getattr(submission, 'link_flair_text', None),
        'locked'                  : getattr(submission, 'locked', None),
        'media_embed'             : getattr(submission, 'media_embed', None),
        'mod_note'                : getattr(submission, 'mod_note', None),
        'mod_reason_by'           : getattr(submission, 'mod_reason_by', None),
        'mod_reason_title'        : getattr(submission, 'mod_reason_title', None),
        'mod_reports'             : getattr(submission, 'mod_reports', None),
        'name'                    : getattr(submission, 'name', None),
        'num_comments'            : getattr(submission, 'num_comments', None),
        'num_crossposts'          : getattr(submission, 'num_crossposts', None),
        'num_reports'             : getattr(submission, 'num_reports', None),
        'over_18'                 : getattr(submission, 'over_18', None),
        'parent_whitelist_status' : getattr(submission, 'parent_whitelist_status', None),
        'permalink'               : getattr(submission, 'permalink', None),
        'pinned'                  : getattr(submission, 'pinned', None),
        'post_hint'               : getattr(submission, 'post_hint', None),
        'quarantine'              : getattr(submission, 'quarantine', None),
        'removal_reason'          : getattr(submission, 'removal_reason', None),
        'report_reasons'          : getattr(submission, 'report_reasons', None),
        'saved'                   : getattr(submission, 'saved', None),
        'score'                   : getattr(submission, 'score', None),
        'secure_media'            : getattr(submission, 'secure_media', None),
        'selftext'                : getattr(submission, 'selftext', "").replace("\r\n", "\n").replace("\r", "\n").replace("\n", " "),
        'spoiler'                 : getattr(submission, 'spoiler', None),
        'stickied'                : getattr(submission, 'stickied', None),
        'subreddit_id'            : getattr(submission, 'subreddit_id', None),
        'suggested_sort'          : getattr(submission, 'suggested_sort', None),
        'title'                   : getattr(submission, 'title', None),
        'ups'                     : getattr(submission, 'ups', None),
        'url'                     : getattr(submission, 'url', None),
        'user_reports'            : getattr(submission, 'user_reports', None),
        'view_count'              : getattr(submission, 'view_count', None),
        'visited'                 : getattr(submission, 'visited', None),
        'whitelist_status'        : getattr(submission, 'whitelist_status', None),
        
        # custom fields ...
        'author'                  : get_author(submission.author),
        'interval'                : interval
    }

def get_comment(comment, submissionId):
    return {
        'approved_at_utc'        : getattr(comment, 'approved_at_utc', None),
        'approved_by'            : getattr(comment, 'approved_by', None),
        'archived'               : getattr(comment, 'archived', None),
        'author_flair_css_class' : getattr(comment, 'author_flair_css_class', None),
        'author_flair_text'      : getattr(comment, 'author_flair_text', None),
        'banned_at_utc'          : getattr(comment, 'banned_at_utc', None),
        'banned_by'              : getattr(comment, 'banned_by', None),
        'body'                   : getattr(comment, 'body', "").replace("\r\n", "\n").replace("\r", "\n").replace("\n", " "),
        'can_gild'               : getattr(comment, 'can_gild', None),
        'can_mod_post'           : getattr(comment, 'can_mod_post', None),
        'collapsed'              : getattr(comment, 'collapsed', None),
        'collapsed_reason'       : getattr(comment, 'collapsed_reason', None),
        'controversiality'       : getattr(comment, 'controversiality', None),
        'created'                : getattr(comment, 'created', None),
        'created_utc'            : getattr(comment, 'created_utc', None),
        'depth'                  : getattr(comment, 'depth', None),
        'distinguished'          : getattr(comment, 'distinguished', None),
        'downs'                  : getattr(comment, 'downs', None),
        'edited'                 : getattr(comment, 'edited', None),
        'gilded'                 : getattr(comment, 'gilded', None),
        'id'                     : getattr(comment, 'id', None),
        'is_submitter'           : getattr(comment, 'is_submitter', None),
        'likes'                  : getattr(comment, 'likes', None),
        'link_id'                : getattr(comment, 'link_id', None),
        'mod_note'               : getattr(comment, 'mod_note', None),
        'mod_reason_by'          : getattr(comment, 'mod_reason_by', None),
        'mod_reason_title'       : getattr(comment, 'mod_reason_title', None),
        'mod_reports'            : getattr(comment, 'mod_reports', None),
        'name'                   : getattr(comment, 'name', None),
        'num_reports'            : getattr(comment, 'num_reports', None),
        'parent_id'              : getattr(comment, 'parent_id', None),
        'permalink'              : getattr(comment, 'permalink', None),
        'removal_reason'         : getattr(comment, 'removal_reason', None),
        'report_reasons'         : getattr(comment, 'report_reasons', None),
        'saved'                  : getattr(comment, 'saved', None),
        'score'                  : getattr(comment, 'score', None),
        'score_hidden'           : getattr(comment, 'score_hidden', None),
        'stickied'               : getattr(comment, 'stickied', None),
        'subreddit_id'           : getattr(comment, 'subreddit_id', None),
        'ups'                    : getattr(comment, 'ups', None),
        'user_reports'           : getattr(comment, 'user_reports', None),
        
        # custom fields ...
        'submissionId'           : submissionId,
        'author'                 : get_author(comment.author)
    }

def get_config(config_file):
    assert type(config_file) == str
    with open(config_file) as f:
        config = json.load(f)
	
    return config

def login(config):
    reddit = praw.Reddit(client_id = config['client_id'], client_secret = config['client_secret'], user_agent = config['user_agent'])
    return reddit

def get_intervals(config, start):
    def give_me_intervals(start, period, end_at):

        # coll. of intervals ...
        intervals = []

        end   = start + period
        intervals.append((start, end))

        padding = 1

        while end <= end_at:

            # lightly move ahead a bit ...
            start = end + padding
            end   = (start - padding) + period

            # stash interval ...
            intervals.append((start, end))

        return intervals

    now       = int(config['now'])
    hour      = int(config['epoch']['hour'])
    full_day  = int(config['epoch']['full_day_in_hours'])
    expand    = int(config['epoch']['expand'])

    period    = (hour * full_day * expand)
    intervals = give_me_intervals(start, period, now)

    return intervals

def get_start(subreddit):
    # search uses utc ... but just take the earliest possible value.
    if subreddit.created_utc < subreddit.created:
        start = subreddit.created_utc
    else:
        start = subreddit.created
    
    return start

def write_to_csv(posts, comments, config, iteration):
    csv_output_path = config['csv_output_path']

    if len(posts) > 0:
        # process posts ...
        df_posts                = pd.DataFrame(posts)
        df_posts.index.names    = ['index']

        posts_csv_path    = csv_output_path + sub + "_posts_" + str(iteration) + ".csv"
        df_posts.to_csv(posts_csv_path, header=True)

    if len(comments) > 0:
        # process comments ...
        df_comments             = pd.DataFrame(comments)
        df_comments.index.names = ['index']

        comments_csv_path = csv_output_path + sub + "_comments_" + str(iteration) + ".csv"
        df_comments.to_csv(comments_csv_path, header=True)

# ********
#  Main:
#   - purpose: walk a subreddit from its start to a date in the future while saving all post/comments in seperate files.
# ********

# pull config ...
config_file = r'./reddit_bot_config.json'
config      = get_config(config_file)

# login to reddit ...
reddit      = login(config)

# find subreddit to focus in on ...
sub         = config['subreddit']
subreddit   = reddit.subreddit(sub)

# pull the earliest search time, that is possible ...
start 		= get_start(subreddit)
intervals   = get_intervals(config, start)

print()
print("configurations:")
print("subreddit", sub)
print("first interval:", intervals[0])
print("last interval:",  intervals[-1])
print("number of intervals", len(intervals))

# get timeouts ...
timeout_after_post_in_secs    = int(config['timeout_after_post'])
timeout_after_comment_in_secs = int(config['timeout_after_comment'])
number_of_iterations_per_file = int(config['number_of_iterations_per_file'])

print("timeout_after_post_in_secs:", timeout_after_post_in_secs)
print("timeout_after_comment_in_secs:", timeout_after_comment_in_secs)
print("number_of_iterations_per_file:", number_of_iterations_per_file)

# initial interval #:
i           = 0

# initial iteration #:
iteration   = 1

print("initial interval #:", i)
print("initial iteration #:", iteration)
print()

print('**** starting ****')
print('**', str(datetime.datetime.now()), '**')
print()

posts          = []
comments       = []

count_1        = 0
posts_count    = 0
comments_count = 0

try:

    for interval in intervals:
        
        start = interval[0]
        end   = interval[1]

        # pull submissions ...
        submissions = subreddit.submissions(start = start, end = end)
        for submission in submissions:

            # create submission ...
            s = get_submission(submission, interval)
            posts.append(s)
            posts_count += 1

            submission.comments.replace_more(limit=None)
            for comment in submission.comments.list():

                # create comment ...
                c = get_comment(comment, s['id'])
                comments.append(c)
                comments_count += 1
                
                if timeout_after_comment_in_secs > 0:
                    print('sleeping', str(timeout_after_comment_in_secs), 'seconds after comment... ')
                    time.sleep(timeout_after_comment_in_secs)
        
        print('completed:', interval, ' - at index:', i, '- time: ', str(datetime.datetime.now()))

        count_1 += 1
        if count_1 == number_of_iterations_per_file:
            count_1 = 0

            # write current posts/comments to csv file ...
            write_to_csv(posts, comments, config, iteration)
            iteration += 1

            # clear out cached contents ...
            posts    = []
            comments = []
			
            print('wrote to file ... ')
		
        if timeout_after_post_in_secs > 0:
            print('sleeping', str(timeout_after_post_in_secs), 'seconds after post... ')
            time.sleep(timeout_after_post_in_secs)

        i += 1

except:
    print('failed at interval index:', i)
    print("Unexpected error:", sys.exc_info()[0])
    print()

# write the what is left to file ...
write_to_csv(posts, comments, config, iteration)

# signal we are done ...
print('**** completed ****')
print('**', str(datetime.datetime.now()), '**')
print()