#! /usr/bin/env python

from datetime import datetime, timedelta
import hb_config
import json
import pandas as pd
import requests
from requests_oauthlib import OAuth1
from urllib import parse

rt_header = """== Popular articles {date7} to {date1} ==
Last updated on ~~~~~

{{| class="wikitable sortable"
!Rank
!Article
!Total weekly views
!Days in top 1k this week
"""

footer = """|}

<!--IMPORTANT add all categories to the top section of the page, not here. Otherwise, they will get removed when the bot runs tomorrow! -->

"""

rt_row = """|-
|{rank}
|[[w:{title}|{title}]]
|{week_total}
|{days_in_topk}
"""

def get_yesterdates(lookback=7):
    """
    Accepts a lookback parameter of how many days ago to gather data for (not including the current day per UTC time)
    Defaults to seven days lookback (val must be at least 1)
    Returns a list of dictionaries with the previous n dates (exclusive), in reverse chronological order
    """

    date_range = []

    for d in range(1, lookback + 1):
        date_parts = {'year': datetime.strftime(datetime.now() - timedelta(d), '%Y'),
           'month' : datetime.strftime(datetime.now() - timedelta(d), '%m'),
           'day': datetime.strftime(datetime.now() - timedelta(d), '%d'),
            }

        date_parts['display_date'] = date_parts['year'] + "-" + date_parts['month'] + "-" + date_parts['day']

        date_parts['api_date'] = date_parts['year'] + date_parts['month'] + date_parts['day'] + "00"

        date_range.append(date_parts)

    return date_range

def get_all_topk_articles(day_range):
    """
    Accepts a list of dicts with year, month, and day values
    Returns a dictionary (article titles as keys)
        with all articles that were in the topk list during those dates
        and the pageview counts for each of the dates the article appeared in the topk

    Example query: https://wikimedia.org/api/rest_v1/metrics/pageviews/top/en.wikipedia.org/all-access/2020/03/31
    """

    q_template= "https://wikimedia.org/api/rest_v1/metrics/pageviews/top/en.wikipedia.org/all-access/{year}/{month}/{day}"

    all_articles = {}

    for day_val in day_range:
        q_string = q_template.format(**day_val)
        r = requests.get(
            url = q_string,
            headers = {'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"},
                )
    #     print(r.headers)
    #     print(r.text)
    #     print(r.url)
        response = r.json()
    #     print(response)

#         response = requests.get(q_string).json()
        top_articles_list = response['items'][0]['articles']

        for ar in top_articles_list:
            if ar['article'] in all_articles.keys():
                all_articles[ar['article']].update({day_val['api_date'] : ar['views']})

            else:
                all_articles.update({ar['article'] : {day_val['api_date'] : ar['views']}})

    return all_articles

def ar_days_in_topk(day_range, ar_dict):
    """
    Accepts a day range dictionary
        And a nested dict with articles as keys
        And as values varying numbers of k,v pairs
    Returns the article dictionary with a new k,v pair value
        that counts the number of existing k,v pairs in that article dict
    """

    for k,v in ar_dict.items():
        v['topk_days'] = len(ar_dict[k])

    return ar_dict

def get_daily_non_topk_counts(day_range, ar_dict):
    """
    Accepts a list of dicts with year, month, and day values
        And a dict with article titles as keys dicts with different numbers of k,v pairs as values
    Returns a dict that contains for each article the difference between the length of the day range
        And the number of keys in each dict
    Example query:         https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/2009_swine_flu_pandemic/daily/2020032500/2020033100
    """

    q_template= "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/{article}/daily/{day7}/{day1}"

    for k,v in ar_dict.items():
        if len(v) < 8: #if this article didn't spend all week among the top 1000

            safe_title = parse.quote(k, safe='') #in case there are weird chars in title
            q_string = q_template.format(article = safe_title, day7 = day_range[6]['api_date'], day1 = day_range[0]['api_date'])
#             print(q_string)
            r = requests.get(
                url = q_string,
                headers = {'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"},
                    )
        #     print(r.headers)
        #     print(r.text)
        #     print(r.url)
            response = r.json()
        #     print(response)
#             response = requests.get(q_string).json()

            ar_views = response['items']
    #         print(ar_views)

            for d in ar_views:
                if d['timestamp'] not in v.keys():
                    v.update({d['timestamp'] : d['views']})
        else:
            pass

    return ar_dict

def fill_null_date_vals(day_range, ar_dict):
    """
    Accepts a list of dicts with year, month, and day values
        And a dict with article titles as keys and gaps in the date keys
    Returns the article dictionary with each sub-dict fully populated
        With pageview values for all dates in range, even if val is 0
    """
    #https://www.geeksforgeeks.org/dictionary-methods-in-python-set-2-update-has_key-fromkeys/
    for day_val in week_of_days:
        for v in ar_dict.values():
            if len(v) < 8: #if we still don't have any pageviews for some days
                v.setdefault(day_val['api_date'], 0) #adds a key with val of 0 if no key present
            else:
                pass

    return ar_dict

def format_row(rank, title, week_total, days_in_topk, row_template):

    table_row = {'rank': rank,
           'title' : title.replace("_"," "),
            'week_total' : week_total,
            'days_in_topk' : days_in_topk
                }

    row = row_template.format(**table_row)
#     print(row)
    return(row)


def get_token(auth1):
    """
    Accepts an auth object for a user
    Returns an edit token for the specified wiki
    """

    result = requests.get(
        url="https://en.wikipedia.org/w/api.php", #TODO add to config
        params={
            'action': "query",
            'meta': "tokens",
            'type': "csrf",
            'format': "json"
            },
        headers={'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"}, #TODO add to config
        auth=auth1,
        ).json()

#     print(result)
    edit_token = result['query']['tokens']['csrftoken']
#     print(edit_token)

    return(edit_token)

def publish_report(output, edit_sum, auth1, edit_token):
    """
    Accepts the page text, credentials and edit token
    Publishes the formatted page text to the specified wiki
    """
    response = requests.post(
    url = "https://en.wikipedia.org/w/api.php", #TODO add to config
    data={
        'action': "edit",
        'title': "User:HostBot/Top_1000_report", #TODO add to config
        'section': "1",
        'summary': edit_sum, #TODO add to config
        'text': output,
        'bot': 1,
        'token': edit_token,
        'format': "json"
        },
    headers={'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"}, #TODO add to config
    auth=auth1
        )

if __name__ == "__main__":

    auth1 = OAuth1("b5d87cbe96174f9435689a666110159c",
               hb_config.client_secret,
               "ca1b222d687be9ac33cfb49676f5bfd2",
               hb_config.resource_owner_secret)

    #get previous week's date info for query and reporting
    week_of_days = get_yesterdates(lookback=7)

    #get all of the articles that appeared on the topk list that week
    all_articles = get_all_topk_articles(week_of_days)

    #get counts for all days each article was in the top 1000
#     all_articles = get_daily_topk_counts(week_of_days, all_articles)

    #add number of days each article appears in the topk list. could do this in first function too
    all_articles = ar_days_in_topk(len(week_of_days), all_articles)

    #add page counts for days the article was not in the topk list
    all_articles = get_daily_non_topk_counts(week_of_days, all_articles)

    all_articles = fill_null_date_vals(week_of_days, all_articles)

    #now we're ready to make a dataframe!
    df_aa = pd.DataFrame.from_dict(all_articles, orient="index")

    #sum across the daily counts
    #https://stackoverflow.com/questions/25748683/pandas-sum-dataframe-rows-for-given-columns
    df_aa['week_total'] = df_aa.sum(axis=1)

    #make the title NOT the index. Should do this when creating the frame, instead
    df_aa.reset_index(inplace=True)

    #rename title column. Should do this when creating the frame, instead
    df_aa.rename(columns = {'index' : 'title'}, inplace=True)

    #remove blacklisted titles--pages we don't care about, for these purposes. Although... we could keep them I guess.
    blacklist = ["Main_Page", "Special:", "Category:", "Portal:", "Template:", "Wikipedia:", "Talk:", "User:", "_talk:", "Help:", "File:", "United_States_Senate",]
    df_aa = df_aa[~df_aa['title'].str.contains('|'.join(blacklist))]

    #sort by weekly views
    df_aa.sort_values('week_total', ascending=False, inplace=True)

    #add rank column based on weekly views
    new_rank = range(1, len(df_aa)+1)
    df_aa['rank'] = list(new_rank)

    #reset the index to reflect the final ranking, dropping the existing index this time
    df_aa.reset_index(drop=True, inplace=True)

    #start and end dates for header and edit comment
    header_dates = {'date1' : week_of_days[0]['display_date'],
                    'date7' : week_of_days[6]['display_date']
                        }

    #format the header template
    header = rt_header.format(**header_dates)

    report_rows = [format_row(a, b, c, d, rt_row) #this is messy
      for a, b, c, d
      in zip(df_aa['rank'],
             df_aa['title'],
             df_aa['week_total'],
             df_aa['topk_days'],
                )]

    rows_wiki = ''.join(report_rows)

    output = header + rows_wiki + footer
#     print(output)

    edit_token = get_token(auth1)

    edit_sum = "Popular articles from {date7} to {date1}".format(**header_dates)

    publish_report(output, edit_sum, auth1, edit_token)


