import os
import re
import json
import time
import math
import random
import logging
import argparse
import operator
import functools
from collections import Counter, OrderedDict

import asyncio
import numpy as np
import pandas as pd
from aiohttp import ClientSession

from angular_flask import app
from angular_flask import helper
from angular_flask.core import mongo, cache
from angular_flask.utils import get_logger, multikeysort, cmp

static_url = 'https://fantasy.premierleague.com/api/bootstrap-static/'
order_dict_by_key = lambda d: OrderedDict(sorted(d.items(), key = lambda x: x[0], reverse=True))
logger = get_logger(__name__)
seed = 1
multiplier = 0.775
season_weights = []
for i in range(19):
	season_weights += [seed]
	seed *= multiplier

async def fetch(session, url):
	resp = await session.request(method="GET", url=url, skip_auto_headers={'USER-AGENT'})
	content = await resp.json()
	return content

async def fetch_all(session, urls):
	tasks = []
	for url in urls:
		task = asyncio.create_task(fetch(session, url))
		tasks.append(task)
	results = await asyncio.gather(*tasks, return_exceptions=True)
	return results

def update_test():
	with app.app_context():
		test = mongo.db.test
		val = test.find_one()['val']
		test.find_one_and_update({'var': 'a'}, {'$set': {'val': val + 1}})

async def update_current_gw():
	with app.app_context():
		currentgw = mongo.db.currentgw
		gw_obj = currentgw.find_one()
		eid = gw_obj['_id']
		if gw_obj['finished']: gw_finished = False
		else: gw_finished = True
		async with ClientSession() as session:
			static_data = await fetch(session, static_url)
			gw = next((x['id'] for x in static_data['events'] if x['is_current'] == True), None)
			currentgw.find_one_and_update({'_id': eid}, {'$set': {'gw': gw, 'finished': gw_finished}})

async def update_epl_teams():
	with app.app_context():
		eplteams = mongo.db.eplteams
		eplteams.delete_many({})
		async with ClientSession() as session:
			static_data = await fetch(session, static_url)
			eplteams.insert_many([{'name': str(team['name']), 'short': str(team['short_name']), 'id': team['id']} for team in static_data['teams']])

async def update_epl_players():
	with app.app_context():
		eplplayers = mongo.db.eplplayers
		async with ClientSession() as session:
			static_data = await fetch(session, static_url)
			for player in static_data['elements']:
				existing_player_obj = eplplayers.find_one({'id': player['id']})
				if existing_player_obj == None:
					eplplayers.insert_one({'id': player['id'], 'name': player['web_name'], 'team': player['team'], 'pos': player['element_type'], 'tsb': float(player['selected_by_percent']), 'prev_cost': player['now_cost'], 'cost': player['now_cost']})
				else:
					eplplayers.find_one_and_update({'id': player['id']}, {'$set': {'team': player['team'], 'tsb': float(player['selected_by_percent']), 'prev_cost': existing_player_obj['cost'], 'cost': player['now_cost']}})

async def update_season_history():
	with app.app_context():
		eplplayers_season_history = mongo.db.eplplayers_season_history
		eplteams = mongo.db.eplteams
		async with ClientSession() as session:
			static_data = await fetch(session, static_url)
			for player in static_data['elements']:
				player['team_name'] = eplteams.find_one({'id': player['team']})['short']
				player['position'] = get_player_position(player['element_type'])
				eplplayers_season_history.update_one({'id': "2020-21", 'player': player['id']}, {'$set': {"history": player}}, upsert=True)

async def update_gw_fixtures():
	with app.app_context():
		gwfixtures = mongo.db.gwfixtures
		gwfixtures.delete_many({})
		gw = mongo.db.currentgw.find_one()['gw']
		fixtures_url = 'https://fantasy.premierleague.com/api/fixtures/?event=%d' % gw
		async with ClientSession() as session:
			fixtures_data = await fetch(session, fixtures_url)
			gwfixtures.insert_many({'id': str(fix['id']), 'started': fix['started'], 'finished_90': fix['finished_provisional'], 'finished': fix['finished'], 'home': fix['team_h'], 'away': fix['team_a'], 'bps': fix['stats'][9]['a'] + fix['stats'][9]['h'] if fix['started'] == True else []} for fix in fixtures_data)

def get_fixture_status():
	with app.app_context():
		gwfixtures = mongo.db.gwfixtures
		fixture_status_dict = {}
		fixture_status_90_dict = {}
		for fixture in gwfixtures.find():
			fixture_status_dict[fixture['id']] = fixture['finished']
			fixture_status_90_dict[fixture['id']] = fixture['finished_90']
		return fixture_status_dict, fixture_status_90_dict

def process_live_points(live_data):
	with app.app_context():
		print('Updating Live Points!')
		livepoints = mongo.db.livepoints
		fixture_status_dict, fixture_status_90_dict = get_fixture_status()
		for player in live_data['elements']:
			player_id = player['id']
			points = player['stats']['total_points']
			fixture = []
			minutes = []
			for f in player['explain']:
				fixture.append(f['fixture'])
				for s in f['stats']:
					if s['identifier'] == 'minutes':
						minutes.append(s['value'])
						break
			live_player_obj = {'live_bonus': {}, 'fixture': fixture, 'minutes': minutes, 'points': points, 'fixture_finished': {str(f): fixture_status_dict[str(f)] for f in fixture}, 'fixture_finished_90': {str(f): fixture_status_90_dict[str(f)] for f in fixture}}
			livepoints.update_one({'id': player_id}, {'$set': live_player_obj}, upsert=True)
		update_live_bonus()
		cache.flush_all()

async def update_live_points():
	with app.app_context():
		await update_gw_fixtures()
		gw = mongo.db.currentgw.find_one()['gw']
		gwpoints = mongo.db.gwpoints
		live_url = 'https://fantasy.premierleague.com/api/event/%d/live' % gw
		async with ClientSession() as session:
			live_data = await fetch(session, live_url)
			gwpoints.update_one({'gw': gw}, {'$set': {'players': live_data['elements']}}, upsert=True)
			process_live_points(live_data)

def update_live_bonus():
	with app.app_context():
		livepoints = mongo.db.livepoints
		gwfixtures = mongo.db.gwfixtures
		for fixture in gwfixtures.find():
			if len(fixture['bps']) > 0:
				bps_dict = {}
				bonus_candidates_dict = {3: [], 2: [], 1: []}
				total_bonus_candidates = 0
				bps_values = sorted(fixture['bps'], key=lambda x: x['value'], reverse=True)
				#Group players by 'bonus points score' values
				for tup in bps_values:
					if tup['value'] in bps_dict:
						bps_dict[tup['value']] += [tup['element']]
					else:
						bps_dict[tup['value']] = [tup['element']]
				bps_dict = order_dict_by_key(bps_dict)
				for bps, players in bps_dict.items():
					#Break if there are at least 3 candidates already
					if total_bonus_candidates >= 3:
						break
					if len(bonus_candidates_dict[3]) == 0:
						bonus_candidates_dict[3] = players
					elif len(bonus_candidates_dict[2]) == 0:
						if total_bonus_candidates == 1:
							bonus_candidates_dict[2] = players
						else:
							bonus_candidates_dict[1] = players
					else:
						bonus_candidates_dict[1] = players
					total_bonus_candidates += len(players)
				for bonus_value, players in bonus_candidates_dict.items():
					for player in players:
						livepoints.update_one({'id': player}, {'$set': {'live_bonus': { fixture["id"]: bonus_value }}})

async def store_gw_points():
	with app.app_context():
		curr_gw = mongo.db.currentgw.find_one()['gw']
		gwpoints = mongo.db.gwpoints
		async with ClientSession() as session:
			for gw in range(1, curr_gw + 1):
				gw_url = 'https://fantasy.premierleague.com/api/event/%d/live/' % gw
				gw_data = await fetch(session, gw_url)
				gwpoints.update_one({'gw': gw}, {'$set': {'players': gw_data['elements']}}, upsert=True)
				print('Stored points for GW: ' + str(gw))

def save_picks(manager_id, manager_name, gw_data, gw_finished):
	with app.app_context():
		eplplayers = mongo.db.eplplayers
		playing, bench = [], []
		captain = None
		vicecaptain = None
		position_map = dict.fromkeys([str(x) for x in range(1, 16)], None)
		transcost = gw_data['entry_history']['event_transfers_cost']
		overall_rank = gw_data['entry_history']['overall_rank']
		total_points = gw_data['entry_history']['total_points'] - gw_data['entry_history']['points']
		if gw_finished:
			total_points += transcost
		team_value = (gw_data['entry_history']['value']) / float(10)
		chip = gw_data['active_chip']
		points = gw_data['entry_history']['points']
		bench_points = gw_data['entry_history']['points_on_bench']
		for pick in gw_data['picks']:
			if pick['is_captain']:
				captain = pick['element']
			if pick['is_vice_captain']:
				vicecaptain = pick['element']
			if pick['position'] >= 12:
				bench.append(pick['element'])
			else:
				playing.append(pick['element'])
			player_position = eplplayers.find_one({'id': pick['element']})['pos']
			position_map[str(pick['position'])] = player_position
		return {'code': manager_id, 'name': manager_name, 'overall_rank': overall_rank, 'total_points': total_points, 'team_value': team_value, 'position_map': position_map, 'points': points, 'bench_points': bench_points, 'captain': captain, 'vicecaptain': vicecaptain, 'chip': chip, 'cost': transcost, 'playing': playing, 'bench': bench}

def process_picks(manager_list, responses, gw_finished):
	tourny_gwpicks = mongo.db.tourny_gwpicks
	tourny_gwpicks.delete_many({})
	for manager_obj, manager_picks in zip(manager_list, responses):
		manager_code, manager_name = manager_obj
		manager_obj = save_picks(manager_code, manager_name, manager_picks, gw_finished)
		tourny_gwpicks.update_one({"code": manager_code}, {'$set': manager_obj}, upsert=True)

async def update_tourny_picks_custom():
	with app.app_context():
		tourny_teams = mongo.db.tourny_teams
		tourny_gwpicks = mongo.db.tourny_gwpicks
		curr_gw = mongo.db.currentgw.find_one()
		gw = curr_gw['gw']
		gw_finished = curr_gw['finished']
		picks_url = 'https://fantasy.premierleague.com/api/entry/{}/event/{}/picks/'
		manager_list, manager_urls = [], []
		team = tourny_teams.find_one({"id": "sparta_Patrick-Abdul-Ed"})
		manager_list += [(manager['code'], manager['manager']) for manager in team['managers']]
		manager_urls += [picks_url.format(manager['code'], gw) for manager in team['managers']]
		async with ClientSession() as session:
			logger.info("Requesting manager picks for tournys")
			responses = await fetch_all(session, manager_urls)
			logger.info("Received manager picks for tournys")
			for manager_obj, manager_picks in zip(manager_list, responses):
				manager_code, manager_name = manager_obj
				manager_obj = save_picks(manager_code, manager_name, manager_picks, gw_finished)
				tourny_gwpicks.update_one({"code": manager_code}, {'$set': manager_obj}, upsert=True)

async def update_tourny_picks():
	with app.app_context():
		tourny_teams = mongo.db.tourny_teams
		curr_gw = mongo.db.currentgw.find_one()
		gw = curr_gw['gw']
		gw_finished = curr_gw['finished']
		picks_url = 'https://fantasy.premierleague.com/api/entry/{}/event/{}/picks/'
		manager_list, manager_urls = [], []
		for team in tourny_teams.find():
			manager_list += [(manager['code'], manager['manager']) for manager in team['managers']]
			manager_urls += [picks_url.format(manager['code'], gw) for manager in team['managers']]
		async with ClientSession() as session:
			logger.info("Requesting manager picks for tournys")
			responses = await fetch_all(session, manager_urls)
			logger.info("Received manager picks for tournys")
			process_picks(manager_list, responses, gw_finished)

async def update_bot_user_picks():
	with app.app_context():
		bot_users = mongo.db.botusers
		logger.info("Requesting manager picks for bot")
		for user in bot_users.find():
			user_fpl_id = user['user_fpl_id']
			username = user['username']
			user_picks = await get_user_picks(user_fpl_id, username)
			bot_users.find_one_and_update({'user_fpl_id': user_fpl_id}, {'$set': {'picks': user_picks}})
		logger.info("Saved manager picks for bot")

async def get_user_picks(user_fpl_id, username):
	with app.app_context():
		curr_gw = mongo.db.currentgw.find_one()
		gw = curr_gw['gw']
		gw_finished = curr_gw['finished']
		picks_url = 'https://fantasy.premierleague.com/api/entry/{}/event/{}/picks/'.format(user_fpl_id, gw)
		history_url = 'https://fantasy.premierleague.com/api/entry/{}/history/'.format(user_fpl_id)
		async with ClientSession() as session:
			manager_data = await fetch_all(session, [picks_url, history_url])
			pick_data, history_data = manager_data
			pick_obj = save_picks(user_fpl_id, username, pick_data, gw_finished)
			pick_obj['current_season'] = history_data['current']
			pick_obj['season_history'] = [season['rank'] for season in history_data['past']]
			pick_obj['chips'] = history_data['chips']
			pick_obj['total_points'] = next((i_gw['total_points'] for i_gw in history_data['current'] if i_gw['event'] == gw - 1), 0)
			pick_obj['custom_points'] = next((i_gw['total_points'] for i_gw in history_data['current'] if i_gw['event'] == gw - 5), 0)
			return pick_obj

'''
def store_manager_history():
	with app.app_context():
		ffcteams = mongo.db.ffcteams
		manager_history = mongo.db.manager_history
		start = time.time()
		for team in teamList:
			obj = ffcteams.find_one({'team': team})
			for code in obj['codes']:
				history_url = 'https://fantasy.premierleague.com/api/entry/%d/history/' % (code)
				try:
					history_data = soupify(history_url)
				except ValueError:
					print(history_url)
				manager_history.insert_one({'code': code, 'chips': history_data['chips'], 'entry': history_data['entry'], 'past_seasons': history_data['season'], 'season_history': history_data['history']})
			print('Processed ' + team)
		print(time.time() - start)
'''

def init_tourny_standings(league):
	with app.app_context():
		tourny = mongo.db.tourny
		tourny_obj = tourny.find_one({'id': league})
		leaguestandings = mongo.db.leaguestandings
		standings = []
		c = 1
		for team in tourny_obj['teams']:
			item = {
				'id': c,
				'position': 1,
				'last_gw_position': 1,
				'team': team,
				'played': 0,
				'won': 0,
				'drawn': 0,
				'lost': 0,
				'gd': 0,
				'total': 0,
				'points': 0
			}
			c += 1
			standings.append(item)
		leaguestandings.insert_one({'league': league, 'standings': standings})
		print('Initialized standings')

def update_ffc_standings(league):
	with app.app_context():
		leaguestandings = mongo.db.leaguestandings
		league_obj = leaguestandings.find_one({'league': league})
		if league_obj == None: return
		standings = league_obj['standings']
		gw_results = [
			[('Liverpool', 573), ('Watford', 500)],
			[('Burnley', 539), ('Newcastle', 512)],
			[('Chelsea', 506), ('Bournemouth', 523)],
			[('Leicester', 537), ('Norwich', 469)],
			[('Sheffield Utd', 555), ('Aston Villa', 542)],
			[('Southampton', 493), ('West Ham', 464)],
			[('Man Utd', 522), ('Everton', 514)],
			[('Wolves', 605), ('Spurs', 526)],
			[('Arsenal', 573), ('Man City', 477)],
			[('Crystal Palace', 504), ('Brighton', 519)]
		]
		standings_dict = {}
		for fixture in gw_results:
			draw = False
			winner_points = 3
			loser_points = 0
			winner_bonus = 0
			loser_bonus = 0
			teamA = fixture[0][0]
			teamB = fixture[1][0]
			teamA_score = fixture[0][1]
			teamB_score = fixture[1][1]
			if abs(teamA_score - teamB_score) <= 9:
				draw = True
			elif teamA_score > teamB_score:
				winner = teamA
				loser = teamB
				winner_score = teamA_score
				loser_score = teamB_score
			else:
				winner = teamB
				loser = teamA
				winner_score = teamB_score
				loser_score = teamA_score
			gd = (abs(teamA_score - teamB_score)) // 10
			if draw:
				standings_dict[teamA] = {'played': 1, 'drawn': 1, 'total_score': teamA_score, 'points': 1}
				standings_dict[teamB] = {'played': 1, 'drawn': 1, 'total_score': teamB_score, 'points': 1}
			else:
				standings_dict[winner] = {'played': 1, 'won': 1, 'gd': gd, 'total_score': winner_score, 'points': winner_points}
				standings_dict[loser] = {'played': 1, 'lost': 1, 'gd': gd * -1, 'total_score': loser_score, 'points': loser_points}
		for obj in standings:
			update_team_obj = standings_dict[obj['team']]
			for prop, value in update_team_obj.items():
				obj[prop] += value
		leaguestandings.update_one({'league': league}, {'$set': {'standings': standings}})
		update_ffc_standing_positions(league)
		print('Updated standings')

def update_ffc_standing_positions(league):
	with app.app_context():
		standings = []
		leaguestandings = mongo.db.leaguestandings
		league_obj = leaguestandings.find_one({'league': league})
		if league_obj == None: return
		new_standings = multikeysort(league_obj['standings'], ['-points', '-gd', '-total_score', '-bonus'])
		for index, item in enumerate(new_standings):
			item['last_gw_position'] = item['position']
			item['position'] = index + 1
		leaguestandings.update_one({'league': league}, {'$set': {'standings': new_standings}})
		print('Updated positions')

def analyze_manager_history():
	manager_dict = {}
	with app.app_context():
		manager_history = mongo.db.manager_history
		for manager in manager_history.find():
			manager_rank = min([gw['rank'] for gw in manager['season_history']])
			manager_name = manager['entry']['player_first_name'] + ' ' + manager['entry']['player_last_name']
			min_gw = [gw['rank'] for gw in manager['season_history']].index(manager_rank) + 1
			min_entry = manager['season_history'][0]['entry']
			manager_dict[min_entry] = (manager_name, manager_rank, min_gw)
		top_managers = OrderedDict(sorted(manager_dict.items(), key = lambda x: x[1][1], reverse=False))
		for k, v in top_managers.items()[:20]:
			print(str(k) + ': %s' % (v,))

async def update_for_gw(update_gw=True):
	start = time.time()
	if update_gw:
		await update_current_gw()
	await update_epl_players()
	await update_live_points()
	await update_tourny_picks()
	# await update_bot_user_picks()
	logger.info("[update_for_gw] time: {}".format(time.time() - start))

def update_teams(league, league_file, sheet_name, column_row=0):
	with app.app_context():
		tourny = mongo.db.tourny
		teams = tourny.find_one({'id': league})['teams']
		tourny_teams = mongo.db.tourny_teams
		df = pd.read_excel(league_file, sheet_name=sheet_name, header=column_row)
		for team in teams:
			team_id = "_".join([league, team])
			team_df = df[df['Team'].str.contains(team)]
			manager_links = team_df['Link'].tolist()
			manager_codes = [int(re.findall(r'\d+', link)[0]) for link in manager_links]
			manager_names = team_df['Player'].tolist()
			manager_objs = [{'manager': manager, 'code': code} for manager, code in zip(manager_names, manager_codes)]
			print(team_id, manager_objs)
			tourny_teams.insert_one({'id': team_id, 'managers': manager_objs})
		print('Updated teams!')

def update_div2_fixtures():
	with app.app_context():
		tourny_fixtures = mongo.db.tourny_fixtures
		df = pd.read_excel('etc/Div2_fixtures.xlsx', header=0)
		fixture_list = df['FIXTURE'].tolist()
		fixture_list = [fixture_list[i:i + 10] for i in range(0, len(fixture_list), 10)]
		for i in range(len(fixture_list)):
			gw = i + 1
			fixtures = [f.replace('FFC', '').split(' vs ') for f in fixture_list[i]]
			fixtures = [[team.strip() for team in fx] for fx in fixtures]
			gw_fixtures = []
			for fx in fixtures:
				obj = {}
				obj['home'] = fx[0]
				obj['away'] = fx[1]
				gw_fixtures.append(obj)
			print(gw_fixtures)
			tourny_fixtures.insert_one({'id': 'div2', 'gw': gw, 'fixtures': gw_fixtures})
		print('Added div2 fixtures!')

def update_captains_bench(league):
	with app.app_context():
		tourny_captains = mongo.db.tourny_captains
		tourny_bench = mongo.db.tourny_bench
		tourny_teams = mongo.db.tourny_teams
		tourny = mongo.db.tourny
		teams = tourny.find_one({'id': league})['teams']
		for team in teams:
			team_id = "_".join([league, team])
			obj = tourny_teams.find_one({'id': team_id})
			captain_code = obj['managers'][0]['code']
			bench_code = obj['managers'][1]['code']
			tourny_captains.insert_one({'team': team_id, 'captain': captain_code})
			tourny_bench.insert_one({'team': team_id, 'bench': bench_code})
		print('Added captains and bench!')

async def get_league_managers(session, league_code, rank, gw):
	url = 'https://users.premierleague.com/accounts/login/'
	picks_url = 'https://fantasy.premierleague.com/api/entry/{}/event/{}/picks/'
	history_url = 'https://fantasy.premierleague.com/api/entry/{}/history/'
	league_page_url = 'https://fantasy.premierleague.com/api/leagues-classic/{}/standings/?page_new_entries=1&page_standings={}&phase=1'
	payload = {
		'password': os.environ.get('FPL_PASSWORD'),
		'login': os.environ.get('FPL_USERNAME'),
		'redirect_uri': 'https://fantasy.premierleague.com/a/login',
		'app': 'plfpl-web'
	}
	managers = []
	break_flag = False
	num_pages = rank // 40 + 2
	league_page_urls = [league_page_url.format(league_code, i) for i in range(1, num_pages)]
	await session.post(url, data=payload)
	logger.info("Requesting managers for league: {}".format(league_code))
	league_page_resp = await fetch_all(session, league_page_urls)
	logger.info("Received managers for league: {}".format(league_code))
	for league_page in league_page_resp:
		if len(league_page['standings']['results']) == 0:
			break
		for manager in league_page['standings']['results']:
			if manager['rank'] > rank:
				break_flag = True
				break
			managers.append({
				'manager_name': manager['player_name'],
				'manager_code': manager['entry'],
				'manager_url': picks_url.format(manager['entry'], gw),
				'manager_history_url': history_url.format(manager['entry'])
			})
		if break_flag:
			break
	return managers

def get_all_time_stats(season_history):
	if len(season_history) == 0: return ('NA', 'NA', 'NA', 'NA', 'NA', 'NA')
	season_history.reverse()
	top10k_finish = len([rank for rank in season_history if rank <= 10000])
	top100k_finish = len([rank for rank in season_history if rank <= 100000])
	total_seasons_played = len(season_history)
	best_rank = min(season_history)
	median = np.median(season_history, axis=0)
	if total_seasons_played == 1: return ('NA', median, top10k_finish, top100k_finish, total_seasons_played, best_rank)
	med_abs_deviation = np.median([abs(x - median) for x in season_history], axis=0)
	mad_score = [0.6745 * abs(rank - median) / med_abs_deviation for rank in season_history]
	thresh = math.sqrt(max(mad_score))
	season_history = [rank for rank in season_history if 0.6745 * abs(rank - median) / med_abs_deviation <= thresh]
	weighted_sum, sum_weights = 0, 0
	for i in range(len(season_history)):
		weighted_sum += season_history[i] * season_weights[i]
		sum_weights += season_weights[i]
	hof_or = round(float(weighted_sum) / float(sum_weights))
	return (hof_or, round(median), top10k_finish, top100k_finish, total_seasons_played, best_rank)

def get_all_time_stats_obj(season_history):
	all_time_stats = get_all_time_stats(season_history)
	all_time_stats_obj = {
		'hof': all_time_stats[0],
		'median': all_time_stats[1],
		'top10k': all_time_stats[2],
		'top100k': all_time_stats[3],
		'total_seasons': all_time_stats[4],
		'best_rank': all_time_stats[5]
	}
	return all_time_stats_obj

async def update_custom_league(league_name, league_code, league_type, rank=1000):
	with app.app_context():
		curr_gw = mongo.db.currentgw.find_one()
		gw = curr_gw['gw']
		gw_finished = curr_gw['finished']
		leaguecodes = mongo.db.leaguecodes
		leaguecodes.delete_one({'league': league_name})
		async with ClientSession() as session:
			c, manager_pick_objs = 0, []

			managers = await get_league_managers(session, league_code, rank, gw)
			manager_codes = [manager['manager_code'] for manager in managers]

			manager_urls = [manager['manager_url'] for manager in managers]
			manager_history_urls = [manager['manager_history_url'] for manager in managers]
			
			manager_picks_task = asyncio.create_task(fetch_all(session, manager_urls))
			manager_history_task = asyncio.create_task(fetch_all(session, manager_history_urls))
			manager_tasks = [manager_picks_task, manager_history_task]

			logger.info("Requesting manager picks and history for league: {}".format(league_name))
			manager_picks, manager_history = await asyncio.gather(*manager_tasks, return_exceptions=True)
			logger.info("Received manager picks and history for league: {}".format(league_name))

			for manager, pick_data, history_data in list(zip(managers, manager_picks, manager_history)):
				c += 1
				pick_obj = save_picks(manager['manager_code'], manager['manager_name'], pick_data, gw_finished)
				season_history = [season['rank'] for season in history_data['past']]
				pick_obj['season_history'] = season_history
				pick_obj['season_history_stats'] = get_all_time_stats_obj(season_history)
				pick_obj['chips'] = history_data['chips']
				pick_obj['total_points'] = next((i_gw['total_points'] for i_gw in history_data['current'] if i_gw['event'] == gw - 1), 0)
				pick_obj['custom_points'] = next((i_gw['total_points'] for i_gw in history_data['current'] if i_gw['event'] == gw - 5), 0)
				manager_pick_objs.append(pick_obj)
				if c % 1000 == 0: print('Processed picks:' + str(c))
			leaguecodes.insert_one({'league_id': league_code, 'league_type': league_type, 'codes': manager_codes, 'league': league_name, 'picks': manager_pick_objs})
			logger.info('Updated League: {}, num of managers: {}'.format(league_name, len(managers)))

async def update_existing_leagues():
	with app.app_context():
		start = time.time()
		leagues = mongo.db.leagues
		tasks = []
		leagues_objs = []
		for league_obj in leagues.find():
			rank = 1000
			if league_obj['league_name'] == 'top10k':
				rank = 10000
			leagues_objs.append((league_obj['league_name'], league_obj['league_code'], league_obj['league_type'], rank))
		for league_name, league_code, league_type, rank in leagues_objs:
			task = asyncio.create_task(update_custom_league(league_name, league_code, league_type, rank=rank))
			tasks.append(task)
		results = await asyncio.gather(*tasks, return_exceptions=True)
		print(time.time() - start)
		print('update_existing_leagues complete!')
		return results

def update_league_names():
	with app.app_context():
		leagues = mongo.db.leagues
		leaguecodes = mongo.db.leaguecodes
		for league_obj in leaguecodes.find():
			leagues.insert_one({'league_code': league_obj['league_id'], 'league_name': league_obj['league'], 'league_type': league_obj['league_type']})

def extract_team_names(div, div_file, sheet_name, column_row=0):
	with app.app_context():
		tourny = mongo.db.tourny
		df = pd.read_excel(div_file, sheet_name=sheet_name, header=column_row)
		team_names = list(set(df['Team'].tolist()))
		team_names = [team.replace(' FFC', '').strip() for team in team_names]
		team_names.sort()
		for x in team_names:
			print(x)
		tourny.update_one({'id': div}, {'$set': {'teams': team_names}})

async def get_bench_points():
	with app.app_context():
		leaguecodes = mongo.db.leaguecodes
		league_obj = leaguecodes.find_one({'league': 'maniacs'})
		history_urls = []
		history_url = 'https://fantasy.premierleague.com/api/entry/{}/history/'
		manager_names = []
		results = []
		for maniac in league_obj['picks']:
			name = maniac['name']
			code = maniac['code']
			history_urls.append(history_url.format(code))
			manager_names.append(name)
		async with ClientSession() as session:
			manager_history_data = await fetch_all(session, history_urls)
			for name, manager_data in zip(manager_names, manager_history_data):
				manager_bench_points = sum([gw['points_on_bench'] for gw in manager_data['current']])
				results.append((name, manager_bench_points))
			results = sorted(results, key=lambda x: x[1], reverse=True)
			for manager, points in results:
				print(manager, points)

def generate_fixtures(teams):
	if len(teams) % 2:
		raise Exception("Number of teams should be even.")
	n = len(teams)
	matches, fixtures, return_matches = ([] for i in range(3))
	for _ in range(1, n):
		for i in range(n // 2):
			matches.append((teams[i], teams[n - 1 - i]))
			return_matches.append((teams[n - 1 - i], teams[i]))
		teams.insert(1, teams.pop())
		fixtures.insert(len(fixtures) // 2, matches)
		fixtures.append(return_matches)
		matches = []
		return_matches = []

	return fixtures

def generate_tourny_fixtures(tourny_id):
	with app.app_context():
		tourny = mongo.db.tourny
		tourny_fixtures = mongo.db.tourny_fixtures
		teams = tourny.find_one({'id': tourny_id})['teams']
		fixtures = generate_fixtures(teams)
		w = 1
		for gw_fixtures in fixtures:
			fixtures_list = []
			for home_team, away_team in gw_fixtures:
				fixtures_list.append({
					'home': home_team,
					'away': away_team
				})
			tourny_fixtures.insert_one({'id': tourny_id, 'gw': w, 'fixtures': fixtures_list})
			w += 1

async def store_gwfixtures():
	with app.app_context():
		tourny_fixtures = mongo.db.tourny_fixtures
		eplteams = mongo.db.eplteams
		for w in range(1, 39):
			fixtures = []
			async with ClientSession() as session:
				fixtures_url = 'https://fantasy.premierleague.com/api/fixtures/?event=%d' % w
				fixtures_data = await fetch(session, fixtures_url)
				for fixture in fixtures_data:
					homeTeam = eplteams.find_one({'id': fixture['team_h']})['name']
					awayTeam = eplteams.find_one({'id': fixture['team_a']})['name']
					fixtures.append({
						'home': homeTeam,
						'away': awayTeam
					})
				tourny_fixtures.insert_one({'id': 'efpl', 'gw': w, 'fixtures': fixtures})

def get_best_form():
	with app.app_context():
		gw = mongo.db.currentgw.find_one()['gw']
		leaguecodes = mongo.db.leaguecodes
		target_league = leaguecodes.find_one({'league': 'top10k'})
		results = []
		best_points = 0
		for manager in target_league['picks']:
			points = manager['total_points'] + manager['points'] - manager['custom_points'] - manager['cost']
			if points >= best_points:
				if points == best_points:
					results += [(manager['code'], points)]
				else:
					best_points = points
					results = [(manager['code'], points)]
		print(results)

def get_lowest_10k():
	with app.app_context():
		gw = mongo.db.currentgw.find_one()['gw']
		leaguecodes = mongo.db.leaguecodes
		target_league = leaguecodes.find_one({'league': 'top10k'})
		results = []
		min_points = float('inf')
		for manager in target_league['picks']:
			points = manager['total_points']
			if points <= min_points:
				if points == min_points:
					results += [(manager['code'], points)]
				else:
					min_points = points
					results = [(manager['code'], points)]
		print(results)

def get_10k_history():
	with app.app_context():
		gw = mongo.db.currentgw.find_one()['gw']
		leaguecodes = mongo.db.leaguecodes
		target_league = leaguecodes.find_one({'league': 'top10k'})
		top1k_count, top10k_count, top100k_count = 0, 0, 0
		top1k_count_flat, top10k_count_flat, top100k_count_flat = 0, 0, 0
		flat_history = []
		for manager in target_league['picks']:
			if any([True for rank in manager['season_history'] if rank <= 1000]):
				top1k_count += 1
			if any([True for rank in manager['season_history'] if rank <= 10000]):
				top10k_count += 1
			if any([True for rank in manager['season_history'] if rank <= 100000]):
				top100k_count += 1
			top1k_count_flat += sum(rank < 1000 for rank in manager['season_history'])
			top10k_count_flat += sum(rank < 10000 for rank in manager['season_history'])
			top100k_count_flat += sum(rank < 100000 for rank in manager['season_history'])
			flat_history += manager['season_history']

		print(top1k_count, top10k_count, top100k_count)
		print(len(flat_history))
		print(top1k_count_flat, top10k_count_flat, top100k_count_flat)
		print(top1k_count_flat / len(flat_history) * 100, top10k_count_flat / len(flat_history) * 100, top100k_count_flat / len(flat_history) * 100)

def update_history_csv(filepath, season):
	with app.app_context():
		eplplayers_history = mongo.db.eplplayers_history
		data = pd.read_csv(filepath)
		data_json = json.loads(data.to_json(orient='records'))
		eplplayers_history.update_one({'id': season}, {'$set': {'history': data_json}}, upsert=True)

def update_history_col():
	with app.app_context():
		eplplayers_history = mongo.db.eplplayers_history
		eplplayers_season_history = mongo.db.eplplayers_season_history
		for season_obj in eplplayers_history.find():
			season = season_obj['id']
			for player_obj in season_obj['history']:
				player_history_data = {}
				player_history_data['id'] = season
				player_history_data['player'] = player_obj['id']
				player_history_data['history'] = player_obj
				eplplayers_season_history.insert_one(player_history_data)

def update_season_gw_history(target_dir, season):
	with app.app_context():
		eplplayers_season_gw_history = mongo.db.eplplayers_season_gw_history
		for subdir, dirs, files in os.walk(target_dir):
			for file in files:
				player_gw_data = {}
				if file == "gw.csv":
					filepath = os.path.join(subdir, file)
					data = pd.read_csv(filepath)
					data_json = json.loads(data.to_json(orient='records'))
					player_id = str(data_json[0]['element'])
					player_gw_data['id'] = season
					player_gw_data['player'] = player_id
					player_gw_data['gw'] = data_json
					eplplayers_season_gw_history.insert_one(player_gw_data)

def update_history_player_teams(season):
	with app.app_context():
		team_map = {1: 'ARS', 2: 'BOU', 3: 'BHA', 4: 'BUR', 5: 'CAR', 6: 'CHE', 7: 'CRY', 8: 'EVE', 9: 'FUL', 10: 'HUD', 11: 'LEI', 12: 'LIV', 13: 'MCI', 14: 'MUN', 15: 'NEW', 16: 'SOU', 17: 'TOT', 18: 'WAT', 19: 'WHU', 20: 'WOL'}
		eplplayers_history = mongo.db.eplplayers_history
		season_obj = eplplayers_history.find_one({'id': season})
		for player_obj in season_obj['history']:
			player_obj['team_name'] = team_map[player_obj['team']]
		eplplayers_history.update_one({'id': season}, {'$set': {'history': season_obj['history']}}, upsert=True)

def update_history_player_gw_opp_teams(season):
	with app.app_context():
		team_map = {}
		eplplayers_history = mongo.db.eplplayers_history
		eplplayers_season_gw_history = mongo.db.eplplayers_season_gw_history
		season_history_obj = eplplayers_history.find_one({'id': season})
		season_gw_objs = eplplayers_season_gw_history.find({'id': season})
		for player_obj in season_history_obj['history']:
			if player_obj['team'] not in team_map:
				team_map[player_obj['team']] = player_obj['team_name']
				if len(team_map) == 20: break
		print(team_map)
		for player_obj in season_gw_objs:
			for gw_obj in player_obj['gw']:
				gw_obj['opponent_team_name'] = team_map[gw_obj['opponent_team']]
			eplplayers_season_gw_history.update_one({'id': season, 'player': player_obj['player']}, {'$set': {'gw': player_obj['gw']}})

def update_player_cost():
	with app.app_context():
		eplplayers_history = mongo.db.eplplayers_history
		for season_obj in eplplayers_history.find():
			for player_obj in season_obj['history']:
				player_obj['now_cost'] /= 10
			eplplayers_history.update_one({'id': season_obj['id']}, {'$set': {'history': season_obj['history']}}, upsert=True)

def get_player_position(element_type):
	pos = "FWD"
	if element_type == 1:
		pos = "GKP"
	elif element_type == 2:
		pos = "DEF"
	elif element_type == 3:
		pos = "MID"
	return pos

def update_player_position():
	with app.app_context():
		eplplayers_history = mongo.db.eplplayers_history
		for season_obj in eplplayers_history.find():
			for player_obj in season_obj['history']:
				player_obj["position"] = get_player_position(player_obj['element_type'])
			eplplayers_history.update_one({'id': season_obj['id']}, {'$set': {'history': season_obj['history']}}, upsert=True)

def create_tourny(tourny_id):
	with app.app_context():
		tourny = mongo.db.tourny
		leagueteams = mongo.db.leagueteams
		tourny_obj = {
			'name': 'sparta',
			'captain': {
				'enabled': False
			}, 
			'bench': {
				'enabled': False
			}, 
			'scoring': 'sum',
			'home_advtg': {
				'enabled': True,
				'multiplier': 0.1
			},
			'teams': leagueteams.find_one({'league': tourny_id})['teams']