from __future__ import division

import ipdb
import os, sys
import itertools
import logging
import random
import collections

import networkx as nx
import numpy as np
from scipy.sparse import csr_matrix, coo_matrix
import matplotlib.pyplot as plt

lib_path = os.path.abspath('YenKSP')
sys.path.append(lib_path)
from YenKSP import graph
from YenKSP import algorithms

from networks.TrafficNetwork import TrafficNetwork
from synth_utils import to_np

__author__ = 'cathywu'

class GridNetwork(TrafficNetwork):

    def __init__(self, ncol=5, nrow=5, nodroutes=2, myseed=None, o_flow=1.0,
                 nnz_oroutes=2, concentration=None):
        TrafficNetwork.__init__(self)
        # we have n*m nodes, (((of which a 5*5 grid is for Caltec and a 5*5 grid
        # is for the streets (for now) --> imagine it as a 5 rows, 10 columns
        # grid, indexed like a matrix)))

        # Generate directed road network
        self.n, self.m, self.r = ncol, nrow, nodroutes
        self.G, self.sensors = self._construct_grid()
        self._add_weights()
        self._add_reverse()
        logging.debug('Graph generated')

        # Generate routes
        H = self._invert_graph_weights()
        self.routes = self._pairwise_shortest_routes(H)
        logging.debug('Routes generated')

        # Routes with non-zero flow
        self.nz_routes = None

        self.bbox = self.get_bounding_box()

        # Save seed for reproducibility
        if myseed is None:
            myseed = random.randint(0,4294967295)
        np.random.seed(myseed)
        random.seed(myseed)
        self.myseed = myseed

        self.o_flow = o_flow
        self.concentration = concentration
        self.nnz_oroutes = nnz_oroutes
        self.sample_OD_flow()

    def num_links(self):
        return len(self.G.edges())

    def get_route_flow(self,i):
        return self.routes[i]['flow']

    def _construct_grid(self):
        sensors = []
        G = nx.DiGraph()

        for j in range(0, self.n):
            for k in range(0, self.m):
                G.add_node(k * self.n + j, pos = (j, -k))

        for j in range(0, self.n-1):
            for k in range(0, self.m-1):
                G.add_edge(k * self.n + j, k * self.n + j + 1, weight=1)
                sensors.append((k * self.n + j, k * self.n + j + 1))
                G.add_edge(k * self.n + j, (k+1) * self.n + j, weight=1)
                sensors.append((k * self.n + j, (k+1) * self.n + j))

        # Add edges to the last row
        for j in range(0, self.n-1):
            G.add_edge((self.m-1)*self.n+j,(self.m-1)*self.n+j+1, weight=1)
            sensors.append(((self.m-1)*self.n + j, (self.m-1)*self.n + j + 1))
            # had to do this for this small node case to get at least 2 highways...

        # Add edges to the last column
        for k in range(0, self.m-1):
            G.add_edge(k*self.n + self.n-1, (k+1)*self.n + self.n-1, weight=1)
            sensors.append((k * self.n + self.n-1, (k+1) * self.n + self.n-1))

        return G, sensors

    def _add_reverse(self):
        """
        Add reverse edges of sensors to graph
        :return:
        """
        for (u, v, data) in self.G.edges(data=True):
            self.G.add_edge(v, u, weight = data['weight'])

        # Add opposite sensors
        sensors_opposite = [(v,u) for (u,v) in self.sensors]
        self.sensors.extend(sensors_opposite)

    def _invert_graph_weights(self):
        # Invert graph weights
        H = graph.DiGraph()
        H._data = {} # FIXME reset data, not entirely sure why this is necessary

        for (u, v) in self.G.edges():
            H.add_edge(u, v, cost = 1./self.G.edge[u][v]['weight'])
        return H

    def _add_weights(self):
        # Set bigger road weights
        for k in range(0, self.m-1):
            for j in range(0, self.n-1):
                # Highways, every 4 rows
                if k % 4 == 0:
                    self.G.edge[k*self.n + j][k*self.n + j+1]['weight'] = 6
                    # sensors.append((k*n + j,k*n + j+1))

                # Big streets, every other 4 rows
                if k % 4 == 2:
                    self.G.edge[k*self.n + j][k*self.n + j+1]['weight'] = 3
                    # Philipp: Hungry for sensors
                    # if j % 3 == 0:
                    # sensors.append((k*n + j,k*n + j+1))

                # Half big streets, every 2 columns
                if j % 2 == 0:
                    self.G.edge[k*self.n + j][(k+1)*self.n + j]['weight'] = 2
                    # oh my gosh, I want more sensors
                    # sensors.append((k*n + j,(k+1)*n + j))

    @staticmethod
    def has_duplicate(path):
        return len(set(path)) == len(path)

    def _pairwise_shortest_routes(self, H):
        # Find k shortest routes between all 2 nodes
        k_shortests = []

        for pair in itertools.product(range(0, self.n*self.m), repeat=2):
            u,v = pair[0], pair[1]
            if u == v:
                continue
            # dijkstra would be routes.append(nx.shortest_path(G,source=v,target=w))
            k_shortest = algorithms.ksp_yen(H, u, v, max_k = self.r)
            saved_routes = []
            for shortest in k_shortest:
                if GridNetwork.has_duplicate(shortest['path']):
                    k_shortest.remove(shortest)
                shortest['o'], shortest['d'], shortest['flow'] = u, v, 0
                saved_routes.append(shortest)
            k_shortests.extend(saved_routes)
        return k_shortests

    def get_route_indices_by_origin(self):
        route_indices_by_origin = collections.defaultdict(list)
        for i, r in enumerate(self.routes):
            route_indices_by_origin[r['o']].append(i)
        return route_indices_by_origin

    def _new_dict_OD(self):
        dict_OD = collections.defaultdict(list)
        for i, r in enumerate(self.routes):
            dict_OD[r['o']] = collections.defaultdict(list)
        return dict_OD

    def get_route_indices_by_OD(self):
        route_indices_by_OD = self._new_dict_OD()
        for i, r in enumerate(self.routes):
            route_indices_by_OD[r['o']][r['d']].append(i)
        return route_indices_by_OD

    def _get_OD_pairs(self):
        OD_pairs = collections.defaultdict(list)
        for i, r in enumerate(self.routes):
            OD_pairs[(r['path'][0],r['path'][-1])] = 1
        return OD_pairs.keys()

    # SAMPLE VARIOUS FLOWS (HELPER)
    # --------------------------------------------------------------------------
    def _get_heavy_edges(self, thresh=5):
        heavy = []
        for o,do in self.G.edge.items():
            for d,v in do.items():
                if v['weight'] >= thresh:
                    heavy.append((o,d))
        return heavy

    def get_heavy_points(self, thresh=5):
        heavy_edges = self._get_heavy_edges(thresh=thresh)
        heavy_points =[(self.G.node[e[0]]['pos'],self.G.node[e[1]]['pos']) \
                       for e in heavy_edges]
        return heavy_points

    def get_bounding_box(self, margin=0.05):
        pos = nx.get_node_attributes(self.G,'pos')
        x, y = zip(*pos.values())
        x_min, x_max, y_min, y_max = min(x), max(x), min(y), max(y)
        w, h = x_max-x_min, y_max-y_min
        x1, x2, y1, y2 = x_min - w*margin, x_max + w*margin, y_min - h*margin, \
                         y_max + h*margin
        return x1, y1, x2, y2

    # SAMPLE VARIOUS FLOWS
    # --------------------------------------------------------------------------
    def sample_OD_flow(self):
        if self.concentration is None:
            self._sample_flows(nnz_oroutes=self.nnz_oroutes)
        else:
            self._sample_flows_dense(concentration=self.concentration)
        self._update_nz_routes()

    def _sample_flows(self, nnz_oroutes=2):
        '''Generate traffic from each origin onto some small fraction of its routes, \
                and compute the amount of flow at each edge.'''

        # collect routes by origin or by OD pair
        # Note: All route indices are with respect to _routes_.
        route_indices_by_origin = self.get_route_indices_by_origin()
        route_indices_by_OD = self.get_route_indices_by_OD()

        flow_portions = [0] * len(self.routes) # from origin
        flow_portions_OD = [0] * len(self.routes) # from origin to destination
        self.od_flows = self._new_dict_OD()

        # initialize the flows, in case a node is not in the interior of any route
        for n in self.G.nodes():
            self.G.node[n]['2nd_flow'] = {}

        # initialize all flows on edges to 0
        for (u,v) in self.G.edges():
            self.G.edge[u][v]['flow'] = 0

        # sample routes and compute aggregate first-order, second-order info on
        # origins and origin-destination pairs

        for origin in self.G.nodes():
            # consider all routes out of origin
            route_indices_from_origin = route_indices_by_origin[origin]
            #    num_nonzero_routes = max(1, int(sparsity * len(route_indices_from_origin)))
            # select routes with non-zero flow
            selected_route_indices = sorted(random.sample(route_indices_from_origin,
                            min(nnz_oroutes,len(route_indices_from_origin))))
            # probability prior (uniform dirichlet) on non-zero flow routes
            selected_route_weights = np.random.dirichlet([1] * nnz_oroutes,
                                                            1)[0]

            self._update_flows(flow_portions, self.o_flow,
                              selected_route_indices,selected_route_weights)

            # normalize flows for each OD pair
            for dest in self.G.nodes():
                # normalize weights per OD pair
                selected_route_indices_OD = route_indices_by_OD[origin][dest]
                total = sum(flow_portions[i] for i in selected_route_indices_OD)
                self.od_flows[origin][dest] = total
                if total == 0:
                    continue
                for i in selected_route_indices_OD:
                    flow_portions_OD[i] = flow_portions[i]/total

    def _sample_flows_dense(self, concentration=0.1):
        '''Generate traffic from each origin onto some small fraction of its routes, \
                and compute the amount of flow at each edge.'''

        # collect routes by origin or by OD pair
        # Note: All route indices are with respect to _routes_.
        route_indices_by_OD = self.get_route_indices_by_OD()

        flow_portions_OD = [0] * len(self.routes) # from origin to destination
        self.od_flows = self._new_dict_OD()
        OD_pairs = self._get_OD_pairs()

        # initialize the flows, in case a node is not in the interior of any route
        for n in self.G.nodes():
            self.G.node[n]['2nd_flow'] = {}

        # initialize all flows on edges to 0
        for (u,v) in self.G.edges():
            self.G.edge[u][v]['flow'] = 0

        # sample OD pairs
        selected_OD_pairs = random.sample(OD_pairs,int(len(OD_pairs) * concentration))
        for origin,dest in selected_OD_pairs:
            self.od_flows[origin][dest] = self.o_flow
            selected_route_indices = route_indices_by_OD[origin][dest]
            num_routes = len(selected_route_indices)
            selected_route_weights = np.random.dirichlet([1] * num_routes, 1)[0]

            self._update_flows(flow_portions_OD, self.o_flow,
                              selected_route_indices,selected_route_weights)

    # FLOW UPDATE
    # --------------------------------------------------------------------------
    def _update_od_flows(self, od_flows):
        self.od_flows = od_flows

    def _update_flows(self, flow_portions, flow_from_each_node, r_ind, r_weights):
        """
        Update route flows, compute link flows and split ratios
        :param flow_portions:
        :param flow_from_each_node:
        :param r_ind:
        :param r_weights:
        :return:
        """
        for i, w in zip(r_ind, r_weights):
            flow_portions[i] = w
            self.routes[i]['flow'] = flow_from_each_node * w

            # add up flows on each link
            for u, v in zip(self.routes[i]['path'], self.routes[i]['path'][1:]):
                self.G.edge[u][v]['flow'] += flow_from_each_node * w

            # add up "turn" information on each transition
            # p = predecessor, n = node, s = successor
            for p, n, s in zip(self.routes[i]['path'], self.routes[i]['path'][1:],
                               self.routes[i]['path'][2:]):
                # add "second order flow"
                node = self.G.node[n]
                current_flow = node['2nd_flow'][(p, s)][0] if (p, s) in \
                                                    node['2nd_flow'] else 0
                current_routes = node['2nd_flow'][(p, s)][1] if (p, s) in \
                                                    node['2nd_flow'] else set()
                node['2nd_flow'][(p, s)] = (current_flow+flow_from_each_node*w,
                                            current_routes | set([i]))

    def _update_nz_routes(self, tol=1e-3):
        self.nz_routes = [i for (i,r) in enumerate(self.routes) if r['flow']>tol]

    # SIMPLEX
    # --------------------------------------------------------------------------
    def simplex_od(self):
        """Build simplex constraints from od flows
        """
        from cvxopt import matrix, spmatrix
        rids = self.get_route_indices_by_OD()
        n = sum([len(v) for v in self.od_flows.values()])
        m = len(self.routes)
        I, J, d, k = [], [], matrix(0.0, (n,1)), 0
        for i in self.od_flows.keys():
            for j, od_flow in self.od_flows[i].iteritems():
                d[k] = od_flow
                for rid in rids[i][j]:
                    I.append(k)
                    J.append(rid)
                k += 1
        T = coo_matrix(([1.0] * len(I),(I,J)), shape=(n,m)).tocsr()
        d = to_np(d)
        return T, d

    # ART
    # --------------------------------------------------------------------------
    def draw_graph(self):
        # draw nodes and edges
        pos = nx.get_node_attributes(self.G,'pos')
        nx.draw(self.G,pos)
        plt.hold()

        # draw edge weights
        edge_to_weight=dict([((u,v,),d['weight']) for u,v,d in \
                             self.G.edges(data=True)])
        from collections import defaultdict
        dd = defaultdict(list)
        for k, v in edge_to_weight.iteritems():
            dd[v].append(k)
        weight_to_edge = dict(dd)
        for k,v in weight_to_edge.iteritems():
            nx.draw_networkx_edges(self.G, pos, edgelist=v, width=k,
                                   alpha=0.5, edge_color='k')
        # draw sensors
        nx.draw_networkx_edges(self.G, pos, edgelist=self.sensors, width=1,
                               alpha=0.5, edge_color='b')

if __name__ == '__main__':
    import unittest
    from tests.test_grid_network import TestGridNetwork
    unittest.main()

    # G = GridNetwork()

    # G.draw_graph()


# python
from typing import cast
# 3rd party
import cv2 as cv # type: ignore
# project
import ganymede.opencv as g_cv
import ganymede.math.convert as gm_convert
import ganymede.math.point2 as gm_p2

from ganymede.opencv import CvPerspective

from ganymede.augmentation.augmentation_data import AugmentationData
from ganymede.augmentation.function.transform.perspective_coord_transformer import PerspectiveCoordTransformer
from ganymede.augmentation.function.transform.delegate import delegate_transform_points

def augmentate_rotate2d(
    data: AugmentationData,
    angle: float,
    interpolation: int = cv.INTER_AREA
) -> AugmentationData:
    img_h, img_w = data.image.shape[0:2]
    img_size_ratio = img_w / img_h

    rel_w = 1.0
    rel_h = 1.0 / img_size_ratio

    c_x = rel_w / 2
    c_y = rel_h / 2

    angle = -gm_convert.deg2rad(angle)

    src_points = ((0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0))

    prepare_points = [(0.0, 0.0), (rel_w, 0.0), (0.0, rel_h), (rel_w, rel_h)]

    rotate_points = [gm_p2.rotate(p, angle, (c_x, c_y))
                     for p in prepare_points]

    dst_points_list = gm_p2.normalize_on_self(rotate_points)

    dst_points = cast(CvPerspective, tuple(dst_points_list))

    l, t, r, b = gm_p2.get_contour(rotate_points)
    w_scale, h_scale = (r - l) / rel_w, (b - t) / rel_h
    output_size = int(w_scale * img_w), int(h_scale * img_h)

    img = g_cv.warp_perspective_on_keypoints(
        data.image, src_points, dst_points, interpolation, output_size)

    transform = PerspectiveCoordTransformer.from_points(
        src_points, dst_points)

    points = delegate_transform_points(data.points, transform)

    return AugmentationData(img, points)


# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
import os
import gym
import gym_donkeycar
import numpy as np
def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.

# Press the green button in the gutter to run the script.
if __name__ == '__main__':

    # SET UP ENVIRONMENT
    # You can also launch the simulator separately
    # in that case, you don't need to pass a `conf` object
    exe_path = "/home/mao/mysim/DonkeySimLinux/donkey_sim.x86_64"
    port = 9091

    conf = {"exe_path": exe_path, "port": port}

    env = gym.make("donkey-generated-track-v0", conf=conf)

    # PLAY
    obs = env.reset()
    for t in range(100):
        action = np.array([0.2, 0.1])  # drive straight with small speed
        # execute the action
        obs, reward, done, info = env.step(action)

    # Exit the scene
    env.close()

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


import pygame
import sys
import constants

def initialize_game():
    pygame.init()
    surface = pygame.display.set_mode((constants.SCREEN_WIDTH, constants.SCREEN_HEIGHT))
    pygame.display.set_caption(constants.TITLE)
    surface.fill(constants.WHITE)
    return surface

def read_board(gridfile):
    with open(gridfile, "r") as f:
        board_map = f.readlines()
    board_map = [line.strip() for line in board_map]
    return board_map

def get_space_color(space_contents):
    space_color = constants.DARKLAVA
    if space_contents == "X":
        space_color = constants.PASTELBROWN
    if space_contents == "-":
        space_color = constants.PALETAUPE
    return space_color

def draw_board(surface, grid_spaces):
    for j, space in enumerate(grid_spaces):
        for i, space_contents in enumerate(space):
            #print("{},{}: {}".format(i, j, space_contents))
            myrect = pygame.Rect(i*constants.BLOCK_WIDTH, j*constants.BLOCK_HEIGHT, constants.BLOCK_WIDTH, constants.BLOCK_HEIGHT)
            pygame.draw.rect(surface, get_space_color(space_contents), myrect)

def draw_lines(surface):
    for i in range(constants.GRID_ROWS):
        new_height = round(i * constants.BLOCK_HEIGHT)
        new_width = round(i * constants.BLOCK_WIDTH)
        pygame.draw.line(surface, constants.DARKLAVA, (0, new_height), (constants.SCREEN_WIDTH, new_height), 2)
        pygame.draw.line(surface, constants.DARKLAVA, (new_width, 0), (new_width, constants.SCREEN_HEIGHT), 2)

def game_loop(surface, board_grid):
    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()
            if event.type == pygame.KEYDOWN:
                pygame.quit()
                sys.exit()
        draw_board(surface, board_grid)
        draw_lines(surface)
        pygame.display.update()

def main():
    board_map = read_board(constants.GRIDFILE)
    surface = initialize_game()
    game_loop(surface, board_map)
"""
pyg.init()

# wood background
woodImage = pyg.image.load("wood_board.jpg")

def wood(x, y):
    surface.blit(woodImage, (x, y))

x = (screen_width // 2000)
y = (screen_height // 2000)

"""

if __name__ == "__main__":
    main()


class Atm(object):
    def __init__(self, cardNumber, pin):
        self.cardNumber = cardNumber
        self.pin = pin

    def balanceInquiry(self):
        print("Your balance is: 5000INR")

    def cashWithdrawl(self, amount):
        new_amount = 5000-amount
        print("You withdrawed: " + str(amount) + "INR Your remaining balance is: " + str(new_amount))

def main():
    name = input("Hello, Enter your name: ")
    print("Hello, " + name)

    cardNumber = input("Insert your card number: ")
    pin = input("Insert your pin: ")

    new_user = Atm(cardNumber, pin)

    print("Choose your activity:")
    print("1. Balance Inquiry")
    print("2. Cash Withdrawel")
    activity = int(input("Enter activity choice: "))

    if(activity == 1):
        new_user.balanceInquiry()
    elif(activity == 2):
        amount = int(input("Enter the amount: "))
        new_user.cashWithdrawl(amount)
    else:
        print("Enter a valid number choice")

main()


import csv

products = []

csv_file_path = "data/products.csv"
#csv_file_path_2 = "data/other_products.csv"

# READ PRODUCTS CSV
with open(csv_file_path, "r") as csv_file:
    reader = csv.DictReader(csv_file)
    for row in reader:
        products.append(row)

#Product Lookup
def lookup_product_by_id(product_id):
    matching_products = [product for product in products if product["id"] == product_id]
    return matching_products[0] # because the line above gives us a list and we want to return a single item

def valid_id(id):
    ids = []
    for product in products:
        ids.append(product["id"])
    while(id not in ids):
        id = input("WRONG PRODUCT INDENTIFIER! Please try again: ")
    return id

#Menu Greeting & Inputs
print("----------------------------------")
print("PRODUCTS APPLICATION")
print("----------------------------------")
print("Welcome User!")
print("\n")
print("There are " + str(len(products)) + " Products in the database. Please choose an operation:" )
print("\n")
print("    operation | description" )
print("    --------- | -----------")
print("    'List'    | Display a list of product identifiers and names.")
print("    'Show'    | Show information abouta product.")
print("    'Create'  | Add a new product.")
print("    'Update'  | Edit an exisiting product.")
print("    'Destroy' | Delete an exisiting product.")

print("\n")
chosen_operation = input("Input Operation: ")
chosen_operation = chosen_operation.title()
print(chosen_operation)

#Menu Definitions
def list_products():
    for product in products:
        print(" + ", dict(product))

def show_product():
    product_id = input("Please input a valid product identifier. ")
    product_id = valid_id(product_id)
    for product in products:
        product_show = lookup_product_by_id(product_id)
    print("SHOW PRODUCT HERE: ", dict(product_show))

def create_product():
    print("CREATING A PRODUCT")
    product_name = input("name is:")
    product_aisle = input("aisle is:")
    product_department = input("department is:")
    product_price = input("price is:")
    new_product = {
        "id": len(products) + 1,
        "name": product_name,
        "aisle": product_aisle,
        "department": product_department,
        "price": product_price
    }
    print("NEW PRODUCT IS", new_product)
    products.append(new_product)

def update_product():
    print("UPDATING A PRODUCT")
    update_product_id = input("Please input a valid product identifier. ")
    product = [i for i in products if i["id"]==update_product_id]
    if product:
        product = product[0]
        print(dict(product))
        update_product_name = input("What should we change the NAME to?: ")
        update_product_aisle = input ("What should we change the AISLE to?: ")
        update_product_department= input("What should we change DEPARTMENT to?: ")
        update_product_price = input("What should we change PRICE to?: ")
        updated_product = {
        "name":update_product_name,
        "aisle":update_product_aisle,
        "department":update_product_department,
        "price":update_product_price
        }
        print("We will update to ",dict(updated_product) )
        confirmation = input("Please type Y if its okay to update: ")
        confirmation = confirmation.capitalize()
        if confirmation == "Y":
            product["name"] = update_product_name
            product["aisle"]= update_product_aisle
            product["department"]=update_product_department
            product["price"]=update_product_price
            print("Updated product")
        else:
            print("Try again")
    else:
            print("Not a valid ID, please try again")

def destroy_product():
    print("DESTROYING A PRODUCT")
    destroy_product = input("Please input a valid product identifier. ")
    product = [i for i in products if i["id"] == destroy_product]
    if product:
        product = product[0]
        print("I am destroying the following product: ")
        print(dict(product))
        confirmation = input("Please type Y if its okay to destroy: ")
        confirmation = confirmation.capitalize()
        if confirmation == "Y":
            del products[products.index(product)]
            print("Updated product list")
        else:
            print("Try again")

#Menu If Statements
if chosen_operation == "List": list_products()
elif chosen_operation == "Show": show_product()
elif chosen_operation == "Create": create_product()
elif chosen_operation == "Update": update_product()
elif chosen_operation == "Destroy": destroy_product()
else: print("OOPS. PLEASE CHOOSE ONE OF THE RECOGNIZED OPERATIONS.")

# OVERWRITING INVENTORY CSV FILE
with open(csv_file_path, "w", newline='') as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=["id","name","aisle","department","price"])
    writer.writeheader() # uses fieldnames set above
    for product in products:
        writer.writerow(product)


import pandas as pd
import sklearn as sk
from sklearn import preprocessing,decomposition
import matplotlib.pyplot as plt
import os

def cleanse_missing_grades(df):
    """A function to replace the missing grades for HW and Exam with 0"""

    for hwno in range(1,4):
        df['Homework {}'.format(hwno)]=df['Homework {}'.format(hwno)].fillna(0)

    df['Exam']=df['Exam'].fillna(0)

def question_one_part_one(df):
    """A function to complete Q1P1"""

    print("""
    -
    QUESTION 1 - P1
    -

    """)

    def df_describe(df):
        """The dataframe before cleansing"""

        answer_df=df.describe().T

        print("The df shape is {}".format(df.shape))
        mean_col_name='mean'
        min_col_name='min'
        max_col_name='max'
        std_col_name='std'
        count_col_name='count'

        for homework_no in range(1,4):
            homework_column="Homework {}".format(homework_no)
            answer_statement="""
    Homework: {}
    Minimum: {}
    Maximum: {}
    Mean: {}
    Std. Deviation: {}
    Count: {}
    """
            filter_df=answer_df[answer_df.index==homework_column]

            min_col=filter_df[min_col_name][0]
            max_col=filter_df[max_col_name][0]
            mean_col=filter_df[mean_col_name][0]
            std_col=filter_df[std_col_name][0]
            cnt_col=filter_df[count_col_name][0]

            print(answer_statement.format(homework_column,
                                          min_col,
                                          max_col,
                                          mean_col,
                                          std_col,
                                         cnt_col))

        return

    df=pd.read_csv("./specs/Students_Results.csv")

    print("""
    ---
    Before Cleansing
    ---

    """)

    df_describe(df=df)

    cleanse_missing_grades(df=df)

    print("""
    ---
    After Cleansing
    ---

    """)

    df_describe(df=df)

    return df

def question_one_part_two(df):

    print("""
    -
    QUESTION 1 - P2
    -

    """)

    df['Homework Avg']=round((
                            (  (df['Homework 1'] +
                                df['Homework 2'] +
                                df['Homework 3'])/3
                            )
                       ),1)

    df['Overall Mark']=round(0.25*df['Homework Avg'] + 0.75*df['Exam'],2)

    return df

def question_one_part_three(df):
    """Create a correlation matrix"""

    print("""
    -
    QUESTION 1 - P3
    -
    """)

    print("Correlation matrix: ")

    non_stud_col=[]

    for col in df.columns:

        if col!='Student ID':
            non_stud_col+=[col]

    corr_df=df[non_stud_col].corr()

    print(corr_df)

    return

def question_one_part_four():
    """See assignment"""
    return

def question_one_part_five(df):
    """Question 1 Part 5"""

    def ucd_grading(row):
        """UCD Computer Science Grading"""

        Ap=95
        A=90
        Am=85
        Bp=80
        B=75
        Bm=70
        Cp=65
        C=60
        Cm=55
        Dp=50
        D=45
        Dm=40
        Ep=35
        E=30
        Em=25
        Fp=20
        F=15
        Fm=10
        Gp=8
        G=5
        Gm=2
        NG=0

        if row['Overall Mark']>=Ap and row['Overall Mark']<=100:
              return 'A+'

        if row['Overall Mark']>=A and row['Overall Mark']<Ap:
              return 'A'

        if row['Overall Mark']>=Am and row['Overall Mark']<A:
              return 'A-'

        if row['Overall Mark']>=Bp and row['Overall Mark']<Am:
              return 'B+'

        if row['Overall Mark']>=B and row['Overall Mark']<Bp:
              return 'B'

        if row['Overall Mark']>=Bm and row['Overall Mark']<B:
              return 'B-'

        if row['Overall Mark']>=Cp and row['Overall Mark']<Bm:
              return 'C+'

        if row['Overall Mark']>=C and row['Overall Mark']<Cp:
              return 'C'

        if row['Overall Mark']>=Cm and row['Overall Mark']<C:
              return 'C-'

        if row['Overall Mark']>=Dp and row['Overall Mark']<Cm:
              return 'D+'

        if row['Overall Mark']>=D and row['Overall Mark']<Dp:
              return 'D'

        if row['Overall Mark']>=Dm and row['Overall Mark']<D:
              return 'D-'

        if row['Overall Mark']>=Ep and row['Overall Mark']<Dm:
              return 'E+'

        if row['Overall Mark']>=E and row['Overall Mark']<Ep:
              return 'E'

        if row['Overall Mark']>=Em and row['Overall Mark']<E:
              return 'E-'

        if row['Overall Mark']>=Fp and row['Overall Mark']<Em:
              return 'F+'

        if row['Overall Mark']>=F and row['Overall Mark']<Fp:
              return 'F'

        if row['Overall Mark']>=Fm and row['Overall Mark']<F:
              return 'F-'

        if row['Overall Mark']>=Gp and row['Overall Mark']<Bm:
              return 'G+'

        if row['Overall Mark']>=G and row['Overall Mark']<Gp:
              return 'G'

        if row['Overall Mark']>=Gm and row['Overall Mark']<G:
              return 'G-'

        if row['Overall Mark']>=NG and row['Overall Mark']<Gm:
              return 'NG'

        return 'Grade Error'

    df['Grade']= df.apply(lambda row: ucd_grading(row), axis=1)

    ax=(df['Overall Mark']
            .plot(
                kind='hist'
                ,title='Histogram of Overall Marks'
                ,xlabel='Overall Mark'
                ,ylabel='No of Students'
                ,bins=10
                )
            .set_xlim((0,100))
        )

    plt.savefig('./output/histogram_of_grades_q1.png')

    df_grade=df['Grade'].value_counts().reindex(index = ['A+','A','A-',
                                                         'B+','B','B-',
                                                         'C+','C','C-',
                                                         'D+','D','D-',
                                                        'E+','E','E-',
                                                        'F+','F','F-',
                                                        'G+','G','G-'])

    new_ax=(df_grade
            .plot(
                kind='bar'
                ,title='Barchart of Grades'
                ,xlabel='Grade'
                ,ylabel='No of Students'
                )
        )

    plt.savefig('./output/barchart_of_grades_q1.png')

    return df

def question_one_part_six(df):
    df.to_csv('./output/question1_out.csv',index=False)

def question_one():
    student_df=pd.read_csv('./specs/Students_Results.csv')

    #Part 1
    student_df=question_one_part_one(df=student_df)

    #Part 2
    student_df=question_one_part_two(df=student_df)

    question_one_part_three(df=student_df)

    question_one_part_four()

    student_df=question_one_part_five(df=student_df)

    question_one_part_six(df=student_df)

    return student_df

def question_two_part_one(df):
    df['Original Input3']=df['Input3']
    df['Original Input12']=df['Input12']
    return df

def question_two_part_two(df):
    """Scale the columns"""

    #Z score
    df['Input3']=(df['Input3'] - df['Input3'].mean())/df['Input3'].std()

    return df

def question_two_part_three(df):
    """Scale the columns"""

    #MinMax
    if df['Input12'].max()-df['Input12'].min()>0:
        #Have to round because of the test_practical only has three decimal places.
        df['Input12']=round((df['Input12']-df['Input12'].min())/(df['Input12'].max()-df['Input12'].min()),3)
    else:
        print("Error: Min=Max")

    return df

def question_two_part_four(df):

    inputs=[]
    for input_no in range(1,13):
        inputs+=['Input{}'.format(input_no)]

    df['Average Input']=df[inputs].mean(axis=1)

    return df

def question_two_part_five(df):

    df.to_csv('./output/question2_out.csv',index=False)

    return

def question_two():
    sensor_df=pd.read_csv('./specs/Sensor_Data.csv')
    sensor_df=question_two_part_one(sensor_df)
    sensor_df=question_two_part_two(sensor_df)
    sensor_df=question_two_part_three(sensor_df)
    sensor_df=question_two_part_four(sensor_df)
    question_two_part_five(sensor_df)
    return

def question_three_part_one(df):

    def pca_given_limit(df,cut_off_limit=0.95):
        """PCA to get components given a cutoff"""

        component_no=1
        explained_variance = {}
        explained_variance['default'] = -1

        #Add in components
        while max(explained_variance.values())<cut_off_limit and component_no<len(df.columns):
            pca = decomposition.PCA(n_components = component_no)
            pca.fit(df)
            post_pca_data = pca.transform(df)

            for component in range(0, len(pca.explained_variance_ratio_)):

                current_feature='PCA_{}'.format(component)

                if component == 0:
                    explained_variance[current_feature]=pca.explained_variance_ratio_[component]

                else:
                    previous_explained_variance=explained_variance['PCA_{}'.format(component-1)]
                    explained_variance[current_feature] = pca.explained_variance_ratio_[component] + previous_explained_variance

            component_no+=1

        return post_pca_data, explained_variance

    pca_data, var_dict = pca_given_limit(df,cut_off_limit=0.95)

    pca_col_names=[]

    for k in range(0,pca_data.shape[1]):
        pca_col_names+=['pca{}'.format(k)]

    pca_df=pd.DataFrame(pca_data)
    pca_df.columns=pca_col_names

    return pca_df, var_dict

def question_three_part_two(original_df,pca_df):
    """Use cut to complete part 2"""

    pca_columns=pca_df.columns

    for column in pca_columns:
        original_df["{}_width".format(column)]=pd.cut(pca_df[column],10)

    return original_df

def question_three_part_three(original_df,pca_df):
    """Use qcut to complete part three"""

    pca_columns=pca_df.columns

    for column in pca_columns:
        original_df["{}_freq".format(column)]=pd.qcut(pca_df[column],10)

    return original_df

def question_three_part_four(df):
    """Complete part 4 by saving the DF"""

    df.to_csv('./output/question3_out.csv',index=False)

    return

def question_three():
    """Question Three"""
    df=pd.read_csv('./specs/DNA_Data.csv')
    pca_df, var_dict=question_three_part_one(df)
    question_three_part_two(original_df=df,pca_df=pca_df)
    question_three_part_three(original_df=df,pca_df=pca_df)
    question_three_part_four(df=df)
    return

def run_all():
    """A function to run all of the tutorial"""

    if not os.path.exists('./output/'):
        os.makedirs('./output/')

    question_one()
    question_two()
    question_three()

run_all()


from __future__ import division
import tensorflow as tf
from utils import *

import tensorflow as tf
import tensorflow.contrib.slim as slim

def instance_norm(x, scope_name = "instance_norm"):
    with tf.variable_scope(scope_name):
        channel = x.get_shape()[3]
        epsilon = 1e-5
        mean, var = tf.nn.moments(x, [1, 2], keep_dims=True)
        scale = tf.get_variable('scale',[channel], initializer=tf.truncated_normal_initializer(mean=1.0, stddev=0.02))
        offset = tf.get_variable('offset',[channel],initializer=tf.constant_initializer(0.0))
        out = scale*tf.div(x-mean, tf.sqrt(var+epsilon)) + offset
        return out

def lrelu(x, leak=0.2, scope_name="lrelu"):
     with tf.variable_scope(scope_name):
         f1 = 0.5 * (1 + leak)
         f2 = 0.5 * (1 - leak)
         return f1 * x + f2 * abs(x)

def ResnetBlock(x, dim, use_dropout, name="resnetblock"):
    """
    so far, only padding_type=reflect is supported for tf.pad
    and padding="VALID" is supported for slim.conv2d
    """
    with tf.variable_scope(name):
        output = tf.pad(x,[[0,0],[1,1],[1,1],[0,0]],"REFLECT")
        output = slim.conv2d(output, num_outputs=dim, kernel_size=[3, 3], stride=1, padding="VALID",
                             normalizer_fn=None, activation_fn = None, scope=name+"_conv1")
        output = instance_norm(output, scope_name = name + "_instance_norm1")
        output = tf.nn.relu(output)

        if use_dropout:
            output = slim.dropout(output, keep_prob=0.5, scope=name+"_dropout")

        output = tf.pad(output, [[0,0],[1,1],[1,1],[0,0]], "REFLECT")
        output = slim.conv2d(output, num_outputs=dim, kernel_size=[3, 3], stride=1, padding="VALID",
                            normalizer_fn=None, activation_fn=None,
                            scope=name+"_conv2")
        output = instance_norm(output, scope_name = name + "_instance_norm2")
        return x + output

def ResnetGenerator(x, output_nc, ngf=64, use_dropout=False, n_blocks=6, name="ResnetGenerator_6"):
    """
    in original pytorch verion, bias is added to conv2d, while bias is ignored here
    """
    with tf.variable_scope(name):
        x = tf.pad(x, [[0,0],[3,3],[3,3],[0,0]], "REFLECT")
        output = slim.conv2d(x, num_outputs=ngf, kernel_size=[7, 7], padding="VALID",
                           normalizer_fn=None, activation_fn=None, scope=name+"_conv")
        output = instance_norm(output, scope_name = name + "_instance_norm")
        output = tf.nn.relu(output)

        n_downsampling = 2
        for i in range(n_downsampling):
            mult = 2**i
            output = slim.conv2d(output, num_outputs=ngf * mult * 2, kernel_size=[3, 3],
                               stride=2, padding="SAME",
                               normalizer_fn=None, activation_fn=None, scope=name+"_conv_"+str(i))
            output = instance_norm(output, scope_name = name + "_instance_norm_" + str(i))
            output = tf.nn.relu(output)

        mult = 2 ** n_downsampling
        for i in range(n_blocks):
            output = ResnetBlock(output, ngf * mult,use_dropout=use_dropout, name=name+"_resnetblock_"+str(i))

        for i in range(n_downsampling):
            mult = 2 ** (n_downsampling - i)
            output = slim.conv2d_transpose(output, num_outputs=int(ngf*mult/2), kernel_size=[3, 3], stride=2,
                                            padding="SAME",normalizer_fn=None, activation_fn=None,
                                            scope=name+"_convtrans_"+str(i))
            output = instance_norm(output, scope_name = name + "_transpose_instance_norm_"+str(i))
            output = tf.nn.relu(output)

        output = tf.pad(output, [[0,0],[3,3],[3,3],[0,0]],"REFLECT")
        output = slim.conv2d(output, num_outputs=output_nc, kernel_size=[7, 7], stride=1, padding="VALID",
                            normalizer_fn=None, activation_fn=None, scope=name+"_conv_last")

        output = tf.nn.tanh(output, name + "_tanh")

        return output

def NLayerDiscriminator(x, ndf=64, n_layers=3, use_sigmoid=False, name="Discriminator"):
    with tf.variable_scope(name):
        kw=4
        output = slim.conv2d(x, num_outputs=ndf, kernel_size=[kw, kw], stride=2, padding="SAME",
                            normalizer_fn=None, activation_fn=None, scope=name+"_conv")
        output = lrelu(output, scope_name = name + "_leakyrelu1")

        nf_mult=1
        nf_mult_prev=1
        for n in range(1, n_layers):
            nf_mult_prev = nf_mult
            nf_mult = min(2**n, 8)
            output = slim.conv2d(output, num_outputs=ndf * nf_mult, kernel_size=[kw, kw], stride=2,padding="SAME",
                                normalizer_fn=None, activation_fn=None, scope=name+"_conv_"+str(n))
            output = instance_norm(output, scope_name = name + "_first_instance_norm"+ str(n))
            output = tf.nn.relu(output, name + "_firstconv_lrelu"+str(n))

        nf_mult_prev = nf_mult
        nf_mult = min(2**n_layers, 8)
        output = slim.conv2d(output, num_outputs=ndf * nf_mult, kernel_size=[kw, kw], stride=1,
                            padding="SAME",
                            normalizer_fn=None, activation_fn=None, scope=name+"_second_conv_"+str(n_layers))
        output = instance_norm(output, scope_name = name + "_second_instance_norm")
        output = lrelu(output, scope_name = name + "_secondconv_lrelu")

        output = slim.conv2d(output, 1, kernel_size=[kw, kw], stride=1,padding="SAME",
                            normalizer_fn=None, activation_fn=None, scope=name+"_last_conv")

        if use_sigmoid:
            output = tf.nn.Sigmoid(output)

        return output

def define_G(x, output_nc, ngf, n_blocks, use_dropout=False, reuse=False, scope_name = "d_G"):
    with tf.variable_scope(scope_name):
        if reuse:
            tf.get_variable_scope().reuse_variables()
        else:
            assert tf.get_variable_scope().reuse is False

        output = ResnetGenerator(x, output_nc=output_nc, ngf=ngf, use_dropout=use_dropout, n_blocks=n_blocks, name = scope_name + "_Generator")
        return output

def define_D(x, ndf, n_layers_D=3, use_sigmoid=False, reuse=False,scope_name = "d_D"):
    with tf.variable_scope(scope_name):
        if reuse:
            tf.get_variable_scope().reuse_variables()
        else:
            assert tf.get_variable_scope().reuse is False
        outputs = NLayerDiscriminator(x, ndf, n_layers_D, use_sigmoid=use_sigmoid, name = scope_name + "_Discriminator")
        return outputs

def abs_criterion(in_, target):
    return tf.reduce_mean(tf.abs(in_ - target))

def mae_criterion(in_, target):
    return tf.reduce_mean((in_-target)**2)

def sce_criterion(logits, labels):
    return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=labels))


class Solution:
    def knapsack(self, w, v, c):
        """
        dynamic programing:
        Given package space c,weights w and corresponding value v,we initialize a two-matrix dp with rows=len(w) and cols=c+1.
        Obvious,elements in the first column of matrix dp are equal to 0 because package(left space=0) can contains nothing.
        Hence,dp[i][j] denotes the max value of deciding i-th items and left space is equal to j.
        Therefore,given i-th item,if j>w[i], dp[i][j]=max{v[i]+dp[i-1][j-w[i]],dp[i-1][j]}.
        otherwise dp[i][j]=dp[i-1][j]
        Finally,return dp[len(w)-1][c].
        And we can know which items are selected by this matrix.For example,this matrix is displayed:
        [0, 1, 1, 1, 1, 1, 1, 1]
        [0, 1, 1, 4, 5, 5, 5, 5]
        [0, 1, 1, 4, 5, 6, 6, 9]
        [0, 1, 1, 4, 5, 7, 8, 9]
        As we known,element dp[3][7] is desirable.
        dp[2][7]=dp[3][7] isn't selected.And dp[2][7]!=dp[1][7] hence,2-nd item is selected.
        Hence we go back to dp[1][7-w[2]]=dp[1][3].dp[1][3]!=dp[0][3],i.e.,1-st item is also selected.
        Up to now,we can make a conclusion that selected items are 1-st and 2-nd items.
        :param w: items weight list
        :param v: corresponding items value list
        :return:
        reference: https://www.youtube.com/watch?v=8LusJS5-AGo&t=0s&list=PLrmLmBdmIlpsHaNTPP_jHHDx_os9ItYXr&index=2
        """
        rows=len(w)
        dp = [[0 for _ in range(c+1)] for _ in range(rows)]
        for i in range(rows):
            for j in range(1, c+1):
                if j < w[i]:
                    dp[i][j] = dp[i-1][j]
                else:
                    dp[i][j] = max(v[i]+dp[i-1][j-w[i]], dp[i-1][j])
        print(dp[rows-1][c])
        k = c
        path = []
        i=rows-1
        while k > 0 and i>0:
            if dp[i][k] != dp[i-1][k]:
                k=k-w[i]
                path.append(i)
            i-=1
        print(path)

s = Solution()
s.knapsack([1, 3, 4, 5], [1, 4, 5, 7], 7)


import os
import pandas as pd
from skimage.metrics import structural_similarity as  compare_ssim
import numpy as np
import cv2
from PIL import Image

def SSIM(img1, img2):
    # 计算两个灰度图像之间的结构相似度
    (score, diff) = compare_ssim(img1, img2, gaussian_weights=True, data_range=255,
                                 full=True)  # 滑动窗口大小101 , gaussian_weights=True , win_size=11  data_range=255,
    # (score, diff) = compare_ssim(img1, img2, multichannel=True, gaussian_weights=True, full=True) # 采用高斯加权计算每一窗口的均值、方差以及协方差，sigma=1.5
    # data[i] = score
    return score

def pearson(image1, image2):
    '''皮尔逊相关系数'''
    X = np.vstack([image1, image2])
    return np.corrcoef(X)[0][1]
def compute_mertics(data_root, gt_dir, pred_dir, flu):
    fl = os.listdir(gt_dir)
    i = 0
    pcc_all = 0
    ssim_all = 0
    a = []
    b = []
    c = []
    data = {}
    for file in fl:
        # print(file)
        i = i + 1
        image1_path = gt_dir + '/' + file
        image2_path = pred_dir + '/' + file

        image1 = cv2.imread(image1_path, 0)
        image2 = cv2.imread(image2_path, 0)
        ssim = SSIM(image1, image2)
        image1 = Image.fromarray(np.uint8(image1))
        image2 = Image.fromarray(np.uint8(image2))
        image1 = image1.resize(image2.size)
        image1 = np.asarray(image1).flatten()
        image2 = np.asarray(image2).flatten()
        pcc = pearson(image1, image2)

        # jcd = jaccard(image1, image2)
        pcc_all = pcc_all + pcc
        ssim_all = ssim_all + ssim
        ssim = round(ssim, 4)
        pcc = round(pcc, 4)
        data['name'] = a.append(file)
        data['pcc'] = b.append(pcc)
        data['ssim'] = c.append(ssim)
        # print(file,f'pcc={pcc},ssim={ssim}')
        # print(pearson(image2, image3))
    pcc_avg = pcc_all / i
    ssim_avg = ssim_all / i
    pcc_std = np.std(b)
    ssim_std = np.std(c)
    pcc_avg = round(pcc_avg, 4)  # 保留小数点后四位
    pcc_std = round(pcc_std, 4)
    ssim_avg = round(ssim_avg, 4)
    ssim_std = round(ssim_std, 4)
    print(f'pcc_avg={pcc_avg},pcc_std={pcc_std}')
    print(f'ssim_avg={ssim_avg},ssim_std={ssim_std}')
    data['name'] = a.append('avg')
    data['pcc'] = b.append(pcc_avg)
    data['ssim'] = c.append(ssim_avg)
    data['name'] = a.append('std')
    data['pcc'] = b.append(pcc_std)
    data['ssim'] = c.append(ssim_std)

    df = pd.DataFrame({'name': a, 'pcc': b, 'ssim': c})
    save_path = data_root + '/' + f'{flu}_pcc_and_ssim.csv'  # _denoise
    df.to_csv(path_or_buf=save_path, sep=',', na_rep='NA',
              columns=['name', 'pcc', 'ssim'])  # headers = False(不保存列名) index = False(不保存索引)

if __name__=="__main__":
    save_dir = './result'
    pb_aug ='no_aug'
    model_name ='cgan'

    cell_and_magnification = '231_20X/'
    fluorescence = 'dapi'
    number = ['4', '8', '16', '32', '64', '128', '216']
    for num in number:
        print(os.path.join(save_dir, cell_and_magnification, f'few_images/{num}',
                                       fluorescence, model_name, 'pretrained'))
        for i in range(5):
            result_path = os.path.join(save_dir, cell_and_magnification, f'few_images/{num}',
                                       fluorescence, model_name, 'pretrained', f'result{i}')
            # if not os.path.exists(result_path):
            #     os.makedirs(result_path)

            compute_mertics(result_path, os.path.join(result_path,'test', 'gt'),
                            os.path.join(result_path,'test', 'pred'), 'd')


#!/bin/env python3

import sys
import os
import os.path
import re
import io

from typing import Tuple

LOW_REGISTERS = ["R0", "R1", "R2", "R3", "R4", "R5", "R6", "R7"]
HIGH_REGISTERS = ["R8", "R9", "R10", "R11", "R12"]
# TODO: Only allow high registers if we run out of low registers
ALL_REGISTERS = LOW_REGISTERS
RESERVED_REGISTERS = ALL_REGISTERS[0:1]

RE_PLONK_BEGIN = re.compile("^!\s+PLONK\(([0-9]+)\):")
RE_PLONK_NAMED = re.compile("\s+([a-zA-Z_][a-zA-Z_0-9]*)\s+=\s+([^,\s]+)(,|$)")
RE_PLONK_END = re.compile("^!!$")
RE_PLONK_INSERT = re.compile("^!\s+INSERT\s+(.+)$")

RE_ASM_COMMENT = re.compile("[;@].*$")

RE_ADDRESS_TARGET = re.compile("\[([^]]+)\]")
def match_address_target(value: str) -> Tuple[bool, str]:
	"""Matches [[^]]+]"""
	match = RE_ADDRESS_TARGET.match(value)

	if match is not None:
		return True, match.group(1)

	return False, None

RE_ADDRESS_CONSTANT = re.compile("=([a-zA-Z_0-9]+)")
def match_address_constant(value: str) -> Tuple[bool, str]:
	"""Matches =ADDR"""
	match = RE_ADDRESS_CONSTANT.match(value)

	if match is not None:
		return True, match.group(1)

	return False, None

RE_NAMED_REGISTER = re.compile("\$([a-zA-Z_][a-zA-Z_0-9]+)")
def match_named_register(value: str) -> Tuple[bool, str]:
	"""Matches $[a-zA-Z_][a-zA-Z_0-9]+"""
	match = RE_NAMED_REGISTER.match(value)

	if match is not None:
		return True, match.group(1)

	return False, None

RE_ARGUMENT_REGISTER = re.compile("\$A([0-9]+)")
def match_argument_register(value: str) -> Tuple[bool, int]:
	"""Matches $A[0-9]+"""
	match = RE_ARGUMENT_REGISTER.match(value)

	if match is not None:
		return True, int(match.group(1))

	return False, None

RE_VARIABLE_REGISTER = re.compile("\$([0-9]+)")
def match_variable_register(value: str) -> Tuple[bool, int]:
	"""Matches $[0-9]+"""
	match = RE_VARIABLE_REGISTER.match(value)

	if match is not None:
		return True, int(match.group(1))

	return False, None

RE_RAW_REGISTER = re.compile("R([0-9]|1[0-2])")
def match_raw_register(value: str) -> Tuple[bool, str]:
	"""Matches R[0-9]|1[0-2]"""
	match = RE_RAW_REGISTER.match(value)

	if match is not None:
		return True, match.group(0)

	return False, None

RE_DEC_NUMBER_LITERAL = re.compile("([0-9_]+)")
RE_HEX_NUMBER_LITERAL = re.compile("(0x[a-fA-F0-9_]+)")
RE_BINARY_NUMBER_LITERAL = re.compile("(0b[0-1_]+)")
def match_number(value: str) -> Tuple[bool, int]:
	match = RE_BINARY_NUMBER_LITERAL.match(value)
	if match is not None:
		number = int(match.group(1), 2)

		return True, number

	match = RE_HEX_NUMBER_LITERAL.match(value)
	if match is not None:
		number = int(match.group(1), 16)

		return True, number

	match = RE_DEC_NUMBER_LITERAL.match(value)
	if match is not None:
		number = int(match.group(1), 10)

		return True, number

	return False, None

RE_STRING_ADDRESS = re.compile("""^("([^"]+)"|'([^']+)')""")
def match_string(value: str) -> Tuple[bool, str]:
	match = RE_STRING_ADDRESS.match(value)
	if match is not None:
		return True, match.group(2)

	return False, None

def code_line(value: str) -> str:
	return f"\t{value}\n"

class PlonkVariable:
	"""Represents a variable with optional initialization code and an assigned register."""
	name: str
	register: str
	code: str

	needs_free: bool

	def __init__(self, name, register, code = None, needs_free = False):
		self.name = name
		self.register = register
		self.code = code

		self.needs_free = needs_free

	def __str__(self):
		return f"{self.name}@{self.register}"

	def __repr__(self):
		return f"{self.__str__()} at 0x{id(self):X}"

class PlonkRegisterContext:
	"""Represents registers available throughout one PLONK context."""

	# Registers with assigned names
	named_registers: dict
	# Registers used for arguments
	argument_registers: list
	# Registers free to be allocated
	variable_registers: dict

	def __init__(
		self,
		arguments = 0,
		named_registers = {},
		available_registers = ALL_REGISTERS
	):
		self.named_registers = {}
		self.argument_registers = []
		self.variable_registers = {}

		registers = available_registers.copy()

		for x in range(arguments):
			self.argument_registers.append(
				PlonkVariable(
					f"$A{x}",
					registers.pop(0)
				)
			)

		for key in named_registers:
			reg = registers.pop()
			val = named_registers[key]

			code = ""
			if type(val) is str:
				code += code_line(f"LDR {reg}, ={val}")
			elif type(val) is int:
				code += code_line(f"LDR {reg}, =0x{val:X}")
			else:
				raise RuntimeError(f"Invalid constant value type {val}")

			self.named_registers[key] = PlonkVariable(
				key,
				reg,
				code
			)

		for register in registers:
			if register not in RESERVED_REGISTERS:
				self.variable_registers[register] = { "free": True }

	def __str__(self):
		return f"PlonkRegisterContext(named = {self.named_registers}, args = {self.argument_registers}, variables = {self.variable_registers})"

	def generate_initialization_code(self):
		code = ""
		for key in self.named_registers:
			code += self.named_registers[key].code

		return code

	def get_named(self, name: str) -> PlonkVariable:
		if not name in self.named_registers:
			return None

		return self.named_registers[name]

	def get_argument(self, index: int) -> PlonkVariable:
		if len(self.argument_registers) <= index:
			return None

		return self.argument_registers[index]

	def allocate_variable(self) -> PlonkVariable:
		for register in self.variable_registers:
			if self.variable_registers[register]["free"]:
				self.variable_registers[register]["free"] = False

				return PlonkVariable(None, register, None, needs_free = True)

		raise RuntimeError(f"Could not allocate register: {self}")

	def check_register(self, register: str):
		if register in RESERVED_REGISTERS:
			return True

		if not register in self.variable_registers:
			return False

		return self.variable_registers[register]["free"]

	def free_variable(self, variable: PlonkVariable):
		if not variable.register in self.variable_registers:
			raise RuntimeError(f"Attempted to free non-existent register {variable}")

		if self.variable_registers[variable.register]["free"]:
			raise RuntimeError(f"Attempted to free already free register {variable}")

		self.variable_registers[variable.register]["free"] = True

class PlonkRegisterStack:
	"""Represents a stack of PlonkVariables that can be pushed and poped with freeing when necessarry."""
	stack: list

	def __init__(self):
		self.stack = []

	def __str__(self):
		return f"{self.stack}"

	def push(self, variable: PlonkVariable):
		self.stack.append(variable)

	def get(self, index: int) -> PlonkVariable:
		if index < len(self.stack):
			return self.stack[index]
		return None

	def get_named(self, name: str) -> PlonkVariable:
		for var in self.stack:
			if var.name == name:
				return var

		return None

	def pop(self, context: PlonkRegisterContext) -> PlonkVariable:
		variable = self.stack.pop()

		if variable.needs_free:
			context.free_variable(variable)

		return variable

class PlonkFileState:
	"""Represents the Plonk state throughout a source file."""

	plonking: bool = False
	context: PlonkRegisterContext = None

	# variables addressable with $[0-9]+ syntax
	scoped_variables: PlonkRegisterStack
	# variables allocated but not addresable
	scoped_internal: PlonkRegisterStack

	def __init__(self):
		self.scoped_variables = PlonkRegisterStack()
		self.scoped_internal = PlonkRegisterStack()

	def process_line(self, line: str):
		# check for plonk beginning
		begin_match = RE_PLONK_BEGIN.match(line)
		if begin_match is not None:
			if self.plonking:
				raise RuntimeError("Can only plonk in one dimension at once")

			argc = int(begin_match.group(1))
			named_registers = {}
			for named_match in RE_PLONK_NAMED.finditer(
				line,
				begin_match.span()[1]
			):
				key = named_match.group(1)
				value = named_match.group(2)

				matches, address = match_address_constant(value)
				if not matches:
					matches, number = match_number(value)
					if not matches:
						raise RuntimeError(f"Could not parse {value} as number nor address constant")
					else:
						value = number
				else:
					value = address

				named_registers[key] = value

			self.plonking = True
			self.context = PlonkRegisterContext(
				arguments = argc,
				named_registers = named_registers
			)

			return self.context.generate_initialization_code()

		# check for plonk insert
		insert_match = RE_PLONK_INSERT.match(line)
		if insert_match is not None:
			path = insert_match.group(1)
			print(f"Inserting \"{path}\"", file = sys.stderr)

			outstream = io.StringIO("")
			with open(path, "r") as file:
				process_file(file, outstream)

			return outstream.getvalue()

		# check for plonk ending
		end_match = RE_PLONK_END.match(line)
		if end_match is not None:
			if not self.plonking:
				raise RuntimeError("Cannot plonk in negative")

			self.plonking = False
			self.context = None

			return None

		# process the line specially when plonking
		if self.plonking:
			return self.process_line_plonking(line)

		# if not plonking then just return the current line
		return line

	def process_line_plonking(self, line: str):
		# strip comments and whitespace
		stripped = RE_ASM_COMMENT.sub("", line).strip()
		# ignore empty lines
		if stripped == '':
			return line

		first_split = stripped.split(" ", maxsplit = 1)
		word = first_split[0].lower()
		rest = None
		if len(first_split) > 1:
			rest = first_split[1]

		# self-contained
		if word == "store":
			return self.process_store(rest)

		if word == "call":
			return self.process_call(rest)
		elif word == "calln":
			return self.process_call(rest, nested = True)

		if word == "ifeq":
			return self.process_if(rest)
		elif word == "ifneq":
			return self.process_if(rest, branch = "BNE")
		elif word == "ifgt":
			return self.process_if(rest, branch = "BGT")
		elif word == "iftestone":
			return self.process_if(rest, test = "TST", branch = "BNE")
		elif word == "iftestzero":
			return self.process_if(rest, test = "TST", branch = "BEQ")

		# scoping
		if word == "read":
			return self.process_read(rest)
		elif word == "write":
			return self.process_write()
		elif word == "discard":
			return self.process_discard()

		if word == "alloc":
			return self.process_alloc(rest)
		elif word == "free":
			return self.process_free(rest)

		# other

		return self.process_other(line)

	def end(self):
		return None

	## MATCHERS ##

	def match_and_scope(self, value: str, *matchers) -> PlonkVariable:
		"""Returns the first match and scopes it into `self.scoped_internal`."""
		for matcher in matchers:
			match = matcher(value)
			if match is not None:
				self.scoped_internal.push(match)
				return match

		return None

	def match_and_code(self, value: str, target: str, *matchers) -> str:
		"""Returns the first match and creates a code line moving the value of the match into the target register"""
		for matcher in matchers:
			match = matcher(value)
			if match is not None:
				if match.needs_free:
					self.context.free_variable(match)

				if match.code is not None:
					return match.code.replace(match.register, target, 1)
				else:
					return code_line(f"MOV {target}, {match.register}")

		return None

	def match_address_constant(self, value: str) -> PlonkVariable:
		matches, address = match_address_constant(value)
		if matches:
			var = self.context.allocate_variable()
			return PlonkVariable(
				f"={address}",
				var.register,
				code_line(f"LDR {var.register}, ={address}"),
				needs_free = True
			)

		return None

	def match_argument_register(self, value: str) -> PlonkVariable:
		matches, index = match_argument_register(value)
		if matches:
			arg = self.context.get_argument(index)
			if arg is None:
				raise RuntimeError(f"Argument $A{index} does not exist")
			return PlonkVariable(f"$A{index}", arg.register)

		return None

	def match_variable_register(self, value: str) -> PlonkVariable:
		matches, index = match_variable_register(value)
		if matches:
			var = self.scoped_variables.get(index)
			if var is None:
				raise RuntimeError(f"Variable ${index} does not exist: {self.scoped_variables}")
			return PlonkVariable(f"${index}", var.register)

		return None

	def match_named_register(self, value: str) -> PlonkVariable:
		matches, name = match_named_register(value)
		if matches:
			named = self.scoped_variables.get_named(name)
			if named is None:
				named = self.context.get_named(name)
				if named is None:
					raise RuntimeError(f"Named register ${name} does not exist")
			return PlonkVariable(name, named.register)

		return None

	def match_raw_register(self, value: str) -> PlonkVariable:
		matches, register = match_raw_register(value)
		if matches:
			return PlonkVariable(register, register)

		return None

	def match_number(self, value: str) -> PlonkVariable:
		matches, number = match_number(value)
		if matches:
			var = self.context.allocate_variable()
			return PlonkVariable(
				f"#0x{number:X}",
				var.register,
				code_line(f"LDR {var.register}, =0x{number:X}"),
				needs_free = True
			)

		return None

	## SELF-CONTAINED ##

	def process_store(self, rest: str):
		"""
		store [=ADDR|$N|$AN|$NAMED|RAW] NUM|$N|$AN|$NAMED|RAW

		store [=ADDR] _ ->
			LDR RI, =ADDR
			STR _ ,[RI]

		store [$N|$AN|$NAMED|RAW] _ ->
			STR _, [$N|$AN|$NAMED|RAW]

		store _ NUM ->
			MOV RI, #NUM
			STR RI, _

		store _ $N|$AN|$NAMED|RAW ->
			STR $N|$AN|$NAMED|RAW, _
		"""
		split = rest.split(" ")
		target_argument_matches, target_argument = match_address_target(split[0])
		if not target_argument_matches:
			raise RuntimeError(f"First argument of store must match pattern `[[^]]+]`: {split[0]}")
		source_argument = split[1]

		code = ""

		target = self.match_and_scope(
			target_argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if target.code is not None:
			code += target.code

		source = self.match_and_scope(
			source_argument,
			self.match_number,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if source.code is not None:
			code += source.code

		code += code_line(f"STR {source.register}, [{target.register}]")

		# pop both source and target
		self.scoped_internal.pop(self.context)
		self.scoped_internal.pop(self.context)

		return code

	def process_call(self, rest: str, nested = False):
		"""
		call|calln LABEL =ADDR|NUMBER|$N|$AN|$NAMED|RAW ...

		call _ =ADDR ->
			LDR R0, =ADDR
			BL _

		call _ NUMBER ->
			MOV R0, #NUMBER
			BL _

		call _ $N|$AN|$NAMED ->
			MOV R0, $N|$AN|$NAMED|RAW
			BL _

		calln _ _ ->
			_
			PUSH { LR }
			BL _
			POP { LR }
		"""
		split = rest.split(" ")
		label = split[0]
		arguments = split[1:]

		overlapping_registers = []
		if len(arguments) > 0:
			# overlapping with the current context arguments
			overlapping_registers = ALL_REGISTERS[
				:min(len(arguments), len(self.context.argument_registers))
			]

			# overlapping with the allocated ones or the constant ones
			remaining_registers = ALL_REGISTERS[len(overlapping_registers) : len(arguments)]
			for reg in remaining_registers:
				if not self.context.check_register(reg):
					overlapping_registers.append(reg)

		code = ""

		if len(overlapping_registers) > 0:
			regs = ", ".join(overlapping_registers)
			code += code_line(f"PUSH {{ {regs} }}")

		for x in range(len(arguments)):
			code += self.match_and_code(
				arguments[x],
				f"R{x}",
				self.match_number,
				self.match_address_constant,
				self.match_argument_register,
				self.match_variable_register,
				self.match_named_register,
				self.match_raw_register
			)

		if nested:
			code += code_line("PUSH { LR }")
		code += code_line(f"BL {label}")
		if nested:
			code += code_line("POP { LR }")

		if len(overlapping_registers) > 0:
			regs = ", ".join(reversed(overlapping_registers))
			code += code_line(f"POP {{ {regs} }}")

		return code

	def process_if(self, rest: str, test = "CMP", branch = "BEQ"):
		"""
		ifeq|ifneq|ifgt|iftestone|iftestzero =ADDR|$N|$AN|$NAMED|RAW =ADDR|NUM|$N|$AN|$NAMED|RAW LABEL

		ifeq =ADDR _ _ ->
			LDR RI, =ADDR
			CMP RI, _
			BEQ _

		ifeq $N|$AN|$NAMED|RAW _ _ ->
			CMP $N|$AN|$NAMED|RAW _
			BEQ _

		ifeq _ =ADDR _ ->
			LDR RI, =ADDR
			CMP _, RI
			BEQ _

		ifeq _ NUM _ ->
			CMP _, #NUM
			BEQ _

		ifeq _ $N|$AN|$NAMED|RAW _ ->
			CMP _, $N|$AN|$NAMED|RAW
			BEQ _

		ifeq _, _, LABEL ->
			CMP _, _
			BEQ LABEL

		ifneq _, _, LABEL ->
			CMP _, _
			BNE LABEL

		ifgt _, _, LABEL ->
			CMP _, _
			BGT LABEL

		iftestone _, _, LABEL ->
			TST _, _
			BNZ LABEL

		iftestzero _, _, LABEL ->
			TST _, _
			BEQ LABEL
		"""
		split = rest.split(" ")
		left_argument = split[0]
		right_argument = split[1]
		label = split[2]

		code = ""

		left = self.match_and_scope(
			left_argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if left.code is not None:
			code += left.code

		right_variable = self.match_and_scope(
			right_argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if right_variable is not None:
			if right_variable.code is not None:
				code += right_variable.code

			code += code_line(f"{test} {left.register}, {right_variable.register}")

			self.scoped_internal.pop(self.context)
		else:
			matches, number = match_number(right_argument)
			if not matches:
				raise RuntimeError(f"Argument {right_argument} does not match any known right side")

			code += code_line(f"{test} {left.register}, #0x{number:X}")

		code += code_line(f"{branch} {label}")

		# pop left
		self.scoped_internal.pop(self.context)

		return code

	## SCOPED ##

	def process_alloc(self, rest: str):
		"""
		alloc NAME =ADDR|NUM|$N|$AN|$NAMED|RAW

		Allocates a variable addressable with `$[0-9]+` and with an optional name syntax.
		"""
		name = None
		value_argument = None
		if rest is not None:
			split = rest.split(" ")
			if RE_NAMED_REGISTER.match(f"${split[0]}") is not None:
				name = split[0]

			if len(split) > 1:
				value_argument = split[1]

		var = self.context.allocate_variable()
		var.name = name
		self.scoped_variables.push(var)

		code = None
		if value_argument is not None:
			code = self.match_and_code(
				value_argument,
				var.register,
				self.match_address_constant,
				self.match_number,
				self.match_argument_register,
				self.match_variable_register,
				self.match_named_register,
				self.match_raw_register
			)

		return code

	def process_free(self, rest: str):
		"""
		free N

		Frees the last `N` addressable variables.
		"""
		count = int(rest or 1)

		for x in range(count):
			self.scoped_variables.pop(self.context)

	def process_read(self, rest: str):
		"""
		Allocates a new addressable variable and loads the value of `[=ADDR|$N|$AN|$NAMED|RAW]` into it.

		read [=ADDR|$N|$AN|$NAMED|RAW] NAME

		read [=ADDR] ->
			LDR RI, =ADDR
			LDR RV, [RI]

		read [$N|$AN|$NAMED|RAW]
			LDR RV, [$N|$AN|$NAMED|RAW]
		"""
		split = rest.split(" ")
		argument_matches, argument = match_address_target(split[0])
		if not argument_matches:
			raise RuntimeError(f"Argument of read must match pattern `[[^]]+]`: {rest}")

		name = None
		if len(split) > 1 and RE_NAMED_REGISTER.match(f"${split[1]}") is not None:
			name = split[1]

		code = ""

		address = self.match_and_scope(
			argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if address.code is not None:
			code += address.code

		variable = self.context.allocate_variable()
		variable.name = name
		self.scoped_variables.push(variable)

		code += code_line(f"LDR {variable.register}, [{address.register}]")

		return code

	def process_write(self):
		"""
		Frees the last allocated addressable variabled and writes its value into the address specified by previous `read`.

		write

		write -> ; with previous `read =ADDR`
			STR RV, [RI]

		write -> ; with previous `read $N|$AN|$NAMED|RAW`
			STR RV, [$N|$AN|$NAMED|RAW]
		"""
		reg = self.scoped_variables.pop(self.context)
		addr = self.scoped_internal.pop(self.context)

		return code_line(f"STR {reg.register}, [{addr.register}]")

	def process_discard(self):
		"""
		Frees the last allocated addressable variable and discards its value. This can be used to end a `read` block without emitting a store.

		discard
		"""
		reg = self.scoped_variables.pop(self.context)
		addr = self.scoped_internal.pop(self.context)

		return None

	## OTHER ##

	def process_other(self, line: str):
		def transform_with(line: str, regex, callback):
			transformed_line = ""
			last_pos = 0

			for match in regex.finditer(line):
				span = match.span()

				transformed_line += line[last_pos:span[0]]
				transformed_line += callback(match)

				last_pos = span[1]

			transformed_line += line[last_pos:]

			return transformed_line

		line = transform_with(
			line,
			RE_BINARY_NUMBER_LITERAL,
			lambda match: f"0x{int(match.group(1), 2):X}"
		)

		# $0 -> RN
		line = transform_with(
			line,
			RE_VARIABLE_REGISTER,
			lambda match: self.scoped_variables.get(int(match.group(1))).register
		)

		# $A0 -> RN
		line = transform_with(
			line,
			RE_ARGUMENT_REGISTER,
			lambda match: self.context.argument_registers[int(match.group(1))].register
		)

		# $NAMED -> RN
		line = transform_with(
			line,
			RE_NAMED_REGISTER,
			lambda match: (
				self.scoped_variables.get_named(match.group(1))
				or
				self.context.get_named(match.group(1))
			).register
		)

		return line

def process_file(stream, outstream):
	context = PlonkFileState()
	line_number = 0

	for line in stream:
		line_number += 1
		try:
			output = context.process_line(line)
		except BaseException as e:
			raise RuntimeError(
				f""" Cannot process line #{line_number}
	{line.rstrip()}

	Reason: {e}
				"""
			)
		if output is not None:
			print(output, file = outstream, end = "")

	final_output = context.end()
	if final_output is not None:
		print(final_output, file = outstream, end = "")

def process_folder(folder):
	files = os.listdir(folder)

	for file in files:
		if os.path.isfile(file):
			base, ext = os.path.splitext(file)
			if ext in [".plonk"]:
				out_path = os.path.join(
					os.path.dirname(folder),
					base + ".s"
				)

				print(f"## Processing {file} > {out_path}")

				with open(file, "r") as in_file:
					with open(out_path, "w") as out_file:
						process_file(in_file, out_file)

def main():
	if len(sys.argv) > 1:
		path = sys.argv[1]
		if os.path.isdir(path):
			process_folder(path)
		else:
			with open(path, "r") as file:
				process_file(file, sys.stdout)
	else:
		process_file(sys.stdin, sys.stdout)

main()


"""
Tests for identifying XSS vulnerabilities.
This is currently done when the "q" method is called.
"""
from __future__ import absolute_import

import os
from mock import patch
import pytest

from bok_choy.page_object import XSSExposureError
from bok_choy.web_app_test import WebAppTest
from .pages import SitePage

class XSSExposureTest(WebAppTest):
    """
    Tests for identifying XSS vulnerabilities.
    """
    def __init__(self, *args, **kwargs):
        super(XSSExposureTest, self).__init__(*args, **kwargs)
        self.site_page = None

    def _visit_page(self, page_name):
        """
        Instantiate a ``SitePage`` for the given base filename and visit it.
        """
        self.site_page = SitePage(self.browser)
        self.site_page.name = page_name
        self.site_page.visit()

    @patch.dict(os.environ, {'VERIFY_XSS': 'True'})
    def test_html_exposure(self):
        self._visit_page("xss_html")
        with pytest.raises(XSSExposureError) as excinfo:
            self.site_page.q(css='.unescaped')
        assert '2 XSS issue' in str(excinfo)

    @patch.dict(os.environ, {'VERIFY_XSS': 'True'})
    def test_js_exposure(self):
        self._visit_page("xss_js")
        with pytest.raises(XSSExposureError) as excinfo:
            self.site_page.q(css='.unescaped')
        assert '1 XSS issue' in str(excinfo)

    @patch.dict(os.environ, {'VERIFY_XSS': 'True'})
    def test_mixed_exposure(self):
        self._visit_page("xss_mixed")
        with pytest.raises(XSSExposureError) as excinfo:
            self.site_page.q(css='.unescaped')
        assert '2 XSS issue' in str(excinfo)

    @patch.dict(os.environ, {'VERIFY_XSS': 'True'})
    def test_escaped(self):
        self._visit_page("xss_safe")
        self.site_page.q(css='.escaped')

    @patch.dict(os.environ, {'VERIFY_XSS': 'False'})
    def test_xss_testing_disabled_explicitly(self):
        self._visit_page("xss_html")
        self.site_page.q(css='.unescaped')

    def test_xss_testing_disabled_by_default(self):
        self._visit_page("xss_html")
        self.site_page.q(css='.unescaped')


# coding: utf-8

# In[1]:

import pandas as pd

# In[2]:

fi="tabula-qcuxk3y7c1ezwo5yylnn.csv"
df = pd.read_csv(fi)

# In[3]:

df.head()

# In[4]:

df.describe()

# In[5]:

df[20:30]

# In[6]:

df.dtypes

# In[7]:

# remove header row (is there because frame is conctenation of each team's table)
df = df[df.Team != 'Team']
df[10:30]

# In[8]:

df.describe()

# In[9]:

# the tallest player :
df['Height'].max()

# In[10]:

# make object-> numeric so can run aggregations on top of it
df.apply(pd.to_numeric, errors='ignore')

# In[11]:

df.dtypes

# In[12]:

df['Height'] = df['Height'].astype('int64')

# In[13]:

df = df.reset_index(drop=True)

# In[14]:

df.head()

# In[15]:

# find index of the tallest player
ind = df['Height'].idxmax()
ind

# In[16]:

pd.__version__

# In[17]:

# and the tallest player is:
df.iloc[149]

# In[18]:

df[149:150]

# In[19]:

df.dtypes

# In[20]:

# and the shortest player is ...
df.iloc[df['Height'].idxmin()]

# In[21]:

# average hight by team
df.groupby('Team', as_index=False)['Height'].mean().sort_values('Height')

# In[22]:

# shortest teams are Peru, Saudi Arabia, Argentina
# tallest Serbia, Denmark, Germany

# In[23]:

# same but add a column for median and the variance (standard deviation)
df.groupby('Team', as_index=False).agg({'Height':['mean','median','std']}).sort_values([("Height","mean")])

# In[24]:

import datetime

# In[25]:

def func(x):
    return datetime.datetime.strptime(x, '%d.%m.%Y').date()

df['birth date corrected']=df['Birth Date'].apply(lambda x: func(x))

# In[26]:

df.head()

# In[27]:

from datetime import date
def calculate_age(born):
    today = date.today()
    return today.year - born.year - ((today.month, today.day) < (born.month, born.day))

df['age']=df['birth date corrected'].apply(lambda x: calculate_age(x))

# In[28]:

df.head()

# In[29]:

def calc_age_as_fraction(birth_date):
    days_in_year = 365.2425
    return (date.today() - birth_date).days / days_in_year
df['age_fraction']=df['birth date corrected'].apply(lambda x: calc_age_as_fraction(x))

# In[30]:

df.head()

# In[31]:

# average age by team with median and variance (standard deviation)
df_age =df.groupby('Team', as_index=False).agg({'age_fraction':['mean','median','std']}).sort_values([("age_fraction","mean")])

# In[32]:

df_age

# In[33]:

import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')

# In[34]:

df_age.unstack(['age_fraction']).head()

# In[35]:

df['Team'] = df['Team'].astype('|S')

# In[36]:

ax = df_age['age_fraction']['mean'].plot(x=df_age['Team'],kind="bar",color=['blue'])
# ax.set_xticklabels(df_age['Team'], rotation=90)

# In[37]:

# ax = df_age['age_fraction']['mean'].plot(x=df_age['Team'],kind="bar",color=['blue'])
# ax.set_xticklabels(df_age['Team'], rotation=90)
fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
width=0.75
# ax.bar(list(df_age['Team']),sorted(list(df_age['age_fraction']['mean'].sort_values())), width , color='b', yerr=list(df_age['age_fraction']['std']))
ax = df_age['age_fraction']['mean'].plot(x=df_age['Team'],kind="bar",color=['blue'],yerr=df_age['age_fraction']['std'])

ax.set_xticklabels(list(df_age['Team']), rotation=90)
ax.set_title("National team age")
plt.show()

# In[38]:

df_height =df.groupby('Team', as_index=False).agg({'Height':['mean','median','std']}).sort_values([("Height","mean")])
df_height

# In[39]:

fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
width=0.75
ax = df_height['Height']['mean'].plot(x=df_height['Team'],kind="bar",color=['blue'],yerr=df_height['Height']['std'])
ax.set_xticklabels(list(df_height['Team']), rotation=90)
ax.set_title("National team height")
plt.show()

# In[40]:

#calc bmi - body mass index
# bmi = mass/height^2
ctn=0
def calc_bmi(x):
#     global ctn

#     ctn+=1
#     bmi=int(x['Weight'])/int(x['Height'])**2
#     print(ctn,bmi)
    mass = x['Weight']
    height = x['Height']
    bmi = int(mass) / int((height/100)**2)
#     print(ctn,mass,height,bmi)

    return bmi

df['BMI']=df.apply(lambda x: calc_bmi(x),axis=1)
# d.apply(lambda row: min([row['A'], row['B']])-row['C'], axis=1)

# In[41]:

df

# In[42]:

df.describe()

# In[43]:

df[df['age']==45.0]

# In[44]:

df[df['age']==19.0]

# In[45]:

df[df['BMI']==20.0]

# In[46]:

# club origin
def get_club_origin(x):
    return str(x['Club']).split("(")[1].split(")")[0]

#test
x0=dict()
x0["Club"] = "Manchester United FC (ENG)"
print(get_club_origin(x0))

# In[47]:

df['Club origin']=df.apply(lambda x:get_club_origin(x),axis=1)

# In[48]:

df.head()

# In[49]:

# get the 3letter code of team
import pycountry
print(pycountry.countries.get(name="England"))

# In[50]:

# remove b from team
df['Team'] = df['Team'].str.decode('utf-8')

# In[51]:

def get_3_letter_code(x):
    # special cases not handled by pycountry (why not??)
    if (str(x['Team'])=='England'):
        return "ENG"
    if (str(x['Team'])=="IR Iran"):
        return "IRN"
    if (str(x['Team'])=="Korea Republic"):
        return "KOR"
    if (str(x['Team'])=="Russia"):
        return "RUS"
    return pycountry.countries.get(name=str(x["Team"])).alpha_3

df['Country_3']=df.apply(lambda x:get_3_letter_code(x),axis=1)

# In[52]:

for p in pycountry.countries:
    print(p)

# In[53]:

df[df['Team']=="Russia"]

# In[54]:

def plays_in_home_country(x):
    return (x['Club origin']==x['Country_3'])
df['Plays_in_home_country']=df.apply(lambda x:plays_in_home_country(x),axis=1)

# In[55]:

df_same_country =df[df['Plays_in_home_country']==True].groupby('Team',as_index=False).agg({'Plays_in_home_country':'count'})
# df.groupby('Team', as_index=False).agg({'Height':['mean','median','std']}).sort_values([("Height","mean")])

# In[56]:

df_same_country

# In[57]:

df[df['Plays_in_home_country']==True]

# In[122]:

df_clubs_origin = df.groupby("Club origin",as_index=False).agg({"Plays_in_home_country":"count"})
df_clubs_origin

# In[127]:

df_clubs_origin_sorted =df_clubs_origin.sort_values(['Plays_in_home_country'],ascending=False)
df_clubs_origin_sorted.head(10)

# In[136]:

df_clubs_origin_sorted['Club origin'][0:10]

# In[133]:

ax = df_clubs_origin_sorted['Plays_in_home_country'][0:10].plot(x=df_clubs_origin_sorted['Club origin'][0:10],kind="bar",color=['blue'])

# In[74]:

df_clubs = df.groupby("Club",as_index=True).agg({'Club':['count']}).sort_values([("Club","count")],ascending=False)
# df.groupby('Team', as_index=False).agg({'Height':['mean','median','std']}).sort_values([("Height","mean")])
df_clubs

# In[75]:

df['Team'].unique()

# In[76]:

round_16_teams=['Argentina', 'Belgium', 'Brazil', 'Colombia',
       'Croatia', 'Denmark', 'England', 'France',
       'Japan',
       'Mexico',
       'Portugal', 'Russia', 'Spain',
       'Sweden', 'Switzerland', 'Uruguay']
len(round_16_teams)

# In[79]:

quarter_finals_temas = ['Belgium', 'Brazil',
       'Croatia',  'England', 'France',
        'Russia',
       'Sweden','Uruguay']
len(quarter_finals_temas)

# In[84]:

df_only_R16 = df[df['Team'].isin( round_16_teams)]
df_only_R16['Team'].unique()

# In[85]:

df_only_QF = df[df['Team'].isin(quarter_finals_temas)]
df_only_QF['Team'].unique()

# In[116]:

def visualise_top_k_represented_clubs(df, k):
    df_grouped = df.groupby("Club",as_index=True).agg({'Club':['count']}).sort_values([("Club","count")],ascending=False)
    df_grouped_top_k=df_grouped[0:k]
    ax = df_grouped_top_k['Club']['count'].plot(x=df_grouped_top_k.index,kind="bar",color=['blue'])

    # ax.set_xticklabels(df_age['Team'], rotation=90)
    return ax

# In[104]:

visualise_top_k_represented_clubs(df,10)

# In[96]:

visualise_top_k_represented_clubs(df_only_R16,10)

# In[97]:

visualise_top_k_represented_clubs(df_only_QF,10)

# In[120]:

fig, ax = plt.subplots()
fig.set_figheight(5)
fig.set_figwidth(15)

plt.subplot(1,3,1)
plt.ylim((0,17))
ax = visualise_top_k_represented_clubs(df,15)
ax = plt.gca()

plt.subplot(1,3,2)
plt.ylim((0,17))
ax2 = visualise_top_k_represented_clubs(df_only_R16,15)
ax2 = plt.gca()

plt.subplot(1,3,3)
plt.ylim((0,17))
ax3 = visualise_top_k_represented_clubs(df_only_QF,15)
ax3 = plt.gca()

# In[143]:

"""
==================
Animated histogram
==================

This example shows how to use a path patch to draw a bunch of
rectangles for an animated histogram.

"""
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.patches as patches
import matplotlib.path as path
import matplotlib.animation as animation
from IPython.display import HTML
fig, ax = plt.subplots()

# histogram our data with numpy
data = np.random.randn(1000)
n, bins = np.histogram(data, 100)

# get the corners of the rectangles for the histogram
left = np.array(bins[:-1])
right = np.array(bins[1:])
bottom = np.zeros(len(left))
top = bottom + n
nrects = len(left)

# here comes the tricky part -- we have to set up the vertex and path
# codes arrays using moveto, lineto and closepoly

# for each rect: 1 for the MOVETO, 3 for the LINETO, 1 for the
# CLOSEPOLY; the vert for the closepoly is ignored but we still need
# it to keep the codes aligned with the vertices
nverts = nrects*(1 + 3 + 1)
verts = np.zeros((nverts, 2))
codes = np.ones(nverts, int) * path.Path.LINETO
codes[0::5] = path.Path.MOVETO
codes[4::5] = path.Path.CLOSEPOLY
verts[0::5, 0] = left
verts[0::5, 1] = bottom
verts[1::5, 0] = left
verts[1::5, 1] = top
verts[2::5, 0] = right
verts[2::5, 1] = top
verts[3::5, 0] = right
verts[3::5, 1] = bottom

barpath = path.Path(verts, codes)
patch = patches.PathPatch(
    barpath, facecolor='green', edgecolor='yellow', alpha=0.5)
ax.add_patch(patch)

ax.set_xlim(left[0], right[-1])
ax.set_ylim(bottom.min(), top.max())

def animate(i):
    # simulate new data coming in
    data = np.random.randn(1000)
    n, bins = np.histogram(data, 100)
    top = bottom + n
    verts[1::5, 1] = top
    verts[2::5, 1] = top
    return [patch, ]

ani = animation.FuncAnimation(fig, animate, 1000, repeat=False, blit=True)
plt.show()
# HTML(ani.to_html5_video())
ani.save('myvideo.mp4', codec='h264')