import networkx as nx
import pandas as pd
import numpy as np
import math
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
from cartopy.io.shapereader import Reader
from cartopy.feature import ShapelyFeature
import cartopy.crs as ccrs
import cartopy.io.img_tiles as cimgt

################################################################################################################
'''Function that creates a network with edge and node attributes for a given year'''

def make_year_network(year):
    # enter year in YYYY string format

    node_attr = pd.read_csv('../data/Attributes.csv')
    # print(water_rights.WDID.to_list())
    path = 'network_csv_files/priorityWdid/v2_network_'
    data = pd.read_csv(path + year + '.csv')

    G = nx.from_pandas_edgelist(df=data, source='priorityWdid', target='analysisWdid', edge_attr = 'sumWtdCount',
                                create_using=nx.DiGraph())
    # add latitude for each node
    lat_dict = node_attr[['wdid', 'latitude']].set_index('wdid')['latitude'].to_dict()
    nx.set_node_attributes(G, name = 'latitude', values = lat_dict)

    # add longitude for each node
    long_dict = node_attr[['wdid', 'longitude']].set_index('wdid')['longitude'].to_dict()
    nx.set_node_attributes(G, name = 'longitude', values = long_dict)

    # add net absolute for each node
    vol_dict = node_attr[['wdid', 'sum_netAbs']].set_index('wdid')['sum_netAbs'].to_dict()
    nx.set_node_attributes(G, name='netAbs', values=vol_dict)

    # add label for each node
    label_dict = node_attr[['wdid', 'structureName']].set_index('wdid')['structureName'].to_dict()
    nx.set_node_attributes(G, name='name', values=label_dict)

    return G

###################################################################################################################

'''Plot network graph'''

def plot_network_graph(G, year):
    # creates gridspec plot for a given network G
    fig = plt.figure(figsize=(18, 12))
    gspec = GridSpec(nrows=6, ncols=9)
    fig.suptitle(year, fontsize=24, fontweight='bold')
    plt.rcParams.update({'font.size': 12})

    # map extent
    extent = [-109.125, -105.625, 38.875, 40.50]
    # background map tiles
    tiles = cimgt.Stamen('terrain-background')
    # MosartWM gridcells
    xlist = np.linspace(-109.125, -105.625, 29)
    ylist = np.linspace(38.875, 40.50, 17)
    X, Y = np.meshgrid(xlist, ylist)
    Xgrid = np.arange(128, 156)
    Ygrid = np.arange(112, 128)
    Z = np.ones([len(Ygrid), len(Xgrid)])
    # Basin shape
    shape_feature = ShapelyFeature(Reader('../data/Water_Districts.shp').geometries(),
                                   ccrs.PlateCarree(), edgecolor='black', facecolor='None')
    # Stream lines
    flow_feature = ShapelyFeature(Reader('../data/UCRBstreams.shp').geometries(),
                                  ccrs.PlateCarree(), edgecolor='royalblue', facecolor='None', alpha=0.5, zorder=1)

    # 1. SUBPLOT OF NETWORK MAP
    # the first subplot will span 6 rows and 4 columns, top left
    crs = ccrs.PlateCarree()
    ax0 = fig.add_subplot(gspec[:, :], projection=crs)

    # Set map extent
    ax0.set_extent([-109.125, -105.625, 38.875, 40.50])
    # Draw background tiles
    ax0.add_image(tiles, 9)
    # Draw basin
    ax0.add_feature(shape_feature, facecolor='#a1a384', alpha=0.5)
    # Draw streams
    ax0.add_feature(flow_feature, alpha=0.7, linewidth=1.5, zorder=1)
    # Draw grid
    # ax0.pcolor(X, Y, Z, facecolor='none', edgecolor='grey', alpha=0.1, linewidth=0.5, transform=ccrs.PlateCarree())

    # Get latitude and longitude attributes for each node
    latitude = nx.get_node_attributes(G, 'latitude')
    longitude = nx.get_node_attributes(G, 'longitude')
    ds = [longitude, latitude]
    pos = {}
    for k in latitude.iterkeys():
        pos[k] = tuple(d[k] for d in ds)
    print(pos)

    Gcc = G

    # Color nodes by out degree
    degs = Gcc.out_degree() # Dict with Node ID, Degree
    nod = Gcc.nodes()
    color = np.asarray([degs[n] for n in nod])
    # print(sorted(color, reverse=True))

    # Size nodes by net absolute volume log-scaled and set size of nodes with out_degree = 0 to 0
    net_abs = nx.get_node_attributes(Gcc, 'netAbs')
    sizes = [np.log10(net_abs[i])*40 if net_abs[i] > 0 and Gcc.out_degree(i) > 0 else 0 for i in Gcc.nodes]

    # Size edge widths by sumWtdCount
    count = nx.get_edge_attributes(Gcc, 'sumWtdCount').values()
    print(count)
    count = [c/200 + 0.01 for c in count]

    # Add labels only for important users (nodes with out_degree > 20)

    labels = {i: name for i, name in list(nx.get_node_attributes(Gcc, 'name').items()) if degs[i] >= 1}

    # Add nodes with out_degree = 0 colored in white
    sizes2 = [0 if (Gcc.out_degree(node)) > 0 else np.log10(net_abs[node])*20 for node in nod]
    nx.draw_networkx_nodes(Gcc, ax = ax0, pos=pos,node_color='white',node_size=sizes2, zorder=40, alpha=0.6)

    # Draw network
    nx.draw_networkx(Gcc, ax = ax0, pos=pos, edge_color='black', alpha=0.9, arrows=True,
                     with_labels=True, node_color = color, cmap=plt.cm.plasma, vmin=1, vmax=2000,
                     node_size=sizes, labels=labels, width = count, font_size=14, font_weight='bold', zorder=50)
    sm = plt.cm.ScalarMappable(cmap=plt.cm.plasma, norm=plt.Normalize(vmin=1, vmax=2000))
    sm._A = []
    cb = plt.colorbar(sm, ax=ax0, orientation='horizontal')
    cb.set_label(label='Out Degree', size=16)

    plt.tight_layout(pad=5)
    plt.savefig( 'by_seniority/' + year  + '.png', dpi=300)

    return

###################################################################################################################

users = pd.read_csv('../data/CDSS_WaterRights.csv')
print(users.columns)

N = 200

# select top 200 users by decree (net absolute)
users_decree = users.sort_values('Net Absolute')
users_decree = users_decree.head(n=N)['WDID'].tolist()
print(users_decree[:10])

# select top 200 users by seniority
users_seniority = users.sort_values('Priority Admin No')
users_seniority = users_seniority.head(n=N)['WDID'].tolist()
print(users_seniority[:10])

years = [str(yr) for yr in range(2000,2019)]
print(years)

for year in years:
    print(year)
    G = make_year_network(year)
    Gcc = G.subgraph(users_seniority)
    plot_network_graph(Gcc, year)
    print(Gcc.number_of_nodes())


__author__ = 'wendy'

class Utils:

    @staticmethod
    def ranklist(l):
        ''' given l=[a, b, c, c, c, b]
        return counts and ordered list of tuples
        result = [(c, 3), (b, 2), (a, 1)]
        '''
        result = []
        for entry in set(l):
            result.append((entry, l.count(entry)))
        result.sort(key = lambda x: -x[1])
        return result


from collections import deque

bridge_length = 2
weight = 10
truck_weights = [7, 4, 5, 6]

res = 0

current_weigt = 0
bridge_weight = deque([0] * bridge_length)
trucks = deque(truck_weights)

while trucks or current_weigt > 0:
    removedTruck = bridge_weight.popleft()
    current_weigt -= removedTruck

    if trucks and current_weigt + trucks[0] <= weight:
        newTruck = trucks.popleft()
        current_weigt += newTruck

        bridge_weight.append(newTruck)

    else:
        bridge_weight.append(0)

    res += 1

print(res)

다리를 지나는 트럭(2).py


from utils import Input

CHAT_NAME = Input.input_chat('请输入要发送消息的人或群：')
MESSAGE = Input.input_message('请输入要发送的消息：')
WE_CHAT_PATH = r"C:\Program Files (x86)\Tencent\WeChat\WeChat.exe"  # 微信路径  C:\Program Files (x86)\Tencent\WeChat
WE_CHAT_ID = 'WeChat.exe'   # 微信程序
BACKEND_UIA = 'uia'   #
CLASS_NAME = 'WeChatMainWndForPC'
TYPE_KEYS = '^a'
SEND_KEYS = '{ENTER}'
CONTROL_TYPE_1 = 'Edit'
CONTROL_TYPE_2 = 'List'
TITLE_1 = '搜索'
TITLE_2 = '会话'
TITLE_3 = '输入'
TITLE_4 = '消息'


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name="home"),
    path('callback', views.callback, name="twitter_callback"),
    path('get_profile', views.get_profile, name="get_profile"),
    path('get_timeline', views.get_timeline, name="get_timeline"),
    path('get_timeline_from_db', views.get_timeline_from_db, name="get_timeline_from_db"),
    path('update_timeline_job', views.update_timeline_job, name="update_timeline_job"),
]


from bumps.names import *
import bumps.curve
from sasmodels.core import load_model
from sasmodels.bumps_model import Model, Experiment
from sasmodels.data import load_data, plot_data
from molgroups import mol
from molgroups import components as cmp
from molgroups import lipids

import numpy

# constants
dimension = 100
stepsize = 1.
z = numpy.linspace(0, 99, 100, endpoint=True)

def bilayer(z, bulknsld, l_lipid1, l_lipid2, sigma=3.0, vf_bilayer=1.0):
    # Scale all SLDs from Refl1D units (1e-6 Ang^-2) to molgroups units (Ang^-2)
    bulknsld *= 1e-6

    blm.fnSet(sigma=sigma, bulknsld=bulknsld, startz=20., l_lipid1=l_lipid1, l_lipid2=l_lipid2, vf_bilayer=vf_bilayer)

    # print(z)

    # Calculate scattering properties of volume occupied by bilayer
    normarea, area, nsl = blm.fnWriteProfile(z)

    # Fill in the remaining volume with buffer of appropriate nSLD
    nsld = nsl / (normarea * np.gradient(z)) + (1.0 - area / normarea) * bulknsld

    # export objects for post analysis, needs to be from this function
    problem.moldat = blm.fnWriteGroup2Dict({}, 'bilayer', np.arange(dimension) * stepsize)
    problem.results = blm.fnWriteResults2Dict({}, 'bilayer')

    # Return nSLD profile in Refl1D units
    return nsld*1e6

def Dummy(x, l_lipid=11., sigma=3.):
    #################################################
    # bilayer update
    # have to figure out how to do a dynamic update
    # that's the code
    sldarr0 = bilayer(z, float(model0.sld_solvent.value), l_lipid, l_lipid, sigma=sigma)
    #sldarr1 = bilayer(z, float(model1.sld_solvent.value), l_lipid, l_lipid, sigma=sigma)
    for i in range(pars0['n']):
        getattr(model0, 'sld'+str(i+1)).value = sldarr0[i]
        #getattr(model1, 'sld'+str(i+1)).value = sldarr1[i]
        getattr(model0, 'thickness'+str(i+1)).value = 1.
        #getattr(model1, 'thickness'+str(i+1)).value = 1.

    result = numpy.array([1., 2., 3.]) # + 0 * l_lipid * x
    return result

### Define bilayer object
palmitoyl = cmp.Component(name='palmitoyl', formula='C15 H31', cell_volume=913./2.0, length=11.0)
DPPC = cmp.Lipid(name='DPPC', headgroup=lipids.PC, tails=2 * [palmitoyl], methyls=[cmp.methyl])
blm = mol.BLM(lipids=[DPPC], lipid_nf=[1.0])

# IMPORT THE DATA USED
data0 = load_data('sim0.dat')
#data1 = load_data('sim1.dat')

qmin = 0.02
qmax = 0.45
setattr(data0, 'qmin', qmin)
setattr(data0, 'qmax', qmax)
#setattr(data1, 'qmin', 0.02)
#setattr(data1, 'qmax', 0.35)

# DEFINE THE MODEL
# copied and initialized the custom model with up to 100 sld and thickness parameters in sasmodels/models
# a second hard-coded limit in sasmodels/modelinfo.py line 594 was manually increased from 20 to 120.
# further in sasmodels/data.py replace all imports from sas.sascalc ... to from sasdata. ...
kernel = load_model('my_core_multi_shell@hardsphere')

pars0 = dict(scale=0.002, background=0.15, sld_core=3.0, sld_solvent=2.4, radius=500.0, radius_pd=0.3, n=100, radius_effective=60.0, volfraction=0.01)
#pars1 = dict(scale=0.002, background=0.15, sld_core=3.0, sld_solvent=3.4, radius=60.0, radius_pd=0.3, n=100, radius_effective=60.0, volfraction=0.01)

model0 = Model(kernel, **pars0)
#model1 = Model(kernel, **pars1)

# Bilayer specific-parameters are passed into a dummy function
xd = numpy.array([1., 2., 3.])
yd= numpy.array([1., 2., 3.])
dyd = numpy.array([0.01, 0.01, 0.01])
M0 = Curve(Dummy, xd, yd, dyd, l_lipid=2)
M0.l_lipid.range(10, 15)
M0.sigma.range(2., 3.5)

# PARAMETER RANGES (ONLY THOSE PARAMETERS ARE FITTED)
background0 = Parameter(name='background0', value=0.4).range(0.01, 0.2)
#background1 = Parameter(name='background1', value=0.4).range(0.01, 0.8)
sld_solvent0 = sld_core0 = Parameter(name='sld_solvent0', value=2.0).range(5.0, 6.4)
#sld_solvent1 = sld_core1 = Parameter(name='sld_solvent1', value=2.0).range(-0.56, 6.4)

model0.scale.range(0.0001, 0.5)
model0.background = background0
model0.sld_solvent = sld_solvent0
model0.sld_core = sld_core0
#model0.radius.range(40., 1200.)
model0.radius_pd.range(0.05, 0.5)
model0.radius_effective = model0.radius + 50.
model0.volfraction.range(0.01, 0.8)

#model1.scale = model0.scale
#model1.background = background1
#model1.sld_solvent = sld_solvent1
#model1.sld_core = sld_core1
#model1.radius = model0.radius
#model1.radius_pd=model0.radius_pd
#model1.radius_effective = model0.radius
#model1.volfraction = model0.scale

M1 = Experiment(data=data0, model=model0)
#M2 = Experiment(data=data1, model=model1)

problem = MultiFitProblem([M0, M1])


from base import *

NICK    = "test-2920"
MAX_AGE = 123456

CONF = """
vserver!2920!nick = %(NICK)s
vserver!2920!document_root = %(droot)s
vserver!2920!hsts = 1
vserver!2920!hsts!max_age = %(MAX_AGE)s
vserver!2920!rule!1!match = default
vserver!2920!rule!1!handler = dirlist
"""

class Test (TestBase):
    def __init__ (self):
        TestBase.__init__ (self, __file__)
        self.name             = "HSTS: Error code and Header"
        self.request          = "HTTP / HTTP/1.0\r\n" + \
                                "Host: %s\r\n" %(NICK)
        self.expected_error   = 301
        self.expected_content = ["Strict-Transport-Security:", "max-age=%d"%(MAX_AGE)]

    def Prepare (self, www):
        droot = self.Mkdir (www, "%s_droot"%(NICK))

        vars = globals()
        vars.update(locals())
        self.conf = CONF %(vars)


row1 = ["🟥", "🟥", "🟥"]
row2 = ["🟥", "🟥", "🟥"]
row3 = ["🟥", "🟥", "🟥"]

rows = [row1, row2, row3]
print(f"{row1}\n{row2}\n{row3}")

position = int(input("Enter the the position of the treasure(columnrow): "))
col = int(position / 10) - 1
row = int(position % 10) - 1
rows[row][col] = "🏆"
print(f"{row1}\n{row2}\n{row3}")


import pandas as pd
import matplotlib.pyplot as plt

df = pd.read_csv("train.csv")

M = df.as_matrix()
#get the first column from row 1 to the end#
im = M[0, 1:]
#reshape it to 28*28 2D matrix#
im = im.reshape(28, 28)

#show the image, set as gray means 0 is color black, and 255 is color white#
plt.imshow(255-im, cmap='gray')
plt.show()


################################################################################

# functionality: perform HOG/SVM testing over a specified dataset and compute the
# resulting prediction/clasification error over that same dataset, using
# pre-saved the SVM model trained on HOG feature descriptors

# This version: (c) 2018 Toby Breckon, Dept. Computer Science, Durham University, UK
# License: MIT License

# Minor portions: based on fork from https://github.com/nextgensparx/PyBOW

################################################################################

import numpy as np
import cv2
import params
import sys
import os
sys.path.append(os.path.abspath("./"))
import utils

################################################################################

def main():

    # load up the SVM stored from prior training

    try:
        svm = cv2.ml.SVM_load(params.HOG_SVM_PATH_SAVED)
    except:
        print("Missing files  SVM")
        print("-- have you performed training to produce this file ?")
        exit()

    # load ** testing ** data sets in the same class order as training
    # (here we perform patch sampling only from the centre of the +ve
    # class and only a single sample is taken
    # hence [0,0] sample sizes and [False,True] centre weighting flags)

    print("Loading test data as a batch ...")

    paths = [params.DATA_testing_path_neg, params.DATA_testing_path_pos]
    use_centre_weighting = [False, True]
    class_names = params.DATA_CLASS_NAMES
    imgs_data = utils.load_images(
        paths, class_names, [0, 0], use_centre_weighting)

    print("Computing HOG descriptors...")  # for each testing image
    start = cv2.getTickCount()
    [img_data.compute_hog_descriptor() for img_data in imgs_data]
    utils.print_duration(start)

    # get the example/sample HOG descriptors and class labels

    samples, class_labels = utils.get_hog_descriptors(
        imgs_data), utils.get_class_labels(imgs_data)

    # perform batch SVM classification over the whole set

    print("Performing batch SVM classification over all data  ...")

    results = svm.predict(samples)
    output = results[1].ravel()

    # compute and report the error over the whole set

    error = ((np.absolute(class_labels.ravel() - output).sum()) /
             float(output.shape[0]))
    print("Successfully trained SVM with {}% testing set error".format(
        round(error * 100, 2)))
    print("-- meaining the SVM got {}% of the testing examples correct!".format(round((1.0 - error) * 100, 2)))

################################################################################

if __name__ == "__main__":
    main()

################################################################################


"""
Load config file and optional .env (which may contain secrets that should not go in config)

"""

from . import parser as cp
from . import loader as cl
from .. import logger as pml

def load_configuration(config_file, dotenv_file=None, include_os_env=True):
    """
    Read configuration files and return dict of settings

    :param config_file: str
    :param dotenv_file: str | None
    :param include_os_env: bool
    :return:

    """
    config = cp.parse_config(config_file, dotenv_file, include_os_env)
    logger = configure_logger(config)
    monitors = cl.load_monitors(config)
    actions = cl.load_actions(config)

    config['monitors'] = monitors
    config['actions'] = actions
    config = configure_listeners(config)

    return config

def configure_listeners(config):
    """
    Attach listeners (actions) to monitors

    :param config:
    :return:
    """
    listeners = config.get('listeners', {})
    monitors = config.get('monitors', {})
    actions = config.get('actions', {})
    logger = pml.get_logger()

    if not listeners:
        raise ValueError("no listeners found in configuration")

    for listener_name, listener_info in listeners.items():
        monitor_name = listener_info['monitor']
        action_name = listener_info['action']

        if monitor_name not in monitors:
            raise ValueError("monitor {} not found "
                             "(for listener {})".format(monitor_name, listener_name))

        if action_name not in actions:
            raise ValueError("action {} not found "
                             "(for listener {})".format(action_name, listener_name))

        monitor = monitors[monitor_name]
        action = actions[action_name]

        monitor.add_listener(listener_name, action)
        logger.debug(f"adding action {action} to monitor {monitor}")

    return config

def configure_logger(config: dict):
    """
    Set up application-wide logger

    """
    general = config.get("general")
    log_level = general.get("log_level", "ERROR")
    log_file = general.get("log_file", None)
    logger = pml.set_logger(log_level, log_file)

    return logger


# Google code jam
import numpy as np

# read number of cases from stdin
T = int(input())

for j in range(1,T+1):

    N = int(input()) # class size

    # read BFF ids (shift down by one for indexing 0..N-1)
    BFF = np.fromstring(input(), dtype=int, sep=' ') - 1

    ids = np.arange(N,dtype=int) # all ids
    subset = np.array([ids[0]])
    subsize = np.array([0])

    # start with first child in class
    thisid = 0
    ids = np.delete(ids,0)
    while True:
        thisid = BFF[thisid]
        #print(ids, subset, thisid, BFF[thisid])
        if thisid in set(subset):
            # cannot seat BFF to the right (and make circle larger),
            # since BFF is already seated, next subset determine
            # largest circle

            # find position of last child's BBF in subset to form closed circle
            pos = np.where(subset==thisid)[0][0]
            # does the last child have its BFF sitting left of it
            if pos == (subset.size - 2):
                subsize = np.append(subsize,subset.size)
            else:
                subsize = np.append(subsize,subset.size - pos)

            # restart with next child (unless there are less than two left)
            if ids.size < 2:
                break
            else:
                thisid = ids[0]
                subset = np.array([ids[0]])
                ids = np.delete(ids,0)

        else:
            # exit if no ids remain
            if ids.size == 0:
                break

            # can seat BFF to the right: add to subset
            # remove id from remaining ids
            #   print("add {}".format(thisid))
            subset = np.append(subset,thisid)
            if thisid in set(ids):
                pos = np.where(ids==thisid)[0][0]
                ids = np.delete(ids,pos)

    maxsize = np.amax(subsize)
    print("Case #{}: {}".format(j,maxsize))


class Hash_table:
    def __init__(self, s):
        self.size = s
        self.T = [None] * s

    def __hash_level1(self, key):
        return key % self.size

    def __hash_level2(self, key):
        return key % (self.size // 10)

    def insert(self, key, value):
        pos_level1 = self.__hash_level1(key)
        if self.T[pos_level1] is None:
            self.T[pos_level1] = [None] * (self.size // 10)
        pos_level2 = self.__hash_level2(key)
        if self.T[pos_level1][pos_level2] is None:
            self.T[pos_level1][pos_level2] = []
        self.T[pos_level1][pos_level2].append((key, value))

    def get(self, key):
        pos_level1 = self.__hash_level1(key)
        if self.T[pos_level1] is not None:
            pos_level2 = self.__hash_level2(key)
            if self.T[pos_level1][pos_level2] is not None:
                for k, value in self.T[pos_level1][pos_level2]:
                    if k == key:
                        return value
        return None

    def print(self):
        print("{")
        for i in range(self.size):
            print("[", end="")
            for j in range(self.size // 10):
                if self.T[i] is not None and self.T[i][j] is not None:
                    items = self.T[i][j]
                    _str = ""
                    for k, value in items:
                        _str += value.to_string() + " "
                    print("[" + _str + "]", end="")
                else:
                    print("[]", end="")
            print("]")
        print("}")


# -*- coding: utf-8 -*-

from myformatter import MyFormatter
import pandas as pd
from males import get_males
from females import get_females
from heightstandperc import get_standards
from getheight import get_heights
from datetime import date
import os

def main():
    standard = get_standards()
    height = get_heights()
    heightsfemale = get_females(standard,height)
    heightsmale = get_males(standard,height)

    last_row = len(heightsfemale) + 1
    last_rowm = len(heightsmale) + 1
    with pd.ExcelWriter("HeightMonitoring{:%d-%b}.xlsx".format(date.today())) as writer:
        workbook = writer.book
        danger_format = workbook.add_format({'border': 1, 'bg_color': '#ff3333'})
        clear_format = workbook.add_format({'border': 1, 'bg_color': '#00ff00'})
        date_format = workbook.add_format({'num_format': 'dd-mmm-yyyy','border': 1})
        bold = workbook.add_format({'bold': True, 'font_color': 'black'})
        bold.set_align("top")
        red = workbook.add_format({'bold': True, 'font_color': '#ff0000'})
        red.set_align('vcenter')
        red.set_text_wrap()
        green = workbook.add_format({'bold': True, 'font_color': '#248f24'})
        green.set_align('vcenter')
        green.set_text_wrap()
        wrap = workbook.add_format({'bold': False, 'font_color': 'black'})
        wrap.set_align('vcenter')
        wrap.set_text_wrap()

        heightsfemale.to_excel(writer, sheet_name="Females")

        worksheet = writer.sheets["Females"]

        # If Height Needed to be in 3rd percentile > Projected Height at final visit
        formula_danger = f'=L2>N2'
        formula_clear = f'=G2>L2'

        worksheet.conditional_format(f'A2:A{last_row}', {'type': 'formula',
                                                        'criteria': formula_danger,
                                                        'format': danger_format})
        worksheet.conditional_format(f'A2:A{last_row}', {'type': 'formula',
                                                    'criteria': formula_clear,
                                                    'format': clear_format})
        worksheet.conditional_format(f'G2:G{last_row}', {'type': 'formula',
                                                    'criteria': formula_clear,
                                                    'format': clear_format})
        worksheet.conditional_format(f'G2:G{last_row}', {'type': 'formula',
                                                    'criteria': formula_danger,
                                                    'format': danger_format})
        worksheet.conditional_format(f'F2:F{last_row}', {'type': 'no_errors','format': date_format})
        worksheet.conditional_format(f'H2:H{last_row}', {'type': 'no_errors','format': date_format})
        formatter = MyFormatter(heightsfemale,workbook,worksheet)
        formatter.spacing()
        formatter.color_columns(color="#b3e0ff")
        formatter.add_borders()
        worksheet.freeze_panes(1,1)
        # Males
        heightsmale.to_excel(writer, sheet_name="Males")

        worksheet = writer.sheets["Males"]
        worksheet.conditional_format(f'A2:A{last_row}', {'type': 'formula',
                                                        'criteria': formula_danger,
                                                        'format': danger_format})
        worksheet.conditional_format(f'A2:A{last_row}', {'type': 'formula',
                                                    'criteria': formula_clear,
                                                    'format': clear_format})
        worksheet.conditional_format(f'G2:G{last_row}', {'type': 'formula',
                                                    'criteria': formula_clear,
                                                    'format': clear_format})
        worksheet.conditional_format(f'G2:G{last_row}', {'type': 'formula',
                                                    'criteria': formula_danger,
                                                    'format': danger_format})
        worksheet.conditional_format(f'F2:F{last_rowm}', {'type': 'no_errors','format': date_format})
        worksheet.conditional_format(f'H2:H{last_rowm}', {'type': 'no_errors','format': date_format})
        formatter = MyFormatter(heightsmale,workbook,worksheet)
        formatter.spacing()
        formatter.color_columns(color="#b3e0ff")
        formatter.add_borders()
        worksheet.freeze_panes(1,1)

        pd.DataFrame().to_excel(writer,sheet_name="Definitions", index = False)
        worksheet=writer.sheets["Definitions"]
        worksheet.set_column(0, 0, 20)
        worksheet.set_column(1, 1, 35)
        worksheet.write(0, 0, 'HNAT:',bold)
        worksheet.write(0, 1, 'Final Week 52 Height needed for a patient to be in the 3rd Percentile.\nCalculated as BaselineWeight + 3rd Percentile Growth Rate for that specific age and sex cohort.',wrap)
        worksheet.write(1,0,"WeeklyGrowthRate:",bold)
        worksheet.write(1,1,"Most Recent Visit Height - Baseline Height/Time between those visits in weeks",wrap)
        worksheet.write(2,0,"ProjectedGrowth:",bold)
        worksheet.write(2,1,"(WeeklyGrowthRate * Number of weeks from most recent vist date to EstimatedWk52Date)\n + Most recent CurrentHeight",wrap)
        worksheet.write(4,1,"Patients Highlighted in Red are not not expected to be above the 3rd percentile based on current GR",red)
        worksheet.write(6,1,"Any CurrentHeight highlighted in Greeen is already above the 3rd percentile",green)
        worksheet.write(8,1,"Percentile Standards are taken from the Table of Yearly growth velocity by age and gender(abstracted  from Tanner and  Davis. J Pediatr. 1985;107(3):317-329)",wrap)
    os.system("start EXCEL.EXE HeightMonitoring{:%d-%b}.xlsx".format(date.today()))

if __name__ == "__main__":
    main()


from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template

def send_notification(template_name, subject, receivers, **kwargs):
    domain = Site.objects.get_current()
    context = kwargs.get('context', {})
    context.update({
        'domain': domain,
    })
    template = get_template(template_name)
    message = template.render(context)
    msg = EmailMultiAlternatives(subject, from_email=settings.EMAIL_HOST_USER, to=receivers)
    msg.attach_alternative(message, "text/html")
    msg.send()


def solution(numbers):
    answer = []
    #select two numbers from list
    for i in range(len(numbers)):
        for j in range(len(numbers)):
            #skip duplicated turn
            if i >= j:
                continue
            #sum selected numbers
            answer.append(numbers[i]+numbers[j])
    #deduplicate
    answer=set(answer)
    answer=list(answer)
    answer.sort()
    return answer

두 게 뽑아서 더하기.py


import argparse
from collections import defaultdict
import numpy as np
import os.path

parser = argparse.ArgumentParser()
parser.add_argument("--babi_dir", default=None, type=str, help="bAbI tasks directory.")
parser.add_argument("--tasks", default=[5, 6], type=int, nargs='+', help="Which tasks to include.")

START_TOKEN = '<START>'

def entropy(vocab: dict, word_count):
    e = 0
    for word, freq in vocab.items():
        e -= freq / word_count * np.log2(freq / word_count)
    return e

def bigram_entropy(bigram_vocab: dict, vocab: dict, bigram_count, start_count):
    e = 0
    for (context_word, word), freq in bigram_vocab.items():
        if context_word == START_TOKEN:
            context_freq = start_count
        else:
            context_freq = vocab[context_word]
        e -= freq / bigram_count * np.log2(freq / context_freq)
    return e

def update(vocab: dict, words: list):
    for word in words:
        vocab[word] += 1

def bigram_update(bigram_vocab: dict, words: list):
    for bigram in zip([START_TOKEN] + words, words):
        bigram_vocab[bigram] += 1

def main(args):
    assert os.path.isdir(args.babi_dir), "Invalid path to bAbI directory."
    test_files = ['dialog-babi-task1-API-calls-trn.txt',
                  'dialog-babi-task2-API-refine-trn.txt',
                  'dialog-babi-task3-options-trn.txt',
                  'dialog-babi-task4-phone-address-trn.txt',
                  'dialog-babi-task5-full-dialogs-trn.txt',
                  'dialog-babi-task6-dstc2-trn.txt']
    paths = [os.path.join(args.babi_dir, test_file) for test_file in test_files]

    data = {}

    for task in args.tasks:
        path = paths[task - 1]

        dialogues = 0
        turns_total = 0
        words_user_total = 0
        words_bot_total = 0
        bigrams_user_total = 0
        bigrams_bot_total = 0
        bot_start_count_total = 0
        vocab_user = defaultdict(int)
        vocab_bot = defaultdict(int)
        bigram_vocab_user = defaultdict(int)
        bigram_vocab_bot = defaultdict(int)
        turn_counts = []
        user_word_counts = []
        bot_word_counts = []
        wpt_user_counter = []
        wpt_bot_counter = []
        words_user = 0
        words_bot = 0

        with open(path, "r") as task_file:
            for line in task_file:
                if not line.strip():
                    turn_counts.append(turns)
                    wpt_user_counter.append(words_user)
                    wpt_bot_counter.append(words_bot)
                    words_user = 0
                    words_bot = 0
                    continue
                if '\t' not in line: continue

                user, bot = line.split('\t')
                ID, *user = user.split()
                ID = int(ID)
                bot = bot.split()

                if ID == 1:
                    dialogues += 1
                    turns = 0

                if user == [] or user[0] == '<SILENCE>':
                    user = []
                else:
                    turns += 1
                    wpt_user_counter.append(words_user)
                    wpt_bot_counter.append(words_bot)
                    words_user = 0
                    words_bot = 0

                bot_start_count_total += 1

                words_user += len(user)
                words_bot += len(bot)
                words_user_total += len(user)
                words_bot_total += len(bot)
                bigrams_user_total += len(user) - 1
                bigrams_bot_total += len(bot) - 1

                update(vocab_user, user)
                update(vocab_bot, bot)
                bigram_update(bigram_vocab_user, user)
                bigram_update(bigram_vocab_bot, bot)

        user_entropy = entropy(vocab_user, words_user_total)
        bot_entropy = entropy(vocab_bot, words_bot_total)

        user_bigram_entropy = bigram_entropy(bigram_vocab_user, vocab_user, bigrams_user_total, sum(turn_counts))
        bot_bigram_entropy = bigram_entropy(bigram_vocab_bot, vocab_bot, bigrams_bot_total, bot_start_count_total)

        user_vocab_size = len(vocab_user.keys())
        print(vocab_user.keys())
        bot_vocab_size = len(vocab_bot.keys())

        data[task] = [dialogues,
                      sum(turn_counts),
                      np.mean(turn_counts),
                      np.std(turn_counts),
                      user_entropy,
                      user_bigram_entropy,
                      words_user_total,
                      user_vocab_size,
                      np.mean(wpt_user_counter),
                      np.std(wpt_user_counter),
                      bot_entropy,
                      bot_bigram_entropy,
                      bot_vocab_size,
                      words_bot_total,
                      np.mean(wpt_bot_counter),
                      np.std(wpt_bot_counter)]

    labels = ['Total Dialogues',
              'Total Truns',
              'Mean Dialogue Length in Turns',
              'Std of Dialogue Length in Turns',
              'User Entropy',
              'User Bigram Entropy',
              'User Words',
              'User Vocabulary Size',
              'User Mean Turn Length in Words',
              'User Std of Turn Length in Words',
              'System Entropy',
              'System Bigram Entropy',
              'System Vocabulary Size',
              'System Words',
              'System Mean Turn Length in Words',
              'System Std of Turn Length in Words']

    print('||', end='')
    for task in args.tasks:
        print(f'Task {task}|', end='')
    print()
    print('|---|' + '---|' * len(args.tasks))
    for idx, label in enumerate(labels):
        print(f'|{label}|', end='')
        for task in args.tasks:
            val = data[task][idx]
            if isinstance(val, int):
                print(val, end='|')
            else:
                print(f'{val:.2f}', end='|')
        print()

if __name__ == "__main__":
    args = parser.parse_args([] if "__file__" not in globals() else None)

    main(args)


import tkinter as tk
from tkinter import filedialog
from tkinter import StringVar
from tkinter import *
from tkinter import ttk
from tkinter import messagebox
import pandas as pd
import os
from proteinplotter import plotprotein
from tables import get_sequence, one_of_each
import tensorflow as tf

def run_protein_analysis():
    biglist_of_everything = []
    for index, row in df.iterrows():
        print(row['#CHROM'], row['POS'], row['ALT'], row['REF'])
        variant_location_list, sequence_list, variant_sequence_list = get_sequence([row['#CHROM'], row['POS']], row['ALT'], row['REF'])
        biglist_of_everything.append([variant_location_list, sequence_list, variant_sequence_list])
    df['protein_analysis'] = biglist_of_everything
    rows = df[['#CHROM', 'POS', 'REF', 'ALT', 'PREDICTIONSCORE', 'protein_analysis']]
    print(rows)
    update_table(rows, 2)

def select_active_items():
    global sequence_location
    #results_storage_dict = {}
    selected_list = []
    for item in curitem:
        selected_list.append(trv.item(item)['values'])
    print(selected_list)
    for item in selected_list:
        print(item)

        # protein plotter here instead
        #results_storage_dict[selection[:2]] = [variant_location_list, sequence_list, variant_sequence_list]
        #listofsequences.append(sequences)
        #listof_variant_sequences(variant_sequence)

    #print(listofsequences)

    #get_sequence("\t".join(selected_list))

def get_plot(sequence):
    def get_model():
        model = tf.keras.Sequential([
            tf.keras.layers.InputLayer(input_shape=(3000,)),
            tf.keras.layers.Dense(5000, activation='relu'),
            tf.keras.layers.Dense(3000, activation='sigmoid')
        ])
        model.compile(loss='binary_crossentropy', optimizer='adam')
        return model
    if len(sequence) > 3000:
        print("protein too big, cant analyse")
    else:
        model = get_model()
        model.load_weights('./models/binding.sav')
        other_binding = model.predict(one_of_each(sequence))
        model = get_model()
        model.load_weights('./models/dna_binding.sav')
        dna_binding = model.predict(one_of_each(sequence))
        model = get_model()
        model.load_weights('./models/metal.sav')
        metal_binding = model.predict(one_of_each(sequence))
        model = get_model()
        model.load_weights('./models/Act_sites.sav')
        active = model.predict(one_of_each(sequence))
        plot_image = plotprotein(sequence, other_binding, dna_binding, metal_binding, active, sequence_location)
        return plot_image

def selectItem(a):
    global curitem
    curitem = trv.selection()
    for item in curitem:
        print(trv.item(item))

def running_analysis():
    print("button clicked with", filename)
    print("sh runningscript.sh {}".format(filename[0]))
    print("analyis status: ", haplotype_var.get())
    if haplotype_var.get() == 1:
        haplonumber = entry.get()
        if len(filename) > 1:
    # change to import maybe instead of different program
            print("python3 Haplotype_file_analysis.py '{}'".format(filename))
            os.system("python3 Haplotype_file_analysis.py '{}' {}".format(filename, haplonumber))
    os.system("sh /Users/iwanhidding/PycharmProjects/Internship-Helsinki/runningscript.sh {}".format(filename[0]))
    #path.os(runstuff)

def on_button_click():
    #filename = entry.get()
    running_analysis()

def UploadAction(event=None):
    global filename
    filename = filedialog.askopenfilenames()
    #filelabelname.set('Selected file: {}'.format(filename))
    file_label.configure(text='Selected file: {}'.format(filename))
    #cst(filename, window).pack()
    print('Selected:', filename)

def Load_UploadAction(event=None):
    global df
    analysis_file = filedialog.askopenfilename()
    print(analysis_file)
    df = pd.read_csv(analysis_file, delim_whitespace=True, header=0)
    # global rows
    # add score
    rows = df[['#CHROM', 'POS', 'REF', 'ALT', 'PREDICTIONSCORE']]
    print(rows)
    update_table(rows, 1)
    # cst(open(analysis_file), window).pack()
    print('Selected:', analysis_file)

def update_table(rows, instance):
    for index, row in rows.iterrows():
        # Expand on this with other columns later
        # add a scrollbar and a search option, coding vs non-coding
        if instance == 1:
            trv.insert('', 'end', values=[row['#CHROM'], row['POS'], row['REF'], row['ALT'], row['PREDICTIONSCORE']])
        elif instance == 2:
            trv.insert('', 'end', values=[row['#CHROM'], row['POS'], row['REF'], row['ALT'], row['PREDICTIONSCORE'], row['protein_analysis']])

def load_image():
    #canvas = Canvas(root, width=300, height=300)
    #canvas.pack()
    #img = plotprotein()
    #canvas.create_image(20, 20, anchor=NW, image=img)
    pass

window = tk.Tk()
window.title("Iwan's analysis pipeline")

wrapper1 = LabelFrame(window, text="Pathogenicity analysis")
wrapper2 = LabelFrame(window, text="Additional analysis")

wrapper1.pack(fill="both", expand="yes", padx=20, pady=10)
wrapper2.pack(fill="both", expand="yes", padx=20, pady=10)

greeting = tk.Label(wrapper1, text="Put your vcf down here:")

global haplotype_var
haplotype_var = IntVar()

entry = tk.Entry(wrapper1)
file_label = tk.Label(wrapper1, text="No file selected")
haplolabel = tk.Label(wrapper1, text="Enter the required number of matching variants for haplotype analysis:")

button = tk.Button(
    wrapper1,
    text="Start analysis!",
    command=on_button_click
)

button_2 = tk.Button(wrapper1, text='Open', command=UploadAction)
button_3 = tk.Button(wrapper2, text='Open', command=Load_UploadAction)
button_selection = tk.Button(wrapper2, text='plot selection', command=select_active_items)
button_protein = tk.Button(wrapper2, text='Run protein analysis', command=run_protein_analysis)
haplocheckmark = Checkbutton(wrapper1, text="Haplotype Analysis", variable=haplotype_var)

trv = ttk.Treeview(wrapper2, columns=(1, 2, 3, 4, 5, 6), show="headings", height="10")

greeting.pack()
button_2.pack()
file_label.pack()
haplocheckmark.pack()
haplolabel.pack()
entry.pack()
button.pack()
button_3.pack()
trv.pack()
button_protein.pack()
button_selection.pack()
trv.heading(1, text="chr")
trv.heading(2, text="pos")
trv.heading(3, text="ref")
trv.heading(4, text="alt")
trv.heading(5, text="Pathogenicity score")
trv.heading(6, text="Protein analysis")
trv.bind('<ButtonRelease-1>', selectItem)

#button_4 = tk.Button(window, text='Open', command=UploadAction)
#f = open("output_annotation/clinpred_pred.txt")

#print(f)
#newtable = cst(f, window)

#newtable.pack()
# on button click run runningscript with the filename from the entry pack
window.mainloop()

# turn to grid over pack to allow better visualisation, add scrolling, maybe less detials


#!/usr/bin/env python
"""Plotting singular vectors of random matrices."""

import numpy as np
from matplotlib import pyplot as plt

MAXITER, M, N = 1_000_000, 2, 2
matrices = np.random.randn(MAXITER, M, N)
us = np.full((MAXITER, M), fill_value=float("nan"))
vs = np.full((MAXITER, N), fill_value=float("nan"))

for k, A in enumerate(matrices):
    U, S, Vh = np.linalg.svd(A)
    u, s, v = U[:, 0], S[0], Vh[0, :]
    us[k] = u
    vs[k] = v

angle_u = np.arctan2(*us.T[::-1])  # takes y as first arg
angle_v = np.arctan2(*vs.T[::-1])  # takes y as first arg

fig, ax = plt.subplots(figsize=(2.5, 2.5), dpi=300, constrained_layout=True)
ax.plot(angle_u, angle_v, ".", ms=2, markeredgecolor="none", alpha=0.2)
ax.set_xlabel("angle u")
ax.set_ylabel("angle v")
ax.set_aspect("equal", adjustable="box")
ax.set_xticks([-np.pi, 0, np.pi], ["-π", "0", "+π"])
ax.set_yticks([-np.pi, 0, np.pi], ["-π", "0", "+π"])
fig.savefig("hexagons.png", dpi=300)


# -*- coding: utf-8 -*-
"""
Created on Sun May  5 19:20:59 2019

@author: Sanchayni
"""

from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('ml-bank').getOrCreate()
spark.conf.set("spark.sql.execution.arrow.enabled", "true")

RDD_01 = spark.read.csv('New_cs_data.csv', header = True, inferSchema = True)
RDD_01.printSchema()

df = RDD_01.select("*").toPandas()
df = df.drop(['_c0'],axis=1)
df = df.drop(['Unnamed: 0'],axis=1)

print(df.columns)
import numpy as np
import pygal
import pandas as pd

df = df.apply(lambda x: x.replace('NA',np.nan))
df = df.apply(lambda x: x.astype(float))

print(df.isna().sum())

# Plotting

# --------------- UNIVARIATE plots ---------------

# Age
age_chart = pygal.Bar(x_label_rotation=-90)
age_chart.title = 'Histogram for variable Age'
age_chart.x_labels = map(str,pd.DataFrame(df.groupby('age').age.count()).index)
age_chart.add('age' ,  pd.DataFrame(df.groupby('age').age.count()).age)
age_chart.render_to_file('EDA_svgs/age_chart.svg')

# Dept Ratio
DebtRatio_chart = pygal.Bar(x_label_rotation=45)
DebtRatio_chart.title = 'Histogram for variable Debt Ratio'
DebtRatio_chart.x_labels = map(str, pd.DataFrame(pd.cut(df['DebtRatio'],20).value_counts().sort_index()).index)
DebtRatio_chart.add('DeptRatio' ,  pd.DataFrame(pd.cut(df['DebtRatio'],20).value_counts().sort_index()).DebtRatio)
DebtRatio_chart.render_to_file('EDA_svgs/DebtRatio_chart.svg')

# Monthly Income
Income_chart = pygal.Bar(x_label_rotation=45)
Income_chart.title = 'Histogram for variable Monthly Income'
Income_chart.x_labels = map(str, pd.DataFrame(pd.cut(df['MonthlyIncome'],20).value_counts().sort_index()).index)
Income_chart.add('MonthlyIncome' ,  pd.DataFrame(pd.cut(df['MonthlyIncome'],20).value_counts().sort_index()).MonthlyIncome)
Income_chart.render_to_file('EDA_svgs/Income_chart.svg')

# NumberOfDependents
NOD_chart = pygal.Bar(x_label_rotation=-90)
NOD_chart.title = 'Histogram for variable Number of Dependents'
NOD_chart.x_labels = map(str,pd.DataFrame(df.groupby('NumberOfDependents').NumberOfDependents.count()).index)
NOD_chart.add('NumberOfDependents',  pd.DataFrame(df.groupby('NumberOfDependents').NumberOfDependents.count()).NumberOfDependents)
NOD_chart.render_to_file('EDA_svgs/NOD_chart.svg')

# SeriousDlqin2yrs
SDy_chart = pygal.Bar(x_label_rotation=-90)
SDy_chart.title = 'Histogram for variable SeriousDlqin2yrs'
SDy_chart.x_labels = map(str,pd.DataFrame(df.groupby('SeriousDlqin2yrs').SeriousDlqin2yrs.count()).index)
SDy_chart.add('SeriousDlqin2yrs',  pd.DataFrame(df.groupby('SeriousDlqin2yrs').SeriousDlqin2yrs.count()).SeriousDlqin2yrs)
SDy_chart.render_to_file('EDA_svgs/SDy_chart.svg')

# RevolvingUtilizationOfUnsecuredLines
RUOUL_chart = pygal.Bar(x_label_rotation=-90)
RUOUL_chart.title = 'Histogram for variable RevolvingUtilization Of Unsecured Lines'

RUOUL_chart.x_labels = map(str,pd.DataFrame(pd.cut(df['RevolvingUtilizationOfUnsecuredLines'],20).value_counts().sort_index()).index)
RUOUL_chart.add('RevolvingUtilizationOfUnsecuredLines',  pd.DataFrame(pd.cut(df['RevolvingUtilizationOfUnsecuredLines'],20).value_counts().sort_index()).RevolvingUtilizationOfUnsecuredLines)
RUOUL_chart.render_to_file('EDA_svgs/RUOUL_chart.svg')

# NumberRealEstateLoansOrLines
NRELL_chart = pygal.Bar(x_label_rotation=45)
NRELL_chart.title = 'Histogram for NumberRealEstateLoansOrLines'
NRELL_chart.x_labels = map(str,pd.DataFrame(df.groupby('NumberRealEstateLoansOrLines').NumberRealEstateLoansOrLines.count()).index)
NRELL_chart.add('NumberRealEstateLoansOrLines',  pd.DataFrame(df.groupby('NumberRealEstateLoansOrLines').NumberRealEstateLoansOrLines.count()).NumberRealEstateLoansOrLines)
NRELL_chart.render_to_file('EDA_svgs/NRELL_chart.svg')

# NumberOfOpenCreditLinesAndLoans
NOCLL_chart = pygal.Bar(x_label_rotation=45)
NOCLL_chart.title = 'Histogram for NumberOfOpenCreditLinesAndLoans'
NOCLL_chart.x_labels = map(str,pd.DataFrame(df.groupby('NumberOfOpenCreditLinesAndLoans'). NumberOfOpenCreditLinesAndLoans.count()).index)
NOCLL_chart.add('NumberOfOpenCreditLinesAndLoans',  pd.DataFrame(df.groupby('NumberOfOpenCreditLinesAndLoans'). NumberOfOpenCreditLinesAndLoans.count()).NumberOfOpenCreditLinesAndLoans)
NOCLL_chart.render_to_file('EDA_svgs/NOCLL_chart.svg')

# NumberOfTime30-59DaysPastDueNotWorse
NT30_59PD_chart = pygal.Bar(x_label_rotation=45)
NT30_59PD_chart.title = 'Histogram for variable NumberOfTime30-59DaysPastDueNotWorse'
NT30_59PD_chart.x_labels = map(str,pd.DataFrame(df['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index()).index)
NT30_59PD_chart.add('NumberOfTime30-59DaysPastDueNotWorse',  pd.DataFrame(df['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index())['NumberOfTime30-59DaysPastDueNotWorse'])
NT30_59PD_chart.render_to_file('EDA_svgs/NT30_59PD_chart.svg')

# NumberOfTime60-89DaysPastDueNotWorse
NT60_89PD_chart = pygal.Bar(x_label_rotation=45)
NT60_89PD_chart.title = 'Histogram for variable NumberOfTime60-89DaysPastDueNotWorse'
NT60_89PD_chart.x_labels = map(str,pd.DataFrame(df['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index()).index)
NT60_89PD_chart.add('NumberOfTime60-89DaysPastDueNotWorse',  pd.DataFrame(df['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index())['NumberOfTime60-89DaysPastDueNotWorse'])
NT60_89PD_chart.render_to_file('EDA_svgs/NT60_89PD_chart.svg')

# NumberOfTimes90DaysLate
NT90DL_chart = pygal.Bar(x_label_rotation=45)
NT90DL_chart.title = 'Histogram for variable NumberOfTimes90DaysLate'
NT90DL_chart.x_labels = map(str,pd.DataFrame(df['NumberOfTimes90DaysLate'].value_counts().sort_index()).index)
NT90DL_chart.add('NumberOfTimes90DaysLate',  pd.DataFrame(df['NumberOfTimes90DaysLate'].value_counts().sort_index()).NumberOfTimes90DaysLate)
NT90DL_chart.render_to_file('EDA_svgs/NT90DL_chart.svg')

#-------------- BOXPLOTS --------------

# Age
BP_age_default = pygal.Box(box_mode="1.5IQR")
BP_age_default.title = 'Age variation across Default'
BP_age_default.add('Defaulter', df[df['SeriousDlqin2yrs'] == 1.0].age)
BP_age_default.add('Non Defaulter', df[df['SeriousDlqin2yrs'] == 0.0].age)
BP_age_default.render_to_file('EDA_svgs/BP_age_default.svg')

# DebtRatio
BP_DR_default = pygal.Box(box_mode="tukey")
BP_DR_default.title = 'Dept Ratio variation across Default'
BP_DR_default.add('Defaulter', df[df['SeriousDlqin2yrs'] == 1.0].DebtRatio)
BP_DR_default.add('Non Defaulter', df[df['SeriousDlqin2yrs'] == 0.0].DebtRatio)
BP_DR_default.render_to_file('EDA_svgs/BP_DR_default.svg')

# MonthlyIncome
# BP_MI_default = pygal.Box(box_mode="1.5IQR", range = (min(df['MonthlyIncome']),max(df['MonthlyIncome']) ) )

BP_MI_default = pygal.Box(box_mode="1.5IQR", range = (-10000,40000 ))

BP_MI_default.title = 'Monthly Income variation across Default'
BP_MI_default.add('Defaulter', df[(df['SeriousDlqin2yrs'] == 1.0) & (df['MonthlyIncome'].notna())].MonthlyIncome)
BP_MI_default.add('Non Defaulter', df[(df['SeriousDlqin2yrs'] == 0.0) & (df['MonthlyIncome'].notna())].MonthlyIncome)
BP_MI_default.render_to_file('EDA_svgs/BP_MI_default.svg')

# NumberOfDependents
BP_NOD_default = pygal.Box(box_mode="tukey", range = (-3,10))
BP_NOD_default.title = 'NumberOfDependents variation across Default'
BP_NOD_default.add('Defaulter', df[ (df['SeriousDlqin2yrs'] == 1.0) & (df['NumberOfDependents'].notna())].NumberOfDependents)
BP_NOD_default.add('Non Defaulter', df[ (df['SeriousDlqin2yrs'] == 0.0) & (df['NumberOfDependents'].notna())].NumberOfDependents)
BP_NOD_default.render_to_file('EDA_svgs/BP_NOD_default.svg')

# RevolvingUtilizationOfUnsecuredLines
BP_RUOUL_default = pygal.Box(box_mode="tukey")
BP_RUOUL_default.title = ' RevolvingUtilizationOfUnsecuredLines variation across Default'
BP_RUOUL_default.add('Defaulter', df[df['SeriousDlqin2yrs'] == 1.0].RevolvingUtilizationOfUnsecuredLines)
BP_RUOUL_default.add('Non Defaulter', df[df['SeriousDlqin2yrs'] == 0.0].RevolvingUtilizationOfUnsecuredLines)
BP_RUOUL_default.render_to_file('EDA_svgs/BP_RUOUL_default.svg')

# NumberOfTimes90DaysLate
Default_90ds = pygal.Bar(x_label_rotation=45)
Default_90ds.title = ' Default across Number of times 90 days late'
Default_90ds.x_labels = map(str, df['NumberOfTimes90DaysLate'].value_counts().sort_index().index)
Default_90ds.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ].NumberOfTimes90DaysLate.value_counts().sort_index())
Default_90ds.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ].NumberOfTimes90DaysLate.value_counts().sort_index())

Default_90ds.render_to_file('EDA_svgs/Default_90ds.svg')

# NumberOfTime60-89DaysPastDueNotWorse
Default_60_89ds = pygal.Bar(x_label_rotation=45)
Default_60_89ds.title = ' Default across NumberOfTime60-89DaysPastDueNotWorse'
Default_60_89ds.x_labels = map(str, df['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index().index)
Default_60_89ds.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ]['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index())
Default_60_89ds.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ]['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index())

Default_60_89ds.render_to_file('EDA_svgs/Default_60_89ds.svg')

# NumberOfTime30-59DaysPastDueNotWorse
Default_30_59ds = pygal.Bar(x_label_rotation=45)
Default_30_59ds.title = ' Default across NumberOfTime30-59DaysPastDueNotWorse'
Default_30_59ds.x_labels = map(str, df['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index().index)
Default_30_59ds.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ]['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index())
Default_30_59ds.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ][ 'NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index())

Default_30_59ds.render_to_file('EDA_svgs/Default_30_59ds.svg')

# NumberOfOpenCreditLinesAndLoans
Default_OpenCredit = pygal.Bar(x_label_rotation=45)
Default_OpenCredit.title = ' Default across NumberOfOpenCreditLinesAndLoans'
Default_OpenCredit.x_labels = map(str, df['NumberOfOpenCreditLinesAndLoans'].value_counts().sort_index().index)
Default_OpenCredit.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ].NumberOfOpenCreditLinesAndLoans.value_counts().sort_index())
Default_OpenCredit.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ].NumberOfOpenCreditLinesAndLoans.value_counts().sort_index())

Default_OpenCredit.render_to_file('EDA_svgs/Default_OpenCredit.svg')

# NumberRealEstateLoansOrLines
Default_RealEstate= pygal.Bar(x_label_rotation=45)
Default_RealEstate.title = ' Default across NumberRealEstateLoansOrLines '
Default_RealEstate.x_labels = map(str, df['NumberRealEstateLoansOrLines'].value_counts().sort_index().index)
Default_RealEstate.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ].NumberRealEstateLoansOrLines.value_counts().sort_index())
Default_RealEstate.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ].NumberRealEstateLoansOrLines.value_counts().sort_index())

Default_RealEstate.render_to_file('EDA_svgs/Default_RealEstate.svg')

print('My work here is done')

# execfile('/home/maria_dev/EDA_CS.py')


from openpyxl import load_workbook
import requests
import re
from lxml import etree
def get_quxian(excel):
    wb = load_workbook(excel)
    for i in wb['SQL Results']['F']:
        if i.value != '区县名称':
            yield i.value
def baidu_ss(arg):
    url1 = 'https://www.baidu.com/s?wd='+str(arg)+'人口'
    url2 = 'https://www.baidu.com/s?wd='+str(arg)+'面积'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Host': 'baike.baidu.com',
        'Accept-Encoding': 'gzip, deflate, br'
    }
    response = requests.get(url1,headers=headers)
    response.encoding = 'utf-8'
    html = response.text
    html = etree.HTML(html)
    renkou = html.xpath('//div[@class="op_exactqa_s_area c-span18 c-span-last"]/div[@class="op_exactqa_s_answer"]/text()')

    # a2 = ''.join(a).replace('\n','')
    # mianji = re.findall('(\d+.\d+)平方千米|(\d+)平方公里',a2)
    # renkou = re.findall('(\d+.\d+)万.*?（\d+年常住人口）|(\d+)万.*?（\d+年常住人口）|(\d+.\d+)万.*?（\d+年）|(\d+)万.*?（\d+年）|(\d+)万人|(\d+.\d+)万人',a2)
    response2 = requests.get(url2,headers=headers)
    response2.encoding = 'utf-8'
    html = response2.text
    html = etree.HTML(html)
    mianji = html.xpath('//div[@class="op_exactqa_s_area c-span18 c-span-last"]/div[@class="op_exactqa_s_answer"]/text()')
    print(url1)
    renkou = ''.join(renkou).replace('\n','').replace(' ','').replace('\t','').replace(' ','')
    mianji = ''.join(mianji).replace('\n','').replace(' ','').replace('\t','').replace(' ','')
    print(renkou,mianji)
    return mianji,renkou
def write_excel(mianji,renkou):
    with open('test4.xlsx','a',encoding='utf-8') as f:
        f.write(mianji+'\t'+renkou+'\n')
def main():
    for i in get_quxian('基准价覆盖城市行政区详情.xlsx'):
        mianji,renkou = baidu_ss(i)
        write_excel(str(mianji),str(renkou))
if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

import unittest
from nose.tools import ok_, eq_, assert_equal, assert_false, assert_true

# noinspection PyUnresolvedReferences
from .._tins import EthernetII, HWAddress, PDU, IP, TCP, RAW, PDUNotFound, UDP, ICMP, Dot3, DNS, SNAP, STP, LLC

import platform
IS_MACOSX = platform.system().lower().strip() == "darwin"

def _f(packet):
    return "".join(chr(i) for i in packet)

expected_packet = _f([
    146, 131, 138, 146, 146, 128, 0, 0, 144, 76, 8, 23, 181, 0, 146, 131,
    120, 128, 0, 0, 144, 76, 8, 23, 181, 128, 1, 15, 0, 20, 0, 2, 0, 0,
    0
])

class STPTest(unittest.TestCase):
    def check_equals(self, bpdu1, bpdu2):
        eq_(bpdu1.priority, bpdu2.priority)
        eq_(bpdu1.ext_id, bpdu2.ext_id)
        eq_(bpdu1.id, bpdu2.id)

    def test_default_constr(self):
        stp = STP()
        eq_(stp.proto_id, 0)
        eq_(stp.proto_version, 0)
        eq_(stp.bpdu_type, 0)
        eq_(stp.bpdu_flags, 0)
        eq_(stp.root_path_cost, 0)
        eq_(stp.port_id, 0)
        eq_(stp.msg_age, 0)
        eq_(stp.max_age, 0)
        eq_(stp.hello_time, 0)
        eq_(stp.fwd_delay, 0)

    def test_constr_buf(self):
        stp = STP.from_buffer(expected_packet)
        bpdu = STP.bpdu_id_t(0x8, 0, "00:90:4c:08:17:b5")
        eq_(stp.proto_id, 0x9283)
        eq_(stp.proto_version, 0x8a)
        eq_(stp.bpdu_type, 0x92)
        eq_(stp.bpdu_flags, 0x92)
        self.check_equals(bpdu, stp.root_id)
        eq_(stp.root_path_cost, 0x928378)
        self.check_equals(bpdu, stp.bridge_id)
        eq_(stp.port_id, 0x8001)
        eq_(stp.msg_age, 15)
        eq_(stp.max_age, 20)
        eq_(stp.hello_time, 2)
        eq_(stp.fwd_delay, 0)

    def test_bpduid(self):
        expected = _f([
            0, 0, 0, 0, 0, 128, 100, 0, 28, 14, 135, 120, 0, 0, 0, 0, 4, 128,
            100, 0, 28, 14, 135, 133, 0, 128, 4, 1, 0, 20, 0, 2, 0, 15, 0, 0,
            0, 0, 0, 0, 0, 0, 0
        ])
        stp = STP.from_buffer(expected)
        bpdu = STP.bpdu_id_t(0x8, 100, "00:1c:0e:87:78:00")
        self.check_equals(bpdu, stp.root_id)

    def test_chained_pdus(self):
        input_s = _f([
            1, 128, 194, 0, 0, 0, 0, 144, 76, 8, 23, 181, 0, 38, 66, 66, 3,
            0, 0, 0, 0, 0, 128, 0, 0, 144, 76, 8, 23, 181, 0, 0, 0, 0, 128,
            0, 0, 144, 76, 8, 23, 181, 128, 1, 0, 0, 20, 0, 2, 0, 0, 0
        ])
        pkt = Dot3.from_buffer(input_s)
        stp = pkt.rfind_pdu(STP)
        assert_true(stp is not None)
        llc = pkt.rfind_pdu(LLC)
        assert_true(llc is not None)
        eq_(stp.port_id, 0x8001)
        eq_(stp.msg_age, 0)
        eq_(stp.max_age, 20)
        eq_(stp.hello_time, 2)
        llc.dsap = 0
        llc.ssap = 0
        eq_(input_s, pkt.serialize())

    def test_serialize(self):
        stp = STP.from_buffer(expected_packet)
        eq_(stp.serialize(), expected_packet)

    def test_protoid(self):
        stp = STP()
        stp.proto_id = 0x9283
        eq_(stp.proto_id, 0x9283)

    def test_proto_version(self):
        stp = STP()
        stp.proto_version = 0x15
        eq_(stp.proto_version, 0x15)

    def test_bpdutype(self):
        stp = STP()
        stp.bpdu_type = 0x15
        eq_(stp.bpdu_type, 0x15)

    def test_bpduflags(self):
        stp = STP()
        stp.bpdu_flags = 0x15
        eq_(stp.bpdu_flags, 0x15)

    def test_root_path_cost(self):
        stp = STP()
        stp.root_path_cost = 0x28378462
        eq_(stp.root_path_cost, 0x28378462)

    def test_port_id(self):
        stp = STP()
        stp.port_id = 0x9283
        eq_(stp.port_id, 0x9283)

    def test_msg_age(self):
        stp = STP()
        stp.msg_age = 15
        eq_(stp.msg_age, 15)

    def test_max_age(self):
        stp = STP()
        stp.max_age = 15
        eq_(stp.max_age, 15)

    def test_fwd_delay(self):
        stp = STP()
        stp.fwd_delay = 15
        eq_(stp.fwd_delay, 15)

    def test_hello_time(self):
        stp  = STP()
        stp.hello_time = 15
        eq_(stp.hello_time, 15)

    def test_root_id(self):
        stp = STP()
        bpdu = STP.bpdu_id_t(0x8, 100, "00:1c:0e:87:78:00")
        stp.root_id = bpdu
        self.check_equals(stp.root_id, bpdu)

    def test_bridge_id(self):
        stp = STP()
        bpdu = STP.bpdu_id_t(0x8, 100, "00:1c:0e:87:78:00")
        stp.bridge_id = bpdu
        self.check_equals(stp.bridge_id, bpdu)


import os
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.colors import Normalize, LogNorm
sns.set_theme()
import numpy as np
import pandas as pd
from gbm.data import TTE
from gbm.binning.unbinned import bin_by_time
from gbm.plot import Lightcurve
from gbm.finder import BurstCatalog
import pickle

tte_path = '/beegfs/rcrupi/zzz_other/tte_pkl/' # tte
list_tte = os.listdir(tte_path)
bool_pkl = False
if bool_pkl:
    burstcat = BurstCatalog()
    df_burst = pd.DataFrame(burstcat.get_table())

    def det_triggered(str_mask):
        list_det = np.array(['n0', 'n1', 'n2', 'n3', 'n4', 'n5', 'n6', 'n7',
                             'n8', 'n9', 'na', 'nb'])
        try:
            idx_det = np.where(np.array([int(i) for i in list(str_mask)]) == 1)
            return list(list_det[idx_det])
        except:
            print("Error, not found detectors triggered.")
            return list(list_det)

    ds_train = []
    max_len_time = 8000
    for tte_tmp in list_tte:
        # Check if detector has event signal counts
        str_det = df_burst.loc[df_burst['trigger_name'] == tte_tmp.split('_')[3], 'bcat_detector_mask'].values[0]
        list_det = det_triggered(str_mask=str_det)
        if tte_tmp.split('_')[2] not in list_det:
            # print(tte_tmp + ' not used.')
            continue
        # read a tte file
        tte = TTE.open(tte_path+tte_tmp)
        print(tte)
        # bin in time
        phaii = tte.to_phaii(bin_by_time, 0.256, time_ref=0.0)
        type(phaii)
        # # plot the lightcurve
        # lcplot = Lightcurve(data=phaii.to_lightcurve())
        # plt.show()
        pdc = phaii.data.counts
        if pdc.shape[0] < max_len_time:
            diff_len = max_len_time - pdc.shape[0]
            pdc = np.pad(pdc, [(0, diff_len), (0, 0)], mode='constant', constant_values=0)
        else:
            print("An event is cut for the sake of dimensions. dim original: " + str(pdc.shape[0]))
            pdc = pdc[0:max_len_time, :]
        ds_train.append(pdc)
        # # Draw a heatmap with the numeric values in each cell
        # index = (phaii.data.time_centroids>=-15) & (phaii.data.time_centroids<15)
        # pd_ctime = pd.DataFrame(phaii.data.counts[index]).T
        # pd_ctime.index = np.around(phaii.data.energy_centroids)
        # pd_ctime.columns = np.around(phaii.data.time_centroids[index], 2)
        # f, ax = plt.subplots(figsize=(12, 8))
        # sns.heatmap(pd_ctime.loc[:, :], annot=False, fmt="d", linewidths=.5, ax=ax, norm=LogNorm()) # Normalize, LogNorm

    # mat_len = []
    # for i in ds_train:
    #     mat_len.append(i.shape)
    # mat_len = np.array(mat_len)
    # # max=8000, q75%=4000
    # plt.boxplot(mat_len[:, 0])

    ds_train = np.array(ds_train)

    for i in range(0, ds_train.shape[0]):
        with open(tte_path + 'ds_train' + str(i) + '.pickle', 'wb') as f:
            pickle.dump(ds_train[i, :, :], f)
    # with open(tte_path+'ds_train.pickle', 'wb') as f:
    #     pickle.dump(ds_train, f)
else:
    ds_train = []
    for i in os.listdir(tte_path):
        with open(tte_path+i, 'rb') as f:
            ds_train.append(pickle.load(f))
    ds_train = np.array(ds_train)

print('end')
pass

import tensorflow as tf
from tensorflow.keras import layers

input_img = tf.keras.Input(shape=(8000, 128, 1))

x = layers.Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)
x = layers.MaxPooling2D((2, 2), padding='same')(x)
x = layers.Conv2D(8, (5, 5), activation='relu', padding='same')(x)
x = layers.MaxPooling2D((2, 2), padding='same')(x)
x = layers.Conv2D(8, (7, 7), activation='relu', padding='same')(x)
encoded = layers.MaxPooling2D((2, 2), padding='same')(x)

# at this point the representation is (4, 4, 8) i.e. 128-dimensional

x = layers.Conv2D(8, (7, 7), activation='relu', padding='same')(encoded)
x = layers.UpSampling2D((2, 2))(x)
x = layers.Conv2D(8, (5, 5), activation='relu', padding='same')(x)
x = layers.UpSampling2D((2, 2))(x)
x = layers.Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = layers.UpSampling2D((2, 2))(x)
decoded = layers.Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)

autoencoder = tf.keras.Model(input_img, decoded)
autoencoder.compile(optimizer='adam', loss='binary_crossentropy')

# Preprocessing
# max_ds_train = ds_train.max()
x_train = ds_train[:1000].astype('float32') / 4000
x_test = ds_train[1000:].astype('float32') / 4000

x_train = np.reshape(x_train, (len(x_train), 8000, 128, 1))
x_test = np.reshape(x_test, (len(x_test), 8000, 128, 1))

from keras.callbacks import TensorBoard

autoencoder.fit(x_train, x_train,
                epochs=50,
                batch_size=4, # 128
                shuffle=True,
                validation_data=(x_test, x_test),
                callbacks=[TensorBoard(log_dir='/tmp/autoencoder')])


from PIL import Image
import img2pdf
import os
from glob import glob

# 各画像を格納したフォルダの直下に、画像があるようにすること
# PDFのファイル名はフォルダ名になる

# 画像を格納したフォルダの親フォルダ
path = "C:\\_Python\\PDFeditor\\img"
# 画像の拡張子 *pngはダメらいしい？未検証
ext = ".jpg"

path += "\\**\\"

for p in [i for i in glob(path)]:
    search_path = p + "*" + ext
    folder_name = os.path.basename(p.rstrip(os.sep))
    print(folder_name)
    for i in glob(search_path):
        with Image.open(i) as src:
            data = src.getdata()
            mode = src.mode
            size = src.size
        with Image.new(mode, size) as dst:
            dst.putdata(data)
            dst.save(i)
    if len([i for i in glob(search_path)]) == 0:
        continue
    else:
        with open(folder_name + ".pdf", "wb") as files:
            files.write(img2pdf.convert([i for i in glob(search_path)]))


#memo

import sys

option= sys.argv[1]
contents = sys.argv[2:]

# -a 를 하면 쓰고
# -v 를 하면 읽게끔 하고싶다.

if option == "-a" :
    with open("C:\doit\memo.txt", 'a') as f :
        f.write(str(contents))
        f.write("\n")

elif option == "-v" :
    with open("C:\doit\memo.txt", 'r') as f :

        print(f.read())

else : print("잘못 입력하셨습니다")


import os, torch, misc

def parse_arguments():
    import argparse
    parser = argparse.ArgumentParser(description='DeepSpeech training')
    # wav16000
    parser.add_argument('--raw_data_dir', default='raw_data', help='Path to read raw dataset')
    parser.add_argument('--data_16000_dir', default='wav_16000', help='Path to save dataset')
    parser.add_argument('--data_dir', default='data', help='Path to save dataset')
    parser.add_argument('--sample_rate', default=16000, type=int, help='Sample rate')
    # manifest
    parser.add_argument('--manifest_dir', default='manifest', help='Path to manifest')
    parser.add_argument('--eval_size', type=int, default=500, help='Size of held-out evaluation set')
    parser.add_argument('--test_size', type=int, default=500, help='Size of held-out test set')
    # data loader
    parser.add_argument('--batch_size', default=20, type=int, help='Batch size for training')
    parser.add_argument('--num_workers', default=4, type=int, help='Number of workers used in data-loading')
    parser.add_argument('--window_size', default=.02, type=float, help='Window size for spectrogram in seconds')
    parser.add_argument('--window_stride', default=.01, type=float, help='Window stride for spectrogram in seconds')
    # parser.add_argument('--crop_audio_length', default=20, type=float, help='Crop audios exceeding this length in seconds')
    # audio model
    parser.add_argument('--hidden_size', default=800, type=int, help='Hidden size of RNNs')
    parser.add_argument('--hidden_layers', default=5, type=int, help='Number of RNN layers')
    parser.add_argument('--num_classes', default=10, type=int, help='Number of audio classes')
    parser.add_argument('--model_path', default='models/best.pth', help='Location to save best validation model')
    # train
    parser.add_argument('--gpu', default=-1, type=int, help='The single device used in training, -1 means using CPU or ALL GPUs if avaiable.')
    parser.add_argument('--train_manifest', default='train.csv', help='path to train manifest csv')
    parser.add_argument('--eval_manifest', default='eval.csv', help='path to eval manifest csv')
    parser.add_argument('--test_manifest', default='test.csv', help='path to test manifest csv')
    parser.add_argument('--epochs', default=100, type=int, help='Number of training epochs')
    parser.add_argument('--lr', default=1e-2, type=float, help='initial learning rate')
    parser.add_argument('--momentum', default=0.9, type=float, help='momentum')
    parser.add_argument('--max_norm', default=400, type=int, help='Norm cutoff to prevent explosion of gradients')
    parser.add_argument('--learning_anneal', default=1.05, type=float, help='Annealing applied to learning rate every epoch')
    parser.add_argument('--logdir', default='log', type=str, help='Log folder')
    parser.add_argument('--continue_from', type=str, help='Continue from last training epoch')
    parser.add_argument('--plot', action='store_true', help='Plot training progress in terms of accuracies')
    # parse
    return parser.parse_args()

class Logger(object):

    def __init__(self, args, init_train_info={}, sub_dir=None):
        self.args = args
        misc.ensure_dir(args.logdir)
        sub_dir = args.continue_from or sub_dir or misc.datetimestr()
        self.logdir = os.path.join(args.logdir, sub_dir)
        misc.ensure_dir(self.logdir)
        self._setup_log_file()
        self._create_train_info(args, init_train_info)

    def _setup_log_file(self):
        import logging
        log_file = os.path.join(self.logdir, 'log.txt')
        self._logger = logging.getLogger()
        self._logger.setLevel('INFO')
        fh = logging.FileHandler(log_file)
        ch = logging.StreamHandler()
        self._logger.addHandler(fh)
        self._logger.addHandler(ch)

    def info(self, str_info):
        self._logger.info(str_info)

    # train_info

    def _create_train_info(self, args, init_train_info):
        import json
        train_info_path = os.path.join(self.logdir, 'train_info.txt')
        if os.path.isfile(train_info_path):
            self.train_info = json.loads(misc.load_file(train_info_path))
        else:
            self.train_info = init_train_info

    def save_train_info(self):
        import json
        train_info_path = os.path.join(self.logdir, 'train_info.txt')
        misc.save_file(train_info_path, json.dumps(self.train_info))

    def plot_progress(self):
        import plot
        pdf_plot_path = os.path.join(self.logdir, 'progress.pdf')
        x_data = torch.Tensor(range(1, 1 + len(self.train_info['train_acc'])))
        y_data = torch.Tensor([self.train_info['train_acc'], self.train_info['eval_acc']])
        # y_data, y_err = plot.smooth2d(y_data, step=3)
        plot.plot(x_data, y_data, y_err=None, legends=['Train', 'Eval'],
                title=None, xlabel='Epoch', ylabel='Acccuracy', filename=pdf_plot_path)

    # model

    def save_model(self, model, model_path=None):
        if isinstance(model, torch.nn.DataParallel):
            model = model.module
        if model_path is None:
            if 'model_path' in self.train_info and os.path.exists(os.join(self.logdir, self.train_info['model_path'])):
                print("Removing old model {}".format(self.train_info['model_path']))
                os.remove(os.join(self.logdir, self.train_info['model_path']))
            model_path = misc.datetimestr() + '.model.pth'
            self.train_info['model_path'] = model_path
        print("Saving model to {}".format(model_path))
        model.save(os.path.join(self.logdir, model_path))

    def create_model(self):
        import model, rnn
        if 'model_path' in self.train_info and os.path.exists(self.train_info['model_path']):
            model = model.AudioModel(rnn.BatchRNNLayers, config=None)
            model.load(self.train_info['model_path'])
        else:
            model = model.AudioModel(rnn.BatchRNNLayers, config=self.args.__dict__)
        if torch.cuda.is_available():
            if self.args.gpu == -1:
                model = torch.nn.DataParallel(model).cuda()
            else:
                model = model.cuda()
        return model

def run_epoch(model, data_loader, is_cuda, optimizer=None, max_norm=None):
    import time
    from tqdm import tqdm
    from torch.autograd import Variable

    is_train = optimizer is not None
    if is_train:
        model.train()
    else:
        model.eval()

    avg_loss = misc.Average()
    avg_acc = misc.Average()

    for i, (input, target) in tqdm(enumerate(data_loader), total=len(data_loader)):
        if is_train:
            input = Variable(input, requires_grad=False)
        else:
            input = Variable(input, volatile=True)
        if is_cuda:
            input = input.cuda()

        output = model(input)
        _, pred = output.data.cpu().max(1)
        batch_size = input.size(0)
        acc = (pred.view(-1) == target).sum() / float(batch_size)
        avg_acc.update(acc, batch_size)

        if is_train:
            target = Variable(target, requires_grad=False)
            if is_cuda:
                target = target.cuda()

            loss = torch.nn.functional.cross_entropy(output, target) / batch_size
            avg_loss.update(loss.data[0], batch_size)
            optimizer.zero_grad()
            loss.backward()
            if max_norm is not None:
                torch.nn.utils.clip_grad_norm(model.parameters(), max_norm)
            optimizer.step()

        if is_cuda:
            torch.cuda.synchronize()

    return avg_acc.avg, avg_loss.avg

def init_train_info():
    train_info = {}
    train_info['epoch'] = 0
    train_info['train_loss'] = []
    train_info['train_acc'] = []
    train_info['eval_acc'] = []
    train_info['test_acc'] = []
    train_info['best_eval_acc'] = 0
    train_info['best_test_acc'] = 0
    return train_info

def anneal_lr(epoch, args, optimizer, logger):
    if epoch > 1:
        optim_state = optimizer.state_dict()
        optim_state['param_groups'][0]['lr'] = optim_state['param_groups'][0]['lr'] / args.learning_anneal
        optimizer.load_state_dict(optim_state)
        logger.info('Learning rate annealed to: {lr:.6f}'.format(lr=optim_state['param_groups'][0]['lr']))

def train(args):
    is_cuda = torch.cuda.is_available()
    logger = Logger(args, init_train_info=init_train_info())

    model = logger.create_model()
    parameters = model.parameters()
    optimizer = torch.optim.SGD(parameters, lr=args.lr, momentum=args.momentum, nesterov=True)
    logger.info("Number of parameters: %d" % misc.get_param_size(model))

    import loader
    train_loader, _, _ = loader.get(args, os.path.join(args.manifest_dir, args.train_manifest))
    eval_loader, _, _ = loader.get(args, os.path.join(args.manifest_dir, args.eval_manifest))
    test_loader, _, _ = loader.get(args, os.path.join(args.manifest_dir, args.test_manifest))

    for epoch in range(args.epochs):
        anneal_lr(epoch + 1, args, optimizer, logger)
        if epoch < logger.train_info['epoch']: continue
        logger.train_info['epoch'] += 1

        model.train()
        acc, loss = run_epoch(model, train_loader, is_cuda, optimizer, args.max_norm)
        logger.info('Epoch: {}, train_acc: {}, train_loss: {}'.format(epoch + 1, acc, loss))
        logger.train_info['train_loss'].append(loss)
        logger.train_info['train_acc'].append(acc)

        model.eval()
        acc, _ = run_epoch(model, eval_loader, is_cuda)
        logger.info('Epoch: {}, eval_acc: {}'.format(epoch + 1, acc))
        logger.train_info['eval_acc'].append(acc)

        if acc > logger.train_info['best_eval_acc']:
            logger.train_info['best_eval_acc'] = acc
            acc, _ = run_epoch(model, test_loader, is_cuda)
            logger.info('Epoch: {}, test_acc: {}'.format(epoch + 1, acc))
            logger.train_info['test_acc'].append(acc)

            if acc > logger.train_info['best_test_acc']:
                print("Found better model (acc=%.5f) with test dataset." % acc)
                logger.train_info['best_test_acc'] = acc
                logger.save_model(model, 'best.model.pth')

        logger.save_train_info()
        logger.save_model(model)
        if args.plot:
            logger.plot_progress()

if __name__ == '__main__':
    args = parse_arguments()
    if args.gpu >= 0:
        misc.prepare_single_device(args)
    train(args)


'''test_model.py shows how to write a standalone script to test various django subroutines.
'''

'''Over here is some boiler plat django to make sure the script executes properly. Don't worry
about it too much. Pretty much pattern match when you need to do this.;
'''
import sys, os, logging
sys.path.append('../ormintro')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ormintro.settings'

import django
django.setup()

#sets up logging, all logs go to the console
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)

'''Now we will import our code
'''
from library.models import *

#addBook('9781497438095','Romeo and Juliet','William Shakespeare', 1597)
#addInventory('9781497438095',5)

#addBook('9780446310789','To Kill a Mockingbird','Harper Lee', 1962)

#addInventory('9781497438095',5)
#addInventory('9780446310789',3)

#checkout('9780446310789')
#checkout('9780446310789')
#checkout('9781497438095')

checkin(1)