import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.dates import (DAILY, DateFormatter, rrulewrapper, RRuleLocator, drange, num2date)
import datetime
from datetime import timedelta
from scipy.optimize import curve_fit
import os

'''
Supporting Functions
'''

def sigmoid(x, L ,x0, k, b):
    y = L / (1 + np.exp(-k*(x-x0)))+b
    return (y)

def quadratic (x, a, b, h, c):
    return(a*((x-h)**2) + b*(x-h) + c)

def exponential(x, a, b, h, c):
    return(a*(b ** (x-h)) + c)

def fit(y, X, X_pred, p0_sig=None, p0_quad=None, p0_exp=None, tr_solver=None, method='trf'):
    FTOL = [1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
    ps = np.array([0,0,0,0])
    pq = np.array([0,0,0,0])
    pe = np.array([0,1,0,0])

    for tol in FTOL:
        try:
            ps, _ = curve_fit(sigmoid, X, y, p0=p0_sig, ftol=tol, method=method)
            break
        except:
            continue

    for tol in FTOL:
        try:
            pq, _ = curve_fit(quadratic, X, y, p0=p0_quad, ftol=tol, method=method, bounds=([0, -np.inf, -np.inf, -np.inf], np.inf))
            break
        except:
            continue

    for tol in FTOL:
        try:
            pe, _ = curve_fit(exponential, X, y, p0=p0_exp, ftol=tol, method=method)
            break
        except:
            continue

    sig = sigmoid(X_pred, *ps)
    quad = quadratic(X_pred, *pq)
    expo = exponential(X_pred, *pe)

    return sig, quad, expo, ps, pq, pe

def predictCases(data, days_predicted = 35, vis=True):
    ##  Input
    #   data: confirmed cases from usafacts
    #   days_predicted: number of days to predict
    #   vis: view graphs

    ##  Output
    #   X: number of days since first case
    #   X_pred: number of days since first case including predicted days
    #   y: number of cases
    #   y_pred: number of cases predicted
    #   dates: actual dates
    #   dates_pred: actual dates including predicted daysill

    # Parameters
    learn_rate = 0.125
    w1=0.5
    w2=0.5
    z1 = w1
    z2 = w2
    learn_period = 14

    county = data['County Name'] + ', ' + data['State'] # County name
    y = np.array(data.to_list()[4:]).astype(np.float) #Number of cases
    y_orig = y
    # Create dates
    start_date = datetime.date(2020,1,22)
    end_date = start_date + timedelta(days=len(y))
    delta = datetime.timedelta(days=1)
    dates = drange(start_date, end_date, delta)
    end_date_pred = start_date + timedelta(days=len(y)+days_predicted)
    dates_pred = drange(start_date, end_date_pred, delta)
    dates_str = [item.strftime('%m/%d/%Y') for item in num2date(dates_pred) ]

    # Need to remove leading 0's and make sure there is enough data to find a meaningful curve fit
    first_case_idx = np.nonzero(y)[0][0] # index of first case
    y_first = y[first_case_idx:] #data since first case
    # Learn from N days
    if len(y_first) < learn_period:
        learn_period = 3
    y_idx_start = len(y_first)-learn_period

    # Initialize variables to sum cumulative observed error for comparing exponential vs quadratic
    quad_pred_err_tot = 0
    exp_pred_err_tot = 0

    # Loop through dates to fit and learn from
    for idx, y_idx in enumerate(np.arange(y_idx_start,len(y_first)+1)):
        y = y_first[:y_idx]
        X = np.linspace(0,len(y)-1,len(y)) #Days
        X_pred = np.linspace(0,len(y)-1+days_predicted,len(y)+days_predicted) #Days
        # Create dates

        if (idx == 0):
            p0_sig = [max(y_first), np.median(X),1,min(y_first)]
            p0_quad = None
            p0_exp = None
        else:
            p0_sig = ps
            p0_quad = pq
            p0_exp = pe

        sig, quad, exp, ps, pq, pe = fit(y, X, X_pred, p0_sig, p0_quad, p0_exp)
        y_pred_quad = w1*quad + w2*sig
        y_pred_exp = z1*exp + z2*sig

        if (y_idx < len(y_first)):
            sig_err = np.sum((y_first[y_idx:] - sig[y_idx:len(y_first)]) ** 2)
            quad_err = np.sum((y_first[y_idx:] - quad[y_idx:len(y_first)]) ** 2)
            exp_err = np.sum((y_first[y_idx:] - exp[y_idx:len(y_first)]) ** 2)

            if (idx > 2):
                quad_pred_err_tot += np.sum((y_first[y_idx:] - y_pred_quad[y_idx:len(y_first)]) ** 2)
                exp_pred_err_tot += np.sum((y_first[y_idx:] - y_pred_exp[y_idx:len(y_first)]) ** 2)

            tot_error = sig_err + quad_err
            w1_new = 1 - (quad_err/tot_error)
            w2_new = 1 - (sig_err/tot_error)

            tot_error = sig_err + exp_err
            z1_new = 1 - (exp_err/tot_error)
            z2_new = 1 - (sig_err/tot_error)

            # Save for plotting
            w1_old = np.copy(w1)
            w2_old = np.copy(w2)
            z1_old = z1
            z2_old = z2

            # Update weights
            w1 = w1 + (w1_new - w1) * learn_rate
            w2 = w2 + (w2_new - w2) * learn_rate
            z1 = z1 + (z1_new - z1) * learn_rate
            z2 = z2 + (z2_new - z2) * learn_rate

        if (vis):
            dates_pred_trunc = dates_pred[first_case_idx:first_case_idx+len(X_pred)]
            # Plot
            fig = plt.figure(figsize=(8, 6))
            ax = fig.add_subplot(1, 1, 1)
            ax.plot_date(dates[first_case_idx:first_case_idx+y_idx], y, color='#e74c3c', ls='solid', marker='o')
            try:
                ax.plot_date(dates[first_case_idx+y_idx:first_case_idx+len(y_first)], y_first[y_idx:], color='#636e72', ls='solid', marker='o')
            except:
                pass
            ax.plot_date(dates_pred_trunc, y_pred_quad, color='#00b894', ls='solid',marker="")
            ax.plot_date(dates_pred_trunc, y_pred_exp, color='#2d3436', ls='solid',marker="")
            ax.plot_date(dates_pred_trunc, sig, color='#0984e3', ls='dashed', marker="")
            ax.plot_date(dates_pred_trunc, quad, color='#6c5ce7', ls='dashed', marker="")
            ax.plot_date(dates_pred_trunc, exp, color='#d63031', ls='dashed', marker="")

            w1r = np.round(w1_old, decimals=2)
            w2r = np.round(w2_old, decimals=2)
            z1r = np.round(z1_old, decimals=2)
            z2r = np.round(z2_old, decimals=2)

            # Format plot
            formatter = DateFormatter('%m/%d')
            rule = rrulewrapper(DAILY, interval=7)
            loc = RRuleLocator(rule)
            ax.xaxis.set_major_locator(loc)
            ax.set_xlabel('Days')
            ax.set_ylabel('Confirmed Cases')
            ax.set_title(county + '-- Q: ' + str(w1r) + ' L1: ' + str(w2r) + ' E: ' + str(z1r) + ' L2: ' + str(z2r))
            ax.xaxis.set_major_formatter(formatter)
            ax.xaxis.set_tick_params(rotation=30, labelsize=8)
            ax.legend(['True [Training]', 'True [Testing]', 'Predicted-Quad.', 'Predicted-Exp.','Logistic', 'Quadratic', 'Exponential'])
            ax.set_ylim([0, np.max(y_first) * 1.5])

    if (quad_pred_err_tot < exp_pred_err_tot):
        y_pred = y_pred_quad
        # growth = 'quadratic'
    else:
        y_pred = y_pred_exp
        # growth = 'exponential'

    pad_width = len(y_orig)- len(y)
    y = np.pad(y, (pad_width,0), 'constant')
    y_pred = np.pad(y_pred, (pad_width,0), 'constant')
    y_pred[y_pred<0] = 0
    pad_width = len(y_pred)- len(y)
    y = np.pad(y, (0,pad_width), 'constant', constant_values=(-1,-1))

    X = np.linspace(0,len(y)-1,len(y)) #Days
    X_pred = np.linspace(0,len(y)-1+days_predicted,len(y)+days_predicted) #Days
    # Create dictionary output
    case_output = {
            # 'X': X,
            # 'X_pred': X_pred,
            'y': y,
            'y_pred': y_pred,
            # 'dates': dates,
            # 'dates_pred':dates_pred,
            'dates_str': dates_str,
            }
    return(case_output)

def check_sufficient_data(county_data):
    enough_data = True
    y = np.array(county_data.to_list()[4:]).astype(np.float) #Number of cases
    ydif = np.diff(y, prepend=[0])
    if(len(np.nonzero(ydif)[0]) < 10):
        enough_data = False
    return(enough_data)

def read_data(file_path='./data/covid_confirmed_usafacts.csv', county_file_path='./data/county_data.csv'):
    data = pd.read_csv(file_path,delimiter=',',converters={'countyFIPS': str})

    data = data.sort_values(by=data.columns[-1], ascending=False)
    data = data.reset_index(drop=True)
    data['countyFIPS'] = data['countyFIPS'].apply(lambda x: x.zfill(5))
    county_data = pd.read_csv(county_file_path,delimiter=',')
    county_data['GEO_ID'] = county_data['GEO_ID'].str.strip().str[-5:]

    return(data, county_data)

##############################################################################
data, county_data = read_data(file_path='./data/usafacts_cases.txt')
data = data[data['countyFIPS'] != '00000']
data = data[data['countyFIPS'] != '00001']

output=[]
vis = False

# import time
# start = time.time()

data_fips = data['countyFIPS'].tolist()
# data_fips = ['36081', '36061', '48023']
# data_fips = ['05081']
# data_fips = ['17031']

for idx, code in enumerate(data_fips):
    print(idx)
    row = data[data['countyFIPS']==code].iloc[0]
    if (check_sufficient_data(row)):
        result = predictCases(row, days_predicted = 28, vis=vis)
        df = pd.DataFrame(result)
        df['countyFIPS'] = row['countyFIPS']
        df['County Name'] = row['County Name']
        df['State'] = row['State']
        df['stateFIPS'] = row['stateFIPS']
    else:
        dates_str = result['dates_str']
        y_pred = np.zeros(len(dates_str))
        y = np.array(row.to_list()[4:]).astype(np.float) #Number of cases
        pad_width = len(y_pred)- len(y)
        y = np.pad(y, (0,pad_width), 'constant', constant_values=(-1,-1))
        df = pd.DataFrame({'y': y, 'y_pred': y_pred, 'dates_str': dates_str})
        df['countyFIPS'] = row['countyFIPS']
        df['County Name'] = row['County Name']
        df['State'] = row['State']
        df['stateFIPS'] = row['stateFIPS']
    if (idx==0):
        df_agg = df
    else:
        df_agg = pd.concat([df_agg,df])

data_fips = pd.unique(data['State']).tolist()

for idx, code in enumerate(data_fips):
    print(idx)
    temp = data[data['State']==code]
    row = temp.sum()
    row['countyFIPS'] = temp['stateFIPS'].iloc[0]
    row['County Name'] = temp['State'].iloc[0]
    row['State'] = temp['State'].iloc[0]
    row['stateFIPS'] = temp['stateFIPS'].iloc[0]

    if (check_sufficient_data(row)):
        result = predictCases(row, days_predicted = 28, vis=vis)
        df = pd.DataFrame(result)
        df['countyFIPS'] = row['countyFIPS']
        df['County Name'] = row['County Name']
        df['State'] = row['State']
        df['stateFIPS'] = row['stateFIPS']

    df_agg = pd.concat([df_agg,df])

# end = time.time()
# total = end-start
df_agg.to_csv('./data/predicted_cases.csv',index=False)


import pyqrcode
import png
from pyqrcode import QRCode

# Links which represent the QR codes
link1 = "https://www.linkedin.com/in/sinchan-panda-aab615212/"
link2 = "https://github.com/sinchan1999"

# Representing my dummy details in QR code
studentDetails = "Name: SINCHAN PANDA\nRoll: 1704522 \n Branch: EEE\n KIIT UNIVERSITY, Bhubaneswar"

# Generating QR codes
url1 = pyqrcode.create(link1)
url2 = pyqrcode.create(link2)

info = pyqrcode.create(studentDetails)

# Saving the png file naming "myProfile.png"
url1.png("myProfile.png", scale=4)
url2.png("myGithubProfile.png", scale=4)

info.png("studentDetails.png", scale=4)


"""
implementation of run length encoding
"""

from bit_io import BitIO

class RunLength(object):
    """class for run-length encoding"""

    JUMP_MAP = {
        0: 1,
        1: 0
    }

    def __init__(self, in_name, out_name):
        """initialization"""

        self.max_count = 256
        self.bit_lenth = 8

        self.in_name = in_name
        self.out_name = out_name

    def compress(self):
        """compress"""

        curr_bit = 0
        count = 0

        with BitIO(self.out_name, "w") as bo:
            with BitIO(self.in_name, "r") as bi:

                while not bi.is_empty():

                    if bi.read_bits(1) == curr_bit:
                        count += 1

                        # check if max count reached
                        if count + 1 >= self.max_count:
                            bo.write_char(count)
                            curr_bit = self.JUMP_MAP[curr_bit]
                            count = 0
                    else:
                        # flush count on bit change
                        bo.write_char(count)
                        curr_bit = self.JUMP_MAP[curr_bit]

                        # count for current value
                        count = 1

    def expand(self):
        """compress"""

        curr_bit = 0

        with BitIO(self.out_name, "w") as bo:
            with BitIO(self.in_name, "r") as bi:

                while not bi.is_empty():
                    # read 8-bit count from standard input
                    run = bi.read_char()

                    # write repeated bits
                    for _ in xrange(run):
                        bo.write_bits(1, curr_bit)

                    # toggle bit
                    curr_bit = self.JUMP_MAP[curr_bit]

if __name__ == "__main__":

    import sys

    in_file = sys.argv[1]
    out_file = "%s.bin" % in_file

    co = RunLength(in_file, out_file)
    co.compress()


# main program of NoteSide

import PySimpleGUI as sg
from task_list import task_list as t_list

new_list = t_list()

window = sg.Window("NoteSide", new_list.layout, use_default_focus=False, font="15")

while True:
    event, values = window.read()

    if event == sg.WINDOW_CLOSED:
        break
    if event == "-ADD-":
        task = values["-INPUT-"]
        if task != "":
            new_list.addTsk(task)
            window["-ERRORMSG-"].update("")
            window.extend_layout(window["-TASK_VIEW-"], [[sg.Checkbox(task)]])
            window["-INPUT-"].update('')
        else:
            window["-ERRORMSG-"].update("Task cannot be empty")

window.close()


try:
    from nrgpy import logger
except ImportError:
    pass
from datetime import datetime
from nrgpy.utils.utilities import string_date_check, draw_progress_bar
from .auth import nrg_api, upload_url
import os
import requests

class nrg_api_upload(nrg_api):
    def __init__(
        self,
        client_id="",
        client_secret="",
        filename="",
        rld_dir="",
        site_filter="",
        site_filter2="",
        start_date="1970-01-01",
        end_date="2150-12-31",
    ):

        super().__init__(client_id, client_secret)

        self.filename = filename
        self.rld_dir = rld_dir
        self.site_filter = site_filter
        self.site_filter2 = site_filter2
        self.start_date = start_date
        self.end_date = end_date
        self.headers = {"Authorization": "Bearer {}".format(self.session_token)}

        if filename:
            self.pad = 1
            self.counter = 1
            self.raw_count = 1
            self.progress_bar = False
            self.start_time = datetime.now()
            self.upload_file()

        if rld_dir:
            self.upload_directory()

    def upload_file(self):
        if self.progress_bar:
            draw_progress_bar(self.counter, self.raw_count, self.start_time)
        elif self.raw_count == 1:
            print(
                "{0} | API | uploading {1} ... ".format(
                    datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                    os.path.basename(self.filename),
                ),
                end="",
                flush=True,
            )
        else:
            print(
                "{0} | API | uploading {1}/{2} ... {3} ... ".format(
                    datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                    str(self.counter).rjust(self.pad),
                    str(self.raw_count).ljust(self.pad),
                    os.path.basename(self.filename),
                ),
                end="",
                flush=True,
            )

        self.encoded_rld_bytes = self.prepare_file_bytes(self.filename)

        data = {"type": self.filename[-3:].upper(), "filebytes": self.encoded_rld_bytes}

        self.response = requests.request(
            "POST", upload_url, headers=self.headers, data=data
        )

        if self.progress_bar is False:
            if int(self.response.status_code) < 300:
                print("[OK]")
            else:
                print(f"[FAILED] {self.response.status_code}")

    def upload_directory(self, progress_bar=True):
        self.progress_bar = progress_bar
        self.start_time = datetime.now()

        self.files = [
            f
            for f in sorted(os.listdir(self.rld_dir))
            if self.site_filter in f
            and self.site_filter2 in f
            and f.lower().endswith(("rwd", "rld"))
            and string_date_check(self.start_date, self.end_date, f)
        ]

        self.raw_count = len(self.files)
        self.pad = len(str(self.raw_count)) + 1
        self.counter = 1

        for rld in self.files:
            self.filename = os.path.join(self.rld_dir, rld)
            self.upload_file()
            self.counter += 1


#data and feature determines limit of ML
#model and algorithm approaches this limit

import gc
import numpy as np
import pandas as pd
import re
import sklearn
import seaborn as sns
import matplotlib.pyplot as plt
#matplotlib inline

import warnings
warnings.filterwarnings('ignore')

from collections import Counter

from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve

sns.set(style='white', context='notebook', palette='deep')

# load data
train = pd.read_csv('Variablity Analysis.csv')
test = pd.read_csv("test.csv")
IDtest = test["Sample"]

# Show the head of the table
train.head()

# look up whole data
# print(train.info())

# outlier detection
def detect_outliers(df, n, features):
    outlier_indices = []

    # iterate over features(columns)
    for col in features:
        # 1st quartile (25%)
        Q1 = np.percentile(df[col], 25)
        # 3rd quartile (75%)
        Q3 = np.percentile(df[col], 75)
        # Interquartile range (IQR)
        IQR = Q3 - Q1

        # outlier step
        outlier_step = 1.5 * IQR

        # Determine a list of indices of outliers for feature col
        outlier_list_col = df[(df[col] < Q1 - outlier_step) | (df[col] > Q3 + outlier_step)].index

        # append the found outlier indices for col to the list of outlier indices
        outlier_indices.extend(outlier_list_col)

        # select observations containing more than 2 outliers
        outlier_indices = Counter(outlier_indices)
        multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)

        return multiple_outliers

        # detect outliers from gender, smoking status, BMI, age
        Outliers_to_drop = detect_outliers(train, 2, ["BMI", "Age"])

        train.loc[Outliers_to_drop] # Show the outliers rows

        # Drop outliers
        train = train.drop(Outliers_to_drop, axis=0).reset_index(drop=True)

# joining train set
train_len = len(train)
dataset =  pd.concat(objs=[train, test], axis=0).reset_index(drop=True)

# check for null and missing values
# Fill empty and NaNs values with NaN
dataset = dataset.fillna(np.nan)

# Check for Null values
dataset.isnull().sum()

# Infos
#train.info()
train.isnull().sum()
train.head()
train.dtypes
### Summarize data
# Summarie and statistics
train.describe()

# Feature analysis
####### numerical values
# Correlation matrix between numerical values (age, BMI) and variance
g = sns.heatmap(train[["Variance","BMI","Age"]].corr(),annot=True, fmt = ".2f", cmap = "coolwarm")
#plt.show()
#age has more significative correlation with variance

# BMI feature vs variance
g = sns.factorplot(x="BMI",y="Variance",data=train,kind="bar", size = 8 ,
palette = "muted")
g.despine(left=True)
g = g.set_ylabels("variance probability")
#plt.show()
#23.2,23.8 and 24.4 are first three have high variance

# Age feature vs variance
g  = sns.factorplot(x="Age",y="Variance",data=train,kind="bar", size = 8 ,
palette = "muted")
g.despine(left=True)
g = g.set_ylabels("variance probability")
#plt.show()
#66,45,22 are first three high values

###### categorical values
# gender vs variance
g = sns.barplot(x="Gender",y="Variance",data=train)
g = g.set_ylabel("variance Probability")
#plt.show()
# Male have higher variance, not significant
train[["Gender","Variance"]].groupby('Gender').mean()
# F       0.517862
# M       0.604516

# smoking status vs variance
g = sns.factorplot(x="Smoking Status",y="Variance",data=train,kind="bar", size = 8 ,
palette = "muted")
g.despine(left=True)
g = g.set_ylabels("variance probability")
#plt.show()
# smoking status vs variance by gender
g = sns.factorplot(x="Smoking Status", y="Variance", hue="Gender", data=train,
                   size=8, kind="bar", palette="muted")
g.despine(left=True)
g = g.set_ylabels("variance probability")
#plt.show()

# no need for filling missing values
# dataset is small, check there is all full

# convert variables in dataset all numbers
# gender
dataset["Gender"].describe()
dataset["Gender"].isnull().sum()
dataset["Gender"][dataset["Gender"].notnull()].head()
g = sns.countplot(dataset["Gender"],order=['F','M'])
dataset = pd.get_dummies(dataset, columns = ["Gender"],prefix="Gender")

# smoking status
dataset["Smoking Status"].describe()
dataset["Smoking Status"].isnull().sum()
dataset["Smoking Status"][dataset["Smoking Status"].notnull()].head()
g = sns.countplot(dataset["Smoking Status"],order=['Non-Smoker',  'Smoker', 'Ex-Smoker'])
dataset = pd.get_dummies(dataset, columns = ["Smoking Status"],prefix="Smoking Status")

# Drop Sample variable
dataset.drop(labels = ["Sample"], axis = 1, inplace = True)

#plt.show()
#print(dataset)

### Modeling
# separate train and test dataset
train = dataset[:train_len]
test = dataset[train_len:]
test.drop(labels=["Variance"],axis = 1,inplace=True)

## Separate train features and label
train["Variance"] = train["Variance"].astype(int)
Y_train = train["Variance"]
X_train = train.drop(labels = ["Variance"],axis = 1)

# Cross validate model with Kfold stratified cross val
kfold = StratifiedKFold(n_splits=7)

# choose the SVC and AdaBoostclassifiers for the ensemble modeling.
# performed a grid search optimization for AdaBoost, and SVC classifiers

### META MODELING  WITH ADABOOST, RF, EXTRATREES and GRADIENTBOOSTING

######### 1. Adaboost
DTC = DecisionTreeClassifier()

#7 or 8?
adaDTC = AdaBoostClassifier(DTC, random_state=7)

ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
              "base_estimator__splitter" :   ["best", "random"],
              "algorithm" : ["SAMME","SAMME.R"],
              "n_estimators" :[1, 2],
              "learning_rate":  [ 0.1, 0.2, 0.3]}  #0.0001, 0.001, 0.01,    1.5

#n_jobs = 4 why not????
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 1, verbose = 1)

gsadaDTC.fit(X_train,Y_train)

ada_best = gsadaDTC.best_estimator_

#gsadaDTC.best_score_

########### 2. ExtraTrees
ExtC = ExtraTreesClassifier()

## Search grid for optimal parameters
ex_param_grid = {"max_depth": [None],
              "max_features": [1, 3, 5],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False],
              "n_estimators" :[100,300],
              "criterion": ["gini"]}

gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 1, verbose = 1)

gsExtC.fit(X_train,Y_train)

ExtC_best = gsExtC.best_estimator_

######## 3. RFC Parameters tunning
RFC = RandomForestClassifier()

## Search grid for optimal parameters
rf_param_grid = {"max_depth": [None],
              "max_features": [1, 3, 5],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False],
              "n_estimators" :[100,300],
              "criterion": ["gini"]}

gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 1, verbose = 1)

gsRFC.fit(X_train,Y_train)

RFC_best = gsRFC.best_estimator_

######### 4. Gradient boosting tunning
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
              'n_estimators' : [100,200,300],
              'learning_rate': [0.1, 0.05, 0.01],
              'max_depth': [4, 8],
              'min_samples_leaf': [100,150],
              'max_features': [0.3, 0.1]
              }

gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 1, verbose = 1)

gsGBC.fit(X_train,Y_train)

GBC_best = gsGBC.best_estimator_

############### special: SVC classifier
SVMC = SVC(probability=True)
svc_param_grid = {'kernel': ['rbf'],
                  'gamma': [ 0.01, 0.1, 1],
                  'C': [1, 10, 50, 100]}

gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 1, verbose = 1)

gsSVMC.fit(X_train,Y_train)

SVMC_best = gsSVMC.best_estimator_

## feature importance of tree based classifiers
# In order to see the most informative features for the prediction
nrows = ncols = 2
fig, axes = plt.subplots(nrows = nrows, ncols = ncols, sharex="all", figsize=(15,15))
names_classifiers = [("AdaBoosting", ada_best),("ExtraTrees",ExtC_best),("RandomForest",RFC_best),("GradientBoosting",GBC_best)]

nclassifier = 0
for row in range(nrows):
    for col in range(ncols):
        name = names_classifiers[nclassifier][0]
        classifier = names_classifiers[nclassifier][1]
        indices = np.argsort(classifier.feature_importances_)[::-1][:40]
        g = sns.barplot(y=X_train.columns[indices][:40],x = classifier.feature_importances_[indices][:40] , orient='h',ax=axes[row][col])
        g.set_xlabel("Relative importance",fontsize=12)
        g.set_ylabel("Features",fontsize=12)
        g.tick_params(labelsize=9)
        g.set_title(name + " feature importance")
        nclassifier += 1

#plt.show()

# Ensemble modeling
#######combining models
votingC = VotingClassifier(estimators=[('rfc', RFC_best), ('extc', ExtC_best),
('svc', SVMC_best), ('adac',ada_best),('gbc',GBC_best)], voting='soft', n_jobs=1)

votingC = votingC.fit(X_train, Y_train)

#print(votingC)

#########predict and submit results
test_Variance = pd.Series(votingC.predict(test), name="Variance")

results = pd.concat([IDtest,test_Variance],axis=1)

results.to_csv("ensemble_python_voting.csv",index=False)


"""
https://www.acmicpc.net/problem/1173
"""
N, m, M, T, R = map(int,input().split())
ex = 0 # 운동
t = 0 # 시간
now = m
while ex < N:
    if m+T > M:
        break
    if now + T <= M:
        now += T
        ex += 1
    else:
        now = max(now-R, m)
    t += 1
print(t if ex == N else -1)

심심풀이 브론즈/1173.운동.py


from torchvision import transforms, datasets
from torch.utils.data import DataLoader
import torch
from PIL import Image
import numpy as np

def load_data(data_directory):
    '''
    Load and preprocess the image data from the specified directory.

    Args:
    data_directory (str): The root directory containing 'train', 'valid', and 'test' subdirectories.

    Returns:
    tuple: A tuple containing train_loader, valid_loader, test_loader, and train_data.
    '''
    train_dir = data_directory + '/train'
    valid_dir = data_directory + '/valid'
    test_dir = data_directory + '/test'
    # Define your transforms for the training, validation, and testing sets
    train_transforms = transforms.Compose([
        transforms.RandomRotation(30),
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    valid_transforms = transforms.Compose([
        transforms.Resize(255),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    test_transforms = transforms.Compose([
        transforms.Resize(255),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    # Load the datasets with ImageFolder
    train_data = datasets.ImageFolder(train_dir, transform=train_transforms)
    valid_data = datasets.ImageFolder(valid_dir, transform=valid_transforms)
    test_data = datasets.ImageFolder(test_dir, transform=test_transforms)

    # Define the dataloaders
    train_loader = torch.utils.data.DataLoader(train_data, batch_size=64, shuffle=True)
    valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=64)
    test_loader = torch.utils.data.DataLoader(test_data, batch_size=64)

    return train_loader, valid_loader, test_loader, train_data


#! /usr/bin/env python
import json
import tensorflow as tf
import numpy as np
import os
import time
import datetime
import data_helpers
from text_cnn import TextCNN
from tensorflow.contrib import learn
import csv

# Parameters
# ==================================================

# Eval Parameters
tf.flags.DEFINE_integer("batch_size", 64, "Batch Size (default: 64)")
tf.flags.DEFINE_string("checkpoint_dir", "", "Checkpoint directory from training run")
tf.flags.DEFINE_boolean("eval_train", False, "Evaluate on all training data")

# Misc Parameters
tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")

FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
    print("{}={}".format(attr.upper(), value))
print("")

x_raw, project_id = data_helpers.load_projects()

# x_raw = x_raw[:64]
# project_id = project_id[:64]

# Map data into vocabulary
vocab_path = os.path.join(FLAGS.checkpoint_dir, "..", "vocab")
vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
x_test = np.array(list(vocab_processor.transform(x_raw)))

print("\nEvaluating...\n")

# Evaluation
# ==================================================
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
graph = tf.Graph()
with graph.as_default():
    session_conf = tf.ConfigProto(
      allow_soft_placement=FLAGS.allow_soft_placement,
      log_device_placement=FLAGS.log_device_placement)
    sess = tf.Session(config=session_conf)
    with sess.as_default():
        # Load the saved meta graph and restore variables
        saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
        saver.restore(sess, checkpoint_file)

        # Get the placeholders from the graph by name
        input_x = graph.get_operation_by_name("input_x").outputs[0]
        # input_y = graph.get_operation_by_name("input_y").outputs[0]
        dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]

        # Tensors we want to evaluate
        predictions = graph.get_operation_by_name("output/predictions").outputs[0]
        scores = graph.get_operation_by_name("output/scores").outputs[0]

        # Generate batches for one epoch
        batches = data_helpers.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False)

        # Collect the predictions here
        all_predictions = []

        count = 0
        for x_test_batch in batches:
            batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
            batch_score = sess.run(scores, {input_x: x_test_batch, dropout_keep_prob: 1.0})
            for idx, score in enumerate(batch_score):
                if all(i < 1 for i in score):
                    batch_predictions[idx] = -1
            all_predictions = np.concatenate([all_predictions, batch_predictions])
output = [{'description': x_raw[i], 'category' : all_predictions[i], 'id' : project_id[i]} for i in range(len(x_raw))]

# Save the evaluation to a csv
# predictions_human_readable = np.column_stack((np.array(x_raw), all_predictions))
out_path = os.path.join(FLAGS.checkpoint_dir, "..", "prediction.json")
print("Saving evaluation to {0}".format(out_path))

with open(out_path, 'w') as f:
    json.dump(output, f)


from aoc.utils import get_input
from aoc.aoc2020.days.day9.part_1 import solve_part_one
from typing import List

def numbers_cont_sum(weak_number: int, numbers: List[int]):
    cont_numbers_list = []

    for number in numbers:
        new_cont = [cont_nums + [number]
                    for cont_nums in cont_numbers_list
                    if sum(cont_nums) + number <= weak_number]
        cont_numbers_list = new_cont + [[number]]
        for cont_nums in new_cont:
            if sum(cont_nums) == weak_number:
                return cont_nums

    return cont_numbers_list

def solve_part_two(numbers_raw: str, preamble: int) -> int:
    weak_number = solve_part_one(numbers_raw, preamble)
    numbers = [int(number) for number in numbers_raw.splitlines()]
    numbers_before = numbers[:numbers.index(weak_number)]
    cont_num_sum = numbers_cont_sum(weak_number, numbers_before)
    return min(cont_num_sum) + max(cont_num_sum)

if __name__ == "__main__":
    numbers_raw = get_input(2020, 9)
    result = solve_part_two(numbers_raw, 25)
    print(result)


import os
import speech_recognition as sr
import moviepy.editor as mp
from pydub import AudioSegment
from pydub.silence import split_on_silence

STAGED_FILE_DIR = 'interviews/staged/'
COMPLETED_FILE_DIR = 'interviews/completed/'

def main():

    global STAGED_FILE_DIR
    global COMPLETED_FILE_DIR

    for mp4 in os.listdir(STAGED_FILE_DIR):
        filename = mp4.split('.')[0]
        print(filename)

        # check if output folder already exists, if not, create it
        if not os.path.exists(COMPLETED_FILE_DIR + filename):
            os.mkdir(COMPLETED_FILE_DIR + filename)
            print('new directory created successfully')
        else:
            print(f'{mp4} has already been parsed! skipping')
            continue

        # convert mp4 to wav
        clip = mp.VideoFileClip(STAGED_FILE_DIR + mp4)
        clip.audio.write_audiofile(COMPLETED_FILE_DIR + filename + '/converted.wav')
        print('.wav file written')

        # chunk and parse wav into text
        text = get_large_audio_transcription(COMPLETED_FILE_DIR + filename + '/converted.wav', r)

        # exporting the result
        with open(COMPLETED_FILE_DIR + filename + '/recognized.txt', 'w') as file:
           file.write("Recognized Speech:")
           file.write("\n")
           file.write(text)
           print(f'{mp4} processed and text file written to {COMPLETED_FILE_DIR + filename + "/recognized.txt"}')

    exit()
    # remove all files from staged directory
    for mp4 in os.scandir(os.getcwd()+'/interviews/staged'):
        os.remove(mp4)

def get_large_audio_transcription(path):

    """
    Splitting the large audio file into chunks
    and apply speech recognition on each of these chunks
    """
    global COMPLETED_FILE_DIR

    r = sr.Recognizer()

    sound = AudioSegment.from_wav(path)
    print('chunking wav...')
    # splits audiofile into chunks on silences
    chunks = split_on_silence(sound,
        # experiment with this value for your target audio file
        min_silence_len = 1000,
        # adjust this per requirement (-16 default)
        silence_thresh = sound.dBFS-16,
        # keep the silence for 1 second, adjustable as well
        keep_silence=100
    )
    print(f'wav chunked into {len(chunks)} parts!')
    folder_name = "audio-chunks"
    # create a directory to store the audio chunks
    if not os.path.isdir(COMPLETED_FILE_DIR + folder_name):
        os.mkdir(COMPLETED_FILE_DIR + folder_name)
    whole_text = []
    # process each chunk
    for i, audio_chunk in enumerate(chunks, start=1):
        last_question = False
        # export audio chunk and save it in
        # the `folder_name` directory.
        chunk_filename = os.path.join(COMPLETED_FILE_DIR, folder_name, f"chunk{i}.wav")
        audio_chunk.export(chunk_filename, format="wav")
        # recognize the chunk
        with sr.AudioFile(chunk_filename) as source:
            audio_listened = r.record(source)
            # try converting it to text
            try:
                text = r.recognize_google(audio_listened)
            except sr.UnknownValueError as e:
                print("Error:", str(e))
                if not last_question:
                    whole_text.append('\n(?)\n')
                    last_question = True
            else:
                last_question = False
                text = f"{text.capitalize()}. "
                print(f'parsing : chunk {i}/{len(chunks)} ... {text}')
                whole_text.append(text.strip())

    # return the text for all chunks detected
    return ' '.join(whole_text)

if __name__ == '__main__':
    main()


#!/usr/bin/env python

import os
import tensorflow as tf
from svd.model import SVDModel
from svd.read_data import read_batch

# config
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_float('learning_rate', 0.01, 'learning rate')
flags.DEFINE_float('lam', 0.01, 'regularization for overcoming overfitting')
flags.DEFINE_integer('max_epoch', 100, ' max train epochs')
flags.DEFINE_integer("batch_size", 100, "batch size for sgd")
flags.DEFINE_integer("valid_batch_size", 100, "validate set batch size")
flags.DEFINE_integer("thread_num", 1, "number of thread to read data")
flags.DEFINE_integer("min_after_dequeue", 100, "min_after_dequeue for shuffle queue")
flags.DEFINE_string("checkpoint_dir", "./checkpoint/", "checkpoint dirctory")
flags.DEFINE_string("tensorboard_dir", "./tensorboard/", "summary data saved for tensorboard")
flags.DEFINE_string("optimizer", "adagrad", "optimization algorithm")
flags.DEFINE_integer('steps_to_validate', 1, 'steps to validate and print')
flags.DEFINE_bool("train_from_checkpoint", False, "reload model from checkpoint and go on training")

if not os.path.exists(FLAGS.checkpoint_dir):
    os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.tensorboard_dir):
    os.makedirs(FLAGS.tensorboard_dir)

# read train data
train_userid, train_itemid, train_rating = read_batch("../data/svd_data/train.data.tfrecord",
                                                FLAGS.max_epoch,
                                                FLAGS.batch_size,
                                                FLAGS.thread_num,
                                                FLAGS.min_after_dequeue)
# read validate data
valid_userid, valid_itemid, valid_rating = read_batch("../data/svd_data/test.data.tfrecord",
                                                FLAGS.max_epoch,
                                                FLAGS.batch_size,
                                                FLAGS.thread_num,
                                                FLAGS.min_after_dequeue)

# define model
max_userid = 671
max_itemid = 163949
embedding_num = 10
model = SVDModel(max_userid+1, max_itemid+1, embedding_num)

# define loss
preference, user_embedding, item_embedding = model.forward(train_userid, train_itemid)
rmse = tf.nn.l2_loss(tf.sub(preference, train_rating))
regularization = tf.add(tf.nn.l2_loss(user_embedding), tf.nn.l2_loss(item_embedding))
lam = tf.constant(FLAGS.lam, dtype=tf.float32, shape=[])
loss = rmse + tf.mul(regularization, lam)

# define optimizer
print("Optimization algorithm: {}".format(FLAGS.optimizer))
if FLAGS.optimizer == "sgd":
    optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
elif FLAGS.optimizer == "adadelta":
    optimizer = tf.train.AdadeltaOptimizer(FLAGS.learning_rate)
elif FLAGS.optimizer == "adagrad":
    optimizer = tf.train.AdagradOptimizer(FLAGS.learning_rate)
elif FLAGS.optimizer == "adam":
    optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
elif FLAGS.optimizer == "ftrl":
    optimizer = tf.train.FtrlOptimizer(FLAGS.learning_rate)
elif FLAGS.optimizer == "rmsprop":
    optimizer = tf.train.RMSPropOptimizer(FLAGS.learning_rate)
else:
    print("Error: unknown optimizer: {}".format(FLAGS.optimizer))
    exit(1)
with tf.device("/cpu:0"):
    global_step = tf.Variable(0, name='global_step', trainable=False)
train_op = optimizer.minimize(loss, global_step=global_step)

# eval rmse
tf.get_variable_scope().reuse_variables()
valid_preference, _, _ = model.forward(valid_userid, valid_itemid)
valid_rmse = tf.nn.l2_loss(tf.sub(valid_preference, valid_rating))

# checkpoint
checkpoint_file = FLAGS.checkpoint_dir + "/checkpoint"
saver = tf.train.Saver()

# summary
tf.scalar_summary('loss', loss)
tf.scalar_summary('rmse', valid_rmse)
summary_op = tf.merge_all_summaries()

# train loop
with tf.Session() as sess:
    init_op = tf.initialize_all_variables()
    writer = tf.train.SummaryWriter(FLAGS.tensorboard_dir, sess.graph)
    sess.run(init_op)
    sess.run(tf.initialize_local_variables())

    if FLAGS.train_from_checkpoint:
        checkpoint_state = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
        if checkpoint_state and checkpoint_state.model_checkpoint_path:
            print("Continue training from checkpoint {}".format(checkpoint_state.model_checkpoint_path))
            saver.restore(sess, checkpoint_state.model_checkpoint_path)

    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)
    try:
        while not coord.should_stop():
            _, loss_value, step = sess.run([train_op, loss, global_step])
            if step % FLAGS.steps_to_validate == 0:
                valid_rmse_value, summary_value = sess.run([valid_rmse, summary_op])
                print("Step: {}, loss: {}, rmse: {}".format(
                    step, loss_value, valid_rmse_value))
                writer.add_summary(summary_value, step)
                saver.save(sess, checkpoint_file, global_step=step)
    except tf.errors.OutOfRangeError:
        print("training done")
    finally:
        coord.request_stop()

    # wait for threads to exit
    coord.join(threads)
    sess.close()


# have a look of the flowers
from IPython.display import Image
Setosa = 'http://upload.wikimedia.org/wikipedia/commons/5/56/Kosaciec_szczecinkowaty_Iris_setosa.jpg'
Versicolor = 'http://upload.wikimedia.org/wikipedia/commons/4/41/Iris_versicolor_3.jpg'
Virginica = 'http://upload.wikimedia.org/wikipedia/commons/9/9f/Iris_virginica.jpg'
Image(Setosa,width=300,height=300)
Image(Versicolor,width=300,height=300)
Image(Virginica,width=300,height=300)

import seaborn as sns
iris = sns.load_dataset('iris')
iris.head()
iris.info()
iris.describe()

sns.pairplot(iris,hue='species',palette='Dark2')

setosa = iris[iris['species']=='setosa']
sns.kdeplot(setosa['sepal_width'],setosa['sepal_length'],
            cmap='plasma',shade=True,shade_lowest=False)

from sklearn.cross_validation import train_test_split
X = iris.drop('species',axis=1)
y = iris['species']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

from sklearn.svm import SVC
model = SVC()
model.fit(X_train, y_train)
pred = model.predict(X_test)

from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(y_test,pred))
print(confusion_matrix(y_test,pred))

from sklearn.grid_search import GridSearchCV
param_grid = {'C':[0.1,1,10,100,1000],'gamma':[1,0.1,0.01,0.001,0.0001]}
grid = GridSearchCV(SVC(),param_grid,verbose=3)
grid.fit(X_train,y_train)
grid.best_params_
grid.best_estimator_
grid.best_score_

pred_grid = grid.predict(X_test)
print(classification_report(y_test,pred_grid))
print(confusion_matrix(y_test,pred_grid))


from config import*
import random
from game.casting.basics.body import Body
from game.casting.basics.actor import Actor
from game.casting.basics.image import Image
from game.casting.specifics.explosion import Explosion
from game.casting.basics.animation import Animation

from game.scripting.action import Action
from game.shared.point import Point

class BulletCollideShipAction(Actor):

    def __init__(self, physics_service, sound_service):
        self._physics_service = physics_service
        self._sound_service = sound_service
        # self._message = Actor()

    def execute(self, cast, script, flags):
        bullets = cast.get_actors(ALIEN_PROJECTILE_GROUP)
        if bullets == []:
            return
        ship = cast.get_first_actor(SHIP_GROUP)
        if not ship:
            return
        stats = cast.get_first_actor(STATS_GROUP)

        for bullet in bullets:
            bullet_body = bullet.get_body()
            ship_body = ship.get_body()

            if self._physics_service.has_collided(bullet_body, ship_body):
                sound_num = random.randint(0, len(EXPLOSION_SOUNDS)-1)
                sound = EXPLOSION_SOUNDS[sound_num]
                self._sound_service.play_sound(sound)

                # remove ship actor from grid, but not from cast.

                ship_position = ship_body.get_position()
                ship_x = ship_position.get_x()
                ship_y = ship_position.get_y()
                explosion_x = ship_x + (SHIP_WIDTH - EXPLOSION_WIDTH) / 2
                explosion_y = ship_y + (SHIP_HEIGHT - EXPLOSION_HEIGHT) / 2
                explosion_position = Point(explosion_x, explosion_y)
                size = Point(EXPLOSION_WIDTH, EXPLOSION_HEIGHT)
                velocity = Point(0, 0)
                body = Body(explosion_position, size, velocity)
                animation = Animation(EXPLOSION_IMAGES["explosions"])
                explosion = Explosion(body, animation)
                cast.remove_actor(SHIP_GROUP, ship)
                cast.remove_actor(ALIEN_PROJECTILE_GROUP, bullet)
                cast.add_actor(EXPLOSION_GROUP, explosion)


import re

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import *

class LanguageDetector():

    def __init__(self,classifier=MultinomialNB()):
        self.classifier = classifier
        self.vectorizer = CountVectorizer(
            lowercase=True,
            analyzer="char_wb",
            ngram_range=(1,2),
            max_features=1000,
            preprocessor=self._remove_noise
        )

    def _remove_noise(self,text):
        noise_pattern = re.compile("|".join(["http\S+","\@\w+","\#\w+","\d+"]))
        clean_text = re.sub(noise_pattern,"",text)
        return clean_text

    def features(self,X):
        return self.vectorizer.transform(X)

    def fit(self,X,y):
        self.vectorizer.fit(X)
        self.classifier.fit(self.features(X).toarray(),y)

    def predict(self,x):
        return self.classifier.predict(self.features([x]).toarray())

    def score(self,X,y):
        return self.classifier.score(self.features(X),y)

if __name__ == '__main__':
    # read data from data.csv
    data_file = open("data.csv",encoding="utf-8")
    lines = data_file.readlines()
    data_file.close()
    dataset = [(line.strip()[:-3],line.strip()[-2:]) for line in lines]
    # x:langeges_sentences
    # y:tagas
    x,y = zip(*dataset)
    x_train,x_test,y_train,y_test = train_test_split(x,y,random_state=1)
    # print(x_train[:20])

    language_dectector = LanguageDetector(classifier=MultinomialNB())
    language_dectector.fit(x_train,y_train)
    print(language_dectector.score(x_test,y_test))

    # English
    print(language_dectector.predict("This is an English sentence"))
    # German
    print(language_dectector.predict("Das ist ein deutscher Satz."))
    # French
    print(language_dectector.predict("C 'est un français."))


from . import conic, helpers, surf
import math
import json
import numpy as np

class NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        if isinstance(obj, set):
            return list(obj)
        return json.JSONEncoder.default(self, obj)

def get_sagf_from_fullspec(fullspec):
    lens = []
    for idx, val in enumerate(fullspec[0]["r"]):
        c = get_conic_from_fullspec(fullspec, idx)
        lens.append(c)
    sag_vector = np.vectorize(lens_sag, excluded=["lens"])
    return lambda RHO, PHI: sag_vector(lens=lens, r=RHO, theta=PHI)

def get_conic_from_fullspec(rows, m):
    """
    Extracts a dict describing the conic section of meridian n from our fullspec definition.

    Where rows is a dict representing:

    - r: [8]
      e: [0.0]
      d: 6.0
    - r: [9]
      e: [0.5]
      d: 10.0
    - r: [11]
      e: [1.1]
      d: 12.0
    """
    rs = [0] * len(rows)
    es = [0] * len(rows)
    hds = [0] * len(rows)
    for idx, row in enumerate(rows):
        hds[idx] = row["d"] / 2.0
        rs[idx] = row["r"][m]
        es[idx] = row["e"][m]
    offsets = conic.calc_offsets(rs, es, hds)
    output = {}
    output["hds"] = hds
    output["es"] = es
    output["rs"] = rs
    output["offsets"] = offsets
    return output

def lens_sag(lens, r, theta):
    """
    Returns the sag of the lens at a given point.

    Args:
        lens: The first parameter.
        param2: The second parameter.

    Returns:
        Sag of the lens, or nan if point is outside lens.

    """
    if r < 0:
        r = abs(r)
        theta = theta + math.pi
    if r > lens[0]["hds"][-1]:
        return np.nan
    if len(lens) == 1:
        lens_val = conic.find_val(
            lens[0]["rs"], lens[0]["es"], lens[0]["hds"], lens[0]["offsets"], r
        )
    elif len(lens) == 2:
        l1 = conic.find_val(
            lens[0]["rs"], lens[0]["es"], lens[0]["hds"], lens[0]["offsets"], r
        )
        l2 = conic.find_val(
            lens[1]["rs"], lens[1]["es"], lens[1]["hds"], lens[1]["offsets"], r
        )
        lens_val = surf.interp_sag(l1, l2, (theta))
    return lens_val

def to_fullspec(rs, es, hds):
    """
    Rotationally symmetric dump of a list of each to fullspec
    """
    output = []
    for idx in range(0, len(rs)):
        row = {"r": [rs[idx]], "e": [es[idx]], "d": hds[idx] * 2}
        output.append(row)
    return output

def get_lens_from_fullspec(rows):
    """
    Same as get_conic_from_fullspec, but gets a list
    """
    rs = [0] * len(rows)
    es = [0] * len(rows)
    hds = [0] * len(rows)

    outputs = []
    for m in range(0, len(rows[0]["r"])):
        output = {}
        for idx, row in enumerate(rows):
            hds[idx] = row["d"] / 2.0
            rs[idx] = row["r"][m]
            es[idx] = row["e"][m]
        offsets = conic.calc_offsets(rs, es, hds)
        output["hds"] = hds
        output["es"] = es
        output["rs"] = rs
        output["offsets"] = offsets
        outputs.append(output)
    return outputs


#!/usr/bin/env python

import os, sys, ntpath;

g = sys.modules[__name__]

# GIMP console path (GIMP 2 bin dir added to path)
# ================================================
GIMP_CONSOLE = "gimp-console-2.10"

# processing method
# =================
PROCESSING_METHOD = "wbrMask"

# wbrMask default options
# =======================
# HARD option removes rectagle selection from image processing
g.SKIP_PIC_HARD = True
# SOFT option uses brush tool to partially remove most of the image (for now - poor quality)
g.SKIP_PIC_SOFT = False
# GLOW effect is being applied only on foil effect mask layer
g.GLOW_ON = True

# wbrMask default config
g.THR_BLACK = 0.5     # 0-1
g.THR_WHITE = 0.95    # 0-1
g.GLOW_RADIUS = 12.4  # 1-50
g.GLOW_BRIGHT = 0.32  # 0-1
g.GLOW_SHARP = 0.7    # 0-1
g.EFFECT_OPACITY = 38 # 0-100%

# TODO: sizes are now adjusted to card size: 357x497 (will be changed to % value)
g.SKIP_PIC_HARD_POS_X = 39
g.SKIP_PIC_HARD_POS_Y = 71
g.SKIP_PIC_HARD_SIZE_X = 278
g.SKIP_PIC_HARD_SIZE_Y = 208

g.SKIP_PIC_SOFT_POS_X = 178
g.SKIP_PIC_SOFT_POS_Y = 171
g.SKIP_PIC_SOFT_SIZE = 343      # 1-10000
g.SKIP_PIC_SOFT_ARATIO = 0.3    # -20-20
g.SKIP_PIC_SOFT_HARDNESS = 0.81 # 0-1

# this method parses different card types in order to decrease foil effect influence on unwanted image parts
def setPortraitBypassParams(cardtype):
    if cardtype == "lotrtcg_character":
        g.EFFECT_OPACITY = 38 # 0-100%
        g.SKIP_PIC_HARD = True
        g.SKIP_PIC_SOFT = False
        g.SKIP_PIC_HARD_POS_X = 39
        g.SKIP_PIC_HARD_POS_Y = 71
        g.SKIP_PIC_HARD_SIZE_X = 278
        g.SKIP_PIC_HARD_SIZE_Y = 208
        g.SKIP_PIC_SOFT_POS_X = 178
        g.SKIP_PIC_SOFT_POS_Y = 171
        g.SKIP_PIC_SOFT_SIZE = 343      # 1-10000
    elif cardtype == "lotrtcg_modifier":
        g.EFFECT_OPACITY = 38 # 0-100%
        g.SKIP_PIC_HARD = True
        g.SKIP_PIC_SOFT = False
        g.SKIP_PIC_HARD_POS_X = 76
        g.SKIP_PIC_HARD_POS_Y = 57
        g.SKIP_PIC_HARD_SIZE_X = 249
        g.SKIP_PIC_HARD_SIZE_Y = 185
        g.SKIP_PIC_SOFT_POS_X = 200
        g.SKIP_PIC_SOFT_POS_Y = 148
        g.SKIP_PIC_SOFT_SIZE = 308      # 1-10000
    elif cardtype == "lotrtcg_site":
        g.SKIP_PIC_HARD = True
        g.SKIP_PIC_SOFT = False
        g.SKIP_PIC_HARD_POS_X = 46
        g.SKIP_PIC_HARD_POS_Y = 52
        g.SKIP_PIC_HARD_SIZE_X = 220
        g.SKIP_PIC_HARD_SIZE_Y = 389
        g.SKIP_PIC_SOFT_POS_X = 165
        g.SKIP_PIC_SOFT_POS_Y = 247
        g.SKIP_PIC_SOFT_SIZE = 407      # 1-10000
        g.SKIP_PIC_SOFT_ARATIO = -1.1   # -20-20
        g.SKIP_PIC_SOFT_HARDNESS = 0.69 # 0-1
    elif cardtype == "lotrtcg_ring":
        g.SKIP_PIC_HARD = False
        g.SKIP_PIC_SOFT = False
    else:
        print("setPortraitBypassParams - card type not supported: " + cardtype)

def wbrMask(fname, cardtype, outputpath):
    # load image
    card = pdb.gimp_file_load(fname, fname)

    # duplicate image in new layer
    card_WB_layer = card.layers[0].copy()
    card.add_layer(card_WB_layer, 0)

    # load effect background as new layer
    effect_layer = pdb.gimp_file_load_layer(card, "effect.png")
    card.add_layer(effect_layer, 0)

    # apply card type parameters
    setPortraitBypassParams(cardtype)

    if SKIP_PIC_HARD:
        # cut off rectangle part of the image from processing
        pdb.gimp_image_select_rectangle(card, 0, g.SKIP_PIC_HARD_POS_X, SKIP_PIC_HARD_POS_Y, g.SKIP_PIC_HARD_SIZE_X, g.SKIP_PIC_HARD_SIZE_Y)
        pdb.gimp_edit_bucket_fill(card.layers[1], 0, 0, 100, 0, FALSE, g.SKIP_PIC_HARD_POS_X, g.SKIP_PIC_HARD_POS_Y)
        pdb.gimp_selection_none(card)
    elif SKIP_PIC_SOFT:
        # use brush to partially remove picture
        # not working -> pdb.gimp_context_set_brush("2. Block 02")
        brush_name = pdb.gimp_brush_new('new')
        pdb.gimp_brushes_set_brush(brush_name)
        pdb.gimp_context_set_brush_size(g.SKIP_PIC_SOFT_SIZE)
        pdb.gimp_context_set_brush_angle(0)
        pdb.gimp_context_set_brush_spacing(10)
        pdb.gimp_context_set_brush_force(0.5)
        pdb.gimp_context_set_brush_aspect_ratio(g.SKIP_PIC_SOFT_ARATIO)
        pdb.gimp_context_set_brush_hardness(g.SKIP_PIC_SOFT_HARDNESS)
        pdb.gimp_paintbrush_default(card.layers[1], 2, [g.SKIP_PIC_SOFT_POS_X, g.SKIP_PIC_SOFT_POS_Y])

    # create white/black mask
    pdb.gimp_drawable_threshold(card.layers[1], 0, g.THR_BLACK, g.THR_WHITE)

    # load mask to an effect image
    pdb.gimp_image_select_item(card, 0, card.layers[1])
    mask = effect_layer.create_mask(ADD_SELECTION_MASK)
    card.layers[0].add_mask(mask)
    pdb.gimp_edit_copy(card.layers[1])
    floating_sel = pdb.gimp_edit_paste(card.layers[0].mask, FALSE)
    pdb.gimp_floating_sel_anchor(floating_sel)
    pdb.gimp_image_remove_layer(card, card.layers[1])

    # add glow to main effect and adjust its visibility
    if GLOW_ON:
        pdb.plug_in_softglow(card, card.layers[0], g.GLOW_RADIUS, g.GLOW_BRIGHT, g.GLOW_SHARP)
    card.layers[0].opacity = float(g.EFFECT_OPACITY)

	# file export
    card.merge_visible_layers(NORMAL_MODE)
    #pdb.file_png_save(card, card.layers[0], outputpath + ntpath.basename(fname), outputpath + ntpath.basename(fname), 0, 9, 1, 0, 0, 1, 1)
    pdb.file_png_save_defaults(card, card.layers[0], os.path.splitext(outputpath + ntpath.basename(fname))[0] + ".png", os.path.splitext(outputpath + ntpath.basename(fname))[0] + ".png")
    pdb.gimp_image_delete(card)

# GIMP auto-execution stub
if __name__ == "__main__":
    import os, sys, subprocess
    if len(sys.argv) < 4:
        print("Missing arguments! Usage: foilmaker.py <card_type> <output_dir_path> <image/images_path>")
        print("Example: python foilmaker.py lotrtcg_character C:/MyFoils/ D:/CardImages/*.jpg")
        sys.exit(-1)
    scrdir = os.path.dirname(os.path.realpath(__file__))
    scrname = os.path.splitext(os.path.basename(__file__))[0]
    outputpath = sys.argv[2]
    images = sys.argv[3:]
    applylines = ""
    for img in images:
        applylines += ";" + scrname + "." + PROCESSING_METHOD + "('" + img + "', '" + sys.argv[1] + "', '" + outputpath + "')"
    shcode = "import sys;sys.path.insert(0, '" + scrdir + "');import " + scrname + applylines
    shcode = GIMP_CONSOLE + " -idf --batch-interpreter python-fu-eval -b \"" + shcode + "\" -b \"pdb.gimp_quit(1)\""
    sys.exit(subprocess.call(shcode, shell=True))
else:
    from gimpfu import *


'''
The base class is a flappy bird game is going to be the ground that the bird is going to be flying over.
The base is going to move from right to left. The logic is going to move two images of the base from right to left to give an illusion that the base is moving.
'''

# Local imports
from constants import BASE_IMG

class Base:
    VEL = 5 # Velocity of the base
    WIDTH = BASE_IMG.get_width() # Width of the base
    IMG = BASE_IMG # Image of the base

    def __init__(self, y): # y is the y position of the base
        self.y = y # y position of the base
        self.x1 = 0 # x position of the base
        self.x2 = self.WIDTH # x position of the base

    # Method to move the base
    def move(self):
        self.x1 -= self.VEL # Move the base from right to left
        self.x2 -= self.VEL # Move the base from right to left

        # If the base is off the screen, reset the base
        if self.x1 + self.WIDTH < 0:
            self.x1 = self.x2 + self.WIDTH # Reset the base

        if self.x2 + self.WIDTH < 0:
            self.x2 = self.x1 + self.WIDTH # Reset the base

    # Method to draw the base
    def draw(self, win):
        win.blit(self.IMG, (self.x1, self.y)) # Draw the base
        win.blit(self.IMG, (self.x2, self.y)) # Draw the base again, so that the base is going to be moving from right to left


import wx
from wx import adv
import os
import os.path
import subprocess
import time
import sys
from pathlib import Path
import winreg
import kh2reg

base_dir = "./"

if getattr(sys, 'frozen', False):
    base_dir = sys._MEIPASS + "/files/"

TRAY_TOOLTIP = 'PuttyAuthSock'
TRAY_ICON_INACTIVE = os.path.join(base_dir, 'icon.png')
TRAY_ICON_ACTIVE = os.path.join(base_dir, 'icon_green.png')

PLINK_CMD = 'plink.exe -ssh -A -batch -i {} {}@{} ' \
            '"rm -rf ~/.auth_sock || 0; ln -s \"$SSH_AUTH_SOCK\" \"~/.auth_sock\"; sleep infinity&; echo $!;"'

# Thanks a lot to FogleBird for getting me started on wxpython
# https://stackoverflow.com/a/6389727/891624

def subprocess_args(include_stdout=True):
    if hasattr(subprocess, 'STARTUPINFO'):
        si = subprocess.STARTUPINFO()
        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
        env = os.environ
    else:
        si = None
        env = None

    if include_stdout:
        ret = {'stdout': subprocess.PIPE}
    else:
        ret = {}

    ret.update({'stdin': subprocess.PIPE,
                'stderr': subprocess.PIPE,
                'startupinfo': si,
                'env': env })
    return ret

def create_menu_item(menu, label, func):
    item = wx.MenuItem(menu, -1, label)
    menu.Bind(wx.EVT_MENU, func, id=item.GetId())
    menu.Append(item)
    return item

class TaskBarIcon(adv.TaskBarIcon):
    def __init__(self):
        super(TaskBarIcon, self).__init__()
        self.set_icon(TRAY_ICON_INACTIVE)
        self.active = False
        self.auth_sock_pid = None
        self.Bind(wx.adv.EVT_TASKBAR_LEFT_DOWN, self.on_left_down)

    def CreatePopupMenu(self):
        menu = wx.Menu()
        # create_menu_item(menu, 'Say Hello', self.on_hello)
        # menu.AppendSeparator()
        create_menu_item(menu, 'Exit', self.on_exit)
        return menu

    def set_icon(self, path):
        icon = wx.Icon()
        icon.LoadFile(path)
        self.SetIcon(icon, TRAY_TOOLTIP)

    def on_left_down(self, event):
        if not self.active:
            self.init_auth_sock()
        else:
            self.kill_auth_sock()

    def init_auth_sock(self):
        print("Starting auth_sock")
        self.run_auth_sock()
        self.active = True
        self.set_icon(TRAY_ICON_ACTIVE)

    def run_auth_sock(self):
        ip = get_wsl_ip()
        user = get_wsl_username()
        ppk_file = get_ppk_file()
        print("Starting agent forwarding")
        process = subprocess.Popen(PLINK_CMD.format(ppk_file, user, ip), **subprocess_args(True))
        time.sleep(1)
        self.auth_sock_pid = int(process.stdout.readline().strip())

    def kill_auth_sock(self):
        print("Killing auth sock")
        kill_pid(self.auth_sock_pid)
        print("Removing auth_sock file")
        remove_auth_sock()
        self.active = False
        self.set_icon(TRAY_ICON_INACTIVE)

    def on_exit(self, event):
        if self.active:
            self.kill_auth_sock()
        wx.CallAfter(self.Destroy)

def main():
    app = wx.App()
    TaskBarIcon()
    app.MainLoop()

def get_ssh_dir():
    home = str(Path.home())
    ssh_dir = os.path.join(home, ".ssh")
    return ssh_dir

def get_ppk_file():
    ppk_file = os.path.join(get_ssh_dir(), "putty_auth_sock.ppk")
    return ppk_file

def get_wsl_ip():
    # we need to read WSL's IP address every time it has been started since it is not static
    ip_cmd = "ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'"
    ip = subprocess.check_output(['bash', '-c', '{}'.format(ip_cmd)], **subprocess_args(False)).decode('ascii')
    print("Using IP address {}".format(ip))
    return ip.strip()

def start_wsl_ssh():
    subprocess.check_output(['wsl', '-u', 'root', '--', 'service', 'ssh', 'start'], **subprocess_args(False))

def register_known_host():
    known_host_entry = obtain_wsl_known_host_entry()
    add_putty_reg_key(known_host_entry)

def obtain_wsl_known_host_entry():
    fingerprints = subprocess.check_output(['ssh-keyscan', get_wsl_ip()], **subprocess_args(False)).decode()
    fingerprints = fingerprints.split("\n")
    for fingerprint in fingerprints:
        if 'ssh-ed25519' in fingerprint:
            return fingerprint
    print("Error: unable to obtain ssh-rsa fingerprint")

def update_known_hosts(known_host_entry):
    known_hosts_file = os.path.join(get_ssh_dir(), "known_hosts")
    with open(known_hosts_file, 'r') as f:
        known_hosts = f.read().split("\n")

    os.remove(known_hosts_file)

    # Filter out eventually outdated known_host lines that match the fingerprint
    # known_host_entry is of format "<IP> <format> <fingerprint>"
    fingerprint = known_host_entry.split(" ")[2].strip()
    known_hosts = [known_host for known_host in known_hosts if fingerprint not in known_host]
    known_hosts.append(known_host_entry)

    with open(known_hosts_file, 'w') as f:
        f.write("\n".join([known_host.strip() for known_host in known_hosts]))

def get_wsl_username():
    username = subprocess.check_output(['bash', '-c', '"whoami"'], **subprocess_args(False)).decode('ascii')
    print("Using username {}".format(username))
    return username.strip()

def remove_auth_sock():
    subprocess.check_output(['wsl', '--', 'rm', '-rf', '~/.auth_sock'], **subprocess_args(False))

def kill_pid(pid):
    subprocess.check_output(['wsl', '--', 'kill',  '{}'.format(pid)], **subprocess_args(False))

# Adds the known_host_entry to the registry
# https://git.tartarus.org/?p=simon/putty.git;a=blob;f=contrib/kh2reg.py;hb=HEAD
def add_putty_reg_key(known_host_entry):
    regkey, regval = kh2reg.handle_line(known_host_entry)
    print(regkey)
    print(regval)
    handle = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'SOFTWARE\SimonTatham\PuTTY\SshHostKeys', 0, winreg.KEY_ALL_ACCESS)
    winreg.SetValueEx(handle, regkey, 0, winreg.REG_SZ, regval)
    winreg.CloseKey(handle)

if __name__ == '__main__':
    start_wsl_ssh()
    register_known_host()
    main()


import requests
import json
import os

access_key = os.getenv('ACCESS_KEY')
base_url = 'https://api.unsplash.com/'

def download_image(url, file_path, output):
    response = requests.get(url)
    with open(output + '/' + file_path, 'wb') as file:
        file.write(response.content)

def search_and_download(query, count, output):
    search_url = base_url + 'search/photos'
    headers = {'Authorization': 'Client-ID ' + access_key}
    params = {'query': query, 'per_page': count}
    response = requests.get(search_url, headers=headers, params=params)
    data = json.loads(response.text)
    # print(data)

    for i, photo in enumerate(data['results']):
        image_url = photo['urls']['regular']
        file_path = f'image_{i}.jpg'
        download_image(image_url, file_path, output)

def main():
    query = input("Enter your search query: ")
    count = int(input("Enter the number of images to download: "))

    cur_dir = os.getcwd()
    output = cur_dir + f'/{query}'
    if not os.path.exists(output):
        os.mkdir(output)

    search_and_download(query, count, output)

if __name__ == '__main__':
    main()


"""
Write a function that takes directory path, a file extension and an optional tokenizer.
It will count lines in all files with that extension if there are no tokenizer.
If a the tokenizer is not none, it will count tokens.

For dir with two files from hw1.py:
# >>> universal_file_counter(test_dir, "txt")
6
# >>> universal_file_counter(test_dir, "txt", str.split)
6

"""
import os
from pathlib import Path
from typing import Callable, Optional

DIR_PATH = Path(__file__).parent

def count_len(tokens):
    try:
        if isinstance(tokens, str):
            return 1
        return len(tokens)
    except TypeError:
        return 1

def universal_file_counter(dir_path: Path, file_extension: str, tokenizer: Optional[Callable] = None) -> int:
    counter = 0

    for file in os.listdir(dir_path):
        if file.endswith(file_extension):
            with open(dir_path / file) as file_read:
                if tokenizer is None:
                    counter += len(file_read.readlines())
                else:
                    counter += sum(count_len(tokenizer(line)) for line in file_read.readlines())

    return counter


from .BaseModel import BaseModel
from ...utils import mic_acc_cal, shot_acc, weighted_mic_acc_cal, weighted_shot_acc, class_count, F_measure
from ...utils.logger import Logger

import torch
import torch.nn.functional as F

import os
import copy
import pickle
import numpy as np
from tqdm import tqdm

class CVModel(BaseModel):

    def __init__(
            self,
            model_name: str,
            dataset_name: str,
            base_dir: str = '../../',
            test_mode: bool = False,
            ) -> None:

        super().__init__(model_name, dataset_name, base_dir)

        self.test_mode = test_mode
        self.networks = None

        self.logger = Logger(self.base_dir, self.model_name, self.dataset_name)
        self.logger.log.info(f'Log will be saved to {self.base_dir}/logs/{self.model_name}_{self.dataset_name}.log')

        self.training_opt = self.config['training_opt']
        self.memory = self.config['memory']
        self.num_gpus = torch.cuda.device_count()
        self.do_shuffle = self.config['shuffle'] if 'shuffle' in self.config else False

        self.scheduler_params = self.training_opt['scheduler_params']

    def load_config(self):
        super().load_config()

    def load_data(self):
        super().load_data()

    def load_pretrained_model(self):
        super().load_pretrained_model()

    def save_latest(self, epoch):
        model_weights = {}
        model_weights['feat_model'] = copy.deepcopy(self.networks['feat_model'].state_dict())
        model_weights['classifier'] = copy.deepcopy(self.networks['classifier'].state_dict())

        model_states = {
            'epoch': epoch,
            'state_dict': model_weights
        }

        model_dir = f'{self.output_path}/latest_model_checkpoint.pth'
        torch.save(model_states, model_dir)

    def save_model(self, epoch, best_epoch, best_model_weights, best_acc, centroids=None):

        model_states = {'epoch': epoch,
                'best_epoch': best_epoch,
                'state_dict_best': best_model_weights,
                'best_acc': best_acc,
                'centroids': centroids}

        model_dir = f'{self.output_path}/final_model_checkpoint.pth'

        torch.save(model_states, model_dir)

    def output_logits(self, openset=False):
        filename = os.path.join(self.output_path,
                                'logits_%s'%('open' if openset else 'close'))
        self.logger.log.info("Saving total logits to: %s.npz" % filename)
        np.savez(filename,
                 logits=self.total_logits.detach().cpu().numpy(),
                 labels=self.total_labels.detach().cpu().numpy(),
                 paths=self.total_paths)

    def reset_model(self, model_state):
        for key, model in self.networks.items():
            weights = model_state[key]
            weights = {k: weights[k] for k in weights if k in model.state_dict()}
            model.load_state_dict(weights)

    def eval_with_preds(self, preds, labels):
        # Count the number of examples
        n_total = sum([len(p) for p in preds])

        # Split the examples into normal and mixup
        normal_preds, normal_labels = [], []
        mixup_preds, mixup_labels1, mixup_labels2, mixup_ws = [], [], [], []
        for p, l in zip(preds, labels):
            if isinstance(l, tuple):
                mixup_preds.append(p)
                mixup_labels1.append(l[0])
                mixup_labels2.append(l[1])
                mixup_ws.append(l[2] * np.ones_like(l[0]))
            else:
                normal_preds.append(p)
                normal_labels.append(l)

        # Calculate normal prediction accuracy
        rsl = {'train_all':0., 'train_many':0., 'train_median':0., 'train_low': 0.}
        if len(normal_preds) > 0:
            normal_preds, normal_labels = list(map(np.concatenate, [normal_preds, normal_labels]))
            n_top1 = mic_acc_cal(normal_preds, normal_labels)
            n_top1_many, \
            n_top1_median, \
            n_top1_low, = shot_acc(normal_preds, normal_labels, self.data['train'])
            rsl['train_all'] += len(normal_preds) / n_total * n_top1
            rsl['train_many'] += len(normal_preds) / n_total * n_top1_many
            rsl['train_median'] += len(normal_preds) / n_total * n_top1_median
            rsl['train_low'] += len(normal_preds) / n_total * n_top1_low

        # Calculate mixup prediction accuracy
        if len(mixup_preds) > 0:
            mixup_preds, mixup_labels, mixup_ws = \
                list(map(np.concatenate, [mixup_preds*2, mixup_labels1+mixup_labels2, mixup_ws]))
            mixup_ws = np.concatenate([mixup_ws, 1-mixup_ws])
            n_top1 = weighted_mic_acc_cal(mixup_preds, mixup_labels, mixup_ws)
            n_top1_many, \
            n_top1_median, \
            n_top1_low, = weighted_shot_acc(mixup_preds, mixup_labels, mixup_ws, self.data['train'])
            rsl['train_all'] += len(mixup_preds) / 2 / n_total * n_top1
            rsl['train_many'] += len(mixup_preds) / 2 / n_total * n_top1_many
            rsl['train_median'] += len(mixup_preds) / 2 / n_total * n_top1_median
            rsl['train_low'] += len(mixup_preds) / 2 / n_total * n_top1_low

        # Top-1 accuracy and additional string
        self.logger.log.info('Training acc Top1: %.3f ' % (rsl['train_all']))
        self.logger.log.info('Many_top1: %.3f' % (rsl['train_many']))
        self.logger.log.info('Median_top1: %.3f' % (rsl['train_median']))
        self.logger.log.info('Low_top1: %.3f' % (rsl['train_low']))

        return rsl

    def eval(self, phase='val', openset=False, save_feat=False):

        self.logger.log.info('-----------------------------------Phase: %s-----------------------------------' % (phase))

        torch.cuda.empty_cache()

        # In validation or testing mode, set model to eval() and initialize running loss/correct
        for model in self.networks.values():
            model.eval()

        self.total_logits = torch.empty((0, self.training_opt['num_classes'])).cuda()
        self.total_labels = torch.empty(0, dtype=torch.long).cuda()
        self.total_paths = np.empty(0)

        get_feat_only = save_feat
        feats_all, labels_all, idxs_all, logits_all = [], [], [], []
        # Iterate over dataset
        for inputs, labels, paths in tqdm(self.data[phase]):
            inputs, labels = inputs.cuda(), labels.cuda()

            # If on training phase, enable gradients
            with torch.set_grad_enabled(False):

                # In validation or testing
                self.__batch_forward(inputs, labels,
                                   centroids=self.memory['centroids'],
                                   phase=phase)
                if not get_feat_only:
                    self.total_logits = torch.cat((self.total_logits, self.logits))
                    self.total_labels = torch.cat((self.total_labels, labels))
                    self.total_paths = np.concatenate((self.total_paths, paths))

                if get_feat_only:
                    logits_all.append(self.logits.cpu().numpy())
                    feats_all.append(self.features.cpu().numpy())
                    labels_all.append(labels.cpu().numpy())
                    idxs_all.append(paths.numpy())

        if get_feat_only:
            typ = 'feat'
            if phase == 'train_plain':
                name = 'train{}_all.pkl'.format(typ)
            elif phase == 'test':
                name = 'test{}_all.pkl'.format(typ)
            elif phase == 'val':
                name = 'val{}_all.pkl'.format(typ)

            fname = f'{self.output_path}/{name}'
            self.logger.log.info('===> Saving feats to ' + fname)
            with open(fname, 'wb') as f:
                pickle.dump({
                             'feats': np.concatenate(feats_all),
                             'labels': np.concatenate(labels_all),
                             'idxs': np.concatenate(idxs_all),
                            },
                            f, protocol=4)
            return
        probs, preds = F.softmax(self.total_logits.detach(), dim=1).max(dim=1)

        # Calculate the overall accuracy and F measurement
        self.eval_acc_mic_top1= mic_acc_cal(preds[self.total_labels != -1],
                                            self.total_labels[self.total_labels != -1])
        self.eval_f_measure = F_measure(preds, self.total_labels, openset=openset,
                                        theta=self.training_opt['open_threshold'])
        self.many_acc_top1, \
        self.median_acc_top1, \
        self.low_acc_top1, \
        self.cls_accs = shot_acc(preds[self.total_labels != -1],
                                 self.total_labels[self.total_labels != -1],
                                 self.data['train'],
                                 acc_per_cls=True)

        rsl = {phase + '_all': self.eval_acc_mic_top1,
               phase + '_many': self.many_acc_top1,
               phase + '_median': self.median_acc_top1,
               phase + '_low': self.low_acc_top1,
               phase + '_fscore': self.eval_f_measure}

        self.logger.log.info('Phase: %s, Evaluation_accuracy_micro_top1: %.3f, Averaged F-measure: %.3f, Many_shot_accuracy_top1: %.3f, Median_shot_accuracy_top1: %.3f, Low_shot_accuracy_top1: %.3f'
            % (phase, self.eval_acc_mic_top1, self.eval_f_measure, self.many_acc_top1, self.median_acc_top1, self.low_acc_top1),)
        if phase != 'val':
            acc_str = "{:.1f} \t {:.1f} \t {:.1f} \t {:.1f}".format(
                self.many_acc_top1 * 100,
                self.median_acc_top1 * 100,
                self.low_acc_top1 * 100,
                self.eval_acc_mic_top1 * 100)
            self.logger.log.info(acc_str)

        if phase == 'test':
            with open(f'{self.output_path}/cls_accs.pkl', 'wb') as f:
                pickle.dump(self.cls_accs, f)
        return rsl

    def centroids_cal(self, data, save_all=False):

        centroids = torch.zeros(self.training_opt['num_classes'],
                                   self.training_opt['feature_dim']).cuda()

        self.logger.log.info('Calculating centroids.')

        torch.cuda.empty_cache()
        for model in self.networks.values():
            model.eval()

        feats_all, labels_all, idxs_all = [], [], []

        # Calculate initial centroids only on training data.
        with torch.set_grad_enabled(False):
            for inputs, labels, idxs in tqdm(data):
                inputs, labels = inputs.cuda(), labels.cuda()

                # Calculate Features of each training data
                self.batch_forward(inputs, feature_ext=True)
                # Add all calculated features to center tensor
                for i in range(len(labels)):
                    label = labels[i]
                    centroids[label] += self.features[i]
                # Save features if requried
                if save_all:
                    feats_all.append(self.features.cpu().numpy())
                    labels_all.append(labels.cpu().numpy())
                    idxs_all.append(idxs.numpy())

        if save_all:
            fname = f'{self.output_path}/feats_all.pkl'
            with open(fname, 'wb') as f:
                pickle.dump({'feats': np.concatenate(feats_all),
                             'labels': np.concatenate(labels_all),
                             'idxs': np.concatenate(idxs_all)},
                            f)
        # Average summed features with class count
        centroids /= torch.tensor(class_count(data)).float().unsqueeze(1).cuda()

        return centroids

    def batch_forward(self, inputs, labels=None, centroids=False, feature_ext=False, phase='train'):
        '''
        This is a general single batch running function.
        '''

        # Calculate Features
        self.features, self.feature_maps = self.networks['feat_model'](inputs)

        # If not just extracting features, calculate logits
        if not feature_ext:

            # During training, calculate centroids if needed to
            if phase != 'test':
                if centroids and 'FeatureLoss' in self.criterions.keys():
                    self.centroids = self.criterions['FeatureLoss'].centroids.data
                    torch.cat([self.centroids] * self.num_gpus)
                else:
                    self.centroids = None

            if self.centroids is not None:
                centroids_ = torch.cat([self.centroids] * self.num_gpus)
            else:
                centroids_ = self.centroids

            # Calculate logits with classifier
            self.logits, self.direct_memory_feature = self.networks['classifier'](self.features, centroids_)

    def batch_backward(self):
        # Zero out optimizer gradients
        self.model_optimizer.zero_grad()
        if self.criterion_optimizer:
            self.criterion_optimizer.zero_grad()
        # Back-propagation from loss outputs
        self.loss.backward()
        # Step optimizers
        self.model_optimizer.step()
        if self.criterion_optimizer:
            self.criterion_optimizer.step()

    def batch_loss(self, labels):
        self.loss = 0

        # First, apply performance loss
        if 'PerformanceLoss' in self.criterions.keys():
            self.loss_perf = self.criterions['PerformanceLoss'](self.logits, labels)
            self.loss_perf *=  self.criterion_weights['PerformanceLoss']
            self.loss += self.loss_perf

        # Apply loss on features if set up
        if 'FeatureLoss' in self.criterions.keys():
            self.loss_feat = self.criterions['FeatureLoss'](self.features, labels)
            self.loss_feat = self.loss_feat * self.criterion_weights['FeatureLoss']
            # Add feature loss to total loss
            self.loss += self.loss_feat

    def shuffle_batch(self, x, y):
        index = torch.randperm(x.size(0))
        x = x[index]
        y = y[index]
        return x, y


import os
import logging
import fnmatch
from typing import Iterable

#
# Class to help convert from absolute path to EDK2 build path
# using workspace and packagepath variables
#

class Edk2Path(object):

    def __init__(self, ws: os.PathLike, packagepathlist: Iterable[os.PathLike], error_on_invalid_pp: bool = True):
        """ An Edk2Path object is an object that can be used to resolve edk2 relative paths

        Args:
            ws: absolute path or cwd relative path of the workspace.
            packagespathlist: list of packages path.
                Entries can be Absolute path, workspace relative path, or CWD relative.
            error_on_invalid_pp: default value is True. If packages path value is invalid raise exception
        """

        self.WorkspacePath = ws
        self.logger = logging.getLogger("Edk2Path")
        if(not os.path.isabs(ws)):
            self.WorkspacePath = os.path.abspath(os.path.join(os.getcwd(), ws))

        if(not os.path.isdir(self.WorkspacePath)):
            self.logger.error("Workspace path invalid.  {0}".format(ws))
            raise Exception("Workspace path invalid.  {0}".format(ws))

        # Set PackagePath
        self.PackagePathList = list()
        for a in packagepathlist:
            if(os.path.isabs(a)):
                self.PackagePathList.append(a)
            else:
                # see if workspace relative
                wsr = os.path.join(ws, a)
                if(os.path.isdir(wsr)):
                    self.PackagePathList.append(wsr)
                else:
                    # assume current working dir relative.  Will catch invalid dir when checking whole list
                    self.PackagePathList.append(os.path.abspath(os.path.join(os.getcwd(), a)))

        error = False
        for a in self.PackagePathList[:]:
            if(not os.path.isdir(a)):
                self.logger.log(logging.ERROR if error_on_invalid_pp else logging.WARNING,
                                "Invalid package path entry {0}".format(a))
                self.PackagePathList.remove(a)  # remove invalid path
                error = True

        # report error
        if(error and error_on_invalid_pp):
            raise Exception("Invalid package path directory(s)")

    def GetEdk2RelativePathFromAbsolutePath(self, abspath):
        ''' Given an absolute path return a edk2 path relative
        to workspace or packagespath.

        If not valid return None
        '''

        relpath = None
        found = False
        if abspath is None:
            return None
        for a in (os.path.normcase(p) for p in self.PackagePathList):
            if os.path.normcase(abspath).startswith(a):
                # found our path...now use original strings to avoid
                # change in case
                relpath = abspath[len(a):]
                found = True
                self.logger.debug("Successfully converted AbsPath to Edk2Relative Path using PackagePath")
                self.logger.debug("AbsolutePath: %s found in PackagePath: %s" % (abspath, a))
                break

        if(not found):
            # Does path start with workspace
            if os.path.normcase(abspath).startswith(os.path.normcase(self.WorkspacePath)):
                # found our path...now use original strings to avoid
                # change in case
                relpath = abspath[len(self.WorkspacePath):]
                found = True
                self.logger.debug("Successfully converted AbsPath to Edk2Relative Path using WorkspacePath")
                self.logger.debug("AbsolutePath: %s found in Workspace: %s" % (abspath, self.WorkspacePath))

        if(found):
            relpath = relpath.replace(os.sep, "/")
            return relpath.lstrip("/")

        # didn't find the path for conversion.
        self.logger.error("Failed to convert AbsPath to Edk2Relative Path")
        self.logger.error("AbsolutePath: %s" % abspath)
        return None

    def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, relpath, log_errors=True):
        ''' Given a edk2 relative path return an absolute path to the file
        in this workspace.

        Note: For case insensitive operating systems the case of the input
        relpath will be used for the return value even if it doesn't match
        the case in the filesystem.

        If not valid or doesn't exist return None'''

        if relpath is None:
            return None
        relpath = relpath.replace("/", os.sep)
        abspath = os.path.join(self.WorkspacePath, relpath)
        if os.path.exists(abspath):
            return abspath

        for a in self.PackagePathList:
            abspath = os.path.join(a, relpath)
            if(os.path.exists(abspath)):
                return abspath
        if log_errors:
            self.logger.error("Failed to convert Edk2Relative Path to an Absolute Path on this system.")
            self.logger.error("Relative Path: %s" % relpath)

        return None

    # Find the package this path belongs to using
    # some Heuristic.  This isn't perfect but at least
    # identifies the directory consistently
    #
    # @param InputPath:  absolute path to module
    #
    # @ret Name of Package that the module is in.
    def GetContainingPackage(self, InputPath):
        self.logger.debug("GetContainingPackage: %s" % InputPath)

        # Make a list that has the path case normalized for comparison.
        # This only does anything on Windows
        NormCasePackagesPathList = [os.path.normcase(x) for x in self.PackagePathList]

        # check InputPath to make sure it is at least in the folder structure of the code tree
        if os.path.normcase(self.WorkspacePath) not in os.path.normcase(InputPath):
            # not in workspace - check all the packages paths
            found_in_pp = False
            for p in NormCasePackagesPathList:
                if p in os.path.normcase(InputPath):
                    found_in_pp = True
                    break
            if(not found_in_pp):
                self.logger.error(f"{InputPath} not in code tree")
                self.logger.info("PackagePath is: %s" % os.pathsep.join(self.PackagePathList))
                self.logger.info("Workspace path is : %s" % self.WorkspacePath)
                return None

        # InputPath is in workspace or PackagesPath for worst case scenario.

        dirpathprevious = os.path.dirname(InputPath)
        dirpath = os.path.dirname(InputPath)
        for _ in range(100):  # 100 is just a counter to avoid infinite loops.  Path nodes are unlikely to exceed 100
            #
            # Check for a DEC file in this folder
            # if here then return the directory name as the "package"
            #
            for f in os.listdir(dirpath):
                if fnmatch.fnmatch(f.lower(), '*.dec'):
                    a = os.path.basename(dirpath)
                    self.logger.debug("Found DEC file at %s.  Pkg is: %s", dirpath, a)
                    return a

            #
            # if at the root of the workspace return the previous dir.
            # this catches cases where a package has no DEC
            #
            if os.path.normcase(dirpath) == os.path.normcase(self.WorkspacePath):
                a = os.path.basename(dirpathprevious)
                self.logger.debug("Reached Workspace Path.  Using previous directory: %s" % a)
                return a

            #
            # if at the root of a packagepath return the previous dir.
            # this catches cases where a package has no DEC
            #
            if os.path.normcase(dirpath) in NormCasePackagesPathList:
                a = os.path.basename(dirpathprevious)
                self.logger.debug("Reached Package Path.  Using previous directory: %s" % a)
                return a

            dirpathprevious = dirpath
            dirpath = os.path.dirname(dirpath)

        self.logger.error("Failed to find containing package for %s" % InputPath)
        self.logger.info("PackagePath is: %s" % os.pathsep.join(self.PackagePathList))
        self.logger.info("Workspace path is : %s" % self.WorkspacePath)
        return None

    # Find the list of modules (infs) that file path is in
    #
    # for now just assume any inf in the same dir or if none
    # then check parent dir.  If InputPath is not in the filesystem
    # this function will try to return the likely containing module
    # but if the entire module has been deleted this isn't possible.
    #
    # @param InputPath:  absolute path to file
    #
    # @ret list of abs path file paths for module infs
    def GetContainingModules(self, InputPath: str) -> list:
        self.logger.debug("GetContainingModules: %s" % InputPath)

        # if INF return self
        if fnmatch.fnmatch(InputPath.lower(), '*.inf'):
            return [InputPath]

        # Before checking the local filesystem for an INF
        # make sure filesystem has file or at least folder
        if not os.path.isfile(InputPath):
            logging.debug("InputPath doesn't exist in filesystem")

        modules = []
        # Check current dir
        dirpath = os.path.dirname(InputPath)
        if os.path.isdir(dirpath):
            for f in os.listdir(dirpath):
                if fnmatch.fnmatch(f.lower(), '*.inf'):
                    self.logger.debug("Found INF file in %s.  INf is: %s", dirpath, f)
                    modules.append(os.path.join(dirpath, f))

        # if didn't find any in current dir go to parent dir.
        # this handles cases like:
        # ModuleDir/
        #   Module.inf
        #   x64/
        #     file.c
        #
        if(len(modules) == 0):
            dirpath = os.path.dirname(dirpath)
            if os.path.isdir(dirpath):
                for f in os.listdir(dirpath):
                    if fnmatch.fnmatch(f.lower(), '*.inf'):
                        self.logger.debug("Found INF file in %s.  INf is: %s", dirpath, f)
                        modules.append(os.path.join(dirpath, f))

        return modules


from abc import ABC
import re

class Validator(ABC):

    @staticmethod
    def check_validators(validators:"dict", value:"any") -> "tuple[bool, list[str]]":

        check_validation = {
            "required": Validator.required_validator(value, validators["required"]) if 'required' in validators else None,
            "minLength": Validator.min_length_validator(value, validators["minLength"]) if 'minLength' in validators else None,
            "maxLength": Validator.max_length_validator(value, validators["maxLength"]) if 'maxLength' in validators else None,
            "min": Validator.min_validator(value, validators["min"]) if 'min' in validators else None,
            "max": Validator.max_validator(value, validators["max"]) if 'max' in validators else None,
            "dataType": Validator.data_type_validator(value, validators["dataType"]) if 'dataType' in validators else None,
            "regex": Validator.regex_validator(value, validators["regex"]) if 'regex' in validators else None
        }
        validations_message:"list[str]" = []

        for validator in validators:
            if validator in check_validation:
                checked, message = check_validation[validator]
                if not checked:
                    validations_message.append(message)

        validation_status = True if len(validations_message) == 0 else False
        return validation_status, validations_message

    @staticmethod
    def required_validator(value:"any", is_required:"bool") -> "tuple[bool, list[str]|None]":
        status = False if is_required and not value else True
        message = None if status else "Required validation error"
        return status, message

    @staticmethod
    def min_length_validator(value:"any", min_length:"any") -> "tuple[bool, list[str]|None]":
        try:
            min_length = min_length if isinstance(min_length, int) else int(min_length)
            status = len(value) > min_length
        except Exception:
            status = False

        message = None if status else "Min length error"
        return status, message

    @staticmethod
    def max_length_validator(value:"any", max_length:"any") -> "tuple[bool, list[str]|None]":
        try:
            max_length = max_length if isinstance(max_length, int) else int(max_length)
            status = len(value) < max_length
        except Exception:
            status = False

        message = None if status else "Max length error"
        return status, message

    @staticmethod
    def min_validator(value:"any", min_value:"int") -> "tuple[bool, list[str]|None]":
        try:
            status = int(value) > min_value
        except Exception as ex:
            status = False

        message = None if status else "Min value error"
        return status, message

    @staticmethod
    def max_validator(value:"any", max_value:"int") -> "tuple[bool, list[str]|None]":
        try:
            status = int(value) < max_value
        except Exception as ex:
            status = False

        message = None if status else "Max value error"
        return status, message

    @staticmethod
    def data_type_validator(value:"any", type:"str") -> "tuple[bool, list[str]|None]":
        message = None
        status = True
        if type in ["int", "float", "text", "ntext", "str"]:
            try:
                if type == "int":
                    int(value)
                elif type == "float":
                    float(value)
                else:
                    str(value)
            except Exception:
                status = False
            if not status:
                message = f"Type of value is not {type}"
        else:
            status = False
            message = f"Function not found for {type} type checker"

        return status, message

    @staticmethod
    def regex_validator(value:"any", regex_expression:"str") -> "tuple[bool, list[str]|None]":
        try:
            status = re.search(value, regex_expression) != None
        except Exception as ex:
            status = False

        message = None if status else "regex error"
        return status, message


from urllib.request import urlopen
from urllib.error import HTTPError
from urllib.error import URLError
from bs4 import BeautifulSoup

codigo = input("Digite o codigo do aerodromo ou 0 para sair:")
while codigo != "0":
    try:
        url = "https://www.aisweb.aer.mil.br/?i=aerodromos&codigo="+codigo
        page = urlopen(url)
        page_content = BeautifulSoup(page.read(), 'html.parser')

        # cartas
        cartas = []
        next_s = page_content.select_one('h4:contains("Cartas")').find_next_sibling("h4")
        while next_s.text.find("Rotas Preferenciais") < 0:
            cartas.append(next_s.text)
            next_s = next_s.find_next_sibling("h4")

        # taf e metar
        taf = page_content.select_one('h5:contains("TAF")').find_next_sibling("p").text
        metar = page_content.select_one('h5:contains("METAR")').find_next_sibling("p").text

        # nascer e por do sol
        sunrise = page_content.find('sunrise').contents[0]
        sunset = page_content.find('sunset').contents[0]

        print("Aerodromo: {}".format(codigo))
        print("_____\n")
        print("CARTAS:")
        for carta in cartas:
            print("-", carta)
        print("_____\n")
        print("Nascer do sol: {}\nPor do sol   : {}".format(sunrise, sunset))
        print("_____\n")
        print("TAF  : {}\nMETAR: {}".format(taf, metar))
        print("_____\n")
    except:
        print("Algo deu errado. Tente novamente.")

    codigo = input("Digite o codigo do aerodromo ou 0 para sair:")