import cv2
import numpy as np

import time
#from Whatsappmassage import Whatappmassagefunction
#from EmailAleert import MailAllertFunc
#number of constant for calculations
#EmptyParking=1;
FullParking=0;
avialibalrparkingspot=1;#inital number->emptyparking=1=number of all the spots
counter_of_frames_leaving_park=0;
counterofspot1_in=0;
counterofspot1_out=0;
counterofspot2_in=0;
counterofspot2_out=0;
counterofspot3_in=0;
counterofspot3_out=0;
counterofspot4_in=0;
counterofspot4_out=0;
counterofspot5_in=0;
counterofspot5_out=0;
counterofspot6_in=0;
counterofspot6_out=0;
counterofspot7_in=0;
counterofspot7_out=0;
counterofspot8_in=0;
counterofspot8_out=0;
bool("Entrparking");
Entrparking=False;#iniatializ
bool("Leavingparking");
bool("catch_spot_num_1");
bool("catch_spot_num_2");
bool("catch_spot_num_3");
bool("catch_spot_num_4");
bool("catch_spot_num_5");
bool("spot_num_1_ready_out");
bool("spot_num_1_ready_in");

#initiliaz catch const as a function of the spots status
#False=>spot caught
#True=>Spot free
catch_spot_num_1=False;
catch_spot_num_2=False;
catch_spot_num_3=False;
catch_spot_num_4=False;
catch_spot_num_5=False;
spot_num_1_ready_out=False;
spot_num_1_ready_in=False;
Leavingparking=False;
cap = cv2.VideoCapture('shalom-alichem-keren-shalom-day-23-10-20_a.avi')#VideoCapture(0)-for camera live
frame_width = int( cap.get(cv2.CAP_PROP_FRAME_WIDTH))

frame_height =int( cap.get( cv2.CAP_PROP_FRAME_HEIGHT))

fourcc = cv2.VideoWriter_fourcc('X','V','I','D')

out = cv2.VideoWriter("output.avi", fourcc, 5.0, (1280,720))#record the output and send it to some path

font = cv2.FONT_HERSHEY_COMPLEX
ret, frame1 = cap.read()
ret, frame2 = cap.read()
print(frame1.shape)

while cap.isOpened():
    diff = cv2.absdiff(frame1, frame2)
    image1=out.write(diff)
    gray = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY)
    blur = cv2.GaussianBlur(gray, (5,5), 0)
    _, thresh = cv2.threshold(blur, 20, 255, cv2.THRESH_BINARY)
    dilated = cv2.dilate(thresh, None, iterations=3)
    contours, _ = cv2.findContours(dilated, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cv2.putText(frame1, "{}".format('Num of available parking spots:'+ str(avialibalrparkingspot)+''), (10, 20), cv2.FONT_HERSHEY_SIMPLEX,
                0.7, (0, 0, 255), 2)
    if avialibalrparkingspot <= 0:
         avialibalrparkingspot=0
    if avialibalrparkingspot==10:
        cv2.putText(frame1, "Status: {}".format('All parking spots available' ),
                    (10, 50), cv2.FONT_HERSHEY_SIMPLEX,
                    0.6, (0, 0, 255), 2)
    if  avialibalrparkingspot==FullParking:
        cv2.putText(frame1, "Status: {}".format('All parking spots caught!'),
                    (10, 50), cv2.FONT_HERSHEY_SIMPLEX,
                    0.6, (0, 0, 255), 2)
    for contour in contours:
        (x, y, w, h) = cv2.boundingRect(contour)
        print('Number of available parking spot:'+ str(avialibalrparkingspot)+'')

        if cv2.contourArea(contour) < 520:#The number is the area of the rectangle
            continue
        cv2.rectangle(frame1, (x, y), (x+w, y+h), (0, 255, 0), 2)
        #showing coordinats on screen
        string = str(x) + " " + str(y)
        cv2.putText(frame1, string, (x, y),font, 0.7, (0, 255, 0))

        outallert='Watch-out! Car going out the parking spot!';
        inallert='Watch-out! Car enter the parking spot!';
        spot1Out= 'Allert:"Hovevi Chion street":Car going out, parking Spots number 1 available'
        spot1In= 'Allert:"Hovevi Chion street": parking Spots number 1 caught'
        spot2Out = 'Allert:"Hovevi Chion street":Car going out, parking Spots number 2 available'
        spot2In = 'Allert:"Hovevi Chion street": parking Spots number 2 caught'
        spot3Out = 'Allert:"Hovevi Chion street":Car going out, parking Spots number 3 available'
        spot3In = 'Allert:"Hovevi Chion street": parking Spots number 3 caught!'
        spot4Out = 'Allert:"Hovevi Chion street":Car going out, parking Spots number 4 available'
        spot4In = 'Allert:"Hovevi Chion street": parking Spots number 4 caught!'
        spot5Out = 'Allert:"Hovevi Chion street":Car going out, parking Spots number 5 available'
        spot5In = 'Allert:"Hovevi Chion street": parking Spots number 5 caught!'
        if (x>40 and x<78) and (y>51 and y<72):
            print(inallert)
            Entrparking=True;
            cv2.putText(frame1, "{}".format('Car/s enter the parking spot!'), (50, 400), cv2.FONT_HERSHEY_SIMPLEX,
                        0.9, (0, 0, 255),3)

            # Spots number 1 Checking
            # if car catch spot num 1
            #option 1-front
        if (x > 470 and x < 475) and (y > 447 and y < 452) and (catch_spot_num_1 == True):
            print(spot1In)
            counterofspot1_in = counterofspot1_in + 1;  # counter of changes of frames at spot parking
            if counterofspot1_in > 4:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                catch_spot_num_1 = False;
                counterofspot1_in = 0;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 1 caught'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

            #option 2-reverse
            ##if car going to catch spot num 1 by reverse
        if (x > 554 and x <559 ) and (y > 447 and y < 452) and (catch_spot_num_1 == True):
            print("car going to catch spot number 1 by reverse,ready==True")
            spot_num_1_ready_in=True;

        if (x > 520 and x < 558) and (y > 442 and y < 452) and (catch_spot_num_1 == True) and (spot_num_1_ready_in==True):
            print(spot1In)
            counterofspot1_in = counterofspot1_in + 1;  # counter of changes of frames at spot parking
            if counterofspot1_in > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                catch_spot_num_1 = False;
                counterofspot1_in = 0;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 1 caught'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

            ##if car going to leave spot num 1
        if (x > 534 and x < 567) and (y > 437 and y < 459) and (catch_spot_num_1 == False):
            print("car going to leave spot number 1,ready==True")
            spot_num_1_ready_out=True;

            ##if car leaving spot num 1
        if (x > 520 and x < 570 ) and (y > 435 and y < 460) and (catch_spot_num_1 == False) and ( spot_num_1_ready_out==True):
            print(spot1Out)
            counterofspot1_out = counterofspot1_out + 1;  # counter of changes of frames at spot parking
            if counterofspot1_out > 2:
                avialibalrparkingspot = avialibalrparkingspot + 1;
                counterofspot1_out = 0;
                catch_spot_num_1 = True;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 1 is available to park'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

            # Spots number 2 Checking
            # if car catch spot num 2
        if (x > 336 and x < 341) and (y > 328 and y < 333) and (catch_spot_num_2 == True):
            print(spot2In)
            counterofspot2_in = counterofspot2_in + 1;  # counter of changes of frames at spot parking
            if counterofspot2_in > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                catch_spot_num_2 = False;
                counterofspot2_in = 0;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 2 caught'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)
            ##if car leaving spot num 2
        if (x > 336 and x < 341) and (y > 328 and y < 333) and (catch_spot_num_2 == False):
            print(spot2Out)
            counterofspot2_out = counterofspot2_out + 1;  # counter of changes of frames at spot parking
            if counterofspot2_out > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                counterofspot2_out = 0;
                catch_spot_num_2 = True;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 2 is available to park'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

            # Spots number 3 Checking
            # if car catch spot num 3
        if (x > 213 and x < 218) and (y > 228 and y < 233) and (catch_spot_num_3 == True):
            print(spot3In)
            counterofspot3_in = counterofspot3_in + 1;  # counter of changes of frames at spot parking
            if counterofspot3_in > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                catch_spot_num_3 = False;
                counterofspot3_in = 0
                cv2.putText(frame1, "Status: {}".format('Parking spot num 3 caught'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)
            ##if car leaving spot num 3
        if (x > 213 and x < 218) and (y > 228 and y < 233) and (catch_spot_num_3 == False):
            print(spot3Out)
            counterofspot3_out = counterofspot3_out + 1;  # counter of changes of frames at spot parking
            if counterofspot3_out > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                counterofspot3_out = 0;
                catch_spot_num_3 = True;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 3 is available to park'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

            # Spots number 4 Checking
            # if car catch spot num 4
        if (x > 156 and x < 161) and (y > 185 and y < 190) and (catch_spot_num_4 == True):
            print(spot4In)
            counterofspot4_in = counterofspot4_in + 1;  # counter of changes of frames at spot parking
            if counterofspot4_in > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                catch_spot_num_4 = False;
                counterofspot4_in = 0;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 4 caught!'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)
            ##if car leaving spot num 4
        if (x > 156 and x < 161) and (y > 185 and y < 190) and (catch_spot_num_4 == False):
            print(spot4Out)
            counterofspot4_out = counterofspot4_out + 1;  # counter of changes of frames at spot parking
            if counterofspot4_out > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                counterofspot4_out = 0;
                catch_spot_num_4 = True;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 4 is available to park'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

            # Spots number 5 Checking
            # if car catch spot num 5
        if (x > 115 and x < 120) and (y > 150 and y < 155) and (catch_spot_num_5 == True):
            print(spot5In)
            counterofspot5_in = counterofspot5_in + 1;  # counter of changes of frames at spot parking
            if counterofspot5_in > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                catch_spot_num_5 = False;
                counterofspot5_in = 0;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 5 caught'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)
            ##if car leaving spot num 5
        if (x > 115 and x < 120) and (y > 150 and y < 155) and (catch_spot_num_5 == False):
            print(spot5Out)
            counterofspot5_out = counterofspot5_out + 1;  # counter of changes of frames at spot parking
            if counterofspot5_out > 2:
                avialibalrparkingspot = avialibalrparkingspot - 1;
                counterofspot5_out = 0;
                catch_spot_num_5 = True;
                cv2.putText(frame1, "Status: {}".format('Parking spot num 5 is available to park'), (10, 100),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.6, (0, 0, 255), 1)

                # if car leaving the parking
        if (x>420 and x<641) and (y>370 and y<480):
            print(outallert)
            Leavingparking=True;
            cv2.putText(frame1, "Status: {}".format('Car/s leaving parking spot'), (50, 370), cv2.FONT_HERSHEY_SIMPLEX,
                        0.8, (0, 0, 255), 3)
            Exitparking=True;

        #Whatappmassagefunction(allert);
    #cv2.drawContours(frame1, contours, -1, (0, 255, 0), 2)

    image = cv2.resize(frame1, (1280,720))
    out.write(image)
    cv2.imshow("feed", frame1)

    frame1 = frame2
    ret, frame2 = cap.read()

    # channels = cv2.mean(frame2)
    # observation = np.array([(channels[2], channels[1], channels[0])])
    # if np.mean(observation)<130 and np.mean(observation)>120:#128 value is gray in RGB
    #     print("Gray image/Empty parking")

    # _, binaryimage = cv2.threshold( frame2, 209, 255, cv2.THRESH_BINARY)
    # print(binaryimage)

    if cv2.waitKey(40) == 27:#Esc for exit camera
        break

cv2.destroyAllWindows()
cap.release()
out.release()


if __name__ =='__main__':
    import sys

    if len(sys.argv) != 4:
        print("Usage: python %s file_in file_fr_out file_en_out"%sys.argv[0], file=sys.stderr)
        sys.exit(1)

    file_in = sys.argv[1]
    file_fr_out = sys.argv[2]
    file_en_out = sys.argv[3]

    with open(file_in) as fp:
        with open(file_fr_out, 'w') as fp_fr, open(file_en_out, 'w') as fp_en:
            for l in fp:
                l = l.strip()
                if not l:
                    continue
                fr, en = l.split('\t')
                fp_en.write("{}\n".format(en))
                fp_fr.write("{}\n".format(fr))


# -*- coding: utf-8 -*-
import json

import numpy as np
import math
import pandas as pd
from sklearn.model_selection import StratifiedKFold
from tensorflow import keras
from joblib import dump, load
from utils.helpers import *
# from RAAC import read_fasta

if __name__ == '__main__':
    # TFPM_val, TFPM_val_labels = read_fasta('../data/TFPM vs TFPNM/TFPM_independent_dataset.txt')
    # TFPNM_val, TFPNM_val_labels = read_fasta('../data/TFPM vs TFPNM/TFPNM_independent_dataset.txt')
    #
    # path = 'F:/BioInformatic/Dataset/TFPM vs TFPNM/PSSM/'
    # TFPM_val_pssm, TFPM_val_pssm_labels = read_data(path + 'TFPM_independent_dataset/', padding='none')
    # TFPNM_val_pssm, TFPNM_val_pssm_labels = read_data(path + 'TFPNM_independent_dataset/', padding='none')
    #
    # reduced_TFPM, _ = reduce_pssms(TFPM_val, TFPM_val_labels, TFPM_val_pssm, TFPM_val_pssm_labels)
    # reduced_TFPNM, _ = reduce_pssms(TFPNM_val, TFPNM_val_labels, TFPNM_val_pssm, TFPNM_val_pssm_labels)
    #
    # data = np.append(reduced_TFPM, reduced_TFPNM, axis=0)
    # labels = np.append(np.ones(len(reduced_TFPM)), np.zeros(len(reduced_TFPNM)))
    #
    # data = data.reshape((data.shape[0], -1))

    TFPM_val, _ = read_fasta('../data/TFPM vs TFPNM/TFPM_independent_dataset.txt')
    TFPNM_val, _ = read_fasta('../data/TFPM vs TFPNM/TFPNM_independent_dataset.txt')

    # TFPM_val, _ = read_fasta('../data/TF vs non-TF/TF_independent_dataset.txt')
    # TFPNM_val, _ = read_fasta('../data/TF vs non-TF/non-TF_independent_dataset.txt')

    gap = 4
    # op = 'op13'
    # ops = ['op5', 'op8', 'op9', 'op11', 'op13']
    ops = ['op13']

    TFPM_val_all = [[] for i in range(len(TFPM_val))]
    TFPNM_val_all = [[] for i in range(len(TFPNM_val))]

    for op in ops:
        reduced_TFPM_val = RAACs(TFPM_val, op=op)
        RGDP_TFPM_val = RGDPs(reduced_TFPM_val, gap=gap, op=op)
        reduced_TFPNM_val = RAACs(TFPNM_val, op=op)
        RGDP_TFPNM_val = RGDPs(reduced_TFPNM_val, gap=gap, op=op)

        TFPM_val_all = np.append(TFPM_val_all, RGDP_TFPM_val, axis=-1)
        TFPNM_val_all = np.append(TFPNM_val_all, RGDP_TFPNM_val, axis=-1)

    # reduced_TFPM_val = RAACs(TFPM_val, op=op)
    # RGDP_TFPM_val = RGDPs(reduced_TFPM_val, gap=gap, op=op)
    # reduced_TFPNM_val = RAACs(TFPNM_val, op=op)
    # RGDP_TFPNM_val = RGDPs(reduced_TFPNM_val, gap=gap, op=op)

    # RGDP_TFPM_val = GDPs(TFPM_val, gap=gap)
    # RGDP_TFPNM_val = GDPs(TFPNM_val, gap=gap)

    # validating data
    data = np.append(TFPM_val_all, TFPNM_val_all, axis=0)
    labels = np.append(np.ones(len(TFPM_val_all)), np.zeros(len(TFPNM_val_all)), axis=0)

    # # BERT
    # path = 'F:/BioInformatic/Dataset/TFPM vs TFPNM/BERT/'
    # data, labels = read_h5py(path, 'TFPM_testing_top100')
    # print(data.shape)
    # data = data[:, 0]

    # DCT_TFPM_GGAP = load('../saved_models/DCT_TFPM_GGAP.joblib')
    # # importance = DCT_TFPM_GGAP.feature_importances_
    # importance = DCT_TFPM_GGAP.coef_[0]
    # importance = np.argsort(importance)[::-1]

    # file = h5py.File('mrmr' + '.h5', 'r')
    # importance = np.asarray(file['mrmr'])

    # pick = importance[:37]
    # data = data[:, pick]

    path = "../saved_models/TFPM/"
    model_paths = os.listdir(path)
    model = []
    i = 0
    for model_path in model_paths:
        model.append(load('../saved_models/TFPM op5/' + str(i) + '_SVM.joblib'))
        i += 1

    i = 0
    a = []
    b = []
    for i in range(len(model)):
        pre = model[i].predict_proba(data)
        print("model: " + str(i))
        sen = sensitivity(labels, pre)
        spe = specificity(labels, pre)
        accc = acc(labels, pre)
        mccc = mcc(labels, pre)
        aucc = auc(labels, pre)
        b.append(math.floor(sen * 1000) / 1000)
        b.append(math.floor(spe * 1000) / 1000)
        b.append(math.floor(accc * 1000) / 1000)
        b.append(math.floor(mccc * 1000) / 1000)
        b.append(math.floor(aucc * 1000) / 1000)
        a.append(b)
        b = []
        i += 1

    vote = voting(model, data)
    ave = average(model, data)
    med = median(model, data)

    print("voting:")
    sen = sensitivity(labels, vote)
    spe = specificity(labels, vote)
    accc = acc(labels, vote)
    mccc = mcc(labels, vote)
    aucc = auc(labels, vote)
    b.append(math.floor(sen * 1000) / 1000)
    b.append(math.floor(spe * 1000) / 1000)
    b.append(math.floor(accc * 1000) / 1000)
    b.append(math.floor(mccc * 1000) / 1000)
    b.append(math.floor(aucc * 1000) / 1000)
    a.append(b)
    b = []

    print("ave:")
    sen = sensitivity(labels, ave)
    spe = specificity(labels, ave)
    accc = acc(labels, ave)
    mccc = mcc(labels, ave)
    aucc = auc(labels, ave)
    b.append(math.floor(sen * 1000) / 1000)
    b.append(math.floor(spe * 1000) / 1000)
    b.append(math.floor(accc * 1000) / 1000)
    b.append(math.floor(mccc * 1000) / 1000)
    b.append(math.floor(aucc * 1000) / 1000)
    a.append(b)
    b = []

    print("med:")
    sen = sensitivity(labels, med)
    spe = specificity(labels, med)
    accc = acc(labels, med)
    mccc = mcc(labels, med)
    aucc = auc(labels, med)
    b.append(math.floor(sen * 1000) / 1000)
    b.append(math.floor(spe * 1000) / 1000)
    b.append(math.floor(accc * 1000) / 1000)
    b.append(math.floor(mccc * 1000) / 1000)
    b.append(math.floor(aucc * 1000) / 1000)
    a.append(b)
    b = []

    pd.DataFrame(a).to_csv('../test.csv')


'''
  Author: Jon Paul Miles
  Date Created: March 15, 2022

  Daily data README:
  https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt
'''

from globals import *
import pandas as pd
import numpy as np
import glob
from bs4 import BeautifulSoup
import urllib.request
from termcolor import colored, cprint
import os

# When parsing rows for the temperature files for this network, these set the bounds for each column
DATA_COLUMNS = [(0,11), (11, 15)] + generate_month_boundaries([5,6,7,8], 19)

check_mark = colored(u'\u2713', 'green', attrs=['bold'])

attention_mark = colored('!', 'yellow', attrs=['bold'])

# All GHCNd .dly station files have 31 days even if some days are missing
DAYS_IN_MONTH = 31

def download_and_compile_uscrn_data(compiled_file, folder_name, url):

  TEMPERATURES_FILE_PATH = ""

  # Check if the compiled USCRN data exists
  matching_compiled_uscrn_files = glob.glob(compiled_file)

  # The compiled daily data file was found
  if len(matching_compiled_uscrn_files):

    TEMPERATURES_FILE_PATH = matching_compiled_uscrn_files[0]

    print(f"{check_mark} Found '{compiled_file}'")

  # If the compiled daily data doesn't exist
  else:

    print(f"{attention_mark} Missing '{compiled_file}'")

    # Check if the folder of USCRN station files has already been downloaded
    matching_extracted_uscrn_files = glob.glob(os.path.join('.', folder_name, '*'))

    # If so, use it when compiling the data
    if len(matching_extracted_uscrn_files):

      print(f"{check_mark} Found '{folder_name}':")

      print(f'  {check_mark} ' + f'\n  {check_mark} '.join(matching_extracted_uscrn_files))

    # If not, download the USCRN data
    else:

      station_files = []

      print(f"{attention_mark} Missing '{folder_name}'. Downloading from {url}")

      # Make a folder to save the USCRN station files to
      if not os.path.exists(folder_name):

        os.mkdir(folder_name)

      # Read the link to all the station files
      soup = BeautifulSoup(urllib.request.urlopen(url), features="html.parser")

      # Get the links for this HTML page
      for a in soup.find_all('a'):

        link = a['href']

        # If the link is a CRN station .txt file:
        if 'CRN' in link and link.endswith('.txt'):

          # Join the link file name with the url
          station_url = os.path.join(url, link)

          # Join the folder name with the file name
          station_file_path = os.path.join(folder_name, link)

          # Download the file
          urllib.request.urlretrieve(station_url, station_file_path)

          # Add to our station file list for displaying in the console later
          station_files.append(station_file_path)

      print(f"  {check_mark} Downloaded:")

      print(f'    {check_mark} ' + f'\n  {check_mark} '.join(station_files))

    TEMPERATURES_FILE_PATH = compile_uscrn_data(VERSION, folder_name)

  return TEMPERATURES_FILE_PATH

def get_files():

  COUNTRIES_FILE_PATH = ""

  STATION_FILE_PATH = 'stations.tsv'

  TEMPERATURES_FILE_PATH = download_and_compile_uscrn_data(
    compiled_file = 'uscrn.tavg.v1.dat',
    folder_name = 'uscrn_stations_v1',
    url = 'https://www.ncei.noaa.gov/pub/data/uscrn/products/monthly01/'
  )

  return STATION_FILE_PATH, TEMPERATURES_FILE_PATH, COUNTRIES_FILE_PATH

def get_stations(station_file_name):

  names = [ 'station_id', 'country_code', 'state', 'LOCATION', 'VECTOR', 'name', 'latitude', 'longitude', 'elevation', 'STATUS', 'COMMISSIONING', 'CLOSING', 'OPERATION', 'PAIRING', 'network', 'other_station_id' ]
  # names=names,

  stations = pd.read_csv(
    station_file_name,
    sep="\t",
    names = names,
    header=None,
    skiprows=[0],
    encoding='utf-8',
  ).reset_index()

  # Pad Station ID
  stations['station_id'] = 'USCRN' + stations['station_id'].astype('str').apply(lambda s: str(s).rjust(6, '0') )

  stations = stations[['country_code', 'station_id', 'latitude', 'longitude', 'elevation', 'name', 'state']]

  stations['country'] = 'United States of America'

  return stations

def has_passing_flags(MFLAG, QFLAG, SFLAG):

  return True

def average_tmax_and_tmin(TMAX_AND_TMIN):

  tavg = math.nan if len(TMAX_AND_TMIN) != 2 or TMAX_AND_TMIN.isnull().any() else int(normal_round(TMAX_AND_TMIN.mean()))

  return tavg

def parse_daily_row(unparsed_row):

  parsed_row = []

  ELEMENT = str(unparsed_row[17:21])

  if not ELEMENT == "TMAX" and not ELEMENT == "TMIN":
    return False

  try:

    STATION_ID = str(unparsed_row[0:11]).strip()
    YEAR = int(unparsed_row[11:15])
    MONTH = int(unparsed_row[15:17])

    parsed_row.append(STATION_ID)
    parsed_row.append(YEAR)
    parsed_row.append(MONTH)
    parsed_row.append(ELEMENT)

    DAYS_IN_MONTH = 31
    FIRST_DAY_INDEX = 21
    CHARACTERS_NEEDED_FOR_READING_AND_FLAGS = 8

    FULL_RANGE = (DAYS_IN_MONTH * CHARACTERS_NEEDED_FOR_READING_AND_FLAGS) + FIRST_DAY_INDEX

    for index in range(FIRST_DAY_INDEX, FULL_RANGE, CHARACTERS_NEEDED_FOR_READING_AND_FLAGS):

      # Temperature comes in tenths of degrees C
      temperature_reading = int(unparsed_row[index:index + 5])

      '''
        https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt
        MFLAG1     is the measurement flag for the first day of the month.  There are
                   ten possible values:

                   Blank = no measurement information applicable
                   B     = precipitation total formed from two 12-hour totals
                   D     = precipitation total formed from four six-hour totals
             H     = represents highest or lowest hourly temperature (TMAX or TMIN)
                     or the average of hourly values (TAVG)
             K     = converted from knots
             L     = temperature appears to be lagged with respect to reported
                     hour of observation
                   O     = converted from oktas
             P     = identified as "missing presumed zero" in DSI 3200 and 3206
                   T     = trace of precipitation, snowfall, or snow depth
             W     = converted from 16-point WBAN code (for wind direction)

      '''
      MFLAG = str(unparsed_row[index+5:index+6])

      '''
        https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt
        QFLAG1     is the quality flag for the first day of the month.  There are
                   fourteen possible values:

                   Blank = did not fail any quality assurance check
                   D     = failed duplicate check
                   G     = failed gap check
                   I     = failed internal consistency check
                   K     = failed streak/frequent-value check
             L     = failed check on length of multiday period
                   M     = failed megaconsistency check
                   N     = failed naught check
                   O     = failed climatological outlier check
                   R     = failed lagged range check
                   S     = failed spatial consistency check
                   T     = failed temporal consistency check
                   W     = temperature too warm for snow
                   X     = failed bounds check
             Z     = flagged as a result of an official Datzilla
                     investigation
      '''
      QFLAG = str(unparsed_row[index+6:index+7])

      '''
        https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt
        SFLAG1     is the source flag for the first day of the month.  There are
                   thirty possible values (including blank, upper and
             lower case letters):

                   Blank = No source (i.e., data value missing)
                   0     = U.S. Cooperative Summary of the Day (NCDC DSI-3200)
                   6     = CDMP Cooperative Summary of the Day (NCDC DSI-3206)
                   7     = U.S. Cooperative Summary of the Day -- Transmitted
                     via WxCoder3 (NCDC DSI-3207)
                   A     = U.S. Automated Surface Observing System (ASOS)
                           real-time data (since January 1, 2006)
             a     = Australian data from the Australian Bureau of Meteorology
                   B     = U.S. ASOS data for October 2000-December 2005 (NCDC
                           DSI-3211)
             b     = Belarus update
             C     = Environment Canada
             D     = Short time delay US National Weather Service CF6 daily
                     summaries provided by the High Plains Regional Climate
               Center
             E     = European Climate Assessment and Dataset (Klein Tank
                     et al., 2002)
                   F     = U.S. Fort data
                   G     = Official Global Climate Observing System (GCOS) or
                           other government-supplied data
                   H     = High Plains Regional Climate Center real-time data
                   I     = International collection (non U.S. data received through
                     personal contacts)
                   K     = U.S. Cooperative Summary of the Day data digitized from
                     paper observer forms (from 2011 to present)
                   M     = Monthly METAR Extract (additional ASOS data)
             m     = Data from the Mexican National Water Commission (Comision
                     National del Agua -- CONAGUA)
             N     = Community Collaborative Rain, Hail,and Snow (CoCoRaHS)
             Q     = Data from several African countries that had been
                     "quarantined", that is, withheld from public release
               until permission was granted from the respective
                     meteorological services
                   R     = NCEI Reference Network Database (Climate Reference Network
                     and Regional Climate Reference Network)
             r     = All-Russian Research Institute of Hydrometeorological
                     Information-World Data Center
                   S     = Global Summary of the Day (NCDC DSI-9618)
                           NOTE: "S" values are derived from hourly synoptic reports
                           exchanged on the Global Telecommunications System (GTS).
                           Daily values derived in this fashion may differ significantly
                           from "true" daily data, particularly for precipitation
                           (i.e., use with caution).
             s     = China Meteorological Administration/National Meteorological Information Center/
                     Climatic Data Center (http://cdc.cma.gov.cn)
                   T     = SNOwpack TELemtry (SNOTEL) data obtained from the U.S.
                     Department of Agriculture's Natural Resources Conservation Service
             U     = Remote Automatic Weather Station (RAWS) data obtained
                     from the Western Regional Climate Center
             u     = Ukraine update
             W     = WBAN/ASOS Summary of the Day from NCDC's Integrated
                     Surface Data (ISD).
                   X     = U.S. First-Order Summary of the Day (NCDC DSI-3210)
             Z     = Datzilla official additions or replacements
             z     = Uzbekistan update

             When data are available for the same time from more than one source,
             the highest priority source is chosen according to the following
             priority order (from highest to lowest):
             Z,R,D,0,6,C,X,W,K,7,F,B,M,m,r,E,z,u,b,s,a,G,Q,I,A,N,T,U,H,S
      '''
      SFLAG = str(unparsed_row[index+7:index+8])

      corrected_temperature = math.nan if temperature_reading == -9999 or not has_passing_flags(MFLAG, QFLAG, SFLAG) else int(temperature_reading)

      parsed_row.append(corrected_temperature)

    return parsed_row

  except:

    print(f"\nCould not parse row:")
    print(unparsed_row)
    print()

    return False

def gather_station_files(FOLDER_WITH_DAILY_DATA):

  # Find all station data files in folder
  station_files = []

  all_stations = os.listdir(FOLDER_WITH_DAILY_DATA)

  # Create an array of all our .dly station file URLs
  for station_file_url in all_stations:

    if 'CRN' in station_file_url and station_file_url.endswith('.txt'):

      station_files.append(os.path.join(FOLDER_WITH_DAILY_DATA, station_file_url))

  return station_files

def convert_to_hundreds(tavg):

  if math.isnan(tavg):

    return math.nan

  elif tavg == -9999.0:

    return -9999

  else:

    return int(normal_round(tavg * 100))

def compile_uscrn_data(VERSION, FOLDER_WITH_DAILY_DATA):

  # Prepare our mega file to save all combined station temperatures too
  OUTPUT_FILE_URL = f"./uscrn.tavg.{VERSION}.dat"

  if os.path.exists(OUTPUT_FILE_URL):

    os.remove(OUTPUT_FILE_URL)

  print(f"\nCompiling USCRN data into a GHCNm-like monthly TAVG file to be named '{OUTPUT_FILE_URL}'\n")

  # Prepare the file for editing
  OUTPUT_CONTENT = open(OUTPUT_FILE_URL, 'a')

  station_files = gather_station_files(FOLDER_WITH_DAILY_DATA)

  total_stations = len(station_files)

  station_iterator = 0

  # For each daily station file
  for station_file_url in station_files:

    station_iterator += 1

    print(f"Composing station {station_iterator} of {total_stations}")

    names = [ 'station_id', 'year', 'month', 'tavg' ]

    colspecs = [ (0,5), (6,10), (10, 12), (56,64) ]

    station_daily_values = pd.read_fwf(station_file_url, names = names, colspecs=colspecs)

    # Convert each tavg into an int
    station_daily_values['tavg'] = station_daily_values['tavg'].apply(convert_to_hundreds).astype('Int64')

    station_daily_values['station_id'] = 'USCRN' + station_daily_values['station_id'].astype('str').apply(lambda s: str(s).rjust(6, '0') )

    # Convert the vertical months, into horizontal ones across each year
    station_daily_values = station_daily_values.pivot(

      index=['station_id', 'year'], columns='month', values='tavg'

    ).reset_index()

    # If the file has 12 columns, one for each month
    if len(station_daily_values.columns) == 14 and len(station_daily_values.values) > 1:

      # For each row in the station's data
      for year, row in station_daily_values.iterrows():

        # Create a representative string line to save in the output file
        output_row_string = f"\n{row['station_id']}{row['year']}TAVG"

        for month in range(1, 13):

          month_string = str(row[month]).replace('<NA>', '-9999').rjust(5, " ")

          output_row_string = output_row_string + f"{month_string}   "

        # Add the representative line to the output file
        OUTPUT_CONTENT.write(output_row_string)

  print(f"\n{check_mark} USCRN station data compiled into '{OUTPUT_FILE_URL}'\n")

  return OUTPUT_FILE_URL


import pandas as pd
from pyspark.sql import SparkSession, Window
from pyspark.sql.functions import col, avg, min, max, count

if __name__ == '__main__':

    class HybridPandasSerie:

        def __init__(self, serie):
            self.serie = serie

        def __mul__(self, other):
            self.serie = self.serie * other
            return self

        def __add__(self, other):
            if isinstance(other, HybridPandasSerie):
                other = other.serie
            self.serie = self.serie + other
            return self

        def __gt__(self, other):
            if isinstance(other, HybridPandasSerie):
                other = other.serie
            self.serie = self.serie > other
            return self

        def __str__(self):
            return str(self.serie)

        def max(self):
            return self.serie.max()

    class HybridPandasDataFrame:

        def __init__(self, df):
            self.df = df

        def __getitem__(self, other):
            #Si es una columna utilizar DataFrame
            if isinstance(other, list):
                return HybridPandasDataFrame(df[other])
            if isinstance(other, HybridPandasSerie):
                return HybridPandasDataFrame(df[other.serie])
            return HybridPandasSerie(df[other])

        def __str__(self):
            return str(self.df)

    df = pd.DataFrame([[1, 2], [2, 2], [3, 3]], columns=['A', 'B'])
    df2 = HybridPandasDataFrame(df)
    print(df2['A'] + 2)
    print(df2[['A']])
    print(df2[df2['A'] > 1])
    print((df2['A'] + df2['B']) * 2)
    print(df2['A'].max())

    class A:

        def __init__(self, value):
            self.value = value

        def __add__(self, other):
            self.value = self.value + other
            return self

        def __mul__(self, other):
            self.value = self.value * other
            return self

        def __str__(self):
            return str(self.value)

    print((A(1) + 1) * 2)

    spark = SparkSession.builder.master("local").getOrCreate()
    df = spark.read.format('csv')\
                  .option('header', True)\
                  .option('sep', ';')\
                  .schema("id INT, name STRING, surname STRING, age INT ")\
                  .load('user.csv')

    import databricks.koalas as ks
    df = ks.read_csv('user.csv', sep=";", header=0)
    print(df[df['age'] > 30])

    (df.to_spark())
    df.toPandas()

    #Pandas to koalas
    ks.from_pandas(df)
    #Spark to koalas
    ks.DataFrame(df.to_spark())

    from pyspark.sql.types import *
    schema = StructType(
        [StructField('id', IntegerType()), StructField('name', StringType()), StructField('surname', StringType()),
         StructField('age', IntegerType(), False)])
    df = spark.read.format('csv').option('header', 'True').option('sep', ';').schema(schema).load(
        'user.csv')
    df.show()

    df.groupby('surname').pivot('name').agg(avg(col('age'))).show()

    df.filter(col('surname').isNull()).show()
    df_aux = df.filter(col('age') < 30)

    df_aux.cache()
    df_aux.count()

    df_aux.show()

    miwindow = Window.partitionBy('surname').orderBy(col('age').asc()).rowsBetween(Window.unboundedPreceding, Window.currentRow )
    df.select("name", "surname", min(col('age')).over(miwindow).alias('min_age')).show()

    b = spark.createDataFrame([[1002], [3001], [4002], [2003], [2002], [3004], [1003], [4006]], ['id']).\
            withColumn("x", col("id") % 1000)

    c = b.groupBy(col("x"))\
    .agg(count("x"), sum("value"))\
    .drop("x")\
    .toDF("count", "total")\
    .orderBy(col("count").desc, col("total"))\
    .limit(1)\
    .show()


import secrets
import canonicaljson
import hashlib
import decimal
import datetime
import logging
import random

from . import apitype

import beeline

STANDARD_SUMMARY_BATCHES = [20, 50]

def traced_function(f):
    name = f.__name__
    return beeline.traced(name=name)(f)

from .models import (
    Fact,
    FactCategory,
    FactType,
    BooleanChallenge,
    NumericChallenge,
    Challenge,
    BooleanResponse,
    NumericResponse,
    Response,
    SummaryScore,
    RESPONSE_MODELS,
    FACT_MODELS,
)

from .stats import (
    calculate_plausibility_of,
)

from .exceptions import (
    BadRequest,
    AlreadyResponded,
    NoFactsAvailable,
    PermissionDenied,
)

logger = logging.getLogger(__name__)

@traced_function
def generate_uid():
    return secrets.token_hex(16)

@traced_function
def select_random_fact():
    cats = [cat for cat in FactCategory.objects.all() if cat.active]
    logger.info(
        "Selecting random facts: categories: %s",
        [(cat.name, float(cat.weight)) for cat in cats],
    )
    cat_name = None
    if not cats:
        qs = Fact.objects.filter(active=True)
        logger.info("No categories -- using legacy random selection.")
    else:
        total_weight = float(sum(cat.weight for cat in cats))
        x = random.random() * total_weight
        for cat in cats:
            x -= float(cat.weight)
            if x < 0:
                break
        cat_name = cat.name
        qs = Fact.objects.filter(active=True, category=cat)
    logger.info("Choosing randomly (from category %s) among %d options", cat_name, qs.count())
    return qs.order_by("?").first()

@traced_function
def _save_and_return(x):
    x.save()
    return x

@traced_function
def create_boolean_challenge(fact):
    return _save_and_return(BooleanChallenge(fact=fact))

@traced_function
def create_numeric_challenge(fact):
    return _save_and_return(NumericChallenge(fact=fact))

@traced_function
def create_challenge_from_fact(user, fact):
    creators = {
        "boolean_fact": create_boolean_challenge,
        "numeric_fact": create_numeric_challenge,
    }
    kw = {
        k.replace("_fact", "_challenge"): f(getattr(fact, k))
        for k, f in creators.items()
        if getattr(fact, k)
    }
    return _save_and_return(Challenge(
        uid = generate_uid(),
        user = user,
        fact = fact,
        challenge_type = fact.fact_type,
        **kw,
    ))

@traced_function
def get_or_create_current_challenge(user):
    challenge = Challenge.objects.filter(
        user = user,
        active = True,
        response__isnull = True,
    ).order_by("-creation_time").first()

    if not challenge:
        fact = select_random_fact()
        if not fact:
            raise NoFactsAvailable()
        challenge = create_challenge_from_fact(user, fact)

    return challenge

@traced_function
def discard_current_challenge(user, challenge_uid=None):
    qs = Challenge.objects.filter(
        user = user,
        active = True,
        response__isnull = True,
    )
    if challenge_uid:
        qs = qs.filter(uid = challenge_uid)
    return qs.update(active = False)

@traced_function
def respond_to_challenge(user, challenge_uid, response):
    challenge = Challenge.objects.get(user = user, uid = challenge_uid)

    try:
        challenge.response
        raise AlreadyResponded(detail = f"Challenge {challenge_uid} has already gotten a response")
    except Challenge.response.RelatedObjectDoesNotExist:
        pass

    k = challenge.challenge_type
    if list(response) != [k]:
        raise BadRequest(detail = f"Challenge is of type {k}; got response of other type ({list(response)})")

    response_cls = RESPONSE_MODELS[k]

    response_core = _save_and_return(response_cls(
        challenge = challenge.challenge,
        **response[k],
    ))
    response_field = k + "_response"

    confidence = response_core.confidence_percent

    is_correct = response_core.get_correctness()

    rv = _save_and_return(Response(
        user = user,
        challenge = challenge,
        confidence_percent = confidence,
        correct = is_correct,
        response_type = k,
        **{response_field: response_core},
    ))

    for batch_size in STANDARD_SUMMARY_BATCHES:
        maybe_summarize_responses(user, batch_size)

    return rv

@traced_function
def get_last_response(user):
    return Response.objects.filter(
        user = user,
    ).order_by("-creation_time").first()

@traced_function
def get_user_responses(user, limit):
    return Response.objects.filter(
        user = user,
    ).order_by("creation_time").reverse()[:limit]

def _maybe_pop(d, k):
    if k in d:
        return d.pop(k)
    return None

@traced_function
def post_fact(fact_data):
    fact_data = dict(fact_data)

    key = fact_data.pop("key")

    category_name = None
    if "category" in fact_data:
        category_name = fact_data.pop("category")

    category = None
    if category_name:
        category, _ = FactCategory.objects.get_or_create(name = category_name)

    logger.info("Attempting to post new fact (with key: %s)", key)

    kwargs = {
        "category": category_name,
        "source_link": _maybe_pop(fact_data, "source_link"),
        "source": _maybe_pop(fact_data, "source"),
        "fine_print": _maybe_pop(fact_data, "fine_print"),
    }

    if len(fact_data) != 1:
        raise BadRequest(f"Invalid fact data (unknown type): {' '.join(fact_data)}")

    fact_type = list(fact_data)[0]
    fact_payload = list(fact_data.values())[0]
    fact_hashable_obj = {
        fact_type: fact_payload,
        **kwargs,
    }
    fact_hashable = canonicaljson.encode_canonical_json(fact_hashable_obj)

    fact_hash = hashlib.sha256(fact_hashable).hexdigest()

    logger.info("Attempting to post new fact (with key: %s, hash: %s)", key, fact_hash)

    old_fact = Fact.objects.filter(
        key = key,
        active = True,
    ).first()
    if old_fact:
        if old_fact.version_hash and old_fact.version_hash == fact_hash:
            logger.info("Attempting to post new fact (with key: %s, hash: %s): rejected, same as active fact", key, fact_hash)
            return old_fact

    field_name = fact_type + "_fact"

    core = FACT_MODELS[fact_type].objects.create(**fact_payload)

    kwargs["category"] = category
    kwargs[field_name] = core

    Fact.objects.filter(key = key).update(active = False)
    return Fact.objects.create(
        key = key,
        active = True,
        version_hash = fact_hash,
        fact_type = fact_type,
        **kwargs,
    )

@traced_function
def get_eval_stats_for_scope(user, limit=None, **kwargs):
    qs = Response.objects.filter(user=user, **kwargs)
    if limit is None:
        objs = list(qs.all())
    else:
        objs = qs.order_by("-creation_time")[:limit]
    correct = [o.correct for o in objs]
    confidences = [o.confidence_percent/100 for o in objs]
    assert len(correct) == len(confidences)
    expected = sum(confidences)
    confidence_correctness = list(zip(confidences, correct))
    plausibility = calculate_plausibility_of(confidence_correctness)
    return {
        "number_of_answers": len(objs),
        "number_of_correct_answers": sum(correct),
        "expected_correct_answers": expected,
        "plausibility": plausibility,
    }

@traced_function
def get_eval_stats(user):
    now = datetime.datetime.now()
    cutoff_24h = now - datetime.timedelta(hours=24)
    rv = {
        "stats": {
            "total": get_eval_stats_for_scope(user),
            "24h": get_eval_stats_for_scope(user, creation_time__gte=cutoff_24h),
            "last50": get_eval_stats_for_scope(user, limit=50),
            "last10": get_eval_stats_for_scope(user, limit=10),
        },
    }
    logger.info("Returning evaluation stats: %s", repr(rv))
    return rv

@traced_function
def get_summarizable_responses_qs(user, batch_size):
    return Response.objects.filter(
        user = user,
        resolved = True,
    ).exclude(
        summary_scores__batch_size = batch_size,
    ).order_by("creation_time")

@traced_function
def get_summarizable_responses(user, batch_size):
    qs = get_summarizable_responses_qs(user, batch_size)
    if qs.count() < batch_size:
        return None
    return qs[:batch_size]

@traced_function
def maybe_summarize_responses(user, batch_size):
    batch = get_summarizable_responses(user, batch_size)
    if not batch:
        return None

    conf_corr = [
        (float(x.confidence_percent/100), x.correct)
        for x in batch
    ]

    actual_correct = sum(corr for _, corr in conf_corr)
    expected_correct = sum(float(conf) for conf, _ in conf_corr)

    resp = calculate_plausibility_of(conf_corr)
    assert resp

    rv = SummaryScore.objects.create(
        user = user,
        batch_size = batch_size,
        actual_correct = actual_correct,
        expected_correct = expected_correct,
        probability_fewer_correct = resp["prob_fewer"],
        probability_same_correct = resp["prob_same"],
        probability_more_correct = resp["prob_more"],
    )
    rv.datapoints.set(batch)
    return rv

@traced_function
def get_last_summary(user, batch_size=None):
    batch_size = batch_size or STANDARD_SUMMARY_BATCHES[0]
    return SummaryScore.objects.filter(
        user = user,
        batch_size = batch_size,
    ).order_by("-creation_time").first()

@traced_function
def get_batch_progress(user, batch_size=None):
    batch_size = batch_size or STANDARD_SUMMARY_BATCHES[0]
    qs = get_summarizable_responses_qs(user, batch_size)
    achieved = min(batch_size, qs.count())
    return achieved, batch_size

@traced_function
def get_largest_standard_summarized_batch_size(user):
    active_batch_sizes = SummaryScore.objects.filter(
        user = user,
    ).values_list("batch_size", flat=True).distinct()
    logger.info("Active batch sizes: %s", repr(active_batch_sizes))

    active_std = set(active_batch_sizes) & set(STANDARD_SUMMARY_BATCHES)

    logger.info("Active standard batch sizes (%s): %s", repr(STANDARD_SUMMARY_BATCHES), repr(active_std))
    if not active_std:
        return None

    return max(active_std)

@traced_function
def get_challenge_by_uid(user, uid):
    return Challenge.objects.get(user = user, uid = uid)

@traced_function
def delete_user_account(user):
    user.delete()

@traced_function
def get_summary_chart_data(user, batch_size = None):
    batch_size = batch_size or get_largest_standard_summarized_batch_size(user)

    if not batch_size:
        return None

    timestamps = []
    lower = []
    upper = []

    scores = SummaryScore.objects.filter(
        user = user,
        batch_size = batch_size,
    ).all()

    if not scores:
        return None

    for score in scores:
        timestamps.append(score.creation_time.strftime("%Y-%m-%d %H:%M"))

    return {
	"type": "line",
	"data": {
                "labels": [str(x) for x in timestamps],
		"datasets": [
                    {
                        "label": "Fewer (underconfident)",
			"data": [x.probability_fewer_correct for x in scores],
                        "borderColor": "#ffaaaa",
                        "backgroundColor": "#ee9999",
                    },
                    {
                        "label": "Same (calibrated)",
			"data": [x.probability_same_correct for x in scores],
                        "borderColor": "#aaffaa",
                        "backgroundColor": "#99ee99",
                    },
                    {
                        "label": "More (overconfident)",
			"data": [x.probability_more_correct for x in scores],
                        "borderColor": "#aaaaff",
                        "backgroundColor": "#9999ee",
                    },
                ],
	},
        "options": {
            "title": {
                "display": True,
                "text": f"Expected correct answers given calibration (batches of {batch_size})",
            },
            "tooltips": {
                "mode": "index",
            },
            "scales": {
                "yAxes": [
                    {
                        "stacked": True,
                    },
                ],
            },
        },
    }

@traced_function
def export_facts(user):
    if not user.is_staff:
        logger.info("User (%s) is not staff user: refusing export", repr(user))
        raise PermissionDenied()

    qs = Fact.objects.filter(
        active = True,
    ).select_related("category")

    logger.info("Exporting %d facts.", qs.count())

    rv = []

    for obj in qs.order_by("pk"):
        attrname = obj.fact_type + "_fact"
        kwargs = getattr(obj, attrname).export()

        logger.info("Exporting fact: %s (cat: %s)", repr(obj), repr(obj.category))

        rv.append(apitype.Fact(
            key = obj.key,
            category = obj.category.name if obj.category else None,
            fine_print = obj.fine_print,
            source = obj.source,
            source_link = obj.source_link,
            **kwargs,
        ))

    return [{k: v for k, v in x.dict().items() if v} for x in rv]

@traced_function
def export_fact_categories(user):
    if not user.is_staff:
        logger.info("User (%s) is not staff user: refusing export", repr(user))
        raise PermissionDenied()

    qs = FactCategory.objects.all()

    logger.info("Exporting %d fact categories.", qs.count())

    rv = []

    for obj in qs.order_by("pk"):
        rv.append(apitype.FactCategory(
            name = obj.name,
            weight = decimal.Decimal(obj.weight),
        ))

    return [{k: v for k, v in x.dict().items() if v} for x in rv]

@traced_function
def post_fact_category(cat_data):
    cat = apitype.FactCategory.parse_obj(cat_data)

    old_cat = FactCategory.objects.filter(
        name = cat.name,
    ).first()

    if old_cat:
        old_cat.weight = cat.weight
        old_cat.save()
        return

    return FactCategory.objects.create(
        name = cat.name,
        weight = cat.weight,
    )


# Biblioteca que cria as APIs e Rotas
from flask import Flask, jsonify
# Biblioteca do Json simples para objetos com varios tipos de elementos (decimal, string, inteiro, etc)
import simplejson as json
# Código Fonte que respode na Rota Default /
from Controller import home
# Código Fonte que respode na Rota Clientes
from Controller import clientes
# Código Fonte que respode na Rota Receitas
from Controller import receita
# Biblioteca que imprime na tela os resultados melhor organizados
from pprint import pprint

app = Flask(__name__)

# Cria a Rota Home
@app.route('/', methods=['GET'])
def index():
    response = home.home()
    return response,200

# Cria Rota CLientes
@app.route('/clientes', methods=['GET'])
def ListaCliente():
    # Busca os CLientes do ERPFlex
    response = clientes.getClientes()
    # Converte o Objeto em Json
    return jsonify(response),200

# Cria Rota das Receitas
@app.route('/receitas', methods=['GET'])
def ListaReceitas():
    # Busca as Receitas no ERPFlex
    response = receita.getReceita()
    # Converte o Objeto em Json
    return jsonify(response),200

# Roda o Server
if __name__ == '__main__':
    app.run(debug=True)


from typing import Optional
from collections import deque

class Node:
    """
    Binary tree node.
    """

    def __init__(self, level: int, data: Optional[int] = None):
        self.level = level
        self.data = data
        self.left = None
        self.right = None

    def print_tree(self):
        output = []
        if not self.data:
            return output
        if self.left:
            output.extend(self.left.print_tree())
        output.append(self.data)
        if self.right:
            output.extend(self.right.print_tree())
        return output

class BinaryTree:
    """
    This is different from a binary search tree
    Because a binary search tree, left is always smaller than right.
    This is not the case here.
    """
    def __init__(self, root=None):
        self.root = root if root else Node(1, 1)

    def insert(self, value, level):
        # works on a binary search tree
        current = self.root
        while current:
            if value > current.data:
                if current.right:
                    current = current.right
                else:
                    current.right = Node(level, value)
                    break
            else:
                if current.left:
                    current = current.left
                else:
                    current.left = Node(level, value)
                    break
        return 1

    def custom_insert(self, indexes):
        queue = deque([self.root])
        for count, value in enumerate(indexes):
            if len(queue) > 0:
                node_of_interest = queue.popleft()
                left_value, right_value = value[0], value[1]
                if left_value > 0:
                    node_of_interest.left = Node(node_of_interest.level + 1, left_value)
                    queue.append(node_of_interest.left)
                if right_value > 0:
                    node_of_interest.right = Node(node_of_interest.level + 1, right_value)
                    queue.append(node_of_interest.right)
        return 1

    def swap(self, level_to_swap):
        queue = deque([self.root])
        while queue:
            node = queue.popleft()
            if node.level % level_to_swap == 0:
                node.left, node.right = node.right, node.left
            if node.left:
                queue.append(node.left)
            if node.right:
                queue.append(node.right)

def print_tree(root: Node):
    output = []
    if not root:
        return output
    if root.left:
        output.extend(print_tree(root.left))
    output.append(root.data)
    if root.right:
        output.extend(print_tree(root.right))
    return output

def swap_nodes(indexes, queries):
    output = []
    binary_tree = BinaryTree(Node(1, 1))
    binary_tree.custom_insert(indexes)
    for query in queries:
        binary_tree.swap(query)
        output.append(binary_tree.root.print_tree())
    return 0

if __name__ == "__main__":
    # above solution is running into a runtime error when given larger data sets
    # swap_nodes([[2, 3], [-1, 4], [-1, 5], [-1, -1], [-1, -1]], [2])
    swap_nodes([[2, 3]], [1])


fname = 'python_para_todos\Cap 9\words_romeo.txt'

try:
    fhand = open(fname)
except:
    print('No se ha encontrado el archivo.')
    exit()

counts = dict()

for linea in fhand:
    palabras = linea.split()
    for palabra in palabras:
        counts[palabra] = counts.get(palabra, 0) + 1

print(counts)


from flask import Flask, jsonify, request

from src.backend import process

app = Flask(__name__)

TEXT = "text"
ROOMS_N_MIN = "rooms_n_min"
ROOMS_N_MAX = "rooms_n_max"
SIZE_MIN = "size_min"
SIZE_MAX = "size_max"
PRICE_MIN = "price_min"
PRICE_MAX = "price_max"

@app.route('/real_estates', methods=['POST'])
def tmp():
    content = request.json
    real_estates = process(
        content[TEXT],
        content[PRICE_MAX],
        content[PRICE_MIN],
        content[SIZE_MAX],
        content[SIZE_MIN],
        content[ROOMS_N_MAX],
        content[ROOMS_N_MIN],
    )

    return jsonify(real_estates)


"""
Escrito por Alexander DIniz dos Santos para avaliação de perfil.

Este script busca solucionar a seguinte questão:
        QUESTÃO 01
    Dado um array de números inteiros, retorne os índices dos dois números de forma que eles se
    somem a um alvo específico.
    Você pode assumir que cada entrada teria exatamente uma solução, e você não pode usar o
    mesmo elemento duas vezes.
"""

#!/bin/sh

def sum_to_target(num_list, alvo):
    """
    Busca somas (dois números) fornecidos na num_list
    que resultem no valor da variável alvo.

    :type num_list: List[int]
    :type alvo: int
    :rtype: List[int]
    """

    # Validação de dados
    if not isinstance(alvo, int):
        raise TypeError("Parâmetro 'alvo' deveria ser um inteiro.")
    if not isinstance(num_list, list):
        raise TypeError("Parâmetro 'num_list' deveria ser uma lista.")
    if not all(isinstance(num, int) for num in num_list):
        raise TypeError("A lista 'num_list' deveria apenas conter inteiros.")

    mapa_valores = {}
    NUM_LIST_LEN = range(len(num_list)) # Tamanho da lista num_list
    for i in NUM_LIST_LEN: # Iterando num_list
        num = num_list[i] # Número que estamos usando para comparação
        par = alvo - num # O número que precisamos achar na lista
        if par in mapa_valores: # Se isto aqui for válido, o par foi encontrado na lista.
            return [mapa_valores[par], i] # Retorna uma lista com o resultado
        mapa_valores[num] = i # Guarda no mapa
    return # Não encontrado.


#!/usr/bin/python3

def safe_print_list(my_list=[], x=0):
    """print x elements of a list and returns the number of element printed"""
    ret = 0

    for num in range(x):
        try:
            print("{}".format(my_list[num]), end="")
            ret += 1
        except IndexError:
                  break
    print("")
    return (ret)


# Construct the QR decomposition of a real, symmetric matrix A and use
# this to find the eigenvalues and eigenvectors.
#
# We follow the procedure outlined in Newman section 6.2

from __future__ import print_function

import numpy as np

def get_QR(A):
    """ given A, return the matrices Q and R, such that Q
        is an orthogonal matrix and R is upper-triangular """

    N = A.shape[0]
    if not A.shape == (N, N):
        print("ERROR: matrix A must be square")
        return None

    Q = np.zeros_like(A)
    R = np.zeros_like(A)

    q = []
    u = []

    # create the orthonormal basis vectors from the
    # columns of A -- note we need to copy A here, since
    # just indexing a column gives a view into the array
    # A, and we'd be changing A as we alter u
    u.append(np.copy(A[:,0]))
    q.append(u[0]/np.sqrt(u[0].dot(u[0])))

    for i in range(1, N):
        u.append(np.copy(A[:,i]))
        for j in range(i):
            u[i] -= q[j].dot(A[:,i])*q[j]

        q.append(u[i]/np.sqrt(u[i].dot(u[i])))

    # create Q
    for n in range(N):
        Q[:,n] = q[n]

    # create R
    for i in range(N):
        for j in range(i,N):
            if i == j:
                R[i,j] = np.sqrt(u[i].dot(u[i]))
            else:
                R[i,j] = q[i].dot(A[j])

    return Q, R

def eigen(A, eps=1.e-6):
    """ use the QR algorithm for a real, symmetric matrix to geth
    the eigenvalues and eigenvectors """

    N = A.shape[0]
    if not A.shape == (N, N):
        print("ERROR: matrix A must be square")
        return None

    V = np.eye(N)

    err = 1.e10
    A_old = np.copy(A)
    while err > eps:
        Q, R = get_QR(A_old)
        A_new = R.dot(Q)
        V = V.dot(Q)

        A2 = A_new**2  # element-by-element
        err = np.sqrt(np.sum(A2) - np.trace(A2))   # this is kinda an L2 norm of the off-diagonals

        A_old = A_new

    return A_new, V

A = np.array([[1, 4, 8, 4],
              [4, 2, 3, 7],
              [8, 3, 6, 9],
              [4, 7, 9, 2]], dtype=np.float64)

Lambda, V = eigen(A, eps=1.e-10)

for n in range(A.shape[0]):
    print("eigenvalue: {}".format(Lambda[n,n]))
    print("    eigenvector:    {}".format(V[:,n]))
    print("    A v - lambda v: {}\n".format(A.dot(V[:,n]) - Lambda[n,n]*V[:,n]))


import json
import logging
import os
from datetime import datetime

import telegram.ext
from dateutil import tz
from telegram import (InlineKeyboardButton, InlineKeyboardMarkup,
                      ReplyKeyboardMarkup, ReplyKeyboardRemove)
from telegram.ext import (CallbackQueryHandler, CommandHandler,
                          ConversationHandler, Filters, MessageHandler,
                          Updater)

import parsedatetime
from common.constants import RegExpressions, RoleType, StateType
from common.exceptions import NoUserNameException, UserBannedException
from common.messages import Errors, Messages
from common.models import Report, Request
from common.mysql_connector import MySQLConnector
from helpers.filters import ResultFilters
from helpers.helpers import (check_blacklist, check_request_exists,
                             check_search_user_valid, get_location,
                             get_location_json, get_locations_json)
from matcher.matcher import Matcher

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

logger = logging.getLogger(__name__)

matcher_session = Matcher()
mysql_connector = MySQLConnector()

## Prod bot
updater = Updater(os.environ.get('TELE_KEY_PROD'), use_context=True)

# Get the dispatcher to register handlers
dp = updater.dispatcher
bot = updater.bot

(
    ROLE, LOCATION_PICKUP, NUM_DROPOFFS, LOCATION_DROPOFF,
    PRICE, TIME, PACKAGE_TYPE, ADDITIONAL_INFO
) = range(8)

def listing_formatter(request):
    if request.role == RoleType.DRIVER:
        return (
            '*Driver*\n'
            f'{request.first_name}\n'
            'Additional information:\n'
            f'{request.additional_info}'
        )
    to_zone = tz.gettz(os.environ['TZ'])
    utc = tz.gettz('UTC')

    time_format = (
        request.time.replace(tzinfo=utc).astimezone(to_zone).strftime('%d, %b %Y %I:%M %p')
    )
    pickup_location = json.loads(request.location_pickup)['text']
    dropoff_locations = json.loads(request.location_dropoffs)
    dropoff_text = 'Dropoffs:'
    for count, dropoff_location in enumerate(dropoff_locations, start=1):
        dropoff_text += '\n{}) {}'.format(count, dropoff_location['text'])

    return (
        '*Customer*\n'
        f'{request.first_name}\n'
        f'Pickup: {pickup_location}\n'
        f'{dropoff_text}\n'
        f'Time: {time_format}\n'
        f'Price: {request.price}\n'
        f'Package info: {request.package_type}\n'
        'Additional information:\n'
        f'{request.additional_info}'
    )

def validate_add_to_db(context, update):
    user = update.message.from_user
    chat_id = update.message.chat_id

    if not user.username:
        raise NoUserNameException

    with mysql_connector.session() as db_session:
        existing_entry = (
            db_session
            .query(Request)
            .filter_by(chat_id=chat_id)
            .one_or_none()
        )
        if existing_entry:
            # Disabled flood checks
            db_session.delete(existing_entry)
            db_session.commit()

        if context.chat_data['role'] == 'Driver':
            new_request = Request(
                chat_id=chat_id,
                additional_info=context.chat_data['additional_info'],
                state=StateType.ACTIVE,
                role=RoleType.DRIVER,
                first_name=user.first_name,
                last_name=user.last_name,
                telegram_handle=user.username
            )
        else:
            new_request = Request(
                location_pickup=get_location_json(context.chat_data['location_pickup']),
                location_dropoffs=get_locations_json(context.chat_data['location_dropoffs']),
                chat_id=chat_id,
                additional_info=context.chat_data['additional_info'],
                state=StateType.ACTIVE,
                time=context.chat_data['time'],
                price=context.chat_data['price'],
                package_type=context.chat_data['package_type'],
                role=RoleType.CUSTOMER,
                first_name=user.first_name,
                last_name=user.last_name,
                telegram_handle=user.username
            )

        db_session.add(new_request)
        db_session.commit()
        update.message.reply_text(
            'Your request has been created\n\n' + listing_formatter(new_request),
            parse_mode=telegram.ParseMode.MARKDOWN
        )

def start(update, context):
    user = update.message.from_user
    with mysql_connector.session() as db_session:
        try:
            check_blacklist(db_session, update.message.chat_id)
        except UserBannedException as e:
            update.message.reply_text(str(e))
            logger.info("User %s was blacklisted and tried to access bot.", user.first_name)
            return ConversationHandler.END
        except Exception:
            return ConversationHandler.END

    reply_keyboard = [['Driver', 'Hitcher or Customer']]
    update.message.reply_text(Messages.WELCOME_MESSAGE)
    update.message.reply_text(
        'State your role',
        reply_markup=ReplyKeyboardMarkup(reply_keyboard, one_time_keyboard=True))

    return ROLE

def role(update, context):
    user = update.message.from_user
    logger.info('Role of %s: %s', user.first_name, update.message.text)
    context.chat_data['role'] = update.message.text

    if context.chat_data['role'] == 'Driver':
        update.message.reply_text(
            Messages.ADDITIONAL_INFO_REQUEST,
            reply_markup=ReplyKeyboardRemove()
        )
        return ADDITIONAL_INFO

    update.message.reply_text(
        Messages.LOCATION_PICKUP_REQUEST,
        reply_markup=ReplyKeyboardRemove()
    )
    return LOCATION_PICKUP

def location_pickup(update, context):
    user = update.message.from_user

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return LOCATION_PICKUP

    user_location = update.message.text
    logger.info('Location of %s: %s', user.first_name, update.message.text)

    context.chat_data['location_pickup'] = user_location

    update.message.reply_text(Messages.LOCATION_NUM_DROPOFF_REQUEST)
    context.chat_data['location_dropoffs'] = []  # Clear the data if there are any
    return NUM_DROPOFFS

def num_dropoffs(update, context):
    user = update.message.from_user
    logger.info('Number of dropoffs for %s: %s', user.first_name, update.message.text)
    context.chat_data['num_dropoffs'] = int(update.message.text)
    update.message.reply_text(Messages.LOCATION_DROPOFF_REQUEST)
    return LOCATION_DROPOFF

def location_dropoff(update, context):
    user = update.message.from_user

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return LOCATION_DROPOFF

    user_location = update.message.text
    logger.info('Location of %s: %s', user.first_name, update.message.text)

    if 'location_dropoffs' not in context.chat_data:
        context.chat_data['location_dropoffs'] = [user_location]
    else:
        context.chat_data['location_dropoffs'].append(user_location)

    if len(context.chat_data['location_dropoffs']) < context.chat_data['num_dropoffs']:
        update.message.reply_text(Messages.LOCATION_DROPOFF_REQUEST_NEXT)
        return LOCATION_DROPOFF

    update.message.reply_text(Messages.PRICE_REQUEST)
    return PRICE

def price(update, context):
    user = update.message.from_user
    logger.info('Price of %s: %s', user.first_name, update.message.text)
    context.chat_data['price'] = float(update.message.text)
    update.message.reply_text(Messages.TIME_REQUEST)

    return TIME

def time(update, context):
    user = update.message.from_user
    cal = parsedatetime.Calendar()
    utc = tz.gettz('UTC')
    time_struct, parse_status = cal.parse(update.message.text)
    local_time = datetime(*time_struct[:6])
    utc_time = local_time.replace(tzinfo=tz.gettz(os.environ['TZ'])).astimezone(utc)
    if not parse_status:
        update.message.reply_text(
            Errors.INCORRECT_TIME
        )
        return TIME

    context.chat_data['time'] = utc_time

    logger.info(
        'Time of %s at %s',
        user.first_name,
        str(utc_time)
    )
    update.message.reply_text(Messages.PACKAGE_REQUEST)

    return PACKAGE_TYPE

def package_type(update, context):

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return PACKAGE_TYPE

    context.chat_data['package_type'] = update.message.text

    update.message.reply_text(Messages.ADDITIONAL_INFO_REQUEST)
    return ADDITIONAL_INFO

def additional_info(update, context):
    user = update.message.from_user
    logger.info("Additional info of %s: %s", user.first_name, update.message.text)

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return ADDITIONAL_INFO

    context.chat_data['additional_info'] = update.message.text

    context.job_queue.run_once(process_request, 0, context=(update, context))
    update.message.reply_text(Messages.THANK_YOU_MESSAGE)

    return ConversationHandler.END

def cancel(update, context):
    user = update.message.from_user
    logger.info("User %s canceled the conversation.", user.first_name)
    with mysql_connector.session() as db_session:
        my_request = db_session.query(Request).filter_by(chat_id=update.message.chat_id).one_or_none()
        if my_request:
            my_request.state = StateType.DONE
        db_session.commit()

    update.message.reply_text('Canceled',
                              reply_markup=ReplyKeyboardRemove())

    return ConversationHandler.END

def help_me(update, context):
    user = update.message.from_user
    # FIXME: Move this with mysql_connector.session() as db_session: chunk to another method
    # wrap it with retry connection
    with mysql_connector.session() as db_session:
        try:
            check_blacklist(db_session, update.message.chat_id)
        except UserBannedException as e:
            update.message.reply_text(str(e))
            logger.info("User %s was blacklisted and tried to access bot.", user.first_name)
            return ConversationHandler.END
        except Exception:
            return ConversationHandler.END

    update.message.reply_text(Messages.HELP_MESSAGE,
                              reply_markup=ReplyKeyboardRemove())

    return ConversationHandler.END

def list_all(update, context):
    user = update.message.from_user
    logger.info("User %s refreshed the list.", user.first_name)

    update.message.reply_text('Refreshing...',
                              reply_markup=ReplyKeyboardRemove())

    context.job_queue.run_once(refresh_handler, 0, context=(update, context))

    return ConversationHandler.END

def search_pickup(update, context):
    args = context.args
    user = update.message.from_user
    logger.info(
        "User {} searching the list for dropoffs that match. {}".format(user.first_name, str(args))
    )

    update.message.reply_text('Searching... {}'.format(' '.join(args)),
                              reply_markup=ReplyKeyboardRemove())

    context.job_queue.run_once(search_handler, 0, context=(update, context, 'pickup'))

    return ConversationHandler.END

def search_dropoff(update, context):
    args = context.args
    user = update.message.from_user
    logger.info(
        "User {} searching the list for pickups that match. {}".format(user.first_name, str(args))
    )

    update.message.reply_text('Searching... {}'.format(' '.join(args)),
                              reply_markup=ReplyKeyboardRemove())

    context.job_queue.run_once(search_handler, 0, context=(update, context, 'dropoff'))

    return ConversationHandler.END

def search_handler(context: telegram.ext.CallbackContext):
    update, context, search_type = context.job.context
    if not context.args:
        update.message.reply_text(Errors.SEARCH_ERROR)
        return
    with mysql_connector.session() as db_session:
        try:
            check_request_exists(db_session, update)
            check_search_user_valid(db_session, update)
        except Exception:
            return

    location_dict = get_location(' '.join(context.args))
    if search_type == 'dropoff':
        get_relevant_requests(update, context, dropoff_dict=location_dict)
    else:
        get_relevant_requests(update, context, pickup_dict=location_dict)

def refresh_handler(context: telegram.ext.CallbackContext):
    update, context = context.job.context
    with mysql_connector.session() as db_session:
        try:
            check_request_exists(db_session, update)
        except Exception:
            return

    get_relevant_requests(update, context)

def get_relevant_requests(update, context, pickup_dict=None, dropoff_dict=None):
    with mysql_connector.session() as db_session:
        curr_request = (
            db_session.query(Request)
            .filter_by(
                chat_id=update.message.chat_id,
                state=StateType.ACTIVE,
            )
            .one_or_none()
        )
        context.chat_data['chat_id'] = update.message.chat_id

        all_relevant_requests = (
            db_session.query(Request)
            .filter_by(
                role=(
                    RoleType.CUSTOMER if curr_request.role == RoleType.DRIVER else RoleType.DRIVER
                ),
                state=StateType.ACTIVE,
            )
            .all()
        )

        if pickup_dict or dropoff_dict:
            all_relevant_requests = ResultFilters.filter_location(
                all_relevant_requests,
                coordinates_pickup=pickup_dict['latlng'] if pickup_dict else None,
                coordinates_dropoff=dropoff_dict['latlng'] if dropoff_dict else None
            )

        update.message.reply_text(f'We have {len(all_relevant_requests)} requests now...')

        for request in all_relevant_requests:
            keyboard = [
                [
                    InlineKeyboardButton("Accept", callback_data=f'accept:{request.chat_id}'),
                    InlineKeyboardButton("Ignore", callback_data=f'ignore:{request.chat_id}')
                ],
                [InlineKeyboardButton("Report", callback_data=f'report:{request.chat_id}')],
            ]
            reply_markup = InlineKeyboardMarkup(keyboard)
            update.message.reply_text(
                f'{listing_formatter(request)}', reply_markup=reply_markup,
                parse_mode=telegram.ParseMode.MARKDOWN
            )

def process_request(context: telegram.ext.CallbackContext):
    update, context = context.job.context
    try:
        validate_add_to_db(context, update)

    except NoUserNameException:
        update.message.reply_text(Errors.NO_USERNAME)
        return

    except Exception as e:
        update.message.reply_text(Errors.GENERIC_ERROR)
        logger.warning(str(e))
        return

    update.message.reply_text(Messages.POST_ADDITIONAL_INFO_REQUEST)

def button(update, context):
    query = update.callback_query
    choice, chat_id = query.data.split(':')[0], int(query.data.split(':')[1])

    def reply_not_active():
        query.answer()
        query.edit_message_text(text=Errors.NO_LONGER_ACTIVE)

    request = None
    with mysql_connector.session() as db_session:
        request = (
            db_session.query(Request)
            .filter_by(
                chat_id=chat_id,
                state=StateType.ACTIVE
            )
            .one_or_none())

        if choice == 'report':
            existing_report = db_session.query(Report).filter_by(chat_id=chat_id).one_or_none()
            if existing_report:
                existing_report.report_count += 1
            else:
                db_session.add(Report(chat_id=chat_id, report_count=1))
            db_session.commit()
            query.answer()
            query.edit_message_text(text='Your feedback has been reported!')
            return

        elif choice == 'accept':
            matcher_session.add_match(context.chat_data['chat_id'], chat_id)
            if matcher_session.check_match(context.chat_data['chat_id'], chat_id):
                my_request = (
                    db_session.query(Request)
                    .filter_by(chat_id=context.chat_data['chat_id'])
                    .one_or_none()
                )
                if request and my_request:
                    try:
                        bot.send_message(
                            chat_id=context.chat_data['chat_id'],
                            text=f'Match found! @{request.telegram_handle}'
                        )
                        bot.send_message(
                            chat_id=chat_id,
                            text=f'Match found! @{my_request.telegram_handle}'
                        )
                        # Cleanup
                        matcher_session.delete_match(chat_id, context.chat_data['chat_id'])
                    except Exception as e:
                        logger.warning(str(e))
                else:
                    reply_not_active()
                    return

        if request:
            query.answer()
            query.edit_message_text(
                text=f'{listing_formatter(request)}\n\n{choice}!',
                parse_mode=telegram.ParseMode.MARKDOWN
            )
        else:
            reply_not_active()

def error(update, context):
    """Log Errors caused by Updates."""
    logger.warning('Update "%s" caused error "%s"', update, context.error)

def main():
    location_info_handler = ConversationHandler(
        entry_points=[
            CommandHandler('start', start),
            CommandHandler('list', list_all),
            CommandHandler('search_dropoff', search_dropoff),
            CommandHandler('search_pickup', search_pickup),
            CommandHandler('cancel', cancel),
            CommandHandler('help', help_me)
        ],
        states={
            ROLE: [MessageHandler(Filters.regex(r'^(Driver|Hitcher or Customer)$'), role)],
            LOCATION_PICKUP: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), location_pickup)
            ],
            NUM_DROPOFFS: [MessageHandler(Filters.regex(RegExpressions.NUMBER), num_dropoffs)],
            LOCATION_DROPOFF: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), location_dropoff)
            ],
            PRICE: [
                MessageHandler(Filters.regex(RegExpressions.PRICE), price)
            ],
            TIME: [MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), time)],
            PACKAGE_TYPE: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), package_type)
            ],
            ADDITIONAL_INFO: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), additional_info)
            ]
        },
        fallbacks=[CommandHandler('cancel', cancel)]
    )

    dp.add_handler(CallbackQueryHandler(button))
    dp.add_handler(location_info_handler)
    dp.add_error_handler(error)

    updater.start_polling()
    updater.idle()

if __name__ == '__main__':
    main()


def to_jaden_case(string):
    # ...
    stringSpl = string.split()

    result = []
    for i in stringSpl:
        result.append(i.capitalize())
    return " ".join(result)

string = "How can mirrors be real if our eyes aren't real"
print(to_jaden_case(string))

#return ' '.join(word.capitalize() for word in string.split())

#import string def to_jaden_case(str): return string.capwords(str)


import sys
input = sys.stdin.readline

# input
l = int(input())
lst = list(input().strip())
h = 0

# for rule
for i, v in enumerate(lst):
    lst[i] = ord(v) - 96

# hash function
for i, v in enumerate(lst):
    h += v * (31 ** i)

# result
print(h%1234567891)


from dataclasses import replace
import sys

sys.stdin = open(r"C:\\Users\\grivi\\vscodes\\.vscode\\input.txt","r")
sys.stdout = open(r"C:\\Users\\grivi\\vscodes\\.vscode\\output.txt","w")

s = "1"*70

while "2222" in s or "1111" in s:
    if "2222" in s:
        s = s.replace("2222","11",1)
    else:
        s = s.replace("1111", "22",1)
print(s)


import infrastructure.my_login as my_login

import applications.open_orders as open_orders

import database.entities as entities

def usd_orders():
    c = my_login.my_login()
    total_orders = open_orders.open_orders(c,'info')
    print("len(total_orders) = ",len(total_orders),flush=True)
    positions = [total_orders[coin]['info'] for coin in range(len(total_orders))]
    print("positions = ",positions,flush=True)
    return positions


'''
#### Name:  Maximum Subarray Sum
Link: [link]()

Given an array of integers (positive and negative) find the largest continuous sum.
'''

def max_continous_sum(arr):
    max_sum=current_sum = arr[0]

    for number in arr[1:]:
        current_sum = max(current_sum+number , number)
        max_sum = max(current_sum, max_sum)
    return max_sum

input_arr  = [1,2,-1,3,4,10,10,-10,-1]
print(max_continous_sum(input_arr))

Data/Arrays/Maximum Subarray Sum/Maximum Subarray Sum.py


def pegPrinter(*pegs,height=-1,width=-1):
    maxWidth = width if width > max(max(pegs)) else max(max(pegs))

    # dynamic height detection based on minimum needed height
    #maxHeight =  len(max(pegs,key=len))
    # dynamic height detection based on maximum needed height
    maxHeight = 0
    for peg in pegs: maxHeight += len(peg)
    # static height detection based input arguments
    maxHeight = height if height > maxHeight else maxHeight

    # top level printing
    for peg in range(len(pegs)):
        print("|" + " "*maxWidth, end="")
    print()

    #the rest of the printing
    for level in range(maxHeight):
        for peg in range(len(pegs)):
            outputString = ""
            if maxHeight-len(pegs[peg]) <= level:
                a = 0
                while a < pegs[peg][maxHeight-level-1]:
                    outputString += "#"
                    a += 1
                while a <= maxWidth:
                    outputString += " "
                    a += 1
            else:
                outputString = "|" + " "*maxWidth
            print(outputString, end="")
        print()
    print()

def hanoi(n, source, helper, target):
    if n > 0:
        pegPrinter(source, helper, target)
        hanoi(n - 1, source, target, helper)
        if source:
            target.append(source.pop())
        hanoi(n - 1, helper, source, target)
        pegPrinter(source, helper, target)

source = [4,3,2,1]
target = []
helper = []
hanoi(len(source),source,helper,target)

print(source, helper, target)
pegPrinter(source, helper, target)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
from scipy import signal
import os
import pylab

# make values from -5 to 5, for this example

UNKNOWN_FLOW_THRESH = 0.5

def readFlowFile(file):
    assert type(file) is str, "file is not str %r" % str(file)
    assert os.path.isfile(file) is True, "file does not exist %r" % str(file)
    assert file[-4:] == '.flo', "file ending is not .flo %r" % file[-4:]
    f = open(file, 'rb')
    flo_number = np.fromfile(f, np.float32, count=1)[0]
    w = np.fromfile(f, np.int32, count=1)
    h = np.fromfile(f, np.int32, count=1)
    # if error try: data = np.fromfile(f, np.float32, count=2*w[0]*h[0])
    # data = np.fromfile(f, np.float32, count=2*w*h)
    data = np.fromfile(f, np.float32, count=2*w[0]*h[0])
    # Reshape data into 3D array (columns, rows, bands)
    flow = np.resize(data, (int(h), int(w), 2))
    f.close()

    u = flow[: , : , 0]
    v = flow[: , : , 1]
    smaller_u = np.where(abs(u) < UNKNOWN_FLOW_THRESH)
    smaller_v = np.where(abs(v) < UNKNOWN_FLOW_THRESH)
    u[smaller_u] = 0
    u[smaller_v] = 0
    v[smaller_u] = 0
    v[smaller_v] = 0
    u = signal.medfilt(u, 5)
    v = signal.medfilt(v, 5)

    smaller_u = np.where(abs(u) < UNKNOWN_FLOW_THRESH)
    smaller_v = np.where(abs(v) < UNKNOWN_FLOW_THRESH)
    u[smaller_u] = -100
    u[smaller_v] = -100
    v[smaller_u] = -100
    v[smaller_v] = -100
    flow[: , : , 0] = u
    flow[: , : , 1] = v
    tmp = []
    for row in flow:
      for element in row:
        tmp.append(element)

    return tmp

fig, ax = plt.subplots(tight_layout=True)

for i in range(660, 825):
  print('../flow/flow%d.flo' % i)
  arr = readFlowFile('../flow/flow%d.flo' % i)

  x = []
  y = []
  for element in arr:
    x.append(element[0])
    y.append(element[1])

  hist = ax.hist2d(x, y, bins=100, range=[[-10, 10],[-10, 10]])
  plt.savefig('./plt/plt%d.png' % i)


#Equation
#((3+4)*7)

class BinaryTree:
    def __init__(self,root):
        self.root = root
        self.left_child = None
        self.right_child = None
    def insert_left(self,newchild):
        newest = BinaryTree(newchild)
        if self.left_child == None:
            self.left_child = newest
        else:
            temp = self.left_child
            self.left_child = newest
            newest.left_child = temp
        return newest
    def insert_right(self,newchild):
        newest = BinaryTree(newchild)
        if self.right_child == None:
            self.right_child = newest
        else:
            temp = self.right_child
            self.right_child = newest
            newest.right_child = temp
        return newest
    def getroot(self):
        return self.root
    def setroot(self,newroot):
        self.root = newroot
    def getleftchild(self):
        return self.left_child
    def getrightchild(self):
        return self.right_child
    def inorder_traversal(self):
        if self.left_child:
            self.left_child.inorder_traversal()
        print(self.root)
        if self.right_child:
            self.right_child.inorder_traversal()

class Stack:
    def __init__(self):
        self.data = []
        self.size =0
    def isEmpty(self):
        return self.size==0
    def push(self,element):
        self.size+=1
        self.data.append(element)
    def pop(self):
        if self.isEmpty():
            return "Stack is Empty"
        else:
            return self.data.pop()

def buildparsetree(inp_expr):
    operators = ['+','-','*','/']
    inp_list = inp_expr.split()                     #Expression list is created.
    root_tree = BinaryTree(" ")                     #Root node so as to come out of expression tree
    S = Stack()
    S.push(root_tree)                               #Root node is pushed in the Stack
    current_tree = root_tree                        #Current tree is the current subtree

    for i in inp_list:
        if i =='(':                                 #Rule 1
            current_tree.insert_left(" ")
            S.push(current_tree)
            current_tree = current_tree.getleftchild()

        elif i in operators:                        #Rule 3
            current_tree.setroot(i)
            current_tree.insert_right(" ")
            S.push(current_tree)
            current_tree = current_tree.getrightchild()
        elif i ==')':                               #Rule 2
            current_tree = S.pop()
        else:                                       #Rule 4
            current_tree.setroot(int(i))
            current_tree = S.pop()
    return root_tree

#expr_tree = buildparsetree("( ( 4 + 3 ) * 7 )")
#expr_tree.inorder_traversal()


class Car:
    def __init__(self, model, color, year, price) -> None:
        self._model = model
        self._color = color
        self._year = year
        self.__price = price
    #파이썬에서 인스턴스 변수를 private로 정의하려면
    # 변수 이름 앞에 __ (언더바 두개)를 붙이면 된다
    # __ 은닉화 된 정보는 Getter Setter로 접근

    def getPrice(self):
        return self.__price

    def setPrice(self, price):
        self.__price = price

newCar = Car("Genesis","Green","2023","6800")

print("자동차의 모델은",newCar._model)
print("자동차의 색상은",newCar._color)
print("자동차의 연식은",newCar._year)
print("자동차의 가격은",newCar.getPrice()) #접근 제한자떄문에 getter로 접근

빅데이터 활용 기초/객체와 클래스/클래스예제2.py


"""RAFT model
"""
import argparse
import logging
import os
from collections import OrderedDict

import h5py
import numpy as np
import torch
from tqdm import tqdm

from python_libs.image import load_image, show_img
from python_libs.optical_flow.raft import RAFT, flow_viz
from python_libs.video import get_frames

class RaftModel:
    """RaftModel"""

    def __init__(
        self,
        model_path: str,
    ) -> None:
        args = argparse.Namespace(model=model_path, small=False, mixed_precision=False)
        self.model = RAFT(args)
        self.pretrained_weights = torch.load(args.model)
        if torch.cuda.is_available():
            self.device = "cuda"
            self.model = torch.nn.DataParallel(self.model)
            self.model.load_state_dict(self.pretrained_weights)
            self.model.to(self.device)
        else:
            self.device = "cpu"
            self.pretrained_weights = self.get_cpu_model(self.pretrained_weights)
            self.model.load_state_dict(self.pretrained_weights)
        logging.basicConfig(format="%(asctime)s %(message)s")

    def get_cpu_model(self, model):
        """Get cpu model

        Args:
            model (RAFT): model

        Returns:
            OrderedDict: The new model
        """
        new_model = OrderedDict()
        # get all layer's names from model
        for name in model:
            # create new name and update new model
            new_name = name[7:]
            new_model[new_name] = model[name]
        return new_model

    @staticmethod
    def visualize_flow(img, flo):
        """Show flow as image

        Args:
            img (NDArray): original image
            flo (NDArray): The up flow
        """
        # permute the channels and change device is necessary
        img = img[0].permute(1, 2, 0).cpu().numpy()
        flo = flo[0].permute(1, 2, 0).cpu().numpy()

        # map flow to rgb image
        flo = flow_viz.flow_to_image(flo)

        img_flo = np.concatenate([img, flo], axis=0)
        img_flo = load_image(img_flo)
        show_img(img_flo)

    @staticmethod
    def save_flow(flo, name, parent_path):
        """Save flow as image

        Args:
            flo (NDArray): The up flow
            name (str): The name
            parent_path (str): folder
        """
        img = load_image(flow_viz.flow_to_image(flo))
        final_path = os.path.join(parent_path, name)
        img.save(final_path)

    def process_video(
        self, path: str, frame_time: int, width: int, height: int, low=True
    ):
        """Process video

        Args:
            path (str): Video path
            frame_time (int): Time between frames
            width (int): Width
            height (int): Height
            low (bool, optional): Save the original flow. Defaults to True.

        Returns:
            Dict: Flow
        """
        frames = get_frames(path, time=frame_time)
        idx = list(frames.keys())
        counter = 0
        flow = {}
        with torch.no_grad():
            frame_1 = (
                torch.from_numpy(
                    np.array(load_image(frames.get(idx[0])).resize((width, height)))
                )
                .permute(2, 0, 1)
                .float()
                .unsqueeze(0)
                .to(self.device)
            )
            for i in range(1, len(idx) - 1):
                frame_2 = (
                    torch.from_numpy(
                        np.array(load_image(frames.get(idx[i])).resize((width, height)))
                    )
                    .permute(2, 0, 1)
                    .float()
                    .unsqueeze(0)
                    .to(self.device)
                )
                flow_low, flow_up = self.model(
                    frame_1, frame_2, iters=12, test_mode=True
                )
                if low:
                    flow[idx[i]] = flow_low[0].permute(1, 2, 0).cpu().numpy()
                else:
                    flow[idx[i]] = flow_up[0].permute(1, 2, 0).cpu().numpy()
                frame_1 = frame_2
                counter += 1
        return flow

    def process_video_hdf5(
        self,
        path: str,
        frame_time: int,
        width: int,
        height: int,
        hdf5_path: str,
        low=True,
    ) -> str:
        """Process video

        Args:
            path (str): _description_
            frame_time (int): _description_
            width (int): _description_
            height (int): _description_
            low (bool, optional): _description_. Defaults to True.

        Returns:
            str: _description_
        """
        if not os.path.isfile(hdf5_path):
            flow = self.process_video(path, frame_time, width, height, low)
            with h5py.File(hdf5_path, "w") as h_file:
                for index, curr_flow in tqdm(flow.items()):
                    h_file.create_dataset(str(index), data=curr_flow)
            return hdf5_path
        else:
            print(f"{hdf5_path} already processed")
            return None

    def process_dataset(
        self,
        frame_time: int,
        width: int,
        height: int,
        list_file: str,
        root_path: str,
        ext: str = "_flow.hdf5",
        save_file=False,
    ):
        """Process the dataset"""

        self.model.eval()
        logging.info("Opening list file: %s with root path: %s", list_file, root_path)
        with open(list_file, "r") as file:
            video_list = file.read().splitlines()
            for video_name in video_list:
                path = os.path.join(root_path, video_name)
                if os.path.isfile(f"{path}"):
                    if save_file:
                        self.process_video_hdf5(
                            path, frame_time, width, height, f"{path}{ext}"
                        )
                    else:
                        self.process_video(path, frame_time, width, height)


n = int(input())

for i in range(n):
    linha = input().strip()
    esq = 0
    dir = 0
    for j in range(len(linha)):
        if linha[j] == '<':
            esq += 1
        elif linha[j] == '>':
            if esq > 0:
                dir += 1
                esq -= 1
    print(dir)


from transformers import AutoModelForSequenceClassification
from transformers import AutoTokenizer, AutoConfig
from scipy.special import softmax
import os
import pandas as pd
from preprocess import preprocess as pp

class SentimentModel:
    def __init__(self, model_name):
        self.model_name = model_name
        # Check if tokenizer already exists, otherwise create it
        tokenizer_saved_path = "cardiffnlp/tokenizer"
        if os.path.exists(tokenizer_saved_path):
            self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_saved_path)
        else:
            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
            self.tokenizer.save_pretrained(tokenizer_saved_path)
        self.config = AutoConfig.from_pretrained(model_name)
        self.model = AutoModelForSequenceClassification.from_pretrained(model_name)

    def score(self, text):
        encoded_input = self.tokenizer(text, return_tensors="pt")
        output = self.model(**encoded_input)
        scores = output[0][0].detach().numpy()
        scores = softmax(scores)
        return scores

    def score_from_tweets_df(self, df):
        # Create a new DataFrame to store the average sentiment score for each date
        sentiment_df = pd.DataFrame(
            columns=[
                "Date",
                "Text",
                "Negative_Score",
                "Natural_Score",
                "Positive_Score",
            ]
        )

        # Iterate over each date and calculate the average sentiment score for all tweets on that date
        for tweet in df.iterrows():
            text = tweet[1].text
            date = tweet[1].Date
            sentiment_score = self.score(text)
            new_row = pd.DataFrame(
                {
                    "Date": date,
                    "Text": text,
                    "Negative_Score": sentiment_score[0],
                    "Natural_Score": sentiment_score[1],
                    "Positive_Score": sentiment_score[2],
                },
                index=[0],
            )
            sentiment_df = pd.concat([sentiment_df, new_row], ignore_index=True)

        sentiment_df = pp.calculate_avarge_rolling(sentiment_df,'Negative_Score',250)
        return sentiment_df