# Utilizar para Acentos e Caractéres especiais o encoding="utf-8"

# Utilizar o r -> (read) para arquivos mais simples:

with open("arquivo.txt", "w") as arquivo:  # Criando o arquivo
    arquivo.write("Rio de Janeiro")
# print(arquivo)

with open("arquivo.txt", "r", encoding="utf-8") as arquivo:  # Lendo e printando no terminal
    file = arquivo.read()
print(file)

# # Utilizar o a -> (append) para adicionar algo no arquivo:

with open("arquivo.txt", "a", encoding="utf-8") as arquivo:
    arquivo.write("\nSão Paulo")

# Podemos utilizar o readlines() para arquivos maiores

with open("arquivo2.txt", "r", encoding="utf-8") as arquivo:  # Arquivo criado anteriormente
    arquivo = arquivo.readlines()
print(arquivo)


import json

import boto3
import ulid as ulid
from Logger import Logger
from constants import *
from FileManager import FileManager
from DataframeManager import DataframeManager

s3 = boto3.resource('s3')
s3_client = boto3.client('s3')

def results(status, job_execution_id, processed, failed):
    return {
        'statusCode': status,
        'body': json.dumps({
            'job_execution_id': job_execution_id,
            'processed_files': [f's3://{f.bucket_name}/{f.key}' for f in processed],
            'failed_files': [f's3://{f.bucket_name}/{f.key}' for f in failed]
        })
    }

def main(event, context):
    logger = Logger(__name__)
    job_execution_id = ulid.ulid()
    file_manager = FileManager(s3, s3_client)
    dataframe_manager = DataframeManager()

    logger.info('start processing')
    logger.info(f'execution id={job_execution_id}')
    landed_files_by_type = file_manager.find_landed_files()
    logger.info(f'landed {len(landed_files_by_type)} file(s)')
    process_files = dataframe_manager.process_landed_files(landed_files_by_type, job_execution_id, file_manager)
    return results(200, job_execution_id, process_files, [])

if __name__ == '__main__':
    main(None, None)


# Generators and Performance

# Generators are really useful when calculating large sets of data, particularly when
# long loops are used and there is no need to story store that memory.

from time import time

def performance(fn):
    def wrapper(*args, **kwargs):
        t1 = time()
        result = fn(*args, **kwargs)
        t2 = time()
        print(f'took {t2-t1}ms')
        return result
    return wrapper

num = 1000000

@performance
def long_time():
    print('Func 1')
    for i in range(num):
        i*5

@performance
def long_time2():
    print('Func 2')
    for i in list(range(num)):
        i*5

long_time()
long_time2()


from django.conf import settings
from django.db import models

class Location(models.Model):
    id = models.AutoField(primary_key=True)
    name = models.TextField()
    description = models.TextField()

def get_default_new_item_location():
    try:
        return Location.objects.get(name=settings.NEW_ITEM_DEFAULT_LOCATION)
    except Location.DoesNotExist:
        return Location.objects.create(
            name=settings.NEW_ITEM_DEFAULT_LOCATION,
            description="Default new item location",
        )


import csv
import psycopg2
import pandas as pd
import os.path
from os import path
from numpy import genfromtxt, savetxt, count_nonzero

# Returns the address of the entry
def locationName(row, city):
    if city == "denver":
        if pd.isna(row['INCIDENT_ADDRESS']):
            return "UNKNOWN"
        else:
            return row['INCIDENT_ADDRESS']
    else:
        if pd.isna(row['HUNDRED_BLOCK']):
            return "UNKNOWN"
        else:
            return row['HUNDRED_BLOCK']

# Returns the X coodinate of the entry
def locationX(row, city):
    if city == "denver":
        if pd.isna(row['GEO_X']):
            return -1
        else:
            return row['GEO_X']
    else:
        return row['X']

# Returns the Y coodinate of the entry
def locationY(row, city):
    if city == "denver":
        if pd.isna(row['GEO_Y']):
            return -1
        else:
            return row['GEO_Y']
    else:
        return row['Y']

# Returns the Neighbourhood name of the entry
def locationNeighbourhood(row, city):
    if city == "denver":
        return row['NEIGHBORHOOD_ID']
    else:
        return row['NEIGHBOURHOOD']

# Returns the Neighbourhood type of the entry
def locationNeighbourhoodType(row, city):
    if city == "denver":
        denverNeighbourhoodType  = ""
        denverInfo = csv.reader(open('../data/denverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverInfo:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverNeighbourhoodType = x[2]
        return denverNeighbourhoodType
    else:
        vancouverNeighbourhoodType = ""
        vancouverInfo = csv.reader(open('../data/vancouverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in vancouverInfo:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverNeighbourhoodType=x[5]
        return vancouverNeighbourhoodType

# Returns the City of the entry
def locationCity(row, city):
    if city == "denver":
        return "Denver"
    else:
        return "Vancouver"

# Returns the crimerate of the entry
def locationCrimerate(row, city):
    if city == "denver":
        denverNumberOfCrime = 0
        denverPopulation = 0

        denverCrime = csv.reader(open('../data/denverCrimeRate2018.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverCrime:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverNumberOfCrime=x[14]

        denverInfo = csv.reader(open('../data/denverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverInfo:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverPopulation = x[1]

        return (int(denverNumberOfCrime)/int(denverPopulation))*100000
    else:
        vancouverNumberOfCrime = 0
        vancouverPopulation = 0

        vancouverCrime = csv.reader(open('../data/vancouverCrimeRate2018.csv', "rt",encoding="utf8"), delimiter=",")
        for x in vancouverCrime:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverNumberOfCrime=x[11]

        vancouverInfo = csv.reader(open('../data/vancouverInfo.csv', "rt",encoding="utf8"), delimiter=",")

        for x in vancouverInfo:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverPopulation = x[3]
        if(vancouverNumberOfCrime == 0 or vancouverPopulation == 0):
            return 0
        else:
            return (int(vancouverNumberOfCrime)/int(vancouverPopulation))*100000

# Returns the number of crimes of the entry
def locationNumberOfCrimes(row, city):
    if city == "denver":
        denverNumberOfCrime = 0

        denverCrime = csv.reader(open('../data/denverCrimeRate2018.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverCrime:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverNumberOfCrime=x[14]

        return denverNumberOfCrime
    else:
        vancouverNumberOfCrime = 0

        vancouverCrime = csv.reader(open('../data/vancouverCrimeRate2018.csv', "rt",encoding="utf8"), delimiter=",")
        for x in vancouverCrime:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverNumberOfCrime=x[11]

        return vancouverNumberOfCrime

# Returns the Avg house hold income of the entry
def locationAvgHouseholdIncome(row, city):
    if city == "denver":
        denverHouseholdIncome = 0

        denverInfo = csv.reader(open('../data/denverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverInfo:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverHouseholdIncome=x[4]

        return denverHouseholdIncome
    else:
        vancouverHouseholdIncome = 0

        vancouverInfo = csv.reader(open('../data/vancouverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in vancouverInfo:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverHouseholdIncome=x[7]

        return vancouverHouseholdIncome

# Returns the Avg Property value of the entry
def locationAvgPropValue(row, city):
    if city == "denver":
        denverAvgPropValue = 0

        denverInfo = csv.reader(open('../data/denverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverInfo:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverAvgPropValue=x[3]

        return denverAvgPropValue
    else:
        vancouverAvgPropValue = 0

        vancouverInfo = csv.reader(open('../data/vancouverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in vancouverInfo:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverAvgPropValue=x[6]

        return vancouverAvgPropValue

# Returns the number of precincts of the entry
def locationNumOfPrecincts(row, city):
    if city == "denver":
        denverNumPrecincts = 0

        denverInfo = csv.reader(open('../data/denverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in denverInfo:
            if x[0] == row['NEIGHBORHOOD_ID']:
                denverNumPrecincts=x[5]

        return denverNumPrecincts
    else:
        vancouverNumPrecincts = 0

        vancouverInfo = csv.reader(open('../data/vancouverInfo.csv', "rt",encoding="utf8"), delimiter=",")
        for x in vancouverInfo:
            if x[0] == row['NEIGHBOURHOOD']:
                vancouverNumPrecincts=x[8]

        return vancouverNumPrecincts

# Returns a list of all the columns(attributes) of a row
def generateColumn(row,counter,city):
    columns =[]
    columns.append(counter)
    columns.append(locationName(row, city))
    columns.append(locationX(row, city))
    columns.append(locationY(row, city))
    columns.append(locationNeighbourhood(row, city))
    columns.append(locationNeighbourhoodType(row, city))
    columns.append(locationCity(row, city))
    columns.append(locationCrimerate(row, city))
    columns.append(locationNumberOfCrimes(row, city))
    columns.append(locationAvgHouseholdIncome(row, city))
    columns.append(locationAvgPropValue(row, city))
    columns.append(locationNumOfPrecincts(row, city))
    return columns

# This function creates the csv files for both cities to populate the database.
# Before calling this function make sure the denverLocation.csv and vancouverLocation.csv files are empty
def createLocationCsv(city):
    # ------------------------------------------------Denver csv file populating-----------------------------------------------------------
    counter = 1
    if(city == "denver"):
        denvor_dataset = pd.read_csv('../data/filteredDenverCrime.csv')

        locationRowsDenver = []

        numberOfRowsDenver=len(denvor_dataset.axes[0])

        for x in range(numberOfRowsDenver):
            denverRow = generateColumn(denvor_dataset.iloc[x],counter, "denver")
            locationRowsDenver.append(denverRow)
            counter = counter + 1

        with open('../data/final/denverLocation.csv','w+', newline= '') as f:
            writer = csv.writer(f)
            writer.writerow(['Location_key','Location_name','GeoX','GeoY','Neighbourhood','locationNeighbourhoodType','City','Crime_rate','NumberOfCrimes','AvgHouseholdIncome','AvgPropValue','NumOfPrecincts'])
            for row in locationRowsDenver:
                writer.writerow(row)

# ------------------------------------------------Vancouver csv file populating-----------------------------------------------------------
    else:
        with open('../data/final/denverLocation.csv') as f:
            denver_count = sum(1 for line in f)
        counter = denver_count
        vancouver_dataset = pd.read_csv('../data/filteredVancouverCrime.csv')

        locationRowsVancouver = []

        numberOfRowsVancouver=len(vancouver_dataset.axes[0])

        for x in range(numberOfRowsVancouver):
            vancouverRow = generateColumn(vancouver_dataset.iloc[x],counter, "vancouver")
            locationRowsVancouver.append(vancouverRow)
            counter = counter + 1

        with open('../data/final/vancouverLocation.csv','w+', newline= '') as f:
            writer = csv.writer(f)
            writer.writerow(['Location_key','Location_name','GeoX','GeoY','Neighbourhood','locationNeighbourhoodType','City','Crime_rate','NumberOfCrimes','AvgHouseholdIncome','AvgPropValue','NumOfPrecincts'])
            for row in locationRowsVancouver:
                writer.writerow(row)

# Creates a list of all the primary keys in the location table.
def locationPrimaryKeys():
    denver_count=0
    vancouver_count=0
    with open('../data/final/denverLocation.csv') as f:
        denver_count = sum(1 for line in f)
    with open('../data/final/vancouverLocation.csv') as f:
        vancouver_count = sum(1 for line in f)
    keys = []
    for x in range(denver_count+vancouver_count-2):
        if(x > 0):
            keys.append(x)
    return keys


#import packages
import pandas as pd
import numpy as np
import matplotlib.pylab as plt
from sklearn.cluster import	KMeans
from scipy.spatial.distance import cdist
from scipy.cluster.hierarchy import linkage
import scipy.cluster.hierarchy as sch

#load dataset
wine = pd.read_csv("wine.csv")

wine.describe()

#Normalization
def norm_func(i):
    x = (i-i.min())	/ (i.max()-i.min()) #min max scaler
    return (x)

# Normalized data frame (considering the numerical part of data)
df_norm = norm_func(wine)
df_norm.describe()

#Hierarchical Clustering
z = linkage(df_norm, method = "complete", metric = "euclidean")

# Dendrogram
plt.figure(figsize=(2, 14));plt.title('Hierarchical Clustering Dendrogram');plt.xlabel('Index');plt.ylabel('Distance')
sch.dendrogram(z,
    leaf_rotation = 0,  # rotates the x axis labels
    leaf_font_size = 10 # font size for the x axis labels
)
plt.show()

# Now applying AgglomerativeClustering choosing 4 as clusters from the above dendrogram
from sklearn.cluster import AgglomerativeClustering

h_complete = AgglomerativeClustering(n_clusters = 4, linkage = 'complete', affinity = "euclidean").fit(df_norm)
h_complete.labels_
h_complete
cluster_labels = pd.Series(h_complete.labels_)
cluster_labels
wine['clust'] = cluster_labels # creating a new column and assigning it to new column

wine = wine.iloc[:, [14,0,1,2,3,4,5,6,7,8,9,10,11,12,13]]
wine.head()

# Aggregate mean of each cluster
wine.groupby(wine.clust).mean()

# creating a csv file
wine.to_csv("PCA-HierarchicalClustering-Assignment.csv", encoding = "utf-8")

import os
os.getcwd()

# K-means clustering
# scree plot or elbow curve
TWSS = []
k = list(range(2, 14))

for i in k:
    kmeans = KMeans(n_clusters = i)
    kmeans.fit(df_norm)
    TWSS.append(kmeans.inertia_)

TWSS

# Scree plot
plt.plot(k, TWSS, 'ro-');plt.xlabel("No_of_Clusters");plt.ylabel("total_within_SS")

# Selecting 3 clusters from the above scree plot which is the optimum number of clusters
model = KMeans(n_clusters = 3)
model.fit(df_norm)

model.labels_ # getting the labels of clusters assigned to each row
mb = pd.Series(model.labels_)  # converting numpy array into pandas series object
wine['clust'] = mb # creating a  new column and assigning it to new column

wine.head()
df_norm.head()

wine = wine.iloc[:,[14,0,1,2,3,4,5,6,7,8,9,10,11,12,13]]
wine.head()

wine.iloc[:, 1:14].groupby(wine.clust).mean()

wine.to_csv("PCA-Kmeans.csv", encoding = "utf-8")

#PCA
from sklearn.decomposition import PCA
from sklearn.preprocessing import scale

# Normalizing the numerical data
uni_normal = scale(wine)
uni_normal

pca = PCA(n_components = 6)
pca_values = pca.fit_transform(uni_normal)

# The amount of variance that each PCA explains is
var = pca.explained_variance_ratio_
var

pca.components_
pca.components_[0]

# Cumulative variance
var1 = np.cumsum(np.round(var, decimals = 4) * 100)
var1

# Variance plot for PCA components obtained
plt.plot(var1, color = "red")

# PCA scores
pca_values

pca_data = pd.DataFrame(pca_values)
pca_data.columns = "comp0", "comp1", "comp2", "comp3", "comp4", "comp5"
final = pd.concat([wine.Univ, pca_data.iloc[:, 0:3]], axis = 1)

# Scatter diagram
import matplotlib.pylab as plt
plt.scatter(x = pca_data.pc0, y = pca_data.pc1)


class Solution(object):
    def findClosestElements(self, arr, k, x):
        """
        :type arr: List[int]
        :type k: int
        :type x: int
        :rtype: List[int]
        """
        arrdif = [(abs(i-x), i) for i in arr]
        arrdif.sort()
        topk = [i[1] for i in arrdif[:k]]
        topk.sort()
        return topk


class UtakaException(Exception):
	def __init__(self, args, httpStatus=200):
		self.args = args
		self.httpStatus = httpStatus

	def __str__(self):
		return str(httpStatus) + ": " + str(self.args)

	def toxml(self, debug=False):
		import xml.dom.minidom
		doc = xml.dom.minidom.Document()
		errorEl = doc.createElement("Error")
		for key, val in self.args.iteritems():
			keyEl = doc.createElement(str(key))
			keyEl.appendChild(doc.createTextNode(str(val)))
			errorEl.appendChild(keyEl)
		if debug:
			debugEl = doc.createElement("Debug")
			import traceback
			tracebackEl = doc.createElement("Traceback")
			tracebackEl.appendChild(doc.createTextNode(str(traceback.format_exc())))
			debugEl.appendChild(tracebackEl)
			if 'devArgs' in self.__dict__.keys():
				for key, val in self.devArgs.iteritems():
					keyEl = doc.createElement(str(key))
					keyEl.appendChild(doc.createTextNode(str(val)))
					debugEl.appendChild(keyEl)
			errorEl.appendChild(debugEl)
		doc.appendChild(errorEl)
		return doc.toxml('utf-8')


from menu import Menu
from direct.gui.DirectGui import *

class Controls(Menu):
    def __init__(self):
        super().__init__()
        self.parentNode = aspect2d.attachNewNode('Save')

        self.backButton = DirectButton(text=("back"), scale = 0.25,
                     command=self.switchToMainMenu, parent=base.a2dTopLeft,
                                       pos=(0.275,0,-0.225))

        self.background = OnscreenImage(parent=render2dp, image='tp/images/controls.png')
        base.cam2dp.node().getDisplayRegion(0).setSort(-20)


import allure

class TestLogin:
    def test_a(self):
        print("\n111")
        assert 1

    @allure.step("测试登录步骤")
    def test_b(self):
        print("\n222")
        allure.attach("描述1","请输入用户名")
        print("\n333")
        allure.attach("描述2","请输入密码")
        assert 0


 # -*- coding: utf-8 -*-
from django import template
from web.models import Project
import re

register = template.Library()

def carousel(value, arg):
    projects_list = Project.objects.filter(image__gt='').filter(incarousel=True)
    return projects_list

def truncatewords_by_chars(value, arg):
    """Truncate the text when it exceeds a certain number of characters.
    Delete the last word only if partial.
    Adds '...' at the end of the text.

    Example:

        {{ text|truncatewords_by_chars:25 }}
    """
    try:
        length = int(arg)
    except ValueError:
        return value

    if len(value) > length:
        if value[length:length + 1].isspace():
            return value[:length].rstrip() + '...'
        else:
            return value[:length].rsplit(' ', 1)[0].rstrip() + '...'
    else:
        return value

def show_person_box(person,width):
    return {'person': person,
            'width':width}

def add_http(value):
    """Add http protocol when is missing in a URL."""
    if re.match('http://\w*', value):
        return value
    else:
        return 'http://'+value

def show_title(current_page,args):
    if current_page == '':
       current_page = 'Home'
    return 'The CulturePlex ─ '+current_page

register.filter('show_title', show_title)
register.filter('add_http', add_http)
register.filter('carousel', carousel)
register.filter('truncatewords_by_chars', truncatewords_by_chars)
register.inclusion_tag('person_box.html')(show_person_box)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 29 10:26:12 2023

@author: mikewang
"""
import cv2
import numpy as np
from tqdm import tqdm
import os

#%% find the bbox from a whole image
# Load image, grayscale, Otsu's threshold
image = cv2.imread('1f8f08ea-b5b3-4f68-94d4-3cc071b7dce8.png')
Path_bagOfCells = 'filter_extracted_cells/'
Path_patchedimage = 'patched_examples/'
original = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = 255 - gray
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]

# Morph open to remove noise
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)

# Find contours, obtain bounding box, extract and save ROI
ROI_number = 0
cnts = cv2.findContours(opening, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]

cnts_length = len(cnts)
bbox_array = np.zeros([4,cnts_length])
overlapped_check_array = np.ones([cnts_length,], dtype = 'int32')

#%% check the relative size and the color of the cells in the image
# first store the bbox coordinate
for i, c in enumerate(cnts):
    x,y,w,h = cv2.boundingRect(c) # bounding box parameters
    bbox_array[:,i] = np.array([x,y,w,h])

# calculate the relateive size of the bbox
bbox_h_var = np.var(bbox_array[3,:])
bbox_w_var = np.var(bbox_array[2,:])
bbox_h_mean = np.mean(bbox_array[3,:])
bbox_w_mean = np.mean(bbox_array[2,:])

# rgb mean and var array
rgb_mean_var_array = np.zeros([6,len(cnts)])
for i, c in enumerate(cnts):
    x,y,w,h = cv2.boundingRect(c)
    cv2.rectangle(image, (x, y), (x + w, y + h), (36,255,12), 2)
    ROI = original[y:y+h, x:x+w]
    ROI_x, ROI_y, channels = ROI.shape
    ROI_linear = ROI.reshape(ROI_x*ROI_y, channels)
    rgb_mean_var_array[0,i], rgb_mean_var_array[3,i] = np.mean(ROI_linear[:,0]), np.var(ROI_linear[:,0])
    rgb_mean_var_array[1,i], rgb_mean_var_array[4,i] = np.mean(ROI_linear[:,1]), np.var(ROI_linear[:,1])
    rgb_mean_var_array[2,i], rgb_mean_var_array[5,i] = np.mean(ROI_linear[:,2]), np.var(ROI_linear[:,2])
rgb_mean_var = np.mean(rgb_mean_var_array,axis=1)

#%% matching for relatively similiar color and size cell in the bag
cell_image_names = []

for name in os.listdir(Path_bagOfCells):
    if ".png" in name:
        cell_image_names.append(name)

num_cell = len(cell_image_names)
cell_index_array = np.array(range(num_cell))
np.random.shuffle(cell_index_array)

# find the matching cell image
for i in range(num_cell):
    cell_index = cell_index_array[i]
    cell_image_name = cell_image_names[cell_index]
    cell_image = cv2.imread(Path_bagOfCells+cell_image_name)
    cell_image_x, cell_image_y, cell_image_channel = cell_image.shape
    cell_image_linear = cell_image.reshape([cell_image_x*cell_image_y, cell_image_channel])
    cell_image_r_mean, cell_image_r_var = np.mean(cell_image_linear[:,0]), np.var(cell_image_linear[:,0])
    cell_image_g_mean, cell_image_g_var = np.mean(cell_image_linear[:,1]), np.var(cell_image_linear[:,1])
    cell_image_b_mean, cell_image_b_var = np.mean(cell_image_linear[:,2]), np.var(cell_image_linear[:,2])
    cell_image_color_mean_var_array = np.array([cell_image_r_mean, cell_image_g_mean, cell_image_b_mean, cell_image_r_var, cell_image_g_var, cell_image_b_var])
    error = np.linalg.norm(rgb_mean_var-cell_image_color_mean_var_array)
    if error < 1:
        break

#%% patching
# example find one of the bbox and replace it with a patch
c_example = cnts[20]
x,y,w,h = cv2.boundingRect(c_example) # bounding box location in the picture
cell_image = cv2.resize(cell_image, (w,h))
patched_image = original.copy()
patched_image[y:y+h, x:x+w] = cell_image

cv2.imwrite(Path_patchedimage+'original_{}.png'.format(cell_image_name), original)
cv2.imwrite(Path_patchedimage+'patched_{}.png'.format(cell_image_name), patched_image)


# -*- coding: utf-8 -*
import os
import time
import numpy as np
import Utils.config_brain as CONFIG
from argparse import ArgumentParser
from collections import OrderedDict
from tqdm import tqdm

import torch
import torch.nn as nn
import torch.optim as optim
from Dataloader_LGNet.dataloader_nodown import ZSData

import matplotlib.pyplot as plt
plt.switch_backend('agg')
from prefetch_generator import BackgroundGenerator
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve, auc
from torch.utils.tensorboard import SummaryWriter
from Model.create_model import create_model

from Utils.utils import seed_torch, transform_valid, SCELoss
import Utils.utils.transform_train as transform

Writer = None
bit = True

class DataloaderX(torch.utils.data.DataLoader):
    def __iter__(self):
        return BackgroundGenerator(super().__iter__())

def run(model, dataloaders, phase, criterion, criterion2, optimizer, n_count):
    with tqdm(total=n_count, desc = phase, unit='img', ncols = 100) as pbar:

        if phase == 'train':
            model.train()
        elif phase == 'valid':
            model.eval()
        else:
            raise Exception('Error Phase')

        all_labels = np.array([])
        all_predicts = np.array([])
        all_values = np.array([])
        running_loss = 0.0
        for input, label in dataloaders:
            input, label = input.cuda(non_blocking = True), label.cuda(non_blocking = True)
            with torch.torch.set_grad_enabled(phase == 'train'):
                pbar.update(len(input))
                output = model(input)
                _, pred = torch.max(output, 1)
                value = output[: , 1]
                loss = criterion(output, label)
                optimizer.zero_grad()
                if phase == 'train':
                    loss = criterion(output, label)
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
                elif phase == 'valid':
                    loss = criterion2(output, label)
                    optimizer.zero_grad()
            pbar.set_postfix(**{'loss (batch)': loss.item()})

            running_loss += loss.item() * input.size(0)
            all_labels = np.concatenate((all_labels, label.cpu().numpy()))
            all_predicts = np.concatenate((all_predicts, pred.cpu().numpy()))
            all_values = np.concatenate((all_values, value.detach().cpu().numpy()))

        Loss = running_loss / len(all_labels)
        Acc1 = (all_predicts == all_labels).sum()
        Acc = Acc1 / len(all_labels)
        TP = ((all_predicts == 1) & (all_labels == 1)).sum()
        TN = ((all_predicts == 0) & (all_labels == 0)).sum()
        FN = ((all_predicts == 0) & (all_labels == 1)).sum()
        FP = ((all_predicts == 1) & (all_labels == 0)).sum()

        p = TP / (TP + FP)
        r = TP / (TP + FN)
        F1 = 2 * r * p / (r + p)
        glioma_p = TN / (TN + FN)
        glioma_r = TN / (TN + FP)
        conf_matrix = confusion_matrix(all_labels, all_predicts, labels=list(range(CONFIG.NUM_CLASSES)))
        print('%s loss:' % phase, Loss)
        print('%s Acc:' % phase, Acc)

        Writer.add_scalar('%s/Loss' % phase, Loss, global_epoch)
        Writer.add_scalar('%s/Acc' % phase, Acc, global_epoch)

        fpr, tpr, theshold = roc_curve(all_labels, all_values, pos_label=1)
        print('AUC', auc(fpr, tpr))

        Writer.add_scalar('{}/AUC'.format(phase.capitalize()), auc(fpr, tpr), global_epoch)
        Writer.add_scalar('%s/lymphoma_Precision' % phase.capitalize(), p, global_epoch)
        Writer.add_scalar('%s/lymphoma_Recall' % phase.capitalize(), r, global_epoch)
        Writer.add_scalar('%s/lymphoma_F1' % phase.capitalize(), F1, global_epoch)
        Writer.add_scalar('%s/glioma_Precision' % phase.capitalize(), glioma_p, global_epoch)
        Writer.add_scalar('%s/glioma_Recall' % phase.capitalize(), glioma_r, global_epoch)
        Writer.add_scalar('%s/sum_Precision' % phase.capitalize(), (p+glioma_p), global_epoch)
        Writer.add_scalar('%s/sum_Recall' % phase.capitalize(), (r+glioma_r), global_epoch)

        return Loss, p, r, F1, Acc, auc(fpr, tpr), conf_matrix, (r+glioma_r)

def start_train(train_loader, valid_loader, model, device, criterion, criterion2, optimizer, scheduler, num_epochs, n_train, n_valid, swa_model, swa_start, swa_scheduler):
    best_F1 = .0
    best_auc = .0
    best_acc = .0
    best_loss = 1000000
    best_recall = .0
    best_sum_r = .0

    for epoch in range(1, num_epochs + 1):

        global global_epoch
        global_epoch = epoch
        print('epoch:' , global_epoch)

        train_loss, train_p, train_r, train_F1, train_acc, train_auc, train_conf_matrix,_ = run(model, train_loader, 'train', criterion, criterion2, optimizer, n_train)
        valid_loss, valid_p, valid_r, valid_F1, valid_acc, valid_auc, valid_conf_matrix,valid_sumr = run(model, valid_loader, 'valid', criterion, criterion2, optimizer, n_valid)

        if epoch >= swa_start:
            swa_model.update_parameters(model)
            swa_scheduler.step()
            last_lr = swa_scheduler.get_last_lr()[0]
            Writer.add_scalar('Learing rate', last_lr, epoch)
        else:
            scheduler.step()
            last_lr = scheduler.get_last_lr()[0]
            Writer.add_scalar('Learing rate', last_lr, epoch)

        if valid_F1 > best_F1:
            best_F1 = valid_F1
            torch.save(model.state_dict(), os.path.join(checkpoint, 'bestF1.pt'))

        if valid_loss < best_loss:
            best_loss = valid_loss
            es = 0
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'bestloss.pt')
            )
        else:
            es += 1
            print("Counter {} of 10".format(es))

        if es > 99:
            print("Early stopping with epoch %i" % epoch)
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'es.pt'))
            break

        if valid_auc > best_auc:
            best_auc = valid_auc
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'bestauc.pt')
            )
        if valid_acc > best_acc:
            best_acc = valid_acc
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'bestacc.pt')
            )
        if valid_auc > best_auc:
            best_auc = valid_auc
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'bestauc.pt')
            )
        if valid_r > best_recall:
            best_recall = valid_r
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'bestrecall.pt')
            )
        if valid_sumr > best_sum_r:
            best_sum_r = valid_sumr
            torch.save(model.state_dict(), os.path.join(
                checkpoint, 'best_sumrecall.pt')
            )
        if epoch % 10 == 0:
            torch.save(model.state_dict(), os.path.join(checkpoint, '%i.pt' % epoch))

        print('epoch %s complete' % global_epoch)
    torch.save(model.state_dict(), os.path.join(checkpoint, 'final.pt'))

    torch.optim.swa_utils.update_bn(train_loader, swa_model)
    torch.save(swa_model.state_dict(), os.path.join(checkpoint, 'swa.pt'))

def make_weights_for_balanced_classes(label_list, nclasses):
    count = [0] * nclasses
    print(label_list)
    for item in label_list:
        count[item] += 1
    weight_per_class = [0.0] * nclasses
    N = float(sum(count))
    print(N)
    for i in range(nclasses):
        weight_per_class[i] = N / float(count[i])
    weight = [0] * len(label_list)
    for num, label in enumerate(label_list):
        weight[num] = weight_per_class[label]
    return weight, weight_per_class

def prepare_data(img_path):
    train_dataset = ZSData(os.path.join(img_path, 'train'), '', transforms = transform, bi = bit)
    valid_dataset = ZSData(os.path.join(img_path, 'valid'), '', transforms = transform_valid, bi = bit)
    nclasses = len(CONFIG.CLASSES)
    weights, weights_per_class = make_weights_for_balanced_classes(train_dataset.get_label_list(), nclasses)
    train_loader = DataloaderX(train_dataset, shuffle = True, batch_size = batch_size, num_workers = 16, pin_memory = True)
    valid_loader = DataloaderX(valid_dataset, shuffle = True, batch_size = batch_size, num_workers = 16, pin_memory = True)

    return train_loader, valid_loader, weights_per_class, train_dataset.__len__(), valid_dataset.__len__()

if __name__ == '__main__':

    parser = ArgumentParser()
    parser.add_argument("--TrainFolder", dest='train_folder', type=str)
    parser.add_argument("--FoldN", dest='foldn', type=int)
    parser.add_argument("--Loss", dest='loss_func', type=str)
    parser.add_argument("--NumEpoch", dest='num_epochs', type=int)
    parser.add_argument("--Seed", dest='seed', type=int)
    parser.add_argument("--Model", dest='model', type=str)
    parser.add_argument("--LearningRate", dest='learning_rate', type=float)
    parser.add_argument("--BatchSize", dest='batch_size', type=int)
    parser.add_argument("--WeightDecay", dest='weight_decay', type=float)# learning rate decay
    parser.add_argument("--DeviceId", dest='device_id', type=str)
    parser.add_argument("--Comment", dest='comment', type=str)
    parser.add_argument("--Pretrain", dest='pretrain', action="store_false")
    parser.add_argument("--Modelpath", dest='modelpath', type=str)

    args = parser.parse_args()

    seed_torch(args.seed)

    batch_size = args.batch_size

    loss_func = args.loss_func

    os.environ['CUDA_VISIBLE_DEVICES'] = args.device_id
    os.environ['TORCH_HOME']='./pretrained_model'#path of imgnet-pretrained model checkpoint
    now = time.strftime("%Y_%m_%d_", time.localtime())

    global checkpoint
    checkpoint = os.path.join(CONFIG.CHECK_POINT, now+'split4-1_direct_classification,'+args.model,
        now+"split4-1_direct_classification,"+args.model+','+str(args.batch_size)+","+"seed"+str(args.seed)+",fold"+str(args.foldn))

    print('Summary write in %s' % checkpoint)

    Writer = SummaryWriter(log_dir=checkpoint)

    train_loader, valid_loader, weight_per_class, n_train, n_valid = prepare_data(os.path.join(args.train_folder, str(args.foldn)))

    print(len(CONFIG.CLASSES), 'classes:', CONFIG.CLASSES)

    print('num train images %d x %d' % (len(train_loader), args.batch_size))

    print('num val images %d x %d' % (len(valid_loader), args.batch_size))

    print("CUDA is_available:", torch.cuda.is_available())

    if args.device_id is not None:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    print('\n### Build model %s' % args.model)

    if args.modelpath != None:
        state_dict = torch.load(args.modelpath)
        state_dict = state_dict['state_dict']
        new_state_dict = OrderedDict()
        for k, v in state_dict.items():
            if 'backbone' in k:
                k = k.replace('module.backbone.', '')
            new_state_dict[k]=v
        model = create_model(args.model, args.pretrain)

        model_dict = model.state_dict()
        ignore = {k: v for k, v in new_state_dict.items() if k not in model_dict}

        weights = {k: v for k, v in new_state_dict.items() if k in model_dict}
        model_dict.update(weights)
        model.load_state_dict(model_dict, True)
    else:
        model = create_model(args.model, args.pretrain)
    if torch.cuda.device_count() == 2:
        print(os.environ['CUDA_VISIBLE_DEVICES'])
        model = torch.nn.DataParallel(model, device_ids=[0,1])
    elif torch.cuda.device_count() == 4:
        print(os.environ['CUDA_VISIBLE_DEVICES'])
        model = torch.nn.DataParallel(model, device_ids=[0,1,2,3])
    model = model.to(device)

    # cross entropy loss
    if loss_func == 'cross':
        weight_per_class = torch.Tensor(weight_per_class).to(device)
        criterion = nn.CrossEntropyLoss(weight = weight_per_class)
        criterion2 = nn.CrossEntropyLoss()
        criterionsce = SCELoss(a=0.1, b=1, num_classes=2)
    optimizer = optim.SGD(model.parameters(),
                            momentum = 0.9,
                            lr=args.learning_rate,
                            weight_decay=args.weight_decay)

    #scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[20, 40], gamma=0.1)
    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.num_epochs, eta_min=0.)

    swa_model = torch.optim.swa_utils.AveragedModel(model)
    swa_start = 95
    swa_scheduler = torch.optim.swa_utils.SWALR(optimizer, swa_lr=0.0001)
    start_train(train_loader, valid_loader, model, device, criterionsce, criterion2,
                optimizer, scheduler, args.num_epochs, n_train, n_valid, swa_model, swa_start, swa_scheduler)


# -*- coding:utf-8 -*-
import configparser
import json
import os
import subprocess
import sys
import time
from datetime import datetime
from threading import Thread

import requests

sys.path.append(os.path.dirname(os.getcwd()))
from video_log_parser import ResultParser

def upload_file(fname):
    print('上传数据文件:', fname)
    f = {"file": open(fname, "rb")}
    r = requests.post(api_server + 'file', files=f)
    print('上传数据文件结果:', r.text)

def upload_status(stat):
    print('上传状态:', stat)
    r = requests.post(api_server + 'status', json=stat)
    print('上传状态结果:', r.text)

def save_local(fname, data):
    # 保存结果数据到本地
    with open(fname, 'w', encoding='utf8') as f:
        json.dump(data, f, ensure_ascii=False)

def parser_log(code, id, loc, pole_length, rtsp_num):
    # 分析日志
    rp = ResultParser('result_log/{}-{}-{}-{}.log'.format(code, id, loc, rtsp_num), code, id, loc, pole_length, rtsp_num)
    while True:
        json_result, end_file = rp.parser()
        # 上传图表数据
        fname = 'result_log/{}-{}-{}-{}-chart.log'.format(code, id, loc, rtsp_num)
        save_local(fname, rp.get_chart_data())
        upload_file(fname)
        # 上传结果json文件
        fname = 'result_log/{}-{}-{}-{}-result.json'.format(code, id, loc, rtsp_num)
        save_local(fname, json_result)
        upload_file(fname)
        # 上传卸钻视频
        list_upload_mp4 = rp.get_video_data()
        while len(list_upload_mp4):
            fn = list_upload_mp4.pop(0)
            if fn not in mp4_uploaded_cache:
                upload_file('result_mp4/{}'.format(fn))
                mp4_uploaded_cache.append(fn)
        # 结束分析
        if end_file:
            # 上传完整的视频文件
            upload_file('result_mp4/{}-{}-{}-{}.mp4'.format(code, id, loc, rtsp_num))
            print('----------结束分析----------\n----------{}-{}-{}-{}----------'.format(code, id, loc, rtsp_num))
            return
        time.sleep(10)

if __name__ == '__main__':
    """
        1. 从服务器获取任务Json
        2. 启动分析服务
        3. 开线程分析结果日志
        4. 开启直播服务
        5. 动态上传结果
    """
    ports = [23333, 23334, 23335, 23336, 23337, 23338, 23339, 23340, 23341, 23342, 23343, 23344, 23345, 23346, 23347, 23348, 23349, 23350]
    conf = configparser.ConfigParser()  # 读取配置信息
    conf.read("config.cfg")
    api_server = conf.get('config', 'api_server')
    data_path = conf.get('cmd', 'data_path')
    cfg_path = conf.get('cmd', 'cfg_path')
    weight_path = conf.get('cmd', 'weight_path')
    mp4_uploaded_cache = []  # 缓存已经上传的mp4，防止重复上传

    while True:
        resp = requests.get(api_server + 'job')  # 从服务器获取任务Json
        resp_json = json.loads(resp.text)
        if resp_json['data']:
            print('{}   拿到任务信息:{},开始执行任务.\n'.format(datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S"), resp.text))

            mine_code = resp_json['data']['cs_mine_code']
            work_location = resp_json['data']['cs_workface_location']
            work_id = resp_json['data']['cs_workface_id']
            pole_length = resp_json['data']['cs_pole_length']
            # hole_num = resp_json['data']['cs_hole_number'] if 'cs_hole_number' in resp_json['data'].keys() else 1

            gpu_num = 0
            for rtsp in resp_json['data']['cs_rtsp_list']:  # 启动多路视频监测分析
                # 根据任务参数生成日志文件（cs_mine_code-cs_workface_id-cs_workface_location-rtsp_number.log）
                log_name = 'result_log/{}-{}-{}-{}.log'.format(mine_code, work_id, work_location, rtsp['cs_channel_number'])

                # 生成分析指令
                cmd_training = 'nohup ./darknet detector demo {0} {1} {2} -i 0 -thresh 0.25 {9} -dont_show -http_port {10} -out_filename result_mp4/{3}-{4}-{5}-{6} -gpus {7} > {8} &'.format(
                    data_path, cfg_path, weight_path, mine_code, work_id, work_location, rtsp['cs_channel_number'], gpu_num, log_name, rtsp['cs_rtsp'], ports[gpu_num])

                print(cmd_training)
                subprocess.Popen(cmd_training, shell=True)

                upload_status({'name': '{}-{}-{}-{}'.format(mine_code, work_id, work_location, rtsp['cs_channel_number']), 'value': {'result': '检测程序启动成功', 'date': datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")}})
                time.sleep(6)

                # 开线程分析结果日志并启动直播服务
                Thread(target=parser_log, args=(mine_code, work_id, work_location, pole_length, rtsp['cs_channel_number'])).start()
                subprocess.Popen('nohup ./enlive_mul {0} {1} {2} {3} {4}&'.format(mine_code, work_id, work_location, rtsp['cs_channel_number'], ports[gpu_num]), shell=True)
                upload_status({'name': '{}-{}-{}-{}'.format(mine_code, work_id, work_location, rtsp['cs_channel_number']), 'value': {'result': '检测程序启动成功，直播服务启动成功', 'date': datetime.strftime(datetime.now(),
                                                                                "%Y-%m-%d %H:%M:%S")}})
                gpu_num += 1
        time.sleep(3)


from flask import current_app
from ldap3 import ObjectDef
from ..sanity import _check_dn_exists

def check_userbase_exists(fix=True):
    """
    Check the User base DN exists, and if not, create it.
    """
    conn = current_app.ldap3_login_manager.connection
    ou = ObjectDef("organizationalunit", conn)
    userbase = "%s,%s" % (
        current_app.config["LDAP_USER_DN"],
        current_app.config["LDAP_BASE_DN"],
    )
    _check_dn_exists(userbase, ou, fix)

def check_group_base_exists(fix=True):
    """
    Check the Group base DN exists, and if not, create it.
    """
    conn = current_app.ldap3_login_manager.connection
    ou = ObjectDef("organizationalunit", conn)
    groupbase = "%s,%s" % (
        current_app.config["LDAP_GROUP_DN"],
        current_app.config["LDAP_BASE_DN"],
    )
    _check_dn_exists(groupbase, ou, fix)


"""Implementation of different stochastic Modules (e.g., DropPath)."""

from typing import Any, Callable, Optional, Tuple

import flax.linen as nn
import jax.numpy as jnp
import jax.random
from jax import random

Array = Any
PRNGKey = Any
Shape = Tuple[int]
Dtype = Any

DEFAULT_KERNEL_INIT = nn.initializers.lecun_normal()
DEFAULT_BIAS_INIT = nn.initializers.zeros

class DropPath(nn.Module):
  """Create a stochastic depth layer.

  Follows dropout implementation from
  flax/linen/stochastic.py

    Attributes:
      rate: the drop probability.  (_not_ the keep rate!)
      deterministic: if false the inputs are scaled by `1 / (1 - rate)` and
        masked, whereas if true, no mask is applied and the inputs are returned
        as is.
  Copied from: https://github.com/google-research/nested-transformer/blob/main/libml/attn_utils.py
  """
  rate: float
  deterministic: Optional[bool] = None

  @nn.compact
  def __call__(self, inputs):
    if self.rate == 0.:
      return inputs
    keep_prob = 1. - self.rate
    if self.deterministic:
      return inputs
    else:
      # just use the same set of naming with dropout
      rng = self.make_rng("dropout")
      mask_shape = [inputs.shape[0]] + [1 for _ in inputs.shape[1:]]
      mask = jax.random.bernoulli(rng, p=keep_prob, shape=mask_shape)
      mask = jnp.tile(mask, [1] + list(inputs.shape[1:]))
      scaled_inputs = inputs / keep_prob
      scaled_inputs = scaled_inputs.astype(inputs.dtype)
      return jax.lax.select(mask, scaled_inputs, jnp.zeros_like(inputs))

class LayerScale(nn.Module):
  """LayerScale was defined in CaiT: https://arxiv.org/abs/2103.17239"""
  features: int
  scale_init: float
  dtype: Dtype = jnp.float32

  @nn.compact
  def __call__(self, inputs):
    _init_fn = lambda key, shape: jnp.ones(shape, dtype=jnp.float32) * self.scale_init
    scale_param = self.param('scale_param', _init_fn, (self.features,))
    scale_param = jnp.asarray(scale_param, self.dtype)
    return inputs * scale_param

class MlpBlock(nn.Module):
  """MLP blocks.

  Based on Flax implementation:
  https://github.com/google/flax/blob/main/examples/lm1b/models.py
  """
  mlp_dim: Optional[int] = None
  out_dim: Optional[int] = None
  dense_fn: Callable = nn.Dense  # pylint: disable=g-bare-generic
  activation_fn: Any = nn.gelu
  proj_drop: float = 0.0
  kernel_init: Callable[[random.PRNGKey, Shape, Dtype], Array] = DEFAULT_KERNEL_INIT
  bias_init: Callable[[random.PRNGKey, Shape, Dtype], Array] = DEFAULT_BIAS_INIT
  use_bias: bool = True
  train: bool = True
  dtype: Any = jnp.float32

  @nn.compact
  def __call__(self, x):
    actual_out_dim = (x.shape[-1] if self.out_dim is None else self.out_dim)
    mlp_dim = x.shape[-1] if self.mlp_dim is None else self.mlp_dim
    x = self.dense_fn(mlp_dim,
                      dtype=self.dtype,
                      kernel_init=self.kernel_init,
                      use_bias=self.use_bias,
                      bias_init=self.bias_init)(x)
    x = self.activation_fn(x)
    x = nn.Dropout(self.proj_drop, deterministic=not self.train)(x)
    x = self.dense_fn(actual_out_dim,
                      dtype=self.dtype,
                      kernel_init=self.kernel_init,
                      use_bias=self.use_bias,
                      bias_init=self.bias_init)(x)
    x = nn.Dropout(self.proj_drop, deterministic=not self.train)(x)
    return x


"""Database wrapper for MongoDB for tasks project."""

import os
import pymongo
import subprocess
import time
from bson.objectid import ObjectId
from typing import Optional, List

class TasksDB_MongoDB():    # noqa: E801
    """Wrapper class for MongoDB.

    The methods in this class need to match
    all database interaction classes.

    So far, this is:
    TasksDB_TinyDB found in tasksdb_tinydb.py.
    """

    def __init__(self, db_path: str) -> None:
        """Start MongoDB client and connect to db."""
        self._process = None
        self._client = None
        # Старт сервера MongoDB и определение процесса self._process
        self._start_mongod(db_path)
        # Соединение с БД, создание БД и пользовательского счетчика
        self._connect()

    def add(self, task: dict) -> int:
        """Add a task dict to db."""
        task['_id'] = self._get_next_task_id()
        return self._db.task_list.insert_one(task).inserted_id

    def get(self, task_id: int) -> dict:
        """Return a task dict with matching id."""
        task_dict = self._db.task_list.find_one({'_id': task_id})
        task_dict['id'] = task_dict.pop('_id')
        return task_dict

    def list_tasks(self, owner: Optional[str]=None) -> List[dict]:
        """Return list of tasks."""
        if owner is None:
            all = self._db.task_list.find()
        else:
            all = self._db.task_list.find({'owner': owner})
        all = list(all)
        # WARNING all - это итератор, тут он опустошается
        for task_dict in all:
            task_dict['id'] = task_dict.pop('_id')
        return all

    def count(self) -> int:
        """Return number of tasks in db."""
        return self._db.task_list.estimated_document_count()

    def last_id(self) -> int:
        """Return last id from db."""
        if self.count():
            return self._db.task_list.find_one(sort=[('_id', -1)])['_id']
        return self.count()

    def update(self, task_id: int, task: dict) -> None:
        """Modify task in db with given task_id."""
        self._db.task_list.find_one_and_update({'_id': task_id}, {'$set': task})

    def delete(self, task_id: int) -> None:
        """Remove a task from db with given task_id."""
        reply = self._db.task_list.delete_one({'_id': task_id})
        if reply.deleted_count == 0:
            raise ValueError('id {} not in task database'.format(str(task_id)))

    def unique_id(self) -> int:
        """Return an integer that does not exist in the db."""
        tail_task_ids = self._db.counters.find_one({'_id': 'tasksid'})
        uid = tail_task_ids['seq'] + 1
        return uid

    def delete_all(self) -> None:
        """Remove all tasks from db."""
        self._db.task_list.drop()

    def stop_tasks_db(self) -> None:
        """Disconnect from db."""
        self._disconnect()
        self._stop_mongod()

    def _start_mongod(self, db_path: str) -> None:
        """Старт сервера mongod в отдельном процессе."""
        self._devnull = open(os.devnull, 'wb')
        self._process = subprocess.Popen(
            ['mongod', '--dbpath', db_path],
            stdout=self._devnull,
            stderr=subprocess.STDOUT
        )
        assert self._process, 'mongod process failed to start'

    def _stop_mongod(self) -> None:
        if self._client:
            self._client = None
        if self._process:
            self._process.terminate()
            self._process.wait()
            self._devnull.close()
            self._process = None

    def _connect(self) -> None:
        if self._process and (not self._client or not self._db):
            for _ in range(3):
                try:
                    # соединение с БД
                    self._client = pymongo.MongoClient()
                except pymongo.errors.ConnectionFailure:
                    time.sleep(0.1)
                    continue
                else:
                    break
            if self._client:
                # создание БД task_db
                self._db = self._client.task_db
                # создание идентификатора tasksid, начинающегося с 0, в коллекции counters (специальной ?)
                # WARNING вероятно, pymongo где-то создает _id: 1 (можно ли его сбросить ?)
                # ANSWER вроде сработало! Просто заменяем встроенный _id своим.
                # self._db.counters.insert_one({'_id': 'tasksid', 'seq': 0})
                # self._reset_task_id()
                self._set_last_field()

    def _reset_task_id(self) -> None:
        self._db.counters.find_one_and_update({'_id': 'tasksid'},
                                              {'$set': {'seq': 0}})

    def _set_last_field(self) -> None:
        self._db.counters.find_one_and_update({'_id': 'tasksid'},
                                              {'$set': {'seq': self.last_id()}})

    def _get_next_task_id(self):
        ret = self._db.counters.find_one_and_update({'_id': 'tasksid'},
                                                    {'$inc': {'seq': 1}},
                                                    return_document=pymongo.ReturnDocument.AFTER)
        return ret['seq']

    def _disconnect(self) -> None:
        self._db = None

def start_tasks_db(db_path: str) -> TasksDB_MongoDB:
    """Connect to db."""
    return TasksDB_MongoDB(db_path)

if __name__ == '__main__':
    from tasks import Task
    task = start_tasks_db('C:\\Users\\DEUS\\tests\\mongo_db')
    print('id на старте (указатель):', task.last_id())
    # Вставляем 3 записи
    print('1-e add:', task.add(Task('do something great')._asdict()))
    print('last id:', task.last_id())
    print('2-e add:', task.add(Task('repeat', owner='Brian')._asdict()))
    print('last id:', task.last_id())
    print('3-e add:', task.add(Task('again and again', owner='Okken')._asdict()))
    print('last id:', task.last_id())
    # Вставляем 4-ю запись
    task_id = (task.add(Task('testing', owner='Brian')._asdict()))
    print('get id=last:', task.get(task.last_id()))
    print('count:', task.count())
    print('Все записи:')
    for entry in task.list_tasks():
        print(entry)
    print('-'*40)
    print('list_tasks(Brian):', task.list_tasks('Brian'))
    task.delete(task.last_id())
    print('count:', task.count())
    print('После удаления последней записи:')
    for entry in task.list_tasks():
        print(entry)
    print('-'*40)
    id2 = Task('again and again +', owner='Okken', done=True)._asdict()
    print('task для update:', id2)
    task.update(task.last_id(), id2)
    print('get id=last after update:', task.get(task.last_id()))
    print('count:', task.count())
    print('-'*40)
    for entry in task.list_tasks():
        print(entry)
    task.delete_all()
    task.stop_tasks_db()


from tkinter import Tk, Label, Button, Entry, CENTER
from phone_iso3166.country import phone_country
import phonenumbers
from phonenumbers import geocoder, carrier
import pycountry

class LocationTracker:
    def __init__(self, app):
        self.window = app
        self.window.title("Phone number Tracker")
        self.window.geometry("500x400")
        self.window.configure(bg="black")
        self.window.resizable(False, False)

        # ___________Application menu_____________
        Label(
            app,
            text="Enter a phone number",
            fg="cyan",
            font=("Times", 20, "bold", "italic"),
            bg="black",
        ).place(x=150, y=30)
        self.phone_number = Entry(
            app,
            bg="black",
            width=16,
            fg="#a6fb3f",
            font=("Times", 18, "bold", "italic"),
            relief="flat",
        )
        self.track_button = Button(
            app, fg="red", text="Track Country", bg="cyan", relief="sunk"
        )
        self.country_label = Label(
            app, fg="black", font=("Times", 20), bg="cyan", relief="flat"
        )
        self.test_label = Label(app, fg="black", font=("Times", 20), bg="cyan")

        # ___________Place widgets on the window______
        self.phone_number.place(x=165, y=120)
        self.track_button.place(x=185, y=200)
        self.country_label.place(x=150, y=280)
        self.test_label.place(x=200, y=320)

        # __________Linking button with countries ________
        self.track_button.bind("<Button-1>", self.phone_numbers)
        # 255757294146

    def phone_numbers(self, event):
        phone_number = self.phone_number.get()
        test = ""
        test_1 = ""
        number = phonenumbers.parse(phone_number)
        operator = carrier.name_for_number(number, "en")
        country = geocoder.description_for_number(number, "en")
        py_country = pycountry.countries.get(alpha_2=phone_country(phone_number))
        tracker = (operator, country, py_country, number)
        print(tracker)
        if py_country:
            test = py_country.official_name
            if operator:
                test_1 = operator
        self.test_label.configure(text=test_1)
        self.country_label.configure(text=test)

PhoneTracker = Tk()
MyApp = LocationTracker(PhoneTracker)
PhoneTracker.mainloop()


#!/usr/bin/python
# coding:utf-8

# @author: wangchangxin
# @contact: 986798607@qq.com
# @software: PyCharm
# @file: scores.py
# @License: GNU Lesser General Public License v3.0
"""
Notes:
    score method.
"""

import copy
import itertools
import warnings

import numpy as np
# import sympy
from sklearn import metrics
from sklearn.exceptions import DataConversionWarning
from sklearn.metrics import r2_score
from sklearn.model_selection import KFold
from sklearn.utils import check_array
from sympy import utilities, Expr, diff
from sympy.core import Function

from bgp.calculation.coefficient import try_add_coef, cla
from bgp.calculation.translate import compile_context
from bgp.functions.dimfunc import dim_map, dless, dnan, Dim
from bgp.functions.gsymfunc import NewArray

def calculate_y(expr01, x, y, terminals, add_coef=True, x_test=None, y_test=None,
                filter_warning=True, inter_add=True, inner_add=False, vector_add=False, out_add=False, flat_add=False,
                np_maps=None, classification=False):
    if filter_warning:
        warnings.filterwarnings("ignore")
    try:
        if add_coef:
            pre_y, expr01 = try_add_coef(expr01, x, y, terminals,
                                         filter_warning=filter_warning,
                                         inter_add=inter_add, inner_add=inner_add,
                                         vector_add=vector_add, out_add=out_add, flat_add=flat_add,
                                         np_maps=np_maps, classification=classification)
        else:
            func0 = utilities.lambdify(terminals, expr01, modules=[np_maps, "numpy"])
            pre_y = func0(*x)
            if classification:
                pre_y = cla(pre_y)

        if x_test is not None and y_test is not None:
            func0 = utilities.lambdify(terminals, expr01, modules=[np_maps, "numpy"])
            pre_y = func0(*x_test)
            if classification:
                pre_y = cla(pre_y)

            pre_y = pre_y.ravel()
            assert y_test.shape == pre_y.shape
            pre_y = check_array(pre_y, ensure_2d=False)
        else:
            pre_y = pre_y.ravel()
            assert y.shape == pre_y.shape
            pre_y = check_array(pre_y, ensure_2d=False)

    except (DataConversionWarning, TypeError, AssertionError, ValueError, AttributeError, KeyError, ZeroDivisionError):
        pre_y = None

    return pre_y, expr01

def calculate_y_unpack(expr01, x, terminals, classification=False):
    try:
        func0 = utilities.lambdify(terminals, expr01)
        pre_y = func0(*x)
        if classification:
            pre_y = cla(pre_y)
        pre_y = pre_y.ravel()
        pre_y = check_array(pre_y, ensure_2d=False)

    except (DataConversionWarning, AssertionError, ValueError, AttributeError, KeyError, ZeroDivisionError):
        pre_y = None
    return pre_y

def uniform_score(score_pen=1):
    """return the worst score"""
    if score_pen >= 0:
        return -np.inf
    elif score_pen <= 0:
        return np.inf
    elif score_pen == 0:
        return 0
    else:
        return score_pen

def calculate_score(expr01, x, y, terminals, scoring=None, score_pen=(1,),
                    add_coef=True, filter_warning=True, inter_add=True,
                    inner_add=False, vector_add=False, out_add=False, flat_add=False, np_maps=None,
                    classification=False, score_object="y", details=False):
    """

    Parameters
    ----------
    vector_add
    expr01: Expr
        sympy expression.
    x: list of np.ndarray
        list of xi
    y: np.ndarray
        y value
    terminals: list of sympy.Symbol
        features and constants
    scoring: list of Callbale, default is [sklearn.metrics.r2_score,]
        See Also sklearn.metrics
    score_pen: tuple of  1 or -1
        1 : best is positive, worse -np.inf
        -1 : best is negative, worse np.inf
        0 : best is positive , worse 0
    add_coef: bool
        bool.
    filter_warning: bool
        bool
    inter_add: bool
        bool
    inner_add: bool
        bool
    np_maps: Callable
        user np.ndarray function

    Returns
    -------
    score:float
        score
    expr01: Expr
        New expr.
    pre_y: np.ndarray or float
        np.ndarray or None
    """
    if filter_warning:
        warnings.filterwarnings("ignore")
    if not scoring:
        scoring = [r2_score, ]
    if not isinstance(scoring, (tuple, list)):
        scoring = [scoring, ]
    if isinstance(score_pen, int):
        score_pen = [score_pen, ]

    assert len(scoring) == len(score_pen), "the scoring and score_pen with same size"

    pre_y, expr01 = calculate_y(expr01, x, y, terminals, add_coef=add_coef,
                                filter_warning=filter_warning, inter_add=inter_add, inner_add=inner_add,
                                vector_add=vector_add, out_add=out_add, flat_add=flat_add,
                                np_maps=np_maps, classification=classification)

    if score_object == "y" or classification is True:
        try:
            sc_all = []
            for si, sp in zip(scoring, score_pen):
                sc = si(y, pre_y)
                if np.isnan(sc):
                    sc = uniform_score(score_pen=sp)
                sc_all.append(sc)

        except (ValueError, RuntimeWarning, TypeError):

            sc_all = [uniform_score(score_pen=i) for i in score_pen]

    else:

        ######dy under test########
        pre_dy, dy = calculate_derivative_y(expr01, x, terminals, np_maps=np_maps)
        if pre_dy is None:
            sc_all = [uniform_score(score_pen=i) for i in score_pen]
        else:
            try:
                sc_all = []
                for si, sp in zip(scoring, score_pen):
                    sc = []
                    for i, j in zip(dy, pre_dy):
                        index = np.isfinite(i) * np.isfinite(j)
                        sci = si(i[index], j[index])
                        sc.append(sci)
                    sc = [uniform_score(score_pen=sp) if np.isnan(i) or i is None else i for i in sc]
                    sc = sum(sc) / len(sc)
                    sc_all.append(sc)

            except (ValueError, RuntimeWarning, TypeError):

                sc_all = [uniform_score(score_pen=i) for i in score_pen]
    if details:
        return sc_all, expr01, pre_y
    else:
        return sc_all, str(expr01), 0

def calculate_derivative_y(expr01, x, terminals, np_maps=None):
    """
    Something error for reference:

    M. Schmidt, H. Lipson, Distilling free-form natural laws from experimental data, Science, 324 (2009),
    81–85.

    Parameters
    ----------
    expr01: Expr
        sympy expression.
    x: list of np.ndarray
        list of xi
    terminals: list of sympy.Symbol
        features and constants
    np_maps: Callable
        user np.ndarray function

    Returns
    -------
    pre_dy_all:np.ndarray or float
        pre-dy
    dy_all: np.ndarray or float
        dy
    """
    warnings.filterwarnings("ignore")
    if not isinstance(expr01, (Expr, NewArray)):
        return None, None
    if len(expr01.free_symbols) < 2:
        return None, None

    free_symbols = list(expr01.free_symbols)[:4] if len(expr01.free_symbols) > 4 else list(expr01.free_symbols)

    free_symbols.sort(key=lambda x: x.name)
    par = list(itertools.combinations(free_symbols, 2))

    # free_symbols = [] # no dependence
    free_symbols = par  # all denpendence
    # free_symbols = [par[0]]
    # free_symbols = par[-1:]
    # free_symbols = par[1:]

    try:
        pre_dy_all = []
        dy_all = []
        for pari in par:
            pari = list(pari)
            pari.sort(key=lambda x: x.name)
            i, j = pari
            subbb = {}
            subbb1 = {k: Function("{}f".format(k.name))(v) for k, v in free_symbols if k is not i and v is i}
            subbb2 = {k: Function("{}f".format(k.name))(v) for v, k in free_symbols if k is not i and v is i}
            subbb.update(subbb1)
            subbb.update(subbb2)
            subb_re = dict(zip(subbb.values(), subbb.keys()))

            fdv1 = diff(expr01.subs(subbb), i, evaluate=True)
            fdv1 = fdv1.subs(subb_re)
            subbb = {}
            subbb3 = {k: Function("{}f".format(k.name))(v) for k, v in free_symbols if k is not j and v is j}
            subbb4 = {k: Function("{}f".format(k.name))(v) for v, k in free_symbols if k is not j and v is j}
            subbb.update(subbb3)
            subbb.update(subbb4)
            subb_re = dict(zip(subbb.values(), subbb.keys()))
            fdv2 = diff(expr01.subs(subbb), j, evaluate=True).subs(subb_re)
            fdv = fdv2 / fdv1

            func0 = utilities.lambdify(terminals, fdv, modules=[np_maps, "numpy"])
            pre_dy = func0(*x)

            ff = diff(i, j, evaluate=False)
            func0 = utilities.lambdify(terminals, ff, modules=[np_maps, "numpy"])
            dy = func0(*x)

            pre_dy_all.append(pre_dy)
            dy_all.append(dy)

        pre_dy_all = np.array(pre_dy_all)
        dy_all = np.array(dy_all)

    except (ValueError, RuntimeWarning, TypeError):
        pre_dy_all, dy_all = None, None

    return pre_dy_all, dy_all

def calculate_cv_score(expr01, x, y, terminals, scoring=None, score_pen=(1,), cv=5, refit=True,
                       add_coef=True, filter_warning=True, inter_add=True,
                       inner_add=False, vector_add=False, out_add=False, flat_add=False, np_maps=None,
                       classification=False, score_object="y", details=False):
    """
    Use cv spilt for score, return the mean_test_score.
    Use cv spilt for predict, return the cv_predict_y.(have not be used)

    Notes:
        if cv and refit, all the data is refit to determine the coefficients.
        Thus, the expression is not compact with the this scores, when re-calculated by this expression.

    Parameters
    ----------

    score_object:
        score by y or delta y
    classification:
        classification or not
    refit: True:
        use forced, refit the coefficient use all data.
    cv:sklearn.model_selection._split._BaseKFold,int
        the shuffler must be False
    vector_add
    expr01: Expr
        sympy expression.
    x: list of np.ndarray
        list of xi
    y: np.ndarray
        y value
    terminals: list of sympy.Symbol
        features and constants
    scoring: list of Callbale, default is [sklearn.metrics.r2_score,]
        See Also sklearn.metrics
    score_pen: tuple of  1 or -1
        1 : best is positive, worse -np.inf
        -1 : best is negative, worse np.inf
        0 : best is positive , worse 0
    add_coef: bool
        bool
    filter_warning: bool
        bool
    inter_add: bool
        bool
    inner_add: bool
        bool
    flat_add:bool
        bool
    out_add:bool
        bool
    np_maps: Callable
        user np.ndarray function

    Returns
    -------
    score:float
        score
    expr01: Expr
        New expr.
    pre_y: np.ndarray or float
        np.array or None
    """
    if filter_warning:
        warnings.filterwarnings("ignore")

    if cv == 1:
        sc_all, expr01, pre_y = calculate_score(expr01, x, y, terminals, scoring=scoring, score_pen=score_pen,
                                                add_coef=add_coef, filter_warning=filter_warning, inter_add=inter_add,
                                                inner_add=inner_add, vector_add=vector_add, out_add=out_add,
                                                flat_add=flat_add, np_maps=np_maps, classification=classification,
                                                score_object=score_object, details=details)
        return sc_all, expr01, pre_y

    else:
        if score_object != "y":
            raise TypeError('cv>1 and score_object !="y" are not compatible')

        if isinstance(cv, int):
            cv = KFold(cv, shuffle=False)

        if True:
            _, expr01, _ = calculate_score(expr01, x, y, terminals, scoring=scoring, score_pen=score_pen,
                                           add_coef=add_coef, filter_warning=filter_warning,
                                           inter_add=inter_add,
                                           inner_add=inner_add, vector_add=vector_add, out_add=out_add,
                                           flat_add=flat_add, np_maps=np_maps, classification=classification,
                                           score_object=score_object, details=details)

        cv_sc_all = []
        # cv_expr01 = []
        cv_pre_y = []
        xx = [xi for xi in x if isinstance(xi, np.ndarray)]
        c = [xi for xi in x if not isinstance(xi, np.ndarray)]
        xx = [xi.reshape((-1, 1)) if xi.ndim == 1 else xi.T for xi in xx]

        for train_index, test_index in cv.split(xx[0], y):

            X_train = [xi[train_index] for xi in xx]
            X_test = [xi[test_index] for xi in xx]
            y_train, y_test = y[train_index], y[test_index]

            X_train.reverse()
            X_test.reverse()
            nc = copy.deepcopy(c)
            nc.reverse()
            X_train = [X_train.pop() if isinstance(xi, np.ndarray) else nc.pop() for index, xi in enumerate(x)]
            nc = copy.deepcopy(c)
            nc.reverse()
            X_test = [X_test.pop() if isinstance(xi, np.ndarray) else nc.pop() for index, xi in enumerate(x)]

            pre_y, expr01 = calculate_y(expr01, X_train, y_train, terminals,
                                        x_test=X_test, y_test=y_test,
                                        add_coef=False,
                                        filter_warning=filter_warning, inter_add=inter_add,
                                        inner_add=inner_add,
                                        vector_add=vector_add, out_add=out_add, flat_add=flat_add,
                                        np_maps=np_maps, classification=classification)

            try:
                sc_all = []
                for si, sp in zip(scoring, score_pen):
                    sc = si(y_test, pre_y)
                    if np.isnan(sc):
                        sc = uniform_score(score_pen=sp)
                    sc_all.append(sc)

            except (ValueError, RuntimeWarning, TypeError):

                sc_all = [uniform_score(score_pen=i) for i in score_pen]

            cv_sc_all.append(sc_all)
            # cv_expr01.append(expr01)
            cv_pre_y.append(pre_y)

        sc_all = list(np.mean(np.array(cv_sc_all), axis=0))
        try:
            pre_y = np.concatenate(cv_pre_y)
        except ValueError:
            pre_y = None

        if refit is True:
            "the refit only use for see the detial of calcualtion after loop"
            sc_all0, expr01, pre_y0 = calculate_score(expr01, x, y, terminals, scoring=scoring, score_pen=score_pen,
                                                      add_coef=add_coef, filter_warning=filter_warning,
                                                      inter_add=inter_add,
                                                      inner_add=inner_add, vector_add=vector_add, out_add=out_add,
                                                      flat_add=flat_add, np_maps=np_maps, classification=classification,
                                                      score_object=score_object, details=details)

        return sc_all, expr01, pre_y

def score_dim(dim_, dim_type, fuzzy=False):
    if dim_type is None:
        return 1
    elif isinstance(dim_type, str):
        if dim_type == 'integer':
            return 1 if dim_.isinteger() else 0
        elif dim_type == 'coef':
            return 1 if not dim_.anyisnan() else 0
        elif dim_type == 'ignore':
            return 1
        else:
            raise TypeError("dim_type should be None,'coef', 'integer', special Dim or list of Dim")
    elif isinstance(dim_type, list):
        return 1 if dim_ in dim_type else 0
    elif isinstance(dim_type, Dim):
        if fuzzy:
            return 1 if dim_.is_same_base(dim_type) else 0
        else:
            return 1 if dim_ == dim_type else 0

    else:
        print(dim_type)
        raise TypeError("dim_type should be None,'coef','integer', special Dim or list of Dim")

def calcualte_dim(expr01, terminals, dim_list, dim_maps=None):
    """

    Parameters
    ----------
    expr01: Expr
        sympy expression.
    terminals: list of sympy.Symbol
        features and constants
    dim_list: list of Dim
        dims of features and constants
    dim_maps: Callable
        user dim_maps

    Returns
    -------
    Dim:
        dimension
    dim_score
        is target dim or not
    """
    terminals = [str(i) for i in terminals]
    if not dim_maps:
        dim_maps = dim_map()
    func0 = utilities.lambdify(terminals, expr01, modules=[dim_maps])
    try:
        dim_ = func0(*dim_list)
    except (ValueError, TypeError, ZeroDivisionError, NameError):
        dim_ = dnan
    if isinstance(dim_, (float, int)):
        dim_ = dless
    if not isinstance(dim_, Dim):
        dim_ = dnan

    return dim_

def calcualte_dim_score(expr01, terminals, dim_list, dim_type, fuzzy, dim_maps=None):
    """

    Parameters
    ----------
    expr01: Expr
        sympy expression.
    terminals: list of sympy.Symbol
        features and constants
    dim_list: list of Dim
        dims of features and constants
    dim_maps: Callable
        user dim_maps
    dim_type:list of Dim
        target dim
    fuzzy:
        fuzzy dim or not

    Returns
    -------
    Dim:
        dimension
    dim_score
        is target dim or not
    """
    dim_ = calcualte_dim(expr01, terminals, dim_list, dim_maps=dim_maps)

    dim_score = score_dim(dim_, dim_type, fuzzy)
    return dim_, dim_score

def calculate_collect_(ind, context, x, y, terminals_and_constants_repr, gro_ter_con,
                       dim_ter_con_list, dim_type, fuzzy, cv=1, refit=True,
                       scoring=None, score_pen=(1,),
                       add_coef=True, filter_warning=True, inter_add=True, inner_add=False,
                       vector_add=False, out_add=False, flat_add=False,
                       np_maps=None, classification=False, dim_maps=None, cal_dim=True, score_object="y",
                       details=False):
    expr01 = compile_context(ind, context, gro_ter_con)

    if cal_dim:
        dim, dim_score = calcualte_dim_score(expr01, terminals_and_constants_repr,
                                             dim_ter_con_list, dim_type, fuzzy,
                                             dim_maps=dim_maps)
    else:
        dim, dim_score = dless, 1

    score, expr01, pre_y = calculate_cv_score(expr01, x, y, terminals_and_constants_repr,
                                              cv=cv, refit=refit,
                                              add_coef=add_coef, inter_add=inter_add,
                                              inner_add=inner_add, vector_add=vector_add, out_add=out_add,
                                              flat_add=flat_add,
                                              scoring=scoring, score_pen=score_pen,
                                              filter_warning=filter_warning,
                                              np_maps=np_maps, classification=classification, score_object=score_object,
                                              details=details
                                              )

    return score, dim, dim_score, expr01, pre_y

score_collection = {'explained_variance': metrics.explained_variance_score,
                    'max_error': metrics.max_error,
                    'neg_mean_absolute_error': metrics.mean_absolute_error,
                    'neg_mean_squared_error': metrics.mean_squared_error,
                    'neg_root_mean_squared_error': metrics.mean_squared_error,
                    'r2': metrics.r2_score,
                    'accuracy': metrics.accuracy_score,
                    'precision': metrics.precision_score,
                    'f1': metrics.f1_score,
                    'balanced_accuracy': metrics.balanced_accuracy_score,
                    'average_precision': metrics.average_precision_score, }


import asn1tools
from eleptic import *
import string
from pygost.gost3410 import CURVE_PARAMS #p, q, a, b, x, y
from pygost.gost3412 import *
from pygost.utils import *
from random import randrange
from pygost.utils import *
import random
from pygost.gost34112012 import GOST34112012 as GostHash

cmd_scheme = asn1tools.compile_files('schemes/cmd_scheme.asn')
challenge_scheme = asn1tools.compile_files('schemes/challenge_scheme.asn')
gost_sign_file = asn1tools.compile_files('schemes/gost_sign.asn')
pub_key_scheme = asn1tools.compile_files('schemes/public_key_scheme.asn')
session_key_scheme = asn1tools.compile_files('schemes/session_scheme.asn')
cert_scheme = asn1tools.compile_files('schemes/cert_scheme.asn')
priv_key_scheme = asn1tools.compile_files('schemes/private_key_scheme.asn')

def RandomString(stringLength=10):
    """Generate a random string of fixed length """
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(stringLength))

def decode_string(res, length):
    vect = []
    for i in range(0, length):
        vect.append(chr(res % 256))
        res = res // 256
    return "".join(reversed(vect))

def EncryptGostOpen(chunk, curv, O):
	num = 0
	for i in chunk:
		num *= 256
		num += ord(i)
	k = randrange(1, curv.p-1)
	pk = curv.mult(k, curv.P)
	qk = curv.mult(k, O)
	l = (num*qk[0])%curv.p
	return pk,l

def DecryptGostOpen(curve, O, l, d):
    D = curve.mult(d, O)
    t = l*(invert(D[0],curve.p))%curve.p
    return t

def GenSign(curve, message, d, Q):
    hash_message = GostHash(message).digest()
    e = int.from_bytes(hash_message, "big", signed=False) % curve.q

    r = 0
    s = 0
    while r == 0 or s == 0:
        k = random.randrange(1, curve.q)
        C = curve.mult(k, curve.P)
        r = C[0] % curve.q
        s = (r*d + k*e) % curve.q

    sign = gost_sign_file.encode('GostSignFile', dict(keyset=
    {
        'key': dict
            (
            algid=b'\x80\x06\x07\x00',
            test='gostSignKey',
            keydata=dict
            (
                qx = Q[0],
                qy = Q[1]
            ),
            param=dict
                (
                fieldparam=dict
                (
                    prime=curve.p
                ),
                curveparam=dict
                    (
                    a=curve.a,
                    b=curve.b
                ),
                genparam=dict
                    (
                    px=curve.P[0],
                    py=curve.P[1]
                ),
                q=curve.q
            ),
            ciphertext=dict
            (
                r=r,
                s=s
            )
        )
    }, last={}))
    # print("sign generated")
    return sign

def AuthSign(curve, message, sign_data, Q):
    sign_str = gost_sign_file.decode('GostSignFile', sign_data)
    r = sign_str['keyset']['key']['ciphertext']['r']
    s = sign_str['keyset']['key']['ciphertext']['s']

    if r > curve.q or s > curve.q:
        return False
    hash = GostHash(message).digest()
    e = int.from_bytes(hash, "big", signed=False) % curve.q
    if e == 0:
        e = 1

    v = invert(e, curve.q)
    z1 = (s*v) % curve.q
    z2 = (-r*v) % curve.q

    C = curve.add(curve.mult(z1, curve.P), curve.mult(z2, Q))

    if  C[0] % curve.q == r:
        # print("sign true")
        return True
    else:
        # print("sign false")
        return False

def EncryptGostSym(text, gost):
    if isinstance(text, str):
        text = text.encode()
    ans = b''
    for i in range(0, len(text), 16):
        st = text[i:i + 16]
        if len(st) < 16:
            st = st + b' ' * (16 - len(st))
        ans += gost.encrypt(st)
    return ans

def DecryptGostSym(encText, gost):
    ans = b''
    for i in range(0, len(encText), 16):
        st = encText[i:i + 16]
        ans += gost.decrypt(st)
    return ans.rstrip()


import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Dropout

def unstringify_arr(df):
    unstringify = lambda li: [float(i) for i in li[1:-1].split('; ')]  # noqa: E731
    return np.array([unstringify(li) for li in df])

# Load le dataset mnist depuis le csv custom
def load_mnist_from_csv(train_file, test_file):
    df_train = pd.read_csv(train_file)
    df_test = pd.read_csv(test_file)

    x_train_str, y_train_num = df_train['inputs'], df_train['labels']
    x_test_str, y_test_num = df_test['inputs'], df_test['labels']

    x_train = unstringify_arr(x_train_str)
    x_test = unstringify_arr(x_test_str)

    print(x_train.shape, x_test.shape)

    input_size = x_train.shape[1]
    num_labels = np.unique(y_test_num).shape[0]

    # convert them to one-hot
    y_train = to_categorical(y_train_num, num_classes=num_labels)
    y_test = to_categorical(y_test_num, num_classes=num_labels)

    return x_train, x_test, y_train, y_test, input_size, num_labels

#
#  Building the model
#

def build_model(input_size, num_labels, hidden_units, dropout):
    # MLP with ReLU and Dropout
    model = Sequential()

    model.add(Dense(hidden_units, input_dim=input_size))
    model.add(Activation('relu'))
    model.add(Dropout(dropout))

    model.add(Dense(hidden_units))
    model.add(Activation('relu'))
    model.add(Dropout(dropout))

    model.add(Dense(num_labels))

    # Activation for output layer
    model.add(Activation('softmax'))

    # Optimization
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

    return model

#
#   Main
#
DATASETS_PATH = os.path.join("datasets")
GRAPHS_PATH = os.path.join("graphs")

# Load data from CSV
ftrain = f"{DATASETS_PATH}/train.csv"
ftest = f"{DATASETS_PATH}/test.csv"

x_train, x_test, y_train, y_test, input_size, num_labels = load_mnist_from_csv(ftrain, ftest)

# Train
hidden_units = 256
dropout = 0.45

# print(input_size, num_labels)
model = build_model(input_size, num_labels, hidden_units, dropout)
model.summary()

# Training
batch_size = 128
history = model.fit(x_train, y_train, epochs=15, batch_size=batch_size)

# Save accuracy and loss graphs
os.makedirs(GRAPHS_PATH, exist_ok=True)

plt.plot(history.history['accuracy'], color='b', label='train set')
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend()
plt.savefig(f"{GRAPHS_PATH}/model_accuracy.jpg")
plt.clf()

plt.plot(history.history['loss'], color='r', label='train set')
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend()
plt.savefig(f'{GRAPHS_PATH}/model_loss.jpg')
plt.clf()

# Evaluation
loss, acc = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=0)
print("\nAccuracy on test set : %.1f%%\n" % (100.0 * acc))

model.save('mnist_model.h5')


from django.forms import *
from core.gastos.models import *

# Formulario de ingreso de datos de la aplicacion Gastos.
class GastosForm(ModelForm):
    # Funcion Init para obtener el usuario y mostrar las categorias creadas por el usuario.
    def __init__(self, *args, **kwargs):
        user = kwargs.pop("user")
        super(GastosForm, self).__init__(*args, **kwargs)
        self.fields['category'].queryset = Categoria.objects.filter(user_creation_id=user)

    class Meta:
        # Especificacion del modelo al cual se le ingresaran datos
        model = Gastos
        # Cuales son los campos de ingreso de datos
        fields = '__all__'
        # Exclusion de campos de la base de datos que no serán mostrados al usuario
        exclude = ('date', 'semana', 'user_creation',)
        # Campos de ingreso de datos, con su texto que sera mostrado
        labels = {
            'amount': 'Monto',
            'category': 'Categoria',
        }
        # Atributos de los campos de ingreso de datos
        widgets = {
            'amount': TextInput(
                attrs={
                    'class': 'form-control',
                    'placeholder': 'Ingrese el monto del gasto',
                }
            ),
            'category': Select(
                attrs={
                    'class': 'form-control'
                }
            ),
        }

    # Funcion de guardado de la información
    def save(self, commit=True):
        data = {}
        form = super()
        try:
            if form.is_valid():

                form.save()
            else:
                data['error'] = form.errors
        except Exception as e:
            data['error'] = str(e)
        return data


import pandas as pd

data=pd.read_csv("car.data")
# print(data.info)
# print(data.shape)
# print(data.head(10))
# print(data.tail(10))

people = {
    "first": ["Corey", 'Jane', 'John'],
    "last": ["Schafer", 'Doe', 'Doe'],
    "email": ["CoreyMSchafer@gmail.com", 'JaneDoe@email.com', 'JohnDoe@email.com']
}

df=pd.DataFrame(people)

# print(df["email"][0])

# print(df.email)

#print(type(df[["email","last"]]))
#print(df[["email","last"]])
#print(df.columns)

# print(df.iloc[[0,1],2])
# print(df.loc[[0,1],["last","email"]])
# print(data["persons"].value_counts())

#print(data.loc[2]["door"])
# print(data.loc[5:12,"maint":"lug_boot"])

# print(data.set_index("maint",inplace=True))
# data.set_index("maint",inplace=True)
# print(data.index)
# print(data.loc["low"])


#!/usr/bin/env python
# coding: utf-8

# # READ a PNG file
# # Extract the german Text
# # Translate to english
# # Save as text file

# In[ ]:

import pytesseract
from PIL import Image
from googletrans import Translator
from google_trans_new import google_translator
import sys, os
from os import listdir
from os.path import isfile, join, isdir
from json.decoder import JSONDecodeError

# Load Tesseract
pytesseract.pytesseract.tesseract_cmd = r'C:/Program Files/Tesseract-OCR/tesseract.exe'
#where are we
print(sys.path[0])

# select all jpg files from current folder

mypath = sys.path[0]
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
onlyjpgfiles = [f for f in onlyfiles if '.png' in f]
print(f'lList of jpg files : {onlyjpgfiles}')

def get_textfromjpg(list_of_files):
    extracted_dict = {}
    for file_name in list_of_files:
        text = pytesseract.image_to_string(Image.open(file_name))
        extracted_dict[file_name] = text.splitlines()

    return extracted_dict

extracted_text_all = get_textfromjpg(onlyjpgfiles)

#text1 =  extracted_text_all['PXL_20201205_160438315.jpg']
#print(text1)

# Translate Text

translator = google_translator()
def translate_please(file_dict):
    translated_files = {}
    for file_name in file_dict:
        translate_list = []
        file = file_dict[file_name]
        for line in file:
            try:
                english_text = translator.translate(line,lang_src='de', lang_tgt='en')
                translate_list.append(english_text)
            except JSONDecodeError:
                pass
        translate_en =  '\n'.join(translate_list)
        translated_files[file_name] = translate_en

    return translated_files

# Print english Translation

translated_files = translate_please(extracted_text_all)

#print(translated_files['PXL_20201205_160438315.jpg'])

#store translated files

def write_txtfiles(list_files, lang):
    for file in list_files:
        write_file = file[:-4] + '_' + lang + '.txt'
        with open(write_file,'w') as f:
            f.write(translated_files[file])
    pass

if not isdir(join(mypath, 'de')):
    get_ipython().system('mkdir de ')
    directory = join(mypath, 'de')
    os.chdir(directory)
    write_txtfiles(translated_files,'de')
    os.chdir(mypath)
else:
    directory = join(mypath, 'de')
    os.chdir(directory)
    write_txtfiles(translated_files,'de')
    os.chdir(mypath)

if not isdir(join(mypath, 'en')):
    get_ipython().system('mkdir en')
    directory = join(mypath, 'en')
    os.chdir(directory)
    write_txtfiles(extracted_text_all,'en')
    os.chdir(mypath)
else:
    directory = join(mypath, 'en')
    os.chdir(directory)
    write_txtfiles(extracted_text_all,'en')
    os.chdir(mypath)

#

#


from PIL import Image
import requests

from transformers import CLIPProcessor, CLIPModel

mode = "openai/clip-vit-large-patch14-336"

model = CLIPModel.from_pretrained(mode)
processor = CLIPProcessor.from_pretrained(mode)

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

inputs = processor(text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True)

print(mode)
outputs = model(**inputs)
logits_per_image = outputs.logits_per_image # this is the image-text similarity score
print(logits_per_image)
probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
print(probs)


# type: ignore
'''
从知乎上下载文章，保存为 Markdown 格式。
'''

import re
import urllib.parse
import requests
from bs4 import BeautifulSoup
from pathlib import Path

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'
}

def escape(s):
    return re.sub(r'[\\`\*_\{\}\[\]\(\)#\+-\.\!\|]', lambda match: '\\' + match.group(), s).replace('\\,', ',')

def link(s):
    if s[:31] == 'https://link.zhihu.com/?target=':
        return urllib.parse.unquote(s[31:])
    return s

def html2md(node):
    if node is None:
        return ''
    if node.name is None:
        return escape(str(node))
    if node.name == 'p':
        return ''.join((html2md(c) for c in node.children)) + '\n\n'
    if node.name == 'a':
        if node.attrs.get('data-reference-link'):
            index = node.attrs['href'].split('_')[1]
            return f'[^{index}]'
        return f'[{node.text}]({link(node.attrs["href"])})' + ('\n' if 'LinkCard' in node.attrs['class'] else '')
    if node.name == 'hr':
        return '\n---\n'
    if node.name == 'h2':
        return f'## {node.text}\n'
    if node.name == 'h3':
        return f'### {node.text}\n'
    if node.name == 'figure':
        return html2md(node.img) + '\n' + html2md(node.figcaption)
    if node.name == 'img':
        if node.attrs.get('eeimg'):
            formula = node.attrs.get('alt').replace('|', '\\|')
            return f'$${formula}$$'
        src = node.attrs['src']
        name = src.split('/')[-1]
        with open(f'./_drafts/{title_quoted}-assets/{name}', 'wb') as img:
            img.write(requests.get(src, headers=headers, stream=True).content)
        return f'\n![{name}](./{title_quoted}-assets/{name})\n'
    if node.name == 'figcaption':
        return f'*{node.text}*\n'
    if node.name == 'b':
        return f"**{''.join((html2md(c) for c in node.children))}**"
    if node.name == 'i':
        return f"*{''.join((html2md(c) for c in node.children))}*"
    if node.name == 'div':
        return ''.join((html2md(c) for c in node.children)) + '\n\n'
    if node.name == 'pre':
        return ('```'
        + node.code.attrs['class'][0].split('-')[-1]
        + '\n' + node.code.text
        + '\n```\n')
    if node.name == 'code':
        return f'`{node.text}`'
    if node.name == 'blockquote':
        content = ''.join((html2md(c) for c in node.children))
        return '> ' + '\n> \n> '.join(content.split('\n')) + '\n\n'
    if node.name == 'br':
        return '\n'
    if node.name == 'ol':
        if node.attrs.get('class') and 'ReferenceList' in node.attrs['class']:
            result = ''
            for li in node.children:
                index = li.attrs['id'].split('_')[1]
                result += f"[^{index}]: {escape(li.text).strip('^')}\n"
            return result
        result = ''.join(f'{index}. {html2md(li)}\n' for index, li in enumerate(node.children, start=1))

        return result
    if node.name == 'ul':
        return '- ' + '\n- '.join((html2md(c) for c in node.children)) + '\n\n'
    if node.name == 'li':
        return ''.join((html2md(c) for c in node.children))
    if node.name == 'span':
        if node.attrs.get('data-tex'):
            return f'$${node.attrs.get("data-tex")}$$'
        return ''.join((html2md(c) for c in node.children))
    if node.name == 'sup':
        return ''.join((html2md(c) for c in node.children))
    return str(node)

if __name__ == '__main__':
    url = input('请输入网址: ').strip()
    req = requests.get(url, headers=headers)
    req.encoding = 'utf-8'
    soup = BeautifulSoup(req.text, features='lxml')
    title = soup.title.text.strip('- 知乎').strip()
    title_quoted = title.replace(' ', '')
    Path(f'./_drafts/{title_quoted}-assets').mkdir(exist_ok=True, parents=True)
    markdown = f'''# 文章 | {title}

:::tip
本文首发于知乎。
:::

'''
    for n in soup.find(class_='RichText'):
        markdown += html2md(n)
    while re.findall(r'\n\n\n', markdown):
        markdown = re.sub(r'\n\n\n', '\n\n', markdown)
    with open(f'./_drafts/{title}.md', 'w', encoding='utf-8') as f:
        f.write(markdown)


from __future__ import annotations
from abc import ABC
from typing import List, Callable

from numpy import ndarray
from itertools import combinations

from Complexes.VR.Base import VRBase

class ExpansionBruteForce(VRBase, ABC):
    def __init__(self, points: List[ndarray], epsilon: float,
                 metric: Callable[[ndarray[float, ...], ndarray[float, ...]], float]):
        super().__init__(points, epsilon, metric)

    def compute_expansion(self, dim: int):
        if not self._is_skeleton_constructed:
            raise ReferenceError("Skeleton not constructed.")
        for simplex_dim in range(2, dim + 1):
            for elem in combinations(self._complex.p_simplices(simplex_dim - 1), simplex_dim + 1):
                flag = True
                max_dist = 0
                for (x, y) in combinations(elem, 2):
                    if x.intersect(y).dim == -1:
                        flag = False
                        break
                    max_dist = max([max_dist, self._complex.get_weight(x), self._complex.get_weight(y)])
                if flag:
                    union = elem[0].union(*elem[1:])
                    if union.dim == simplex_dim:
                        self._complex.add_simplex(union, max_dist)
            if self._complex.p_simplex_count(simplex_dim) < simplex_dim + 1:
                break


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 开发人员   ：Davis Niu
# 开发时间   ：7/1/2020  9:44 AM
# 文件名称   ：check_result.py
from celery.result import AsyncResult
from celery_tasks.celery import cel

async_result = AsyncResult(id='9b8612fa-ae61-4776-aadd-2bcbea49b70e', app=cel)

if async_result.successful():
    result = async_result.get()
    print(result)
elif async_result.failed():
    print('执行失败')
elif async_result.status == 'PENDING':
    print('等待任务执行')
elif async_result.status == 'RETRY':
    print('任务异常后正在重试')
elif async_result.status == 'STARTED':
    print('任务已经开始被执行')


import argparse
import codecs

parser = argparse.ArgumentParser()
parser.add_argument('-src', required=True, type=str, help='the path of the data')
parser.add_argument('-tgt', required=True, type=str, help='the path of the data')
parser.add_argument('-pred', required=True, type=str, help='the path of the data')

opt = parser.parse_args()
print(opt)

def readSentences(src):
    with codecs.open(src, 'r', 'utf-8') as f:
        lines = f.readlines()
    return [line.strip() for line in lines]

def main():
    src_lines, tgt_lines, pred_lines = readSentences(opt.src), readSentences(opt.tgt), readSentences(opt.pred)
    index = 1
    for s, t, p in zip(src_lines, tgt_lines, pred_lines):
        print(index)
        print('src:  ' + s)
        print('tgt:  ' + t)
        print('pred: ' + p)
        index += 1

if __name__ == '__main__':
    main()


import json

def get_key(_request: str) -> str:
    """
    Retrieves the key of the command from a request.

    Args:
        _request (str): The request containing the key.

    Returns:
        str: The extracted key.
    """
    msg = json.loads(_request)['commands']
    results = msg.split(" ", 3)
    key = results[1].replace(":", "").replace("\"", "")
    return key

def check_id_exist(request: str, _id: int) -> bool:
    """
    Checks if an ID exists in the replication nodes list of a request.

    Args:
        request (str): The request to check.
        _id (int): The ID to check.

    Returns:
        bool: True if the ID exists in the replication nodes, False otherwise.

    Raises:
        json.JSONDecodeError: If the request is not a valid JSON.
    """
    try:
        msg = json.loads(request)
        replication_nodes = msg.get("rep_ids", [])
        return str(_id) in replication_nodes
    except json.JSONDecodeError as e:
        raise e


def build_max_heap(lst):
	"""
	Takes and array LST and returns a max heap.
	"""
	size = len(lst)
	for pos in reversed(range(size // 2)):
		max_heapify(lst, pos, size)

def max_heapify(lst, pos, bound):
	"""
	Corrects at most one violation of the heap property in a sub-tree's root,
	where LST is an array which contains the heap upto but not inclduing the
	elment at index BOUND, and POS is the index of a node which exibits this
	potential violation. Notice that POS < BOUND.
	"""
	left = 2 * (pos + 1) - 1
	right = 2 * (pos + 1)
	if right < bound and lst[right] > lst[left] and lst[right] > lst[pos]:
		lst[pos], lst[right] = lst[right], lst[pos]
		max_heapify(lst, right, bound)
	elif left < bound and lst[left] > lst[pos]:
		lst[pos], lst[left] = lst[left], lst[pos]
		max_heapify(lst, left, bound)

def heap_sort(lst):
	"""
	Sorts the array LST using the heap sort algorithm.
	+ Runtime: O(n log n)

	>>> lst = [21, 5, 6, 7, 5]
	>>> heap_sort(lst)
	>>> lst
	[5, 5, 6, 7, 21]
	"""
	size = len(lst)
	build_max_heap(lst)
	for bound in reversed(range(size)):
		lst[0], lst[bound] = lst[bound], lst[0]
		max_heapify(lst, 0, bound)


#!/usr/bin/env python
import os
import argparse
from seppuku import create_app, logger
parser = argparse.ArgumentParser()
parser.add_argument('-e', help='Environment variables', type=str, nargs='*')

if __name__ == '__main__':
    args = parser.parse_args()
    if args.e is not None:
        for env_str in args.e:
            name, value = env_str.split('=')
            os.environ[name] = value
    app = create_app()
    logger.info('Serving /api/v1/seppuku-message at port 5002')
    app.run(port='5002')