from django.db.models.query import QuerySet
from django.dispatch.dispatcher import receiver
from app.tasks import fetch_data_from_plc, get_queryset_to_fetch, procesar_datos
from app.models import Area, Dato, DatoProcesado, Fila, Plc, end_fetching_data
from typing import List, Tuple
import unittest
import pytest
from django.test import TestCase
from snap7.server import mainloop
from multiprocessing import Process
import time
import snap7
import struct

ip = "127.0.0.1"
rack = 0
slot = 0
port = 1102

class TestDataFetchingAndProcessing(TestCase):

    @classmethod
    def setUpClass(cls):
        cls.process = Process(target=mainloop)
        cls.process.start()
        time.sleep(2)  # wait for server to start

    @classmethod
    def tearDownClass(cls):
        cls.process.terminate()
        cls.process.join(1)
        if cls.process.is_alive():
            cls.process.kill()

    def setUp(self):
        self.client : snap7.client.Client = snap7.client.Client()
        self.client.connect(ip, rack, slot, port)

        self.createBasicModels()
        data = bytearray(10)
        data[0:0+2] = struct.pack(">h", 100)
        data[2:2+2] = struct.pack(">h", 230)
        data[4:4+2] = struct.pack(">h", 25)
        self.client.db_write(1, 0, data)

    def tearDown(self):
        self.client.disconnect()
        self.client.destroy()

    def createBasicModels(self):
        self.plc: Plc = Plc.objects.create(ip=ip, rack=rack, slot=slot, port=port, nombre="test plc")
        self.area: Area = Area.objects.create(plc=self.plc, area=Area.DB, nombre="area 1", numero=1, offset=10)
        self.fila_1: Fila = Fila.objects.create(area=self.area, name="fila 1", byte=0, tipo_dato=Fila.INT)
        self.fila_2: Fila = Fila.objects.create(area=self.area, name="fila 2", byte=2, tipo_dato=Fila.INT)
        self.fila_3: Fila = Fila.objects.create(area=self.area, name="fila 3", byte=4, tipo_dato=Fila.INT)

    def test_class(self):
        assert self.client.get_connected()
        data = self.client.db_read(1, 0, 6)
        assert 100 == snap7.util.get_int(data, 0)
        assert 230 == snap7.util.get_int(data, 2)
        assert 25 == snap7.util.get_int(data, 4)

    def test_get_queryset_to_fetch(self):
        queryset = get_queryset_to_fetch(ids=[self.area.pk])
        assert queryset[0].get("plc__ip") == ip
        assert queryset[0].get("numero") == 1
        assert queryset[0].get("offset") == 10
        assert queryset[0].get("area") == snap7.types.areas.DB

    def test_fetch_data_from_plc(self):
        fetch_data_from_plc(ids=[self.area.pk])

        dato = Dato.objects.first()
        assert dato
        assert snap7.util.get_int(dato.value, 0) == 100
        assert snap7.util.get_int(dato.value, 2) == 230
        assert snap7.util.get_int(dato.value, 4) == 25

    def test_procesar_datos(self):
        fetch_data_from_plc(ids=[self.area.pk])
        queryset = Dato.objects.all()
        assert queryset.exists()
        procesar_datos()
        assert queryset.first().procesado

        datos_procesados: List[DatoProcesado] = list(DatoProcesado.objects.all())

        assert datos_procesados[0].value == 100
        assert datos_procesados[1].value == 230
        assert datos_procesados[2].value == 25

    def test_end_fetching_data_signal(self):
        test = []
        @receiver(signal=end_fetching_data)
        def fire_process_data_after_fetching(sender, ids, **kwargs):
            test.append((sender, ids))

        fetch_data_from_plc(ids=[self.area.pk])

        assert test[0] == ("fetch_data_from_plc", [self.area.pk])


"""Seed file to make sample data for db."""

from models import Playlist, Song, PlaylistSong, db
from app import app

# Create all tables
db.drop_all()
db.create_all()

Playlist1 = Playlist(name="Walking", description="Walking in the park")
Playlist2 = Playlist(name="Gym", description="Gym workout")
Playlist3 = Playlist(name="Meditation", description="Meditation session")
Song1 = Song(title="Individual Entity", artist="Rylei Nathaniel")
Song2 = Song(title="Runnin' Down A Dream", artist="Tom Petty")
Song3 = Song(title="This Love", artist="Maroon 5")
PlaylistSong1 = PlaylistSong(playlist_id=1, song_id=1)
PlaylistSong2 = PlaylistSong(playlist_id=2, song_id=2)
PlaylistSong3 = PlaylistSong(playlist_id=3, song_id=3)

db.session.add_all([Playlist1, Playlist2, Playlist3, Song1,
                   Song2, Song3, PlaylistSong1, PlaylistSong2, PlaylistSong3])

db.session.commit()


# Python program to implement Playfair Cipher

# Function to convert the string to lowercase

def toLowerCase(text):
	return text.lower()

# Function to remove all spaces in a string

def removeSpaces(text):
	newText = ""
	for i in text:
		if i == " ":
			continue
		else:
			newText = newText + i
	return newText

# Function to group 2 elements of a string
# as a list element

def Diagraph(text):
	Diagraph = []
	group = 0
	for i in range(2, len(text), 2):
		Diagraph.append(text[group:i])

		group = i
	Diagraph.append(text[group:])
	return Diagraph

# Function to fill a letter in a string element
# If 2 letters in the same string matches

def FillerLetter(text):
	k = len(text)
	if k % 2 == 0:
		for i in range(0, k, 2):
			if text[i] == text[i+1]:
				new_word = text[0:i+1] + str('x') + text[i+1:]
				new_word = FillerLetter(new_word)
				break
			else:
				new_word = text
	else:
		for i in range(0, k-1, 2):
			if text[i] == text[i+1]:
				new_word = text[0:i+1] + str('x') + text[i+1:]
				new_word = FillerLetter(new_word)
				break
			else:
				new_word = text
	return new_word

list1 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm',
		'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']

# Function to generate the 5x5 key square matrix

def generateKeyTable(word, list1):
	key_letters = []
	for i in word:
		if i not in key_letters:
			key_letters.append(i)

	compElements = []
	for i in key_letters:
		if i not in compElements:
			compElements.append(i)
	for i in list1:
		if i not in compElements:
			compElements.append(i)

	matrix = []
	while compElements != []:
		matrix.append(compElements[:5])
		compElements = compElements[5:]

	return matrix

def search(mat, element):
	for i in range(5):
		for j in range(5):
			if(mat[i][j] == element):
				return i, j

def encrypt_RowRule(matr, e1r, e1c, e2r, e2c):
	char1 = ''
	if e1c == 4:
		char1 = matr[e1r][0]
	else:
		char1 = matr[e1r][e1c+1]

	char2 = ''
	if e2c == 4:
		char2 = matr[e2r][0]
	else:
		char2 = matr[e2r][e2c+1]

	return char1, char2

def encrypt_ColumnRule(matr, e1r, e1c, e2r, e2c):
	char1 = ''
	if e1r == 4:
		char1 = matr[0][e1c]
	else:
		char1 = matr[e1r+1][e1c]

	char2 = ''
	if e2r == 4:
		char2 = matr[0][e2c]
	else:
		char2 = matr[e2r+1][e2c]

	return char1, char2

def encrypt_RectangleRule(matr, e1r, e1c, e2r, e2c):
	char1 = ''
	char1 = matr[e1r][e2c]

	char2 = ''
	char2 = matr[e2r][e1c]

	return char1, char2

def encryptByPlayfairCipher(Matrix, plainList):
	CipherText = []
	for i in range(0, len(plainList)):
		c1 = 0
		c2 = 0
		ele1_x, ele1_y = search(Matrix, plainList[i][0])
		ele2_x, ele2_y = search(Matrix, plainList[i][1])

		if ele1_x == ele2_x:
			c1, c2 = encrypt_RowRule(Matrix, ele1_x, ele1_y, ele2_x, ele2_y)
			# Get 2 letter cipherText
		elif ele1_y == ele2_y:
			c1, c2 = encrypt_ColumnRule(Matrix, ele1_x, ele1_y, ele2_x, ele2_y)
		else:
			c1, c2 = encrypt_RectangleRule(
				Matrix, ele1_x, ele1_y, ele2_x, ele2_y)

		cipher = c1 + c2
		CipherText.append(cipher)
	return CipherText

text_Plain = 'instruments'
text_Plain = removeSpaces(toLowerCase(text_Plain))
PlainTextList = Diagraph(FillerLetter(text_Plain))
if len(PlainTextList[-1]) != 2:
	PlainTextList[-1] = PlainTextList[-1]+'z'

key = "Monarchy"
print("Key text:", key)
key = toLowerCase(key)
Matrix = generateKeyTable(key, list1)

print("Plain Text:", text_Plain)
CipherList = encryptByPlayfairCipher(Matrix, PlainTextList)

CipherText = ""
for i in CipherList:
	CipherText += i
print("CipherText:", CipherText)

# This code is Contributed by Boda_Venkata_Nikith

# key = input("Enter key: ")
# key = key.replace(" ", "")
# key = key.upper()

# def matrix(x, y, initial):
#     return [[initial for i in range(x)] for j in range(y)]

# result = list()
# for i in key:  # storing key
#     if i not in result:
#         if i == "J":
#             result.append("I")
#         else:
#             result.append(i)
# flag = 0
# for i in range(65, 91):  # storing other character
#     if chr(i) not in result:
#         if i == 73 and chr(74) not in result:
#             result.append("I")
#             flag = 1
#     elif flag == 0 and i == 73 or i == 74:
#         pass
#     else:
#         result.append(chr(i))
# k = 0
# my_matrix = matrix(5, 5, 0)  # initialize matrix
# for i in range(0, 5):  # making matrix
#     for j in range(0, 5):
#         my_matrix[i][j] = result[k]
#         k += 1
# print("\nPLAY-FAIR KEYWORD MATRIX:")
# for i in range(0, 5):
#     for j in range(0, 5):
#         print(my_matrix[i][j], end=" ")
#     print()

# def locindex(c):  # get location of each character
#     loc = list()
#     if c == "J":
#         c = "I"
#     for i, j in enumerate(my_matrix):
#         for k, l in enumerate(j):
#             if c == l:
#                 loc.append(i)
#                 loc.append(k)
#                 return loc

# def encrypt():  # Encryption
#     msg = str(input("Enter the plaintext: "))
#     msg = msg.upper()
#     msg = msg.replace(" ", "")
#     i = 0
#     for s in range(0, len(msg) + 1, 2):
#         if s < len(msg) - 1:
#             if msg[s] == msg[s + 1]:
#                 msg = msg[: s + 1] + "X" + msg[s + 1:]
#     if len(msg) % 2 != 0:
#         msg = msg[:] + "X"
#     print("\nCipher text:", end=" ")
#     while i < len(msg):
#         loc = list()
#         loc = locindex(msg[i])
#         loc1 = list()
#         loc1 = locindex(msg[i + 1])
#         if loc[1] == loc1[1]:
#             print(
#                 "{}{}".format(
#                     my_matrix[(loc[0] + 1) % 5][loc[1]],
#                     my_matrix[(loc1[0] + 1) % 5][loc1[1]],
#                 ),
#                 end=" ",
#             )
#         elif loc[0] == loc1[0]:
#             print(
#                 "{}{}".format(
#                     my_matrix[loc[0]][(loc[1] + 1) % 5],
#                     my_matrix[loc1[0]][(loc1[1] + 1) % 5],
#                 ),
#                 end=" ",
#             )
#         else:
#             print(
#                 "{}{}".format(my_matrix[loc[0]][loc1[1]],
#                               my_matrix[loc1[0]][loc[1]]),
#                 end=" ",
#             )
#             i = i + 2
#     print()

# def decrypt():  # decryption
#     msg = str(input("Enter Cipher text: "))
#     msg = msg.upper()
#     msg = msg.replace(" ", "")
#     decrypted_msg = ""
#     i = 0
#     while i < len(msg):
#         loc = list()
#         loc = locindex(msg[i])
#         loc1 = list()
#         loc1 = locindex(msg[i + 1])
#         if loc[1] == loc1[1]:
#             decrypted_msg += "{}{}".format(my_matrix[(loc[0] - 1) %
#                                                      5][loc[1]], my_matrix[(loc1[0] - 1) % 5][loc1[1]]) + " "
#         elif loc[0] == loc1[0]:
#             decrypted_msg += "{}{}".format(my_matrix[loc[0]][(loc[1] - 1) %
#                                                              5], my_matrix[loc1[0]][(loc1[1] - 1) % 5]) + " "
#         else:
#             decrypted_msg += "{}{}".format(my_matrix[loc[0]]
#                                            [loc1[1]], my_matrix[loc1[0]][loc[1]]) + " "
#             i = i + 2
#         print("\nPlain text:", decrypted_msg)
#         decrypted_msg = decrypted_msg.replace('X', '').replace(" ", "")
#         print("\nPlain text after romoving all X and spces:", decrypted_msg)

# while 1:
#     choice = int(input("\nChoose one: \n1. Encryption \n2. Decryption \n3. Exit\n\n"))
#     if choice == 1:
#          encrypt()
#     elif choice == 2:
#         decrypt()
#     elif choice == 3:
#         break
#     else:
#         print("Please, choose correct choice.")


import traci
import sys
import threading
from api.v2i import traffic_light as traffic_light_manager
from api.v2i import glosa as glosa_manager
from api.v2v import communication as v2v_client
from api.output import logger,plotter
from api.sim import helper
from api.sim import visualizer

plot_data_v1 = []
plot_data_v2 = []

def run_sim():
    '''Main function that performs simulation steps and executes the v2i and v2v logic'''
    thread = threading.Thread(target=plotter.plot_speed)
    thread.start()

    thread2 = threading.Thread(target=plotter.plot_speed_v2)
    thread2.start()

    step = 0 # step in simulation count
    ttc = 0 # time to change (traffic light event)
    while traci.simulation.getMinExpectedNumber() > 0:
        visualizer.clear_all_polylines()
        vehicles = traci.vehicle.getIDList()
        v2v_client.send_messages()
        v2v_client.collect_messages()

        if(ttc <= 0): # handle traffic lights at intersection
            state, ttc = traffic_light_manager.get_current_phases()
            traci.trafficlight.setRedYellowGreenState('tli', state)
            print(f"TLI at intersection changed! Next event in {ttc}s")

        if(step > 0 and step % 3 == 0): # handle vehicles following green light optimal speed advisory (glosa)
            for vehicle in helper.get_super_vehicles(vehicles):
                glosa_manager.move_according_to_glosa(vehicle)

        if(step > 0 and 'v2v2i.0' in vehicles):
            plot_data_v1.append(traci.vehicle.getSpeed('v2v2i.0'))
            plotter.plot_queue_v1.put(plot_data_v1[:])  # add the data to the queue

        if(step > 0 and 'v2v2i.1' in vehicles):
            plot_data_v2.append(traci.vehicle.getSpeed('v2v2i.1'))
            plotter.plot_queue_v2.put(plot_data_v2[:])  # add the data to the queue

        logger.printlog("Next traffic signal event in " + str(ttc) + "seconds.")

        traci.simulationStep()

        step += 1
        ttc -= 1

    end_sequence()

def end_sequence():
    traci.close()
    sys.stdout.flush()

def start_simulation(sumo_binary, sumo_config_file):
    '''Starts the Sumo simulation using the provided sumo config file in TraCI'''
    traci.start([sumo_binary, "-c", sumo_config_file, "--start"])
    run_sim()


from datetime import datetime
from dotenv import load_dotenv

from airflow import DAG
from airflow.operators.python import PythonOperator

from download_data import download_data
from data_processing import raw_data_processing, feature_engineering
from ml_training import ml_training

DAG_ID = "stock_etf_data_pipeline"

default_args = {
    "owner": "stock_market",
    "depends_on_past": False,
    "retries": 0,
}

load_dotenv()

with DAG(
    DAG_ID,
    default_args=default_args,
    schedule_interval=None,
    start_date=datetime(2023, 7, 5),
    catchup=False,
    tags=["stocks", "etfs", "data-pipeline"],
) as dag:
    download_data = PythonOperator(
        task_id="download_data",
        python_callable=download_data,
    )

    etfs_raw_data_processing = PythonOperator(
        task_id="etfs_raw_data_processing",
        python_callable=raw_data_processing,
        op_kwargs={"stockType": "etfs"},
    )

    stocks_raw_data_processing = PythonOperator(
        task_id="stocks_raw_data_processing",
        python_callable=raw_data_processing,
        op_kwargs={"stockType": "stocks"},
    )

    feature_engineering = PythonOperator(
        task_id="feature_engineering",
        python_callable=feature_engineering,
    )

    ml_training = PythonOperator(
        task_id="ml_training",
        python_callable=ml_training,
    )

    (
        download_data
        >> [etfs_raw_data_processing, stocks_raw_data_processing]
        >> feature_engineering
        >> ml_training
    )


f = open("../inputs/day20.txt", "r")

tiles = f.read().split("\n\n")

tiles = [tile.split("\n") for tile in tiles]

ids = []
for i in range(len(tiles)):
    ids.append(int(tiles[i][0][5:9]))
    tiles[i] = tiles[i][1:]

def top(tile):
    return tile[0]

def bottom(tile):
    return tile[-1][::-1]

def right(tile):
    return "".join(tile[i][-1] for i in range(0, len(tile)))

def left(tile):
    return "".join(tile[i][0] for i in reversed(range(0, len(tile))))

adj = []

ans1 = 1
start = -1
for i in range(len(tiles)):
    possible = []
    for j in range(len(tiles)):
        if i == j:
            continue
        sides = [top, right, bottom, left]
        for x in range(4):
            for y in range(4):
                a, b = sides[x](tiles[i]), sides[y](tiles[j])
                if a == b or a == b[::-1]:
                    possible.append((j, x, y))
    adj.append(possible)
    if len(possible) == 2:
        ans1 *= ids[i]
        if start == -1:
            start = i

print("p1:", ans1)

def rotate(m):
    m = [[char for char in line] for line in m]
    return ["".join(row) for row in [[m[j][i] for j in range(len(m))] for i in range(len(m[0])-1, -1, -1)]]

def flip_hor(m):
    return [line[::-1] for line in m]

def flip_vert(m):
    return m[::-1]

def rotateK(m, k):
    return rotateK(rotate(m), k-1) if k > 0 else m

size = 12
image = list([-1] * size for _ in range(size))

remaining_tiles = set(range(len(tiles)))

def get_orientation(prev, i, j):
    prev_side = right if j > 0 else bottom
    curr_side = left if j > 0 else top
    flip = flip_hor if j > 0 else flip_vert

    for curr in remaining_tiles:
        for k in range(4):
            m = rotateK(tiles[curr], k)
            if curr_side(m) == prev_side(prev)[::-1]:
                return curr, k, None
            elif prev_side(m) == prev_side(prev):
                return curr, k, flip
    print("NONONO")

prev = start
for i in range(size):
    for j in range(size):
        if i + j == 0:
            image[i][j] = rotateK(tiles[start], 2)
            remaining_tiles.remove(start)
        else:
            prev = image[i][j-1] if j > 0 else image[i-1][j]
            curr, k, flip = get_orientation(prev, i, j)

            image[i][j] = rotateK(tiles[curr], k)
            if flip:
                image[i][j] = flip(image[i][j])
            remaining_tiles.remove(curr)

reduced_image = []

for i in range(size):
    for k in range(1, 9):
        line = ""
        for j in range(size):
            line += image[i][j][k][1:9]
        reduced_image.append(line)

monster =  \
[
"                  # ",
"#    ##    ##    ###",
" #  #  #  #  #  #   ",
]

def match(i, j, img):
    for k in range(3):
        for h in range(20):
            if i + k >= len(img) or j + h >= len(img[0]):
                return False
            if monster[k][h] == "#" and img[i+k][j+h] != monster[k][h]:
                return False
    return True

def solve(img):
    total = 0
    count = 0
    for i in range(len(img)):
        for j in range(len(img[0])):
            total += img[i][j] == "#"
            count += match(i, j, img)
    if count == 0:
        return -1
    return total - count * 15

oriented = flip_hor(rotateK(reduced_image, 3))
print(solve(oriented))


import numpy as np
import pickle

def unpickle(filename):
    import pickle
    with open(filename, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

def load_cifar100_data(data_home='./cifar100_dataset/cifar-100-python/',label_mode='coarse'):
    """
    data : (50000,3072) (10000,3072)
    labels : (50000,) (10000,)
    :param data_home:
    :return:
    """
    if label_mode == 'coarse':
        label_mode = b'coarse_labels'
    elif label_mode == 'fine':
        label_mode = b'fine_labels'

    train_dict = unpickle(data_home+'train')
    test_dict = unpickle(data_home+'test')

    train_datas = train_dict[b'data']
    train_labels = train_dict[label_mode]
    test_datas = test_dict[b'data']
    test_labels = test_dict[label_mode]
    # datas 是uint8 的ndarray 但是label需要我们自己转换
    train_labels = np.array(train_labels,dtype=np.uint8)
    test_labels = np.array(test_labels,dtype=np.uint8)

    # image data 归一化 不然影响cnn的卷积参数 导致梯度过大 不同与标准化 不会改变分布
    # 这个原始数据没有归一化
    data_max = np.max(train_datas)
    data_min = np.min(train_datas)
    train_datas = (train_datas - data_min) / (data_max - data_min)
    test_datas = (test_datas - data_min) / (data_max - data_min)

    randidx = np.random.permutation(train_datas.shape[0])
    # randidx = np.random.shuffle(np.range(mnist.data.shape[0]))
    train_datas = train_datas[randidx]
    train_labels = train_labels[randidx]

    randidx = np.random.permutation(test_datas.shape[0])
    # randidx = np.random.shuffle(np.range(mnist.data.shape[0]))
    test_datas = test_datas[randidx]
    test_labels = test_labels[randidx]

    return (train_datas,train_labels,test_datas,test_labels)

if __name__ == '__main__':
    data_home = './cifar100_dataset/cifar-100-python/'
    # train_dict = unpickle(data_home+'train')
    # test_dict = unpickle(data_home+'test')
    # desc_dict = unpickle(data_home+'meta')
    # print(len(desc_dict[b'fine_label_names']),len(desc_dict[b'coarse_label_names']))
    # print(len(train_dict[b'data']))
    # print(len(test_dict[b'data']))
    # print(train_dict[b'batch_label'])
    # print(test_dict[b'batch_label'])
    #
    # print(train_dict[b'fine_labels'][:10])
    # print(train_dict[b'coarse_labels'][:10])
    # # 100 fine classes   20 coarse classes
    #
    # print(train_dict[b'data'].shape) # (50000, 3072)
    # print(np.max(train_dict[b'data']),np.min(train_dict[b'data'])) # 255 0
    # print(type(train_dict[b'data'][0][0])) # <class 'numpy.uint8'>
    # print(type(train_dict[b'coarse_labels'][0])) # <class 'numpy.uint8'>

    load_cifar100_data(data_home=data_home)


""" edb dosyalarini star-charts'in istedigi formata cevirir """
import logging
import os
import math
import ephem

EXT = ".edb"
PATH = "edb/"

def get_lines(file_path):
    """ dosyadaki tum satirlari doner """
    with open(file_path, "r") as edb_file:
        return edb_file.readlines()

def angle_to_right_ascension(angle):
    """ aciyi star-charts'in istedigi formata cevirir """
    return (1 - angle / (2 * math.pi)) * 24

def convert(date, lat, lon, elevation):
    """ degistirme isini yapar """
    gatech = ephem.Observer()
    gatech.lon = lat
    gatech.lat = lon
    gatech.elevation = elevation
    gatech.date = date

    catalog_stars = []
    for edb_file in os.listdir(PATH):
        if edb_file.endswith(EXT):
            catalog_stars.extend(get_lines(PATH + edb_file))

    new_catalog = ""
    for catalog_star in catalog_stars:
        if catalog_star.startswith("#") or catalog_star.strip() == "":
            continue

        try:
            star = ephem.readdb(catalog_star)
            star.compute(gatech)

            ra = angle_to_right_ascension(star.az)
            dec = math.degrees(star.alt)

            new_catalog += f"{ra},{dec},{star.mag},{star.name}\n"
        except Exception as ex:
            print(catalog_star)
            logging.exception(ex)

    planets = [
        ephem.Jupiter(),
        ephem.Mars(),
        ephem.Mercury(),
        ephem.Moon(),
        ephem.Neptune(),
        ephem.Pluto(),
        ephem.Saturn(),
        ephem.Sun(),
        ephem.Uranus(),
        ephem.Venus(),
    ]
    for planet in planets:
        planet.compute(gatech)
        ra = angle_to_right_ascension(planet.az)
        dec = math.degrees(planet.alt)
        new_catalog += f"{ra},{dec},{planet.mag},{planet.name}\n"

    return new_catalog


from django.shortcuts import get_object_or_404, redirect, render
from agenda.models import Compromisso
from agenda.forms import CompromissoForm

# Create your views here.
def listagem(request):
    if request.user.is_authenticated:
        context = {}
        compromissos = Compromisso.objects.all().order_by('-data')
        context['compromissos'] = compromissos
        return render(request, 'agenda/listagem.html', context)
    else:
        redirect('accounts/login')

def create(request):
    if request.user.is_authenticated:
        if request.method == 'POST':
            form = CompromissoForm(request.POST)
            if form.is_valid():
                form.save()
                return redirect('../')
            else:
                return render(request, 'agenda/create.html', {'form': form})
        else:
            form = CompromissoForm()
            return render(request, 'agenda/create.html', {'form': form})
    else:
        return redirect('login')

def delete(request, compromisso_id):
    objeto = Compromisso.objects.get(pk=compromisso_id)
    objeto.delete()
    return redirect('../')


import json
import requests
from decouple import config
import datetime

# 기본, 게임, 엔터테인먼트, 뷰티, 스포츠, 먹망, 키즈, 동물, 일상, IT
NECESSARY_WORD = [
    ['유튜브', '유튜버', '유투버', '채널'],
    ['게임', 'game'],
    ['엔터테인먼트', '예능', 'entertainment'],
    ['뷰티', '화장', '패션'],
    ['운동', 'Sports', '스포츠', '헬스'],
    ['먹방', '음식', '푸드', 'Food'],
    ['키즈', '어린이', 'Kids'],
    ['동물', '애니멀', 'Animal'],
    ['일상', '브이로그', 'V-log', 'Vlog'],
    ['IT', 'SW', '소프트웨어', '기술', '신제품', '노트북', '컴퓨터', '시스템', '스마트폰', '무선']
]

MONTH = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
YOUTUBER = 'workman'    # DB 에서 yno 에 해당하는 channelName or youtubeName 을 입력
CATEGORY = [2]     # 해당 유튜버의 카테고리가 게임과 엔터테인먼트

URL = 'https://openapi.naver.com/v1/search/news.json?'
params = {
    'query': YOUTUBER,
    'display': 100,
    'start': 1,
    'sort': 'date'
}

headers = {
    'X-Naver-Client-Id': config('X_NAVER_CLIENT_ID'),
    'X-Naver-Client-Secret': config('X_NAVER_CLIENT_SECRET'),
}

category_keyword = NECESSARY_WORD[0]
for num in CATEGORY:
    category_keyword += NECESSARY_WORD[num]

# 만약 last updatedDate 가 2019-03-15 면 3월 15일 이후의 기사만 가져와야함
updatedDate = datetime.datetime(2019, 3, 15)
news_list = []
while True:
    response = requests.get(URL, params=params, headers=headers).text  # str type
    total = json.loads(response)["total"]
    newses = json.loads(response)["items"]

    for (index, news) in enumerate(newses):
        isCorrect = False
        date = datetime.datetime(int(news["pubDate"][12:16]), MONTH.index(news["pubDate"][8:11]), int(news["pubDate"][5:7]))

        # 업데이트되어있는 기사는 추가하지 않음
        if (date - updatedDate).days < 0:
            break

        # Description 에 특정 키워드가 없는 기사는 추가하지 않음
        for keyword in category_keyword:
            if keyword in news["description"]:
                isCorrect = True
                break
        if not isCorrect:
            continue

        newNews = {
            'newsLink': news["link"],
            'newsTitle': news["title"],
            'newsDescription': news["description"],
            'newsDate': str(date)[:10]
        }
        news_list.append(newNews)
    if params['start'] >= min(1000, total):
        break
    params['start'] += params['display']


#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import subprocess
import xml.etree.ElementTree as ET
from xml.etree.ElementTree import ParseError
from WifiBox import WifiBox
import logging

"""
A tab to stock each network
"""
tab_mixte = []

def sup(box_1,box_2):
    """
    Compare two WifiBox instance
    :param box_1: first box you want to compare
    :param box_2: second box you want to compare

    .. note:: A x10 factor is applyed to WEP Wi-FI because they're easyier to Hack
    """
    wep_factor = 10
    data1 = abs( box_1._Data )
    pow1 = abs(box_1._PWR)
    data2 = abs( box_2._Data )
    pow2 = abs(box_2._PWR)
    if str(box_1._Encryption).find('WEP') != -1 :
        data1 = data1 * wep_factor
        pow1 = pow1 * wep_factor
    if str(box_2._Encryption).find('WEP') != -1 :
        data2 = data2 * wep_factor
        pow2 = pow2 * wep_factor

    if data1 > data2:
        return -1
    elif pow1 > pow2 :
        return -1
    elif data1 == data2 and pow1 > pow2 :
        return 0
    else :
        return 1

def parsing_global():
    """
    Parsing of the xml file from global_listening
    stock each WPA and WEP networks in tab_mixte
    Other boxes are not keep in memory
    """
    nom_ecoute = "TestBox/global/record"
    version = "-01"
    nom_fichier_xml_ecoute_global = nom_ecoute + version + ".kismet.netxml"
    try:
        tree = ET.parse(nom_fichier_xml_ecoute_global)
        root = tree.getroot()
        for child in root:
            BSSID = str(child.find('BSSID').text)
            for network in child.findall('SSID'):
                ESSID = str(network.find('essid').text)
                Encryption = str(network.find('encryption').text)
                Data = int(child.find('packets').find('data').text)
                Channel = int(child.find('channel').text)
                PWR = int(child.find('snr-info').find('last_signal_dbm').text)
                if BSSID != "None" and str(ESSID) != "None":
                    nouveau_reseau = WifiBox(BSSID, ESSID, Encryption, Data, Channel, PWR)
                    if nouveau_reseau in tab_mixte:
                        index = tab_mixte.index(nouveau_reseau)
                        tab_mixte[index] = nouveau_reseau
                    else:
                        tab_mixte.append(nouveau_reseau)
    except ParseError:
            logging.warning("Can't parse : %s", nom_fichier_xml_ecoute_global)

def prepare_wep_listining():
    """
    sorte tab_mixte with sup function
    """
    global tab_mixte
    tab_mixte = sorted(tab_mixte, sup)
    return tab_mixte


from three.A import Turtle, draw_star, draw_polygon
from math import sin, radians, atan, degrees, sqrt, cos, sin
from functools import partial
import svgwrite
from itertools import combinations

length = 100

def inscribed_polygram_rel(turtle, number_of_vertices=5, length=length):
    star_angle = 180 / number_of_vertices
    polygon_side_length = sin(radians(star_angle))/sin(radians(star_angle*2))*length #law of sines

    draw_star(turtle, number_of_vertices, length)
    turtle.right(star_angle*2)
    draw_polygon(turtle, number_of_vertices, length=polygon_side_length)

def inscribed_polygram_abs(turtle, number_of_vertices=5, x=35,y=5, length=length/2):
    vertices = get_polygon_points(number_of_vertices=5, x=x, y=y, length=length)
    for i in combinations(vertices, 2):
        turtle.image.add(turtle.line(i[0], i[1]))

def recursive_squares(turtle, rec_depth=200, offset = 0.2, length=length):
    angle =  degrees(atan(length*offset/(length-length*offset)))
    for i in range(rec_depth):
        for j in range(4): #draw square
            turtle.forward(length)
            turtle.right(90)
        turtle.penup()
        turtle.forward(length*offset)
        turtle.pendown()
        turtle.right(angle)
        _length = length*offset
        length = sqrt(_length**2+(length-_length)**2) #calculate length of inner square

def circle_grid(turtle, length=100, radius=50):
    g = svgwrite.container.Group(clip_path='url(#circle)') #grid
    for x in range(0, length, int(length/10)):
        g.add(turtle.line((0, x), (length, x)))
        g.add(turtle.line((x, 0), (x, length)))

    clip = svgwrite.masking.ClipPath(id='circle')
    clip.add(svgwrite.shapes.Circle((length/2, length/2), r=radius)) #clipping circle
    turtle.image.defs.add(clip)
    turtle.image.add(g)

def recursive_triangles(turtle, rec_depth=12, length=100):
    side = length
    step = 5
    for i in range(rec_depth):
        for i in range(3):
            turtle.forward(side)
            turtle.right(120)
        turtle.penup()
        turtle.right(30)
        turtle.forward(step)
        turtle.left(30)
        turtle.pendown()
        side -= 9 #magic

'''Calculates coordinates for regular polygons
'''
def get_polygon_points(number_of_vertices, start_angle=0, x=255, y=255, length=length, assign_angles=False):
    points =[]
    for i in range(1, number_of_vertices+1):
        angle = i*360/number_of_vertices+start_angle
        rad = radians(angle)
        new_x   = length * cos(rad)
        new_y   = length * sin(rad)
        if assign_angles:
            points.append(((x+new_x, y+new_y), angle))
        else:
            points.append((x+new_x, y+new_y))
        x += new_x
        y += new_y
    return points

def flower_polygon(turtle, number_of_vertices=16, number_of_polygons=16, length=7):
    angle = 360.0/number_of_polygons
    for i in range(0, number_of_polygons):
        turtle.image.add(svgwrite.shapes.Polygon(points=get_polygon_points(number_of_vertices, x=turtle.x+length*7, y=turtle.y+length*7, start_angle=angle*i, length=length), stroke='black', style="fill-opacity:0;"))

def draw_object(draw_func, length=length, start_x=65, start_y=65):
    turtle = Turtle(draw_func.__name__+'.svg', start_x=start_x, start_y=start_y)
    draw_func(turtle=turtle)
    turtle.save_image()

if __name__ == "__main__":
    draw_object(draw_func=recursive_triangles, start_x=5, start_y=5)
    draw_object(draw_func=flower_polygon, start_x=0, start_y=0)
    draw_object(draw_func=inscribed_polygram_rel, start_x=5, start_y=40)
    draw_object(draw_func=inscribed_polygram_abs)
    draw_object(draw_func=recursive_squares, start_x=5, start_y=5)
    draw_object(draw_func=circle_grid, start_x=10, start_y=10)


from typing import Optional

# Definition for singly-linked list.
class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    def addTwoNumbers(
        self, l1: Optional[ListNode], l2: Optional[ListNode]
    ) -> Optional[ListNode]:

        s1 = ""
        t1 = l1
        s2 = ""
        t2 = l2
        while t1 or t2:
            if t1:
                s1 += str(t1.val)
                t1 = t1.next
            if t2:
                s2 += str(t2.val)
                t2 = t2.next

        sum_val = int(s1) + int(s2)

        l3 = ListNode("")
        t3 = l3
        for i in str(sum_val):
            node = ListNode(i)
            t3.next = node
            t3 = t3.next
        return l3.next


from lucy import __appname__, __version__
from setuptools import setup

long_description = ""

setup(
    name=__appname__,
    version=__version__,
    scripts=[],
    packages=[
        'lucy',
    ],
    author="Paul Tagliamonte",
    author_email="tag@pault.ag",
    long_description=long_description,
    description='Lucy!',
    license="Expat",
    url="http://deb.io/",
    platforms=['any'],
    entry_points={
        'console_scripts': [
            'lucy-nuke = lucy.cli.nuke:main',
            'lucy-process-incoming = lucy.cli.incoming:main',
            'lucy-init = lucy.cli.init:main',
            'lucyd = lucy.server:main',
        ],
    }
)


#  Sample emp doc and a custom map on it -
#  https://gist.github.com/arunapiravi/044d6547b7853ad6c14a
import random
import copy
from TestInput import TestInputSingleton

EMP_FIELDS = {
    'text': ["name", "dept", "languages_known", "email"],
    'number': ["mutated", "salary"],
    'boolean': ["is_manager"],
    'datetime': ["join_date"],
    'object': ["manages"]  # denote nested fields
}

TOTAL_EMP_FIELDS = 9

EMP_NESTED_FIELDS = {
    'manages': {
        'text': ["reports"],
        'number': ["team_size"]
    }
}

# Need to work on double nested field "revision_text_text"

WIKI_FIELDS = {
    'text': ["title", "type"],
    'number': ["mutated"],
    #'object': ["revision", "text", "contributor"]
    'object': ["revision"]
}

TOTAL_WIKI_FIELDS = 6

WIKI_NESTED_FIELDS = {
    'revision': {
        'datetime': ["timestamp"]
    },
    #'text': {
    #    'text': ["#text"]
    #},
    #'contributor': {
    #    'text': ["username"]
    #}
}

FULL_FIELD_NAMES = {
    'reports': 'manages_reports',
    'team_size': 'manages_team_size',
    'timestamp': 'revision_timestamp',
    #'#text': 'revision_text_text',
    #'username': 'revision_contributor_username'
}

CUSTOM_ANALYZER_TEMPLATE = {
    "analyzers": {},
    "token_filters": {
        "back_edge_ngram": {
            "back":True,
            "max": 5,
            "min": 3,
            "type": "edge_ngram"
        },
        "dict_compound_en": {
            "dict_token_map": "stop_en",
            "type": "dict_compound"
        },
        "dict_compound_fr": {
            "dict_token_map": "articles_fr",
            "type": "dict_compound"
        },
        "front_edge_ngram": {
            "back":False,
            "max": 5,
            "min": 3,
            "type": "edge_ngram"
        },
        "keyword_marker": {
            "keywords_token_map": "stopwords",
            "type": "keyword_marker"
        },
        "stopwords": {
            "stop_token_map": "stopwords",
            "type": "stop_tokens"
        },
        "length": {
            "max": 5,
            "min": 3,
            "type": "length"
        },
        "ngram": {
            "max": 5,
            "min": 3,
            "type": "ngram"
        },
        "shingle": {
            "filler": "",
            "max": 5,
            "min": 2,
            "output_original": "false",
            "separator": "",
            "type": "shingle"
        },
        "truncate": {
            "length": 10,
            "type": "truncate_token"
        }
    },
    "token_maps": {
        "stopwords": {
            "tokens": ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves',
                 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him',
                 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its',
                 'itself', 'they', 'them', 'their', 'theirs', 'themselves',
                 'what', 'which', 'who', 'whom', 'this', 'that', 'these',
                 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been',
                 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did',
                 'doing', 'would', 'should', 'could', 'ought', "i'm", "you're",
                 "he's", "she's", "it's", "we're", "they're", "i've", "you've",
                 "we've", "they've", "i'd", "you'd", "he'd", "she'd", "we'd",
                 "they'd", "i'll", "you'll", "he'll", "she'll", "we'll",
                 "they'll", "isn't", "aren't", "wasn't", "weren't", "hasn't",
                 "haven't", "hadn't", "doesn't", "don't", "didn't", "won't",
                 "wouldn't", "shan't", "shouldn't", "can't", 'cannot',
                 "couldn't", "mustn't", "let's", "that's", "who's", "what's",
                 "here's", "there's", "when's", "where's", "why's", "how's",
                 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as',
                 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about',
                 'against', 'between', 'into', 'through', 'during', 'before',
                 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in',
                 'out', 'on', 'off', 'over', 'under', 'again', 'further',
                 'then', 'once', 'here', 'there', 'when', 'where', 'why',
                 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most',
                 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own',
                 'same', 'so', 'than', 'too', 'very'],
            "type": "custom"
        }
    },
    "char_filters": {
        "mapping": {
            "regexp": "[f]",
            "replace": "ph",
            "type": "regexp"
        }
    },
    "tokenizers": {
        "alphanumeric": {
            "regexp": "[0-9a-zA-Z_]*",
            "type": "regexp"
        }
    }
}

ANALYZERS = ["standard", "simple", "keyword", "en"]

LANG_ANALYZERS = ["ar", "cjk", "fr", "fa", "hi", "it", "pt", "en", "web"]

CHAR_FILTERS = ["html", "mapping"]

TOKENIZERS = ["letter", "single", "unicode", "web", "whitespace", "alphanumeric"]

TOKEN_FILTERS = ["apostrophe", "elision_fr", "to_lower", "ngram",
                 "front_edge_ngram", "back_edge_ngram", "shingle",
                 "truncate", "stemmer_porter", "length", "keyword_marker",
                 "stopwords", "cjk_bigram", "stemmer_it_light",
                 "stemmer_fr_light", "stemmer_fr_min", "stemmer_pt_light"]

class CustomMapGenerator:
    """
    # Generates an FTS and equivalent ES custom map for emp/wiki datasets
    """
    def __init__(self, seed=0, dataset="emp", num_custom_analyzers=0,multiple_filters=False,
                 custom_map_add_non_indexed_fields=True, text_analyzer=None, type_mapping=None, collection_index=False):
        random.seed(seed)
        self.fts_map = {"types": {}}
        self.es_map = {}
        self.num_field_maps = random.randint(1, 10)
        self.queryable_fields = {}
        self.num_custom_analyzers = num_custom_analyzers
        self.text_analyzer = text_analyzer
        self.custom_map_add_non_indexed_fields = custom_map_add_non_indexed_fields
        # Holds the list of custom analyzers created by
        # build_custom_analyzer method
        self.custom_analyzers=[]
        self.multiple_filters = multiple_filters

        for n in range(0, self.num_custom_analyzers, 1):
            self.custom_analyzers.append("customAnalyzer"+str(n+1))

        if dataset == "emp":
            self.fields = EMP_FIELDS
            self.nested_fields = EMP_NESTED_FIELDS
            self.max_fields = TOTAL_EMP_FIELDS
            type_mapping_val = type_mapping + "." + dataset if collection_index else dataset
            self.fts_map['types'][type_mapping_val] = {
                                        "dynamic": False,
                                        "enabled": True,
                                        "fields": [],
                                        "properties": {}
                                    }
            self.es_map[dataset] = {
                        "dynamic": False,
                        "properties": {}
                    }
            if collection_index:
                self.build_custom_map(dataset, type_mapping)
            else:
                self.build_custom_map(dataset)
        elif dataset == "wiki":
            self.fields = WIKI_FIELDS
            self.nested_fields = WIKI_NESTED_FIELDS
            self.max_fields = TOTAL_WIKI_FIELDS
            type_mapping_val = type_mapping + "." + dataset if collection_index else dataset
            self.fts_map['types'][type_mapping_val] = {
                                        "dynamic": False,
                                        "enabled": True,
                                        "fields": [],
                                        "properties": {}
                                    }
            self.es_map[dataset] = {
                        "dynamic": False,
                        "properties": {}
                    }
            if collection_index:
                self.build_custom_map(dataset, type_mapping)
            else:
                self.build_custom_map(dataset)
        elif dataset == "all":
            self.fields = EMP_FIELDS
            self.nested_fields = EMP_NESTED_FIELDS
            self.max_fields = TOTAL_EMP_FIELDS
            type_mapping_val = type_mapping+".emp" if collection_index else 'emp'
            self.fts_map['types'][type_mapping_val] = {
                                        "dynamic": False,
                                        "enabled": True,
                                        "fields": [],
                                        "properties": {}
                                    }
            self.es_map['emp'] = {
                        "dynamic": False,
                        "properties": {}
                    }
            if collection_index:
                self.build_custom_map("emp", type_mapping)
            else:
                self.build_custom_map("emp")
            if int(TestInputSingleton.input.param("doc_maps", 1)) > 1:
                self.fields = WIKI_FIELDS
                self.nested_fields = WIKI_NESTED_FIELDS
                self.max_fields = TOTAL_WIKI_FIELDS
                type_mapping_val = type_mapping+".wiki" if collection_index else 'wiki'
                self.fts_map['types'][type_mapping_val] = {
                                        "dynamic": False,
                                        "enabled": True,
                                        "fields": [],
                                        "properties": {}
                                    }
                self.es_map['wiki'] = {
                        "dynamic": False,
                        "properties": {}
                    }
                if collection_index:
                    self.build_custom_map("wiki", type_mapping)
                else:
                    self.build_custom_map("wiki")
            else:
                if not TestInputSingleton.input.param("default_map", False):
                    # if doc_maps=1 and default map is disabled, force single
                    # map on ES by disabling wiki map
                    self.es_map['wiki'] = {
                            "dynamic": False,
                            "properties": {}
                        }

    def get_random_value(self, list):
        return list[random.randint(0, len(list)-1)]

    def get_map(self):
        return self.fts_map, self.es_map

    def get_smart_query_fields(self):
        """
        Smart querying refers to generating queries on
        fields referenced in the custom map
        """
        return self.queryable_fields

    def add_to_queryable_fields(self, field, field_type):
        if field in FULL_FIELD_NAMES:
            # if nested field, then fully qualify the field name
            field = FULL_FIELD_NAMES[field]
        if field_type not in list(self.queryable_fields.keys()):
            self.queryable_fields[field_type] = []
        if field not in self.queryable_fields[field_type]:
            self.queryable_fields[field_type].append(field)

    def build_custom_map(self, dataset, collection_type=None):
        for x in range(0, self.num_field_maps):
            field, _type = self.get_random_field_name_and_type(self.fields)
            type_val = collection_type + "." + dataset if collection_type else dataset
            if field not in iter(list(self.nested_fields.keys())):
                fts_child, es_child = self.get_child_field(field, _type)
            else:
                fts_child, es_child = self.get_child_map(field, type_val, dataset)
            self.fts_map['types'][type_val]['properties'][field] = fts_child
            self.es_map[dataset]['properties'][field] = es_child

        if self.custom_map_add_non_indexed_fields:
            self.add_non_indexed_field_to_query()

    def add_non_indexed_field_to_query(self):
        """
        Add 1 or 2 non-indexed fields(negative test for custom mapping)
        Query on non-indexed fields to see if 0 results are returned
        """
        count = 0
        if self.num_field_maps < self.max_fields:
            while count < self.max_fields:
                count += 1
                field, field_type = self.get_random_field_name_and_type(
                    self.fields)
                if field_type != 'object' and \
                   field_type not in list(self.queryable_fields.keys()):
                    print(("Adding an extra non-indexed field '%s' to" \
                          " list of queryable fields" % field))
                    self.queryable_fields[field_type] = [field]
                    break
                if field_type != 'object' and \
                   field not in self.queryable_fields[field_type]:
                    print(("Adding an extra non-indexed field '%s' to" \
                          " list of queryable fields" % field))
                    self.queryable_fields[field_type].append(field)
                    break
            else:
                print(("Unable to add a non-indexed field after %s retries" \
                      % self.max_fields))

    def get_child_map(self, field, dataset, es_dataset):
        """
        Child maps are for nested json structures i.e, any higher level field
        having another nested structure as its value
        """
        current_prop = self.fts_map['types'][dataset]['properties']
        if field not in iter(current_prop.keys()):
            fts_child_map = {}
            fts_child_map['dynamic'] = False
            fts_child_map['enabled'] = True
            fts_child_map['fields'] = []
            fts_child_map['properties'] = {}

            es_child_map = {}
            es_child_map['dynamic'] = False
            es_child_map['enabled'] = True
            es_child_map['type'] = "object"
            es_child_map['properties'] = {}
        else:
            fts_child_map = self.fts_map['types'][dataset]['properties'][field]
            es_child_map = self.es_map[es_dataset]['properties'][field]

        field, type = self.get_nested_child_field(field)
        fts_child, es_child = self.get_child_field(field, type)
        fts_child_map['properties'][field] = fts_child
        es_child_map['properties'][field] = es_child

        return fts_child_map, es_child_map

    def get_child_field(self, field, type):
        """
        Encapsulate the field map with 'dynamic', 'enabled', 'properties'
        and fields
        """
        fts_child = {}
        fts_child['dynamic'] = False
        fts_child['enabled'] = True
        fts_child['properties'] = {}
        fts_child['fields'] = []

        fts_field, es_field = self.get_field_map(field, type)
        fts_child['fields'].append(fts_field)

        return fts_child, es_field

    def get_field_map(self, field, field_type):
        """
        Set Index properties for any field in json and return the field map
        """
        is_indexed = bool(random.getrandbits(1))
        fts_field_map = {}
        fts_field_map['include_in_all'] = True
        fts_field_map['include_term_vectors'] = True
        fts_field_map['index'] = True
        fts_field_map['name'] = field
        fts_field_map['store'] = False
        fts_field_map['type'] = field_type
        if self.num_custom_analyzers:
            analyzer = self.get_random_value(self.custom_analyzers)
        elif self.text_analyzer:
            analyzer = self.text_analyzer
        else:
            analyzer = self.get_random_value(ANALYZERS)
        if field_type == "text":
            fts_field_map['analyzer'] = analyzer
        else:
            fts_field_map['analyzer'] = ""

        es_field_map = {}
        es_field_map['type'] = field_type
        if field_type == "number":
            es_field_map['type'] = "float"
        if field_type == "datetime":
            es_field_map['type'] = "date"
        es_field_map['store'] = False
        #if is_indexed:
        es_field_map['index'] = 'analyzed'
        if field_type == "boolean":
            es_field_map['index'] = 'not_analyzed'
        #else:
        #    es_field_map['index'] = 'no'
        if field_type == "text":
            es_field_map['type'] = "string"
            es_field_map['term_vector'] = "yes"

            es_field_map['analyzer'] = analyzer
            if analyzer == "en":
                es_field_map['analyzer'] = "english"
            if analyzer == "standard":
                es_field_map['analyzer'] = "default"

        # add to list of queryable fields
        self.add_to_queryable_fields(field, field_type)

        return fts_field_map, es_field_map

    def get_random_field_name_and_type(self, fields):
        type = self.get_random_value(list(fields.keys()))
        field = self.get_random_value(fields[type])
        return field, type

    def get_nested_child_field(self, nested_field):
        if nested_field in iter(list(self.nested_fields.keys())):
            return self.get_random_field_name_and_type(
                self.nested_fields[nested_field])

    def build_custom_analyzer(self):
        analyzer_map = {}
        if self.multiple_filters:
            num_token_filters = random.randint(1, min(3, len(TOKEN_FILTERS)))
            num_char_filters = random.randint(1, min(3, len(CHAR_FILTERS)))
        else:
            num_token_filters = 1
            num_char_filters = 1

        for custom_analyzer in self.custom_analyzers:
            analyzer_map[custom_analyzer] = {}
            analyzer_map[custom_analyzer]["char_filters"] = []
            analyzer_map[custom_analyzer]["token_filters"] = []
            analyzer_map[custom_analyzer]["tokenizer"] = ""

            for num in range(0, num_char_filters, 1):
                char_filter = self.get_random_value(CHAR_FILTERS)
                if not analyzer_map[custom_analyzer]["char_filters"].count(char_filter):
                    analyzer_map[custom_analyzer]["char_filters"].append(char_filter)

            tokenizer = self.get_random_value(TOKENIZERS)
            analyzer_map[custom_analyzer]["tokenizer"] = tokenizer

            for num in range(0, num_token_filters, 1):
                token_filter = self.get_random_value(TOKEN_FILTERS)
                if not analyzer_map[custom_analyzer]["token_filters"].count(token_filter):
                    analyzer_map[custom_analyzer]["token_filters"].append(token_filter)
            analyzer_map[custom_analyzer]["type"] = "custom"

        analyzer = CUSTOM_ANALYZER_TEMPLATE
        analyzer["analyzers"]=analyzer_map
        return analyzer

if __name__ == "__main__":
    import json
    custom_map = CustomMapGenerator(seed=1).get_map()
    print((json.dumps(custom_map, indent=3)))


"""Used in `SaveFile.plots`."""
from __future__ import annotations

from typing import Any, List

import attrs

from civ4save.vanilla import enums as e

@attrs.define(slots=True)
class Plot:
    """Represents a plot."""

    x: int
    y: int
    ownership_duration: int
    improvement_duration: int
    starting_plot: bool
    hills: bool
    can_be_worked: bool
    irrigated: bool
    owner: int
    plot_type: e.PlotType
    terrain_type: e.TerrainType
    feature_type: e.FeatureType
    bonus_type: e.BonusType
    improvement_type: e.ImprovementType
    yields: List[int]

    @classmethod
    def from_struct(cls, data: Any) -> Plot:
        """Return `Plot` from parsed struct."""
        plot_type = e.PlotType[data.plot_type]
        terrain_type = e.TerrainType[data.terrain_type]
        feature_type = e.FeatureType[data.feature_type]
        bonus_type = e.BonusType[data.bonus_type]
        improvement_type = e.ImprovementType[data.improvement_type]
        return cls(
            data.x,
            data.y,
            data.ownership_duration,
            data.improvement_duration,
            data.starting_plot,
            data.hills,
            data.potential_city_work,
            data.irrigated,
            data.owner,
            plot_type,
            terrain_type,
            feature_type,
            bonus_type,
            improvement_type,
            data.yields,
        )


# 进制转换 将int以不同进制表现出来
# 类型转换 将一个数据类型转换为其他类型的数据
# int -> str  str -> int

age = input('请输入您的年龄： ')

new_age = int(age)

# 使用int内置类可以将其他类型的数据转换为整数

print((new_age + 1))

09-数据类型的转换.py


import json

class Database:
    def __init__(self, filename):
        self.filename = filename
        self.prod_name = self._load_data()

    def _load_data(self):
        try:
            with open(self.filename, 'r') as file:
                return json.load(file)
        except FileNotFoundError:
            return []

    def _save_data(self):
        with open(self.filename, 'w') as file:
            json.dump(self.prod_name, file, indent=2)

    def create(self, product):
        self.prod_name.append(product)
        self._save_data()

    def read(self, product_id=None):
        if product_id is None:
            return self.prod_name
        for product in self.prod_name:
            if product['id'] == product_id:
                return product
        return None

    def update(self, product_id, new_name):
        for product in self.prod_name:
            if product['id'] == product_id:
                product.update(new_name)
                self._save_data()
                return True
        return False

    def delete(self, product_id):
        for index, product in enumerate(self.prod_name):
            if product['id'] == product_id:
                del self.prod_name[index]
                self._save_data()
                return True
        return False

class Model:
    def __init__(self, data):
        self.prod_name = data

    def validate(self):
        return True

class View:
    @staticmethod
    def print_products(products):
        for product in products:
            print(product)

    @staticmethod
    def print_message(message):
        print(message)

class Controller:
    def __init__(self, db):
        self.db = db

    def create_product(self, name):
        model = Model(name)
        if model.validate():
            self.db.create(name)
            View.print_message("Запись успешно создана.")
        else:
            View.print_message("Ошибка валидации данных. Запись не создана.")

    def read_products(self, product_id=None):
        if product_id is not None:
            product = self.db.read(product_id)
            if product:
                View.print_products([product])
            else:
                View.print_message("Запись не найдена.")
        else:
            products = self.db.read()
            View.print_products(products)

    def update_product(self, product_id, new_name):
        if self.db.update(product_id, new_name):
            View.print_message("Запись успешно обновлена.")
        else:
            View.print_message("Запись не найдена. Обновление не выполнено.")

    def delete_product(self, product_id):
        if self.db.delete(product_id):
            View.print_message("Запись успешно удалена.")
        else:
            View.print_message("Запись не найдена. Удаление не выполнено.")

db = Database("data.json")
controller = Controller(db)

data1 = {"id": 1, "name": "John"}
# data1 = {"id": 2, "name": "John", "age": 30}
controller.create_product(data1)

# controller.read_products()

# new_name = {'id': 2 ,"name": "John Doe", "age": 31}
# controller.update_product(2, new_name)

# controller.delete_product(2)
# controller.delete_product(1)
controller.read_products()


# LC 320. Generalized Abbreviation

'''
A word's generalized abbreviation can be constructed by taking any number of non-overlapping and non-adjacent substrings and replacing them with their respective lengths.

For example, "abcde" can be abbreviated into:
"a3e" ("bcd" turned into "3")
"1bcd1" ("a" and "e" both turned into "1")
"5" ("abcde" turned into "5")
"abcde" (no substrings replaced)
However, these abbreviations are invalid:
"23" ("ab" turned into "2" and "cde" turned into "3") is invalid as the substrings chosen are adjacent.
"22de" ("ab" turned into "2" and "bc" turned into "2") is invalid as the substring chosen overlap.
Given a string word, return a list of all the possible generalized abbreviations of word. Return the answer in any order.

Example 1:

Input: word = "word"
Output: ["4","3d","2r1","2rd","1o2","1o1d","1or1","1ord","w3","w2d","w1r1","w1rd","wo2","wo1d","wor1","word"]

Example 2:

Input: word = "a"
Output: ["1","a"]
'''

class Solution:
    # O(n * 2^n) time | O(2^n) space (O(n) using append/pop)
    def generateAbbreviations(self, word: str) -> List[str]:
        def backtrack(idx, count, path):
            if len(word) == idx:
                ct_str = str(count) if count > 0 else ""
                res.append(path + ct_str)
                return

            backtrack(idx + 1, count + 1, path)
            ct_str = str(count) if count > 0 else ""
            backtrack(idx + 1, 0, path + ct_str + word[idx])

        res = []
        backtrack(0, 0, "")
        return res

    # O(n * 2^n) time | O(n) space
    def bitmask(self, word: str) -> List[str]:
        def bitWord(bit):
            new_word = ""
            count = 0
            for i, b in enumerate(bit):
                if b == "0":
                    new_word += str(count) if count > 0 else ""
                    new_word += word[i]
                    count = 0
                else:
                    count += 1

            new_word += str(count) if count > 0 else ""
            return new_word

        res = []
        mask = 2**(len(word))

        for i in range(2**len(word)):
            bit = mask | i
            new_word = bitWord(bin(bit)[3:])
            res.append(new_word)

        return res

1. Problems/i. Recursion & Backtrack/b. Backtrack - Path - Generalized Abbreviation.py


# -*- coding: utf-8 -*-
import io
import sys
import datetime
import names
import csv
from gen_random_values import *

""" List of values for use in choices in models. """
person_list = []
address_list = []
repeat = 200

''' Lendo os dados de enderecos_.csv '''
with open('fixtures/enderecos_.csv', 'r') as f:
    r = csv.DictReader(f)
    for dct in r:
        address_list.append(dct)
    f.close()

with io.open('fixtures/clientes.csv', 'wt') as f:
    f.write(
        u'id,gender,first_name,last_name,cpf,birthday,email,phone,cell,address,district,city,uf,cep,created_at,modified_at\n')
    for i in range(repeat):
        g = random.choice(['M', 'F'])
        if g == 'M':
            first_name = names.get_first_name(gender='male')
        else:
            first_name = names.get_first_name(gender='female')
        last_name = names.get_last_name()
        cpf = gen_doc()
        birthday = gen_date()
        email = first_name[
            0].lower() + '.' + last_name.lower() + '@example.com'
        phone = gen_phone()
        cell = gen_phone()
        n = random.randint(1, 5)  # escolhe um id de enderecos_.csv
        address = '"' + address_list[n]['address'] + '"'
        district = address_list[n]['district']
        city = address_list[n]['city']
        uf = address_list[n]['uf']
        cep = address_list[n]['cep']
        date = datetime.datetime.now().isoformat(" ") + "+00"
        person_list.append(
            (i + 1, g, first_name, last_name, cpf, birthday, email, phone, cell, address, district, city, uf, cep, date, date))
    for l in person_list:
        s = str(l[0]) + "," + str(l[1]) + "," + str(l[2]) + \
            "," + str(l[3]) + "," + str(l[4]) + "," + str(l[5]) + \
            "," + str(l[6]) + "," + str(l[7]) + "," + str(l[8]) + \
            "," + str(l[9]) + "," + str(l[10]) + "," + str(l[11]) + \
            "," + str(l[12]) + "," + str(l[13]) + "," + str(l[14]) + \
            "," + str(l[15]) + "\n"
        f.write(str(s))


''' file name : sobel.py

Description : This sample shows how to find derivatives of an image

This is Python version of this tutorial : http://opencv.itseez.com/doc/tutorials/imgproc/imgtrans/sobel_derivatives/sobel_derivatives.html#sobel-derivatives

Level : Beginner

Benefits : Learn to use Sobel and Scharr derivatives

Usage : python sobel.py

Written by : Abid K. (abidrahman2@gmail.com) , Visit opencvpython.blogspot.com for more tutorials '''

import cv2
import numpy as np

scale = 1
delta = 0
ddepth = cv2.CV_16S

img = cv2.imread('messi5.jpg')
img = cv2.GaussianBlur(img,(3,3),0)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

# Gradient-X
grad_x = cv2.Sobel(gray,ddepth,1,0,ksize = 3, scale = scale, delta = delta,borderType = cv2.BORDER_DEFAULT)
#grad_x = cv2.Scharr(gray,ddepth,1,0)

# Gradient-Y
grad_y = cv2.Sobel(gray,ddepth,0,1,ksize = 3, scale = scale, delta = delta, borderType = cv2.BORDER_DEFAULT)
#grad_y = cv2.Scharr(gray,ddepth,0,1)

abs_grad_x = cv2.convertScaleAbs(grad_x)   # converting back to uint8
abs_grad_y = cv2.convertScaleAbs(grad_y)

dst = cv2.addWeighted(abs_grad_x,0.5,abs_grad_y,0.5,0)
#dst = cv2.add(abs_grad_x,abs_grad_y)

cv2.imshow('dst',dst)
cv2.waitKey(0)
cv2.destroyAllWindows()

# To see the results, visit : http://opencvpython.blogspot.com/2012/06/image-derivatives-sobel-and-scharr.html


from pyspark.sql import SparkSession
from pyspark.sql.functions import col, round, expr

# Cria uma sessão Spark
spark = SparkSession.builder.getOrCreate()

# Define a lista de transações
transacoes = [
    {'transacao_id': 1, 'total_bruto': 3000, 'desconto_percentual': 6.99},
    {'transacao_id': 2, 'total_bruto': 57989, 'desconto_percentual': 1.45},
    {'transacao_id': 4, 'total_bruto': 1, 'desconto_percentual': None},
    {'transacao_id': 5, 'total_bruto': 34, 'desconto_percentual': 0.0}
]

# Cria um dataframe do Spark a partir da lista de transações
df = spark.createDataFrame(transacoes)

# Calcula o total líquido da empresa
total_liquido = df.select(
    round((col('total_bruto') - expr('IFNULL(desconto_percentual, 0) * total_bruto / 100')), 2).alias('total_liquido')
).agg({'total_liquido': 'sum'}).collect()[0][0]

# Formata o total líquido com duas casas decimais
total_liquido_formatado = '{:.2f}'.format(total_liquido)

# Exibe o total líquido da empresa
print("Total líquido da empresa:", total_liquido_formatado)


from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from django.shortcuts import redirect
from django.template.response import TemplateResponse
from django.urls import reverse

from core.utility import go_to_previous_url_or_search_register
from demand.key_model_forms import (ChargeForm, ChargeFormForExtraSales,
                                    DepositForm, PaymentForm)
from demand.key_models import Charge, Deposit, Payment
from demand.sales_model_forms import (ExtraSalesRealDayCameOutForm,
                                      FirstCenterRegisterForm,
                                      RealDayCameOutForm, SpecialRegisterForm)
from demand.sales_models import ExtraSales, Order, Register

@login_required
def car_number_modal(request):
    car_number = request.GET.get("car_number", None)
    registers = Register.objects.filter(car_number=car_number)
    return TemplateResponse(
        request, "demand/modals/car_number_modal.html",
        context={'registers': registers})

@login_required
def came_out_modal(request, pk):
    order = Order.objects.get(pk=pk)
    register = order.register
    # prefix is need because of JS
    payment_prefix = "payment-0"
    if request.method == "GET":
        real_day_came_out_form = RealDayCameOutForm(instance=register)
        special_register_form = SpecialRegisterForm(instance=register)
        payment_form = PaymentForm(
            instance=order.payment, prefix=payment_prefix)
        return TemplateResponse(
            request, "demand/modals/came_out_modal.html",
            context={"order": order,
                     "real_day_came_out_form": real_day_came_out_form,
                     "special_register_form": special_register_form,
                     "payment_form": payment_form})
    else:
        real_day_came_out_form = RealDayCameOutForm(
            request.POST, instance=register)
        special_register_form = SpecialRegisterForm(
            request.POST, instance=register)
        real_day_came_out_form.save()
        special_register_form.save()
        payment_form = PaymentForm(
            request.POST, instance=order.payment, prefix=payment_prefix)
        if payment_form.is_valid():
            payment = payment_form.save()
            if payment.is_default():
                order.payment = None
                payment.delete()
            else:
                order.payment = payment
            order.save()
        else:
            return JsonResponse({"error_message": "Payment Formset is not valid"})
        return go_to_previous_url_or_search_register(request, register)

@login_required
def charge_modal(request, pk):
    order = Order.objects.get(pk=pk)
    if request.method == "GET":
        first_center_register_form = FirstCenterRegisterForm(
            instance=order.register)
        charge_form = ChargeForm(instance=order.charge, order=order)
        return TemplateResponse(request, "demand/modals/charge_modal.html",
                                context={"order": order,
                                         "first_center_register_form": first_center_register_form,
                                         "charge_form": charge_form})
    else:
        first_center_register_form = FirstCenterRegisterForm(
            request.POST, instance=order.register)
        first_center_register_form.save()
        charge_form = ChargeForm(
            request.POST, instance=order.charge, order=order)
        if charge_form.is_valid():
            charge = charge_form.save()
            order.charge = charge
            order.save()
        else:
            return JsonResponse({"error_message": "Charge Form is not valid"})
        return go_to_previous_url_or_search_register(request, order.register)

@login_required
def deposit_modal(request, pk):
    order = Order.objects.get(pk=pk)
    if request.method == "GET":
        deposit_form = DepositForm(instance=order.deposit, order=order)
        return TemplateResponse(request, "demand/modals/deposit_modal.html",
                                context={"order": order, "deposit_form": deposit_form})
    else:
        deposit_form = DepositForm(
            request.POST, instance=order.deposit, order=order)
        if deposit_form.is_valid():
            deposit = deposit_form.save()
            order.deposit = deposit
            order.save()
        else:
            return JsonResponse({"error_message": "Deposit Form is not valid"})
        return go_to_previous_url_or_search_register(request, order.register)

@login_required
def delete_payment(request, pk):
    payment = Payment.objects.get(pk=pk)
    order = payment.order
    payment.delete()
    return go_to_previous_url_or_search_register(request, order.register)

@login_required
def delete_charge(request, pk):
    charge = Charge.objects.get(pk=pk)
    order = charge.order
    charge.delete()
    return go_to_previous_url_or_search_register(request, order.register)

@login_required
def delete_deposit(request, pk):
    deposit = Deposit.objects.get(pk=pk)
    order = deposit.order
    deposit.delete()
    return go_to_previous_url_or_search_register(request, order.register)

@login_required
def extra_sales_came_out_modal(request, pk):
    extra_sales = ExtraSales.objects.get(pk=pk)
    payment_prefix = "payment-0"
    if request.method == "GET":
        real_day_came_out_form = ExtraSalesRealDayCameOutForm(
            instance=extra_sales)
        payment_form = PaymentForm(
            instance=extra_sales.payment, prefix=payment_prefix)
        return TemplateResponse(
            request, "demand/modals/extra_sales_came_out_modal.html",
            context={"extra_sales": extra_sales,
                     "real_day_came_out_form": real_day_came_out_form,
                     "payment_form": payment_form})
    else:
        real_day_came_out_form = ExtraSalesRealDayCameOutForm(
            request.POST, instance=extra_sales)
        real_day_came_out_form.save()
        payment_form = PaymentForm(
            request.POST, instance=extra_sales.payment, prefix=payment_prefix)
        if payment_form.is_valid():
            payment = payment_form.save()
            if payment.is_default():
                extra_sales.payment = None
                payment.delete()
            else:
                extra_sales.payment = payment
            extra_sales.save()
        else:
            return JsonResponse({"error_message": "Payment Formset is not valid"})
        return redirect(reverse("demand:search_extra_sales"))

@login_required
def extra_sales_charge_modal(request, pk):
    extra_sales = ExtraSales.objects.get(pk=pk)
    if request.method == "GET":
        charge_form = ChargeFormForExtraSales(instance=extra_sales.charge)
        return TemplateResponse(request, "demand/modals/extra_sales_charge_modal.html",
                                context={"extra_sales": extra_sales,
                                         "charge_form": charge_form})
    else:
        charge_form = ChargeFormForExtraSales(
            request.POST, instance=extra_sales.charge)
        if charge_form.is_valid():
            charge = charge_form.save()
            extra_sales.charge = charge
            extra_sales.save()
        else:
            return JsonResponse({"error_message": "Charge Form is not valid"})
    return redirect(reverse("demand:search_extra_sales"))

@login_required
def extra_sales_delete_payment(request, pk):
    payment = Payment.objects.get(pk=pk)
    payment.delete()
    return redirect(reverse("demand:search_extra_sales"))

@login_required
def extra_sales_delete_charge(request, pk):
    charge = Charge.objects.get(pk=pk)
    charge.delete()
    return redirect(reverse("demand:search_extra_sales"))


# -*- coding: utf-8 -*-
"""
Created on Tue May 11 10:00:46 2021

@author: jvano

"""

#%% import and explore data

import pandas as pd
import numpy as np
import geopandas as gpd
import glob, os

os.chdir('C:/...')
gemeenten = []
for shpfile in glob.glob("*.shp"):
    filename = os.path.splitext(shpfile)[0]
    gemeenten.append(filename)

#%% load Files & edit Material DataBase
M_total_NL = pd.DataFrame()
M_total_gem = pd.DataFrame()

building_types = ['vrijstaand', 'serieel', 'appartement', 'winkel', 'kantoor', 'industrie', 'overheid', 'overig']

# Building stock data per municipality (355 in total)
os.chdir('C:/...')

all_years = np.arange(1900,2019,1)
I_gebouwen = pd.DataFrame(0, index = all_years, columns = building_types)

#%%
gemeenten = []
for shpfile in glob.glob("*.shp"):
    filename = os.path.splitext(shpfile)[0]
    gemeenten.append(filename)

# Reclassified_II is "overig" aan toegevoegd, voor ongeclassificeerde BAG elementen (vaak schuurtjes etc.), is gemiddelde van alle andere gebouwen genomen
MI_db = pd.read_excel(r'C:/...MI_historic.xlsx')

MI_db['Type gebouw'] = MI_db['Type gebouw'].fillna(method='ffill')
tuples = list(zip(MI_db['Type gebouw'], MI_db['Cohort']))

MI_db.index = tuples
MI_db.drop(['Type gebouw', 'Cohort'], axis = 1, inplace = True)

material_columns = list(MI_db.columns)

material_index = list(MI_db.index)

#%% Loop door gemeenten Shapefiles

for file in glob.glob("*.shp"):

    BAGx = gpd.read_file(file)

    BAG = BAGx[['bouwjaar', 'status', 'gebruiks_1', 'geometry', 'GO','shape_Area', 'gebr_woonf' ]].copy()

    #2018 is start jaar, dus bouw na 2018 uit bestand verwijderen
    built_after_2018 = BAG[ BAG['bouwjaar'] > 2018].index
    BAG.drop(built_after_2018, inplace = True)

    filename = os.path.splitext(file)[0]
    print(filename)
    woningtypering = gpd.read_file(r'C:/.../'+filename+'.shp')

    # appartement : woning waaraan meerdere verblijfsobjecten zijn gerelateerd, ongeacht het gebruiksdoel van deze verblijfsobjecten, minimaal één 'woonfunctie'
    # vrijstaande woning: woning zonder andere verbonden verblijfsobjecten
    # tussen- of geschakelde woning: woning die met meerdere panden met een verblijsobject is verbonden, ook geschakelde woningen.
    # hoekwoning: eerste of laatste woning in een serie panden.
    # twee-onder-een-kap: deze woning is verbonden met een enkel pand met een verblijsobject en dit pand is alleen met het eerstgenoemde pand verbonden.
    woningtypering = woningtypering[['Woningtype', 'geometry']]

#%% Spatial join woningtypering & BAG

    BAG_1 = gpd.sjoin(BAG, woningtypering, how = 'left', op = 'intersects')
    BAG_1 = BAG_1[~BAG_1.index.duplicated(keep='first')]

#%% Classificatie gebouwen MB

# Classificatie (op basis van GO)

    # grofweg 1/4 van de data is ongeclassificeerd: het overgrootte schuurtjes/kleine huisjes in tuinen.

    BAG_1['Class'] = np.NaN

    BAG_1['Class'].loc[((BAG_1.gebruiks_1 == 'kantoorfunctie') |\
                        (BAG_1.gebruiks_1 == 'bijeenkomstfunctie') |\
                            (BAG_1.gebruiks_1 == 'logiesfunctie'))] = 'kantoor' # voor m2 kantoor & bijeenkomst is < 5.000 gekozen, dit is het miden van de range in m2 voor de categorie middelgroot

    BAG_1['Class'].loc[((BAG_1.gebruiks_1 == 'onderwijsfunctie') | \
                        (BAG_1.gebruiks_1 == 'gezondheidszorgfunctie') | \
                            (BAG_1.gebruiks_1 == 'celfunctie') | \
                                (BAG_1.gebruiks_1 == 'overige gebruiksfunctie') | \
                                        (BAG_1.gebruiks_1 == 'sportfunctie'))] = 'overheid'

    BAG_1['Class'].loc[(BAG_1['gebruiks_1']).isna()] = 'overig'

    BAG_1['Class'].loc[(BAG_1.gebruiks_1 == 'winkelfunctie')] = 'winkel'

    BAG_1['Class'].loc[(BAG_1.gebruiks_1 == 'industriefunctie')] = 'industrie' # aannemened industrie is gelijk aan bedrijfshal / distributiecentrum

    BAG_1['Class'].loc[((BAG_1.Woningtype == 'vrijstaande woning') | (BAG_1.Woningtype == 'twee-onder-een-kap' ))] = "vrijstaand"

    BAG_1['Class'].loc[((BAG_1.Woningtype == 'tussenwoning/geschakeld') | (BAG_1.Woningtype == 'hoekwoning' ))] = "serieel"

    BAG_1['Class'].loc[(BAG_1.Woningtype == 'appartement')] = "appartement" # tot 6 lagen, 6*3 meter per etage = 18m

#%% Sort per building year

    for building_type in building_types:
        summed_buildingtype = BAG_1['GO'].loc[(BAG_1.bouwjaar <= 1900) & (BAG_1.Class == building_type)].sum()
        I_gebouwen.at[1900, building_type] = I_gebouwen.at[1900, building_type] + summed_buildingtype
        for year in all_years:
            if year != 1900:
                summed_buildingtype = BAG_1['GO'].loc[(BAG_1.bouwjaar == year) & (BAG_1.Class == building_type)].sum()
                I_gebouwen.at[year, building_type] = I_gebouwen.at[year, building_type] + summed_buildingtype

#%% Add building material intensities

    BAG_copy = BAG_1.copy()
#    floor_height = 3 # MI in kg/m2, Materiaal(kg/m2) = opp(m)*hoogte(m)/floor_height(mi)*MI (kg/m2)

    cohort_names = ['<1945', '1945-1970', '1970-2000', '>2000']
    cohort_ranges = [[0,1945], [1945,1970], [1970, 2000],[2000,3000]]

    #BAG_copy['floor_area'] = BAG_copy.loc[:,'Opp_BVO']

    for material in material_columns:
        BAG_copy[material]=np.nan

    for index,cohort in enumerate(cohort_ranges): #  maakt 2 kolommen, index 1,2,3,4,... + waarde (cohort)
        for building_type in building_types:
            BAG_class_cohort_filter = ((BAG_copy['bouwjaar'] >= cohort[0]) & (BAG_copy['bouwjaar'] < cohort[1]) & (BAG_copy['Class'] == building_type))
            BAG_filtered = BAG_copy[BAG_class_cohort_filter]
            for material in material_columns:
                index_name = (building_type, cohort_names[index])
                material_intensity = MI_db.at[index_name, material]
                BAG_filtered[material] = BAG_filtered.loc[:,'GO']*material_intensity
                BAG_copy[BAG_class_cohort_filter] = BAG_filtered

    BAG_copy.to_file("C:\.../"+filename+"_voorraden.shp")

#%% Calculate total materialintensities in municipality for each class

    M_total_class = pd.DataFrame(columns = material_columns, index = material_index)

    for material in material_columns:
        value = BAG_copy[material].sum()
        M_total_gem.at[os.path.splitext(filename)[0], material] = value
        for building_type in building_types:
            for index,cohort in enumerate(cohort_ranges): #  maakt 2 kolommen, index 1,2,3,4,... + waarde (cohort)
                BAG_class_filter = ((BAG_copy['bouwjaar'] >= cohort[0]) & (BAG_copy['bouwjaar'] < cohort[1]) & (BAG_copy['Class'] == building_type))
                BAG_filtered = BAG_copy[BAG_class_filter]
                value1 = BAG_filtered[material].sum()
                index_name = (building_type, cohort_names[index])
                M_total_class.at[index_name, material]=value1

    M_total_NL = M_total_NL.add(M_total_class, fill_value=0)

    print("Einde loop: "+filename)

M_total_gem.to_excel('C:/.../M_totalen_gemeentes.xlsx')
M_total_NL.to_excel('C:/.../M_totalen_MB.xlsx')

I_gebouwen.to_excel('C:/.../Inflow_gebouwen_hist.xlsx')


"""
Profile a data file
"""
import logging
from collections import OrderedDict

from django.conf import settings

from opendp_apps.model_helpers.basic_err_check import BasicErrCheck
from opendp_apps.profiler.csv_reader import CsvReader
from opendp_apps.profiler.static_vals import \
    (KEY_SAVE_ROW_COUNT,
     VAR_TYPE_BOOLEAN,
     VAR_TYPE_CATEGORICAL,
     VAR_TYPE_FLOAT,
     VAR_TYPE_INTEGER,
     VAR_TYPE_NUMERICAL)

logger = logging.getLogger(settings.DEFAULT_LOGGER)

class VariableInfoHandler(BasicErrCheck):

    def __init__(self, dataframe, **kwargs):
        """
        Given a dataframe, create a variable profile dictionary
        :param df: Dataframe
        """
        self.df = dataframe
        self.num_variables = None
        self.data_profile = None

        self.save_num_rows = kwargs.get(KEY_SAVE_ROW_COUNT, True)

    def run_profile_process(self):
        """
        Process the dataframe into a complete data profile dictionary
        :return:
        {
            'dataset': {
                'rowCount': 1500,
                'variableCount': 132,
                'variableOrder': [(0, 'some_column'), ....]
            },
            'variables': {
                'some_column': {
                    'label': '',
                    'name': 'some_column',
                    'type': 'Categorical'
                }
            }
        }
        """
        profile_dict = {'dataset': {}}
        num_rows, self.num_variables = self.df.shape
        if self.save_num_rows is True:
            profile_dict['dataset']['rowCount'] = int(num_rows)
        else:
            profile_dict['dataset']['rowCount'] = None

        profile_dict['dataset']['variableCount'] = int(self.num_variables)
        variable_order = [(i, x) for i, x in enumerate(self.df.columns)]
        profile_dict['dataset']['variableOrder'] = variable_order
        profile_dict['variables'] = {}

        sort_order = -1
        for col_name in self.df.columns:
            sort_order += 1
            logger.info(f'profiling column {sort_order}: {col_name}')
            column = self.df[col_name]
            column_info = OrderedDict({
                "name": col_name,
                "sort_order": sort_order,
                "label": ""
            })
            # category_limit = 5
            # Use type checking to filter out numpy Nan
            # Comment out categories for now
            #unique_vals = column.unique()

            if column.unique().shape[0] == 2:
                column_info['categories'] = []
                column_info['type'] = VAR_TYPE_BOOLEAN
            elif str(column.dtypes) == 'object':
                column_info['categories'] = []  # list([x for x in column.unique() if type(x) == str])[:category_limit]
                column_info['type'] = VAR_TYPE_CATEGORICAL
            elif hasattr(column.dtypes, 'categories'):
                column_info[
                    'categories'] = []  # list([x for x in column.dtypes.categories if type(x) == str])[:category_limit]
                column_info['type'] = VAR_TYPE_CATEGORICAL
            elif 'int' in str(column.dtype):
                column_info['type'] = VAR_TYPE_INTEGER
            elif 'float' in str(column.dtype):
                column_info['type'] = VAR_TYPE_FLOAT
            elif str(column.dtypes) != 'object':
                column_info['type'] = VAR_TYPE_NUMERICAL
            profile_dict['variables'][col_name] = column_info

        self.data_profile = profile_dict
        return profile_dict

if __name__ == '__main__':
    import os
    from pprint import pprint
    from server.opendp_project.settings.base import BASE_DIR

    # fname = 'Fatigue_data.tab'
    fname = 'teacher_climate_survey_lwd.csv'
    csv_reader = CsvReader(os.path.join(BASE_DIR, 'test_data', fname))
    df = csv_reader.read()
    v = VariableInfoHandler(df)
    profile_info = v.run_profile_process()
    pprint(profile_info)


from pathlib import Path

from mpi4py import MPI
from petsc4py import PETSc

import cffi
import numba
import numba.core.typing.cffi_utils as cffi_support
import numpy as np

import ufl
from basix.ufl import element
from dolfinx import default_real_type, geometry
from dolfinx.cpp.fem import (Form_complex64, Form_complex128, Form_float32,
                             Form_float64)
from dolfinx.fem import (Form, Function, IntegralType, dirichletbc, form,
                         functionspace, locate_dofs_topological)
from dolfinx.fem.petsc import (apply_lifting, assemble_matrix, assemble_vector,
                               set_bc)
from dolfinx.io import XDMFFile
from dolfinx.jit import ffcx_jit
from dolfinx.mesh import locate_entities_boundary, meshtags

if default_real_type == np.float32:
    print("float32 not yet supported for this demo.")
    exit(0)

infile = XDMFFile(MPI.COMM_WORLD, Path(Path(__file__).parent, "data",
                  "cooks_tri_mesh.xdmf"), "r", encoding=XDMFFile.Encoding.ASCII)
msh = infile.read_mesh(name="Grid")
infile.close()

# Stress (Se) and displacement (Ue) elements
Se = element("DG", msh.basix_cell(), 1, shape=(2, 2), symmetry=True)
Ue = element("Lagrange", msh.basix_cell(), 2, shape=(2,))

S = functionspace(msh, Se)
U = functionspace(msh, Ue)

# Get local dofmap sizes for later local tensor tabulations
Ssize = S.element.space_dimension
Usize = U.element.space_dimension

sigma, tau = ufl.TrialFunction(S), ufl.TestFunction(S)
u, v = ufl.TrialFunction(U), ufl.TestFunction(U)

# Locate all facets at the free end and assign them value 1. Sort the
# facet indices (requirement for constructing MeshTags)
free_end_facets = np.sort(locate_entities_boundary(msh, 1, lambda x: np.isclose(x[0], 48.0)))
mt = meshtags(msh, 1, free_end_facets, 1)

ds = ufl.Measure("ds", subdomain_data=mt)

# Homogeneous boundary condition in displacement
u_bc = Function(U)
u_bc.x.array[:] = 0.0

# Displacement BC is applied to the left side
left_facets = locate_entities_boundary(msh, 1, lambda x: np.isclose(x[0], 0.0))
bdofs = locate_dofs_topological(U, 1, left_facets)
bc = dirichletbc(u_bc, bdofs)

# Elastic stiffness tensor and Poisson ratio
E, nu = 1.0, 1.0 / 3.0

def sigma_u(u):
    """Constitutive relation for stress-strain. Assuming plane-stress in XY"""
    eps = 0.5 * (ufl.grad(u) + ufl.grad(u).T)
    sigma = E / (1. - nu ** 2) * ((1. - nu) * eps + nu * ufl.Identity(2) * ufl.tr(eps))
    return sigma

a00 = ufl.inner(sigma, tau) * ufl.dx
a10 = - ufl.inner(sigma, ufl.grad(v)) * ufl.dx
a01 = - ufl.inner(sigma_u(u), tau) * ufl.dx

f = ufl.as_vector([0.0, 1.0 / 16])
b1 = form(- ufl.inner(f, v) * ds(1))

# JIT compile individual blocks tabulation kernels
nptype, ffcxtype = None, None
if PETSc.ScalarType == np.float32:  # type: ignore
    nptype = "float32"
    ffcxtype = "float"
elif PETSc.ScalarType == np.float64:  # type: ignore
    nptype = "float64"
    ffcxtype = "double"
elif PETSc.ScalarType == np.complex64:  # type: ignore
    nptype = "complex64"
    ffcxtype = "float _Complex"
elif PETSc.ScalarType == np.complex128:  # type: ignore
    nptype = "complex128"
    ffcxtype = "double _Complex"
else:
    raise RuntimeError(f"Unsupported scalar type {PETSc.ScalarType}.")  # type: ignore

ufcx_form00, _, _ = ffcx_jit(msh.comm, a00, form_compiler_options={"scalar_type": ffcxtype})
kernel00 = getattr(ufcx_form00.form_integrals[0], f"tabulate_tensor_{nptype}")
ufcx_form01, _, _ = ffcx_jit(msh.comm, a01, form_compiler_options={"scalar_type": ffcxtype})
kernel01 = getattr(ufcx_form01.form_integrals[0], f"tabulate_tensor_{nptype}")
ufcx_form10, _, _ = ffcx_jit(msh.comm, a10, form_compiler_options={"scalar_type": ffcxtype})
kernel10 = getattr(ufcx_form10.form_integrals[0], f"tabulate_tensor_{nptype}")

ffi = cffi.FFI()
cffi_support.register_type(ffi.typeof('double _Complex'), numba.types.complex128)
c_signature = numba.types.void(
    numba.types.CPointer(numba.typeof(PETSc.ScalarType())),  # type: ignore
    numba.types.CPointer(numba.typeof(PETSc.ScalarType())),  # type: ignore
    numba.types.CPointer(numba.typeof(PETSc.ScalarType())),  # type: ignore
    numba.types.CPointer(numba.typeof(PETSc.RealType())),  # type: ignore
    numba.types.CPointer(numba.types.int32),
    numba.types.CPointer(numba.types.uint8))

@numba.cfunc(c_signature, nopython=True)
def tabulate_condensed_tensor_A(A_, w_, c_, coords_, entity_local_index, permutation=ffi.NULL):
    # Prepare target condensed local element tensor
    A = numba.carray(A_, (Usize, Usize), dtype=PETSc.ScalarType)

    # Tabulate all sub blocks locally
    A00 = np.zeros((Ssize, Ssize), dtype=PETSc.ScalarType)
    kernel00(ffi.from_buffer(A00), w_, c_, coords_, entity_local_index, permutation)

    A01 = np.zeros((Ssize, Usize), dtype=PETSc.ScalarType)
    kernel01(ffi.from_buffer(A01), w_, c_, coords_, entity_local_index, permutation)

    A10 = np.zeros((Usize, Ssize), dtype=PETSc.ScalarType)
    kernel10(ffi.from_buffer(A10), w_, c_, coords_, entity_local_index, permutation)

    # A = - A10 * A00^{-1} * A01
    A[:, :] = - A10 @ np.linalg.solve(A00, A01)

# Prepare a Form with a condensed tabulation kernel
formtype = None
if PETSc.ScalarType == np.float32:  # type: ignore
    formtype = Form_float32
elif PETSc.ScalarType == np.float64:  # type: ignore
    formtype = Form_float64
elif PETSc.ScalarType == np.complex64:  # type: ignore
    formtype = Form_complex64
elif PETSc.ScalarType == np.complex128:  # type: ignore
    formtype = Form_complex128
else:
    raise RuntimeError(f"Unsupported PETSc ScalarType '{PETSc.ScalarType}'.")  # type: ignore

cells = np.arange(msh.topology.index_map(msh.topology.dim).size_local)
integrals = {IntegralType.cell: [(-1, tabulate_condensed_tensor_A.address, cells)]}
a_cond = Form(formtype([U._cpp_object, U._cpp_object], integrals, [], [], False, None))

A_cond = assemble_matrix(a_cond, bcs=[bc])
A_cond.assemble()

b = assemble_vector(b1)
apply_lifting(b, [a_cond], bcs=[[bc]])
b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)  # type: ignore
set_bc(b, [bc])

uc = Function(U)
solver = PETSc.KSP().create(A_cond.getComm())  # type: ignore
solver.setOperators(A_cond)
solver.solve(b, uc.vector)

# Pure displacement based formulation
a = form(- ufl.inner(sigma_u(u), ufl.grad(v)) * ufl.dx)
A = assemble_matrix(a, bcs=[bc])
A.assemble()

# Create bounding box for function evaluation
bb_tree = geometry.bb_tree(msh, 2)

# Check against standard table value
p = np.array([[48.0, 52.0, 0.0]], dtype=np.float64)
cell_candidates = geometry.compute_collisions_points(bb_tree, p)
cells = geometry.compute_colliding_cells(msh, cell_candidates, p).array

uc.x.scatter_forward()
if len(cells) > 0:
    value = uc.eval(p, cells[0])  # type: ignore
    print(value[1])
    assert np.isclose(value[1], 23.95, rtol=1.e-2)

# Check the equality of displacement based and mixed condensed global
# matrices, i.e. check that condensation is exact
assert np.isclose((A - A_cond).norm(), 0.0)