# -*- coding: utf-8 -*-
# +
import requests
import feast
import pandas as pd
from feast import Entity, Feature, FeatureView, FileSource, ValueType
import torch
import pymysql
from datetime import date
from sqlalchemy import create_engine
import numpy as np
pymysql.install_as_MySQLdb()
import MySQLdb
import math

import warnings
warnings.filterwarnings("ignore")

# -

def get_entities(path_,from_time_):
    parquet_ = pd.read_parquet(path_, engine='pyarrow')
    orders = parquet_[['ticket_id','event_timestamp']]

    ### 뽑을 rows의 entity == key 라고 볼 수 있음
    new_orders = orders[orders['event_timestamp']>= from_time_]
    new_orders.drop_duplicates(ignore_index=True, inplace=True)
    return new_orders

def get_df_from_online(feast_path_, entity_df_):
    fs_ = feast.FeatureStore(repo_path=feast_path_)

    online_ = fs_.get_online_features(
                entity_rows=[{"ticket_id": i} for i in entity_df_['ticket_id']],
                features=[
                    "dr_lauren_stat:time",
                    "dr_lauren_stat:weekday",
                    "dr_lauren_stat:weekend",
                    "dr_lauren_stat:instlo_1",
                    "dr_lauren_stat:instlo_2",
                    "dr_lauren_stat:inst_code",
                    "dr_lauren_stat:sysname_lo",
                    "dr_lauren_stat:sysname_eq",
                    "dr_lauren_stat:ntt_label",
            ],
    )

    df = pd.DataFrame.from_dict(online_.to_dict())

    return df[['time', 'weekday', 'weekend', 'instlo_1', 'instlo_2', 'inst_code', 'sysname_lo', 'sysname_eq','ntt_label']]

# ### request

# +
def get_from_bentoml(online_df_):
    response = requests.post("http://127.0.0.1:5000/predict", data=online_df_.iloc[:,:-1].to_json())
    infer_ = response.text
    predict_ = infer_[1:-1].split(', ')
    return predict_

def get_infer_df_(entity_df_, online_df_):
    infer_df = entity_df_.reset_index(drop=True)
    infer_df['label'] = online_df_['ntt_label']
    infer_df['predict'] = get_from_bentoml(online_df_)
    return infer_df

# +
### data create, insert, update
def get_commit_sql(conn_, sql_):
    with conn_.cursor() as cursor:
        cursor.execute(sql_)
        result = cursor.fetchall()
        conn.commit()

### data read
def get_data_sql(conn_, sql_):
    with conn_.cursor() as cursor:
        cursor.execute(sql_)
        result = cursor.fetchall()
        return result

# -

# ### create table

# +
# sql = '''
#     CREATE TABLE infer_df (
#         ticket_id int(11) NOT NULL AUTO_INCREMENT PRIMARY KEY,
#         event_timestamp varchar(255) NOT NULL,
#         label int(11) NOT NULL,
#         predict int(11) NOT NULL
#     ) ENGINE=InnoDB DEFAULT CHARSET=utf8
# '''
# get_commit_sql(conn, sql)
# -

def get_columns_(df_):
    columns_str = ', '.join([i for i in infer_df.columns])
    return columns_str

def get_values_(df_):
    value_ = []
    for row_ in df_.values:
        value_.append("("+", ".join(["'"+str(i)+"'" for i in row_]) + ")")

    values_str = ', '.join(value_)
    return values_str

def pred_into_db(conn_, infer_df_):
    columns_ = get_columns_(infer_df_)

    values_ = get_values_(infer_df_)

    sql = f"INSERT INTO infer_df ({columns_}) VALUES {values_} ;"
    get_commit_sql(conn_, sql)

def main():
    path = 'workspace/ML_Ops/feast_fea_/data/ppr_data_.parquet'
    from_time = '2021-07-01'

    entity_df = get_entities(path, from_time)

    feast_path = './'
    online_df = get_df_from_online(feast_path, entity_df)

    infer_df = get_infer_df_(entity_df, online_df)

    conn = pymysql.connect(host='localhost', port=3306, user='root', bind_address='127.0.0.1', passwd='', db='infer_db', charset='utf8')

    pred_into_db(conn,infer_df)

main()


#!/usr/bin/env python
# coding: utf-8

# # Creación de Dataset de Municipios de Colombia

# #### Tarea #1: Obtener la información para un municipio

# In[1]:

# Importar librerías principales

import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
import numpy as np
import pickle

# In[3]:

# Cargar el contenido de la pagina web (Recurso: Wikipedia)
w = requests.get("https://es.wikipedia.org/wiki/Bogot%C3%A1")

# Convertir a un objeto "BeautifulSoup"
bsobj = bs(w.content)

# Imprimir (Revisar estructura del código html)
print(bsobj.prettify())

# In[4]:

# Revisar la información de la "tabla de datos" usando una clase específica

info_caja = bsobj.find(class_="infobox geography vcard")
info_filas = info_caja.find_all("tr")
for fila in info_filas:
    print(fila.prettify())

# In[5]:

# Función para obtener los valores dados diferentes casos, además, se reemplazan caracteres extraños (unicode).
def obtener_valores(fila_data):
    if fila_data.find("div") and fila_data.find("br"):
        return [texto for texto in fila_data.stripped_strings]
    else:
        return fila_data.get_text(" ").replace("\xa0", " ").replace("\u200b", "").strip()

# Crear función para eliminar etiquetas [1] (span) solo en el caso de que no tengan información relevante
def limpiar_etiquetas(bs_objeto):
    if bs_objeto.find_all("sup", {'class': 'reference separada'}):
        for etiqueta in bs_objeto.find_all("sup"):
            etiqueta.decompose()
    pass

limpiar_etiquetas(bsobj)

municipios_info = {}

# Crear una lista de títulos para tomar el indice desde el cuál empezar a obtener los datos
titulos = [fila.get('title') for fila in info_caja.find_all("a") if fila.get('title') is not None]
contador = 0
for indice, fila in enumerate(info_filas):
    if indice == 0 and fila.find("th"):
        municipios_info['municipio'] = fila.find("th").get_text(" ", strip=True)
    # En caso de que el tag "th" no contenga información o no exista, tomar el nombre del municipio del encabezado (tag "h1")
    elif indice == 0 and not fila.find("th"):
        municipios_info['municipio'] = bsobj.find("h1").get_text(" ", strip=True)
    elif indice < titulos.index('Coordenadas geográficas'): # Imágenes y otros datos irrelevantes no serán tenidos en cuenta
        continue
    else: # Las filas de tabla (tr) que no contengan elementos "th" & "td" serán evitadas
        try:
            # Se obtiene las llaves y se reemplazan caracteres extraños.
            # Se anexa un número a aquellas llaves que posean el mismo nombre "* Total" para evitar errores en los datos
            if fila.find("th").get_text(" ", strip=True) == "• Total":
                llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").strip() + f"_{contador}"
                valores = obtener_valores(fila.find("td"))
                municipios_info[llaves] = valores
                contador += 1
            else:
                llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").replace("\u200b", "").strip()
                valores = obtener_valores(fila.find("td"))
                municipios_info[llaves] = valores
        except AttributeError as e:
            pass

municipios_info

# #### Tarea #2: Obtener la información para todos los municipios de Colombia

# In[24]:

# Cargar el contenido de la página (Lista de municipios de Colombia por población)
url = requests.get("https://es.wikipedia.org/wiki/Anexo:Municipios_de_Colombia_por_poblaci%C3%B3n")

# Convertir a un objeto BS
bso = bs(url.content)

# Imprimir HTML
print(bso.prettify())

# In[25]:

# Extraer las etiquetas "href" para la clase especificada
municipios = bso.select(".wikitable.sortable td")
rutas = [m.a['href'] for m in municipios if m.a and m.get('align') is None]
print(rutas)

# In[26]:

# Extraer las rutas de cada municipio y formar junto al dominio, los enlaces completos.

dominio = "https://es.wikipedia.org"
enlaces = []
for ruta in rutas:
    enlaces.append(dominio + ruta)

# In[27]:

# En algunos casos, las rutas estaban conectadas a municipalidades de otros paises. Se cambian por los enlaces correctos
for n, i in enumerate(enlaces):
    if i == 'https://es.wikipedia.org/wiki/Aquitania':
        enlaces[n] = 'https://es.wikipedia.org/wiki/Aquitania_(Boyac%C3%A1)'
    elif i == 'https://es.wikipedia.org/wiki/Saman%C3%A1':
        enlaces[n] = 'https://es.wikipedia.org/wiki/Saman%C3%A1_(Caldas)'
    elif i == 'https://es.wikipedia.org/wiki/Roncesvalles':
        enlaces[n] = 'https://es.wikipedia.org/wiki/Roncesvalles_(Tolima)'

# In[28]:

# Crear función para obtener los valores dependiendo de la etiqueta, además, se eliminan caracteres extraños.
def obtener_valores(fila_data):
    if fila_data.find("div") and fila_data.find("br"):
        return [texto for texto in fila_data.stripped_strings]
    else:
        return fila_data.get_text(" ").replace("\xa0", " ").replace("\u200b", "").strip()

# Crear función para eliminar etiquetas [1] (span) solo en el caso de que no tengan información relevante
def limpiar_etiquetas(bs_objeto):
    if bs_objeto.find_all("sup", {'class': 'reference separada'}):
        for etiqueta in bs_objeto.find_all("sup"):
            etiqueta.decompose()
    pass

# Crear función para obtener la información de un municipio dada su dirección url.
def obtener_info(url):

    w = requests.get(url)

    bsobj = bs(w.content)
    info_caja = bsobj.find(class_="infobox geography vcard")
    info_filas = info_caja.find_all("tr")

    limpiar_etiquetas(bsobj)

    municipios_info = {}

    # Crear una lista de títulos para tomar el indice desde el cuál empezar a obtener los datos
    titulos = [fila.get('title') for fila in info_caja.find_all("a") if fila.get('title') is not None]
    contador = 0
    for indice, fila in enumerate(info_filas):
        if indice == 0 and fila.find("th"):
            municipios_info['municipio'] = fila.find("th").get_text(" ", strip=True)
        # En el caso de que no exista la etiqueta "th", será necesario tomar el nombre de la cabecera "h1" del código HTML.
        elif indice == 0 and not fila.find("th"):
            municipios_info['municipio'] = bsobj.find("h1").get_text(" ", strip=True)
        elif indice < titulos.index('Coordenadas geográficas'): # Imágenes y otros datos irrelevantes no serán tenidos en cuenta
            continue
        # Manejo de excepciones y reemplazo de strings en la obtención de los datos
        else:
            try:# Caso especial: En el caso de que una dos etiquetas "th" contengan el mismo texto, se le agrega un caracter
                # adicional al texto para diferenciarlos, y así evitar errores en la conexión con los "td" relacionados.
                if fila.find("th").get_text(" ", strip=True) == "• Total":
                    llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").strip() + f"_{contador}"
                    valores = obtener_valores(fila.find("td"))
                    municipios_info[llaves] = valores
                    contador += 1
                # Evitar tomar los datos de IDH y Población Metropolitana, debido a que solo se encuentran contabilizados en pocos municipios (Ciudades Capitales)
                elif 'IDH' in fila.find("th").get_text(" ", strip=True).upper() or 'METROPOLITANA' in fila.find("th").get_text(" ", strip=True).upper():
                    continue
                else:
                    llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").replace("\u200b", "").strip()
                    valores = obtener_valores(fila.find("td"))
                    municipios_info[llaves] = valores
            except AttributeError as e:
                pass

    return municipios_info

# In[11]:

# Probar la función "obtener_info()" para solo un municipio. Verificar que funcione correctamente.
obtener_info("https://es.wikipedia.org/wiki/Medell%C3%ADn")

# In[29]:

# Se inicia el proceso de extracción de datos para todos los municipios (Guardar todos los diccionarios en una lista)

municipios = []
for enlace in enlaces:
    try:
        municipios.append(obtener_info(enlace))
    except Exception as e:
        print(enlace)
        print(e)

# In[30]:

len(municipios)
# Se logra obtener los datos de 1118 municipios de 1122 existentes.
# Para el caso de los municipios de "Isla de San Andrés" e "Isla de Providencia" las tablas no soportan el método "find_all"
# En los otros dos casos, no existe una tabla para la obtención de los datos.

# In[31]:

# Definir una función para guardar el archivo .json y otra para cargar el archivo .json

import json

def guardar_datos(titulo, datos):
    with open(titulo, 'w', encoding='utf-8') as a:
        json.dump(datos, a, ensure_ascii=False, indent=2)

def cargar_datos(titulo):
    with open(titulo, encoding="utf-8") as a:
        return json.load(a)

# In[32]:

# Guardar resultados obtenidos hasta el momento.

guardar_datos("municipios_col.json", municipios)

# #### Tarea #3: Limpieza de Datos

# In[16]:

# Cargar datos

municipios_col = cargar_datos("municipios_col.json")

# In[17]:

# Imprimir todas las distintas llaves de la lista de dictionarios

all_keys = set().union(*(d.keys() for d in municipios_col))
all_keys

# Se identifican diversas llaves que no contienen información relevante, otras que contienen valores faltantes excesivos,
# algunas difieren en el nombre pero guardan valores similares.

# In[144]:

# Para este ejercicio, se trabaja con la información bajo un DataFrame (para facilitar algunas operaciones de limpieza)

municipios_df = pd.DataFrame(municipios_col)

# In[73]:

# Aquitania Boyacá (Cambiar link) https://es.wikipedia.org/wiki/Aquitania_(Boyac%C3%A1)
# Samaná Caldas (Cambiar link) https://es.wikipedia.org/wiki/Saman%C3%A1_(Caldas)
# Roncesvalles Tolima (Cambiar link) https://es.wikipedia.org/wiki/Roncesvalles_(Tolima)
# Eliminar key "Aeropuerto"
# Eliminar key "Autoridad tradicional Ne'jwesx 2019 - 2021"
# Eliminar "Flor", "Arbol", "Ave", "Cabecera", "Cabecera municipal", "Capital", "Barrios", "Veredas", "Caseríos"
# Eliminar "Centros Poblados", "Hermanada con", "Idioma oficial", "Moneda", "Máxima", "Mínima", "Matrícula"
# Unir Alcalde/Corregidor
# Unir localidades, comunas, barrios
# Eliminar "Otras S(s)uperficies" 'PIB (nominal)','PIB per cápita','Partidos gobernantes'
# Eliminar 'Presidente regional','Presupuesto anual', 'Temperatura', 'Temperatura promedio', 'Total_2'
# Ajustar verdadero nombre '[[Departamentos de \nColombia|Departamento]]','[xau[Subregiones de Antioquia'

# In[145]:

municipios_df.head()

# ##### ** Eliminar columnas con información irrelevante, incompleta o nula **

# In[146]:

columnas_erradas = ["Aeropuerto", "Flor", "Arbol", "Árbol", "Ave", "Cabecera", "Cabecera municipal",
        "Capital", "Barrios", "Veredas", "Caseríos", "Centros Poblados", "Nombre", "Fiestas mayores",
        "Hermanada con","Idioma oficial","Moneda","Máxima","Mínima","Matrícula","Patrono(a)","Localidades",
        "Otras Superficies:","PIB (nominal)","PIB per cápita","Partidos gobernantes","Corregimientos",
        "Otras superficies:", "Presidente regional","Presupuesto anual", "Temperatura", "Creación",
        "Temperatura promedio", "Total_2", "Eventos históricos","Superficie","Altitud", "Comunas",
        "Curso de agua","Código Dane (Divipola)","Código Dian","Correspondencia actual","Subdivisiones",
        "Autoridad tradicional Ne'jwesx 2019 - 2021", "Otros idiomas", "", "Prefijo telefónico",
        "Región", "Provincia", "Zona"]

for col in columnas_erradas:
    if col in municipios_df.columns:
        municipios_df = municipios_df.drop(col, 1)
    continue

#municipio['Superficie'] = municipio.pop('Total_0')
#municipio['Altitud Media'] = municipio.pop('Media')
# Unir Región, Subregión y Zona

# In[147]:

# Las columnas 'Total_0', Total_1', 'Urbana' y 'Media' contiene la información correspondiente
# a 'Superficie', 'Población', 'Pobl. Urbana' y 'Altitud Media' respectivamente, por lo tanto,
# cambiamos el nombre de las columnas.

municipios_df = municipios_df.rename(columns={'Total_0': 'Superficie', 'Total_1': 'Población Total',
                                              'Media': 'Altitud Media', 'Urbana': 'Pobl. Urbana'})

# ##### ** Unir columnas con igual información pero diferente nombre **

# In[149]:

# Reducir a una sola columna aquellas que contienen la misma información pero difieren en su encabezado

from re import search

columnas = {'lcald': 'Alcalde', 'epartamento': 'Departamento', 'unicipio': 'Municipio',
           'oblación': 'Población Total', 'Subregi': 'Subregión', 'Código': 'Código postal'}

for col in municipios_df.columns:
    for key, value in columnas.items():
        if search(key, col) and col != value:
            municipios_df[value].fillna(municipios_df[col], inplace=True)
            municipios_df = municipios_df.drop(col, 1)
        continue

municipios_df = municipios_df.drop("Subregión", 1)

# In[151]:

# Unir las columnas Alcalde/Corregidor

municipios_df['Alcalde'].fillna(municipios_df['Corregidor'], inplace=True)
municipios_df = municipios_df.drop('Corregidor', 1)

# ##### Extraer y añadir la información de códigos postales para todos los municipios

# In[153]:

# Extraer el contenido de la página y crear el objeto BS (Departamentos)

url = requests.get("https://codigo-postal.co/colombia/")
soup = bs(url.content)
print(soup.prettify())

# In[170]:

# Identificar todas las etiquetas "li" dentro de contenido de la clase especificada

info_box = soup.find(class_="column-list")
info_rows = info_box.find_all("li")
for row in info_rows:
    print(row.prettify())

# In[155]:

# Extraer y guardar los enlaces de cada departamento

dptos = []

for row in info_rows:
    dptos.append(row.a['href'])

dptos

# In[194]:

# Se define la función "get_urls" la cual accede al enlace del departamento, extrae y guarda los enlaces de todos sus municipios

def get_urls(dpto, mpios):
    url = requests.get(dpto)
    soup = bs(url.content)

    info_box = soup.find(class_="col-md-8")
    info_rows = info_box.find_all("ul")

    info_lists = []
    for row in info_rows:
        info_lists.append(row.find_all("li"))

    for info_list in info_lists:
        for renglon in info_list:
            mpios.append(renglon.a['href'])

    return mpios

# Extraer los enlaces de todos los municipios de cada departamento. Guardar en una misma lista todos los enlaces.
mpios = []
for dpto in dptos:
    get_urls(dpto, mpios)

# In[197]:

# En este caso, se procede a crear una nueva columna que contenga los nombres de cada municipio pero sin tildes (´) en caso...
# ... de que existan. Esto para facilitar el proceso de asignación del código postal para su correspondiente. Mas adelante...

import re
from unicodedata import normalize

municipios_df['New_municipios'] = ""

series = municipios_df['Municipio']
for index, value in series.items():
    municipios_df['New_municipios'][index] = normalize('NFC', re.sub(
        r"([^n\u0300-\u036f]|n(?!\u0303(?![\u0300-\u036f])))[\u0300-\u036f]+", r"\1",
        normalize( "NFD", value), 0, re.I))

# In[198]:

# Se asigna la nueva columna como indice del DataFrame
municipios_df = municipios_df.set_index('New_municipios')

# In[199]:

municipios_df.head()

# In[204]:

# La función "get_codes" accede al enlace del municipio, extrae y guarda el nombre del municipio y su código postal.

def get_codes(mpio, postales):
    try:
        url = requests.get(mpio)
        soup = bs(url.content)

        info_box_head = soup.find(class_="breadcrumb")
        info_rows_head = info_box_head.find_all("li")

        llave = info_rows_head[3].find("span").get_text(" ", strip=True)

        info_box_body = soup.find(class_="table-responsive")
        info_rows_body = info_box_body.find_all("tr")

        valor = info_rows_body[1].find("a").get_text(" ", strip=True)

        postales[llave] = valor

        return postales

    except AttributeError as e:
        print(f"¡{llave} no tiene código postal asignado!")
        pass

# Extraer el código postal para todos los municipio. Guardar la información en un diccionario.
postales = {}
for mpio in mpios:
    get_codes(mpio, postales)

# In[206]:

# La razón por la que se creó una nueva columna con los nombres de cada municipio sin tilde es porque es necesario asignarlos...
# ... usando el nombre de la llave, cuyos nombres no poseen tildes en los casos en los que debería tenerla. Por lo tanto, ...
# ... se utilizan los nuevos indices para

municipios_df = municipios_df.drop("Código postal", 1)

municipios_df['Código Postal'] = 0
for key, value in postales.items():
    for index, val in municipios_df['Clima'].items():
        if search(key, index):
            municipios_df['Código Postal'][index] = value

# In[210]:

# Se elimina la columna indice que fue tomada solo como referencia
municipios_df.reset_index(drop=True, inplace=True)

# In[217]:

municipios_df.head()

# ##### ** Convertir fechas en string a datetime **

# In[370]:

# Extraer la fecha de cada string de la columna "Fundación" ("5 de junio de 1935" --> "5/junio/1935")

for idx in municipios_df.index:
    try:
        if len(municipios_df['Fundación'][idx].split()) > 4:
            municipios_df['Fundación'][idx] = ' '.join(municipios_df['Fundación'][idx].split()[:5]).replace(' de', '').replace(' ', '/')
        continue
    except Exception as e:
        print(f"{idx}: {e}")

# In[371]:

# Extraer la fecha de cada string de la columna "Erección" ("5 de junio de 1935" --> "5/junio/1935")

for idx in municipios_df.index:
    try:
        if len(municipios_df['Erección'][idx].split()) > 4:
            municipios_df['Erección'][idx] = ' '.join(municipios_df['Erección'][idx].split()[:5]).replace(' de', '').replace(' ', '/')
        continue
    except Exception as e:
        print(f"{idx}: {e}")

# In[372]:

# Convertir las fechas de las columnas "Fundación" y "Erección" de formato string a datetime

from datetime import datetime
import locale # Usar locale para configurar la lectura de las fechas en idioma español.

locale.setlocale(locale.LC_TIME, 'es_ES.UTF-8')

columnas = ['Fundación', 'Erección']

for col in columnas:
    for idx in municipios_df.index:
        try:
            municipios_df[col][idx] = datetime.strptime(municipios_df[col][idx], '%d/%B/%Y')
        except Exception as e:
            print(f"{idx}: {e}")

# In[373]:

# Para aquellas fechas incompletas extraer, por lo menos, el año.

from re import search

columnas = ['Fundación', 'Erección']

for col in columnas:
    for idx in municipios_df.index:
        try:
            if not isinstance(municipios_df[col][idx], datetime):
                municipios_df[col][idx] = search('\d{4}', municipios_df[col][idx]).group(0)
        except Exception as e:
            print(f"{idx}: {e}")

# ##### ** Conversión de datos númericos, modificación de strings y  manejo de valores faltantes (NaN) **

# In[375]:

# Convertir valores de la columna superficie a flotantes, creando una nueva columna que muestre la unidad de medida en (km²)

municipios_df['Superficie (km²)'] = np.NaN

for idx in municipios_df.index:
    try:
        municipios_df['Superficie (km²)'][idx] = float(municipios_df['Superficie'][idx].split()[0])
    except ValueError:
        try:
            municipios_df['Superficie (km²)'][idx] = float(municipios_df['Superficie'][idx].split()[0].replace(',', '.'))
        except Exception as e:
            print(f"{idx}: {e}")

# Eliminar columna 'Superficie'
municipios_df = municipios_df.drop("Superficie", 1)

# In[376]:

# Convertir valores de la columna 'Altitud Media' a enteros, creando una nueva columna que muestre la unidad de medida en (msnm)

municipios_df['Altitud Media (m.s.n.m)'] = np.NaN

for idx in municipios_df.index:
    try:
        municipios_df['Altitud Media (m.s.n.m)'][idx] = float(municipios_df['Altitud Media'][idx].split()[0].replace(',', '.'))
    except Exception as e:
        print(f"{idx}: {e}")

# Eliminar columna 'Altitud Media'
municipios_df = municipios_df.drop("Altitud Media", 1)

# In[377]:

# Convertir valores de columna 'Densidad' a flotantes, creando una nueva columna que muestre la unidad de medida en (hab/km²)

municipios_df['Densidad (hab/km²)'] = np.NaN

for idx in municipios_df.index:
    try:
        municipios_df['Densidad (hab/km²)'][idx] = float(municipios_df['Densidad'][idx].split()[0])
    except (ValueError, AttributeError):
        try:
            municipios_df['Densidad (hab/km²)'][idx] = float(municipios_df['Densidad'][idx].split()[0].replace(',', '.'))
        except Exception as e:
            print(f"{idx}: {e}")

# Eliminar columna 'Densidad'
municipios_df = municipios_df.drop("Densidad", 1)

# In[397]:

# Convertir los valores de las columnas 'Población Total' y 'Pobl. Urbana' de string a enteros

columnas = {'Población Total': 'Población Total (hab)', 'Pobl. Urbana': 'Pobl. Urbana (hab)'}

municipios_df['Pobl. Urbana (hab)'] = np.NaN
municipios_df['Población Total (hab)'] = np.NaN

for key, value in columnas.items():
    for idx in municipios_df.index:
        try:
            municipios_df[value][idx] = int(municipios_df[key][idx].split(' hab.')[0].replace(' ', '').replace(',', ''))
        except Exception as e:
            print(f"{idx}: {e}")

    # Eliminar la columna correspondiente (anterior)
    municipios_df = municipios_df.drop(key, 1)

# In[432]:

# Acortar el tamaño de las coordenadas, mostrando solo la latitud y la longitud.

for idx in municipios_df.index:
    try:
        municipios_df['Coordenadas'][idx] = municipios_df['Coordenadas'][idx].split('/')[0].replace('\ufeff', '').strip()
    except Exception as e:
        print(f"{idx}: {e}")

# In[431]:

# Extraer solo el nombre del alcalde de cada municipio (originalmente aparece el año de posesión y el partido político)

for idx in municipios_df.index:
    try:
        municipios_df['Alcalde'][idx] = municipios_df['Alcalde'][idx].split('(')[0].strip()
    except Exception as e:
        print(f"{idx}: {e}")

# In[418]:

# En el caso de las columnas de huso horario, país y entidad los valores son iguales para todos los indices, es decir,
# Uso horario: UTC -5, Entidad: Municipio y País: Colombia. Se usa esta información para rellenar los valores nulos.

municipios_df['Huso horario'].fillna("UTC -5", inplace=True)
municipios_df['Entidad'].fillna("Municipio", inplace=True)
municipios_df['País'].fillna("Colombia", inplace=True)

# In[433]:

# Cambiar el orden de las columnas dependiendo de la relevancia de la información contenida.

municipios_df = municipios_df[['Municipio', 'Departamento', 'Entidad', 'País', 'Fundación', 'Erección', 'Superficie (km²)',
                              'Altitud Media (m.s.n.m)', 'Población Total (hab)', 'Pobl. Urbana (hab)', 'Densidad (hab/km²)',
                              'Coordenadas', 'Clima', 'Gentilicio', 'Alcalde', 'Código Postal', 'Huso horario']]

# ##### ** ¡Trabajo finalizado! Ahora, guardar los datos en formato CSV **

# In[21]:

# Guardar datos en formato CSV

municipios_df.to_csv("municipios_colombia_df.csv")


from __future__ import print_function
from numpy import array as Vector
from collections import namedtuple

from .vector_ops import *
from .kpclasses import Bead,chainDeepCopy
from .writer import *
from .reader import *
from .config import EPSILON,VERBOSE
from knot_pull.finder import find_frame

def too_far(p1 ,p2 ,p3 ,i0 ,i1):
    """Checks if crossing is possible - 9 A from all triangle points is impossible due to single connection
    length restrictions"""
    return all([point_distance(p1 ,i0 )>=9. ,point_distance(p2 ,i0 ) >9. ,point_distance(p3 ,i0 ) >9.]) \
            or all([point_distance(p1 ,i1 )>=9. ,point_distance(p2 ,i1 ) >9. ,point_distance(p3 ,i1 ) >9.])

def inspectingOtherLines(p1, p2, p3, i0, i1, greedy=0, log=0,but_not=[]):
    """Looks for connections going through the triangle. Starts with second connection from each triangle end"""

    #but_not = [x.vec for x in but_not]
    potentials = []
    potids = []

    p123normal = get_normal(p1, p2, p3)

    inspected = i0
    if inspected and not inspected.Nhand:
        if found_crossing(p1,p2,p3, inspected.vec, l1=None, p123normal=None) is not False:
            potentials.append(inspected.vec)

    while inspected and inspected.Nhand:
        if inspected.Nhand.end or (inspected in but_not and inspected.Nhand in but_not):#Skip "Virtual" connection - between chain ends
            inspected = inspected.Nhand
            continue
        l0, l1 = inspected.vec, inspected.Nhand.vec
        #greedy can have distances larger than allowed in too_far()
        i = False if (greedy == 0 and too_far(p1, p2, p3, l0, l1)) else found_crossing(p1, p2, p3, l0, l1, p123normal)

        if i is not False:
            potentials.append(i)
        inspected = inspected.Nhand

    inspected = i1
    if inspected and not inspected.Chand:
        if found_crossing(p1,p2,p3, inspected.vec, l1=None, p123normal=None) is not False:
            potentials.append(inspected.vec)
    while inspected and inspected.Chand:
        if inspected.end or (inspected in but_not and inspected.Chand in but_not):
            inspected = inspected.Chand
            continue
        l0, l1 = inspected.vec, inspected.Chand.vec
        #greedy can have distances larger than allowed in too_far()
        i = False if (greedy == 0 and too_far(p1, p2, p3, l0, l1)) else found_crossing(p1, p2, p3, l0, l1, p123normal)
        if i is not False:
            potentials.append(i)
        inspected = inspected.Chand
    return potentials

def filter_greedy(atoms, outfile='', len_frames=0):
    """Removes all superfluous atoms(kept for PyMOL purposes, to better detect topology"""
    current = atoms[0]
    latom = 0 #num of removed atoms
    changed = False
    while current.Chand and current.Chand.Chand:
        if current.Chand.end: #Skips "virtual" connections at chain ends
            current = current.Chand.Chand
            continue
        if current.end:
            current = current.Chand
            continue

        i0 = current.Nhand
        i1 = current.Chand.Chand.Chand if current.Chand and current.Chand.Chand else False

        potentials = inspectingOtherLines(current.vec, current.Chand.vec, current.Chand.Chand.vec, i0, i1, greedy=True)

        if potentials:
            current = current.Chand
            continue
        else:
            changed = changed or (not current.Chand.recent)
            current.setChand(current.Chand.Chand)
            current.Chand.setNhand(current)

            latom += 1
            continue
            ## TODO specjalnie nie zamieniam current - watpliwe, ale a noz widelec jeszcze jeden?

    while current.Chand: #renumbering needs to start from the C terminus
        current = current.Chand

    atoms = []
    while current:
        current.recent = max(0, current.recent - 1)
        atoms.append(current)
        current = current.Nhand
    atoms.reverse()

    for i, at in enumerate(atoms):
        at.setId(i)

    return atoms, changed

def prefilter_with_adding(atoms):
    """Removes atoms to reduce the backbone where possible. Adding new joints to long connections
    allows for reducing bumps"""

    first_bead_in_structure = atoms[0]

    next_chain_handle = first_bead_in_structure
    changed = False
    cnt=0
    while next_chain_handle is not None:
        cnt+=1
        currentN = next_chain_handle
        currentC = currentN.Chand
        if currentC is None: #to bypass that currentN can be an end
            break
        while currentC.Chand and not currentC.end:
            currentC = currentC.Chand
        next_chain_handle = currentC.Chand
        assert currentC.end or next_chain_handle is None
        #print(cnt,(currentN.Nhand,(currentN.id,currentN.end),currentN.Chand),(currentC.Nhand,(currentC.id,currentC.end),currentC.Chand))
        verbose = 0

        Ntriangle = [currentN,currentN.Chand,currentN.Chand.Chand if currentN.Chand else None]
        Ctriangle = [currentC,currentC.Nhand,currentC.Nhand.Nhand if currentC.Nhand else None]
        #if 1 or verbose:
            #print("starting with",len(atoms))
        #    print([(a.id,a.end) for a in atoms])
         #   print("handle",next_chain_handle,next_chain_handle.id if next_chain_handle is not None else None)

        while None not in Ntriangle and None not in Ctriangle:# and not any(x.end for x in Ntriangle+Ctriangle): #maybe just one of them?
            if verbose: print(currentN.id,currentN,currentC.id,currentC)
            if verbose:  print(Ntriangle,Ctriangle)
            if Ntriangle[0] == Ctriangle[0]: #just passed the overlap (for odd) or lack of it (even)
                break
            # TODO: multichain
            """
                    if current.Chand.end: #skip "Virtual" connections at chain ends
                current = current.Chand.Chand
                continue
            if current.end:
                current = current.Chand
                continue"""
            overlap = sum(nt in Ctriangle for nt in Ntriangle)
            #if overlap: print("OVERLAP",overlap)

            #TODO dodac overalp 2 gdy a' w a,(b,c'),(c,b')

            if overlap == 3:
                if verbose: print("overlap=3")
                potentials = inspectingOtherLines(Ntriangle[0].vec, Ntriangle[1].vec, Ntriangle[2].vec,
                                                  Ntriangle[0].Nhand,#.vec,
                                                  Ntriangle[2].Chand)#.vec)
                if potentials:
                    if verbose: print("have potetnails")
                    if point_distance(Ntriangle[0].vec, Ntriangle[1].vec) > 3.: #new join on long connection
                        #breaks join Ntraingle[0],Ntriangle[1] (same as Ctirange[1], Ctriangle[2])
                        new_N = get_middlepoint(Ntriangle[0].vec, Ntriangle[1].vec)
                        if verbose: print("adding nowy at", new_N)
                        nowy_id = (Ntriangle[0].original_id + Ntriangle[1].original_id) / 2.
                        if int(nowy_id) != nowy_id:
                            if int(nowy_id) != Ntriangle[0].original_id:
                                nowy_id = int(nowy_id)
                            else:
                                nowy_id = int(nowy_id) + 1
                        nowy = Bead(Vector(new_N), "CA", nowy_id)
                        nowy.recent = 2
                        nowy.setId((Ntriangle[0].id + Ntriangle[1].id) / 2.)
                        nowy.setNhand(Ntriangle[0])
                        nowy.setChand(Ntriangle[1])
                        Ntriangle[1].setNhand(nowy)
                        Ntriangle[0].setChand(nowy)
                    if point_distance(Ntriangle[1].vec, Ntriangle[2].vec) > 3.:  # new join on long connection
                        # breaks join Ntraingle[1],Ntriangle[2] (same as Ctriangle[0], Ctriangle[1])
                        new_N = get_middlepoint(Ntriangle[1].vec, Ntriangle[2].vec)
                        if verbose: print("adding nowy at", new_N)
                        nowy_id = (Ntriangle[1].original_id + Ntriangle[2].original_id) / 2.
                        if int(nowy_id) != nowy_id:
                            if int(nowy_id) != Ntriangle[1].original_id:
                                nowy_id = int(nowy_id)
                            else:
                                nowy_id = int(nowy_id) + 1
                        nowy = Bead(Vector(new_N), "CA", nowy_id)
                        nowy.recent = 2
                        nowy.setId((Ntriangle[1].id + Ntriangle[2].id) / 2.)
                        nowy.setNhand(Ntriangle[1])
                        nowy.setChand(Ntriangle[2])
                        Ntriangle[2].setNhand(nowy)
                        Ntriangle[1].setChand(nowy)
                    currentN = currentN.Chand
                    Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                    currentC = currentC.Nhand
                    Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                    continue
                else:
                    if verbose: print("dont have potetnails")
                    if point_distance(Ntriangle[0].vec, Ntriangle[2].vec) < 4.:  # TODO may be too far?
                        # if can be remove, and the distance is not so great - remove the atom
                        changed = changed or (not Ntriangle[1].recent)
                        Ntriangle[0].setChand(Ntriangle[2])
                        Ntriangle[2].setNhand(Ntriangle[0])
                        ## specjalnie nie zamieniam current - watpliwe, ale a noz widelec jeszcze jeden?
                    else:
                        position = get_middlepoint(Ntriangle[0].vec, Ntriangle[2].vec)
                        if verbose: print("adding nowy at", position)
                        # if we can reduce but the distance is too great - remove the atom, put a new one on the connection
                        # reduces traingles to straight line
                        changed = changed or point_distance(Ntriangle[1].vec, position) > EPSILON
                        nowy_id = (Ntriangle[0].original_id + Ntriangle[2].original_id) / 2.

                        nowy = Bead(Vector(position), "CA", nowy_id)
                        nowy.recent = 2
                        nowy.setId(Ntriangle[1].id)  # replaces current atom
                        nowy.setNhand(Ntriangle[0])
                        nowy.setChand(Ntriangle[2])
                        Ntriangle[2].setNhand(nowy)
                        Ntriangle[0].setChand(nowy)
                        currentN = nowy
                        Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                    break #removed middle from overlappig triangle, nothing more to do
            else:
                Npotentials = inspectingOtherLines(Ntriangle[0].vec, Ntriangle[1].vec, Ntriangle[2].vec,
                                                   Ntriangle[0].Nhand,#.vec if Ntriangle[0].Nhand else None,
                                                   Ntriangle[2].Chand,#.vec if Ntriangle[2].Chand else None,
                                                   but_not=Ctriangle)
                Cpotentials = inspectingOtherLines(Ctriangle[0].vec, Ctriangle[1].vec, Ctriangle[2].vec,
                                                   Ctriangle[2].Nhand,#.vec if Ctriangle[2].Nhand else None,
                                                   Ctriangle[0].Chand,#.vec if Ctriangle[0].Chand else None,
                                                   but_not=Ntriangle)
                if verbose: print("overlap < 3")
                if Npotentials:
                    if verbose: print("have Nportenials")
                    if Cpotentials:
                        if verbose: print("have Cportenials")
                        #nothing can be done
                        currentC=currentC.Nhand
                        Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                        currentN = currentN.Chand
                        Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                        continue # it should break at the beginning of next loop pass if overlap was 2
                    else:
                        C123normal = get_normal(Ctriangle[0].vec, Ctriangle[1].vec, Ctriangle[2].vec)
                        # check if they ARE crossing each othe
                        c0v, c1v, c2v = [c.vec for c in Ctriangle]
                        n0v, n1v, n2v = [n.vec for n in Ntriangle]
                        if any(map(lambda x: x is not False, [found_crossing(c0v, c1v, c2v, n0v, n1v, C123normal),
                                                              found_crossing(c0v, c1v, c2v, n1v, n2v, C123normal)])):
                            if verbose: print("N jest w C")
                            currentC = currentC.Nhand
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            currentN = currentN.Chand
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            continue
                        if verbose: print("dont have other Cportenials")
                        if point_distance(Ctriangle[0].vec, Ctriangle[2].vec) < 4.:  # TODO may be too far?
                            # if can be remove, and the distance is not so great - remove the atom
                            changed = changed or (not Ctriangle[1].recent)
                            Ctriangle[0].setNhand(Ctriangle[2])
                            Ctriangle[2].setChand(Ctriangle[0])
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            currentN = currentN.Chand
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            continue
                             ## specjalnie nie zamieniam current - watpliwe, ale a noz widelec jeszcze jeden?
                        else:
                            position = get_middlepoint(Ctriangle[0].vec, Ctriangle[2].vec)
                            if verbose: print("adding nowy at", position)
                            # if we can reduce but the distance is too great - remove the atom, put a new one on the connection
                            # reduces traingles to straight line
                            changed = changed or point_distance(Ctriangle[1].vec, position) > EPSILON
                            nowy_id = (Ctriangle[0].original_id + Ctriangle[2].original_id) / 2.
                            nowy = Bead(Vector(position), "CA", nowy_id)
                            nowy.recent = 2
                            nowy.setId(Ctriangle[1].id)  # replaces current atom
                            nowy.setChand(Ctriangle[0])
                            nowy.setNhand(Ctriangle[2])
                            Ctriangle[2].setChand(nowy)
                            Ctriangle[0].setNhand(nowy)
                            currentC = nowy
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            currentN = currentN.Chand
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            continue
                else:
                    if verbose: print("dont have other Nportenials")
                    if Cpotentials:
                        if verbose: print("have Cportenials")
                        N123normal = get_normal(Ntriangle[0].vec, Ntriangle[1].vec, Ntriangle[2].vec)
                        # check if they ARE crossing each othe
                        c0v, c1v, c2v = [c.vec for c in Ctriangle]
                        n0v, n1v, n2v = [n.vec for n in Ntriangle]
                        if any(map(lambda x: x is not False, [found_crossing(n0v, n1v, n2v, c0v, c1v, N123normal),
                                                              found_crossing(n0v, n1v, n2v, c1v, c2v, N123normal)])):
                            if verbose: print("C jest w N")
                            currentN = currentN.Chand
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            currentC = currentC.Nhand
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            continue
                        if point_distance(Ntriangle[0].vec, Ntriangle[2].vec) < 4.:  # TODO may be too far?
                            # if can be remove, and the distance is not so great - remove the atom
                            changed = changed or (not Ntriangle[1].recent)
                            Ntriangle[0].setChand(Ntriangle[2])
                            Ntriangle[2].setNhand(Ntriangle[0])
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            currentC = currentC.Nhand
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            continue
                             ## specjalnie nie zamieniam current - watpliwe, ale a noz widelec jeszcze jeden?
                        else:
                            position = get_middlepoint(Ntriangle[0].vec, Ntriangle[2].vec)
                            if verbose: print("adding nowy at", position)
                            # if we can reduce but the distance is too great - remove the atom, put a new one on the connection
                            # reduces traingles to straight line
                            changed = changed or point_distance(Ntriangle[1].vec, position) > EPSILON
                            nowy_id = (Ntriangle[0].original_id + Ntriangle[2].original_id) / 2.

                            nowy = Bead(Vector(position), "CA", nowy_id)
                            nowy.recent = 2
                            nowy.setId(Ntriangle[1].id)  # replaces current atom
                            nowy.setNhand(Ntriangle[0])
                            nowy.setChand(Ntriangle[2])
                            Ntriangle[2].setNhand(nowy)
                            Ntriangle[0].setChand(nowy)
                            currentN = nowy
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            currentC = currentC.Nhand
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            continue
                    else:
                        if verbose: print("dont have any portenials")
                        if overlap == 2:
                            if verbose: print("overlap==2")
                            break

                            new_BC = get_middlepoint(Ntriangle[1].vec, Ctriangle[1].vec)
                            if verbose: print("adding nowy at", new_BC)
                            changed = True
                            nowy_id = (Ntriangle[1].original_id +Ctriangle[1].original_id)/2.
                            nowy = Bead(Vector(new_BC), "CA", nowy_id)
                            nowy.recent = 2
                            nowy.setId(Ntriangle[1].id)  # replaces current atom

                            Ntriangle[0].setChand(nowy)
                            Ctriangle[0].setNhand(nowy)
                            nowy.setNhand(Ntriangle[0])
                            nowy.setChand(Ctriangle[0])
                            Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                            Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                            continue #tera bedzie overlap N.A,nowy,C.A
                        else: #separate triangles
                            if verbose: print("overlap<2")
                            N123normal = get_normal(Ntriangle[0].vec, Ntriangle[1].vec, Ntriangle[2].vec)
                            C123normal = get_normal(Ctriangle[0].vec, Ctriangle[1].vec, Ctriangle[2].vec)
                            #check if they ARE crossing each othe
                            c0v,c1v,c2v = [c.vec for c in Ctriangle]
                            n0v,n1v,n2v = [n.vec for n in Ntriangle]
                            if verbose: print("checking",[found_crossing(n0v,n1v,n2v,c0v,c1v,N123normal),
                                    found_crossing(n0v,n1v,n2v,c1v,c2v,N123normal),
                                    found_crossing(c0v, c1v, c2v, n0v, n1v, C123normal),
                                    found_crossing(c0v, c1v, c2v, n1v, n2v, C123normal)])
                            if any(map(lambda x: x is not False,[found_crossing(n0v,n1v,n2v,c0v,c1v,N123normal),
                                    found_crossing(n0v,n1v,n2v,c1v,c2v,N123normal),
                                    found_crossing(c0v, c1v, c2v, n0v, n1v, C123normal),
                                    found_crossing(c0v, c1v, c2v, n1v, n2v, C123normal)])):
                                if verbose: print("traingles cross each other")
                                currentN = currentN.Chand
                                currentC = currentC.Nhand
                                Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                                Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                                #to be determined if move them
                                continue

                            # check if they would cross each other
                            xt1 = found_crossing(Ntriangle[0].vec, Ntriangle[1].vec, Ntriangle[2].vec,Ctriangle[0].vec,Ctriangle[2].vec,N123normal)
                            xt2 = found_crossing(Ctriangle[0].vec, Ctriangle[1].vec, Ctriangle[2].vec,Ntriangle[0].vec,Ntriangle[2].vec,C123normal)
                            #print("xts:",xt1,xt2)
                            #changed to or below, bo mzoe czegos nei widze
                            if xt1 is not False or xt2 is not False: #should always be bool(xt1)==bool(xt2)
                                currentN = currentN.Chand
                                currentC = currentC.Nhand
                                Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                                Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                                # to be on the safe side
                                continue

                                X = get_middlepoint(xt1,xt2) #moving lines should meet here, last safe point
                                changed = True
                                new_N_B = get_middlepoint(xt1,X)
                                if verbose: print("adding nowy at", new_N_B)
                                nowy_N = Bead(Vector(new_N_B), "CA", Ntriangle[1].original_id)
                                nowy_N.recent = 2
                                nowy_N.setId(Ntriangle[1].id)
                                Ntriangle[0].setChand(nowy_N)
                                Ntriangle[2].setNhand(nowy_N)
                                nowy_N.setNhand(Ntriangle[0])
                                nowy_N.setChand(Ntriangle[2])
                                new_C_B = get_middlepoint(xt2,X)
                                if verbose: print("adding nowy at", new_C_B)
                                nowy_C = Bead(Vector(new_C_B), "CA", Ctriangle[1].original_id)
                                nowy_C.recent = 2
                                nowy_C.setId(Ctriangle[1].id)
                                Ctriangle[0].setNhand(nowy_C)
                                Ctriangle[2].setChand(nowy_C)
                                nowy_C.setNhand(Ctriangle[2])
                                nowy_C.setChand(Ctriangle[0])
                                #move on, wont move further here
                                currentN = currentN.Chand
                                Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                                currentC = currentC.Nhand
                                Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]

                            else: #safe to do both triangles
                                if verbose: print("no xts,distances are",point_distance(Ntriangle[0].vec, Ntriangle[2].vec),point_distance(Ctriangle[0].vec, Ctriangle[2].vec))
                                if point_distance(Ntriangle[0].vec, Ntriangle[2].vec) < 4.:  # TODO may be too far?
                                    # if can be remove, and the distance is not so great - remove the atom
                                    changed = changed or (not Ntriangle[1].recent)
                                    Ntriangle[0].setChand(Ntriangle[2])
                                    Ntriangle[2].setNhand(Ntriangle[0])
                                    Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                                    ## specjalnie nie zamieniam current - watpliwe, ale a noz widelec jeszcze jeden?
                                else:
                                    position = get_middlepoint(Ntriangle[0].vec, Ntriangle[2].vec)
                                    if verbose: print("adding nowy at",position)
                                    # if we can reduce but the distance is too great - remove the atom, put a new one on the connection
                                    # reduces traingles to straight line
                                    changed = changed or point_distance(Ntriangle[1].vec, position) > EPSILON
                                    nowy_id = (Ntriangle[0].original_id + Ntriangle[2].original_id) / 2.

                                    nowy = Bead(Vector(position), "CA", nowy_id)
                                    nowy.recent = 2
                                    nowy.setId(Ntriangle[1].id)  # replaces current atom
                                    nowy.setNhand(Ntriangle[0])
                                    nowy.setChand(Ntriangle[2])
                                    Ntriangle[2].setNhand(nowy)
                                    Ntriangle[0].setChand(nowy)
                                    currentN = nowy
                                    Ntriangle = [currentN, currentN.Chand, currentN.Chand.Chand if currentN.Chand else None]
                                if point_distance(Ctriangle[0].vec, Ctriangle[2].vec) < 4.:  # TODO may be too far?
                                    # if can be remove, and the distance is not so great - remove the atom
                                    changed = changed or (not Ctriangle[1].recent)
                                    Ctriangle[0].setNhand(Ctriangle[2])
                                    Ctriangle[2].setChand(Ctriangle[0])
                                    Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                                    continue
                                    ## specjalnie nie zamieniam current - watpliwe, ale a noz widelec jeszcze jeden?
                                else:
                                    position = get_middlepoint(Ctriangle[0].vec, Ctriangle[2].vec)
                                    if verbose: print("adding nowy at", position)
                                    # if we can reduce but the distance is too great - remove the atom, put a new one on the connection
                                    # reduces traingles to straight line
                                    changed = changed or point_distance(Ctriangle[1].vec, position) > EPSILON
                                    nowy_id = (Ctriangle[0].original_id + Ctriangle[2].original_id) / 2.
                                    nowy = Bead(Vector(position), "CA", nowy_id)
                                    nowy.recent = 2
                                    nowy.setId(Ctriangle[1].id)  # replaces current atom
                                    nowy.setChand(Ctriangle[0])
                                    nowy.setNhand(Ctriangle[2])
                                    Ctriangle[2].setChand(nowy)
                                    Ctriangle[0].setNhand(nowy)
                                    currentC = nowy
                                    Ctriangle = [currentC, currentC.Nhand, currentC.Nhand.Nhand if currentC.Nhand else None]
                                    continue
        if verbose: print("changed is",changed)

    #print(list((x.id,x.original_id) for x in atoms))

    #while currentN.Chand:
    #    currentN = currentN.Chand

    currentN = first_bead_in_structure
    atoms = []
    while currentN:
        currentN.recent = max(0, currentN.recent - 1)
        atoms.append(currentN)
        currentN = currentN.Chand
    #atoms.reverse()

    #print("after filtering atoms count:",len(atoms),atoms[0],atoms[0].Chand)

    for i, at in enumerate(atoms):
        at.setId(i)
    #exit()
    #exit()

    return atoms, changed

def run_through_filtering(atoms, config, greedy=0):
    """Governs reduction process and when it should end (when there is no improvement)"""

    frames = []
    frames.append([(x.original_id, x.vec, x.end) for x in atoms])
    o_and_t = config.get('outfile',0) and config.get('trajectory',0)

    if o_and_t:
        #print_out_last_pdb(config, frames, atoms)
        print_out_last_frame(config, atoms, len(frames))

    if not config or not config['no_pulling']:
        atoms, changed = prefilter_with_adding(atoms)

        frames_without_shortening = 0
        atoms_num = len(set(_.tuple() for _ in atoms))

        while changed:  # TODO should check RMSD up to 2 steps back
            frames.append([(x.original_id, list(x.vec), x.end) for x in atoms])
            if len(frames) >= 3 and len(frames[-1]) == len(frames[-3]):
                if all([all(i == j for i, j in zip(x[1], y[1])) for x, y in zip(frames[-1], frames[-3])]):
                    # if two frames back we had the exact same positions
                    break
                if len(set(_.tuple() for _ in atoms)) == atoms_num:
                    frames_without_shortening += 1
                else:
                    atoms_num = len(set(_.tuple() for _ in atoms))
                    frames_without_shortening = 0
                if frames_without_shortening > 2*len(atoms):
                    if VERBOSE: print ("Breaking filtering due to to not enough change")
                    break

            if o_and_t:
                #print_out_last_pdb(config, frames, atoms)
                print_out_last_frame(config, atoms, len(frames))

            atoms, changed = prefilter_with_adding(atoms)

        #    frames.append([(x.original_id, x.vec) for x in atoms])

    if VERBOSE:
        print ("Finished filtering, have {} atoms".format(len(atoms)))

    if greedy: #reduce number of atoms (improves topology detection)
        atoms, _ = filter_greedy(atoms, config, len(frames))
        frames.append([(x.original_id, x.vec, x.end) for x in atoms])
        if VERBOSE:
            print ("Finished greedy filtering, have {} atoms".format(len(atoms)))

    if o_and_t:
        #print_out_last_pdb(config, frames, atoms)
        print_out_last_frame(config, atoms, len(frames))

    return frames,atoms

def run_through_division(atoms):
    """Finds connections which can be safely cut (connected segments can't be simplified even separately) """
    if VERBOSE: print ("Starting with {} atoms".format(len(atoms)))
    cuts = []
    for i in range(1, len(atoms) - 2):  # No sense in checking last edges
        j = i + 1
        if VERBOSE: print("Cutting edge {}".format(j))
        _middle = get_middlepoint(atoms[i].vec, atoms[j].vec)
        _tmp1 = chainDeepCopy(atoms[:j] + [Bead(_middle, "CA")])
        _, _tmp1a = run_through_filtering(_tmp1, {}, greedy=1)
        if VERBOSE: print("Left from {} to {}".format(len(_tmp1),len(_tmp1a)))
        _tmp2 = chainDeepCopy([Bead(_middle, "CA")] + atoms[j:])  #
        _, _tmp2a = run_through_filtering(_tmp2, {}, greedy=1)
        if VERBOSE: print("Right from {} to {}".format(len(_tmp2),len(_tmp2a)))
        if _tmp1 == _tmp1a and _tmp2 == _tmp2a:  # no reduction
            # print i,
            cuts.append(i)

    cuts = [0] + cuts + [len(atoms) - 1]  # fix to get proper segment indices
    atom_lists = []
    if VERBOSE: print("Cuts: {}".format(cuts))

    for i in range(0, len(cuts) - 1):
        cutl = cuts[i]
        cutr = cuts[i + 1]
        lista = []
        if cutl:  # add middlepoints of our breakable edges
            _mid_l = get_middlepoint(atoms[cutl].vec, atoms[cutl + 1].vec)
            lista.append(Bead(_mid_l, "CA"))
        lista += atoms[cutl + bool(cutl):cutr + 1]
        if cutr < len(atoms) - 2:
            _mid_r = get_middlepoint(atoms[cutr].vec, atoms[cutr + 1].vec)
            lista.append(Bead(_mid_r, "CA"))
        lista = chainDeepCopy(lista)
        atom_lists.append(lista)

    return atom_lists  # returns all separateable segments

def pull(atoms, config, chain_names, get_representative=0,greedy=0):#, greedy_file="",trajectory=True,quiet=False, chain_names=(),rna=False):
    outfile = config['outfile']

    frames, atoms = run_through_filtering(atoms, config, greedy=greedy)
    quiet = config['quiet']
    repr = []
    if get_representative:
        Atom = namedtuple("Atom","original_id x y z end")
        repr = find_frame(frames)
        repr = [Atom(x[0],x[1][0],x[1][1],x[1][2],x[2]) for x in repr]

    chains = []
    model_num = len(frames)+1
    separate_chains = divide_into_bead_chains(atoms)
    for c, chain in enumerate(separate_chains):
        ch = Chain(c,chain)
        if chain_names: ch.setChainName(chain_names[c])
        if outfile:
            if not quiet: model_num = ch.print2file(config,model_num)
        chains.append(ch)

    for _,c1 in enumerate(chains):
        for c2 in chains[_+1:]:
            if are_chains_linked(c1,c2):
                c1.neighbours.append(c2)
                c2.neighbours.append(c1)

    return chains,repr

def same_vecs(l1,l2):
    return all(np.array_equal(x.vec,y.vec) for x,y in zip(l1,l2))

def are_chains_linked(ch1,ch2):
    _tmp_both = chainDeepCopy(ch1.atoms+ch2.atoms)
    _tmp_both[len(ch1.atoms)-1].end = True
    _tmp_ch1 = chainDeepCopy(ch1.atoms)
    _tmp_ch2 = chainDeepCopy(ch2.atoms)

    _,_both = run_through_filtering(_tmp_both, {}, greedy=1)
    _,_ch1 = run_through_filtering(_tmp_ch1, {}, greedy=1)
    _,_ch2 = run_through_filtering(_tmp_ch2, {}, greedy=1)
    return not same_vecs(_both,_ch1+_ch2)

class Chain:
    def __init__(self,chain,atoms):
        self.chain = chain
        self.atoms = atoms
        self.atom_lists = run_through_division(atoms)
        self.knots = []
        self.neighbours = []
        self.closing_path = closeTheCurve(atoms)
        self.closing_paths = [closeTheCurve(a) for a in self.atom_lists]
        self.wanda = None
        self.dowker = None
        self.dowker_code = None
        self.chain_name = ""

    def setChainName(self,c):
        self.chain_name = "({})".format(c)

    def print2file(self,config,model_num):
        #print_out_one_frame(outfile, self.atoms, model_num,self.chain)
        model_num+=1
        for alist in self.atom_lists:
            #print_out_one_frame(outfile, alist, model_num,self.chain,self.rna)
            print_out_last_frame(config, alist, model_num, cur_chain=self.chain_name[1:-1], final=True)
            model_num+=1
        return model_num

if __name__ == "__main__":
    import sys
    with open(sys.argv[1]) as input:
        atoms = input.readlines()
    if len(sys.argv) > 2:
        atoms = atoms[int(sys.argv[2]):int(sys.argv[3]) + 1]
    print ("Result is", run_through_filtering(atoms, sys.argv[1] + (
        "{}_{}".format(sys.argv[2], sys.argv[3]) if len(sys.argv) > 2 else "") + ".outall.pdb"))


import sys
import json
from pubcontrol import Item
from gripcontrol import GripPubControl, WebSocketEvent, \
    WebSocketMessageFormat, decode_websocket_events, encode_websocket_events

pub = GripPubControl({'control_uri': 'http://localhost:5561'})

opening = False
out_headers = []
out = []
for e in decode_websocket_events(sys.stdin.read()):
    if e.type == 'OPEN':
        if not opening:
            opening = True

            # enable GRIP
            out_headers.append(('Sec-WebSocket-Extensions', 'grip'))

            # ack the open
            out.append(e)

            # subscribe connection to channel
            cm = {'type': 'subscribe', 'channel': 'room'}
            out.append(WebSocketEvent('TEXT', 'c:%s' % json.dumps(cm)))
    elif e.type == 'CLOSE':
        out.append(e) # ack
        break
    elif e.type == 'TEXT':
        # broadcast to everyone
        pub.publish('room', Item(WebSocketMessageFormat(e.content)))

out_headers.append(('Content-Type', 'application/websocket-events'))

for header in out_headers:
    cm = {
        'type': 'setHeader',
        'payload': {
            'name': header[0],
            'value': header[1]
        }
    }
    sys.stderr.write('%s\n' % json.dumps(cm))

sys.stdout.write(encode_websocket_events(out))


import sys
import time
import mdi
try:
    import numpy
    use_numpy = True
except ImportError:
    use_numpy = False
try:
    from mpi4py import MPI
    use_mpi4py = True
except ImportError:
    use_mpi4py = False

niterations = 10000
natoms = 10
get_coords = False

# initialize MPI
if use_mpi4py:
    mpi_world = MPI.COMM_WORLD
else:
    mpi_world = None

# get the Angstrom to Bohr conversion factor
conversion_factor = mdi.MDI_Conversion_Factor("angstrom","bohr")
print("Conversion Factor: " + str(conversion_factor))

# initialize the socket
mdi.MDI_Init(sys.argv[2],mpi_world)
if use_mpi4py:
    mpi_world = mdi.MDI_Get_Intra_Code_MPI_Comm()
    world_rank = mpi_world.Get_rank()
else:
    world_rank = 0

# connect to the production codes
ncodes = 1
if world_rank == 0:
    for icode in range(ncodes):
        comm = mdi.MDI_Accept_Communicator()

        # get the name of the code
        mdi.MDI_Send_Command("<NAME", comm)
        name = mdi.MDI_Recv(mdi.MDI_NAME_LENGTH, mdi.MDI_CHAR, comm)
        print('Received communicator: ' + str(name))

        if name.strip() == 'PONG' or name.strip() == "PONG_F90" or name.strip() == "PONG_Py":
            mm_comm = comm
        else:
            raise ValueError('Production code name not recognized')

if world_rank == 0:

    initial_time = time.clock();

    for iiter in range(niterations):

        # get the MM energy
        mdi.MDI_Send_Command("<NAME", comm)
        name = mdi.MDI_Recv(mdi.MDI_NAME_LENGTH, mdi.MDI_CHAR, comm)

        # get the MM coordinates
        if get_coords:
            mdi.MDI_Send_Command("<COORDS", comm)
            if use_numpy:
                coords = mdi.MDI_Recv(3*natoms, mdi.MDI_DOUBLE_NUMPY, comm)
            else:
                coords = mdi.MDI_Recv(3*natoms, mdi.MDI_DOUBLE, comm)

        if iiter % 1000 == 0:
            print("Iteration: " + str(iiter) + "   " + name )

    # close the production codes
    mdi.MDI_Send_Command("EXIT", mm_comm)

    wall_time = time.clock() - initial_time
    if get_coords:
        print( str(coords) )
    print("Ping-pong time: " + str(wall_time))
    print("   us: " + str(1000000.0*wall_time/(1.0*(2*niterations))))


import cv2
import numpy as np

image_obj = cv2.imread(r"C:\Users\User\Desktop\Feature Maps\ALLSTIMULI\ALLSTIMULI\i1252592787.jpeg")

gray = cv2.cvtColor(image_obj, cv2.COLOR_BGR2GRAY)

kernel = np.ones((4, 4), np.uint8)
dilation = cv2.dilate(gray, kernel, iterations=1)

blur = cv2.GaussianBlur(dilation, (5, 5), 0)

thresh = cv2.adaptiveThreshold(blur, 255, 1, 1, 11, 2)

# Now finding Contours         ###################
contours, _ = cv2.findContours(thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

coordinates = []

for cnt in contours:
    area = cv2.contourArea(cnt)
    if area > 1000:
        approx = cv2.approxPolyDP(
            cnt, 0.07 * cv2.arcLength(cnt, True), True)
        if len(approx) == 3:
            coordinates.append([cnt])
            cv2.drawContours(image_obj, [cnt], 0, (0, 0, 255), 3)

cv2.imshow("result.png", image_obj)
cv2.waitKey(0)


from os import name
import sys
import xml.etree.ElementTree as ET
from xml.dom import minidom

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

class DFA:
    Q            = [[0, "q0"], [1, "q1"], [2, "q2"], [3, "q3"], [4, "q4"]]                           ## list of states as ID and name
    sigma        = [0, 1]                        ## list of alphabet
    delta        = [                            ## transition function list of sets in format start id, input code, end id
                    [0,0,0],
                    [0,1,1],
                    [1,0,1],
                    [1,1,2],
                    [2,0,2],
                    [2,1,3],
                    [3,0,3],
                    [3,1,4],
                    [4,0,4],
                    [4,1,4]

                    ]
    q0           = 0                            ## start state as ID
    F            = [3]                          ## accept states
    currentState = q0                           ## this starts on q0 and will be changed as the dfa is calculated

    def genXML(self):

        root = ET.Element("structure")
        automaton = ET.Element("automaton")
        root.append(automaton)
        for thing in self.Q:
            state = ET.Element("state")
            state.set("id", str(thing[0]))
            state.set("name", thing[1])
            automaton.append(state)
            if thing[0] == self.q0:
                initialTagger = ET.Element("initial")
                state.append(initialTagger)
            if thing[0] in self.F:
                finalTagger = ET.Element("final")
                state.append(finalTagger)
        for transSet in self.delta:
            transition = ET.Element("transition")
            fromTag = ET.Element("from")
            toTag = ET.Element("to")
            readTag = ET.Element("read")
            fromTag.text = str(transSet[0])
            toTag.text = str(transSet[2])
            readTag.text = str(transSet[1])
            transition.append(fromTag)
            transition.append(toTag)
            transition.append(readTag)
            automaton.append(transition)

        tree = ET.ElementTree(root)
        with open ("genned.xml", "wb") as files :
            tree.write(files)
        xmlstr = ET.tostring(element=automaton, encoding='unicode', method='xml')
        print(xmlstr)

myDFA = DFA()

myDFA.genXML()


import random
import numpy as np
import torch
from torch import Tensor
import torch.nn as nn
from torch.nn import Parameter
import torch.nn.functional as F
from tqdm import tqdm

class Node2Vec:
    def __init__(self, G, dim=128):
        self.G = G
        self.dim = dim
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.Embeddings = torch.zeros(self.G.n, self.dim).to(self.device)
        self.Embeddings.requires_grad = False
        nn.init.normal_(self.Embeddings, mean=0.0, std=1.0/(self.dim**0.5))
        self.UnigramTable = list()
        print("InitUnigramTable...")
        self.initUnigramTable()
        self.ridx = 0

    def initUnigramTable(self, size=10000000):
        count = [len(self.G.A[i])+1 for i in range(self.G.n)]
        count = np.array(count) ** 0.75
        total_pow = np.sum(count)
        i = 0
        prefix_sum = count[i] / total_pow
        for idx in tqdm(range(size)):
            self.UnigramTable.append(i)
            if idx > prefix_sum * size:
                i += 1
                if i >= self.G.n:
                    i = self.G.n - 1
                prefix_sum += count[i] / total_pow
        print('shuffling...')
        random.shuffle(self.UnigramTable)

    def training(self, max_iter=100, negative=100, rewalk=True, weight_decay=1e-4):
        unisize = len(self.UnigramTable)
        lr = 0.01
        iter_gradient_norm = 0
        for iter in tqdm(range(max_iter)):
            iter_length = 0
            if rewalk:
                print('Random walking...')
                self.G.clean_walk()
                for i in tqdm(range(self.G.n)):
                    self.G.perform_walk(i, 2, 0.5, 20)
                iter_length = len(self.G.Walks)
            else:
                iter_length = self.G.n
            random.shuffle(self.G.Walks)

            for walk in tqdm(self.G.Walks[:iter_length]):
                u = walk[0]
                embedu = self.Embeddings[u]

                idx_p = torch.tensor(walk[1:], dtype=torch.int64).to(self.device)
                embedvs_p = torch.index_select(self.Embeddings, 0, idx_p)
                p_p = 1.0 - torch.sigmoid(torch.matmul(embedvs_p, embedu))

                # negative sample
                neighbors = list()
                count = 0
                while count < negative:
                    v = self.UnigramTable[self.ridx]
                    if not v in self.G.A_set[u]:
                        neighbors.append(v)
                        count += 1
                    self.ridx += 1
                    if self.ridx >= len(self.UnigramTable):
                        self.ridx = 0

                idx_n = torch.tensor(neighbors, dtype=torch.int64).to(self.device)
                embedvs_n = torch.index_select(self.Embeddings, 0, idx_n)
                p_n = 1.0 - torch.sigmoid(-torch.matmul(embedvs_n, embedu))

                # gradient descent
                dEu = lr * (torch.matmul(p_p, embedvs_p) - torch.matmul(p_n, embedvs_n)) - weight_decay * embedu
                self.Embeddings[u] += dEu
                self.Embeddings.index_add_(0, idx_p, lr * (p_p.view(-1, 1) * (embedu - dEu)) - weight_decay * embedvs_p)
                self.Embeddings.index_add_(0, idx_n, -lr * (p_n.view(-1, 1) * (embedu - dEu)) - weight_decay * embedvs_n)

                if (iter + 1) % 10 == 0:
                    iter_gradient_norm += (torch.norm(dEu) / lr).item()

            if (iter + 1) % 25 == 0:
                lr *= 0.5
            if (iter + 1) % 10 == 0:
                print('iter : {} ; average_gradient_norm : {}'.format(iter+1,
                    iter_gradient_norm / iter_length))
                iter_gradient_norm = 0

    def save_embeddings(self, pth):
        torch.save(self.Embeddings, pth)


# Code written to reproduce and repurpose what I already found online at:
    # https://stackoverflow.com/questions/42329766/python-nlp-british-english-vs-american-english
    # https://gist.github.com/rcortini/0d05417339bc74300ce3a971442a4d3c
    # https://github.com/hyperreality/American-British-English-Translator/blob/master/data/british_spellings.json

    # Credits to the creators of those SO answer and the dictionary!

######### OPTION 1: Use online version of the data #########

import requests

def americanize(string):
    url ="https://raw.githubusercontent.com/hyperreality/American-British-English-Translator/master/data/british_spellings.json"
    british_to_american_dict = requests.get(url).json()

    for british_spelling, american_spelling in british_to_american.items():
        string = string.replace(british_spelling, american_spelling)

    return string

def britishize(string):
    url ="https://raw.githubusercontent.com/hyperreality/American-British-English-Translator/master/data/american_spellings.json"
    american_to_british_dict = requests.get(url).json()

    for american_spelling, british_spelling in american_to_british.items():
        string = string.replace(american_spelling, british_spelling)

    return string

######### OPTION 2: Use local version of the data #########

# Read in the JSON from wherever it is locally
def replace_all(text, mydict):
    for gb, us in mydict.items():
        text = text.replace(us, gb)
    return text

# Example
gb_text = "I realise I can see the world in colour but I can't vocalise its splendour"
us_text = replace_all(gb_text, us2gb)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Demo for the TP-Link A19-LB130 RBGW WiFi bulb
'''

import time
from tplight import LB130

def main():
    '''
    Main program function
    '''

    # create an instance of the light with its IP address
    light = LB130("10.0.0.130")

    # fetch the details for the light
    print("Device ID: " + light.device_id)
    print("Alias: " + light.alias)
    print("Wattage: " + str(light.wattage))

    # set the transition period for any changes to 1 seconds
    light.transition_period = 0

    # set the brightness to 50%
    light.brightness = 50
    time.sleep(2)

    # set the saturation to 100%
    light.saturation = 90
    time.sleep(2)

    # cycle through the colours
    light.hue = 0
    time.sleep(1)
    light.hue = 60
    time.sleep(1)
    light.hue = 120
    time.sleep(1)
    light.hue = 180
    time.sleep(1)
    light.hue = 240
    time.sleep(1)
    light.hue = 300
    time.sleep(1)

    # set the colour to warm white and the brightness to 0
    light.temperature = 3800
    light.brightness = 0

if __name__ == "__main__":
    main()


import json
import os

from typing import Dict, Any, Union, Optional
from typing_extensions import Literal
from functools import lru_cache

from .. import load_shared_data, get_shared_data_root

from .pipette_definition import (
    PipetteConfigurations,
    PipetteLiquidPropertiesDefinition,
)
from .model_constants import MOUNT_CONFIG_LOOKUP_TABLE, _MAP_KEY_TO_V2
from .types import (
    PipetteChannelType,
    PipetteModelType,
    PipetteGenerationType,
    PipetteVersionType,
    PipetteModelMajorVersion,
    PipetteModelMinorVersion,
    LiquidClasses,
)

LoadedConfiguration = Dict[str, Union[str, Dict[str, Any]]]

def _get_configuration_dictionary(
    config_type: Literal["general", "geometry", "liquid"],
    channels: PipetteChannelType,
    model: PipetteModelType,
    version: PipetteVersionType,
    liquid_class: Optional[LiquidClasses] = None,
) -> LoadedConfiguration:
    if liquid_class:
        config_path = (
            get_shared_data_root()
            / "pipette"
            / "definitions"
            / "2"
            / config_type
            / channels.name.lower()
            / model.value
            / liquid_class.name
            / f"{version.major}_{version.minor}.json"
        )
    else:
        config_path = (
            get_shared_data_root()
            / "pipette"
            / "definitions"
            / "2"
            / config_type
            / channels.name.lower()
            / model.value
            / f"{version.major}_{version.minor}.json"
        )
    return json.loads(load_shared_data(config_path))

@lru_cache(maxsize=None)
def _geometry(
    channels: PipetteChannelType,
    model: PipetteModelType,
    version: PipetteVersionType,
) -> LoadedConfiguration:
    return _get_configuration_dictionary("geometry", channels, model, version)

@lru_cache(maxsize=None)
def _liquid(
    channels: PipetteChannelType,
    model: PipetteModelType,
    version: PipetteVersionType,
) -> Dict[str, LoadedConfiguration]:
    liquid_dict = {}
    for liquid_class in LiquidClasses:
        try:
            liquid_dict[liquid_class.name] = _get_configuration_dictionary(
                "liquid", channels, model, version, liquid_class
            )
        except FileNotFoundError:
            continue

    return liquid_dict

@lru_cache(maxsize=None)
def _physical(
    channels: PipetteChannelType,
    model: PipetteModelType,
    version: PipetteVersionType,
) -> LoadedConfiguration:
    return _get_configuration_dictionary("general", channels, model, version)

@lru_cache(maxsize=None)
def load_serial_lookup_table() -> Dict[str, str]:
    """Load a serial abbreviation lookup table mapped to model name."""
    config_path = get_shared_data_root() / "pipette" / "definitions" / "2" / "general"
    _lookup_table = {}
    _channel_shorthand = {
        "eight_channel": "M",
        "single_channel": "S",
        "ninety_six_channel": "H",
    }
    _channel_model_str = {
        "single_channel": "single",
        "ninety_six_channel": "96",
        "eight_channel": "multi",
    }
    _model_shorthand = {"p1000": "p1k", "p300": "p3h"}
    for channel_dir in os.listdir(config_path):
        for model_dir in os.listdir(config_path / channel_dir):
            for version_file in os.listdir(config_path / channel_dir / model_dir):
                version_list = version_file.split(".json")[0].split("_")
                built_model = f"{model_dir}_{_channel_model_str[channel_dir]}_v{version_list[0]}.{version_list[1]}"

                model_shorthand = _model_shorthand.get(model_dir, model_dir)

                if (
                    model_dir == "p300"
                    and int(version_list[0]) == 1
                    and int(version_list[1]) == 0
                ):
                    # Well apparently, we decided to switch the shorthand of the p300 depending
                    # on whether it's a "V1" model or not...so...here is the lovely workaround.
                    model_shorthand = model_dir
                serial_shorthand = f"{model_shorthand.upper()}{_channel_shorthand[channel_dir]}V{version_list[0]}{version_list[1]}"
                _lookup_table[serial_shorthand] = built_model
    return _lookup_table

def load_liquid_model(
    model: PipetteModelType,
    channels: PipetteChannelType,
    version: PipetteVersionType,
) -> Dict[str, PipetteLiquidPropertiesDefinition]:
    liquid_dict = _liquid(channels, model, version)
    return {
        k: PipetteLiquidPropertiesDefinition.parse_obj(v)
        for k, v in liquid_dict.items()
    }

def _change_to_camel_case(c: str) -> str:
    # Tiny helper function to convert to camelCase.
    config_name = c.split("_")
    if len(config_name) == 1:
        return config_name[0]
    return f"{config_name[0]}" + "".join(s.capitalize() for s in config_name[1::])

def update_pipette_configuration(  # noqa: C901
    base_configurations: PipetteConfigurations,
    v1_configuration_changes: Dict[str, Any],
    liquid_class: Optional[LiquidClasses] = None,
) -> PipetteConfigurations:
    """Helper function to update 'V1' format configurations (left over from PipetteDict).

    #TODO (lc 7-14-2023) Remove once the pipette config dict is eliminated.
    Given an input of v1 mutable configs, look up the equivalent keyed
    value of that configuration."""
    quirks_list = []
    dict_of_base_model = base_configurations.dict(by_alias=True)

    for c, v in v1_configuration_changes.items():
        lookup_key = _change_to_camel_case(c)
        if c == "quirks" and isinstance(v, dict):
            quirks_list.extend([b.name for b in v.values() if b.value])
        elif liquid_class:
            if lookup_key == "tipLength":
                new_names = _MAP_KEY_TO_V2[lookup_key]
                top_name = new_names["top_level_name"]
                nested_name = new_names["nested_name"]
                # This is only a concern for OT-2 configs and I think we can
                # be less smart about handling multiple tip types by updating
                # all tips.
                for k in dict_of_base_model["liquid_properties"][liquid_class][
                    new_names["top_level_name"]
                ].keys():
                    dict_of_base_model["liquid_properties"][liquid_class][top_name][k][
                        nested_name
                    ] = v
            else:
                dict_of_base_model["liquid_properties"][liquid_class].pop(lookup_key)
                dict_of_base_model["liquid_properties"][liquid_class][lookup_key] = v
        else:
            try:
                dict_of_base_model.pop(lookup_key)
                dict_of_base_model[lookup_key] = v
            except KeyError:
                # The name is not the same format as previous so
                # we need to look it up from the V2 key map
                new_names = _MAP_KEY_TO_V2[lookup_key]
                top_name = new_names["top_level_name"]
                nested_name = new_names["nested_name"]
                if new_names.get("liquid_class"):
                    # isinstances are needed for type checking.
                    liquid_class = LiquidClasses[new_names["liquid_class"]]
                    dict_of_base_model[top_name][liquid_class][nested_name] = v
                else:
                    # isinstances are needed for type checking.
                    dict_of_base_model[top_name][nested_name] = v
    dict_of_base_model["quirks"] = list(
        set(dict_of_base_model["quirks"]) - set(quirks_list)
    )

    # re-serialization is not great for this nested enum so we need
    # to perform this workaround.
    if not liquid_class:
        liquid_class = LiquidClasses.default
    dict_of_base_model["liquid_properties"][liquid_class]["supportedTips"] = {
        k.name: v
        for k, v in dict_of_base_model["liquid_properties"][liquid_class][
            "supportedTips"
        ].items()
    }
    dict_of_base_model["liquid_properties"] = {
        k.name: v for k, v in dict_of_base_model["liquid_properties"].items()
    }
    dict_of_base_model["plungerPositionsConfigurations"] = {
        k.name: v
        for k, v in dict_of_base_model["plungerPositionsConfigurations"].items()
    }
    return PipetteConfigurations.parse_obj(dict_of_base_model)

def load_definition(
    model: PipetteModelType,
    channels: PipetteChannelType,
    version: PipetteVersionType,
) -> PipetteConfigurations:
    if (
        version.major not in PipetteModelMajorVersion
        or version.minor not in PipetteModelMinorVersion
    ):
        raise KeyError("Pipette version not found.")

    geometry_dict = _geometry(channels, model, version)
    physical_dict = _physical(channels, model, version)
    liquid_dict = _liquid(channels, model, version)

    generation = PipetteGenerationType(physical_dict["displayCategory"])
    mount_configs = MOUNT_CONFIG_LOOKUP_TABLE[generation][channels]

    return PipetteConfigurations.parse_obj(
        {
            **geometry_dict,
            **physical_dict,
            "liquid_properties": liquid_dict,
            "version": version,
            "mount_configurations": mount_configs,
        }
    )


"""Visualize data from multiple files on same plot.

A few things to note:

- Ensure:
  - the pertinent data is located in a single directory
  - data format is consistent across files

- type_of_plot argument can be:
  - distance
  - performance
  - dempref

  Note: Choose one of distance or performance even if both types of data
        are within your directory; the plotting can handle that.
"""
from data_utils import *
from inquire.utils.datatypes import Modality

""" Required arguments: """
domains=["linear_combo", "linear_system", "lander", "pizza"]
statics=[[False],[True,False],[True,False],[True,False]]

costs = {Modality.NONE: 0, Modality.DEMONSTRATION: 20, Modality.PREFERENCE: 10, Modality.CORRECTION: 15, Modality.BINARY: 5}

types = ["dempref", "inquire-weighted","pref", "corr", "demo", "bnry", "weighted-inquire"]
def main():
    for d in range(len(domains)):
        domain = domains[d]
        static_vals = statics[d]
        for static in static_vals:
            if static:
                static_name="static_"
            else:
                static_name=""
            directory = "output/static_betas_results/" + static_name + domain + "/"
            for t in types:
                if t == "dempref":
                    prefix = "--" + static_name + domain + "_"
                elif domain == "pizza":
                    prefix = "--" + static_name + domain + "_alpha-0.001_"
                else:
                    prefix = "--" + static_name + domain + "_alpha-0.005_"
                p = Path(directory + t + prefix + "distance.csv")
                if p.is_file():
                    main_data = get_data(file=t+prefix+"distance.csv", directory=directory)[0]
                    query_data = get_data(file=t+prefix+"query_types.csv", directory=directory)[0]
                    converted_data = convert_x_to_cost_axis(main_data, query_data, costs)
                    converted_data.to_csv(directory + t + prefix + "cost.csv")

if __name__ == "__main__":
    main()


import frappe

# This patch updates the Gym Settings with additional email address
def execute():
    # Get the updated version of the Doctype
     frappe.reload_doc("frappe_gym", "doctype", "Gym Settings")
     settings = frappe.get_doc("Gym Settings")

    # Did we find the attribute? If not set it the same as settings email
     if not hasattr(settings,"finance_email"):
        frappe.set_value("Gym Settings", "Gym Settings", "finance_email", settings.email)


import random, time

random.seed(time.time())

randomInput = open("/home/pi/Desktop/GUI/examples.txt", "r")
lat = open("/home/pi/Desktop/Coordinates.txt", "w")
coordinates = []

while True:
    data = randomInput.readline()
    if data == '':
        break
    else:
        data = data.strip('\n')
        coordinates.append(data)

# x  = Latitude
# y = longitude
# SouthWest Latitude < x < SouthEast Latitude
# SouthWest Lognitude < y < NorthWest Longitude
# NW Longitude = SW Longitude
# SE Latitude = SW Latitude
# NE Latitude = SE Latitude
# NE Longitude = NW Longitude

lat.write(coordinates[2] + ',' + coordinates[3] + '\n')
for i in range(0,int(coordinates[8])):
    latitude = random.uniform(float(coordinates[2]),float(coordinates[0]))
    longitude = random.uniform(float(coordinates[1]),float(coordinates[5]))
    lat.write(str(latitude) + ',' + str(longitude) + '\n')

lat.write(coordinates[2] + ',' + coordinates[3] + '\n')
lat.close()
randomInput.close()


from collections import namedtuple
import numpy as np
import math

Move = namedtuple("Move", ["dest"])
Wait = namedtuple("Wait", [])

STATE_GOING_UP = "up"
STATE_GOING_DOWN = "down"
STATE_EMPTY = "both"

ControllerStatus = namedtuple("ControllerStatus",
                              ["wait_up", "wait_down",
                               "locations", "members",
                               "status"])

Person = namedtuple("Person",
                    ["dest_floor", "spawn_floor", "spawn_tick"])

def make_person_from_jsval(v):
    return Person(
        dest_floor=int(v['df']),
        spawn_floor=int(v['sf']),
        spawn_tick=int(v['st']))

class World:
    def __init__(self, conf):
        self.nfloors = conf['num_floors']
        self.nlifts = conf['num_lifts']
        self.ticks_per_day = conf['tick_per_day']
        self.max_people_per_lift = conf['max_people_per_lift']
        self.lift_inv_speed = conf['lift_inv_speed']

        self.lift_stop_duration = conf['lift_stop_duration']

        self.spawn_lambdas = np.zeros((self.ticks_per_day, self.nfloors))
        self.destination_probs = {}
        for dest_conf in conf['destinations']:
            key = dest_conf['type']
            val = np.array(dest_conf['matrix'])
            val = val / np.array([val.sum(axis=1)]).T # normalize
            self.destination_probs[key] = val

        self.destination_types = [None for _ in range(self.ticks_per_day)]
        for spawn_conf in conf['people_incoming']:
            beg, end = spawn_conf['range']
            lambdas = np.array(spawn_conf['incoming_lambda'])
            self.spawn_lambdas[beg:end, :] = lambdas
            for t in range(beg, end):
                self.destination_types[t] = spawn_conf['destination_type']

    def spawn_people(self, tick):
        people = np.random.poisson(self.spawn_lambdas[tick])

        ret = [[] for _ in range(self.nfloors)]
        probmat = self.destination_probs[self.destination_types[tick]]
        for i, (n, ps) in enumerate(zip(people, probmat)):
            if n == 0:
                continue

            for _ in range(n):
                d = np.random.choice(range(self.nfloors), p=ps)
                ret[i].append(Person(dest_floor=d, spawn_floor=i, spawn_tick=tick))
        return ret

def next_int(fr):
    if fr.denominator == 1:
        return fr.numerator + 1
    else:
        return math.ceil(fr)

def prev_int(fr):
    if fr.denominator == 1:
        return fr.numerator - 1
    else:
        return math.floor(fr)


from flask_wtf import FlaskForm
from wtforms import StringField
from wtforms.validators import InputRequired, Length, Regexp, Email

LETTERS_REGEXP = r'^[A-Za-ząćęłńóśźżĄĆĘŁŃÓŚŹŻ]+$'
PH_NUM_MESS = 'Number should look like YYXXXXXXXXX,' \
              ' where YYYY is the dialing code e.g. 0048 for Poland, and X’s are the digits of phone number'

# Employee's personal data form
class EmployeePersonalForm(FlaskForm):
    first_name = StringField('First Name', validators=[InputRequired(), Length(max=255), Regexp(LETTERS_REGEXP)])
    last_name = StringField('Last Name', validators=[InputRequired(), Length(max=255), Regexp(LETTERS_REGEXP)])
    tel_number = StringField('Phone number', validators=[InputRequired(), Regexp(r'^[0-9]{11}$', message=PH_NUM_MESS)])
    email = StringField('Email', validators=[InputRequired(), Length(max=100), Email()])
    position = StringField('Position', validators=[InputRequired()], render_kw={'readonly': True})


import tensorflow as tf
import keras

def create_model(input_globalvars,input_cpf,input_npf,input_sv,nlabels):

    cpf = input_cpf
    for nfilters in [32,32,8]:
        cpf = keras.layers.Conv1D(nfilters,1,padding='same',activation='selu')(cpf)
        cpf = keras.layers.Dropout(0.1)(cpf)
    cpf = keras.layers.Flatten()(cpf)

    npf = input_npf
    for nfilters in [16,16,4]:
        npf = keras.layers.Conv1D(nfilters,1,padding='same',activation='selu')(npf)
        npf = keras.layers.Dropout(0.1)(npf)
    npf = keras.layers.Flatten()(npf)

    sv = input_sv
    for nfilters in [32,32,8]:
        sv = keras.layers.Conv1D(nfilters,1,padding='same',activation='selu')(sv)
        sv = keras.layers.Dropout(0.1)(sv)
    sv = keras.layers.Flatten()(sv)

    globalvars = input_globalvars
    features = keras.layers.Concatenate(axis=1)([cpf,npf,sv,globalvars])
    for nnodes in [200,100,100]:
        features = keras.layers.Dense(nnodes,activation='selu')(features)
        features = keras.layers.Dropout(0.1)(features)
    predicted_labels = keras.layers.Dense(nlabels)(features)

    model = keras.models.Model(
        inputs=[
            input_globalvars,
            input_cpf,
            input_npf,
            input_sv
        ],
        outputs=[predicted_labels]
    )

    return model


import math
import logging
import numpy as np

import helpers

def get_angle_between_points(first, second):
    """Returns the absolute angle between two points"""
    angle = math.atan2(second[1] - first[1], second[0] - first[0])
    angle = helpers.maths.norm_pi(angle)
    return angle

def get_distance_between_points(first, second):
    """Returns the distance between two points"""
    part_one = math.pow(second[0] - first[0], 2)
    part_two = math.pow(second[1] - first[1], 2)
    distance = math.sqrt(part_one + part_two)
    return distance

def is_near(pos, center, threshold=0.05):
    """
    Returns a true if the robot is inside a certain area determined by an xy center and a given radius.
    """
    if (type(pos) != np.ndarray) and (type(pos) != list):
        pos = [pos.x.value, pos.y.value]
    return math.pow(pos[0] - center[0], 2) + math.pow(pos[1] - center[1], 2) <= math.pow(threshold, 2)

def is_near_angle(robot_th, th, threshold=0.05):
    """ Returns True if the robot's angle matches that of the one specified in the th argument (with a default threshold value of 5.7º)"""
    if not isinstance(robot_th, (int, float)):
        robot_th = robot_th.th.value
    if abs(helpers.maths.norm_pi(robot_th)-helpers.maths.norm_pi(th)) < threshold:
        return True
    return (math.pi - abs(helpers.maths.norm_pi(robot_th)) + math.pi - abs(helpers.maths.norm_pi(th)) < threshold)

def absolute_offset(robot, distance=0):
    """Returns the position of a point at a certain distance in front of the robot"""
    final_pos = np.array([robot.x.value + distance*math.sin(robot.th.value),
                         robot.y.value + distance*math.cos(robot.th.value)])
    return final_pos

def get_robot_quadrant(robot, index = False):
    """Returns the nearest n*pi/2"""
    th = robot.th.value
    ob_th = th
    if is_near_angle(th, 0, threshold=math.pi/4):
        if index:
            ob_th = 1
        else:
            ob_th = 0
    elif is_near_angle(th, math.pi/2, threshold=math.pi/4):
        if index:
            ob_th = 0
        else:
            ob_th = math.pi/2
    elif is_near_angle(th, math.pi, threshold=math.pi/4):
        if index:
            ob_th = 3
        else:
            ob_th = math.pi
    elif is_near_angle(th, -math.pi/2, threshold=math.pi/4):
        if index:
            ob_th = 2
        else:
            ob_th = -math.pi/2
    return ob_th

def print_angle(th, deg = False):
    """turns any given angle in radians to a friendly format for printing, can also convert radians to deg"""
    if deg:
        return ('{}º'.format(math.degrees(th)))
    else:
        return ('{} * π rad'.format(th/math.pi))


class Solution:
    def rotateString(self, s: str, goal: str) -> bool:
        if s == goal:
            return True
        if len(s) != len(goal):
            return False
        s, goal = [*s], [*goal]
        print(s)
        for i in range(len(s)):
            # shift the current first letter
            s = s[1:] + [s[0]]
            if s == goal:
                return True
        return False

Algorithms - Python/String/Matching /796. Rotate String.py