"""
app/services/term_service.py

This module contains term-methods.
"""

from app.repositorys.term_repository import term_repository
from app.repositorys.lender_repository import lender_repository
from app.repositorys.debt_repository import debt_repository
from app.repositorys.debtor_repository import debtor_repository
from app.repositorys.address_repository import address_repository
from app.repositorys.parcels_repository import parcels_repository
from app.services.render_template_service import render
from app.models import Credor, Devedor, Divida, Endereco, TermoDivida, Parcela, Parcelamento
from datetime import date, timedelta

def adicionar_meses(data, n):
    # Adicionar n meses à data fornecida
    novo_ano = data.year + (data.month + n - 1) // 12
    novo_mes = (data.month + n - 1) % 12 + 1

    # Garantir que o dia não ultrapasse o último dia do novo mês
    ultimo_dia_do_mes = (data.replace(year=novo_ano, month=novo_mes, day=1) + timedelta(days=31)).replace(day=1) - timedelta(days=1)
    novo_dia = min(data.day, ultimo_dia_do_mes.day)

    nova_data = data.replace(year=novo_ano, month=novo_mes, day=novo_dia)
    return nova_data

class TermService:
    """
    Term Service
    """
    def get_term_by_id(self, id_termo:int):
        """
        Get data term by id
        """
        return term_repository.get_term_by_id(id_termo)

    def get_dados(self):
        """
        Get data term by id
        """
        return {'nome':'John Doe', 'cpf':'225.815.220-89', 'nacionalidade': 'Brasileira', 'estado_civil':'Solteiro', 'endereco': 'Avenida João Naves de Ávila, 2121, Santa Mônica, Uberlândia', 'num_contrato':'3127313921',
                'data_divida': '2023-09-16', 'data_renegociacao':'2023-09-17', 'produto':'Internet 5G', 'montante_valor':'500.00', 'montante_atrasado': '500.00'}

    def get_all_term_by_id(self, id_user: str):
        """
        Get data term by id
        """
        return term_repository.get_all_term_by_id(id_user)

    def post_term(self, termo:TermoDivida, parcela:Parcelamento):
        """
        Insert new data term
        """
        divida = debt_repository.get_debt_by_id(termo.id_divida)
        devedor = lender_repository.get_lerder_by_id(divida.id_devedor)
        credor = debtor_repository.get_debtor_by_id(divida.id_credor)
        end_cred = address_repository.get_address_by_id(credor.id_endereco)
        end_dev = address_repository.get_address_by_id(devedor.id_endereco)

        data = date(date.today().year,date.today().month,10)

        parcelasDb = []

        for i in range(parcela.qtd):
            parcelasDb.append(Parcela(id_parcela = parcels_repository.generate_id_parcela()+i,
                                id_termo = termo.id_termo,
                                valor_parcela = parcela.valor,
                                data_vencimento = adicionar_meses(data,i+1),
                                parcela_paga=False))

        render(devedor,credor, divida, end_dev, end_cred,termo)

        ret = term_repository.post_term(termo)

        for i in parcelasDb:
            parcels_repository.post_parcels(i)

        # return ret
        return True

    def update_term(self):
        """
        Update term data
        """
        return term_repository.update_term()

term_service = TermService()


nums = int(input())
lst = []
vals = input().split()

def maximumPairwise(numbers, values):

    for i in range(numbers):
        if numbers != len(values):
            return
        else:
            lst.append(int(values[i]))

    lst.sort()
    maxNumber = lst.pop()
    maxNumber2 = lst.pop()

    product = maxNumber * maxNumber2

    return print(product)

maximumPairwise(nums, vals)


import os
import random
import asyncio
import logging
import uuid
import itertools
from io import BytesIO
import openai
import replicate
import aiohttp
from PIL import Image

from .cfg import MAX_OUTPUT_IMAGES

# openai token setup
openai.organization = os.getenv("OPENAI_ORG")
openai.api_key = os.getenv("OPENAI_API_KEY")

# replicate model setup
stable_diffusion_model = replicate.models.get("stability-ai/stable-diffusion")
clip_caption_model = replicate.models.get("rmokady/clip_prefix_caption")

async def generate_images(images: list[Image]) -> list[Image]:
    """
    Abstraction of image generation pipeline.

    :param images: list of images to use.
    :return: extension of image set.
    """
    # captioning list of images
    async def caption_image(image: Image) -> str:  # noqa: F
        file = BytesIO()
        image.save(file, "PNG")
        prediction = replicate.predictions.create(
            version=clip_caption_model.versions.list()[0],
            input={
                "image": file,
                "model": "conceptual-captions",
                "use_beam_search": True,
            },
        )
        while prediction.status not in ["succeeded", "failed", "canceled"]:
            await asyncio.sleep(0.5)
            prediction.reload()
        return prediction.output

    clip_outputs = await asyncio.gather(*map(caption_image, images))
    logging.debug(f"CLIP outputs: {clip_outputs}")
    captions = "".join(pred.strip() + "\n" for pred in clip_outputs)

    # pass captions through gpt3
    gpt3_response = openai.Completion.create(
        model="text-davinci-002", prompt=captions, temperature=0.8, max_tokens=128
    )
    logging.debug(f"GPT3 response: {gpt3_response}")
    # limit to 2 prompts for cost reasons
    diffusion_prompts = list(
        filter(
            None,
            itertools.chain(
                *[p["text"].splitlines() for p in gpt3_response["choices"]]
            ),
        )
    )[:MAX_OUTPUT_IMAGES]
    logging.debug(f"Diffusion prompts: {diffusion_prompts}")

    async def download_image(session: aiohttp.ClientSession, url: str) -> Image:
        async with session.get(url) as response:
            raw = await response.read()
        return Image.open(BytesIO(raw))

    # generate new images using stable diffusion
    async def create_image(prompt: str) -> list[Image]:
        init_image = random.choice(images)
        file = BytesIO()
        init_image.save(file, "PNG")
        try:
            prediction = replicate.predictions.create(
                version=stable_diffusion_model.versions.list()[0],
                input={
                    "prompt": prompt,
                    "width": 512,
                    "height": 512,
                    "init_image": file,
                    "prompt_strength": 0.65,
                },
            )
            while prediction.status not in ["succeeded", "failed", "canceled"]:
                await asyncio.sleep(0.5)
                prediction.reload()
            async with aiohttp.ClientSession() as session:
                image_list = await asyncio.gather(
                    *[download_image(session, url) for url in prediction.output]
                )
            logging.debug(f"Created {len(image_list)} new images")
            return image_list  # noqa: E
        except Exception as e:
            logging.warning(e)
            return []

    new_images = await asyncio.gather(*map(create_image, diffusion_prompts))
    return list(itertools.chain(*new_images))

# script to use for testing
if __name__ == "__main__":
    test_dir = "app/photos"
    save_dir = "app/results"
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    original_images = []
    for path in os.listdir(test_dir):
        image = Image.open(f"{test_dir}/{path}")
        if image.mode != "RGB":
            image = image.convert(mode="RGB")
        original_images.append(image)
    output_images = asyncio.run(generate_images(original_images))
    for image in output_images:
        image.save(f"{save_dir}/{uuid.uuid4().hex}.png")


from flask import jsonify, make_response, request
import sqlalchemy
import models
from flask import Blueprint
person_bp = Blueprint('person', __name__)

@person_bp.route("/person")
def get_all_person():
    person_list = models.Person.query.all()
    return jsonify([models.row2dict(person) for person in person_list])

@person_bp.route("/person/<person_name>")
def get_person_by_name(person_name):
    # id is a primary key, so we'll have max 1 result row
    person = models.Person.query.filter_by(name=person_name).first()
    if person:
        return jsonify(models.row2dict(person))
    else:
        return make_response(jsonify({"code": 404, "msg": "Cannot find this person name."}), 404)

@person_bp.route("/person/<int:person_id>")
def get_person(person_id):
    # id is a primary key, so we'll have max 1 result row
    person = models.Person.query.filter_by(id=person_id).first()
    if person:
        return jsonify(models.row2dict(person))
    else:
        return make_response(jsonify({"code": 404, "msg": "Cannot find this person id."}), 404)

@person_bp.route("/person", methods={"PUT"})
def create_person():
    # get the name first, if no name then fail
    name = request.form.get("name")
    if not name:
        return make_response(jsonify({"code": 403,
                                      "msg": "Cannot put person. Missing mandatory fields."}), 403)
    person_id = request.form.get("id")
    if not person_id:
        p = models.Person(name=name)
    else:
        p = models.Person(id=person_id, name=name)

    models.db.session.add(p)
    try:
        models.db.session.commit()
    except sqlalchemy.exc.SQLAlchemyError as e:
        error = "Cannot put person. "
        print(person_bp.config.get("DEBUG"))
        if person_bp.config.get("DEBUG"):
            error += str(e)
        return make_response(jsonify({"code": 404, "msg": error}), 404)
    return jsonify({"code": 200, "msg": "success"})

@person_bp.route("/person/update", methods={"PUT"})
def update_person():
    # get the movie name first, if no movie name then fail
    person_name = request.form.get("name")
    person_id = request.form.get("id")
    if not person_id:
        return make_response(jsonify({"code": 403,
                                      "msg": "Cannot update person. Missing mandatory fields."}), 403)

    person = models.Person.query.filter_by(id=person_id).first()
    if not person:
        return make_response(jsonify({"code": 404, "msg": "Cannot find this person id."}), 404)

    person.name = person_name

    try:
        models.db.session.commit()
    except sqlalchemy.exc.SQLAlchemyError as e:
        error = "Cannot update person. "
        print(person_bp.config.get("DEBUG"))
        if person_bp.config.get("DEBUG"):
            error += str(e)
        return make_response(jsonify({"code": 404, "msg": error}), 404)
    return jsonify({"code": 200, "msg": "success"})

@person_bp.route("/person/delete", methods={"PUT"})
def delete_person():
    person_id = request.form.get("person_id")
    # id is a primary key, so we'll have max 1 result row
    person = models.Person.query.filter_by(id=person_id).first()
    if person:
        models.db.session.delete(person)
        try:
            models.db.session.commit()
        except sqlalchemy.exc.SQLAlchemyError as e:
            error = "Cannot delete person. "
            print(person_bp.config.get("DEBUG"))
            if person_bp.config.get("DEBUG"):
                error += str(e)
            return make_response(jsonify({"code": 404, "msg": error}), 404)
        return jsonify({"code": 200, "msg": "person is successfully deleted"})
    else:
        return make_response(jsonify({"code": 404, "msg": "Cannot find this person id."}), 404)


#!/usr/bin/env python_cyecca
import sys
import time
import numpy as np

import rclpy
from rclpy.node import Node
from rclpy.parameter import Parameter
from rclpy.time import Time
import rclpy.clock
from rclpy.lifecycle import LifecycleNode

from rosgraph_msgs.msg import Clock
from sensor_msgs.msg import Imu, NavSatFix, MagneticField
from actuator_msgs.msg import Actuators
from nav_msgs.msg import Odometry

import cyecca
import casadi as ca

class QuadEstimator(LifecycleNode):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        # parameters
        self.declare_parameter("dt", 0.005)
        self.set_parameters([
            Parameter("use_sim_time", Parameter.Type.BOOL, True),
        ])

        # actuators publication
        self.pub_odom = self.create_publisher(Odometry, "/odom", 10)
        self.dt = self.get_parameter("dt").value
        self.create_timer(
            timer_period_sec=self.dt,
            callback=self.update_odom,
            clock=self.get_clock())
        self.msg_odom = Odometry()
        self.msg_odom.header.frame_id = 'map'
        self.msg_odom.child_frame_id = 'base_link'

    def update_odom(self):
        t = self.get_clock().now().nanoseconds*1e-9
        self.msg_odom.header.stamp = self.get_clock().now().to_msg()
        self.msg_odom.pose.pose.position.x = np.sin(t)
        self.msg_odom.pose.pose.position.y = np.cos(t)
        self.msg_odom.pose.pose.position.z = 1.0
        self.msg_odom.pose.pose.orientation.x = 0.0
        self.msg_odom.pose.pose.orientation.y = 0.0
        self.msg_odom.pose.pose.orientation.z = 0.0
        self.msg_odom.pose.pose.orientation.w = 1.0
        self.pub_odom.publish(self.msg_odom)

def main(args=None):
    try:
        rclpy.init(args=args)
        estimator = QuadEstimator(node_name="quad_estimator")
        rclpy.spin(estimator)
    except KeyboardInterrupt:
        pass
    estimator.destroy_node()
    rclpy.try_shutdown()

if __name__ == "__main__":
    main()


"""
Module to implement a plugin that ensures that all Unordered List Items
start at the beginning of the line.
"""
from typing import List, cast

from pymarkdown.general.parser_helper import ParserHelper
from pymarkdown.plugin_manager.plugin_details import PluginDetails
from pymarkdown.plugin_manager.plugin_scan_context import PluginScanContext
from pymarkdown.plugin_manager.rule_plugin import RulePlugin
from pymarkdown.tokens.block_quote_markdown_token import BlockQuoteMarkdownToken
from pymarkdown.tokens.list_start_markdown_token import ListStartMarkdownToken
from pymarkdown.tokens.markdown_token import MarkdownToken

class RuleMd006(RulePlugin):
    """
    Class to implement a plugin that ensures that all Unordered List Items
    start at the beginning of the line.
    """

    def __init__(self) -> None:
        super().__init__()
        self.__token_stack: List[MarkdownToken] = []

    def get_details(self) -> PluginDetails:
        """
        Get the details for the plugin.
        """
        return PluginDetails(
            plugin_name="ul-start-left",
            plugin_id="MD006",
            plugin_enabled_by_default=False,
            plugin_description="Consider starting bulleted lists at the beginning of the line",
            plugin_version="0.5.0",
            plugin_interface_version=1,
            plugin_url="https://github.com/jackdewinter/pymarkdown/blob/main/docs/rules/rule_md006.md",
        )

    def starting_new_file(self) -> None:
        """
        Event that the a new file to be scanned is starting.
        """
        self.__token_stack = []

    def __calculate_expected_indent(self) -> int:
        expected_indent = 0
        if len(self.__token_stack) > 1:
            if self.__token_stack[-2].is_block_quote_start:
                block_quote_token = cast(
                    BlockQuoteMarkdownToken, self.__token_stack[-2]
                )
                assert block_quote_token.bleading_spaces is not None
                split_spaces = block_quote_token.bleading_spaces.split(
                    ParserHelper.newline_character
                )
                expected_indent = len(split_spaces[0]) + (
                    block_quote_token.column_number - 1
                )
            else:
                list_token = cast(ListStartMarkdownToken, self.__token_stack[-2])
                expected_indent = list_token.indent_level
        return expected_indent

    def next_token(self, context: PluginScanContext, token: MarkdownToken) -> None:
        """
        Event that a new token is being processed.
        """
        if token.is_list_start or token.is_block_quote_start:
            self.__token_stack.append(token)
            if token.is_unordered_list_start:
                expected_indent = self.__calculate_expected_indent()
                if token.column_number != (1 + expected_indent):
                    self.report_next_token_error(context, token)
        elif token.is_list_end or token.is_block_quote_end:
            del self.__token_stack[-1]
        elif token.is_new_list_item:
            if self.__token_stack[-1].is_unordered_list_start:
                expected_indent = self.__calculate_expected_indent()
                if token.column_number != (1 + expected_indent):
                    self.report_next_token_error(context, token)


import os
import sys
from uuid import uuid1
import json
import click
import requests
import traceback

from flask import Response

from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
from flow_client.flow_cli.utils.cli_utils import check_abs_path

def prettify(response, verbose=True):
    if verbose:
        if isinstance(response, requests.models.Response):
            try:
                response_dict = response.json()
            except TypeError:
                response_dict = response.json
        else:
            response_dict = response
        try:
            click.echo(json.dumps(response_dict, indent=4, ensure_ascii=False))
        except TypeError:
            click.echo(json.dumps(response_dict.json, indent=4, ensure_ascii=False))
        click.echo('')
    return response

# def dataset_upload(url, config_data):
#     file = config_data['file']
#     config_temp_path = f"/tmp/{uuid1()}"
#     with open(config_temp_path, "w", encoding="utf-8") as fp:
#         json.dump(config_data, fp)
#     with open(file, 'rb') as fp:
#         data = MultipartEncoder(
#             fields={'file': (file, fp, 'application/octet-stream')}
#         )
#         tag = [0]

#         def read_callback(monitor):
#             if config_data.get('verbose') == 1:
#                 sys.stdout.write("\r UPLOADING:{0}{1}".format("|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len)))
#                 sys.stdout.flush()
#                 if monitor.bytes_read / monitor.len == 1:
#                     tag[0] += 1
#                     if tag[0] == 2:
#                         sys.stdout.write('\n')

#         data = MultipartEncoderMonitor(data, read_callback)
#         return access_server('post', url, 'data/upload', json_data=None, data=data,
#                         params=config_data, headers={'Content-Type': data.content_type})
def dataset_upload(file_name, params, url):
    with open(f"/tmp/{file_name}", 'rb') as fp:
        data = MultipartEncoder(
            fields={'file': (file_name, fp, 'application/octet-stream')}
        )
        tag = [0]
        def read_callback(monitor):
            if params.get('verbose') == 1:
                sys.stdout.write("\r UPLOADING:{0}{1}".format("|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len)))
                sys.stdout.flush()
                if monitor.bytes_read / monitor.len == 1:
                    tag[0] += 1
                    if tag[0] == 2:
                        sys.stdout.write('\n')
        data = MultipartEncoderMonitor(data, read_callback)
        resp = access_server(
            'post',
            url,
            'data/upload',
            data=data,
            params=params,
            headers={"Content-Type": data.content_type}
        )
        return resp

def access_server(method, url, postfix, json_data=None, echo=True, **kwargs):
    try:
        url = "/".join([url, postfix])
        response = {}
        if method == 'get':
            response = requests.get(url=url, json=json_data, **kwargs)
        elif method == 'post':
            response = requests.post(url=url, json=json_data, **kwargs)
        if echo:
            prettify(response)
            return response
        else:
            return response
    except Exception as e:
        exc_type, exc_value, exc_traceback_obj = sys.exc_info()
        response = {'retcode': 100, 'retmsg': str(e),
                    'traceback': traceback.format_exception(exc_type, exc_value, exc_traceback_obj)}
        if 'Connection refused' in str(e):
            response['retmsg'] = 'Connection refused. Please check if the fate flow service is started'
            del response['traceback']
        if 'Connection aborted' in str(e):
            response['retmsg'] = 'Connection aborted. Please make sure that the address of fate flow server ' \
                                    'is configured correctly. The configuration file path is: ' \
                                    '{}.'.format(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                                os.pardir, os.pardir, 'settings.yaml')))
            del response['traceback']
        if echo:
            prettify(response)
            return ""
        else:
            return Response(json.dumps(response), status=500, mimetype='application/json')


import time
import numpy as np
import pandas as pd
import elephant as el
import pyqtgraph as pg

from scipy import stats
from multiprocessing import Pool
from scipy.spatial.distance import cdist
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from PyQt5 import QtWidgets, QtCore

from swan.gui.parameter_input_dialog_ui import ParameterInputDialogUI

def p2p_amplitude(waves):
    return np.amax(waves[:, 10:], axis=1) - np.amin(waves[:, 10:], axis=1)

def asymmetry(waves):
    return np.amax(waves[:, 10:], axis=1) / np.amin(waves[:, 10:], axis=1)

def square_sum(waves):
    return np.sum(np.einsum('ij, ij -> ij', waves, waves), axis=1)

def diff(waves):
    return np.gradient(waves, axis=1).mean(axis=0)

def diffdiff(waves):
    return np.gradient(np.gradient(waves, axis=1), axis=1).mean(axis=0)

def isi(train):
    return el.statistics.isi(train)

def lv(train):
    intervals = isi(train)
    return el.statistics.lv(intervals)

def cv2(train):
    intervals = isi(train)
    return el.statistics.cv2(intervals)

def fr(train):
    return el.statistics.mean_firing_rate(train)

def isih(train, bin_max, bin_step):
    intervals = isi(train)
    hist, bins = np.histogram(intervals, bins=range(0, bin_max, bin_step), density=True)
    return hist

def run_kmeans_solver(data, n_clusters, n_init=10):
    solver = KMeans(n_clusters=n_clusters, n_init=n_init)
    solver.fit(data)
    clusters = solver.cluster_centers_
    labels = solver.predict(data)
    return solver, clusters, labels

def run_kmeans_solver_no_prediction(data, n_clusters, n_init=10):
    solver = KMeans(n_clusters=n_clusters, n_init=n_init)
    solver.fit(data)
    return solver, n_clusters

def generate_feature_vectors(parent_class, feature_dictionary, additional_dictionary):
    feature_vectors = []

    for u, (waves, train, reduced_mean, mean) in \
            enumerate(zip(parent_class.all_waveforms,
                          parent_class.all_spiketrains,
                          parent_class.reduced_means,
                          parent_class.mean_waveforms)):
        feature_vectors.append([])
        if feature_dictionary['p2p amplitude']:
            feature_vectors[u].extend(list(stats.describe(p2p_amplitude(waves)))[2:])
        if feature_dictionary['asymmetry']:
            feature_vectors[u].extend(list(stats.describe(asymmetry(waves)))[2:])
        if feature_dictionary['square sum']:
            feature_vectors[u].extend(list(stats.describe(square_sum(waves)))[2:])
        if feature_dictionary['spike width']:
            feature_vectors[u].extend(list(stats.describe(spike_width(waves)))[2:])
        if feature_dictionary['cv2']:
            feature_vectors[u].append((cv2(train)))
        if feature_dictionary['lv']:
            feature_vectors[u].append((lv(train)))
        if feature_dictionary['firing rate']:
            feature_vectors[u].append((fr(train)))
        if feature_dictionary['mean']:
            feature_vectors[u].extend(mean)
        if feature_dictionary['reduced mean']:
            feature_vectors[u].extend(reduced_mean)
        if feature_dictionary['first derivative']:
            feature_vectors[u].extend(diff(waves))
        if feature_dictionary['second derivative']:
            feature_vectors[u].extend(diffdiff(waves))
        if feature_dictionary['described isi']:
            feature_vectors[u].extend(list(stats.describe(isih(train,
                                                               additional_dictionary['bin max'],
                                                               additional_dictionary['bin step']
                                                               )
                                                          )
                                           )[2:]
                                      )
        if feature_dictionary['isi']:
            feature_vectors[u].extend(isih(train,
                                           additional_dictionary['bin max'],
                                           additional_dictionary['bin step']
                                           )
                                      )

    feature_vectors = np.array(feature_vectors)
    feature_vectors = (feature_vectors - feature_vectors.mean(axis=0, keepdims=True)) / feature_vectors.std(axis=0)

    return np.nan_to_num(feature_vectors)

def get_session_ids(blocks):
    session_ids = []
    for b, block in enumerate(blocks):
        units = block.channel_indexes[0].units
        for u, unit in enumerate(units):
            if 'noise' not in unit.description.split() and 'unclassified' not in unit.description.split():
                session_ids.append(b)

    return session_ids

def get_real_unit_ids(blocks):
    unit_ids = []
    for block in blocks:
        units = block.channel_indexes[0].units
        for u, unit in enumerate(units):
            if 'noise' not in unit.description.split() and 'unclassified' not in unit.description.split():
                unit_ids.append(u)

    return unit_ids

def get_mean_waveforms(blocks):
    mean_waveforms = []
    for block in blocks:
        units = block.channel_indexes[0].units
        for unit in units:
            if 'noise' not in unit.description.split() and 'unclassified' not in unit.description.split():
                waves = unit.spiketrains[0].waveforms.magnitude[:, 0, :]
                waves = waves - waves.mean(axis=1, keepdims=True)
                mean_waveforms.append(waves.mean(axis=0))

    return mean_waveforms

def get_all_waveforms(blocks):
    waveforms = []
    for block in blocks:
        units = block.channel_indexes[0].units
        for unit in units:
            if 'noise' not in unit.description.split() and 'unclassified' not in unit.description.split():
                waves = unit.spiketrains[0].waveforms.magnitude[:, 0, :]
                waves = waves - waves.mean(axis=1, keepdims=True)
                waveforms.append(waves)
    return waveforms

def get_all_spiketrains(blocks):
    spiketrains = []
    for block in blocks:
        units = block.channel_indexes[0].units
        for unit in units:
            if 'noise' not in unit.description.split() and 'unclassified' not in unit.description.split():
                train = unit.spiketrains[0].times.magnitude
                spiketrains.append(train)
    return spiketrains

def get_time_stamps(blocks):
    all_time_stamps = []
    for block in blocks:
        units = block.channel_indexes[0].units
        for unit in units:
            if 'noise' not in unit.description.split() and 'unclassified' not in unit.description.split():
                all_time_stamps.append(block.rec_datetime)

    corrected_time_stamps = []
    for ts in all_time_stamps:
        cts = time.mktime(ts.timetuple())
        while cts in corrected_time_stamps:
            cts += 1.
        corrected_time_stamps.append(cts)

    time_stamps = [ts - corrected_time_stamps[0] for ts in corrected_time_stamps]

    return time_stamps

def spike_width(waves):
    return np.argmax(waves[:, 10:], axis=1) - np.argmin(waves[:, 10:], axis=1)

class SwanImplementation:

    # noinspection PyArgumentList
    def __init__(self, neodata, parent=None):

        self.blocks = neodata.blocks
        self.parent = parent

        self.feature_dict = {'mean': True,

                             'reduced mean': True,

                             'first derivative': True,
                             'second derivative': True,

                             'p2p amplitude': True,
                             'asymmetry': True,
                             'square sum': True,
                             'spike width': True,

                             'cv2': True,
                             'lv': True,
                             'firing rate': True,

                             'isi': True,
                             'described isi': True,

                             'clusters': 2,

                             'time split': 10
                             }

        self.additional_dict = {'reduced mean dims': 5,
                                'bin max': 15000,
                                'bin step': 60,
                                'max_clusters': 20}

        self.mean_waveforms = get_mean_waveforms(self.blocks)
        self.all_waveforms = get_all_waveforms(self.blocks)
        self.all_spiketrains = get_all_spiketrains(self.blocks)
        self.timestamps = get_time_stamps(self.blocks)
        self.unit_ids = get_real_unit_ids(self.blocks)
        self.session_ids = get_session_ids(self.blocks)

        self.pca = PCA(n_components=self.additional_dict['reduced mean dims'])
        self.pca.fit(np.vstack(self.all_waveforms))
        self.reduced_means = self.pca.transform(self.mean_waveforms)

        QtWidgets.QApplication.restoreOverrideCursor()
        self.feature_dict, solver, okay = ParameterInputDialog(parent=self.parent, algorithm=self).exec_()
        QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)

        if okay:
            self.feature_vectors = generate_feature_vectors(self, self.feature_dict, self.additional_dict)

            if solver is not None:
                self.solver = solver

                clusters = self.solver.cluster_centers_
                labels = self.solver.predict(self.feature_vectors)
            else:
                self.solver, clusters, labels = run_kmeans_solver(self.feature_vectors,
                                                                  self.feature_dict['clusters'], n_init=100)

            labels = self.remove_duplicates(self.feature_vectors, self.session_ids, self.unit_ids, labels, clusters)

            if self.feature_dict['time split'] is not None:
                labels = self.time_splits(days=self.feature_dict['time split'],
                                          labels=labels, timestamps=self.timestamps)

            self.result = self.generate_result_dataframe(self.session_ids, self.unit_ids, labels)
        else:
            self.result = pd.DataFrame({})

    def time_splits(self, days, labels, timestamps):
        threshold = days * 86400.0

        labels_dict = self.generate_labels_dictionary(labels, timestamps)

        while self.has_time_conflicts(labels_dict, threshold):
            for label in sorted(labels_dict.keys()):
                timestamps = labels_dict[label]
                if timestamps.size > 1:
                    time_diffs = timestamps[1:] - timestamps[:-1]
                    if np.any(time_diffs > threshold):
                        cuts = np.where(time_diffs > threshold)[0]
                        for cut in cuts:
                            max_label = max(labels_dict.keys())
                            labels_dict[max_label + 1] = timestamps[cut + 1:]
                            labels_dict[label] = timestamps[:cut + 1]
                            break
                else:
                    continue
        return self.recreate_labels_list(labels_dict)

    @staticmethod
    def has_time_conflicts(dictionary, threshold):
        for label in dictionary.keys():
            timestamps = dictionary[label]
            if timestamps.size > 1 and np.any((timestamps[1:] - timestamps[:-1]) > threshold):
                return True
        else:
            return False

    @staticmethod
    def recreate_labels_list(dictionary):
        outstamps, outlabels = [], []
        for key in dictionary.keys():
            vals = dictionary[key]
            outstamps.extend(vals.tolist())
            outlabels.extend([key] * vals.size)

        outstamps, outlabels = np.array(outstamps), np.array(outlabels)
        return outlabels[np.argsort(outstamps)].tolist()

    def remove_duplicates(self, feature_vectors, session_ids, unit_ids, labels, clusters):
        def get_duplicates(sessions, units, unit_labels):
            df = pd.DataFrame({
                'session': sessions,
                'unit': units,
                'label': unit_labels
            })

            return df.sort_values(by=['label', 'session']).loc[
                df[['session', 'label']].duplicated(keep=False)
            ]

        duplicates = get_duplicates(session_ids, unit_ids, labels)

        visited = []

        count = 1
        while not duplicates.empty:
            clusters = clusters
            labels = labels

            maximum_distances = self.compute_intracluster_maximum_distance(clusters=clusters,
                                                                           feature_vectors=feature_vectors,
                                                                           labels=labels)
            cross_distances = cdist(feature_vectors, clusters)

            for label in np.unique(duplicates.label):
                cluster_df = duplicates.loc[duplicates.label == label]
                for session in np.unique(cluster_df.session):
                    session_duplicates = cluster_df.loc[cluster_df.session == session]
                    conflicting_unit_distances = cross_distances[session_duplicates.index, label]

                    discard_unit_position = np.argmax(conflicting_unit_distances)
                    discard_unit_id = session_duplicates.unit[session_duplicates.index[discard_unit_position]]

                    visited.append((session, discard_unit_id, label))

                    other_distances = cross_distances[discard_unit_position]
                    differences = other_distances - maximum_distances
                    candidates = []

                    for d, difference in enumerate(differences):
                        if d == label:
                            candidates.append(0.0)
                        elif difference < 0.0:
                            candidates.append(difference)
                        else:
                            candidates.append(0.0)

                    while np.any(np.array(candidates) < 0.0):
                        new_cluster_position = np.argmin(candidates)
                        # noinspection PyTypeChecker
                        candidates[new_cluster_position] = 0.0
                        if (session, discard_unit_id, new_cluster_position) not in visited:
                            labels = self.assign_new_cluster(sessions=session_ids,
                                                             units=unit_ids,
                                                             labels=labels,
                                                             session_id=session,
                                                             unit_id=discard_unit_id,
                                                             new_cluster=new_cluster_position)
                            break
                    else:
                        labels = self.create_new_cluster(sessions=session_ids,
                                                         units=unit_ids,
                                                         labels=labels,
                                                         session_id=session,
                                                         unit_id=discard_unit_id)

                    clusters = self.recompute(feature_vectors, labels)
                    break
                break

            duplicates = get_duplicates(session_ids, unit_ids, labels)
            count += 1

        return labels

    @staticmethod
    def compute_intracluster_maximum_distance(clusters, feature_vectors, labels):
        maximum_distances = []
        for c, center in enumerate(clusters):
            members = feature_vectors[np.where(labels == c)[0]]
            max_dist = 0.0
            for member in members:
                distance = np.linalg.norm(member - center)
                if distance > max_dist:
                    max_dist = distance
            maximum_distances.append(max_dist)

        return np.array(maximum_distances)

    @staticmethod
    def assign_new_cluster(sessions, units, labels, session_id, unit_id, new_cluster):
        pair_list = [(s, u) for s, u in zip(sessions, units)]
        position = [pos for pos, val in enumerate(pair_list) if val == (session_id, unit_id)]

        labels[position] = new_cluster

        return labels

    @staticmethod
    def create_new_cluster(sessions, units, labels, session_id, unit_id):
        pair_list = [(s, u) for s, u in zip(sessions, units)]
        position = [pos for pos, val in enumerate(pair_list) if val == (session_id, unit_id)]

        max_label = np.amax(labels)
        labels[position] = max_label + 1

        return labels

    @staticmethod
    def recompute(feature_vectors, labels):
        cluster_labels = np.unique(labels)
        total_clusters = cluster_labels.size
        label_list = labels

        cluster_centers = np.zeros((total_clusters, feature_vectors.shape[1]))

        for idx, label in enumerate(cluster_labels):
            vectors = feature_vectors[label_list == label]
            cluster_centers[idx] = vectors.mean(axis=0)

        return cluster_centers

    @staticmethod
    def generate_result_dataframe(session_ids, unit_ids, labels):
        dataframe = pd.DataFrame({'session': session_ids,
                                  'unit': unit_ids,
                                  'label': labels
                                  })

        return dataframe.sort_values(by=['session', 'unit'])

    @staticmethod
    def generate_labels_dictionary(labels, timestamps):
        labels = np.array(labels)
        timestamps = np.array(timestamps)

        labels_dict = {}
        for label in np.unique(labels):
            labels_dict[label] = timestamps[np.where(labels == label)[0]]

        return labels_dict

# noinspection PyArgumentList
class ParameterInputDialog(QtWidgets.QDialog):

    def __init__(self, parent=None, algorithm=None):
        # noinspection PyArgumentList
        QtWidgets.QDialog.__init__(self, parent=parent)

        self.setWindowTitle("Automatic algorithm: SWAN Implementation")
        self.algorithm = algorithm

        self.interface = ParameterInputDialogUI(self)
        self.plot_widget = self.interface.elbow_curve_plot.pg_canvas

        self.interface.clusters.textChanged.connect(self.update_plot)

        self._solvers = []
        self._clusters = []
        self._inertias = []
        self.chosen_solver = None

        self.values_dict = {}
        self.final_state = False

        self.spot_size = 10
        self.default_pen = pg.mkPen(color='w',
                                    width=2,
                                    style=QtCore.Qt.DotLine)
        self.default_symbol_pen = pg.mkPen(color='b')
        self.default_brush = pg.mkBrush((0, 255, 0, 128))
        self.default_symbol = 'o'

        self.current_clusters_value = 2

    def on_confirm(self):
        self.final_state = True
        self.accept()

    def on_cancel(self):
        self.final_state = False
        self.reject()

    # @QtCore.pyqtSlot(object, object)
    # def on_plot_clicked(self, data_item, ev):
    #     if ev.button() == 1:
    #         x_pos = ev.pos().x()
    #         closest_val = self._get_closest_val(x_pos)
    #         if closest_val is not None:
    #             self.interface.clusters.setValue(closest_val)
    #
    # @staticmethod
    # def _get_closest_val(input_value):
    #     lower = np.floor(input_value)
    #     lower_diff = input_value - lower
    #     upper = np.ceil(input_value)
    #     upper_diff = upper - input_value
    #
    #     pprint({"lower": lower,
    #             "lower_diff": lower_diff,
    #             "upper": upper,
    #             "upper_diff": upper_diff})
    #
    #     if lower_diff < upper_diff and lower_diff < 0.1:
    #         return lower
    #     elif upper_diff < lower_diff and upper_diff < 0.1:
    #         return upper
    #     else:
    #         return None

    @QtCore.pyqtSlot()
    def update_plot(self):
        clusters = self._clusters
        inertias = self._inertias
        sizes = []
        pens = []
        brushes = []
        symbols = []

        self.current_clusters_value = int(self.interface.clusters.value())

        for cluster in clusters:
            sizes.append(self.spot_size)
            symbols.append(self.default_symbol)
            if cluster == self.current_clusters_value:
                pens.append(pg.mkPen('r'))
                brushes.append(pg.mkBrush((255, 0, 0, 128)))
            else:
                pens.append(self.default_symbol_pen)
                brushes.append(self.default_brush)

        self._plot(clusters, inertias,
                   symbolPen=pens,
                   symbolSize=sizes,
                   symbol=symbols,
                   symbolBrush=brushes,
                   pxMode=True,
                   clear=True)

    def plot(self):
        solvers = []
        clusters = []
        inertias = []

        QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
        self.update_values_dict()
        feature_vectors = generate_feature_vectors(self.algorithm, self.values_dict, self.algorithm.additional_dict)

        with Pool(processes=None) as pool:
            results = pool.starmap_async(run_kmeans_solver_no_prediction,
                                         [(feature_vectors, n_clusters, 100)
                                          for n_clusters in range(2, feature_vectors.shape[0])])

            for result in results.get():
                solver, n_clusters = result[0], result[1]
                solvers.append(solver)
                clusters.append(n_clusters)
                inertias.append(solver.inertia_)

        self._solvers = solvers
        self._clusters = clusters
        self._inertias = inertias

        self.update_plot()

        QtWidgets.QApplication.restoreOverrideCursor()

    def _plot(self, x, y, *args, **kwargs):
        self.interface.elbow_curve_plot.pg_canvas.clear()
        plot_item = self.interface.elbow_curve_plot.pg_canvas.plot(x, y, *args, **kwargs)
        # plot_item.sigClicked.connect(self.on_plot_clicked)

    def update_values_dict(self):
        output_dict = {}
        for key in self.interface.key_names:
            output_dict[key] = self.interface.checkboxes[key].isChecked()
        output_dict["clusters"] = int(self.interface.clusters.value())
        if self.interface.time_split_box.isChecked():
            output_dict["time split"] = int(self.interface.time_split_threshold.value())
        else:
            output_dict["time split"] = None
        self.values_dict = output_dict

    def exec_(self):
        QtWidgets.QDialog.exec_(self)
        self.update_values_dict()
        if self._solvers is not None:
            self.chosen_solver = self._solvers[self._clusters.index(self.current_clusters_value)]
        return self.values_dict, self.chosen_solver, self.final_state

def calculate_mapping_bu(virtual_unit_map, data, storage):
    """
    Calculates a mapping for the units based on features like distance.

    The units will be compared pare-wise and sequential.

    **Arguments**

        *data* (:class:`src.neodata.NeoData`):
            This object is needed to get the data
            which will be used to compare the units.

    """
    wave_length = data.get_wave_length()

    for i in range(len(data.blocks) - 1):
        sessions = np.zeros((sum(data.total_units_per_block), 2, wave_length))

        for j, val in enumerate(storage.get_map().mapping[i]):
            if val != 0:
                runit = virtual_unit_map.get_realunit(i, j, data)
                avg = data.get_data("average", runit)
                sessions[j][0] = avg / np.max(avg)
            else:
                sessions[j][0] = np.zeros(wave_length)

        for j, val in enumerate(storage.get_map().mapping[i + 1]):
            if val != 0:
                runit = virtual_unit_map.get_realunit(i + 1, j, data)
                avg = data.get_data("average", runit)
                sessions[j][1] = avg / np.max(avg)
            else:
                sessions[j][1] = np.zeros(wave_length)

        distances = cdist(sessions[:, 0], sessions[:, 1], metric='euclidean')
        threshold = np.mean(distances[distances > 0])
        print(threshold)
        extend = 0
        exclude = []
        for j, val in enumerate(storage.get_map().mapping[i + 1]):
            # print("Executing this in session {}".format(i))
            # print("J: {}, Val: {}".format(j, val))

            if val != 0:
                print(distances[j])

                min_arg = np.argmin(distances[j])
                # print("Min Arg: {}".format(min_arg))

                if min_arg == j:
                    pass
                elif distances[j, min_arg] < threshold and (j, min_arg) not in exclude:
                    # print("Swapping {} and {}".format(j, min_arg))
                    virtual_unit_map.swap(i + 1, j, min_arg)
                    exclude.append((j, min_arg))
                    exclude.append((min_arg, j))
                elif distances[j, min_arg] >= threshold:
                    # print("Swapping {} and {}".format(j, data.total_units_per_block[i + 1] + extend))
                    virtual_unit_map.swap(i + 1, j, data.total_units_per_block[i + 1] + extend)
                    extend += 1
                    print(extend)
                else:
                    print("Logic flawed, check again!")
        print(exclude)
    storage.change_map()

def swan_implementation(virtual_unit_map, data, storage):
    swaps = 0
    # Retrieve mapping from base
    backup_mapping = np.array(storage.get_map().mapping.copy()).T
    pre_mapping = backup_mapping.copy()
    # loop over columns/sessions
    for s in range(pre_mapping.shape[1]):
        # initialize array of mean waveform max amplitudes
        averages = np.zeros((pre_mapping.shape[0]))

        # loop over rows/units
        for j in range(len(averages)):
            # ignore zeros in the map (correspond to non-existent units)
            if pre_mapping[j][s] != 0:
                # Retrieve unit
                runit = virtual_unit_map.get_realunit(s, j, data)
                # Retrieve mean waveform for the unit
                wave = data.get_data("average", runit)
                # Store max amplitude of mean waveform
                averages[j] = np.amax(np.abs(wave))

        #
        # Sort the units in each session by max amplitude of mean waveform
        # and swap the corresponding units in the base.
        #

        # loop over rows/units
        for i in range(len(averages)):
            # retrieve index of max value in session
            max_index = np.argmax(averages[i:]) + i
            if max_index != i:
                virtual_unit_map.swap(s, i, max_index)
                tmp = averages[max_index]
                averages[max_index] = averages[i]
                averages[i] = tmp
                swaps += 1

    storage.change_map()
    # Refresh the mapping using the values in base
    mapping = np.array(storage.get_map().mapping.copy()).T

    # Initialize an array shaped like "mapping" and set all values to np.nan
    history = np.zeros_like(mapping, dtype=np.float)
    history[history == 0.0] = None
    # print("Initial Mapping: {}".format(mapping))

    # Loop over all columns/sessions except the last one
    for i in range(mapping.shape[1] - 1):
        # Loop over all rows/units
        # print("\nFirst loop: {}".format(i))
        for j in range(mapping.shape[0]):
            # ignore zeros in the map (correspond to non-existent units)
            # print("\tSecond Loop: {}".format(j))
            if mapping[j][i] > 0:  # to avoid errors in float comparison
                # Choose (j, i)th unit as the actor and obtain it's mean waveform
                runit_actor = virtual_unit_map.get_realunit(i, j, data)
                actor_wave = data.get_data("average", runit_actor)
                actor_rp = data.get_data("rate profile", runit_actor)

                # Initialize matrix to store distances to the actor unit
                distances_waves = np.zeros((mapping.shape[0], mapping.shape[1]))
                distances_rp = np.zeros((mapping.shape[0], mapping.shape[1]))

                # Loop over all columns/sessions again
                for k in range(mapping.shape[1]):
                    # Loop over all rows/units again
                    # print("\t\tThird Loop: {}".format(k))
                    for l in range(mapping.shape[0]):
                        if mapping[l][k] > 0:
                            # Choose (l, k)th units as the support and obtain it's mean waveform
                            # print("\t\t\tFourth Loop {}\n".format(l))
                            runit_support = virtual_unit_map.get_realunit(k, l, data)
                            support_wave = data.get_data("average", runit_support)
                            support_rp = data.get_data("rate profile", runit_support)
                            #
                            # Calculate and store the Euclidean distance to
                            # support from actor (multiplied by a distance
                            # factor)
                            #
                            distances_waves[l][k] = np.linalg.norm(
                                np.subtract(actor_wave, support_wave))  # * np.exp(abs(k-i))
                            distances_rp[l][k] = np.linalg.norm(np.subtract(actor_rp, support_rp))
                #                            if mapping[l][k] > 0:
                #                                history[l][k] = distances[l][k]

                # print("Distances:\n {}\n".format(distances))

                #
                # Extract the dataset corresponding to all distance measures
                # with non-zero mapping values and calculate the reject threshold
                #

                distances = distances_waves  # + distances_rp

                threshold_dataset = distances[mapping > 0]
                dataset_reject_threshold = np.mean(threshold_dataset)
                # dataset_std = np.std(threshold_dataset)
                # threshold_range_L = threshold_mean + 0.5 * threshold_std
                # dataset_reject_threshold = dataset_mean + 0.5 * dataset_std

                # print("Threshold: {}\n".format(dataset_reject_threshold))

                #
                # The following loop is the main part of the mapping algorithm,
                # which uses the distances between units to rearrange (swap) them.
                #

                # Generate loop range for the loop
                if mapping.shape[1] < 10:
                    loop_range = range(mapping.shape[1])
                else:
                    loop_range = [x for x in range(i + 9) if (i + x) <= mapping.shape[1]]

                # Start looping over sessions/columns
                for k in loop_range:
                    # Find the argument of the minimum in the kth column
                    min_arg = np.argmin(distances[:, k])
                    # print("i: {}, j: {}, k: {}, min_arg: {}\n".format(i, j, k, min_arg))
                    if min_arg == j:
                        # Do nothing if the minimum argument is the same as the row we're looping over
                        pass
                    elif distances[min_arg][k] <= dataset_reject_threshold:
                        # If the distance between the two waveforms falls within
                        # the threshold, we swap the two positions in the kth col
                        if np.isnan(history[min_arg][k]):
                            # If the unit has not been swapped earlier.
                            history[j][k] = distances[min_arg][k]
                            virtual_unit_map.swap(k, min_arg, j)
                            # mapping = backup_mapping.copy()
                            swaps += 1
                        else:
                            # If the unit has been swapped earlier, swap only if
                            # the new distance is less than the old distance
                            prev_dist = history[min_arg][k]
                            curr_dist = distances[min_arg][k]
                            if curr_dist < prev_dist:
                                history[j][k] = distances[min_arg][k]
                                virtual_unit_map.swap(k, min_arg, j)
                                # mapping = backup_mapping.copy()
                                swaps += 1
                    elif distances[min_arg][k] > dataset_reject_threshold:
                        # If the distance between the two waveforms is greater than
                        # the threshold, move the unit to the first available free
                        # plot
                        loc = 0
                        first_zero = np.where(mapping[:, k] == 0)[0][loc]

                        try:
                            while np.sum(mapping[first_zero]) > 0.5:
                                loc += 1
                                first_zero = np.where(mapping[:, k] == 0)[0][loc]
                        except IndexError:
                            first_zero = 0
                        virtual_unit_map.swap(k, first_zero, min_arg)
                        history[min_arg][k] = np.nan
                        mapping = backup_mapping.copy()
                        swaps += 1
                        # print("Swapped {} with empty plot {} on day {}".format(min_arg, first_zero, k))
                        # print("Mapping: {}".format(mapping))
                    else:
                        # If none of the above cases yield true, something is wrong
                        print("Exception reached")
                        print(distances_waves)
                        print(distances_rp)
                        print(distances)
                        mapping = backup_mapping.copy()
    storage.change_map()
    # print("Total swaps: {}\n".format(swaps))

"""
def calculate_mapping(self, data, base):

    Calculates a mapping for the units based on features like distance.

    The units will be compared pare-wise and sequential.

    **Arguments**

        *data* (:class:`src.neodata.NeoData`):
            This object is needed to get the data
            which will be used to compare the units.

    print("Map being calculated")
    #dis = {}
    #for each block except the last one
    for i in range(len(data.blocks) - 1):
        swapped1 = []
        swapped2 = []

        #do it so often that each real unit can find a partner
        for n in range(data.total_units_per_block[i]):
            distances = []

            #for each unit in block i
            for j in range(self.maximum_units):
                if self.mapping[i][j] != 0:
                    unit1 = self.get_realunit(i, j, data)
                    y1 = data.get_data("average", unit1)[0]

                    #for each unit in block i+1
                    for k in range(self.maximum_units):
                        if self.mapping[i+1][k] != 0:
                            unit2 = self.get_realunit(i+1, k, data)
                            y2 = data.get_data("average", unit2)[0]
                            #calculates the distance between the average waveforms
                            distance = np.linalg.norm(np.subtract(y2, y1))

                            #calculate cross correlation {
                            #calculates the inter-spike-interval
                            isi1 = data.get_data("units", unit1)[0]
                            isi2 = data.get_data("units", unit2)[0]
                            len1 = len(isi1)
                            len2 = len(isi2)
                            if len1 > len2:
                                isi1 = isi1[:len2]
                            else:
                                isi2 = isi2[:len1]
                            cor1 = np.corrcoef(y1, y2)
                            cor2 = np.corrcoef(isi1, isi2)
                            c = ((cor1 + cor2) / 2.0)[1][0]
                            #print("unit {} vs. unit {} = {}, crosscorr = {}".format(j, k, distance, c))
                            # }

                            #calculate norm {
                            y = y1 + y2
                            isi = isi1 + isi2
                            av1 = np.mean(y)
                            std1 = np.std(y)
                            av2 = np.mean(isi)
                            std2 = np.std(isi)
                            yn1 = (y1 - av1) / std1
                            yn2 = (y2 - av1) / std1
                            isin1 = (isi1 - av2) / std2
                            isin2 = (isi2 - av2) / std2
                            #fig = plt.figure("dist: {}; corr: {}".format(distance, c))
                            #fig.add_subplot(111)
                            #plt.plot(yn1)
                            #plt.plot(yn2)
                            #plt.plot(isin1)
                            #plt.plot(isin2)
                            #plt.show()
                            #print("unit {} vs. unit {} = {}, y = {} {}, isi = {} {}".format(j, k, distance,
                            #                                                                yn1[:2], yn2[:2],
                            #                                                                isin1[:2], isin2[:2]))
                            #  }

                            if j not in swapped1 and k not in swapped2:
                                distances.append((j, k, distance))
                                #dis[c] = distance
            if distances:
                #finds the min of the distances
                mini = min(distances, key=lambda x: x[2])
                #swap only if distance is under a threshold
                if mini[2] < 200:
                    #swaps the units to have them in the same unit row
                    self.swap(i+1, mini[0], mini[1])
                    swapped1.append(mini[0])
                    swapped2.append(mini[0])
                #otherwise search for an empty row and swap there
                #but only if the units are in the same row
                else:
                    if mini[0] == mini[1]:
                        for u, v in enumerate(self.mapping[i+1]):
                            testing = True
                            for l in range(i+2):
                                if self.mapping[l][u] != 0:
                                    testing = False
                            if testing:
                                self.swap(i+1, mini[1], u)
                                swapped1.append(mini[0])
                                swapped2.append(u)

    #plt.plot(dis.keys(), dis.values(), "o")
    #plt.show()
"""


import argparse
import re
import subprocess

import pytest

from cotainr.cli import CotainrCLI, Info
from ..util.patches import patch_empty_system, patch_system_with_actual_file

class TestExecute:
    def test_report_structure(self, capsys):
        Info().execute()
        stdout = capsys.readouterr().out

        # Dependency report section
        assert stdout.startswith(
            "Dependency report\n"
            "-------------------------------------------------------------------------------\n"
        )
        # One line with python version check
        assert (
            sum(line.startswith("    - Running python ") for line in stdout.split("\n"))
            == 1
        )
        # One line with singularity / apptainer version check
        assert (
            sum(
                any(provider in line for provider in ["singularity", "apptainer"])
                for line in stdout.split("\n")
            )
            == 1
        )

        # Systems info section
        assert (
            "\n"
            "System info\n"
            "-------------------------------------------------------------------------------\n"
        ) in stdout

    def test_with_systems(self, capsys, patch_system_with_actual_file):
        Info().execute()
        stdout = capsys.readouterr().out
        assert stdout.endswith(
            "System info\n"
            "-------------------------------------------------------------------------------\n"
            "Available system configurations:\n"
            "    - some_system_6021\n"
            "    - another_system_6021\n"
        )

    def test_without_systems(self, capsys, patch_empty_system):
        Info().execute()
        stdout = capsys.readouterr().out
        assert stdout.endswith(
            "System info\n"
            "-------------------------------------------------------------------------------\n"
            "No system configurations available\n"
        )

class TestAddArguments:
    def test_no_arguments_added(self):
        parser = argparse.ArgumentParser()
        Info.add_arguments(parser=parser)
        args = parser.parse_args(args=[])
        assert not vars(args)

class TestHelpMessage:
    def test_CLI_subcommand_help_message(self, argparse_options_line, capsys):
        with pytest.raises(SystemExit):
            CotainrCLI(args=["info", "--help"])
        stdout = capsys.readouterr().out
        assert stdout == (
            # Capsys apparently assumes an 80 char terminal (?) - thus extra '\n'
            "usage: cotainr info [-h]\n\n"
            "Obtain info about the state of all required dependencies for building a\n"
            "container.\n\n"
            f"{argparse_options_line}"
            "  -h, --help  show this help message and exit\n"
        )

class Test_check_python_dependency:
    def test_formatting(self):
        # Check for formatting like "Running python 3.11.0 >= 3.8.0, OK"
        assert re.match(
            (
                r"^Running python \d+\.\d+\.\d+ (>=)|(<) \d+\.\d+\.\d+, "
                r"(\\033\[92mOK\\033\[0m)|(\\033\[91mERROR\\033\[0m)$"
            ),
            Info()._check_python_dependency(),
        )

class Test_check_singularity_dependency:
    def test_found_apptainer(self, monkeypatch):
        monkeypatch.setattr(
            subprocess,
            "check_output",
            lambda *args, **kwargs: "apptainer version 1.0.0",
        )
        assert (
            Info()._check_singularity_dependency()
            == "Found apptainer 1.0.0 >= 1.0.0, \x1b[92mOK\x1b[0m"
        )

    def test_found_singularity(self, monkeypatch):
        monkeypatch.setattr(
            subprocess,
            "check_output",
            lambda *args, **kwargs: "singularity version 3.7.4-1",
        )
        assert (
            Info()._check_singularity_dependency()
            == "Found singularity 3.7.4-1 >= 3.7.4, \x1b[92mOK\x1b[0m"
        )

    def test_found_unknown(self, monkeypatch):
        monkeypatch.setattr(
            subprocess,
            "check_output",
            lambda *args, **kwargs: "container_tool_6021 version 60.2.1",
        )
        assert (
            Info()._check_singularity_dependency()
            == "Found unknown singularity provider: container_tool_6021 60.2.1"
        )

    def test_not_found(self, monkeypatch):
        def not_found(*args, **kwargs):
            raise FileNotFoundError()

        monkeypatch.setattr(
            subprocess,
            "check_output",
            not_found,
        )
        assert (
            Info()._check_singularity_dependency()
            == "apptainer/singularity not found, \033[91mERROR\033[0m"
        )

class Test_check_version:
    @pytest.mark.parametrize(
        "version,min_version,cmp_result",
        [
            ((3, 8, 0), (3, 8, 0), ">= 3.8.0, \x1b[92mOK\x1b[0m"),
            ((0, 1, 12), (0, 0, 1), ">= 0.0.1, \x1b[92mOK\x1b[0m"),
            ((3, 7, 14), (3, 8, 0), "< 3.8.0, \033[91mERROR\033[0m"),
            ((1, 0, 0), (3, 8, 0), "< 3.8.0, \033[91mERROR\033[0m"),
        ],
    )
    def test_version_comparison(self, version, min_version, cmp_result):
        assert (
            Info()._check_version(version=version, min_version=min_version)
            == cmp_result
        )

    def test_wrong_input_len(self):
        with pytest.raises(TypeError) as exc_info:
            Info()._check_version(version=(1, 0, 0, 0), min_version=(1, 0, 0))
        assert str(exc_info.value) == (
            "The 'version=(1, 0, 0, 0)' argument must be "
            "a (major, minor, patchlevel) tuple of integers"
        )

    def test_wrong_input_type(self):
        with pytest.raises(TypeError) as exc_info:
            Info()._check_version(version=(1, 0, 0), min_version="1.0.0")
        assert str(exc_info.value) == (
            "The 'min_version=1.0.0' argument must be "
            "a (major, minor, patchlevel) tuple of integers"
        )

    def test_wrong_part_type(self):
        with pytest.raises(TypeError) as exc_info:
            Info()._check_version(version=("1", "0", "0"), min_version=(1, 0, 0))
        assert str(exc_info.value) == (
            "The 'version=('1', '0', '0')' argument must be "
            "a (major, minor, patchlevel) tuple of integers"
        )


# coding: utf-8

"""
    MBTA

    MBTA service API. https://www.mbta.com Source code: https://github.com/mbta/api  # noqa: E501

    The version of the OpenAPI document: 3.0
    Contact: developer@mbta.com
    Generated by: https://openapi-generator.tech
"""

from __future__ import absolute_import

import unittest
import datetime

import openapi_client
from openapi_client.models.trip_resource_relationships import TripResourceRelationships  # noqa: E501
from openapi_client.rest import ApiException

class TestTripResourceRelationships(unittest.TestCase):
    """TripResourceRelationships unit test stubs"""

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def make_instance(self, include_optional):
        """Test TripResourceRelationships
            include_option is a boolean, when False only required
            params are included, when True both required and
            optional params are included """
        # model = openapi_client.models.trip_resource_relationships.TripResourceRelationships()  # noqa: E501
        if include_optional :
            return TripResourceRelationships(
                shape = openapi_client.models.trip_resource_relationships_shape.TripResource_relationships_shape(
                    links = openapi_client.models.trip_resource_relationships_shape_links.TripResource_relationships_shape_links(
                        self = '0',
                        related = '0', ),
                    data = openapi_client.models.trip_resource_relationships_shape_data.TripResource_relationships_shape_data(
                        type = '0',
                        id = '0', ), ),
                service = openapi_client.models.trip_resource_relationships_service.TripResource_relationships_service(
                    links = openapi_client.models.trip_resource_relationships_service_links.TripResource_relationships_service_links(
                        self = '0',
                        related = '0', ),
                    data = openapi_client.models.trip_resource_relationships_service_data.TripResource_relationships_service_data(
                        type = '0',
                        id = '0', ), ),
                route_pattern = openapi_client.models.trip_resource_relationships_route_pattern.TripResource_relationships_route_pattern(
                    links = openapi_client.models.trip_resource_relationships_route_pattern_links.TripResource_relationships_route_pattern_links(
                        self = '0',
                        related = '0', ),
                    data = openapi_client.models.trip_resource_relationships_route_pattern_data.TripResource_relationships_route_pattern_data(
                        type = '0',
                        id = '0', ), ),
                route = openapi_client.models.prediction_resource_relationships_route.PredictionResource_relationships_route(
                    links = openapi_client.models.prediction_resource_relationships_route_links.PredictionResource_relationships_route_links(
                        self = '0',
                        related = '0', ),
                    data = openapi_client.models.prediction_resource_relationships_route_data.PredictionResource_relationships_route_data(
                        type = '0',
                        id = '0', ), )
            )
        else :
            return TripResourceRelationships(
        )

    def testTripResourceRelationships(self):
        """Test TripResourceRelationships"""
        inst_req_only = self.make_instance(include_optional=False)
        inst_req_and_optional = self.make_instance(include_optional=True)

if __name__ == '__main__':
    unittest.main()


import os
import argparse
from functools import partial

import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from torch.optim.lr_scheduler import ReduceLROnPlateau

from torch.utils.tensorboard import SummaryWriter

from model import Im2LatexModel
from model.training import LSTMTrainer, TransformerTrainer
from model.model_transformer import Im2LatexModelTransformer
from utils import collate_fn, collate_transformer_fn, get_checkpoint
from data import Im2LatexDataset
from build_vocab import Vocab, load_vocab

def next_nonexistent_dir(d):
    i = 0
    d_new = d
    while os.path.exists(d_new):
        i += 1
        d_new = '%s_%i' % (d, i)
    return d_new

def get_tensorboard_writer(experiment, verbose=True):
    log_dir = next_nonexistent_dir("./runs/experiment{}".format(experiment))
    os.mkdir(log_dir)
    if verbose:
        print("\nLogging directory:", log_dir, "\n")
    return SummaryWriter(log_dir=log_dir), log_dir

def main():

    # get args
    parser = argparse.ArgumentParser(description="Im2Latex Training Program")
    # parser.add_argument('--path', required=True, help='root of the model')

    # row_encoder: Whether to have the row encoder or not, if not, feed directly from CNN to decoder
    # paper_emb: Fixed embeddings applied as the encoder initial hidden state
    # new_emb: Embeddings applied to the output of the CNN or row encoder (depending on row_encoder)
    experiments = {

        # Row encoder, paper_emb - CLOSEST TO THE PAPER
        1: {"use_transformer": False, "row_encoder": True, "paper_emb": True, "new_emb": False},

        # Transformer, 6+6 layers, use patching
        2: {"use_transformer": True, "size": "small", "use_patches": True, "patch_size":2},

        # Row encoder, new_emb
        3: {"use_transformer": False, "row_encoder": True, "paper_emb": False, "new_emb": True},

        # No row encoder, new_emb
        4: {"use_transformer": False, "row_encoder": False, "paper_emb": False, "new_emb": True},

        # Transformer, 6+6 layers, no patching
        5: {"use_transformer": True, "size": "small", "use_patches": False},

        # Row encoder, no embeddings
        6: {"use_transformer": False, "row_encoder": True, "paper_emb": False, "new_emb": False},

        # Transformer, 8+8 layers, no patching
        7: {"use_transformer": True, "size": "medium", "use_patches": False},

        # Row encoder, paper_emb, new_emb
        8: {"use_transformer": False, "row_encoder": True, "paper_emb": True, "new_emb": True},

        # No row encoder, no embeddings
        9: {"use_transformer": False, "row_encoder": False, "paper_emb": False, "new_emb": False},

        # Transformer, 12+12 layers, use patching
        10: {"use_transformer": True, "size": "large", "use_patches": True, "patch_size": 2},

        # Transformer, 12+12 layers, use patching
        11: {"use_transformer": True, "size": "large", "use_patches": False},

        # Transformer, 8+8 layers, use patching
        12: {"use_transformer": True, "size": "medium", "use_patches": True, "patch_size": 2},
    }
    # experiment args
    parser.add_argument("experiment", type=int, help="Specify an experiment")
    parser.add_argument("--debug", action='store_true', default=False,
                        help="Debugging mode using small dataset")

    # LSTM encoder model args
    parser.add_argument("--emb_dim", type=int,
                        default=80, help="Embedding size")
    parser.add_argument("--enc_rnn_h", type=int, default=256,
                        help="The hidden state of the row encoder RNN")
    parser.add_argument("--dec_rnn_h", type=int, default=512,
                        help="The hidden state of the decoder RNN")
    parser.add_argument("--data_path", type=str,
                        default="./data/", help="The dataset's dir")

    # transformer model args
    parser.add_argument("--use_row_embeddings", action='store_true', default=False,
                        help="Whether or not to use row embedding")
    parser.add_argument("--weight_decay", type=float,
                        default=0.01, help="Optimizer weight decay")
    parser.add_argument("--feat_size", type=int,
                        default=512, help="Feature_Embedding size")
    parser.add_argument("--encoder_nheads", type=int, default=8,
                        help="Number of heads of transormer encoder")
    parser.add_argument("--encoder_dim_feedforward", type=int, default=1024,
                        help="Hidden Dimension for encoder fc layer")
    parser.add_argument("--encoder_num_layers", type=int, default=6,
                        help="Number of encoder attention layers")
    parser.add_argument("--encoder_dropout", type=float, default=0.2,
                        help="Dropout to be used for encoder")
    parser.add_argument("--encoder_max_sequence_length", type=int, default=2048,
                        help="Max output sequence length expected for encoder input")

    parser.add_argument("--decoder_nheads", type=int, default=8,
                        help="Number of decoder heads for each layer")
    parser.add_argument("--decoder_dim_feedforward", type=int, default=1024,
                        help="Hidden dimension for decoder fc layer")
    parser.add_argument("--decoder_num_layers", type=int, default=6,
                        help="Number of decoder attention layers")
    parser.add_argument("--decoder_dropout", type=float, default=0.2,
                        help="Dropout to be used for decoder")
    parser.add_argument("--decoder_max_sequence_length", type=int, default=1024,
                        help="Max output sequence length expected for decoder output")

    # training args
    parser.add_argument("--augment", action='store_true',
                        default=False, help="Perform data augmentation")
    parser.add_argument("--max_len", type=int,
                        default=200, help="Max size of formula")
    parser.add_argument("--dropout", type=float,
                        default=0., help="Dropout probility")
    parser.add_argument("--cuda", action='store_true',
                        default=True, help="Use cuda or not")
    parser.add_argument("--batch_size", type=int, default=32)
    parser.add_argument("--epoches", type=int, default=15)
    parser.add_argument("--lr", type=float, default=3e-4,
                        help="Learning Rate")
    parser.add_argument("--min_lr", type=float, default=3e-5,
                        help="Learning Rate")
    parser.add_argument("--sample_method", type=str, default="teacher_forcing",
                        choices=('teacher_forcing', 'exp', 'inv_sigmoid'),
                        help="The method to schedule sampling")
    parser.add_argument("--decay_k", type=float, default=1.,
                        help="Base of Exponential decay for Schedule Sampling. "
                        "When sample method is Exponential deca;"
                        "Or a constant in Inverse sigmoid decay Equation. "
                        "See details in https://arxiv.org/pdf/1506.03099.pdf")
    parser.add_argument("--lr_decay", type=float, default=0.5,
                        help="Learning Rate Decay Rate")
    parser.add_argument("--lr_patience", type=int, default=3,
                        help="Learning Rate Decay Patience")
    parser.add_argument("--clip", type=float, default=2.0,
                        help="The max gradient norm")
    parser.add_argument("--save_dir", type=str,
                        help="The dir to save checkpoints")
    parser.add_argument("--print_freq", type=int, default=100,
                        help="The frequency to print message")
    parser.add_argument("--seed", type=int, default=2020,
                        help="The random seed for reproducing ")
    parser.add_argument("--from_check_point", action='store_true',
                        default=False, help="Training from checkpoint or not")

    args = parser.parse_args()

    # Args preprocessing
    writer, log_dir = get_tensorboard_writer(args.experiment)

    if args.save_dir == None:
        args.save_dir = "./ckpts/ckpts_" + log_dir[len("./runs/"):]

    print("\nCheckpoint directory:", args.save_dir, "\n")

    if args.augment:
        print("Using data augmentation.")

    # Run training
    if experiments[args.experiment]["use_transformer"]:
        TrainTransformer(experiments, args, writer)
    else:
        TrainLSTMEncoder(experiments, args, writer)

def TrainTransformer(experiments, args, writer):
    max_epoch = args.epoches
    from_check_point = args.from_check_point
    if from_check_point:
        checkpoint_path = get_checkpoint(args.save_dir)
        checkpoint = torch.load(checkpoint_path)
        args = checkpoint['args']
    print("Training args:", args)

    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    # Building vocab
    print("Load vocab...")
    vocab = load_vocab(args.data_path)

    use_cuda = True if args.cuda and torch.cuda.is_available() else False
    device = torch.device("cuda" if use_cuda else "cpu")
    print("Device:", device)

    # Multi-processing doesn't function well with Windows OS
    if os.name == 'nt':
        num_workers = 0
    else:
        num_workers = 4

    # data loader
    if args.debug:
        print('Running in debug mode')
    print("Construct transformer data loader...")
    train_loader = DataLoader(
        Im2LatexDataset(args.data_path,'test' if args.debug else 'train', args),
        batch_size=args.batch_size,
        collate_fn=partial(collate_transformer_fn, vocab.sign2id),
        pin_memory=True if use_cuda else False,
        num_workers=num_workers)
    val_loader = DataLoader(
        Im2LatexDataset(args.data_path, 'validate', args),
        batch_size=args.batch_size,
        collate_fn=partial(collate_transformer_fn, vocab.sign2id),
        pin_memory=True if use_cuda else False,
        num_workers=num_workers)

    # construct model
    print("Constructing model: ")
    vocab_size = len(vocab)
    print("Using row embeddings: ", args.use_row_embeddings)
    if experiments[args.experiment]['size'] == 'small':
        print("Model type: small")
        args.encoder_num_layers = 6
        args.decoder_num_layers = 6
    elif experiments[args.experiment]['size'] == 'medium':
        print("Model type: medium")
        args.encoder_num_layers = 8
        args.decoder_num_layers = 8
    elif experiments[args.experiment]['size'] == 'large':
        print("Model type: large")
        args.encoder_num_layers = 12
        args.decoder_num_layers = 12

    if experiments[args.experiment]['use_patches']:
        args.__dict__.update({
            "use_patches": True,
            "patch_size":  experiments[args.experiment]['patch_size'] })
    else:
        args.__dict__.update({
            "use_patches": False})

    args.__dict__.update({"vocab_size": vocab_size})
    # config = Config(
    #     vocab_size=vocab_size,
    #     feat_size=512,
    #     encoder_nheads=8,
    #     encoder_dim_feedfoward=2048,
    #     encoder_num_layers=6,
    #     encoder_dropout=0.2,
    #     encoder_max_sequence_length=1024,
    #     decoder_nheads=8,
    #     decoder_dim_feedfoward=2048,
    #     decoder_num_layers=8,
    #     decoder_dropout=0.2,
    #     decoder_max_sequence_length=1024
    # )
    model = Im2LatexModelTransformer(args)
    model = model.to(device)
    mem_params = sum([param.nelement()*param.element_size() for param in model.parameters()])
    mem_bufs = sum([buf.nelement()*buf.element_size() for buf in model.buffers()])
    mem = mem_params + mem_bufs # in bytes
    print("Model memory footprint: {}".format(mem))

    # print("Model Summary:")
    # print(model)
    # construct optimizer
    optimizer = optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)

    lr_scheduler = ReduceLROnPlateau(
        optimizer,
        "min",
        factor=args.lr_decay,
        patience=args.lr_patience,
        verbose=True,
        min_lr=args.min_lr
    )

    if from_check_point:
        model.load_state_dict(checkpoint['model_state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        epoch = checkpoint['epoch']
        lr_scheduler.load_state_dict(checkpoint['lr_sche'])
        # init trainer from checkpoint
        trainer = TransformerTrainer(optimizer, model, lr_scheduler,
                          train_loader, val_loader, args,
                          device, use_cuda=use_cuda,
                          init_epoch=epoch, last_epoch=max_epoch,
                          writer=writer)
    else:
        trainer = TransformerTrainer(optimizer, model, lr_scheduler,
                          train_loader, val_loader, args,
                          device, use_cuda=use_cuda,
                          init_epoch=1, last_epoch=args.epoches,
                          writer=writer)
    # begin training
    print('Beginning training for transformer...')
    trainer.train()

def TrainLSTMEncoder(experiments, args, writer):
    max_epoch = args.epoches
    from_check_point = args.from_check_point
    if from_check_point:
        checkpoint_path = get_checkpoint(args.save_dir)
        checkpoint = torch.load(checkpoint_path)
        args = checkpoint['args']
    print("Training args:", args)

    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    # Building vocab
    print("Load vocab...")
    vocab = load_vocab(args.data_path)

    use_cuda = True if args.cuda and torch.cuda.is_available() else False
    device = torch.device("cuda" if use_cuda else "cpu")
    print("Device:", device)

    # Multi-processing doesn't function well with Windows OS
    if os.name == 'nt':
        num_workers = 0
    else:
        num_workers = 4

    # data loader
    if args.debug:
        print('Running in debug mode')
    print("Construct data loader...")
    train_loader = DataLoader(
        Im2LatexDataset(args.data_path, 'test' if args.debug else 'train', args),
        batch_size=args.batch_size,
        collate_fn=partial(collate_fn, vocab.sign2id),
        pin_memory=True if use_cuda else False,
        num_workers=num_workers)
    val_loader = DataLoader(
        Im2LatexDataset(args.data_path, 'validate', args),
        batch_size=args.batch_size,
        collate_fn=partial(collate_fn, vocab.sign2id),
        pin_memory=True if use_cuda else False,
        num_workers=num_workers)

    # construct model
    print("Construct model")
    vocab_size = len(vocab)
    model = Im2LatexModel(
        experiments[args.experiment],
        vocab_size, args.emb_dim,
        args.enc_rnn_h, args.dec_rnn_h,
        device, dropout=args.dropout,
    )

    model = model.to(device)
    print("Model Settings:")
    print(model)

    # construct optimizer
    optimizer = optim.Adam(model.parameters(), lr=args.lr)

    lr_scheduler = ReduceLROnPlateau(
        optimizer,
        "min",
        factor=args.lr_decay,
        patience=args.lr_patience,
        verbose=True,
        min_lr=args.min_lr)

    if from_check_point:
        model.load_state_dict(checkpoint['model_state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        epoch = checkpoint['epoch']
        lr_scheduler.load_state_dict(checkpoint['lr_sche'])
        # init trainer from checkpoint
        trainer = LSTMTrainer(optimizer, model, lr_scheduler,
                          train_loader, val_loader, args,
                          device, use_cuda=use_cuda,
                          init_epoch=epoch, last_epoch=max_epoch,
                          writer=writer)
    else:
        trainer = LSTMTrainer(optimizer, model, lr_scheduler,
                          train_loader, val_loader, args,
                          device, use_cuda=use_cuda,
                          init_epoch=1, last_epoch=args.epoches,
                          writer=writer)
    # begin training
    print('Beginning training...')
    trainer.train()

if __name__ == "__main__":
    main()


import os
import xarray as xr
from py_wake.site.xrsite import XRSite
import matplotlib.pyplot as plt
import yaml
import numpy as np
from plant.examples.example_data.make_example_data import xr2yml

xr2yml

def yml2xr(file):
    pass

if __name__ == '__main__':

    def load(f):
        yml_site = XRSite(xr.open_dataset(example_data_path + f + ".nc"))
        with open(example_data_path + f + '.yml') as fid:
            xr_site = XRSite(xr.Dataset.from_dict(yaml.safe_load(fid)))
        assert yml_site.ds.equals(xr_site.ds)
        return yml_site

    # UniformSite
    site = load('UniformSite')
    plt.figure()
    site.plot_wd_distribution(ws_bins=4)
    plt.title('UniformSite')

    # UniformWeibullSite
    f = 'UniformWeibullSite'
    site = load(f)
    plt.figure()
    site.plot_wd_distribution(ws_bins=4)
    plt.title(f)
    plt.figure()
    site.plot_ws_distribution()
    plt.title(f)

    # NonGriddedSite
    f = 'NonGriddedSite'
    site = load(f)
    x = y = np.arange(16)  # dummy positions
    lw = site.local_wind(x, y, wd=0, ws=10)
    plt.figure()
    plt.title(f)
    lw.WS.plot()

    # GriddedSite
    f = 'GriddedSite'
    site = load(f)
    x = y = np.arange(-2000, 2500, 500)
    X, Y = np.meshgrid(x, y)
    lw = site.local_wind(X.flatten(), Y.flatten(), wd=0, ws=10)
    plt.figure()
    plt.title(f)
    plt.contourf(X, Y, np.reshape(lw.WS, X.shape))

    plt.show()


"Operador division"

"Uso del operador módulo /"

var1 = 40
var2 = 9

"Operación para división"
resultado = var1 / var2
print("El valor de mi división es: {}".format(resultado))

var3 = 8.0

resultado2 = var1 / var3
print("El valor de mi variable 'resultado2' es: {}".format(resultado2))

"""Esto no será posible"""
var4 = 0
resultado3 = var1 / var4
print("El valor de mi variable 'resultado3' es: {}".format(resultado3))


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 24 07:42:08 2020

@author: Piotr.Idzik and Maria Paszkiewicz
Puzzle 24 from https://adventofcode.com/2020/day/24

--- Day 24: Lobby Layout ---

Part 1: Go through the renovation crew's list and determine which tiles they need
to flip. After all of the instructions have been followed, how many tiles are left
with the black side up?

Part 2: The tile floor in the lobby is meant to be a living art exhibit. Every day,
the tiles are all flipped according to the following rules:

- Any black tile with zero or more than 2 black tiles immediately adjacent to it
  is flipped to white.
- Any white tile with exactly 2 black tiles immediately adjacent to it is flipped
  to black.

Here, tiles immediately adjacent means the six tiles directly touching the tile
in question.

How many tiles will be black after 100 days?
"""
import math

def str_to_list(in_str):
    res_list = []

    def inner(in_str):
        if in_str:
            if in_str[0] in ['w', 'e']:
                res_list.append(in_str[0])
                inner(in_str[1:])
            else:
                assert in_str[0:2] in ['ne', 'nw', 'se', 'sw']
                res_list.append(in_str[0:2])
                inner(in_str[2:])
    inner(in_str)
    return res_list

def single_move(in_pos, in_dir):
    def get_shift(in_angle):
        rad_angle = math.radians(in_angle)
        return [math.cos(rad_angle), math.sin(rad_angle)]
    angle_dict = \
        {'e': 0, 'ne': 60, 'nw': 120, 'w': 180, 'sw': 240, 'se': 300}
    cur_shift = get_shift(angle_dict[in_dir])
    return [a+b for a, b in zip(in_pos, cur_shift)]

def list_to_pos(in_list):
    cur_pos = [0, 0]
    for _ in in_list:
        cur_pos = single_move(cur_pos, _)
    return cur_pos

def pos_to_key(in_pos):
    return tuple([round(2*in_pos[0]), round(2*in_pos[1]/(3**0.5))])

def key_to_pos(in_key):
    return [in_key[0]/2, in_key[1]*3**0.5/2]

def make_flips(in_str_list):
    tiles_dict = {}

    def proc_single_line(in_line_str):
        cur_key = pos_to_key(list_to_pos(str_to_list(in_line_str)))
        if cur_key in tiles_dict:
            if tiles_dict[cur_key] == 'B':
                del tiles_dict[cur_key]
            elif tiles_dict[cur_key] == 'W':
                tiles_dict[cur_key] = 'B'
            else:
                raise ValueError('wtf?!')
        else:
            tiles_dict[cur_key] = 'B'
    for _ in in_str_list:
        proc_single_line(_)
    return tiles_dict

def count_black(in_tiles_dict):
    return sum(1 for _ in in_tiles_dict if in_tiles_dict[_] == 'B')

def solve_1(in_str_list):
    '''
    Part 1
    '''
    return count_black(make_flips(in_str_list))

def get_data_m():
    with open('input_m.txt', 'r') as in_file:
        data_str = in_file.read()
    return data_str.split('\n')

def get_data_p():
    with open('input_p.txt', 'r') as in_file:
        data_str = in_file.read()
    return data_str.split('\n')

def get_dir_list():
    return ['e', 'ne', 'nw', 'w', 'sw', 'se']

def next_day(in_tiles_data):
    def get_color(in_dict, in_key):
        if in_key in in_dict:
            res = in_dict[in_key]
        else:
            res = 'W'
        return res

    def get_sur(in_key):
        dir_list = get_dir_list()
        in_pos = key_to_pos(in_key)
        res_list = [pos_to_key(single_move(in_pos, _)) for _ in dir_list]
        return res_list

    def count_black_sur(in_key):
        return sum(1 for _ in get_sur(in_key)
                   if get_color(in_tiles_data, _) == 'B')
    active_pos_set = set()
    for _ in in_tiles_data:
        active_pos_set.update(get_sur(_))
        active_pos_set.add(_)
    res_dict = {}
    for _ in active_pos_set:
        black_num = count_black_sur(_)
        if get_color(in_tiles_data, _) == 'B':
            if not (black_num == 0 or black_num > 2):
                res_dict[_] = 'B'
        elif black_num == 2:
            res_dict[_] = 'B'
    return res_dict

def simulate(in_str_list, max_day_num):
    '''
    Part 2
    '''
    tiles_data = make_flips(in_str_list)
    for _ in range(max_day_num):
        tiles_data = next_day(tiles_data)
    return count_black(tiles_data)

assert str_to_list('seswneswswsenwwnwse') == \
    ['se', 'sw', 'ne', 'sw', 'sw', 'se', 'nw', 'w', 'nw', 'se']

TEST_STR = """sesenwnenenewseeswwswswwnenewsewsw
neeenesenwnwwswnenewnwwsewnenwseswesw
seswneswswsenwwnwse
nwnwneseeswswnenewneswwnewseswneseene
swweswneswnenwsewnwneneseenw
eesenwseswswnenwswnwnwsewwnwsene
sewnenenenesenwsewnenwwwse
wenwwweseeeweswwwnwwe
wsweesenenewnwwnwsenewsenwwsesesenwne
neeswseenwwswnwswswnw
nenwswwsewswnenenewsenwsenwnesesenew
enewnwewneswsewnwswenweswnenwsenwsw
sweneswneswneneenwnewenewwneswswnese
swwesenesewenwneswnwwneseswwne
enesenwswwswneneswsenwnewswseenwsese
wnwnesenesenenwwnenwsewesewsesesew
nenewswnwewswnenesenwnesewesw
eneswnwswnwsenenwnwnwwseeswneewsenese
neswnwewnwnwseenwseesewsenwsweewe
wseweeenwnesenwwwswnew"""

assert solve_1(TEST_STR.split('\n')) == 10
assert solve_1(get_data_m()) == 341
assert solve_1(get_data_p()) == 322

assert simulate(TEST_STR.split('\n'), 1) == 15
assert simulate(TEST_STR.split('\n'), 2) == 12
assert simulate(TEST_STR.split('\n'), 10) == 37
assert simulate(TEST_STR.split('\n'), 20) == 132
assert simulate(TEST_STR.split('\n'), 100) == 2208

assert simulate(get_data_p(), 100) == 3831
assert simulate(get_data_m(), 100) == 3700

print(solve_1(get_data_m()))
print(simulate(get_data_m(), 100))


#import numpy as np
import cv2

cap = cv2.VideoCapture(0)

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # Our operations on the frame come here
    # display in gray
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #cv2.imshow('frame', gray)

    # Display the resulting frame
    #print(frame)
    #print(np.shape(frame))
    #print(type(frame))
    #print(ret)
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #print(gray)
    #print(np.shape(gray))
    #edges = np.zeros_like(gray)
    #e = np.zeros((10,2), int)
    #print(e)
    #b = []
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #edges = cv2.goodFeaturesToTrack(gray, max_corners, quality_lev, min_dist)
    ##edges = cv2.Canny(frame, edges)
    ##edges = cv2.canny(frame, edges, 10, 30)
    ##print(gray)
    #print(edges)
    #print(np.shape(gray))
    cv2.imshow('frame', frame)
    #cv2.imshow('frame', frame)
    #break
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break


# -*- coding: utf-8 -*-
"""
Created on Thu Nov 22 10:58:51 2018
使用 ImageDataGenerator 增加圖像
@author: mars0925
"""

# Convolutional Neural Network

# Installing Theano
# pip install --upgrade --no-deps git+git://github.com/Theano/Theano.git

# Installing Tensorflow
# Install Tensorflow from the website: https://www.tensorflow.org/versions/r0.12/get_started/os_setup.html

# Installing Keras
# pip install --upgrade keras

# Part 1 - Building the CNN

from keras.models import Sequential
from keras.layers import Convolution2D #Convolution layer
from keras.layers import MaxPooling2D #Pooling layer
from keras.layers import Flatten ##扁平化
from keras.layers import Dense #神經網路層
from keras.layers import Dropout

trainPath = r"D:\AI\train512"#訓練集路徑
testPath = r"D:\AI\test512"#訓練集路徑
img_size = 512#照片大小
batch_size = 32#一次要學習幾張
trainSample = 800#訓練習樣本數
testSample = 190#測試集樣本數
epochs = 20 #訓練回數

# Initialising the CNN
classifier = Sequential()

# Step 1 - Convolution
#添加捲積層
#filter 特徵探測器個數
#kernel_size 探測器的大小
#activation= "relu" 激活函數
#input_shape = (64, 64, 3) 輸入的圖片大小,第一層需要設定
classifier.add(Convolution2D(filters= 32, kernel_size=(3,3), activation= "relu", input_shape = (img_size, img_size, 1),padding='same'))
classifier.add(Dropout(rate=0.3))

# Step 2 - Pooling
#添加池化層
classifier.add(MaxPooling2D(pool_size = (2, 2)))

# Adding a second convolutional layer
#第二層
classifier.add(Convolution2D(filters= 32, kernel_size=(3,3), activation = 'relu',padding='same'))
classifier.add(Dropout(rate=0.3))
classifier.add(MaxPooling2D(pool_size = (2, 2)))

# Step 3 - Flattening
#扁平化
classifier.add(Flatten())
classifier.add(Dropout(rate=0.25))

# Step 4 - Full connection
#連結到神經網路
#units=128神經元個數
classifier.add(Dense(units=32, activation = 'relu'))#隱藏層
classifier.add(Dense(units=1, activation = 'sigmoid'))#輸出層

print(classifier.summary())

try:
    classifier.load_weights("./xrayCnnModel.h5")
    print("載入模型成功!繼續訓練模型")
except Exception as e:
    print("出錯了", e)
    print("載入模型失敗!開始訓練一個新模型")

# Compiling the CNN
# optimizer = 'adam' 優化算法
# loss = 'binary_crossentropy' 損失函數
# 要依據分類個數區分 如果是多類別要設定為categorical_crossentropy
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

# Part 2 - Fitting the CNN to the images
# 對圖像進行預先處理
# 使用keras來產生較多的train image
#rescale=1./255 像素除以255 特徵縮放
#shear_range=0.2 #圖片轉向
#zoom_range=0.2 #圖片縮放
#horizontal_flip=True #圖片水平翻轉
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,shear_range = 0.2, zoom_range = 0.2,horizontal_flip = True)

test_datagen = ImageDataGenerator(rescale = 1./255)#測試集只需要特徵縮放處理即可

#target_size = (64, 64) 圖片大小
#batch_size = 32每次生成的張數
# class_mode = 'binary' 分類的種類 如果是多類別 設定categorical
#color_mode: "grayscale", "rbg" 之一。默认："rgb"。图像是否被转换成 1 或 3 个颜色通道。
training_set = train_datagen.flow_from_directory(trainPath,target_size = (img_size, img_size),batch_size = batch_size,class_mode = 'binary',color_mode = "grayscale")

test_set = test_datagen.flow_from_directory(testPath,target_size = (img_size, img_size), batch_size = batch_size, class_mode = 'binary',color_mode = "grayscale")

#用生成的圖片訓練模型
#steps_per_epoch=250,它通常應等於數據集的樣本數除以bathc_size 8000/32
#epochs=25,期數
#validation_data = test_set,測試集
#validation_steps = 62.5 測試集個數/batch_size 2000/32

train_history = classifier.fit_generator(training_set, steps_per_epoch=trainSample/batch_size, epochs=epochs,validation_data = test_set,validation_steps = testSample/batch_size)

classifier.save_weights("./xrayCnnModel.h5")

from tool import show_train_history

#劃出訓練圖形
show_train_history(train_history,'acc','val_acc')
show_train_history(train_history,'loss','val_loss')

#評估模型準確率
score = classifier.evaluate_generator(test_set, testSample/batch_size, workers=12)

## 進行預測
scores = classifier.predict_generator(test_set, testSample/batch_size, workers=12)

print("＝＝＝＝＝＝＝列出預測機率＝＝＝＝")

correct = 0
for i, n in enumerate(test_set.filenames):
    if n.startswith("normal") and scores[i][0] <= 0.5:
        correct += 1
    if n.startswith("abnormal") and scores[i][0] > 0.5:
        correct += 1

print("Correct:", correct, " Total: ", len(test_set.filenames))
print("Loss: ", score[0], "Accuracy: ", score[1])