# On inclut la bibliotheque permettant de manipuler de l'aleatoire.
from random import random

# On inclut la bibliotheque graphique
from matplotlib.pyplot import hist, show, plot, title, axis

def unif(a,b):
    """ Simule une variable aleatoire uniforme sur [a,b]."""
    return a+(b-a)*random()

def unif(N):
    """Renvoie une variable uniforme sur l'ensemble {0,1,...,N-1}"""
    return int(N*random())

def bernoulli(p):
        """Renvoie une la valeur 0 avec probabilite 1-p, et la valeur 1 avec probabilite p.
        Il s'agit d'une variable de Bernoulli de parametre p."""
        if random()<p:
            return 1
        else:
            return 0

def binomiale(n,p):
        """Renvoie une realisation d'une variable aleatoire de loi binomiale de parametres n et p."""
        return sum( [bernoulli(p) for i in range(n)] )

def simu_X():
        """Simule une variable aleatoire a valeurs dans {0,1,2,3}, avec le probabilites suivantes :
        p_0 = 1/2   ;   p_1 = 1/4   ;   p2 = 1/6   ;   p3 = 1/12"""
        U = random()
        if U<1./2:
            return 0
        elif U<1./2 + 1./4:
            return 1
        elif U<1./2 + 1./4 + 1./6:
            return 2
        else:
            return 3

def loi_discrete(pi):
    """Renvoie une simulation de la loi (p_0,...,p_N).
    L'argument a fournir a la fonction est le vecteur (p_0,...,p_{N-1}) (la valeur de p_N est calculee).
    Ce vecteur doit etre positif, et la somme de ses coefficients inferieure a 1."""
    if min(pi)<0 or sum(pi)>1:
        print("pi n'est pas une probabilite")
        return
    x = random()
    somme = 0
    for i in range(len(pi)):
        somme += pi[i]
        if x<somme:
            return i
    return len(pi)

if __name__ == "__main__":

    ##### Uniforme #####

    n = 30
    hist([unif(n) for k in range(100)], bins=30, range=(-0.5,29.5))
    title("100 simulations d'une variable uniforme sur {0,1,...,29}")
    axis([-0.5,29.5,0,10])
    show()

    hist([unif(n) for k in range(1000)], bins=30, range=(-0.5,29.5))
    title("1000 simulations d'une variable uniforme sur {0,1,...,29}")
    axis([-0.5,29.5,0,100])
    show()

    hist([unif(n) for k in range(10000)], bins=30, range=(-0.5,29.5))
    title("10000 simulations d'une variable uniforme sur {0,1,...,29}")
    axis([-0.5,29.5,0,1000])
    show()

    ##### Bernoulli #####

    p = 0.3
    print("Dix variables de Bernoulli de parametre %f" % p)
    for i in range(10):
        print(bernoulli(p))
    print

    ##### Binomiale #####

    n = 20
    p = 0.3
    nb_simus = 10000
    hist([binomiale(n,p) for i in range(nb_simus)], normed=True, bins=21, range=(-0.5,20.5))
    p_theorique = [1] * (n+1)
    for k in range(n+1):
        p_theorique[k] *= (p ** k) * ((1-p)**(n-k))
        fact = 1
        for q in range(k):
            fact *=  (n-q) / (q+1.)
        p_theorique[k] *= fact
    plot(p_theorique, 'or')
    axis([-0.5,20.5,0,0.2])
    title("Histogramme de %d simulations d'une\nbinomiale(%d, %.2f) et valeur theorique" % (nb_simus,n,p))
    show()

    ##### Convergence par LFGN #####

    p = 0.4
    n = 1000
    X_n = [bernoulli(p) for i in range(n)]
    Y_n = [sum(X_n[:i])/float(i) for i in range(1,n+1)]
    plot(Y_n)
    plot([0,n],[p, p], '--k')
    axis([0,n,0,1])
    title("Convergence de 1/n \sum X_i vers p, pour\ndes Bernoulli(%.2f), avec n=%d" % (p,n))
    show()

    ##### Loi arbitraire #####

    hist([simu_X() for k in range(1000)], bins=4, range=(-0.5,3.5), normed=True)
    plot([0,1,2,3], [1./2, 1./4, 1./6, 1./12], 'ro')
    axis([-0.5,3.5,0,0.6])
    title("Histogramme de 1000 realisations de la loi (1/2,1/4,1/6,1/12)")
    show()


from django.db import models
from users.models import CustomUser

class Private_Message(models.Model):
    sender = models.ForeignKey(CustomUser, related_name="message_sender", on_delete=models.CASCADE)
    receiver = models.ForeignKey(CustomUser, related_name="message_receiver", on_delete=models.CASCADE)
    message = models.CharField(default="", max_length=100)
    datetime = models.DateTimeField(auto_now_add=True)
    def __str__(self):
        return "sender: " + str(self.sender.username) + " receiver: " + str(self.receiver.username) + " message: " + str(self.message) + " datetime: " + str(self.datetime)
    # date = models.DateField(auto_now_add=True)
    # time = models.TimeField(auto_now_add=True)

def getAllMessages(user1, user2):
    messages = list(Private_Message.objects.filter(sender=user1, receiver=user2) | Private_Message.objects.filter(sender=user2, receiver=user1))
    messages = sorted(messages, key=lambda x:x.datetime)
    # messages.order_by('datetime')
    #messages.sort(key=lambda x: (x.date, x.time))
    return messages


# C3 = 1210 % 3 = 1 - тип: StringBuffer
# C17 = 1210 % 17 = 3 - В усіх питальних реченнях заданого тексту
# знайти та надрукувати без повторень слова заданої довжини.
import re

class Target:
    @staticmethod
    def vikonavchiymethod():
        text_in = open('2.txt').read()
        # text_in = input('input text: ')

        ###############################################################################

        alphabets = "([A-Za-z])"
        prefixes = "(Mr|St|Mrs|Ms|Dr)[.]"
        suffixes = "(Inc|Ltd|Jr|Sr|Co)"
        starters = "(Mr|Mrs|Ms|Dr|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
        acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
        websites = "[.](com|net|org|io|gov)"

        def split_into_sentences(text):
            text = " " + text + "  "
            text = text.replace("\n", " ")
            text = re.sub(prefixes, "\\1<prd>", text)
            text = re.sub(websites, "<prd>\\1", text)
            if "Ph.D" in text:
                text = text.replace("Ph.D.", "Ph<prd>D<prd>")
            text = re.sub("\s" + alphabets + "[.] ", " \\1<prd> ", text)
            text = re.sub(acronyms + " " + starters, "\\1<stop> \\2", text)
            text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]", "\\1<prd>\\2<prd>\\3<prd>", text)
            text = re.sub(alphabets + "[.]" + alphabets + "[.]", "\\1<prd>\\2<prd>", text)
            text = re.sub(" " + suffixes + "[.] " + starters, " \\1<stop> \\2", text)
            text = re.sub(" " + suffixes + "[.]", " \\1<prd>", text)
            text = re.sub(" " + alphabets + "[.]", " \\1<prd>", text)
            if "”" in text:
                text = text.replace(".”", "”.")
            if "\"" in text:
                text = text.replace(".\"", "\".")
            if "!" in text:
                text = text.replace("!\"", "\"!")
            if "?" in text:
                text = text.replace("?\"", "\"?")
            text = text.replace(".", ".<stop>")
            text = text.replace("?", " ?<stop>")
            text = text.replace("!", "!<stop>")
            text = text.replace("<prd>", ".")
            sentences = text.split("<stop>")
            sentences = sentences[:-1]
            sentences = [s.strip() for s in sentences]
            return sentences

        ################################################################################
        text_in = text_in.replace('“', '')
        text_in = text_in.replace('”', '')
        text_in = text_in.replace(',', '')
        text_in = text_in.replace(':', '')
        text_in = text_in.replace(';', '')
        sentences = split_into_sentences(text_in)
        q_Sentences = []
        for i in sentences:
            if i[-1] == '?':
                q_Sentences.append(i)
        print('interrogative sentences: ')
        for i in q_Sentences:
            print(i)
        print('')
        while True:
            try:
                n = int(input('word length = '))
                break
            except:
                print('you have to input integer number, try again')
        print('')
        words_In_q = []
        answer = []
        for i in q_Sentences:
            words_In_q.append(i.split())
        for i in words_In_q:
            for j in i:
                if len(j) == n:
                    answer.append(j)
        print('words with length = ' + str(n) + ':')
        answer = set(answer)
        for i in answer:
            print(i)
        if answer == []:
            print('there is no words with such length')

Target.vikonavchiymethod()


class ContextMenuItem:
    def __init__(self, menu=None, context_register_callback=None, **kwargs):
        print("init context")
        self.menu = menu
        self.contextRegisterCallback = context_register_callback

    def contextMenuEvent(self, event):
        # if not self.activated:
        #     return
        print("menu event")
        if self.contextRegisterCallback:
            self.contextRegisterCallback(self)

        self.menu.exec_(event.screenPos())


students = {
    "Alice": ["ID001", 26, "A"],
    "Bob": ["ID002", 27, "B"],
    "Claire": ["ID003", 17, "C"],
    "Dan": ["ID004", 21, "D"],
    "Emma": ["ID005", 22, "E"]
    }

print(students["Claire"][0]) #get id
print(students["Dan"][0:1]) #age + Grade

# giving each key it's own dictionary as a value

new_students = {
    "Alice":{"id":"ID001","age":26, "grade":"A"},
    "Bob":{"id":"ID002","age":27, "grade":"B"},
    "Claire":{"id":"ID003","age":17, "grade":"C"},
    "Dan":{"id":"ID004","age":21, "grade":"D"},
    "Emma":{"id":"ID005","age":22, "grade":"E"}
    }

new_students(["Dan"]["age"])
new_students(["Emma"]["id"], new_students["Emma"]["grade"])


from flask import Flask, render_template
from flask_bootstrap import Bootstrap
from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField, SelectField
from wtforms.fields.html5 import URLField, TimeField, IntegerField
from wtforms.validators import DataRequired, URL
import csv

app = Flask(__name__)
app.config['SECRET_KEY'] = '8BYkEfBA6O6donzWlSihBXox7C0sKR6b'
Bootstrap(app)

class CafeForm(FlaskForm):
    cafe = StringField('Cafe name', validators=[DataRequired()])
    url_location = URLField('Google Maps URL', validators=[DataRequired(), URL(message="Must be valid URL")])
    open_time = TimeField('Opening Time', validators=[DataRequired()])
    close_time = TimeField('Closing Time', validators=[DataRequired()])
    coffee_rating = SelectField('Coffee Rating', validators=[DataRequired()], choices=[('✘', '✘'), ('☕', '☕'), ('☕☕', '☕☕'), ('☕☕☕', '☕☕☕'), ('☕☕☕☕', '☕☕☕☕'), ('☕☕☕☕☕', '☕☕☕☕☕')])
    wifi_rating = SelectField('WIFI Rating', validators=[DataRequired()], choices=[('✘', '✘'), ('💪', '💪'), ('💪💪', '💪💪'), ('💪💪💪', '💪💪💪'), ('💪💪💪💪', '💪💪💪💪'), ('💪💪💪💪💪', '💪💪💪💪💪')])
    power_rating = SelectField('Power Rating', validators=[DataRequired()], choices=[('✘', '✘'), ('🔌', '🔌'), ('🔌🔌', '🔌🔌'), ('🔌🔌🔌', '🔌🔌🔌'), ('🔌🔌🔌🔌', '🔌🔌🔌🔌'), ('🔌🔌🔌🔌🔌', '🔌🔌🔌🔌🔌')])
    submit = SubmitField('Submit')

# Exercise:
# add: Location URL, open time, closing time, coffee rating, wifi rating, power outlet rating fields
# make coffee/wifi/power a select element with choice of 0 to 5.
#e.g. You could use emojis ☕/💪/✘/🔌
# make all fields required except submit
# use a validator to check that the URL field has a URL entered.
# ---------------------------------------------------------------------------

# all Flask routes below
@app.route("/")
def home():
    return render_template("index.html")

@app.route('/add', methods=["GET", "POST"])
def add_cafe():
    success = False
    form = CafeForm()
    if form.validate_on_submit():
        open_time = f"{form.open_time.data.strftime('%I:%M %p')}"
        close_time = form.close_time.data.strftime('%I:%M %p')
        success = True
        cafe_data = [form.cafe.data, form.url_location.data, open_time, close_time, form.coffee_rating.data, form.wifi_rating.data, form.power_rating.data]
        with open(file="cafe-data.csv", mode="a", encoding="utf-8") as f:
            writer = csv.writer(f)
            writer.writerow(cafe_data)
    return render_template('add.html', form=form, success=success)

@app.route('/cafes')
def cafes():
    with open('cafe-data.csv', newline='', encoding="utf-8") as csv_file:
        csv_data = csv.reader(csv_file, delimiter=',')
        list_of_rows = []
        for row in csv_data:
            list_of_rows.append(row)
    return render_template('cafes.html', cafes=list_of_rows)

if __name__ == '__main__':
    app.run(debug=True)

Python Projects/062 - Flask WTForms Coffee and Wifi Proj/main.py


import matplotlib.pyplot as plt
import numpy as np

mutation=0.01
crossover_rate=0.8
pop_size=100
generation=100
DNA_length=10
X_BOUND=[0,5]
compare_times=10

def F(x):
    return 2*np.sin(x)+np.cos(x)

def fitness(pop):
    return F(pop)+4

def mutate(DNA):
    for i in range(DNA_length):
        if np.random.uniform(0,1,1)<mutation:
            DNA[i] = 0 if DNA[i] ==1 else 1
    return DNA

def crossover(DNA):
    for i in range(compare_times):
        if np.random.uniform(0,1,1)<crossover_rate:
            i=np.random.randint(0,pop_size,size=1)[0]
            j=np.random.randint(0,pop_size,size=1)[0]
            if fitness(translateDNA(DNA)[i])>fitness(translateDNA(DNA)[j]):
                winner=DNA[i]
                loser=DNA[j]
            else:
                winner=DNA[j]
                loser=DNA[i]
            crosspoint=np.random.randint(0,DNA_length,size=1)[0]
            loser[crosspoint:]=winner[crosspoint:]
            loser=mutate(loser)
    return DNA

def translateDNA(pop):
    return pop.dot(2 ** np.arange(DNA_length)[::-1]) / float(2**DNA_length-1) * X_BOUND[1]

DNA = np.random.randint(2, size=(pop_size, DNA_length))
x = np.linspace(*X_BOUND, 200)

for j in range(generation):
    pop=translateDNA(DNA) #pop.shape=(500,)
    plt.plot(x, F(x))
    plt.plot(pop,F(pop),"ro")
    plt.show()
    DNA=crossover(DNA)


from flask_app import app
from flask import request ,render_template, session, redirect, flash
from flask_app.models.user import User
from flask_bcrypt import Bcrypt
from flask_app.models.address import Address
from flask_app.models.pet import Pet
from flask_app.models.sitter import Sitter
from flask_app.models import user
from flask_app.controllers import users
from flask_app.models.service import Service

bcrypt = Bcrypt(app)     # we are creating an object called bcrypt,
                        # which is made by invoking the function Bcrypt with our app as an argument

# SHOW SITTER FORM
@app.route('/sitter/new')
def new_sitter():
    data = {
        'user_id' : session['user_id']
    }
    one_address = Address.get_one_address_sitter(data)
    one_user = User.get_one_user(data)
    return render_template('new_sitter.html',one_address=one_address,one_user=one_user)

# CREATE SITTER IN DB

@app.route("/users/sitter/create", methods=['POST'])
def create_sitter():
    if(Sitter.validate(request.form)):
        data = {
                **request.form,
                'user_id':session['user_id']
            }
        print(data)
        sitter_id = Sitter.create(data)
        session['sitter_id'] = sitter_id
        return redirect('/users/sitter/profile')
    return redirect('/sitter/new')

# SITTER PROFILE PAGE

@app.route('/users/sitter/profile')
def display_my_sitter_profile():
    data = {
        'user_id' : session['user_id']
    }
    one_user = User.get_one_user(data)
    one_sitter = Sitter.get_by_id(data)
    session['sitt_id'] = one_sitter.id
    print(session['sitt_id'])
    return render_template('sitterprofile.html',one_sitter=one_sitter,one_user=one_user)

# SHOW FORM EDIT SITTER

@app.route('/sitters/<int:user_id>/edit')
def edit_sitter(user_id):
    data = {
        'user_id' : user_id
    }
    one_address = Address.get_one_address_sitter(data)
    one_user = User.get_one_user(data)
    one_sitter = Sitter.get_one_sitter(data)
    return render_template('editsitterprofile.html',one_address=one_address,one_user=one_user,one_sitter=one_sitter)

# UPDATE SITTER PROFILE

@app.route('/sitters/<int:user_id>/edit', methods=['POST'])
def update_sitter(user_id):
    data = {
        **request.form,
        'user_id' : user_id
    }
    Sitter.update(data)

    return redirect('/users/sitter/profile')

#request


"""
Determine the slit curvature from a sky observation
"""
import sys
from os.path import basename, dirname
from typing import Any, Dict

import matplotlib.pyplot as plt
import numpy as np
from cpl.core import Msg
from cpl.ui import (
    Frame,
    FrameSet,
    ParameterList,
    ParameterValue,
    PyRecipe,
    ParameterEnum,
)
from numpy.polynomial.polynomial import polyval2d
from scipy import signal
from scipy.interpolate import interp1d
from scipy.ndimage import gaussian_filter1d, median_filter
from scipy.optimize import least_squares
from scipy.linalg import svd
from skimage import exposure

sys.path.append(dirname(__file__))
from cr2res_recipe import CR2RES_RECIPE
Msg.increase_indent = lambda : Msg.set_config(indent=Msg.get_config()["indent"] + 1)
Msg.decrease_indent = lambda : Msg.set_config(indent=Msg.get_config()["indent"] - 1)

def extend_orders(orders, nrow):
    """Extrapolate extra orders above and below the existing ones

    Parameters
    ----------
    orders : array[nord, degree]
        order tracing coefficients
    nrow : int
        number of rows in the image

    Returns
    -------
    orders : array[nord + 2, degree]
        extended orders
    """

    nord, ncoef = orders.shape

    if nord > 1:
        order_low = 2 * orders[0] - orders[1]
        order_high = 2 * orders[-1] - orders[-2]
    else:
        order_low = [0 for _ in range(ncoef)]
        order_high = [0 for _ in range(ncoef - 1)] + [nrow]

    return np.array([order_low, *orders, order_high])

def fix_extraction_width(xwd, orders, cr, ncol):
    """Convert fractional extraction width to pixel range

    Parameters
    ----------
    extraction_width : array[nord, 2]
        current extraction width, in pixels or fractions (for values below 1.5)
    orders : array[nord, degree]
        order tracing coefficients
    column_range : array[nord, 2]
        column range to use
    ncol : int
        number of columns in image

    Returns
    -------
    extraction_width : array[nord, 2]
        updated extraction width in pixels
    """

    if not np.all(xwd > 1.5):
        # if extraction width is in relative scale transform to pixel scale
        x = np.arange(ncol)
        for i in range(1, len(xwd) - 1):
            for j in [0, 1]:
                if xwd[i, j] < 1.5:
                    k = i - 1 if j == 0 else i + 1
                    left = max(cr[[i, k], 0])
                    right = min(cr[[i, k], 1])

                    if right < left:
                        raise ValueError(
                            f"Check your column ranges. Orders {i} and {k} are weird"
                        )

                    current = np.polyval(orders[i], x[left:right])
                    below = np.polyval(orders[k], x[left:right])
                    xwd[i, j] *= np.min(np.abs(current - below))

        xwd[0] = xwd[1]
        xwd[-1] = xwd[-2]

    xwd = np.ceil(xwd).astype(int)

    return xwd

def fix_column_range(column_range, orders, extraction_width, nrow, ncol):
    """Fix the column range, so that no pixels outside the image will be accessed (Thus avoiding errors)

    Parameters
    ----------
    img : array[nrow, ncol]
        image
    orders : array[nord, degree]
        order tracing coefficients
    extraction_width : array[nord, 2]
        extraction width in pixels, (below, above)
    column_range : array[nord, 2]
        current column range
    no_clip : bool, optional
        if False, new column range will be smaller or equal to current column range, otherwise it can also be larger (default: False)

    Returns
    -------
    column_range : array[nord, 2]
        updated column range
    """

    ix = np.arange(ncol)
    # Loop over non extension orders
    for i, order in zip(range(1, len(orders) - 1), orders[1:-1]):
        # Shift order trace up/down by extraction_width
        coeff_bot, coeff_top = np.copy(order), np.copy(order)
        coeff_bot[-1] -= extraction_width[i, 0]
        coeff_top[-1] += extraction_width[i, 1]

        y_bot = np.polyval(coeff_bot, ix)  # low edge of arc
        y_top = np.polyval(coeff_top, ix)  # high edge of arc

        # find regions of pixels inside the image
        # then use the region that most closely resembles the existing column range (from order tracing)
        # but clip it to the existing column range (order tracing polynomials are not well defined outside the original range)
        points_in_image = np.where((y_bot >= 0) & (y_top < nrow))[0]

        if len(points_in_image) == 0:
            raise ValueError(
                f"No pixels are completely within the extraction width for order {i}"
            )

        regions = np.where(np.diff(points_in_image) != 1)[0]
        regions = [(r, r + 1) for r in regions]
        regions = [
            points_in_image[0],
            *points_in_image[(regions,)].ravel(),
            points_in_image[-1],
        ]
        regions = [[regions[i], regions[i + 1] + 1] for i in range(0, len(regions), 2)]
        overlap = [
            min(reg[1], column_range[i, 1]) - max(reg[0], column_range[i, 0])
            for reg in regions
        ]
        iregion = np.argmax(overlap)
        column_range[i] = np.clip(
            regions[iregion], column_range[i, 0], column_range[i, 1]
        )

    column_range[0] = column_range[1]
    column_range[-1] = column_range[-2]

    return column_range

def fix_parameters(xwd, cr, orders, nrow, ncol, nord, ignore_column_range=False):
    """Fix extraction width and column range, so that all pixels used are within the image.
    I.e. the column range is cut so that the everything is within the image

    Parameters
    ----------
    xwd : float, array
        Extraction width, either one value for all orders, or the whole array
    cr : 2-tuple(int), array
        Column range, either one value for all orders, or the whole array
    orders : array
        polynomial coefficients that describe each order
    nrow : int
        Number of rows in the image
    ncol : int
        Number of columns in the image
    nord : int
        Number of orders in the image
    ignore_column_range : bool, optional
        if true does not change the column range, however this may lead to problems with the extraction, by default False

    Returns
    -------
    xwd : array
        fixed extraction width
    cr : array
        fixed column range
    orders : array
        the same orders as before
    """

    if xwd is None:
        xwd = 0.5
    if np.isscalar(xwd):
        xwd = np.tile([xwd, xwd], (nord, 1))
    else:
        xwd = np.asarray(xwd)
        if xwd.ndim == 1:
            xwd = np.tile(xwd, (nord, 1))

    if cr is None:
        cr = np.tile([0, ncol], (nord, 1))
    else:
        cr = np.asarray(cr)
        if cr.ndim == 1:
            cr = np.tile(cr, (nord, 1))

    orders = np.asarray(orders)

    xwd = np.array([xwd[0], *xwd, xwd[-1]])
    cr = np.array([cr[0], *cr, cr[-1]])
    orders = extend_orders(orders, nrow)

    xwd = fix_extraction_width(xwd, orders, cr, ncol)
    if not ignore_column_range:
        cr = fix_column_range(cr, orders, xwd, nrow, ncol)

    orders = orders[1:-1]
    xwd = xwd[1:-1]
    cr = cr[1:-1]

    return xwd, cr, orders

def polyfit2d(x, y, z, w=None, degree=1, x0=None, loss="arctan", method="trf"):

    x = np.asarray(x).ravel()
    y = np.asarray(y).ravel()
    z = np.asarray(z).ravel()
    if w is not None:
        w = np.asarray(w).ravel()
    else:
        w = np.ones_like(x)

    if np.isscalar(degree):
        degree_x = degree_y = degree + 1
    else:
        degree_x = degree[0] + 1
        degree_y = degree[1] + 1

    polyval2d = np.polynomial.polynomial.polyval2d

    def func(c):
        c = c.reshape(degree_x, degree_y)
        value = polyval2d(x, y, c)
        return (value - z) / w

    if x0 is None:
        x0 = np.zeros(degree_x * degree_y)
    else:
        x0 = x0.ravel()

    res = least_squares(func, x0, loss=loss, method=method)
    coef = res.x
    coef = coef.reshape(degree_x, degree_y)

    return coef

class cr2res_util_slit_curv_sky(PyRecipe, CR2RES_RECIPE):
    _name = "cr2res_util_slit_curv_sky"
    _version = "1.0"
    _author = "Ansgar Wehrhahn"
    _email = "ansgar.wehrhahn@astro.su.se"
    _copyright = "GPL-3.0-or-later"
    _synopsis = "Determine the slit curvature from a sky observation"
    _description = "This recipe determines the slit curvature from a sky emission spectrum instead of a wavelength calibration spectrum"

    peak_width = 1
    sigma_cutoff = 3
    curv_degree = 1
    threshold = 1.5
    window_width = 41
    fit_degree = (1, 1)
    mode = "2D"
    peak_function = "spectrum"
    plot = True
    chip = 0
    order = 0
    wl_set = ""

    def __init__(self) -> None:
        super().__init__()
        self.parameters = ParameterList(
            [
                ParameterValue(
                    name="detector",
                    context="cr2res_util_slit_curv_sky",
                    description="Detector to run",
                    default=-1,
                ),
                ParameterValue(
                    name="order",
                    context="cr2res_util_slit_curv_sky",
                    description="Order to run",
                    default=-1,
                ),
                ParameterEnum(
                    name="fit_mode",
                    context="cr2res_util_slit_curv_sky",
                    description="Whether to perform a 1d or 2d polynomial fit",
                    default="2D",
                    alternatives=("1D", "2D"),
                ),
                ParameterValue(
                    name="degree_x",
                    context="cr2res_util_slit_curv_sky",
                    description="Polynomial degree along x",
                    default=1,
                ),
                ParameterValue(
                    name="degree_y",
                    context="cr2res_util_slit_curv_sky",
                    description="Polynomial degree along y",
                    default=1,
                ),
                ParameterValue(
                    name="plot",
                    context="cr2res_util_slit_curv_sky",
                    description="Activates plots",
                    default=True,
                ),
            ]
        )

    def determine_slit_curvature(self, trace_wave, science, extracted, bpm=None):
        tilts = {}
        shears = {}

        header = science[0].header
        self.wl_set = header["ESO INS WLEN ID"]

        for chip in self.detectors:
            Msg.info(self.name, f"Processing Detector {chip}")
            Msg.increase_indent()
            self.chip = chip

            ext = self.get_chip_extension(chip)
            exterr = self.get_chip_extension(chip, error=True)

            tw_data = trace_wave[ext].data
            data = science[ext].data
            err = science[exterr].data
            # blaze_data = blaze[ext].data
            extract_data = extracted[ext].data
            if bpm is not None:
                bpm_data = bpm[ext].data != 0
                data[bpm_data] = np.nan

            order_traces = []
            extraction_width = []
            column_range = []
            upper_ycen = []
            lower_ycen = []
            orders = self.get_orders_trace_wave(tw_data)
            xsize = len(extract_data)

            for order in orders:
                lower, middle, upper = self.get_order_trace(tw_data, order)
                upper_ycen += [upper]
                lower_ycen += [lower]
                height_upp = int(np.ceil(np.min(upper - middle)))
                height_low = int(np.ceil(np.min(middle - lower)))
                idx = tw_data["Order"] == order
                order_traces += [tw_data[idx]["All"][0][::-1]]
                column_range += [[10, xsize - 10]]
                extraction_width += [[height_upp, height_low]]

            order_traces = np.asarray(order_traces)
            column_range = np.asarray(column_range)
            extraction_width = np.asarray(extraction_width)

            spectrum = np.array(
                [
                    extract_data[self.get_table_column(order, 1, "SPEC")]
                    for order in orders
                ]
            )

            for i, order in enumerate(orders):
                # Smooth spectrum for easier peak detection
                spectrum[i][np.isnan(spectrum[i])] = 0
                spectrum[i] = gaussian_filter1d(spectrum[i], 3)
                # Limit the size of the extraction width
                # to avoid the interorder area
                extraction_width[i, 0] -= 10
                extraction_width[i, 1] -= 10

            order_range = (0, order_traces.shape[0])
            data = np.ma.array(data, mask=~np.isfinite(data))
            tilt, shear = self.execute(
                spectrum,
                data,
                order_traces,
                extraction_width,
                column_range,
                order_range,
                chip=chip,
                order_nrs=orders,
            )

            tilts[chip] = tilt
            shears[chip] = shear
            Msg.decrease_indent()

        # Create output
        for chip in self.detectors:
            ext = self.get_chip_extension(chip)
            tw_data = trace_wave[ext].data
            orders = self.get_orders_trace_wave(tw_data)
            x = np.arange(1, 2048 + 1)
            for i, order in enumerate(orders):
                t = tilts[chip][i]
                s = shears[chip][i]
                _, ycen, _ = self.get_order_trace(tw_data, order, clip=False)
                # Convert to the global reference system
                t -= 2 * ycen * s

                # Fit overarching polynomial
                ct = np.polyfit(x, t, 2)
                cs = np.polyfit(x, s, 2)

                Msg.debug(
                    self.name,
                    f"CHIP: {chip} ORDER: {order} TILT: {np.array2string(ct, precision=2)}",
                )

                # Write the data back to the fits file
                # The indexing is chosen such that the data is actually
                # stored in the tw object and not lost
                idx = tw_data["Order"] == order
                trace_wave[ext].data["SlitPolyA"][idx] = [0, 1, 0]
                trace_wave[ext].data["SlitPolyB"][idx] = ct[::-1]
                trace_wave[ext].data["SlitPolyC"][idx] = cs[::-1]

        return trace_wave

    def run(self, frameset: FrameSet, settings: Dict[str, Any]) -> FrameSet:
        # Check the input parameters
        self.parameters = self.parse_parameters(settings)
        self.detectors = self.parameters["detector"].value
        if self.detectors <= 0:
            self.detectors = [1, 2, 3]
        else:
            self.detectors = [
                self.detectors,
            ]

        self.orders = self.parameters["order"].value

        self.mode = self.parameters["fit_mode"].value
        if self.mode == "1D":
            self.fit_degree = self.parameters["degree_x"].value
        elif self.mode == "2D":
            self.fit_degree = (
                self.parameters["degree_x"].value,
                self.parameters["degree_y"].value,
            )
        else:
            raise ValueError

        self.plot = self.parameters["plot"].value

        Msg.debug(self.name, f"Fit degree {self.fit_degree}")

        frames = self.filter_frameset(
            frameset,
            science="UTIL_CALIB",
            trace_wave="CAL_FLAT_TW",
            spectrum="UTIL_CALIB_EXTRACT_1D",
            bpm="CAL_DARK_BPM",
            validate=False,
        )
        # Validate the input
        if len(frames["science"]) != 1:
            raise ValueError("Expected exactly 1 science frame")
        if len(frames["trace_wave"]) != 1:
            raise ValueError("Expected exactly 1 trace wave frame")
        if len(frames["spectrum"]) != 1:
            raise ValueError("Expected exactly 1 spectrum frame")

        # Run the actual work
        trace_wave = frames["trace_wave"][0].as_hdulist()
        science = frames["science"][0].as_hdulist()
        spectrum = frames["spectrum"][0].as_hdulist()
        if len(frames["bpm"]) >= 1:
            bpm = frames["bpm"][0].as_hdulist()
        else:
            bpm = None

        trace_wave = self.determine_slit_curvature(trace_wave, science, spectrum, bpm)

        # Save the results
        outfile = basename(frames["trace_wave"][0].file)
        trace_wave.writeto(outfile, overwrite=True)

        # Return data back to PyEsorex
        fs = FrameSet(
            [
                Frame(outfile, tag="CAL_SLIT_CURV_TW"),
            ]
        )
        return fs

    @staticmethod
    def _fix_inputs(shape, orders, extraction_width, column_range, order_range):
        nrow, ncol = shape
        nord = len(orders)

        extraction_width, column_range, orders = fix_parameters(
            extraction_width, column_range, orders, nrow, ncol, nord
        )

        column_range = column_range[order_range[0] : order_range[1]]
        extraction_width = extraction_width[order_range[0] : order_range[1]]
        orders = orders[order_range[0] : order_range[1]]

        n = order_range[1] - order_range[0]
        order_range = (0, n)

        return orders, extraction_width, column_range, order_range

    def _find_peaks(self, vec, cr):
        # TODO: Test this heuristic on other spectra?
        # First find all peaks regardless of height
        # Then only use the peaks above the mean
        # We specifically want the mean and not the median
        # as we need the inluence of the large peaks to be stronger
        # This should work well for "natural" spectra that have random small peaks
        # But how well does it work on well performing wavelength calibration images?
        vec -= np.nanpercentile(vec, 10)
        vec[~np.isfinite(vec)] = 0
        peaks, stats = signal.find_peaks(
            vec, width=self.peak_width, distance=self.window_width
        )
        prominence = stats["prominences"]
        # height = np.mean(prominence)
        height = np.median(prominence)
        # height = np.nanpercentile(vec, 20)
        Msg.info(self.name, f"Peak detection threshold height: {height}")
        peaks, _ = signal.find_peaks(
            vec, prominence=height, width=self.peak_width, distance=self.window_width
        )

        # Remove peaks at the edge
        peaks = peaks[
            (peaks >= self.window_width + 1)
            & (peaks < len(vec) - self.window_width - 1)
        ]

        # Manually remove bad areas of peaks
        if self.wl_set == "L3262":
            if self.chip == 3 and self.order == 2:
                peaks = peaks[peaks < 1000]
            elif self.chip == 3 and self.order == 3:
                peaks = peaks[peaks > 150]
                peaks = peaks[(peaks < 700) | (peaks > 1100)]
                peaks = peaks[(peaks < 1600) | (peaks > 1880)]
        elif self.wl_set == "L3340":
            if self.chip == 1 and self.order == 3:
                peaks = peaks[peaks > 200]
                peaks = peaks[(peaks < 300) | (peaks > 600)]
            elif self.chip == 3 and self.order == 2:
                peaks = peaks[peaks < 1200]
            elif self.chip == 3 and self.order == 3:
                peaks = peaks[peaks < 1500]
        elif self.wl_set == "L3426":
            if self.chip == 2 and self.order == 2:
                peaks = []
        elif self.wl_set == "M4318":
            if self.chip == 1 and self.order == 3:
                peaks = peaks[peaks > 400]

        if self.plot:
            plt.clf()
            plt.plot(vec)
            plt.plot(peaks, vec[peaks], "d")
            plt.savefig(f"cr2res_util_slit_curv_sky_peaks_c{self.chip}_o{self.order}.png")

        # Remove the offset, due to vec being a subset of extracted
        peaks += cr[0]
        return vec, peaks

    def _determine_curvature_single_line(
        self, original, extracted, peak, ycen, ycen_int, xwd, chip=0, order=0, npeak=0
    ):
        """
        Fit the curvature of a single peak in the spectrum

        This is achieved by fitting a model, that consists of gaussians
        in spectrum direction, that are shifted by the curvature in each row.

        Parameters
        ----------
        original : array of shape (nrows, ncols)
            whole input image
        peak : int
            column position of the peak
        ycen : array of shape (ncols,)
            row center of the order of the peak
        xwd : 2 tuple
            extraction width above and below the order center to use

        Returns
        -------
        tilt : float
            first order curvature
        shear : float
            second order curvature
        """
        _, ncol = original.shape

        # look at +- width pixels around the line
        # Extract short horizontal strip for each row in extraction width
        # Then fit a gaussian to each row, to find the center of the line
        x = peak + np.arange(-self.window_width, self.window_width + 1)
        x = x[(x >= 0) & (x < ncol)]
        xmin, xmax = x[0], x[-1] + 1

        # Look above and below the line center
        y = np.arange(-xwd[0], xwd[1] + 1)[:, None] - ycen[xmin:xmax][None, :]

        x = x[None, :]
        idx = self.make_index(ycen_int - xwd[0], ycen_int + xwd[1], xmin, xmax)
        img = original[idx]

        # Remove the background
        sl = np.ma.median(img, axis=1)
        sl = gaussian_filter1d(sl.data.ravel(), 10)
        img = img - sl[:, None]

        # Normalize the image
        img_compressed = np.ma.compressed(img)
        img_min = np.percentile(img_compressed, 5)
        img_max = np.percentile(img_compressed, 95)
        img -= img_min
        img /= img_max - img_min

        img[:] = exposure.equalize_hist(img.data, mask=~img.mask)
        img[np.isnan(img)] = np.ma.masked

        # Normalize the spectrum
        sp = extracted
        sp -= sp[xmin:xmax].min()
        sp /= sp[xmin:xmax].max()
        sp_x = np.arange(0, len(sp))

        peak_func = {
            "gaussian": self.gaussian,
            "lorentzian": self.lorentzian,
            "spectrum": self.collapsed,
        }
        peak_func = peak_func[self.peak_function]

        def model(coef):
            A, middle, sig, *curv = coef
            mu = middle + shift(curv)
            if self.peak_function in ["gaussian", "lorentzian"]:
                mod = peak_func(x, A, mu, sig)
            elif self.peak_function in ["spectrum"]:
                mod = peak_func(x, A, sp, sp_x - peak, mu)
            # mod = mod * sl[:, None]
            return mod

        def model_compressed(coef):
            return np.ma.compressed((model(coef) - img).ravel())

        A = 1  # np.nanpercentile(img_compressed, 95)
        sig = (xmax - xmin) / 4  # TODO
        if self.curv_degree == 1:
            shift = lambda curv: curv[0] * y
        elif self.curv_degree == 2:
            shift = lambda curv: (curv[0] + curv[1] * y) * y
        else:
            raise ValueError("Only curvature degrees 1 and 2 are supported")
        # res = least_squares(model, x0=[A, middle, sig, 0], loss="soft_l1", bounds=([0, xmin, 1, -10],[np.inf, xmax, xmax, 10]))
        x0 = [A, peak, sig] + [0] * self.curv_degree
        bounds = [(-np.inf, np.inf), (xmin, xmax), (0, 10 * sig), (-0.1, 0.1)]
        if self.curv_degree == 2:
            bounds += [(-1, 1)]
        bounds = np.array(bounds).T
        res = least_squares(
            model_compressed,
            x0=x0,
            method="trf",
            loss="soft_l1",
            f_scale=0.1,
            bounds=bounds,
        )

        # To get the ucnertainties of the fit
        # Do Moore-Penrose inverse discarding zero singular values.
        # See scipy curve_fit
        _, s, VT = svd(res.jac, full_matrices=False)
        threshold = np.finfo(float).eps * max(res.jac.shape) * s[0]
        s = s[s > threshold]
        VT = VT[:s.size]
        pcov = np.dot(VT.T / s**2, VT)
        unc = np.sqrt(np.diag(pcov))

        if self.curv_degree == 1:
            tilt, shear = res.x[3], 0
            # utilt, ushear = unc[3], 1
            model_data = model(res.x)
            utilt = unc[3]
            ushear = 1
        elif self.curv_degree == 2:
            tilt, shear = res.x[3], res.x[4]
            utilt, ushear = unc[3], unc[4]
        else:
            tilt, shear = 0, 0
            utilt, ushear = 1, 1

        if self.plot:
            nrows, _ = img.shape
            model_data = model(res.x)
            middle = nrows // 2
            y = np.arange(-middle, -middle + nrows)
            x = res.x[1] - xmin + (tilt + shear * y) * y
            y += middle
            vmin, vmax = 0, 1
            plt.clf()
            plt.subplot(121)
            plt.imshow(img, interpolation="none", origin="lower", vmin=vmin, vmax=vmax)
            plt.plot(x, y, "r")
            plt.subplot(122)
            plt.imshow(model_data, interpolation="none", origin="lower", vmin=vmin, vmax=vmax)
            plt.plot(x, y, "r")
            plt.suptitle(f"UNC: {utilt}")
            plt.savefig(f"cr2res_util_slit_curv_sky_c{chip}_o{order}_p{npeak}.png")

        return tilt, shear, utilt, ushear

    def _fit_curvature_single_order(self, peaks, tilt, shear):
        try:
            middle = np.median(tilt)
            sigma = np.percentile(tilt, (32, 68))
            sigma = middle - sigma[0], sigma[1] - middle
            mask = (tilt >= middle - 5 * sigma[0]) & (tilt <= middle + 5 * sigma[1])
            peaks, tilt, shear = peaks[mask], tilt[mask], shear[mask]

            coef_tilt = np.zeros(self.fit_degree + 1)
            res = least_squares(
                lambda coef: np.polyval(coef, peaks) - tilt,
                x0=coef_tilt,
                loss="arctan",
            )
            coef_tilt = res.x

            coef_shear = np.zeros(self.fit_degree + 1)
            res = least_squares(
                lambda coef: np.polyval(coef, peaks) - shear,
                x0=coef_shear,
                loss="arctan",
            )
            coef_shear = res.x

        except:
            coef_tilt = np.zeros(self.fit_degree + 1)
            coef_shear = np.zeros(self.fit_degree + 1)

        return coef_tilt, coef_shear, peaks

    def _determine_curvature_all_lines(
        self,
        original,
        extracted,
        orders,
        extraction_width,
        column_range,
        order_range,
        chip=0,
        order_nrs=None,
    ):
        ncol = original.shape[1]
        # Store data from all orders
        all_peaks = []
        all_tilt = []
        all_shear = []
        all_utilt = []
        all_ushear = []
        plot_vec = []

        n = order_range[1] - order_range[0]
        if order_nrs is None:
            order_nrs = np.arange(n)

        for j in range(n):
            Msg.debug(self.name, f"Processing Order {order_nrs[j]}")
            self.order = order_nrs[j]

            cr = column_range[j]
            xwd = extraction_width[j]
            ycen = np.polyval(orders[j], np.arange(ncol))
            ycen_int = ycen.astype(int)
            ycen -= ycen_int

            # Find peaks
            vec = extracted[j, cr[0] : cr[1]]
            vec, peaks = self._find_peaks(vec, cr)

            npeaks = len(peaks)
            Msg.info(self._name, f"{npeaks} peaks found in Order {order_nrs[j]}")

            # Determine curvature for each line seperately
            tilt = np.zeros(npeaks)
            shear = np.zeros(npeaks)
            utilt = np.zeros(npeaks)
            ushear = np.zeros(npeaks)

            mask = np.full(npeaks, True)
            for ipeak, peak in enumerate(peaks):
                Msg.debug(self.name, f"Processing Peak {ipeak}")
                try:
                    tilt[ipeak], shear[ipeak], utilt[ipeak], ushear[ipeak] = self._determine_curvature_single_line(
                        original,
                        extracted[j],
                        peak,
                        ycen,
                        ycen_int,
                        xwd,
                        chip=chip,
                        order=order_nrs[j],
                        npeak=ipeak,
                    )
                except RuntimeError:  # pragma: no cover
                    mask[ipeak] = False

            # Store results
            all_peaks += [peaks[mask]]
            all_tilt += [tilt[mask]]
            all_shear += [shear[mask]]
            all_utilt += [utilt[mask]]
            all_ushear += [ushear[mask]]
            plot_vec += [vec]
        return all_peaks, all_tilt, all_shear, all_utilt, all_ushear, plot_vec

    def fit(self, peaks, tilt, shear, utilt, ushear, order_range):
        n = order_range[1] - order_range[0]
        if self.mode == "1D":
            coef_tilt = np.zeros((n, self.fit_degree + 1))
            coef_shear = np.zeros((n, self.fit_degree + 1))
            for j in range(n):
                coef_tilt[j], coef_shear[j], _ = self._fit_curvature_single_order(
                    peaks[j], tilt[j], shear[j]
                )
        elif self.mode == "2D":
            x = np.concatenate(peaks)
            y = [np.full(len(p), i) for i, p in enumerate(peaks)]
            y = np.concatenate(y)
            z = np.concatenate(tilt)
            w = np.concatenate(utilt)
            coef_tilt = polyfit2d(x, y, z, w=w, degree=self.fit_degree, loss="soft_l1")

            zs = np.concatenate(shear)
            ws = np.concatenate(ushear)
            coef_shear = polyfit2d(x, y, zs, w=ws, degree=self.fit_degree, loss="arctan")

            if self.plot:
                plt.clf()
                orders = np.unique(y)
                zm = [np.median(z[y==o]) for o in orders]
                zstep = np.median(np.diff(zm)) * 10
                X, Y = np.meshgrid(
                    np.linspace(0, 2048, 100), orders
                )
                Z = np.polynomial.polynomial.polyval2d(X, Y, coef_tilt)
                for i, o in enumerate(orders):
                    l1 = plt.plot(x[y==o], z[y==o] + i * zstep, "+")
                    plt.plot(X[Y==o], Z[Y==o] + i * zstep, "--", color=l1[0]._color)
                plt.savefig(f"cr2res_util_slit_curv_sky_tilt_c{self.chip}.png")

        Msg.debug(self.name, f"Tilt Coef:\n{np.array2string(coef_tilt, precision=2)}")
        Msg.debug(self.name, f"Shear Coef:\n{coef_shear}")

        return coef_tilt, coef_shear

    def eval(self, peaks, order, coef_tilt, coef_shear):
        if self.mode == "1D":
            tilt = np.zeros(peaks.shape)
            shear = np.zeros(peaks.shape)
            for i in np.unique(order):
                idx = order == i
                tilt[idx] = np.polyval(coef_tilt[i], peaks[idx])
                shear[idx] = np.polyval(coef_shear[i], peaks[idx])
        elif self.mode == "2D":
            tilt = polyval2d(peaks, order, coef_tilt)
            shear = polyval2d(peaks, order, coef_shear)
        return tilt, shear

    def execute(
        self,
        extracted,
        original,
        orders,
        extraction_width,
        column_range,
        order_range,
        chip=0,
        order_nrs=None,
    ):

        (orders, extraction_width, column_range, order_range,) = self._fix_inputs(
            original.shape,
            orders,
            extraction_width,
            column_range,
            order_range,
        )

        peaks, tilt, shear, utilt, ushear, vec = self._determine_curvature_all_lines(
            original,
            extracted,
            orders,
            extraction_width,
            column_range,
            order_range,
            chip=chip,
            order_nrs=order_nrs,
        )

        coef_tilt, coef_shear = self.fit(peaks, tilt, shear, utilt, ushear, order_range)

        iorder, ipeaks = np.indices(extracted.shape)
        tilt, shear = self.eval(ipeaks, iorder, coef_tilt, coef_shear)

        return tilt, shear

    @staticmethod
    def gaussian(x, A, mu, sig):
        """
        A: height
        mu: offset from central line
        sig: standard deviation
        """
        return A * np.exp(-np.power(x - mu, 2.0) / (2 * np.power(sig, 2.0)))

    @staticmethod
    def lorentzian(x, A, x0, mu):
        """
        A: height
        x0: offset from central line
        mu: width of lorentzian
        """
        return A * mu / ((x - x0) ** 2 + 0.25 * mu ** 2)

    @staticmethod
    def collapsed(x, A, sp, sp_x, mu):
        interp = interp1d(sp_x, sp, bounds_error=False, fill_value="extrapolate")
        res = A * interp(x - mu)
        return res

if __name__ == "__main__":
    fs = FrameSet("/scratch/ptah/anwe5599/CRIRES/2022-12-23_M4318/extr/slit_curv.sof")
    settings = {"degree_x": 2, "degree_y": 1, "detector": 1}
    module = cr2res_util_slit_curv_sky()
    module.run(fs, settings)


import glob
import os
import shutil

IN_DIR = 'C:\\Users\\mbargury\\Downloads\\New\\Slowmotion_180322_160952\\JPG\\'
OUT_DIR = 'C:\\Users\\mbargury\\Downloads\\New\\Slowmotion_180322_160952\\FILTERED_JPG\\'
EXT = '.jpg'

# locate in files
files = glob.glob(os.path.join(IN_DIR, '*%s' % EXT))

# create out dir
if not os.path.exists(OUT_DIR):
    os.makedirs(OUT_DIR)

# move wanted files
for f in files:
    if (('FrameID(02)' in f) or
       ('FrameID(04)' in f) or
       ('FrameID(06)' in f)):
        shutil.copyfile(f, os.path.join(OUT_DIR, os.path.basename(f)))

print('DONE')


import matplotlib.pyplot as plt
import numpy
from plotter import Plotter
from dimensionality_reduction import DimensionalityReduction
from validator import Validation
from evaluator import Evaluator
from mlFunc import *

def validation(DTR,LTR, VA, dimRed):
    print("##################################")
    print("VALIDATION")
    print("##################################")
    print("***MVG VALIDATION***")
    VA.MVG_validation(DTR,LTR, 0.5, 1,10)
    for i in range (7,10):
         print("PCA con", i)
         DPA = dimRed.PCA(DTR, i)
         VA.MVG_validation(DPA.T, LTR, 0.5, 1, 10)
    print("***LR VALIDATION***")
    VA.LR_validation(DTR, LTR, 0.5, 1, 10, plot=True)
    for i in range (7,10):
         print("PCA con", i)
         DPA = dimRed.PCA(DTR, i)
         VA.LR_validation(DPA.T,LTR, 0.1,1,10, False)

    K_arr = [0.1, 1.0, 10.0]
    C_arr = [0.01, 0.1, 1.0, 10.0]
    for k in K_arr:
        for c in C_arr:
            print("SVM, K: ",k," C: ", c)
            VA.SVM_validation(DTR, LTR, 0.9, 1, 10, k, c)
    VA.SVM_validation(DTR, LTR, 0.5, 1, 10, 10, 10, False)

    VA.GMM_validation(DTR,LTR, 0.5,1,10, 1,4, 0.1, 0.01, True)
    for i in range (7,10):
         print("PCA con", i)
         DPA = dimRed.PCA(DTR, i)
         VA.GMM_validation(DPA.T,LTR, 0.5,1,10, 1,4, 0.1, 0.01)

    VA.plot_ROC(DTR, LTR, 0.5)
    print("Comparing DCF")
    VA.plot_minDCF_cal_score(DTR, LTR, 0.5)
    return

def evaluation(DTE, LTE, DTR, LTR, EV,  dimRed):
    print("##################################")
    print("EVALUATION")
    print("##################################")
    print("***MVG EVALUATION***")
    EV.MVG_evaluation(DTE, LTE, DTR, LTR, 0.5, 1, 10)
    for i in range(7, 10):
       print("PCA con", i)
       DPA = dimRed.PCA(DTR, i)
       DPE = dimRed.PCA_DTE(DTR, i,DTE)
       EV.MVG_evaluation(DPE.T, LTE, DPA.T, LTR, 0.5, 1, 10)

    print("***LR EVALUATION***")

    EV.LR_evaluation(DTE, LTE, DTR, LTR, 0.1, 1, 10, False)
    for i in range(7, 10):
        print("PCA con", i)
        DPA = dimRed.PCA(DTR, i)
        DPE = dimRed.PCA_DTE(DTR, i,DTE)
        EV.LR_evaluation(DPE.T, LTE, DPA.T, LTR, 0.5, 1, 10)
    print("***SVM EVALUATION***")
    EV.SVM_evaluation(DTE.T, LTE, DTR.T, LTR, 0.5, 1, 10)
    for i in range(7, 10):
       print("PCA con", i)
       DPA = dimRed.PCA(DTR, i)
       DPE = dimRed.PCA_DTE(DTR, i,DTE)
       EV.SVM_evaluation(DPE, LTE, DPA, LTR, 0.9, 1, 10)

    print("***GMM EVALUATION***")
    EV.GMM_evaluation(DTE, LTE, DTR, LTR, 0.9, 1, 10, 1, 16, 0.1, 0.01)
    for i in range(7, 10):
        print("PCA con", i)
        DPA = dimRed.PCA(DTR, i)
        DPE = dimRed.PCA_DTE(DTR, i,DTE)
        EV.GMM_evaluation(DPE.T, LTE, DPA.T, LTR, 0.9, 1, 10, 1, 16, 0.1, 0.01)

    EV.plot_ROC(DTR, LTR, DTE,LTE,0.5)
    print("Comparing DCF")
    EV.plot_minDCF_cal_score(DTR, LTR, DTE,LTE,0.5)

if __name__ == "__main__":

    DTR,LTR = load("Train.txt")

    DTE, LTE = load("Test.txt")

    print("TRAINING AUTHENTIC: ", DTR.T[:,LTR==1].shape[1])
    print("TRAINING SPOFFED", DTR.T[:, LTR==0].shape[1])

    print("TEST AUTHENTIC: ", DTE.T[:, LTE == 1].shape[1])
    print("TEST SPOFFED", DTE.T[:, LTE == 0].shape[1])

    plt = Plotter()
    dimRed = DimensionalityReduction()
    VA = Validation()
    EV = Evaluator()

    plt.plot_correlations(DTR.T, "heatmap")
    plt.plot_correlations(DTR.T[:, LTR == 0], "heatmap_spoofed_", cmap="Reds")
    plt.plot_correlations(DTR.T[:, LTR == 1], "heatmap_authentic_", cmap="Blues")

    validation(DTR, LTR, VA, dimRed)
    evaluation(DTE, LTE, DTR, LTR, EV, dimRed)


from __future__ import annotations

import base64
import io
import json
import os
import signal
import socket
import sys
import tempfile
import threading
import uuid
from abc import ABC, abstractmethod
from pathlib import Path
from time import time
from typing import Any

import numpy as np
from dash import Dash, callback_context, dcc, no_update
from dash.dependencies import Input, Output, State
from dash.development.base_component import Component
from flask import Flask
from geoh5py.data import Data
from geoh5py.groups import PropertyGroup
from geoh5py.objects import ObjectBase
from geoh5py.shared import Entity
from geoh5py.shared.utils import fetch_active_workspace, is_uuid
from geoh5py.ui_json import InputFile
from geoh5py.workspace import Workspace
from PySide2 import QtCore, QtWebEngineWidgets, QtWidgets

from geoapps_utils.application.layout import object_selection_layout
from geoapps_utils.driver.params import BaseParams

class BaseDashApplication(ABC):
    """
    Base class for geoapps dash applications
    """

    _params = None
    _param_class = BaseParams
    _driver_class = None
    _workspace = None
    _app_initializer: dict | None = None
    _ui_json_data: dict | None = None

    def __init__(
        self,
        ui_json: InputFile | None = None,
        ui_json_data: dict | None = None,
        params: BaseParams | None = None,
    ):
        """
        Set initial ui_json_data from input file and open workspace.
        """
        if params is not None:
            # Launched from notebook
            # Params for initialization are coming from params
            # ui_json_data is provided
            self.params = params
        elif (
            ui_json is not None
            and getattr(ui_json, "path", None) is not None
            and Path(ui_json.path).exists()
        ):
            # Launched from terminal
            # Params for initialization are coming from ui_json
            # ui_json_data starts as None
            self.params = self._param_class(ui_json)
            input_file = self.params.input_file
            if input_file is not None:
                ui_json_data = input_file.demote(self.params.to_dict())
        self._ui_json_data = ui_json_data

        assert self.params is not None, "No parameters provided."
        self.workspace = self.params.geoh5
        self.workspace.open()
        if self._driver_class is not None:
            self.driver = self._driver_class(self.params)

    @property
    @abstractmethod
    def app(self) -> Dash:
        """Dash app"""

    def get_data_options(
        self,
        ui_json_data: dict,
        object_uid: str | None,
        object_name: str = "objects",
        trigger: str | None = None,
    ) -> list:
        """
        Get data dropdown options from a given object.

        :param ui_json_data: Uploaded ui.json data to read object from.
        :param object_uid: Selected object in object dropdown.
        :param object_name: Object parameter name in ui.json.
        :param trigger: Callback trigger.

        :return options: Data dropdown options.
        """
        obj = None
        if trigger == "ui_json_data" and object_name in ui_json_data:
            if is_uuid(ui_json_data[object_name]):
                object_uid = ui_json_data[object_name]
            elif self.workspace.get_entity(ui_json_data[object_name])[0] is not None:
                object_uid = self.workspace.get_entity(ui_json_data[object_name])[0].uid

        if object_uid is not None and is_uuid(object_uid):
            for entity in self.workspace.get_entity(uuid.UUID(object_uid)):
                if isinstance(entity, ObjectBase):
                    obj = entity

        if obj:
            options = []
            for child in obj.children:
                if isinstance(child, Data):
                    if child.name != "Visual Parameters":
                        options.append(
                            {"label": child.name, "value": "{" + str(child.uid) + "}"}
                        )
            options = sorted(options, key=lambda d: d["label"])

            return options
        return []

    def get_params_dict(self, update_dict: dict) -> dict:
        """
        Get dict of current params.

        :param update_dict: Dict of parameters with new values to convert to a params dict.

        :return output_dict: Dict of current params.
        """
        if self.params is None:
            return {}

        output_dict: dict[str, Any] = {}
        # Get validations to know expected type for keys in self.params.
        validations = self.params.validations

        # Loop through self.params and update self.params with locals_dict.
        for key in self.params.to_dict():
            if key not in update_dict:
                continue
            if validations is not None:
                if bool in validations[key]["types"] and isinstance(
                    update_dict[key], list
                ):
                    # Convert from dash component checklist to bool
                    if not update_dict[key]:
                        output_dict[key] = False
                    else:
                        output_dict[key] = True
                    continue
                if (
                    float in validations[key]["types"]
                    and int not in validations[key]["types"]
                    and isinstance(update_dict[key], int)
                ):
                    # Checking for values that Dash has given as int when they should be floats.
                    output_dict[key] = float(update_dict[key])
                    continue
            if is_uuid(update_dict[key]):
                output_dict[key] = self.workspace.get_entity(
                    uuid.UUID(update_dict[key])
                )[0]
            else:
                output_dict[key] = update_dict[key]
        return output_dict

    @staticmethod
    def init_vals(layout: list[Component], ui_json_data: dict):
        """
        Initialize dash components in layout from ui_json_data.

        :param layout: Dash layout.
        :param ui_json_data: Uploaded ui.json data.
        """
        for comp in layout:
            BaseDashApplication._init_component(comp, ui_json_data)

    @staticmethod
    def _init_component(comp: Component, ui_json_data: dict):
        if hasattr(comp, "children") and not isinstance(comp, dcc.Markdown):
            BaseDashApplication.init_vals(comp.children, ui_json_data)
            return

        if hasattr(comp, "id") and comp.id in ui_json_data:
            if isinstance(comp, dcc.Store):
                comp.data = ui_json_data[comp.id]
                return

            if isinstance(comp, dcc.Dropdown):
                comp.value = ui_json_data[comp.id]
                if comp.value is None:
                    comp.value = "None"
                if not hasattr(comp, "options"):
                    comp_option = ui_json_data[comp.id]
                    if comp_option is None:
                        comp_option = "None"
                    comp.options = [comp_option]
                return

            if isinstance(comp, dcc.Checklist):
                if ui_json_data[comp.id]:
                    comp.value = [True]
                else:
                    comp.value = []
            else:
                comp.value = ui_json_data[comp.id]

    @staticmethod
    def update_visibility_from_checklist(checklist_val: list[bool]) -> dict:
        """
        Update visibility of a component from a checklist value.

        :param checklist_val: Checklist value.

        :return visibility: Component style.
        """
        if checklist_val:
            return {"display": "block"}
        return {"display": "none"}

    @property
    def params(self) -> BaseParams | None:
        """
        Application parameters
        """
        return self._params

    @params.setter
    def params(self, params: BaseParams):
        assert isinstance(
            params, BaseParams
        ), f"Input parameters must be an instance of {BaseParams}"

        self._params = params

    @property
    def workspace(self):
        """
        Current workspace.
        """
        return self._workspace

    @workspace.setter
    def workspace(self, workspace):
        # Close old workspace
        if self._workspace is not None:
            self._workspace.close()
        self._workspace = workspace

class ObjectSelection:
    """
    Dash app to select workspace and object.

    Creates temporary workspace with the object, and
    opens a Qt window to run an app.
    """

    _app_initializer: dict | None = None

    def __init__(
        self,
        app_name: str,
        app_initializer: dict,
        app_class: type[BaseDashApplication],
        param_class: type[BaseParams],
        **kwargs,
    ):
        self._app_name = None
        self._app_class = BaseDashApplication
        self._param_class = BaseParams
        self._workspace = None

        self.app_name = app_name
        self.app_class = app_class
        self.param_class = param_class

        app_initializer.update(kwargs)
        self.params = self.param_class(**app_initializer)
        self._app_initializer = {
            key: value
            for key, value in app_initializer.items()
            if key not in self.params.param_names
        }
        self.workspace = self.params.geoh5

        external_stylesheets = ["https://codepen.io/chriddyp/pen/bWLwgP.css"]
        server = Flask(__name__)
        self.app = Dash(
            server=server,
            url_base_pathname=os.environ.get("JUPYTERHUB_SERVICE_PREFIX", "/"),
            external_stylesheets=external_stylesheets,
        )
        self.app.layout = object_selection_layout

        # Set up callbacks
        self.app.callback(
            Output(component_id="objects", component_property="options"),
            Output(component_id="objects", component_property="value"),
            Output(component_id="ui_json_data", component_property="data"),
            Output(component_id="upload", component_property="filename"),
            Output(component_id="upload", component_property="contents"),
            Input(component_id="ui_json_data", component_property="data"),
            Input(component_id="upload", component_property="filename"),
            Input(component_id="upload", component_property="contents"),
            Input(component_id="objects", component_property="value"),
        )(self.update_object_options)
        self.app.callback(
            Output(component_id="launch_app_markdown", component_property="children"),
            State(component_id="objects", component_property="value"),
            Input(component_id="launch_app", component_property="n_clicks"),
        )(self.launch_qt)

    def update_object_options(
        self,
        ui_json_data: dict,
        filename: str,
        contents: str,
        objects: str,
        trigger: str | None = None,
    ) -> tuple:
        """
        This function is called when a file is uploaded. It sets the new workspace, sets the dcc
        ui_json_data component, and sets the new object options and values.

        :param filename: Uploaded filename. Workspace or ui.json.
        :param contents: Uploaded file contents. Workspace or ui.json.
        :param trigger: Dash component which triggered the callback.

        :return object_options: New object dropdown options.
        :return object_value: New object value.
        :return ui_json_data: Uploaded ui_json data.
        :return filename: Return None to reset the filename so the same file can be chosen twice
            in a row.
        :return contents: Return None to reset the contents so the same file can be chosen twice
            in a row.
        """
        if trigger is None:
            trigger = callback_context.triggered[0]["prop_id"].split(".")[0]

        if trigger != "":
            # Remove entities from ui_json_data
            inp_ui_json_data = ui_json_data.copy()
            for key, value in inp_ui_json_data.items():
                if is_uuid(value) or isinstance(value, Entity):
                    setattr(self.params, key, None)
                    del ui_json_data[key]
        if trigger == "objects":
            return no_update, objects, ui_json_data, None, None

        object_options, object_value = no_update, None
        if contents is not None and filename is not None:
            if filename.endswith(".ui.json"):
                # Uploaded ui.json
                _, content_string = contents.split(",")
                ui_json = json.loads(base64.b64decode(content_string))
                self.workspace = Workspace(ui_json["geoh5"], mode="r")
                # Create ifile from ui.json
                ifile = InputFile(ui_json=ui_json)
                self.params = self.param_class(ifile)
                # Demote ifile data so it can be stored as a string
                if getattr(ifile, "data", None) is not None:
                    ui_json_data = ifile.demote(ifile.data.copy())
                    # Get new object value for dropdown from ui.json
                    object_value = ui_json_data["objects"]
            elif filename.endswith(".geoh5"):
                # Uploaded workspace
                _, content_string = contents.split(",")
                self.workspace = Workspace(
                    io.BytesIO(base64.b64decode(content_string)), mode="r"
                )
                # Update self.params with new workspace, but keep unaffected params the same.
                new_params = self.params.to_dict()
                for key, value in new_params.items():
                    if isinstance(value, Entity):
                        new_params[key] = None
                new_params["geoh5"] = self.workspace
                self.params = self.param_class(**new_params)
        elif trigger == "":
            ui_json_data = self._ui_json_data_from_params()
            object_value = ui_json_data["objects"]

            # Get new options for object dropdown
            if self.workspace is not None:
                object_options = [
                    {
                        "label": obj.parent.name + "/" + obj.name,
                        "value": "{" + str(obj.uid) + "}",
                    }
                    for obj in self.workspace.objects
                ]

        return object_options, object_value, ui_json_data, None, None

    def _ui_json_data_from_params(self) -> dict[str, Any]:
        """Returns app json data initialized from self.params"""
        assert (
            self.params.input_file is not None
        ), "No input file provided in parameters."

        ifile = InputFile(
            ui_json=self.params.input_file.ui_json,
            validate=False,
        )
        ifile.update_ui_values(self.params.to_dict())
        ui_json_data = ifile.demote(ifile.data)
        if self._app_initializer is not None:
            ui_json_data.update(self._app_initializer)
        return ui_json_data

    @staticmethod
    def get_port() -> int:
        """
        Loop through a list of ports to find an available port.

        :return port: Available port.
        """
        open_port = None
        for port in np.arange(8050, 8101):
            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
                in_use = sock.connect_ex(("localhost", port)) == 0
            if in_use is False:
                open_port = port
                break
        if open_port is None:
            raise RuntimeError("No open port found.")
        return open_port

    @staticmethod
    def start_server(
        port: int,
        app_class: type[BaseDashApplication],
        ui_json: InputFile | None = None,
        ui_json_data: dict | None = None,
        params: BaseParams | None = None,
    ):
        """
        Launch dash app server using given port.

        :param port: Port for where to launch server.
        :param app_class: Type of app to create.
        :param ui_json: ifile corresponding to the ui_json_data.
        :param ui_json_data: Dict of current params to provide to app init.
        :param params: Current params to pass to new app.
        """
        app = app_class(ui_json=ui_json, ui_json_data=ui_json_data, params=params)
        app.app.run(jupyter_mode="external", host="127.0.0.1", port=port)

    @staticmethod
    def make_qt_window(app_name: str | None, port: int):
        """
        Make Qt window and load dash url with the given port.

        :param app_name: App name to display as Qt window title.
        :param port: Port where the dash app has been launched.
        """
        app = QtWidgets.QApplication(sys.argv)
        browser = QtWebEngineWidgets.QWebEngineView()

        browser.setWindowTitle(app_name)
        browser.load(QtCore.QUrl("http://127.0.0.1:" + str(port)))
        # Brings Qt window to the front
        browser.setWindowFlags(QtCore.Qt.WindowStaysOnTopHint)
        # Setting window size
        browser.resize(1200, 800)
        browser.show()

        app.exec_()  # running the Qt app
        os.kill(os.getpid(), signal.SIGTERM)  # shut down dash server and notebook

    def launch_qt(self, objects: str, n_clicks: int) -> str:
        """
        Launch the Qt app when launch app button is clicked.

        :param objects: Selected object uid.
        :param n_clicks: Number of times button has been clicked; triggers callback.

        :return launch_app_markdown: Empty string since callbacks must have output.
        """
        del n_clicks  # unused argument

        triggers = [c["prop_id"].split(".")[0] for c in callback_context.triggered]
        if not (
            "launch_app" in triggers
            and objects is not None
            and self.workspace is not None
        ):
            return ""

        # Make new workspace with only the selected object
        obj = self.workspace.get_entity(uuid.UUID(objects))[0]
        if not isinstance(obj, Entity):
            return ""

        temp_geoh5 = self.workspace.name + "_" + f"{time():.0f}.geoh5"
        with tempfile.TemporaryDirectory() as tempdir_name:
            tempdir = Path(tempdir_name)
            temp_workspace = Workspace.create(tempdir / temp_geoh5)

            # Update ui.json with temp_workspace to pass initialize scatter plot app
            param_dict = self.params.to_dict()
            param_dict["geoh5"] = temp_workspace

            with fetch_active_workspace(temp_workspace), fetch_active_workspace(
                self.workspace
            ):
                param_dict["objects"] = obj.copy(
                    parent=temp_workspace, copy_children=True
                )

                if hasattr(obj, "property_groups"):
                    self._copy_property_groups(obj.property_groups, param_dict)

            new_params = self.param_class(**param_dict)

            temp_ui_json_name = temp_geoh5.replace(".geoh5", ".ui.json")
            new_params.write_input_file(
                name=temp_ui_json_name,
                path=tempdir_name,
                validate=False,
            )

            ifile = InputFile.read_ui_json(tempdir / temp_ui_json_name)
            ui_json_data = ifile.demote(ifile.data)

        # Start server
        port = ObjectSelection.get_port()
        threading.Thread(
            target=self.start_server,
            args=(port, self.app_class, ifile, ui_json_data, new_params),
            daemon=True,
        ).start()

        # Make Qt window
        self.make_qt_window(self.app_name, port)
        return ""

    @staticmethod
    def _copy_property_groups(source_groups: ObjectBase, param_dict: dict):
        """Copy any property groups over to param_dict of temporary workspace"""
        temp_prop_groups = param_dict["objects"].property_groups
        for group in source_groups:
            if not isinstance(group, PropertyGroup):
                continue
            for temp_group in temp_prop_groups:
                if temp_group.name != group.name:
                    continue
                for key, value in param_dict.items():
                    if hasattr(value, "uid") and value.uid == group.uid:
                        param_dict[key] = temp_group.uid

    @staticmethod
    def run(app_name: str, app_class: BaseDashApplication, ui_json: InputFile):
        """
        Launch Qt app from terminal.

        :param app_name: Name of app to display as Qt window title.
        :param app_class: Type of app to create.
        :param ui_json: Input file to pass to app for initialization.
        """
        # Start server
        port = ObjectSelection.get_port()
        threading.Thread(
            target=ObjectSelection.start_server,
            args=(port, app_class, ui_json),
            daemon=True,
        ).start()

        # Make Qt window
        ObjectSelection.make_qt_window(app_name, port)

    @property
    def app_name(self) -> str | None:
        """
        Name of app that appears as Qt window title.
        """
        return self._app_name

    @app_name.setter
    def app_name(self, val):
        if not isinstance(val, str) and (val is not None):
            raise TypeError("Value for attribute `app_name` should be 'str' or 'None'")
        self._app_name = val

    @property
    def app_class(self) -> type[BaseDashApplication]:
        """
        The kind of app to launch.
        """
        return self._app_class

    @app_class.setter
    def app_class(self, val):
        if not issubclass(val, BaseDashApplication):
            raise TypeError(
                "Value for attribute `app_class` should be a subclass of "
                ":obj:`geoapps_utils.application.BaseDashApplication`"
            )
        self._app_class = val

    @property
    def param_class(self) -> type[BaseParams]:
        """
        The param class associated with the launched app.
        """
        return self._param_class

    @param_class.setter
    def param_class(self, val):
        if not issubclass(val, BaseParams):
            raise TypeError(
                "Value for attribute `param_class` should be a subclass of "
                ":obj:`geoapps_utils.driver.params.BaseParams`"
            )
        self._param_class = val

    @property
    def workspace(self) -> Workspace | None:
        """
        Input workspace.
        """
        return self._workspace

    @workspace.setter
    def workspace(self, val):
        if not isinstance(val, Workspace) and (val is not None):
            raise TypeError(
                "Value for attribute `workspace` should be :obj:`geoh5py.workspace.Workspace`"
                " or 'None'"
            )
        if self._workspace is not None:
            # Close old workspace
            self._workspace.close()
        self._workspace = val


import re
import random
with open(r'C:\Users\asus\progrexp\lab_2_1\locations.list', 'r') as file:
    with open('locations_for_documentation.list', 'w') as file2:
        names= set()
        pat = '\([1-2][0-9][0-9][0-9]\)'
        for line in file:
            if re.search(pat, line):
                name = line[:(re.search(pat, line).start() - 1)].strip('"').strip("'")
                if name not in names and random.random() < 0.0001:
                    names.add(name)
                    file2.write(line)


import gc
import itertools
import matplotlib.pyplot as plt
import numpy as np

from impl.vis import Plot3DPoints
from impl.sfm.corrs import Find2D3DCorrespondences, GetPairMatches, UpdateReconstructionState
from impl.sfm.geometry import DecomposeEssentialMatrix, EstimateEssentialMatrix, TriangulatePoints, TriangulateImage, EstimateImagePose
from impl.sfm.image import Image
from impl.sfm.io import ReadFeatureMatches, ReadKMatrix
from impl.sfm.vis import PlotImages, PlotWithKeypoints, PlotImagePairMatches, PlotCameras

def main():

  np.set_printoptions(linewidth=10000, edgeitems=100, precision=3)

  data_folder = '../code/data'
  image_names = [
    '0000.png',
    '0001.png',
    '0002.png',
    '0003.png',
    '0004.png',
    '0005.png',
    '0006.png',
    '0007.png',
    '0008.png',
    '0009.png']

  # Read images
  images = {}
  for im_name in image_names:
    images[im_name] = (Image(data_folder, im_name))

  # Read the matches
  matches = {}
  for image_pair in itertools.combinations(image_names, 2):
    matches[image_pair] = ReadFeatureMatches(image_pair, data_folder)

  K = ReadKMatrix(data_folder)

  init_images = [5, 6]

  # Visualize images and features
  # You can comment these lines once you verified that the images are loaded correctly
  """
  # Show the images
  PlotImages(images)

  # Show the keypoints
  for image_name in image_names:
    PlotWithKeypoints(images[image_name])

  # Show the feature matches
  for image_pair in itertools.combinations(image_names, 2):
    PlotImagePairMatches(images[image_pair[0]], images[image_pair[1]], matches[(image_pair[0], image_pair[1])])
    gc.collect()
  """

  e_im1_name = image_names[init_images[0]]
  e_im2_name = image_names[init_images[1]]
  e_im1 = images[e_im1_name]
  e_im2 = images[e_im2_name]
  e_matches = GetPairMatches(e_im1_name, e_im2_name, matches)

  # Estimate Fundamental matrix
  E = EstimateEssentialMatrix(K, e_im1, e_im2, e_matches)

  # Extract the relative pose from the essential matrix.
  # This gives four possible solutions and we need to check which one is the correct one in the next step
  possible_relative_poses = DecomposeEssentialMatrix(E)

  # For each possible relative pose, try to triangulate points.
  # We can assume that the correct solution is the one that gives the most points in front of both cameras
  # Be careful not to set the transformation in the wrong direction
  most_points = 0
  best_R = np.eye(3)
  best_t = np.array([0, 0, 0])
  e_im2.SetPose(best_R, best_t)
  for _, r_pose in enumerate(possible_relative_poses):
    (R, t) = r_pose
    e_im1.SetPose(R, t)
    points3D, _, _ = TriangulatePoints(K, e_im1, e_im2, e_matches)
    if np.shape(points3D)[0] > most_points:
      most_points = np.shape(points3D)[0]
      best_R = R
      best_t = t

  # Set the image poses in the images (image.SetPose(...))
  e_im1.SetPose(best_R, best_t)

  # Triangulate initial points
  points3D, im1_corrs, im2_corrs = TriangulatePoints(K, e_im1, e_im2, e_matches)

  # Add the new 2D-3D correspondences to the images
  e_im1.Add3DCorrs(im1_corrs, list(range(points3D.shape[0])))
  e_im2.Add3DCorrs(im2_corrs, list(range(points3D.shape[0])))

  # Keep track of all registered images
  registered_images = [e_im1_name, e_im2_name]

  for reg_im in registered_images:
    print(f'Image {reg_im} sees {images[reg_im].NumObserved()} 3D points')

  # Register new images + triangulate
  # Run until we can register all images
  while len(registered_images) < len(images):
    for image_name in images:
      if image_name in registered_images:
        continue

      # Find 2D-3D correspondences
      image_kp_idxs, point3D_idxs = Find2D3DCorrespondences(image_name, images, matches, registered_images)

      # With two few correspondences the pose estimation becomes shaky.
      # Keep this image for later
      if len(image_kp_idxs) < 50:
        continue

      print(f'Register image {image_name} from {len(image_kp_idxs)} correspondences')

      # Estimate new image pose
      R, t = EstimateImagePose(images[image_name].kps[image_kp_idxs], points3D[point3D_idxs], K)

      # Set the estimated image pose in the image and add the correspondences between keypoints and 3D points
      images[image_name].SetPose(R, t)
      images[image_name].Add3DCorrs(image_kp_idxs, point3D_idxs)

      # Triangulate new points wth all previously registered images
      image_points3D, corrs = TriangulateImage(K, image_name, images, registered_images, matches)

      # Update the 3D points and image correspondences
      points3D, images = UpdateReconstructionState(image_points3D, corrs, points3D, images)

      registered_images.append(image_name)

  # Visualize
  fig = plt.figure()
  ax3d = fig.add_subplot(111, projection='3d')
  Plot3DPoints(points3D, ax3d)
  PlotCameras(images, registered_images, ax3d)

  # Delay termination of the program until the figures are closed
  # Otherwise all figure windows will be killed with the program
  plt.show(block=True)

if __name__ == '__main__':
  main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os
import shutil

import numpy as np
import pandas as pd

def main():
    info = [
        ("validation", "validation-annotations-human-imagelabels.csv"),
        ("test", "test-annotations-human-imagelabels.csv"),
    ]

    df_desc = pd.read_csv("oidv6-class-descriptions.csv")
    id_label = dict(zip(df_desc["LabelName"], df_desc["DisplayName"]))

    for dname, anno_file in info:
        df = pd.read_csv(anno_file, sep=",")
        df = df[df["Confidence"]==1]

        group = df[["ImageID","LabelName"]].groupby("ImageID")
        df_uniq = group.count()
        df_uniq = df_uniq[df_uniq["LabelName"]==1]

        df = df[df["ImageID"].isin(df_uniq.index)]

        for _, row in df.iterrows():
            file_name = "%s.jpg" % row.ImageID
            path = os.path.join(dname, file_name)
            if not os.path.exists(path):
                continue

            label = id_label.get(row.LabelName, None)
            if not label:
                continue

            print(path, label)

            dst_path = os.path.join("open_image", label, file_name)
            if not os.path.exists(os.path.dirname(dst_path)):
                os.makedirs(os.path.dirname(dst_path))

            shutil.copy(path, dst_path)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for `Particles` package."""

import unittest
from nose.tools import assert_equals, assert_raises

from pexmd import particles
import numpy as np

class TestParticles(unittest.TestCase):
  """Tests for `Particles` module."""

  def setUp(self):
    """Set up test fixtures, if any."""
    self.four_by3 = np.array([[0.0, 0.0, 0.0], [1.0, 0.0, 0.0],
                              [-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    self.three_by3 = np.array([[0.0, 0.0, 0.0], [1.0, 0.0, 0.0],
                               [-1.0, 0.0, 0.0]])
    self.lfour_by3 = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0],
                      [-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]
    self.lthree_by3 = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0],
                       [-1.0, 0.0, 0.0]]
  def tearDown(self):
    """Tear down test fixtures, if any."""

  def test_create_base_particles(self):
    """Create an empty object of generic particles."""
    part = particles.Base(3)

  def test_create_point_particles(self):
    part = particles.PointParticles(3)
    np.testing.assert_array_equal(part.idx, np.arange(3))

  def test_set_position_from_array(self):
    part = particles.PointParticles(4)
    part.x = self.four_by3
    assert_equals(part.n, 4)

  def test_set_position_from_list(self):
    part = particles.PointParticles(4)
    part.x = self.lfour_by3
    assert_equals(part.n, 4)
    assert_equals(type(part.x), np.ndarray)

  def test_set_position_wrong_size(self):
    part = particles.PointParticles(4)
    sttr = lambda x: part.__setattr__("x", x)
    assert_raises(ValueError, sttr, self.three_by3)

  def test_modify_position(self):
    part = particles.PointParticles(4)
    me = self.four_by3.copy()
    part.x = self.four_by3
    np.testing.assert_array_equal(part.x, self.four_by3)
    part.x[0, 0] = 1000
    np.testing.assert_array_equal(me, self.four_by3)

  def test_modify_velocity(self):
    part = particles.PointParticles(4)
    part.x = self.four_by3
    part.v = self.lfour_by3
    assert_equals(part.n, 4)
    sttr = lambda v: part.__setattr__("v", v)
    assert_raises(ValueError, sttr, self.three_by3)
    np.testing.assert_array_equal(part.v, self.four_by3)

  def test_modify_force(self):
    part = particles.PointParticles(4)
    part.f = self.four_by3
    assert_equals(part.n, 4)
    sttr = lambda f: part.__setattr__("f", f)
    assert_raises(ValueError, sttr, self.three_by3)
    np.testing.assert_array_equal(part.f, self.four_by3)

  def test_calculate_acceleration(self):
    part = particles.PointParticles(4)
    part.f = self.four_by3
    part.mass = 2.0
    np.testing.assert_array_equal(part.a, self.four_by3/2)

  def test_modify_type(self):
    part = particles.PointParticles(4)
    part.t = np.array([1, 1, 2, 2], dtype=np.int32)
    np.testing.assert_array_equal(part.t, np.array([1, 1, 2, 2]))
    part.t = 1
    np.testing.assert_array_equal(part.t, np.array([1, 1, 1, 1], dtype=np.int32))
    sttr = lambda t: part.__setattr__("t", t)
    assert_raises(ValueError, sttr, np.array([1, 1, 1]))

  def test_modify_mass(self):
    part = particles.PointParticles(4)
    part.mass = np.array([1.0, 1.0, 2.0, 2.0], dtype=np.float32)
    assert_equals(part.n, 4)
    part.mass = 1.0
    np.testing.assert_array_equal(part.mass, np.array([1, 1, 1, 1], dtype=np.float32))
    sttr = lambda mass: part.__setattr__("mass", mass)
    assert_raises(ValueError, sttr, np.array([1.0, 1.0, 1.0]))


import numpy as np
import matplotlib.pyplot as plt

def morlet(sigsize, location, scale, w0=2 * np.pi):
    """
    Description:
        Compute a Morlet wavelet at the specified time and scale in the time-scale plane.

    Params:
        sigsize: Length of the output wavelet.
        location: Time location of the wavelet.
        scale: Scale of the wavelet.
        w0: Central frequency of the wavelet (default is 2 * pi).

    Returns:
        Complex 1D array representing the values of the Morlet wavelet at the specified time and scale.
    """
    t = np.arange(sigsize) - location
    # Calculate the real and imaginary parts of the Morlet wavelet
    real_part = np.exp(-(t**2) / (2 * scale**2)) * np.cos(w0 * t)
    imag_part = np.exp(-(t**2) / (2 * scale**2)) * np.sin(w0 * t)
    morlet_wavelet = real_part + 1j * imag_part

    return morlet_wavelet

if __name__ == "__main__":
    sigsize = 1024
    location = 512
    scale = 20
    morlet = morlet(sigsize, location, scale)
    plt.plot(morlet.real)
    plt.show()


# -*- coding: utf-8 -*-
"""
@Brief: This is a vision module(single robot) for RoboCup Small Size League
@Version: RoboCup Small Size League 2018
@author: Wang Yunkai
"""

import socket
from time import sleep
import math
from proto.grSim_Packet_pb2 import grSim_Packet
import proto.messages_robocup_ssl_wrapper_pb2 as messages_wrapper
import proto.zss_cmd_pb2 as zss

# MULTI_GROUP = '224.5.23.2'
# VISION_PORT = 10020
# ROBOT_ID = 6

class VisionModule:
    def __init__(self, MULTI_GROUP, VISION_PORT, STATUS_PORT, SENDERIP = '0.0.0.0'):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
        self.sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
        self.sock.bind((SENDERIP,VISION_PORT))
        self.sock.setsockopt(socket.IPPROTO_IP,
            socket.IP_ADD_MEMBERSHIP,
            socket.inet_aton(MULTI_GROUP) + socket.inet_aton(SENDERIP))

        # self.status_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
        # self.status_sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
        # self.status_sock.bind(('127.0.0.1',STATUS_PORT))
        # self.status_sock.setsockopt(socket.IPPROTO_IP,socket.IP_ADD_MEMBERSHIP,socket.inet_aton(MULTI_GROUP) + socket.inet_aton(SENDERIP))
        # self.status_sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
        # self.status_sock.settimeout(0.5)

        self.status_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
        self.status_sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
        self.status_sock.bind((SENDERIP,STATUS_PORT))
        self.status_sock.setsockopt(socket.IPPROTO_IP,
            socket.IP_ADD_MEMBERSHIP,
            socket.inet_aton(MULTI_GROUP) + socket.inet_aton(SENDERIP))
        self.status_sock.settimeout(0.2)

        self.robot_info = [0, 0, 0, 0]
        self.ball_info = [0, 0]

    def receive(self):
        print('receive')
        data, addr = self.sock.recvfrom(1024)
        # print(data)
        # sleep(0.001) # wait for reading
        # print('receive2')
        try:
            robot_Data, addr = self.status_sock.recvfrom(1024)
            print('receive2')
        except:
            robot_Data = None
        return data, robot_Data

    def get_info(self, ROBOT_ID):
        data, robot_Data = self.receive()
        package = messages_wrapper.SSL_WrapperPacket()
        try:
            package.ParseFromString(data)

            detection = package.detection
            #print('camera id:', detection.camera_id)

            robots = detection.robots_blue # repeat
            robot_max_conf = 0
            for robot in robots:
                if robot.robot_id == ROBOT_ID and robot.confidence > robot_max_conf:
                    self.robot_info[0] = robot.x/1000.0
                    self.robot_info[1] = robot.y/1000.0
                    self.robot_info[2] = robot.orientation
                    #print('Robot', robot.confidence)

            balls = detection.balls # repeat
            ball_max_conf = 0
            for ball in balls:
                if ball.confidence >= ball_max_conf:
                    self.ball_info[0] = ball.x/1000.0
                    self.ball_info[1] = ball.y/1000.0

        except:
            print("vision error")
            pass
                #print('Ball', ball.confidence)
        package = zss.Robot_Status
        try:
            package.ParseFromString(robot_Data)
            for robot in package:
                if robot.robot_id == ROBOT_ID:
                    self.robot_info[3] = robot.indrared
        except:
            print("status error")
            pass

if __name__ == '__main__':
    MULTI_GROUP = '224.5.23.2'
    VISION_PORT = 10094
    STATUS_PORT = 30011
    ROBOT_ID = 6
    vision = VisionModule(MULTI_GROUP, VISION_PORT, STATUS_PORT)
    while True:
        vision.get_info(ROBOT_ID)
        print(vision.robot_info)
        # print(vision.ball_info)
    print(vision.robot_info)


from __future__ import annotations

import abc
from typing import *
from dataclasses import dataclass

T = TypeVar("T")
C = TypeVar("C", bound=type)
D = TypeVar("D", bound=type)
X = TypeVar("X")
Y = TypeVar("Y")
Z = TypeVar("Z")

class Morphism(Generic[X, Y], abc.ABC):
    """A mapping from one object in T to another."""

    def __call__(self, e: X) -> Y:
        """This is the functional implementation of m(x)."""

class Identity(Morphism):
    """A generic identity mapping we'll use for type hinting."""

def identity(x: X) -> X:
    return x

def compose(f: Morphism[X, Y], g: Morphism[Y, Z]) -> Morphism[X, Z]:
    return lambda x: f(g(x))

class FunctorCast(Generic[C, D], abc.ABC):
    """Represents a typecast for mapping x to F(x)."""

    def __getitem__(self, x: C) -> D:
        """Should yield F(x) in D."""

class Functor(Generic[C, D], abc.ABC):
    """A relationship between elements and morphisms in categories."""

    Cast: FunctorCast[C, D]

    @classmethod
    @abc.abstractmethod
    def lift(cls, f: Morphism[X, Y]) -> Morphism[Cast[X], Cast[Y]]:
        """Associate morphisms across the categories."""

    @classmethod
    def map(cls, f: Morphism[X, Y], a: Cast[X]) -> Cast[Y]:
        """Actually apply the lifted morphism on a casted X."""

        return cls.lift(f)(a)

@dataclass
class Maybe(Generic[T]):
    value: Optional[T] = None

class MaybeFunctor(Functor):
    """A functor that operates on maybes."""

    Cast = Maybe

    @classmethod
    def lift(cls, f: Morphism[X, Y]) -> Morphism[Cast[X], Cast[Y]]:
        return lambda a: Maybe(f(a.value)) if a.value is not None else a

class IterableFunctor(Functor):
    """Formalize how map() and iter() are functors."""

    Cast = Iterable

    @classmethod
    def lift(cls, f: Morphism[Morphism[X, Y]]) -> Morphism[Cast[X], Cast[Y]]:
        return lambda a: map(f, a)

def test():
    assert MaybeFunctor.Cast[int] == Maybe[int]

    f: Morphism[int, int] = lambda x: x + 1
    assert MaybeFunctor.map(f, Maybe(1)) == Maybe(2)
    assert MaybeFunctor.map(f, Maybe(None)) == Maybe(None)

    # F(id_x) = id_Fx
    identity_int: Identity[int] = identity
    identity_maybe_int: Identity[List[int]] = identity
    assert MaybeFunctor.map(identity_int, Maybe(1)) == identity_maybe_int(Maybe(1))

    # F(f * g) = F(f) * F(g)
    mf = MaybeFunctor
    g: Morphism[int, int] = lambda x: x * 2
    assert mf.map(compose(f, g), Maybe(1)) == compose(mf.lift(f), mf.lift(g))(Maybe(1))
    assert mf.map(compose(f, g), Maybe(None)) == compose(mf.lift(f), mf.lift(g))(Maybe(None))

    assert list(map(f, range(10))) == list(IterableFunctor.map(f, range(10)))

if __name__ == "__main__":
    test()