import numpy as np
import pandas as pd
import igraph as ig
import multires_consensus_clustering as mcc
import time

def multiresolution_graph(clustering_data, settings_data, list_resolutions, neighbour_based, single_resolution):
    """
    Creates a multi-resolution graph based on the resolutions given in the list_resolutions.
    Can either create a graph where all resolution vertices are connected or only the neighbouring resolutions are connected.

    @param settings_data: Settings data about the clusters.
    @param clustering_data: The cluster data.
    @param neighbour_based: Boolean to decided on the way to connect the vertices across resolutions.
    @param list_resolutions: The list containing the different resolutions, e.g. [3,5,9,20, ... ] or "all"
    @param single_resolution: Resolution Parameter for the meta-graph (leiden community detection resolution parameter)
    @return: The mulit-graph as a iGraph graph.
    """

    # set maximum level for hierarchy plot
    len_list_resolutions = len(list_resolutions)
    level_count = len_list_resolutions + 1

    # check if list resolution contains less the two resolutions
    if len_list_resolutions <= 1:
        print("More then one resolution needed for multi resolution graph.")

        return mcc.meta_graph(clustering_data, settings_data, list_resolutions[0], single_resolution)

    # create the multi graph
    else:
        if list_resolutions == "all":
            bins_clusterings = mcc.bin_n_clusters(settings_data["n_clusters"])
            list_resolutions = [int(first_number_clusters[0]) for first_number_clusters in bins_clusterings]
        else:
            # sort resolution list in cases not sorted
            list_resolutions.sort()

        # create first graph and assign the level
        resolution_1 = mcc.meta_graph(clustering_data, settings_data, list_resolutions[0], single_resolution)
        resolution_1.vs["level"] = [level_count] * resolution_1.vcount()

        # create new attribute to save the cell probabilities in a meta node
        probability_df = mcc.graph_nodes_cells_to_df(resolution_1, clustering_data)
        resolution_1.vs["probability_df"] = [probability_df[column].values for column in probability_df.columns]

        resolution_1.vs[0]["cell_index"] = probability_df.index.tolist()

        # delete all edges of the old graph
        mcc.delete_edges_single_resolution(resolution_1)

        # change level count
        level_resolution_2 = level_count - 1

        # select all resolutions except the first
        list_resolutions = list_resolutions[1:]

        # create multi-graph using the rest of the list_resolutions
        for resolution in list_resolutions:

            # create graph and assign the level
            resolution_2 = mcc.meta_graph(clustering_data, settings_data, resolution, single_resolution)
            resolution_2.vs["level"] = [level_resolution_2] * resolution_2.vcount()

            # delete all edges of the old graph
            mcc.delete_edges_single_resolution(resolution_2)

            # create multi graph based on neighbours or connect all vertices
            if neighbour_based:
                resolution_1 = mcc.merge_two_resolution_graphs(resolution_1, resolution_2,
                                                               current_level=level_resolution_2 + 1,
                                                               neighbours=True, clustering_data=clustering_data)
            else:
                # connect all vertices
                resolution_1 = mcc.merge_two_resolution_graphs(resolution_1, resolution_2, current_level=None,
                                                               neighbours=False, clustering_data=clustering_data)

            # set level for next graph
            level_resolution_2 -= 1

        # return the final multi-graph
        return resolution_1

def delete_edges_single_resolution(graph):
    """
    Deletes all edges with an edge weight above 0 -> these are all edges of the graph.
    @param graph: The given graph, iGraph graph.
    @return: The graph without edges; only vertices with attributes
    """

    return graph.delete_edges()

def merge_two_resolution_graphs(graph_1, graph_2, current_level, neighbours, clustering_data):
    """
    Merges two graphs;
    Either connects all vertices or only the bins neighbouring each other. All edges are based on the jaccard-index.

    @param clustering_data: The clustering data from which the meta graphs are build.
    @param neighbours: Connects all vertices or only neighbouring resolutions; Boolean
    @param current_level: The level of the last added vertices.
        Need so the new vertices are only connected to the latest vertices and not all.
    @param graph_1: The first graph, iGraph graph.
    @param graph_2: The second graph, iGraph graph.
    @return: The merged graph.
    """

    # create edge lists
    edge_list, edge_weights = [], []

    # to check the graph merger visually
    graph_1.vs["graph"] = [1] * graph_1.vcount()
    graph_2.vs["graph"] = [2] * graph_2.vcount()

    # add cell probability to the second graph with is added in the new layer (level)
    probability_df = mcc.graph_nodes_cells_to_df(graph_2, clustering_data)
    graph_2.vs["probability_df"] = [probability_df[column].values for column in probability_df.columns]

    # creates a graph based on the two given resolutions
    graph = graph_1.disjoint_union(graph_2)

    # connects vertices based on neighbouring resolutions.
    if neighbours:
        for vertex_1 in graph.vs.select(graph=1):
            for vertex_2 in graph.vs.select(graph=2):
                # connects only bins next to each other
                if vertex_1["level"] == current_level and vertex_2["level"] == current_level - 1:
                    # calculate edge weight
                    edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                    # if the edge_weight is greater 0 the edge is added
                    if edge_weight != 0:
                        edge_list.append((vertex_1, vertex_2))
                        edge_weights.append(edge_weight)

    # connects all vertices
    else:
        for vertex_1 in graph.vs.select(graph=1):
            for vertex_2 in graph.vs.select(graph=2):
                # calculate edge weight
                # edge_weight = mcc.jaccard_index_two_vertices(vertex_1, vertex_2)
                edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                # if the edge_weight is greater 0 the edge is added
                if edge_weight != 0:
                    edge_list.append((vertex_1, vertex_2))
                    edge_weights.append(edge_weight)

    # add edges to the graph
    graph.add_edges(edge_list)

    # add edge weights to the graph
    graph.es["weight"] = edge_weights

    return graph

def reconnect_graph(graph):
    """
    Reconnects the graph. Useful if after merging the graph is just a set of separated nodes.

    @param graph: The graph, in the ideal case a set of discrete nodes.
    @return: The connected graph. Edges are chosen that in the end the graph has a tree structure,
        based on the level of the nodes.
    """

    # check if graph has more than one level
    if len(set(graph.vs["level"])) != 1:
        # if the graph has more then one level a tree structure can be created, using the resolution of the graphs
        for vertex_1 in graph.vs:
            level_vertex_1 = vertex_1["level"]
            for vertex_2 in graph.vs:
                if vertex_1 != vertex_2:
                    # connects only bins next to each other
                    if vertex_2["level"] < level_vertex_1:
                        # calculate edge weight
                        edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                        # if the edge_weight is greater 0 the edge is added
                        if edge_weight != 0:
                            index_1 = vertex_1.index
                            index_2 = vertex_2.index
                            graph.add_edge(index_1, index_2, weight=edge_weight)

                        # check if better edge is available for tree structure
                        edge_list_vertex_2 = vertex_2.all_edges()

                        # if the vertex has only one edge no better is available
                        if len(edge_list_vertex_2) > 1:
                            edges_to_delete = []
                            current_best_weight = 0
                            current_best_edge = None

                            for edge in edge_list_vertex_2:
                                edges_to_delete.append(edge)
                                edge_weight = edge["weight"]

                                # only check edge going from a lower resolution to a higher resolution -> tree structure
                                if graph.vs[edge.target]["level"] < vertex_2["level"]:
                                    edges_to_delete.pop(len(edges_to_delete) - 1)
                                else:
                                    # choose the edge with the best edge weight
                                    if edge_weight > current_best_weight:
                                        edges_to_delete.pop(len(edges_to_delete) - 1)
                                        if current_best_edge is not None:
                                            edges_to_delete.append(current_best_edge)
                                        current_best_weight = edge_weight
                                        current_best_edge = edge

                                    # if the edges have the same edge weight choose the higher resolution one
                                    elif edge_weight == current_best_weight:
                                        last_edge = edges_to_delete.pop(len(edges_to_delete) - 1)
                                        if graph.vs[last_edge.target]["level"] > graph.vs[edge.target]["level"]:
                                            edges_to_delete.append(last_edge)
                                        else:
                                            edges_to_delete.append(edge)

                            graph.delete_edges(edges_to_delete)

    # otherwise returns the maximum spanning tree
    else:
        # calculate all edge weights for every vertex in the graph
        for vertex_1 in graph.vs:
            for vertex_2 in graph.vs:
                if vertex_1 != vertex_2:
                    # calculate edge weight
                    edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                    # if the edge_weight is greater 0 the edge is added
                    if edge_weight != 0:
                        index_1 = vertex_1.index
                        index_2 = vertex_2.index
                        graph.add_edge(index_1, index_2, weight=edge_weight)

        # invert edge weights
        inverted_weights = [1 - edge_weight for edge_weight in graph.es["weight"]]

        # create the minimum spanning tree using the inverted edge weight -> maximum spanning tree
        graph = graph.spanning_tree(weights=inverted_weights, return_tree=True)

    return graph

def multires_community_detection(graph, clustering_data, community_detection, multi_resolution):
    """
    Uses louvain community detection on the multi-resolution graph and creates a clustering tree with reconnect_graph.
    Optional clean up of the clustering tree with edge merging.

    @param multi_resolution: Resolution parameter for the community detection function.
    @param clustering_data: The clustering data on which the graph is based
    @param community_detection: "leiden", "hdbscan" or else automatically louvain,
        detects community with the named algorithms
    @param graph: The multi resolution graph, iGraph graph.
    @return: A clustering tree, igraph Graph.
    """

    # apply probability outlier detection to the graph
    graph = mcc.filter_by_node_probability(graph)

    # community detection
    if community_detection == "leiden":
        # uses the leiden algorithm for community detection
        vertex_clustering = ig.Graph.community_leiden(graph, weights="weight", resolution_parameter=multi_resolution)
    elif community_detection == "hdbscan" and graph.vcount() > 1 and graph.ecount() > 0:
        # use hdbscan for community detection
        vertex_clustering = mcc.hdbscan_community_detection(graph)
    else:
        # if nothing is selected uses louvain community detection
        vertex_clustering = ig.Graph.community_multilevel(graph, weights="weight")

    # combines attributes to a list
    graph = mcc.merge_by_list(vertex_clustering)

    # recalculate the probabilities based on the new merged nodes
    new_probabilities = mcc.graph_nodes_cells_to_df(graph, clustering_data)
    graph.vs["probability_df"] = [new_probabilities[column].values for column in new_probabilities.columns]

    # delete edges and create graph tree structure if there is more than one node
    if graph.vcount() != 1:
        graph.delete_edges()
        graph = reconnect_graph(graph)

        if graph.ecount() > 0:
            # calculate max and min edge weights
            max_edge = max(graph.es["weight"])
            min_edge = min(graph.es["weight"])

            # calculate max-min average
            max_min_average = (max_edge + min_edge) / 2

            # set the upper quantile for the merge edges threshold
            merge_edges_threshold = (max_edge + max_min_average) / 2

            # clean up graph by merging some edges
            graph = mcc.merge_edges_weight_above_threshold(graph, threshold=merge_edges_threshold)

            # recalculate the probabilities based on the new merged nodes
            new_probabilities = mcc.graph_nodes_cells_to_df(graph, clustering_data)
            graph.vs["probability_df"] = [new_probabilities[column].values for column in new_probabilities.columns]

            # check if graph has more then vertices, if so recalculates the edges
            if graph.vcount() != 1:
                graph.delete_edges()
                graph = reconnect_graph(graph)

    return graph


import placement
import services
import sims4.math
import sims4.random
from objects.object_enums import ResetReason
from placement import create_starting_location
from routing import get_routing_surface_at_or_below_position, SurfaceType, SurfaceIdentifier
from sims4.tuning.tunable import HasTunableSingletonFactory, AutoFactoryInit, Tunable

DEFAULT_ON_LOT_SEARCH_FLAGS = placement.FGLSearchFlag.CALCULATE_RESULT_TERRAIN_HEIGHTS | placement.FGLSearchFlag.DONE_ON_MAX_RESULTS | placement.FGLSearchFlag.STAY_OUTSIDE | placement.FGLSearchFlag.STAY_IN_LOT

class TunableCoordinates(HasTunableSingletonFactory, AutoFactoryInit):
    FACTORY_TUNABLES = {
        'x': Tunable(
            tunable_type=float,
            default=None,
            allow_empty=True,
        ),
        'z': Tunable(
            tunable_type=float,
            default=None,
            allow_empty=True,
        ),
    }

    __slots__ = ('x', 'z',)

    @classmethod
    def create_from_target(cls, obj):
        return cls.create_from_position(obj.position)

    @classmethod
    def create_from_position(cls, position):
        return cls(x=position.x, z=position.z)

    def move_sim_to(self, sim, facing_coordinates=None):
        orientation = self._get_orientation(facing_coordinates)
        location = self._get_location(orientation)

        sim.routing_component.on_slot = None
        sim.set_location(location)
        sim.reset(ResetReason.RESET_EXPECTED, None, 'Command')
        return True

    def move_object_to(self, obj, facing_coordinates=None, use_fgl=False, search_flags=DEFAULT_ON_LOT_SEARCH_FLAGS):
        orientation = self._get_orientation(facing_coordinates)

        if use_fgl:
            location = self._get_starting_location()
            fgl_context = placement.create_fgl_context_for_object(location, obj, search_flags=search_flags)
            (position, _, _) = fgl_context.find_good_location()
        else:
            location = self._get_location(orientation)
            position = location.transform.translation

        if position is None:
            return False
        obj.move_to(translation=position, orientation=orientation, routing_surface=location.routing_surface)
        obj.reset(ResetReason.RESET_EXPECTED)
        return True

    def move_object_to_off_lot(self, obj, facing_coordinates=None, use_fgl=False, max_distance=3):
        orientation = self._get_orientation(facing_coordinates)
        use_world_routing_surface = True
        if use_fgl:
            self.set_object_location(obj, orientation, use_world_routing_surface=use_world_routing_surface)
            location = self.get_starting_location(use_world_routing_surface=use_world_routing_surface)
            fgl_context = placement.create_fgl_context_for_object_off_lot(location, obj, max_distance=max_distance)
            (position, _, _) = fgl_context.find_good_location()
        else:
            location = self.get_location(orientation, use_world_routing_surface=use_world_routing_surface)
            position = location.transform.translation

        if position is None:
            return False

        obj.move_to(translation=position, orientation=orientation, routing_surface=location.routing_surface)
        obj.reset(ResetReason.RESET_EXPECTED)
        return True

    def _get_y_axis(self):
        return services.terrain_service.terrain_object().get_height_at(self.x, self.z)

    def _get_position(self):
        return sims4.math.Vector3(self.x, self._get_y_axis(), self.z)

    def _get_routing_surface(self):
        position = self._get_position()
        return get_routing_surface_at_or_below_position(position)

    def _get_world_routing_surface(self):
        return SurfaceIdentifier(services.current_zone_id(), 0, SurfaceType.SURFACETYPE_WORLD)

    def _get_starting_location(self, use_world_routing_surface=False):
        position = self._get_position()
        routing_surface = self._get_world_routing_surface() if use_world_routing_surface else self._get_routing_surface()
        return create_starting_location(position=position, routing_surface=routing_surface)

    def _get_orientation_to_position(self, target_position):
        starting_position = self._get_position()
        vec_to_target = starting_position - target_position
        theta = sims4.math.vector3_angle(vec_to_target)
        return sims4.math.angle_to_yaw_quaternion(theta)

    def _get_transform(self, orientation):
        starting_location = self._get_starting_location()
        return sims4.math.Transform(starting_location.position, orientation)

    def _get_orientation(self, facing_coordinates=None):
        if facing_coordinates:
            facing_position = facing_coordinates.get_position()
            return self._get_orientation_to_position(facing_position)
        else:
            return sims4.random.random_orientation()

    def _get_location(self, orientation, use_world_routing_surface=False):
        transform = self._get_transform(orientation)
        routing_surface = self._get_world_routing_surface() if use_world_routing_surface else self._get_routing_surface()
        return sims4.math.Location(transform, routing_surface)

    def _set_object_location(self, obj, orientation, use_world_routing_surface=False):
        obj.location = self._get_location(orientation, use_world_routing_surface=use_world_routing_surface)


#! /usr/bin/python3

import sys
import time
import copy

#for the stack
class SaveState:
    def __init__(self, index, possibilities, board):
        self.index = index
        self.possibilities = possibilities
        self.board = board

#globals
Cliques=[[0,1,2,3,4,5,6,7,8],[9,10,11,12,13,14,15,16,17],[18,19,20,21,22,23,24,25,26],[27,28,29,30,31,32,33,34,35],[36,37,38,39,40,41,42,43,44],[45,46,47,48,49,50,51,52,53],[54,55,56,57,58,59,60,61,62],[63,64,65,66,67,68,69,70,71],[72,73,74,75,76,77,78,79,80],[0,9,18,27,36,45,54,63,72],[1,10,19,28,37,46,55,64,73],[2,11,20,29,38,47,56,65,74],[3,12,21,30,39,48,57,66,75],[4,13,22,31,40,49,58,67,76],[5,14,23,32,41,50,59,68,77],[6,15,24,33,42,51,60,69,78],[7,16,25,34,43,52,61,70,79],[8,17,26,35,44,53,62,71,80],[0,1,2,9,10,11,18,19,20],[3,4,5,12,13,14,21,22,23],[6,7,8,15,16,17,24,25,26],[27,28,29,36,37,38,45,46,47],[30,31,32,39,40,41,48,49,50],[33,34,35,42,43,44,51,52,53],[54,55,56,63,64,65,72,73,74],[57,58,59,66,67,68,75,76,77],[60,61,62,69,70,71,78,79,80]]

Neighbors = {}  # key is cell-id, value is set of neighbors.  Neighbors[2] = set(0,1,3,4,5,6,7,8,11,20,29,38,47,56,65,74,9,10,18,19)
Boards = {}  # all boards read in
allVals = set([1,2,3,4,5,6,7,8,9])

#make Neighbors
def makeNeighbors():
    global Neighbors, Cliques
    for cell in range(81):
        nb = set()
        for clique in Cliques:
            if cell in clique:
                nb.update(clique) #add all the values from that clique, but ofc no repatitions
        nb.discard(cell) #you shouldn't be checking if that cell itself == value of that cell
        Neighbors[cell] = nb #now add this buffer to the dictionary

#to check if my resulting board is actually correct
def getIncorrect(board):
    global Neighbors
    # returns a list of the positions on this board whose values are incorrect
    inc = []
    for pos in range(len(board)):
        val = board[pos] #this is the value of that cell
        for neighbor in Neighbors[pos]:
            if board[neighbor] == val: #if in that cell's neighbors there is a duplicate
                inc.append(pos) #mark this cell as problematic. then when this goes through again, the duplicate will be marked as problematic
                break
    return inc

def loadBoards(argv = None):
    global Boards
    if not argv:
        argv = sys.argv
    with open(argv[1], 'r') as f_in:
        lines = f_in.read().split('\n')
        i = 0
        while i<len(lines):
            if (len(lines[i].split(',')) == 3): #so that's the name of our board
                name = lines[i].split(',') #aka so if it's titled "A1-1, NYTIMES, Easy", that's th name but as a list
                #print("processing " + name[0]);
                newboard = []
                for j in range(i+1, i+10):
                    buffer = lines[j].split(',')
                    for num in range(9):
                        try:
                            newboard.append(int(buffer[num]))
                        except: #aka if the position is a '_'
                            newboard.append(0)
                Boards[name[0]] = newboard
                i+=9
            else:
                i+=1

def make_p(index, board):
    global Neighbors, allVals
    possibilities = list(allVals)
    for pos in Neighbors[index]:
        if board[pos] in possibilities:
            possibilities.remove(board[pos])
    return possibilities

def printBoard(title,b):
    print(title)
    for row in range(9):
        arow = []
        for col in range(9):
            arow.append(str(b[9*row+col])) #like, at first row == 0. so first append(b[9*0+0]), then append(b[9*0+1]) etc
        print(','.join(arow)) #print row by row
    print() #print blank space

def fill_output(argv, board):
    with open(argv[2], 'w') as f_out:
        #f_out.write("for "+argv[3]+'\n')
        for row in range(9):
            arow = []
            for col in range(9):
                arow.append(str(board[9*row+col])) #like, at first row == 0. so first append(b[9*0+0]), then append(b[9*0+1]) etc
            f_out.write(','.join(arow) + '\n') #print row by row

def main(argv = None):
    #note args like this: arg[0]:program arg[1]:input arg[2]:output arg[3]:name of board to solve
    start_time = time.time()
    #SETTING UP BOARD WE'RE GONNA SOLVE
    global Boards, Neighbors
    if not argv:
        argv = sys.argv
    loadBoards(argv)
    makeNeighbors()
    board = Boards[argv[3]]
    #print(Boards[toSolve])

    #Now SOLVE
    myStack = [] #we'll just use append and pop to treat this like a stack
    i = 0
    backtrack = False
    numtrials = 0
    numbacks = 0
    possibilities = []
    while i < len(board):
        #first check if it needs to be switched -- remember if you're gonna backtrack, set the value of board[prev_index] to 0!
        if board[i] == 0:
            #print("board[%d] == 0"%i)
            #make possibilities, only if you're not backtracking otherwise you already have possibilities
            if not backtrack:
                possibilities = make_p(i, board)
                #print("for index %d, possibilities:"%i)
                #print(possibilities)
            if len(possibilities) > 1:
                #add to stack BEFORE YOU MAKE ANY ADJUSTMENTS (so that when you backtrack, you don't have to overwrite)
                save = SaveState(i, possibilities[:-1], board[:])
                myStack.append(save)
                #add first possibility
                board[i] = possibilities.pop() #this way the list shortens on its own
                backtrack = False
                i+=1
            elif len(possibilities) == 1: #aka forced
                board[i] = possibilities.pop() #no reason to save, bc can't change this decision
                backtrack = False
                i+=1
            else: #you have nothing to do, back track
                saved = myStack.pop()
                i = saved.index
                possibilities = saved.possibilities
                board = saved.board
                backtrack = True
                numbacks+=1
        else:
            i+=1

        numtrials+=1
    #hey! finished the board
    #checking...
    numwrong = getIncorrect(board)
    if len(numwrong)>0:
        print("did not pass check")
    else:
        print("board is correct!")
    ttime = time.time() - start_time
    fill_output(argv, board)
    print("numtrials: %d, numbacktracks: %d, time elapsed: %f"%(numtrials, numbacks, ttime))
    return 0
main()


#!/usr/bin/env python

"""Module that is used for getting the events
that occured throughout games.
"""

import mlbgame.data

import lxml.etree as etree

def game_events(game_id):
    """Return dictionary of events for a game with matching id."""
    # get data from data module
    data = mlbgame.data.get_game_events(game_id)
    # parse XML
    parsed = etree.parse(data)
    root = parsed.getroot()
    # empty output file
    output = {}
    # loop through innings
    innings = root.findall('inning')
    for x in innings:
        # top info
        topinfo = []
        # loop through the top half
        top = x.findall('top')[0]
        for y in top.findall('atbat'):
            atbat = {}
            # loop through and save info
            for i in y.attrib:
                atbat[i] = y.attrib[i]
            atbat['pitches'] = []
            for i in y.findall('pitch'):
                pitch = {}
                # loop through pitch info
                for n in i.attrib:
                    pitch[n] = i.attrib[n]
                atbat['pitches'].append(pitch)
            topinfo.append(atbat)
        # bottom info
        botinfo = []
        # loop through the bottom half
        bot = x.findall('bottom')[0]
        for y in bot.findall('atbat'):
            atbat = {}
            # loop through and save info
            for i in y.attrib:
                atbat[i] = y.attrib[i]
            atbat['pitches'] = []
            for i in y.findall('pitch'):
                pitch = {}
                # loop through pitch info
                for n in i.attrib:
                    pitch[n] = i.attrib[n]
                atbat['pitches'].append(pitch)
            botinfo.append(atbat)
        output[x.attrib['num']] = {'top': topinfo, 'bottom': botinfo}
    return output

class AtBat(object):
    """Class that holds information about at bats in games.

    Properties:

    - num = Number of at bat in game
    - b = balls (at end of at bat or currently if live)
    - s = strikes (at end of at bat or currently if live)
    - o = outs (at end of at bat)
    - batter = batter id number
    - pitcher = pitcher id number
    - des = description of at bat
    - event_num = number that corresponds to type of event
    - event = name of event
    - home_team_runs = home team runs (at end of at bat)
    - away_team_runs = away team runs (at end of at bat)
    - pitches = list of pitches during at bat
    - b1
    - b2
    - b3
    """

    def __init__(self, data):
        """Creates an event object that matches the corresponding info in `data`.

        `data` should be an dictionary of values.
        """
        # loop through data
        for x in data:
            # remove spanish info (causes text encoding errors)
            if '_es' in x:
                continue
            # create pitches list if attribute name is pitches
            if x == 'pitches':
                self.pitches = []
                for y in data[x]:
                    self.pitches.append(Pitch(y))
                continue
            # set information as correct data type
            try:
                setattr(self, x, int(data[x]))
            except ValueError:
                try:
                    setattr(self, x, float(data[x]))
                except ValueError:
                    # string if not number
                    setattr(self, x, str(data[x]))

    def nice_output(self):
        """Prints basic event info in a nice way."""
        return self.des

    def __str__(self):
        return self.nice_output()

class Pitch(object):
    """Class that holds information about individual pitches.

    Properties of pitches are wildly inconsistent,
    sometimes they have a value, sometimes they don't.
    Properties:

    - sv_id
    - des = description of pitch outcome
    - type = ball (B), strike (S), or in play (X)
    - start_speed = pitch speed
    - pitch_type = type of pitch (fastball, curve, etc.)
    """

    def __init__(self, data):
        """Creates a pitch object that matches the corresponding info in `data`.

        `data` should be an dictionary of values.
        """
        # loop through data
        for x in data:
            # remove spanish info (causes text encoding errors)
            if '_es' in x:
                continue
            # set information as correct data type
            try:
                setattr(self, x, int(data[x]))
            except ValueError:
                try:
                    setattr(self, x, float(data[x]))
                except ValueError:
                    # string if not number
                    setattr(self, x, str(data[x]))

    def nice_output(self):
        """Prints basic event info in a nice way."""
        return "Pitch: %s at %s: %s" % (self.pitch_type, self.start_speed, self.des)

    def __str__(self):
        return self.nice_output()


import os.path
import requests
import csv
import time
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
import asyncio
from random import uniform
from fake_useragent import UserAgent

ua = UserAgent()

headers = {
    'User-Agent': ua.random
}

DIR_PATH = os.path.abspath(os.path.dirname(__file__))
folder_name = "_Output"
file_name = 'all_job_data.csv'

def fetch_job_ids(url, max_pages=2):
    try:
        all_job_ids = []
        page = 1

        while page <= max_pages:
            response = goto_next_page(url, page)

            if response is not None:
                soup = BeautifulSoup(response, 'html.parser')
                job_elements = soup.find_all('li', class_='has-pointer-d')
                if not job_elements:
                    print("No job IDs found.")
                    break  # No more job IDs to fetch
                for job_element in job_elements:
                    job_id = job_element.get("data-job-id")
                    if job_id:
                        all_job_ids.append(job_id)

                page += 1
                time.sleep(uniform(1, 2))  # Add a random delay (1-2 seconds) to avoid overloading the website
            else:
                print(f"Failed to fetch data from the next page. Exiting the loop.")
                break

        return all_job_ids

    except Exception as e:
        print(f"Error occurred: {str(e)}")
        return []

def goto_next_page(url, page, retries=3, backoff_factor=2):
    try:
        headers['User-Agent'] = ua.random  # Rotate user-agent
        response = requests.get(url + f'?page={page}', headers=headers, timeout=30)  # Increase timeout to 30 seconds

        if response.status_code == 200:
            return response.content
        elif response.status_code == 429 and retries > 0:
            retry_after = int(response.headers.get('Retry-After', 5))
            print(f"Rate limited. Retrying after {retry_after} seconds...")
            time.sleep(retry_after)
            return goto_next_page(url, page, retries - 1, backoff_factor * 2)

        print(f"Failed to retrieve data from {url} (Page: {page}). Status code: {response.status_code}")
        return None

    except requests.Timeout:
        print(f"Request timed out while fetching data from {url} (Page: {page}).")
        return None

    except Exception as e:
        print(f"Error occurred while fetching data from {url} (Page: {page}). {str(e)}")
        return None

def fetch_data_for_job_id(job_id, retries=3, backoff_factor=2):
    try:
        headers['User-Agent'] = ua.random  # Rotate user-agent
        url = f'https://www.bayt.com/en/job/{job_id}/'
        with requests.Session() as session:
            response = session.get(url, headers=headers)

        if response.status_code == 200:
            soup = BeautifulSoup(response.content, 'html.parser')
            details_desc_mapping = {'Job ID': job_id}

            job_elements = soup.find_all('dl', class_='dlist is-spaced is-fitted t-small')

            for job_element in job_elements:
                job_attributes = job_element.find_all('dt')
                job_desc = job_element.find_all('dd')

                for title, data in zip(job_attributes, job_desc):
                    title_name = title.text.strip()
                    data_text = data.text.strip()
                    details_desc_mapping[title_name] = data_text

            return details_desc_mapping
        elif response.status_code == 429 and retries > 0:
            retry_after = int(response.headers.get('Retry-After', 5))
            print(f"Rate limited. Retrying after {retry_after} seconds...")
            time.sleep(retry_after)
            return fetch_data_for_job_id(job_id, retries - 1, backoff_factor * 2)

        print(f"Failed to retrieve data for Job ID: {job_id}. Status code: {response.status_code}")
        return {}

    except Exception as e:
        print(f"Error occurred while fetching data for Job ID: {job_id}. {str(e)}")
        return {}

def save_to_csv(all_data, csv_filename):
    try:
        with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = list(all_data[0].keys())
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(all_data)
        print(f"Data has been successfully saved to '{csv_filename}'.")

    except Exception as e:
        print(f"Error occurred while saving to CSV: {str(e)}")

async def main():
    url = 'https://www.bayt.com/en/saudi-arabia/jobs/'
    job_ids = fetch_job_ids(url)

    if job_ids:
        all_data = []
        field_names = set()  # Set to store all unique field names

        with ThreadPoolExecutor() as executor:
            loop = asyncio.get_event_loop()
            futures = [loop.run_in_executor(executor, fetch_data_for_job_id, job_id) for job_id in job_ids]

            for result in await asyncio.gather(*futures):
                all_data.append(result)
                field_names.update(result.keys())  # Update field names set

        # Add missing fields with empty values to all_data
        for details_desc_mapping in all_data:
            for field_name in field_names:
                if field_name not in details_desc_mapping:
                    details_desc_mapping[field_name] = ''

        folder_name = "_Output"
        path = os.path.join(DIR_PATH, folder_name)
        try:
            os.mkdir(path)
        except OSError as error:
            print(error)

        csv_filename = "all_job_data.csv"
        save_to_csv(all_data, os.path.join(path, csv_filename))
    else:
        print("No job IDs found.")

if __name__ == '__main__':
    asyncio.run(main())


from collections import defaultdict
from contextlib import contextmanager
import csv
from itertools import tee
import json
import math
import os
import tempfile
import time

import folium
import joblib
from selenium import webdriver
from shapely.geometry import shape, Point

SEASONS = ["winter", "spring", "summer", "autumn"]

@contextmanager
def _tmp_html(data):
    """Yields the path of a temporary HTML file containing data."""
    filepath = ''
    try:
        fid, filepath = tempfile.mkstemp(suffix=".html", prefix="folium_")
        os.write(fid, data.encode("utf8"))
        os.close(fid)
        yield filepath
    finally:
        if os.path.isfile(filepath):
            os.remove(filepath)

def dump_geojson(data, path, **dump_kwargs):
    with open(path, "w") as f:
        json.dump(data, f, separators=(",", ":"), **dump_kwargs)

def _is_point_in_multigon(point, polygons):
    for polygon in polygons:
        if polygon.contains(point):
            return True
    return False

def _parse_sswi_row(row, karst_polygons):
    try:
        point, lat, lng, _, horizon, season, sswi, _ = row
    except ValueError:
        return
    if point.startswith("#"):
        return

    season = SEASONS[int(season) - 1]
    lat = float(lat)
    lng = float(lng)
    sswi = float(sswi)

    in_karst = _is_point_in_multigon(Point(lng, lat), karst_polygons)
    risk_level = 0
    if sswi < -1.4 and in_karst:
        risk_level = 3
    elif sswi < -1.4 and not in_karst:
        risk_level = 2
    elif sswi < -0.7:
        risk_level = 1

    return (horizon, season, {
        "type": "Feature",
        "geometry": {
            "type": "Point",
            "coordinates": [lng, lat],
        },
        "properties": {
            "riskLevel": risk_level,
            "sswi": sswi,
            "inKarst": in_karst,
        }
    })

def compute_risks(sswi_file, karst_file, n_jobs=-1, batch_size="auto", verbose=50, logger=None):
    if logger is not None:
        logger.info("Loading data...")
        sswi_file, sswi_file2 = tee(sswi_file)
        logger.info("Number of points:", sum(1 for line in sswi_file2))

    sswi_reader = csv.reader(sswi_file, delimiter=";")
    karst_polygons = []
    for feature in json.load(karst_file)["features"]:
        karst_polygons.append(shape(feature["geometry"]).buffer(0))

    if logger is not None:
        logger.info("Number of karstic polygons:", len(karst_polygons))
        logger.info("Parsing data...")

    points = joblib.Parallel(
        n_jobs=n_jobs,
        verbose=verbose,
        backend="multiprocessing",
        batch_size=batch_size,
    )(
        joblib.delayed(_parse_sswi_row)(row, karst_polygons)
        for row in sswi_reader
    )

    min_sswi = math.inf
    max_sswi = -math.inf
    features = defaultdict(  # Horizon
        lambda: defaultdict(list)  # Season
    )
    for point in points:
        if point is None:
            continue
        h, s, p = point
        features[h][s].append(p)
        min_sswi = min(p["properties"]["sswi"], min_sswi)
        max_sswi = max(p["properties"]["sswi"], max_sswi)

    geojson = defaultdict(dict)
    for horizon, horizon_data in features.items():
        horizon_points = defaultdict(list)
        for season, points in horizon_data.items():
            geojson[horizon][season] = {
                "type": "FeatureCollection",
                "features": points,
            }
            for p in points:
                horizon_points[tuple(p["geometry"]["coordinates"])].append(p)

        worst_horizon_points = []
        best_horizon_points = []
        for coords, points in horizon_points.items():
            if len(points) != len(horizon_data): # Not all seasons have this point
                continue
            worst_p = best_p = None
            for p in points:
                risk = p["properties"]["riskLevel"]
                if worst_p is None or risk > worst_p["properties"]["riskLevel"]:
                    worst_p = p
                if best_p is None or risk < best_p["properties"]["riskLevel"]:
                    best_p = p
            worst_horizon_points.append(worst_p)
            best_horizon_points.append(best_p)

        geojson[horizon]["more_risky"] = {
            "type": "FeatureCollection",
            "features": worst_horizon_points,
        }
        geojson[horizon]["less_risky"] = {
            "type": "FeatureCollection",
            "features": best_horizon_points,
        }

    metadata = {
        "sswi": dict(min=min_sswi, max=max_sswi),
        "riskLevels": [
            "",
            "Limitations de tous les prélèvements d'eau",
            "Interdiction d'utiliser l'eau pour des usages non prioritaires",
            "Menaces de pénuries en eau potable"
        ],
    }

    return geojson, metadata

def filter_karst_by_type(f, type_=1, logger=None):
    if logger is not None:
        logger.info("Loading karst data")
    data = json.load(f)

    if logger is not None:
        logger.info("Filtering karst data")
    features = []
    for feature in data["features"]:
        if feature["properties"]["TypeZK"] == type_:
            features.append(feature)

    data["features"] = features
    return data

def display(risk, colorscale, karst=None, map_location=[46.2, 2.4], map_zoom=5, width=350, height=350, **map_kwargs):
    m = folium.Map(
        location=map_location,
        zoom_start=map_zoom,
        width=width,
        height=height,
        prefer_canvas=True,
        **map_kwargs
    )

    risk_layer = folium.FeatureGroup(name="Risk")
    for p in risk["features"]:
        color = colorscale[p["properties"]["riskLevel"]]
        lng, lat = p["geometry"]["coordinates"]
        folium.CircleMarker(
            location=[lat, lng],
            radius=0.5,
            fill=True,
            fill_color=color,
            color=color,
        ).add_to(risk_layer)
    risk_layer.add_to(m)

    if karst is not None:
        folium.GeoJson(karst, name="Karst").add_to(m)
        folium.LayerControl().add_to(m)

    return m

def map_to_png(m, path, delay=3):
    # https://github.com/python-visualization/folium/blob/master/folium/folium.py#L296
    # geckodriver must be installed:
    # https://selenium-python.readthedocs.io/installation.html#drivers
    options = webdriver.firefox.options.Options()
    options.add_argument("--headless")
    driver = webdriver.Firefox(options=options)
    html = m.get_root().render()
    with _tmp_html(html) as fname:
        # We need the tempfile to avoid JS security issues.
        driver.get("file:///{path}".format(path=fname))
        driver.maximize_window()
        time.sleep(delay)
        el = driver.find_elements_by_class_name("folium-map")[0]
        el.screenshot(path)
        driver.quit()


import uuid
import socket
from flask import Flask
from config import config
from managerLib.sheduler import BackgroundManager
from zeroconf import Zeroconf, ServiceInfo

# Routes
from routes import attendeces, classroom, recog

app = Flask(__name__)

def page_not_found(error):
    return "
URL Not found
", 404

ip_address = socket.gethostbyname(socket.gethostname()+".local")
info = ServiceInfo(
    "_http._tcp.local.",
    "flaskServer" + str(uuid.uuid4()) + "._http._tcp.local.",
    addresses=[socket.inet_aton(ip_address)],
    port=5001,
)

zeroconf = Zeroconf()
zeroconf.register_service(info)

if __name__ == '__main__':
    bSheduler = BackgroundManager()
    app.config.from_object(config['development'])

    # Blueprints
    app.register_blueprint(attendeces.main, url_prefix='/attendece')
    app.register_blueprint(classroom.main, url_prefix='/classroom')
    app.register_blueprint(recog.main, url_prefix='/recog')

    # Error handlers
    app.register_error_handler(404, page_not_found)
    app.run(host="0.0.0.0", port=5000, use_reloader=False)


from __future__ import annotations

import logging
from itertools import combinations
from typing import Any
from typing import Optional

import numpy as np
from nptyping import Float
from nptyping import NDArray
from scipy import linalg
from sklearn.base import BaseEstimator
from sklearn.base import TransformerMixin
from sklearn.utils.validation import check_array
from sklearn.utils.validation import check_is_fitted

logger = logging.getLogger(__name__)

def _jade(
    arr: NDArray[(Any, ...), Float], n_components: int = 1
) -> NDArray[(Any, ...), Float]:
    """Blind separation of real signals with JADE.

    jadeR implements JADE, an Independent Component Analysis (ICA) algorithm developed
    by Jean-Francois Cardoso. See http://www.tsi.enst.fr/~cardoso/guidesepsou.html, and
    papers cited at the end of the source file.

    Translated into NumPy from the original Matlab Version 1.8 (May 2005) by Gabriel
    Beckers, http://gbeckers.nl .

    Parameters
    ----------
    arr : NDArray
        a data matrix (n_features, n_samples)
    n_components : int
        output matrix B has size mxn so that only m sources are extracted.  This is done
        by restricting the operation of jadeR to the m first principal components.
        Defaults to None, in which case :math:`m = n`.

    Returns
    -------
    NDArray
        An m*n matrix B (NumPy matrix type), such that Y=B*X are separated sources
        extracted from the n*T data matrix X. If m is omitted, B is a square n*n matrix
        (as many sources as sensors). The rows of B are ordered such that the columns of
        :math:`pinv(B)` are in order of decreasing norm; this has the effect that the
        `most energetically significant` components appear first in the rows of
        :math:`Y = B * X`.

    Notes
    -----
    Quick notes (more at the end of this file):

    - This code is for REAL-valued signals.  A MATLAB implementation of JADE for both
      real and complex signals is also available from
      http://sig.enst.fr/~cardoso/stuff.html

    - This algorithm differs from the first released implementations of JADE in that it
      has been optimized to deal more efficiently
        1) with real signals (as opposed to complex)
        2) with the case when the ICA model does not necessarily hold.

    - There is a practical limit to the number of independent components that can be
      extracted with this implementation.  Note that the first step of JADE amounts to a
      PCA with dimensionality reduction from `n` to `m` (which defaults to n).  In
      practice m cannot be *very large* (more than 40, 50, 60... depending on available
      memory)

    - See more notes, references and revision history at the end of this file and more
      stuff at http://sig.enst.fr/~cardoso/stuff.html

    - For more info on NumPy translation, see the end of this file.

    - This code is supposed to do a good job!  Please report any problem relating to
      the NumPY code gabriel@gbeckers.nl

    Copyright original Matlab code : Jean-Francois Cardoso <cardoso@sig.enst.fr>
    Copyright Numpy translation : Gabriel Beckers <gabriel@gbeckers.nl>
    """
    logger.info("jade -> Looking for %d sources", n_components)
    logger.info("jade -> Removing the mean value")
    n_samples, _ = arr.shape
    arr -= arr.mean(axis=0)

    # whitening & projection onto signal subspace
    # ===========================================
    logger.info("jade -> Whitening the data")

    # --- PCA  ----------------------------------------------------------
    u, s, vh = linalg.svd(arr, full_matrices=False)
    u, s, vh = u[:, :n_components], s[:n_components], vh[:n_components]
    B = linalg.inv(np.diag(s)) @ vh * np.sqrt(n_samples)
    arr = u * np.sqrt(n_samples)

    del u, s, vh
    # NOTE: At this stage, X is a PCA analysis in m components of the real data, except
    # that all its entries now have unit variance.  Any further rotation of X will
    # preserve the property that X is a vector of uncorrelated components.  It remains
    # to find the rotation matrix such that the entries of X are not only uncorrelated
    # but also `as independent as possible".  This independence is measured by
    # correlations of order higher than 2.  We have defined such a measure of
    # independence which
    #   1) is a reasonable approximation of the mutual information
    #   2) can be optimized by a `fast algorithm"
    # This measure of independence also corresponds to the `diagonality" of a set of
    # cumulant matrices.  The code below finds the `missing rotation " as the matrix
    # which best diagonalizes a particular set of cumulant matrices.

    # Estimation of the cumulant matrices.
    # ====================================
    logger.info("jade -> Estimating cumulant matrices")

    # Dim. of the space of real symm matrices
    dimsymm = n_components * (n_components + 1) // 2
    nbcm = dimsymm  # number of cumulant matrices
    # Storage for cumulant matrices
    CM = np.zeros((n_components, n_components * nbcm))
    R = np.eye(n_components)

    # I am using a symmetry trick to save storage.  I should write a short note one of
    # these days explaining what is going on here.
    # will index the columns of CM where to store the cum. mats.
    Range = np.arange(n_components)
    sqrt2 = np.sqrt(2)

    for im in range(n_components):
        Xim = np.c_[arr[:, im]]
        Xijm = Xim ** 2
        # Note to myself: the -R on next line can be removed: it does not affect
        # the joint diagonalization criterion
        Rim = np.c_[R[:, im]]
        CM[:, Range] = (Xijm * arr).T @ arr / n_samples - R - 2 * (Rim @ Rim.T)
        Range += n_components
        for jm in range(im):
            Xijm = Xim * np.c_[arr[:, jm]]
            Rjm = np.c_[R[:, jm]]
            CM[:, Range] = (
                sqrt2 * (Xijm * arr).T @ arr / n_samples - Rim @ Rjm.T - Rjm @ Rim.T
            )
            Range = Range + n_components

    # Now we have nbcm = m(m+1)/2 cumulants matrices stored in a big m x m*nbcm array.

    V = np.eye(n_components)

    On = np.zeros(CM.shape[0])
    Range = np.arange(n_components)
    for _ in range(nbcm):
        diag = np.diag(CM[:, Range])
        On += np.sum(diag ** 2, axis=0)
        Range += n_components
    Off = np.sum(CM ** 2) - On

    # % A statistically scaled threshold on `small" angles
    seuil = 1e-6 / np.sqrt(n_samples)
    encore = True
    sweep = 0  # % sweep number
    updates = 0  # % Total number of rotations

    # Joint diagonalization proper
    logger.info("jade -> Contrast optimization by joint diagonalization")

    counters = list(combinations(range(n_components), 2))
    while encore:
        encore = False
        logger.info("jade -> Sweep #%3d", sweep)
        sweep += 1
        upds = 0  # % Number of rotations in a given seep

        for p, q in counters:
            Ip = np.arange(p, n_components * nbcm, n_components)
            Iq = np.arange(q, n_components * nbcm, n_components)

            # computation of Givens angle
            g = np.c_[[CM[p, Ip] - CM[q, Iq], CM[p, Iq] + CM[q, Ip]]]
            gg = g @ g.T
            ton = gg[0, 0] - gg[1, 1]
            toff = gg[0, 1] + gg[1, 0]
            theta = 0.5 * np.arctan2(toff, ton + np.sqrt(ton * ton + toff * toff))
            Gain = (np.sqrt(ton * ton + toff * toff) - ton) / 4.0

            # Givens update
            if abs(theta) > seuil:
                encore = True
                upds += 1
                c = np.cos(theta)
                s = np.sin(theta)
                G = np.array([[c, -s], [s, c]])
                pair = np.array([p, q])
                V[:, pair] = V[:, pair] @ G
                CM[pair, :] = G.T @ CM[pair, :]
                cIp = np.c_[CM[:, Ip]]
                cIq = np.c_[CM[:, Iq]]
                CM[:, np.r_[Ip, Iq]] = np.c_[c * cIp + s * cIq, -s * cIp + c * cIq]
                On += Gain
                Off -= Gain

        logger.info("completed in %d rotations", upds)
        updates = updates + upds
    logger.info("jade -> Total of %d Givens rotations", updates)

    # A separating matrix
    # ===================
    B = V.T @ B

    # Permute the rows of the separating matrix B to get the most energetic components
    # first. Here the **signals** are normalized to unit variance.  Therefore, the sort
    # is according to the norm of the columns of A = pinv(B)

    logger.info("jade -> Sorting the components")

    A = linalg.pinv(B)
    keys = np.argsort((A * A).sum(axis=0))[::-1]
    B = B[keys, :]

    logger.info("jade -> Fixing the signs")
    b = B[:, 0]
    signs = np.sign(np.sign(b) + 0.1)  # just a trick to deal with sign=0
    B = np.diag(signs) @ B

    return B

    # Revision history of MATLAB code:
    #
    # - V1.8, May 2005
    #  - Added some commented code to explain the cumulant computation tricks.
    #  - Added reference to the Neural Comp. paper.
    #
    # -  V1.7, Nov. 16, 2002
    #   - Reverted the mean removal code to an earlier version (not using
    #     repmat) to keep the code octave-compatible.  Now less efficient,
    #     but does not make any significant difference wrt the total
    #     computing cost.
    #   - Remove some cruft (some debugging figures were created.  What
    #     was this stuff doing there???)
    #
    #
    # -  V1.6, Feb. 24, 1997
    #   - Mean removal is better implemented.
    #   - Transposing X before computing the cumulants: small speed-up
    #   - Still more comments to emphasize the relationship to PCA
    #
    # -  V1.5, Dec. 24 1997
    #   - The sign of each row of B is determined by letting the first element be
    #     positive.
    #
    # -  V1.4, Dec. 23 1997
    #   - Minor clean up.
    #   - Added a verbose switch
    #   - Added the sorting of the rows of B in order to fix in some reasonable way the
    #     permutation indetermination.  See note 2) below.
    #
    # -  V1.3, Nov.  2 1997
    #   - Some clean up.  Released in the public domain.
    #
    # -  V1.2, Oct.  5 1997
    #   - Changed random picking of the cumulant matrix used for initialization to a
    #     deterministic choice.  This is not because of a better rationale but to make
    #     the ouput (almost surely) deterministic.
    #   - Rewrote the joint diag. to take more advantage of Matlab"s tricks.
    #   - Created more dummy variables to combat Matlab"s loose memory management.
    #
    # -  V1.1, Oct. 29 1997.
    #    Made the estimation of the cumulant matrices more regular. This also corrects a
    #    buglet...
    #
    # -  V1.0, Sept. 9 1997. Created.
    #
    # Main references:
    # @article{CS-iee-94,
    #  title 	= "Blind beamforming for non {G}aussian signals",
    #  author       = "Jean-Fran\c{c}ois Cardoso and Antoine Souloumiac",
    #  HTML 	= "ftp://sig.enst.fr/pub/jfc/Papers/iee.ps.gz",
    #  journal      = "IEE Proceedings-F",
    #  month = dec, number = 6, pages = {362-370}, volume = 140, year = 1993}
    #
    #
    # @article{JADE:NC,
    #  author  = "Jean-Fran\c{c}ois Cardoso",
    #  journal = "Neural Computation",
    #  title   = "High-order contrasts for independent component analysis",
    #  HTML    = "http://www.tsi.enst.fr/~cardoso/Papers.PS/neuralcomp_2ppf.ps",
    #  year    = 1999, month =	jan,  volume =	 11,  number =	 1,  pages =  "157-192"}
    #
    #
    #  Notes:
    #  ======
    #
    #  Note 1) The original Jade algorithm/code deals with complex signals in Gaussian
    #  noise white and exploits an underlying assumption that the model of independent
    #  components actually holds.  This is a reasonable assumption when dealing with
    #  some narrowband signals.  In this context, one may i) seriously consider dealing
    #  precisely with the noise in the whitening process and ii) expect to use the small
    #  number of significant eigenmatrices to efficiently summarize all the 4th-order
    #  information.  All this is done in the JADE algorithm.
    #
    #  In *this* implementation, we deal with real-valued signals and we do NOT expect
    #  the ICA model to hold exactly.  Therefore, it is pointless to try to deal
    #  precisely with the additive noise and it is very unlikely that the cumulant
    #  tensor can be accurately summarized by its first n eigen-matrices.  Therefore,
    #  we consider the joint diagonalization of the *whole* set of eigen-matrices.
    #  However, in such a case, it is not necessary to compute the eigenmatrices at all
    #  because one may equivalently use `parallel slices" of the cumulant tensor.  This
    #  part (computing the eigen-matrices) of the computation can be saved: it suffices
    #  to jointly diagonalize a set of cumulant matrices.  Also, since we are dealing
    #  with reals signals, it becomes easier to exploit the symmetries of the cumulants
    #  to further reduce the number of matrices to be diagonalized. These
    #  considerations, together with other cheap tricks lead to this version of JADE
    #  which is optimized (again) to deal with real mixtures and to work `outside the
    #  model'.  As the original JADE algorithm, it works by minimizing a `good set' of
    #  cumulants.
    #
    #  Note 2) The rows of the separating matrix B are resorted in such a way that the
    #  columns of the corresponding mixing matrix A=pinv(B) are in decreasing order of
    #  (Euclidian) norm.  This is a simple, `almost canonical" way of fixing the
    #  indetermination of permutation.  It has the effect that the first rows of the
    #  recovered signals (ie the first rows of B*X) correspond to the most energetic
    #  *components*.  Recall however that the source signals in S=B*X have unit
    #  variance.  Therefore, when we say that the observations are unmixed in order of
    #  decreasing energy, this energetic signature is to be found as the norm of the
    #  columns of A=pinv(B) and not as the variances of the separated source signals.
    #
    #  Note 3) In experiments where JADE is run as B=jadeR(X,m) with m varying in range
    #  of values, it is nice to be able to test the stability of the decomposition.  In
    #  order to help in such a test, the rows of B can be sorted as described above. We
    #  have also decided to fix the sign of each row in some arbitrary but fixed way.
    #  The convention is that the first element of each row of B is positive.
    #
    #  Note 4) Contrary to many other ICA algorithms, JADE (or least this version) does
    #  not operate on the data themselves but on a statistic (the full set of 4th order
    #  cumulant). This is represented by the matrix CM below, whose size grows as m^2 x
    #  m^2 where m is the number of sources to be extracted (m could be much smaller
    #  than n).  As a consequence, (this version of) JADE will probably choke on a
    #  `large' number of sources. Here `large' depends mainly on the available memory
    #  and could be something like 40 or so.  One of these days, I will prepare a
    #  version of JADE taking the `data' option rather than the `statistic' option.

    # Notes on translation (GB):
    # =========================
    #
    # Note 1) The function jadeR is a relatively literal translation from the original
    # MATLABcode. I haven't really looked into optimizing it for NumPy. If you have any
    # time to look at this and good ideas, let me know.
    #
    # Note 2) A test module that compares NumPy output with Octave (MATLAB
    # clone) output of the original MATLAB script is available

class JadeICA(TransformerMixin, BaseEstimator):
    """Perform blind source separation using joint diagonalization."""

    def __init__(self, *, n_components: Optional[int] = None) -> None:
        """Perform a blind signal separation using joint diagonalization.

        Parameters
        ----------
        n_components : int, default=None
            Number of signals to extract. `None` assumes all components
        """
        super().__init__()

        self.n_components: Optional[int] = n_components
        self.mean_: NDArray[(Any, ...), Float]
        self.components_: NDArray[(Any, ...), Float]

    def fit(
        self,
        arr: NDArray[(Any, ...), Float],
        y: Optional[NDArray[(Any, ...), Float]] = None,
    ) -> JadeICA:
        """Calculate the unmixing matrix.

        Parameters
        ----------
        arr : NDArray
            mixed signal array
        y : NDArray, optional
            unused

        Returns
        -------
        self

        Raises
        ------
        IndexError
            if :math:`m > n_features`
        """
        # GB: we do some checking of the input arguments and copy data to new variables to
        # avoid messing with the original input. We also require double precision (float64)
        # and a numpy matrix type for X.
        origtype = arr.dtype  # remember to return matrix B of the same type
        arr = check_array(arr, dtype=float)

        # GB: n is number of input signals, T is number of samples
        _, n_features = arr.shape

        if self.n_components is None:
            self.n_components = n_features  # Number of sources defaults to # of sensors
        elif self.n_components > n_features:
            raise IndexError(
                f"More sources ({self.n_components}) than sensors ({n_features})"
            )

        self.mean_ = arr.mean(axis=0)
        self.components_ = _jade(arr, n_components=self.n_components).astype(origtype)
        return self

    def transform(self, arr: NDArray[(Any, ...), Float]) -> NDArray[(Any, ...), Float]:
        """Project the unmixing matrix onto `arr` to give independent signals.

        Parameters
        ----------
        arr : NDArray
            Unmixed signals

        Returns
        -------
        NDArray
            Unmixed signal array
        """
        check_is_fitted(self)

        arr -= self.mean_
        signal: NDArray[(Any, ...), Float] = arr @ self.components_.T
        return signal

    def fit_transform(
        self,
        arr: NDArray[(Any, ...), Float],
        y: Optional[NDArray[(Any, ...), Float]] = None,
        **fit_params: str,
    ) -> NDArray[(Any, ...), Float]:
        """Determine the independent signals using joint diagonalization.

        Parameters
        ----------
        arr : NDArray
            Mixed signal array
        y : NDArray, optional
            unused
        fit_params : dict
            unused

        Returns
        -------
        NDArray
            Unmixed signal array

        Raises
        ------
        IndexError
            if :math:`m > n_features`
        """
        # GB: we do some checking of the input arguments and copy data to new variables to
        # avoid messing with the original input. We also require double precision (float64)
        # and a numpy matrix type for X.
        origtype = arr.dtype  # remember to return matrix B of the same type
        arr1 = check_array(arr, dtype=float)

        # GB: n is number of input signals, T is number of samples
        _, n_features = arr.shape

        if self.n_components is None:
            self.n_components = n_features  # Number of sources defaults to # of sensors
        elif self.n_components > n_features:
            raise IndexError(
                f"More sources ({self.n_components}) than sensors ({n_features})"
            )

        self.mean_ = arr.mean(axis=0)
        self.components_ = _jade(arr1, n_components=self.n_components).astype(origtype)

        arr -= self.mean_
        signal: NDArray[(Any, ...), Float] = arr @ self.components_.T
        return signal

    def inverse_transform(
        self, arr: NDArray[(Any, ...), Float]
    ) -> NDArray[(Any, ...), Float]:
        """Find the original mixed signals.

        Parameters
        ----------
        arr : NDArray
            Unmixed signal array

        Raises
        ------
        NotImplementedError
            If called because Jade does not provide inversion
        """
        raise NotImplementedError(
            "Inverse transformation is currently not implemented."
        )


import openpyxl
from lxml import etree as ET

# Открытие файла Excel
workbook = openpyxl.load_workbook('test_input.xlsx', data_only=True)
sheet = workbook.active

# Создание корневого элемента CERTDATA
certdata = ET.Element('CERTDATA')

# Добавление элемента FILENAME
filename = ET.SubElement(certdata, 'FILENAME')
filename.text = sheet['B3'].value

# Создание элемента ENVELOPE
envelope = ET.SubElement(certdata, 'ENVELOPE')

# Чтение данных из таблицы Excel и создание элементов ECERT
for row in sheet.iter_rows(min_row=6, values_only=True):
    ecert = ET.SubElement(envelope, 'ECERT')
    certno = ET.SubElement(ecert, 'CERTNO')
    certno.text = str(row[0])
    certdate = ET.SubElement(ecert, 'CERTDATE')
    certdate.text = row[1].strftime('%Y-%m-%d')
    status = ET.SubElement(ecert, 'STATUS')
    status.text = row[2]
    iec = ET.SubElement(ecert, 'IEC')
    iec.text = str(row[3])
    expname = ET.SubElement(ecert, 'EXPNAME')
    expname.text = f'"{row[4]}"'
    billid = ET.SubElement(ecert, 'BILLID')
    billid.text = row[5]
    sdate = ET.SubElement(ecert, 'SDATE')
    sdate.text = row[6].strftime('%Y-%m-%d')
    scc = ET.SubElement(ecert, 'SCC')
    scc.text = row[7]
    svalue = ET.SubElement(ecert, 'SVALUE')
    svalue.text = str(row[8]).replace(',', '.')

# Создание объекта ElementTree и запись в файл
tree = ET.ElementTree(certdata)
tree.write('output_xml.xml', encoding='utf-8', xml_declaration=True,pretty_print=True)