from icesatReader import get_atl03_struct
from icesatReader import get_atl08_struct
from icesatReader import get_atl09_struct
from icesatReader import read_atl03_geolocation
from icesatReader import append_atl03_geolocation

from icesatIO import readAtlH5
import os
import re
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy import signal
from scipy.stats import skew, kurtosis, shapiro
from tqdm import tqdm

def get_beam_feature(atl09):

    if atl09.orbit_info.sc_orient == 0:
        beam = np.full(atl09.df.shape[0], int(atl09.gtNum[2]))

    else:
        if atl09.gtNum[2] == '1':
            beam = np.full(atl09.df.shape[0], 3)
        elif atl09.gtNum[2] == '2':
            beam = np.full(atl09.df.shape[0], 2)
        else:
            beam = np.full(atl09.df.shape[0], 1)

    return beam

def plot_cab(atl09, x_column='northing', remove_surface=False,
             show_layers=True, show_detected_surface=True, show_dem=True,
             overlay_alpha=1):

    if atl09.df.solar_elevation.mean() > -7:
        cmap_range = (1e-10, 1.1e-5)
    else:
        cmap_range = (1e-10, 0.9e-5)

    if remove_surface:
        cab = np.copy(atl09.cab_no_surf)
    else:
        cab = np.copy(atl09.cab_prof)

    cab[cab > 1e30] = 0
    begin_val = atl09.df.loc[:, x_column][0]
    end_val = atl09.df.loc[:, x_column][-1:].iloc[0]

    if begin_val > end_val:
        x = np.flip(atl09.df.loc[:, x_column])
        y = atl09.ds_va_bin_h
        z = np.fliplr(cab.T)

    else:
        x = atl09.df.loc[:, x_column]
        y = atl09.ds_va_bin_h
        z = (cab.T)

    fig, ax = plt.subplots(figsize=(12, 6))

    cmap = plt.get_cmap('viridis')  # plasma, inferno also good
    im = plt.pcolormesh(x,
                        y,
                        z,
                        vmin=cmap_range[0],
                        vmax=cmap_range[1],
                        cmap=cmap)

    fig.colorbar(im, ax=ax)

    styles = ['w.', 'w.', 'w.',
              'w.', 'w.', 'w.',
              'w.', 'w.', 'w.',
              'r.']
    if show_layers:
        for layer_num, style in zip(np.arange(10), styles):
            top_col = 'layer_top_' + str(layer_num)
            bot_col = 'layer_bot_' + str(layer_num)
            mask = atl09.df.loc[:, top_col] < 1e30
            if layer_num == 0:
                plt.plot(atl09.df.loc[mask, x_column],
                         atl09.df.loc[mask, top_col],
                         style,
                         alpha=overlay_alpha,
                         label='DDA Layers')
            else:
                plt.plot(atl09.df.loc[mask, x_column],
                         atl09.df.loc[mask, top_col],
                         style,
                         alpha=overlay_alpha)

            plt.plot(atl09.df.loc[mask, x_column],
                     atl09.df.loc[mask, bot_col],
                     style,
                     alpha=overlay_alpha)

    if show_dem:
        plt.plot(atl09.df.loc[:, x_column],
                 atl09.df.dem_h,
                 '.',
                 color='lightsteelblue',
                 alpha=overlay_alpha,
                 label='DEM')

    if show_detected_surface:
        mask = atl09.df.surface_bin < 1e3
        heights = atl09.df.loc[mask, 'surface_bin'].apply(lambda x: atl09.ds_va_bin_h[x])
        plt.plot(atl09.df.loc[mask, x_column],
                 heights,
                 '.',
                 color='red',
                 alpha=1,
                 label='ATL09 Detected Surface')

    plt.ylim([-1000, 17500])

    if begin_val > end_val:
        ax = plt.gca()
        xlim = ax.get_xlim()
        plt.xlim([xlim[1], xlim[0]])

    plt.xlabel(x_column.upper())
    plt.ylabel('Elevation (m)')
    plt.title(atl09.atlFileName + ', Profile ' + atl09.gtNum[2])
    plt.legend()
    ax = plt.gca()

    return ax

def smooth(x, window_len=11, window='hanning'):
    """
    From scipy-cookbook.readthedocs.io/items/SignalSmooth.html
    """

    if x.ndim != 1:
        print("smooth only accepts 1 dimension arrays.")

    if x.size < window_len:
        print("Input vector needs to be bigger than window size.")

    if window_len < 3:
        return x

    if window not in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
        print("Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")

    s = np.r_[x[window_len - 1:0:-1], x, x[-2:-window_len - 1:-1]]

    if window == 'flat':  # moving average
        w = np.ones(window_len, 'd')
    else:
        w = eval('np.' + window + '(window_len)')

    y = np.convolve(w / w.sum(), s, mode='valid')
    return y

def get_cab_no_surf(atl09,
                    show_plots=False,
                    idxing=None,
                    column_search_radius=5,
                    track_search_radius=5,
                    window_len=7,
                    peak_threshold=2e-5):

    if idxing is None:
        idxing = atl09.df.index

    cab_no_surf = pd.DataFrame(np.copy(atl09.cab_prof), index=atl09.df.index)
    cab_no_surf[cab_no_surf > 1e30] = 0
    # Surface estimate algorithm
    cab = pd.DataFrame(atl09.cab_prof,
                       index=atl09.df.index)
    cab[cab > 1e30] = 0

    cutoff_bins = np.full(atl09.df.shape[0], 700)
    surf_removal_method = np.zeros_like(cutoff_bins)

    # Don't accidentally terrorize matplotlib
    if idxing.shape[0] > 20:
        show_plots = False

    for idx in tqdm(idxing):
        # Get surface bin and CAB profile
        surf_bin_i = atl09.df.loc[idx, 'surface_bin'] - 1
        cab_i = cab.loc[idx, :]
        lowest_layer_h = min(atl09.df.loc[idx,
                                          ['layer_bot_' + str(x)
                                            for x in np.arange(10)]])
        lowest_layer_bin = np.around((max(atl09.ds_va_bin_h) - lowest_layer_h) / 30) + 2

        if surf_bin_i < 701:
            # If surface was found by ATL09, confirm location with CAB profile

            # too overzealous with low clouds
            # surf_bin_check = cab_i.iloc[surf_bin_i - column_search_radius:\
            #                             surf_bin_i + column_search_radius].idxmax()
            # cutoff_bin = int(surf_bin_check - 1)  # 30m buffer

            cutoff_bin = int(surf_bin_i - 3)
            surf_removal_method[idx] = 1

            if show_plots:
                # Visualize
                plt.figure()
                plt.title(str(idx) + ', ATL09 Located surface')
                plt.plot(cab_i)
                # plt.axvline(surf_bin_check, color='green', label='Found surface')
                plt.axvline(surf_bin_i, color='blue', label='ATL09 surface')
                plt.axvline(cutoff_bin, color='red', label='Cutoff point')
                plt.axvline(surf_bin_i + column_search_radius, color='yellow')
                plt.axvline(surf_bin_i - column_search_radius, color='yellow')
                plt.xlim([surf_bin_i - 10, surf_bin_i + 10])
                plt.legend()

        else:
            # If ATL09 couldn't find surface
            # See if ATL09 found the surface for nearby neighbors
            local_surf_bin = atl09.df.loc[idx - track_search_radius:
                                          idx + track_search_radius, 'surface_bin'] - 1
            min_local_bin = local_surf_bin[local_surf_bin < 701].min()
            max_local_bin = local_surf_bin[local_surf_bin < 701].max()
            mean_local_bin = local_surf_bin[local_surf_bin < 701].mean()

            if np.isnan(mean_local_bin):
                # IF no peaks identified,
                # OR no singular peak identified
                # OR no nearby surface detected
                # Use DEM to estimate surface location
                # Check granule-level surface height difference
                # Use 2 sigma difference above DEM for cutoff
                # Check DEM for estimated surface location
                dem_bin = np.around((max(atl09.ds_va_bin_h) - atl09.df.loc[idx, 'dem_h']) / 30) + 1
                error_h = atl09.df.dem_h[atl09.df.surface_bin < 800] - \
                    atl09.df.surface_height[atl09.df.surface_bin < 800]
                if len(error_h) == 0:
                    # ATL09 did not find any surface points in this granule
                    cutoff_bin = int(dem_bin)
                else:
                    stdev = np.std(error_h)
                    dem_above_bins = np.around((error_h.mean() + 2 * stdev) / 30)
                    dem_below_bins = np.around((error_h.mean() - 2 * stdev) / 30)

                    bin_2sigma_above_dem = dem_bin + dem_below_bins

                    cutoff_bin = int(bin_2sigma_above_dem)
                    surf_removal_method[idx] = 4

                if show_plots:
                    plt.figure()
                    plt.title(str(idx) + ', DEM Approach')
                    plt.plot(cab_i, label='CAB')
                    plt.axvline(dem_bin, label='DEM Surface', color='magenta')
                    plt.axvline(cutoff_bin, label='Cutoff', color='red')
                    plt.xlim([min([cutoff_bin, dem_bin]) - 30, max([cutoff_bin, dem_bin]) + 30])
                    plt.legend()

            else:
                # Use neighbors to define window to search for peak in CAB
                cab_smooth = smooth(cab_i, window_len=window_len)
                bin_smooth = np.linspace(0, 700 + window_len / 3, 699 + window_len)
                cab_search = cab_smooth[min_local_bin - column_search_radius:
                                        max_local_bin + column_search_radius]

                pk_idx, pk_h = signal.find_peaks(cab_search, height=peak_threshold)

                if len(pk_idx) == 1:
                    # IF a peak is found based on CAB, define this as surface
                    peak = np.around(bin_smooth[min_local_bin - column_search_radius + pk_idx - 1]) - 1
                    cutoff_bin = int(peak)
                    surf_removal_method[idx] = 3
                    if show_plots:
                        plt.figure()
                        plt.title(str(idx) + ', Smoothed peak finding')
                        plt.plot(cab_i, label='CAB')
                        plt.plot(bin_smooth, cab_smooth, label='Smoothed CAB profile')
                        plt.axvline(np.around(mean_local_bin), color='cyan', label='Avg nearby surface location')
                        plt.axvline(cutoff_bin, color='green', label='Cutoff bin')
                        plt.axvline(min_local_bin - column_search_radius, color='red', label='Search window')
                        plt.axvline(min_local_bin + column_search_radius, color='red')
                        plt.axvline(surf_bin_i - column_search_radius, color='red')
                        plt.axvline(lowest_layer_bin, color='black')
                        plt.xlim([min([cutoff_bin, lowest_layer_bin, min_local_bin, surf_bin_i]) - 100,
                                  max([cutoff_bin, lowest_layer_bin, min_local_bin, surf_bin_i]) + 100])
                        plt.legend()

                else:
                    # IF no clear peak identified, USE DEM approach
                    dem_bin = np.around((max(atl09.ds_va_bin_h) - atl09.df.loc[idx, 'dem_h']) / 30) + 1
                    error_h = atl09.df.dem_h[atl09.df.surface_bin < 800] - \
                        atl09.df.surface_height[atl09.df.surface_bin < 800]
                    if len(error_h) == 0:
                        # ATL09 did not find any surface points in this granule
                        cutoff_bin = int(dem_bin)
                    else:
                        stdev = np.std(error_h)
                        dem_above_bins = np.around((error_h.mean() + 2 * stdev) / 30)
                        dem_below_bins = np.around((error_h.mean() - 2 * stdev) / 30)

                        bin_2sigma_above_dem = dem_bin + dem_below_bins

                        cutoff_bin = int(bin_2sigma_above_dem)
                        surf_removal_method[idx] = 4

                    if show_plots:
                        plt.figure()
                        plt.title(str(idx) + ', DEM approach, finding peaks w smooth CAB')
                        plt.plot(cab_i, label='CAB')
                        found_peaks = np.around(bin_smooth[min_local_bin - column_search_radius + pk_idx - 1])
                        if len(found_peaks) > 1:
                            for pk in found_peaks:
                                if pk == found_peaks[0]:
                                    plt.axvline(pk, color='cyan', label='Peaks found')
                                else:
                                    plt.axvline(pk, color='cyan')

                        plt.axvline(lowest_layer_bin, color='black', label='Lowest cloud layer')
                        plt.axvline(dem_bin, label='DEM Surface', color='magenta')
                        plt.axvline(cutoff_bin, label='Cutoff', color='red')
                        plt.xlim([min([min(found_peaks), cutoff_bin, lowest_layer_bin]) - 15,
                                  max([max(found_peaks), cutoff_bin, lowest_layer_bin]) + 15])
                        plt.legend()

        if np.abs(lowest_layer_bin - cutoff_bin) < 5:
            # safety check if clouds are low
            cutoff_bin = int(lowest_layer_bin + 5)
            surf_removal_method[idx] = 5

        cutoff_bins[idx] = cutoff_bin
        cab_no_surf.iloc[idx, cutoff_bin:] = 0

    return cab_no_surf.iloc[idxing], surf_removal_method
def get_ml_features(atl09):
    df_ml = pd.DataFrame(index=atl09.df.index,
                         columns=['profile', 'beam', 'file', 'cab_int',
                                  'cab_skew', 'cab_kurt', 'cab_shapiro'])

    df_ml.loc[:, 'profile'] = np.full(df_ml.shape[0], int(atl09.gtNum[2]))
    df_ml.loc[:, 'file'] = np.full(df_ml.shape[0], atl09.atlFileName)
    df_ml.loc[:, 'beam'] = get_beam_feature(atl09)
    df_ml.loc[:, 'cab_skew'] = skew(atl09.cab_no_surf, axis=1)
    df_ml.loc[:, 'cab_kurt'] = kurtosis(atl09.cab_no_surf, axis=1)
    df_ml.loc[:, 'cab_int'] = -np.trapz(atl09.cab_no_surf, x=atl09.ds_va_bin_h)
    df_ml.loc[:, 'cab_shapiro'] = [i[0] for i in atl09.cab_no_surf.apply(shapiro, axis=1)]

    return df_ml

def get_rgt_matches(input_file_name, filenames_to_search, use_full_rgtcs=False):
    # Find the set of 8 numbers that is RGT, cycle, and segment data, in the filename, store rgtcc
    if use_full_rgtcs:
        rgtcc_str = '_(\d{8})_'
    else:
        rgtcc_str = '_(\d{6})\d{2}_'

    rgtcc_match = re.search(rgtcc_str, input_file_name)
    matches = dict()
    if rgtcc_match == None:
        print('Could not identify rgt for input file')
        matches = []

    else:
        # Get the 4-digit RGT CC combo
        rgtcc = rgtcc_match.group(1)

        # Search for RGTCC set in input list
        if use_full_rgtcs:
            search_str = '_(' + rgtcc + ')_'
        else:
            search_str = '_(' + rgtcc + '\d{2})_'

        # Returns list of matching files
        found = [x for x in filenames_to_search if re.search(search_str, x)]
        matches[input_file_name] = found

    return matches

def check_for_matching_segments(atl09file, atl03file_list, atl09basepath, atl03basepath):
    atl03_overlap_with_atl09 = pd.DataFrame(columns=['starting_seg', 'ending_seg'])

    if len(atl03file_list) == 0:
        print('WARNING No matching atl03 file found')
    else:
        atl09_segment_ids = readAtlH5(atl09basepath + atl09file,
                                      'profile_1/high_rate/segment_id')
        sc_orient = readAtlH5(atl09basepath + atl09file,
                              'orbit_info/sc_orient')

        if len(sc_orient) == 1 and sc_orient[0] == 0:
            gt = 'gt1l'
        elif len(sc_orient) == 1 and sc_orient[0] == 1:
            gt = 'gt1r'
        else:
            print('WARNING the spacecraft transitions orientation during this granule')

        for atl03file in atl03file_list:
            atl03_segment_ids = readAtlH5(atl03basepath + atl03file,
                                          gt + '/geolocation/segment_id')

            segments_in_both = np.isin(atl09_segment_ids, atl03_segment_ids)
            if np.any(segments_in_both):
                starting_seg = atl09_segment_ids[segments_in_both][0]
                ending_seg = atl09_segment_ids[segments_in_both][-1]
                atl03_overlap_with_atl09.loc[atl03file, 'starting_seg'] = starting_seg
                atl03_overlap_with_atl09.loc[atl03file, 'ending_seg'] = ending_seg

    return atl03_overlap_with_atl09

def atl09_overlap_info(atl09file, atl09basepath, atl08basepath, atl03basepath):
    """Returns dataframe of where the ATL09 file overlaps with ATL03/08 files"""

    matches_03 = get_rgt_matches(input_file_name=atl09file,
                                 filenames_to_search=np.unique(os.listdir(atl03basepath)),
                                 use_full_rgtcs=False)

    if len(matches_03[atl09file]) != 0:
        atl03file_list = matches_03[atl09file]
        overlap_with_atl09_df = check_for_matching_segments(atl09file,
                                                            atl03file_list,
                                                            atl09basepath,
                                                            atl03basepath)

        # Find corresponding atl08 file for each atl03 file
        for atl03file in overlap_with_atl09_df.index:
            matches_08 = get_rgt_matches(input_file_name=atl03file,
                                         filenames_to_search=np.unique(os.listdir(atl08basepath)),
                                         use_full_rgtcs=True)

            if len(matches_08) != 0:
                print('Found ATL08 file for ' + atl03file)
                overlap_with_atl09_df.loc[atl03file, 'atl08file'] = matches_08[atl03file]

            else:
                print('UNABLE TO FIND MATCHING ATL08 FOR ' + atl03file)
    else:
        print('UNABLE TO FIND MATCHING ATL03 FILE')

    overlap_with_atl09_df.reset_index(inplace=True)
    overlap_with_atl09_df.rename({'index': 'atl03file'},
                                 axis='columns',
                                 inplace=True)

    return overlap_with_atl09_df

# %%
if __name__ == "__main__":
    print('Test')


#!/usr/bin/env python3

# Origin Write Metadata
# author: hypermodified
# This python script loops through a directory opens the associated origin file, gets the meta data from that and writes it to the vorbis tags in the flac files in the directory
# This script writes to Album Artist, Album, Year, Label, Catalog Number for all albums.
# It will write to Artist as well if you specify that the albums are not Various Artists, DJ or Classical albums.
# If the folder it is checking starts with CD it will use the number of the folder to write to disc number.
# Additionally it will check folders that start with CD for cover art and if it is missing will copy it from the album folder.
# This has only been tested to work with flac files.
# It can handle albums with artwork folders or multiple disc folders in them. It can also handle specials characters.
# It has been tested and works in both Ubuntu Linux and Windows 10.

# Before running this script install the dependencies
# pip install mutagen

# Import dependencies
import os  # Imports functionality that let's you interact with your operating system
import yaml  # Imports yaml
import shutil  # Imports functionality that lets you copy files and directory
import datetime  # Imports functionality that lets you make timestamps
import mutagen  # Imports functionality to get metadata from music files

import origin_script_library as osl  # Imports common code used across all origin scripts

#  Set your directories here
album_directory = "M:\PROCESS"  # Which directory do you want to start with?
log_directory = "M:\PROCESS-LOGS\Logs"  # Which directory do you want the log in?

# Set whether you are using nested folders or have all albums in one directory here
# If you have all your ablums in one music directory Music/Album_name then set this value to 1
# If you have all your albums nest in a Music/Artist/Album style of pattern set this value to 2
# The default is 1
album_depth = 1

# Set whether you are retagging albums that are various artist, dj, or normal
# Setting this to 1 will only set the Album Artist to the artist name which will be Various Artists
# Setting this to 2 will set the Album Artist tag to the DJ name
# Setting this to 3 will set both Album Artist and Artist tag to the artists name
# 1 = VA
# 2 = DJ
# 3 = Normal
# BE CAREFULL you could accidentally overwrite metadata that will be hard to get back if you have it set to 3
# The default is 1
album_type = 3

# Establishes the counters for completed albums and missing origin files
count = 0
total_count = 0
error_message = 0
good_missing = 0
bad_missing = 0
parse_error = 0
origin_old = 0
copy_cover = 0
disc_number_count = 0
missing_cover = 0

# identifies album directory level
path_segments = album_directory.split(os.sep)
segments = len(path_segments)
album_location_check = segments + album_depth

# A function to log events
def log_outcomes(directory, log_name, message, log_list):
    global log_directory

    script_name = "Origin Write Tags Script"
    today = datetime.datetime.now()
    log_name = f"{log_name}.txt"
    album_name = directory.split(os.sep)
    album_name = album_name[-1]
    log_path = os.path.join(log_directory, log_name)
    with open(log_path, "a", encoding="utf-8") as log_name:
        log_name.write(f"--{today:%b, %d %Y} at {today:%H:%M:%S} from the {script_name}.\n")
        log_name.write(f"The album folder {album_name} {message}.\n")
        if log_list != None:
            log_name.write("\n".join(map(str, log_list)))
            log_name.write("\n")
        log_name.write(f"Album location: {directory}\n")
        log_name.write(" \n")
        log_name.close()

# A function that determines if there is an error
def error_exists(error_type):
    global error_message

    if error_type >= 1:
        error_message += 1  # variable will increment if statement is true
        return "Warning"
    else:
        return "Info"

# A function that writes a summary of what the script did at the end of the process
def summary_text():
    global count
    global total_count
    global error_message
    global parse_error
    global bad_missing
    global good_missing
    global origin_old
    global disc_number_count
    global copy_cover
    global missing_cover

    print("")
    print(f"This script wrote tags to {count} tracks from {total_count} albums.")
    if disc_number_count != 0:
        print(f"It added disc numbers to {disc_number_count} sub folders.")
    if copy_cover != 0:
        print(f"It copied cover art to {copy_cover} sub folders.")
    print("This script looks for potential missing files or errors. The following messages outline whether any were found.")

    error_status = error_exists(parse_error)
    print(f"--{error_status}: There were {parse_error} albums skipped due to not being able to open the yaml. Redownload the yaml file.")
    error_status = error_exists(origin_old)
    print(f"--{error_status}: There were {origin_old} origin files that do not have the needed metadata and need to be updated.")
    error_status = error_exists(bad_missing)
    print(f"--{error_status}: There were {bad_missing} folders missing an origin files that should have had them.")
    error_status = error_exists(good_missing)
    print(f"--Info: Some folders didn't have origin files and probably shouldn't have origin files. {good_missing} of these folders were identified.")
    if missing_cover != 0:
        error_status = error_exists(missing_cover)
        print(f"--{error_status}: There were {missing_cover} folders missing cover art that should have had them.")

    if error_message >= 1:
        print("Check the logs to see which folders had errors and what they were and which tracks had metadata written to them.")
    else:
        print("There were no errors.")

# A function to check whether the directory is a an album or a sub-directory and returns an origin file location and album name
def level_check(directory):
    global total_count
    global album_location_check
    global album_directory

    print(f"--Directory: {directory}")
    print(f"--The albums are stored {album_location_check} folders deep.")

    path_segments = directory.split(os.sep)
    folder_name = path_segments[-1]
    directory_location = len(path_segments)

    print(f"--This folder is {directory_location} folders deep.")

    # Checks to see if a folder is an album or subdirectory by looking at how many segments are in a path and returns origin location and album name
    if album_location_check == directory_location and album_depth == 1:
        print("--This is an album.")
        origin_location = os.path.join(directory, "origin.yaml")
        album_name = path_segments[-1]
        total_count += 1  # variable will increment every loop iteration
        return origin_location, album_name, folder_name
    elif album_location_check == directory_location and album_depth == 2:
        print("--This is an album.")
        origin_location = os.path.join(directory, "origin.yaml")
        album_name = os.path.join(path_segments[-2], path_segments[-1])
        total_count += 1  # variable will increment every loop iteration
        return origin_location, album_name, folder_name
    elif album_location_check < directory_location and album_depth == 1:
        print("--This is a sub-directory")
        origin_location = os.path.join(album_directory, path_segments[-2], "origin.yaml")
        album_name = os.path.join(path_segments[-2], path_segments[-1])
        return origin_location, album_name, folder_name
    elif album_location_check < directory_location and album_depth == 2:
        print("--This is a sub-directory")
        origin_location = os.path.join(album_directory, path_segments[-3], path_segments[-2], "origin.yaml")
        album_name = os.path.join(path_segments[-3], path_segments[-2], path_segments[-1])
        return origin_location, album_name, folder_name
    elif album_location_check > directory_location and album_depth == 2:
        print("--This is an artist folder.")
        origin_location = None
        album_name = None
        return origin_location, album_name, folder_name

# Rethink this so it checks all files and if any end in flac go forth
# A function to check whether a directory has flac and should be checked further
def flac_check(directory):

    # Loop through the directory and see if any file is a flac
    for fname in os.listdir(directory):
        if fname.lower().endswith(".flac"):
            print("--There are flac in this directory.")
            return True

    print("--There are no flac in this directory.")
    return False

# A function to check if the origin file is there and to determine whether it is supposed to be there.
def check_file(directory):
    global good_missing
    global bad_missing
    global album_location_check

    # check to see if there is an origin file
    file_exists = os.path.exists("origin.yaml")
    # if origin file exists, load it, copy, and rename
    if file_exists == True:
        return True
    else:
        # split the directory to make sure that it distinguishes between folders that should and shouldn't have origin files
        current_path_segments = directory.split(os.sep)
        current_segments = len(current_path_segments)
        # create different log files depending on whether the origin file is missing somewhere it shouldn't be
        if album_location_check != current_segments:
            # log the missing origin file folders that are likely supposed to be missing
            print("--An origin file is missing from a folder that should not have one.")
            print("--Logged missing origin file.")
            log_name = "good-missing-origin"
            log_message = "origin file is missing from a folder that should not have one.\nSince it shouldn't be there it is probably fine but you can double check"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            good_missing += 1  # variable will increment every loop iteration
            return False
        else:
            # log the missing origin file folders that are not likely supposed to be missing
            print("--An origin file is missing from a folder that should have one.")
            print("--Logged missing origin file.")
            log_name = "bad-missing-origin"
            log_message = "origin file is missing from a folder that should have one"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            bad_missing += 1  # variable will increment every loop iteration
            return False

#  A function that gets the directory and then opens the origin file and extracts the needed variables
def get_metadata(directory, origin_location, album_name):
    global count
    global parse_error
    global origin_old
    global bad_missing

    print(f"--Getting metadata for {album_name}")
    print(f"--From: {origin_location}")

    # check to see if there is an origin file is supposed to be in this specific directory
    file_exists = check_file(directory)
    # check to see the origin file location variable exists
    location_exists = os.path.exists(origin_location)

    if location_exists == True:
        print("--The origin file location is valid.")
        # open the yaml
        try:
            with open(origin_location, encoding="utf-8") as f:
                data = yaml.load(f, Loader=yaml.FullLoader)
        except:
            print("--There was an issue parsing the yaml file and the cover could not be downloaded.")
            print("--Logged missing cover due to parse error. Redownload origin file.")
            log_name = "parse-error"
            log_message = "had an error parsing the yaml and the cover art could not be downloaded. Redownload the origin file"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            parse_error += 1  # variable will increment every loop iteration
            return
        # check to see if the origin file has the corect metadata
        if "Cover" in data.keys():
            print("--You are using the correct version of gazelle-origin.")

            # turn the data into variables
            origin_metadata = {
                "artist_name": data["Artist"],
                "album_name": data["Name"],
                "release_type": data["Release type"],
                "edition": data["Edition"],
                "edition_label": data["Record label"],
                "edition_cat": data["Catalog number"],
                "edition_year": data["Edition year"],
                "djs": data["DJs"],
                "composers": data["Composers"],
                "conductors": data["Conductors"],
                "original_year": data["Original year"],
                "media": data["Media"],
                "dl_directory": data["Directory"],
            }
            f.close()
            return origin_metadata
        else:
            print("--You need to update your origin files with more metadata.")
            print("--Switch to the gazelle-origin fork here: https://github.com/spinfast319/gazelle-origin")
            print("--Then run: https://github.com/spinfast319/Update-Gazelle-Origin-Files")
            print("--Then try this script again.")
            print("--Logged out of date origin file.")
            log_name = "out-of-date-origin"
            log_message = "origin file out of date"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            origin_old += 1  # variable will increment every loop iteration
    else:
        # log the missing origin file folders that are not likely supposed to be missing
        print("--An origin file is missing from a folder that should have one.")
        print("--Logged missing origin file.")
        log_name = "bad-missing-origin"
        log_message = "origin file is missing from a folder that should have one"
        log_list = None
        log_outcomes(directory, log_name, log_message, log_list)
        bad_missing += 1  # variable will increment every loop iteration

def write_tags(directory, origin_metadata, album_name):
    global count
    global album_type

    print("--Retagging files.")
    # Clear the list so the log captures just this albums tracks
    retag_list = []

    if origin_metadata != None:
        # Loop through the directory and rename flac files
        for fname in os.listdir(directory):
            if fname.lower().endswith(".flac"):
                tag_metadata = mutagen.File(fname)
                print(f"--Track Name: {fname}")
                # log track that was retagged
                retag_list.append(f"--Track Name: {fname}")
                #  retag the metadata
                if origin_metadata["artist_name"] != None and album_type == 1:
                    tag_metadata["ALBUM ARTIST"] = origin_metadata["artist_name"]
                    tag_metadata["ALBUMARTIST"] = origin_metadata["artist_name"]
                if origin_metadata["djs"] != None and album_type == 2:
                    tag_metadata["ALBUM ARTIST"] = origin_metadata["djs"]
                    tag_metadata["ALBUMARTIST"] = origin_metadata["djs"]
                if origin_metadata["artist_name"] != None and album_type == 3:
                    tag_metadata["ALBUM ARTIST"] = origin_metadata["artist_name"]
                    tag_metadata["ALBUMARTIST"] = origin_metadata["artist_name"]
                    tag_metadata["ARTIST"] = origin_metadata["artist_name"]
                if origin_metadata["album_name"] != None:
                    tag_metadata["ALBUM"] = origin_metadata["album_name"]
                if origin_metadata["release_type"] != None:
                    tag_metadata["GROUPING"] = origin_metadata["release_type"]
                    tag_metadata["RELEASETYPE"] = origin_metadata["release_type"]
                if origin_metadata["edition_label"] != None:
                    tag_metadata["ORGANIZATION"] = origin_metadata["edition_label"]
                    tag_metadata["LABEL"] = origin_metadata["edition_label"]
                if origin_metadata["edition_cat"] != None:
                    tag_metadata["LABELNO"] = origin_metadata["edition_cat"]
                    tag_metadata["CATALOGNUMBER"] = origin_metadata["edition_cat"]
                if origin_metadata["media"] != None:
                    tag_metadata["MEDIA"] = str(origin_metadata["media"])
                if origin_metadata["original_year"] != None:
                    tag_metadata["ORIGINALDATE"] = str(origin_metadata["original_year"])
                    tag_metadata["ORIGINALYEAR"] = str(origin_metadata["original_year"])
                    tag_metadata["YEAR"] = str(origin_metadata["original_year"])
                if origin_metadata["edition_year"] != None:
                    tag_metadata["DATE"] = str(origin_metadata["edition_year"])
                elif origin_metadata["edition_year"] == None and origin_metadata["original_year"] != None:
                    tag_metadata["DATE"] = str(origin_metadata["original_year"])
                tag_metadata.save()
                count += 1  # variable will increment every loop iteration
    else:
        print(f"Origin metadata unexpectedly missing.")

    # figure out how many tracks were renamed
    tracks_retagged = len(retag_list)
    if tracks_retagged != 0:
        print(f"--Tracks Retagged: {tracks_retagged}")
    else:
        print(f"--There were no flac in this folder.")
    # log the files that were retagged with metadata
    log_name = "files_retagged"
    log_message = f"had {tracks_retagged} files retagged"
    log_list = retag_list
    log_outcomes(directory, log_name, log_message, log_list)

#  This function adds the disc number tag based off of the number of the folder for multidisc albums using the format CD1, CD2, etc
def add_disc_number(directory, folder_name, album_name):
    global disc_number_count

    print("Adding Disc Number")

    # Clear the list so the log captures just this albums tracks
    adddisc_list = []

    path_segments = folder_name.split()
    disc = path_segments[0]
    disc_number = disc[2:]
    print(f"--The disc number is: {disc_number}")
    if disc_number != None:
        for fname in os.listdir(directory):
            if fname.lower().endswith(".flac"):
                tag_metadata = mutagen.File(fname)
                print(f"--Track Name: {fname}")
                # log track that was retagged
                adddisc_list.append(f"--Track Name: {fname}")
                tag_metadata["DISCNUMBER"] = disc_number
                tag_metadata.save()

    # figure out how many tracks were given disc numbers
    tracks_retagged = len(adddisc_list)
    if tracks_retagged != 0:
        print(f"--Tracks Retagged: {tracks_retagged}")
        disc_number_count += 1  # variable will increment every loop iteration
    else:
        print(f"--There were no flac in this folder.")
    # log the files that were retagged with disc numbers
    log_name = "files_retagged"
    log_message = f"had {tracks_retagged} files retagged with disc number"
    log_list = adddisc_list
    log_outcomes(directory, log_name, log_message, log_list)

# Finds the path of an image with the name 'cover' and an extension of .jpg, .jpeg, .gif, or .png in the given directory.
def check_cover(directory):

    # Check if directory exists
    if not os.path.isdir(directory):
        return None

    # Look for files with the name 'cover' and an appropriate extension
    for filename in os.listdir(directory):
        name, ext = os.path.splitext(filename)
        if name.lower() == "cover" and ext.lower() in (".jpg", ".jpeg", ".gif", ".png"):
            return os.path.join(directory, filename)

    # If no appropriate file is found, return None
    return None

#  This function adds the disc number tag based off of the number of the folder for multidisc albums using the format CD1, CD2, etc
#  This function prioritises cover.jpg over any other cover files in the folders
def copy_cover_art(directory, album_name):
    global copy_cover
    global missing_cover

    # Check to see if cover art exists in the sub directory
    sub_folder_cover = check_cover(directory)

    # If no cover is present copy the cover art from the folder above to the folder
    if sub_folder_cover == None:
        print("--This subfolder is missing a cover.")

        # Get the path of the source cover art file in the album folder.
        directory_above = os.path.dirname(directory)
        # Try cover.jpg first, if it doesn't exist get the cover art path
        guess_cover = os.path.join(directory_above, "cover.jpg")
        if os.path.exists(guess_cover):
            source_cover_path = guess_cover
        else:
            source_cover_path = check_cover(directory_above)

        # If cover art exists in the album folder copy it to the subfolder
        if source_cover_path:
            #  Build destination directory
            name, ext = os.path.splitext(source_cover_path)
            if ext.lower() == ".jpeg":
                ext = ".jpg"
            destination_cover_art = directory + os.sep + "cover" + ext.lower()

            # Copy cover art
            print("--Copying album cover art to subfolder.")
            shutil.copy(source_cover_path, destination_cover_art)
            print(f"----Source: {source_cover_path}")
            print(f"----Destination: {destination_cover_art}")
            copy_cover += 1  # variable will increment every loop iteration
            # log the sub folders that had cover art copied to them
            log_name = "files_retagged"
            log_message = f"had cover art copied to it"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
        else:
            print("--The cover art is missing from the album folder.")
            missing_cover += 1  # variable will increment every loop iteration
            # log missing cover art
            log_name = "missing_cover_art"
            log_message = f"was missing cover art"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
    else:
        print("--There is a cover in this subfolder.")

# The main function that controls the flow of the script
def main():

    try:
        # intro text
        print("")
        print("I will shred this universe down to its last atom and then, with the stones you've collected for me, create a new one...")
        print("")

        # Get all the subdirectories of album_directory recursively and store them in a list
        directories = osl.set_directory(album_directory)

        #  Run a loop that goes into each directory identified in the list and runs the function that sorts the folders
        for i in directories:
            os.chdir(i)  # Change working Directory
            print("")
            print("Retagging starting.")
            # establish directory level
            origin_location, album_name, folder_name = level_check(i)
            # check for flac
            is_flac = flac_check(i)
            # check for meta data and sort
            if is_flac == True:
                origin_metadata = get_metadata(i, origin_location, album_name)  # get metadata
                write_tags(i, origin_metadata, album_name)  # write metadata to flac
                if folder_name.startswith("CD"):
                    add_disc_number(i, folder_name, album_name)  # write disc number to flac
                    copy_cover_art(i, album_name)  # copy cover art to CD folders if missing
                print("Retagging complete.")
            else:
                print("No retagging.")

    finally:
        # Summary text
        print("")
        print("...It is not what is lost but only what it is been given... a grateful universal.")
        # run summary text function to provide error messages
        summary_text()
        print("")

if __name__ == "__main__":
    main()


from PIL import Image
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
import numpy as np

# Load the digits dataset
digits = load_digits()

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, test_size=0.2)

# Train an SVM classifier on the training data
clf = SVC()
clf.fit(X_train, y_train)

# Load the image file
img = Image.open("C:/Users/elvis/OneDrive/Desktop/Handwriting/images/handwritten_digit.png").convert("L")

# Resize the image to 8x8 pixels
img = img.resize((8, 8))

# Convert the image to a numpy array
image = np.array(img)

# Flatten the image into a 64-element array
image = image.reshape((64,))

# Predict the digit using the classifier
prediction = clf.predict([image])[0]

# Print the predicted digit
print("Predicted digit:", prediction)

# Test the classifier on the testing data and print the accuracy
accuracy = clf.score(X_test, y_test)
print("Accuracy:", accuracy)


import click
from cli import cli
from cli.deploy.actions import actions

help_txt = """
Usage: project deploy [OPTIONS] SUBCOMMAND

Options:
  --help Show this message and exit.

Subcommands:
  frontend   Build and deploy frontend assets
  models     Deploy built models
  services   Deploy backend services
  database   Deploy database migrations and changes
"""

@cli.command()
@click.argument('subcommand')
def deploy(subcommand):
    """
    Deployment options
    """
    sc = subcommand.lower()

    if sc in actions: actions[sc]()
    else: print(help_txt)


import difflib
import json
import os
import re
from pathlib import Path
from unittest.mock import Mock, patch

import pytest
import yaml  # type: ignore

from skops.card import parse_modelcard
from skops.card._parser import PandocParser, check_pandoc_installed

try:
    check_pandoc_installed()
except FileNotFoundError:
    # not installed, skip
    pytest.skip(reason="These tests require a recent pandoc", allow_module_level=True)

EXAMPLE_CARDS = [
    # actual model cards from HF hub
    "bert-base-uncased.md",
    "clip-vit-large-patch14.md",
    "gpt2.md",
    "specter.md",
    "vit-base-patch32-224-in21k.md",
    # not a model card
    "toy-example.md",
]

def _assert_meta_equal(meta0, meta1):
    # we cannot guarantee the order of metadata items, so we compare parsed
    # dicts, but not strings directly
    assert yaml.safe_load("".join(meta0)) == yaml.safe_load("".join(meta1))

def assert_readme_files_almost_equal(file0, file1, diff):
    """Check that the two model cards are identical, but allow differences as
    defined in the ``diff`` file

    The metainfo is compared separately, as the order of the items is not
    guaranteed to be stable.
    """
    with open(file0, "r") as f:
        readme0 = f.readlines()

    with open(file1, "r") as f:
        readme1 = f.readlines()

    sep = "---\n"
    # we look for 2nd occurrence, so skip first char to not match 1st occurrence
    if sep in readme0[1:]:  # only check if metainfo is present
        idx0, idx1 = readme0[1:].index(sep) + 1, readme1[1:].index(sep) + 1
        meta0, meta1 = readme0[1:idx0], readme1[1:idx1]
        readme0, readme1 = readme0[idx0:], readme1[idx1:]
        _assert_meta_equal(meta0, meta1)

    # exclude trivial case of both being empty
    assert readme0
    assert readme1

    diff_actual = list(difflib.unified_diff(readme0, readme1, n=0))

    with open(diff, "r") as f:
        diff_expected = f.readlines()

    assert diff_actual == diff_expected

@pytest.mark.parametrize("file_name", EXAMPLE_CARDS, ids=EXAMPLE_CARDS)
def test_example_model_cards(tmp_path, file_name):
    """Test that the difference between original and parsed model card is
    acceptable

    For this test, model cards for some of the most popular models on HF Hub
    were retrieved and stored in the ./examples folder. This test checks that
    these model cards can be successfully parsed and that the output is *almost*
    the same.

    We don't expect the output to be 100% identical, see the limitations listed
    in ``parse_modelcard``. Instead, we assert that the diff corresponds to the
    expected diff, which is also checked in.

    So e.g. for "specter.md", we expect that the diff will be the same diff as
    in "specter.md.diff".

    """
    path = Path(os.getcwd()) / "skops" / "card" / "tests" / "examples"
    file0 = path / file_name
    diff = (path / file_name).with_suffix(".md.diff")
    parsed_card = parse_modelcard(file0)
    file1 = tmp_path / "readme-parsed.md"
    parsed_card.save(file1)

    assert_readme_files_almost_equal(file0, file1, diff)

def test_unknown_pandoc_item_raises():
    source = json.dumps(
        {
            "pandoc-api-version": [1, 22, 2, 1],
            "meta": {},
            "blocks": [
                {
                    "t": "Header",
                    "c": [1, ["section", [], []], [{"t": "Str", "c": "section"}]],
                },
                {"c": "valid", "t": "Str"},
                {"t": "does-not-exist", "c": []},
                {"c": "okay", "t": "Str"},
            ],
        }
    )
    parser = PandocParser(source)
    msg = (
        "The parsed document contains 'does-not-exist', which is not "
        "supported yet, please open an issue on GitHub"
    )
    with pytest.raises(ValueError, match=re.escape(msg)):
        parser.generate()

def test_content_without_section_raises():
    source = json.dumps(
        {
            "pandoc-api-version": [1, 22, 2, 1],
            "meta": {},
            "blocks": [
                {"c": "whoops", "t": "Str"},
            ],
        }
    )
    parser = PandocParser(source)
    msg = (
        "Trying to add content but there is no current section, this is probably a "
        "bug, please open an issue on GitHub"
    )
    with pytest.raises(ValueError, match=re.escape(msg)):
        parser.generate()

def test_unsupported_markup_raises():
    match = re.escape("Markup of type does-not-exist is not supported (yet)")
    with pytest.raises(ValueError, match=match):
        PandocParser(source="", markup_type="does-not-exist")

def test_check_pandoc_installed_no_min_version_works():
    # check that it doesn't raise
    check_pandoc_installed(min_version=None)

def test_check_pandoc_installed_min_version_too_high_raises():
    match = re.escape("Pandoc version too low, expected at least 999.9.9, got")
    with pytest.raises(ValueError, match=match):
        check_pandoc_installed(min_version="999.9.9")

def test_pandoc_not_installed():
    def raise_filenotfound(*args, **kwargs):
        # error raised when trying to run subprocess on non-existing command
        raise FileNotFoundError("[Errno 2] No such file or directory: 'pandoc'")

    with patch("subprocess.run", raise_filenotfound):
        match = re.escape(
            "This feature requires the pandoc library to be installed on your system"
        )
        with pytest.raises(FileNotFoundError, match=match):
            check_pandoc_installed()

def test_pandoc_version_cannot_be_determined():
    mock = Mock()
    with patch("subprocess.run", mock):
        match = re.escape("Could not determine version of pandoc")
        with pytest.raises(RuntimeError, match=match):
            check_pandoc_installed()


import xml.sax
from xml.dom import minidom

def example3():
    'minidom.parse'
    domtree = minidom.parse("people.xml")
    root = domtree.documentElement

    persons = root.getElementsByTagName("Person")
    for person in persons:
        print("<<Person>>")
        if(person.hasAttribute("id")):
            print("id = ", person.getAttribute("id"))
        name = person.getElementsByTagName("Name")[0].childNodes[0].data
        age = person.getElementsByTagName("Age")[0].childNodes[0].data
        wage = person.getElementsByTagName("Wage")[0].childNodes[0].data
        active = person.getElementsByTagName("Active")[0].childNodes[0].data
        print("Name = ", name)
        print("Age = ", age)
        print("Wage = ", wage)
        print("Active = ", active)

        print()

    print()

def example2():
    'same as example1()'
    class PersonContentHandler(xml.sax.ContentHandler):
        def __init__(self):
            self.currentTag = None
            self.name = None
            self.age = None
            self.wage = None
            self.active = None

        def startElement(self, name, attrs):
            self.currentTag = name
            if(name == "Person"):
                print("\n*** Person***")
                id = attrs["id"]
                print("id = ", id)

        def endElement(self, name):
             if(self.currentTag == "Name"):
                 print("Name = ", self.name)
             elif(self.currentTag == "Age"):
                 print("Age = ", self.age)
             elif(self.currentTag == "Wage"):
                 print("Wage = ", self.wage)
             elif(self.currentTag == "Active"):
                 print("Active = ", self.active)

             self.currentTag = ""

        def characters(self, content):
             if(self.currentTag == "Name"):
                 self.name = content
             elif(self.currentTag == "Age"):
                 self.age = content
             elif(self.currentTag == "Wage"):
                 self.wage = content
             elif(self.currentTag == "Active"):
                 self.active = content

    parser = xml.sax.make_parser()
    parser.setFeature(xml.sax.handler.feature_namespaces, 0)
    parser.setContentHandler(PersonContentHandler())
    parser.parse("people.xml")

def example1():
    'xml sax'
    class CustomContenHandler(xml.sax.ContentHandler):
        def __init__(self):
            self.CurrentTag = ""
            self.name = ""
            self.age = ""
            self.wage = ""
            self.active = ""

        def startElement(self, name, attrs):
            self.CurrentTag = name
            if name == "Person":
                print("\n*** Person ***")
                id = attrs["id"]
                print("id = ", id)

        def characters(self, content):
            if(self.CurrentTag == "Name"):
                self.name = content
            if(self.CurrentTag == "Age"):
                self.age = content
            if(self.CurrentTag == "Wage"):
                self.wage = content
            if(self.CurrentTag == "Active"):
                self.active = content

        def endElement(self, name):
            if(self.CurrentTag == "Name"):
                print("Name = ", self.name)
            if(self.CurrentTag == "Age"):
                print("Age = ", self.age)
            if(self.CurrentTag == "Wage"):
                print("Wage = ", self.wage)
            if(self.CurrentTag == "Active"):
                print("Active = ", self.active)
            self.CurrentTag = ""

    parser = xml.sax.make_parser()
    parser.setFeature(xml.sax.handler.feature_namespaces, 0)
    parser.setContentHandler(CustomContenHandler())
    parser.parse("people.xml")


from itertools import chain
from itertools import combinations

import numpy as np
from scipy import linalg
from scipy.spatial.distance import squareform
from sklearn.cluster import AgglomerativeClustering
from sklearn.utils import Bunch

from regain.datasets.gaussian import make_ell
from regain.norm import l1_od_norm
from regain.utils import is_pos_def

def make_exp_sine_squared(n_dim_obs=5, n_dim_lat=0, T=1, **kwargs):
    """Make precision matrices using a temporal sine kernel."""
    from regain.bayesian.gaussian_process_ import sample as samplegp
    from sklearn.gaussian_process import kernels

    L, K_HO = make_ell(n_dim_obs, n_dim_lat)

    periodicity = kwargs.get("periodicity", np.pi)
    length_scale = kwargs.get("length_scale", 2)
    epsilon = kwargs.get("epsilon", 0.5)
    sparse = kwargs.get("sparse", True)
    temporal_kernel = kernels.ExpSineSquared(periodicity=periodicity, length_scale=length_scale)(np.arange(T)[:, None])

    u = samplegp(temporal_kernel, p=n_dim_obs * (n_dim_obs - 1) // 2)[0]
    K, K_obs = [], []
    for uu in u.T:
        theta = squareform(uu)

        if sparse:
            # sparsify
            theta[np.abs(theta) < epsilon] = 0

        theta += np.diag(np.sum(np.abs(theta), axis=1) + 0.01)
        K.append(theta)

        assert is_pos_def(theta)
        theta_observed = theta - L
        assert is_pos_def(theta_observed)
        K_obs.append(theta_observed)

    thetas = np.array(K)
    return thetas, np.array(K_obs), np.array([L] * T)

def make_RBF(n_dim_obs=5, n_dim_lat=0, T=1, **kwargs):
    """Make precision matrices using a temporal RBF kernel."""
    from regain.bayesian.gaussian_process_ import sample as samplegp
    from sklearn.gaussian_process import kernels

    length_scale = kwargs.get("length_scale", 1.0)
    length_scale_bounds = kwargs.get("length_scale_bounds", (1e-05, 100000.0))
    epsilon = kwargs.get("epsilon", 0.8)
    sparse = kwargs.get("sparse", True)
    temporal_kernel = kernels.RBF(length_scale=length_scale, length_scale_bounds=length_scale_bounds)(
        np.arange(T)[:, None]
    )

    n = n_dim_obs + n_dim_lat
    u = samplegp(temporal_kernel, p=n * (n - 1) // 2)[0]
    K = []
    for i, uu in enumerate(u.T):
        theta = squareform(uu)
        if sparse:
            theta_obs = theta[n_dim_lat:, n_dim_lat:]
            theta_lat = theta[:n_dim_lat, :n_dim_lat]
            theta_OH = theta[n_dim_lat:, :n_dim_lat]

            # sparsify
            theta_obs[np.abs(theta_obs) < epsilon] = 0
            theta_lat[np.abs(theta_lat) < epsilon / 3] = 0
            theta_OH[np.abs(theta_OH) < epsilon / 3] = 0
            theta[n_dim_lat:, n_dim_lat:] = theta_obs
            theta[:n_dim_lat, :n_dim_lat] = theta_lat
            theta[n_dim_lat:, :n_dim_lat] = theta_OH
            theta[:n_dim_lat, n_dim_lat:] = theta_OH.T
        if i == 0:
            inter_links = theta[n_dim_lat:, :n_dim_lat]
        theta[n_dim_lat:, :n_dim_lat] = inter_links
        theta[:n_dim_lat, n_dim_lat:] = inter_links.T
        theta += np.diag(np.sum(np.abs(theta), axis=1) + 0.01)
        K.append(theta)

        assert is_pos_def(theta)

    thetas = np.array(K)

    theta_obs = []
    ells = []
    for t in thetas:
        L = (
            theta[n_dim_lat:, :n_dim_lat]
            .dot(linalg.pinv(t[:n_dim_lat, :n_dim_lat]))
            .dot(theta[:n_dim_lat, n_dim_lat:])
        )
        theta_obs.append(t[n_dim_lat:, n_dim_lat:] - L)
        ells.append(L)
    return thetas, theta_obs, ells

def genInvCov(size, low=0.3, upper=0.6, portion=0.2, symmetric=True):
    """Generate sparse inverse covariance matrix."""
    import networkx as nx

    portion = portion / 2
    S = np.zeros((size, size))
    n_edges = int((size * (size - 1)) * portion)
    G = nx.gnm_random_graph(size, n_edges)
    for src, dest in G.edges:
        S[src, dest] = (np.random.randint(2) - 0.5) * 2 * (low + (upper - low) * np.random.rand(1)[0])
    if symmetric:
        S += S.T
    # vals = alg.eigvalsh(S)
    # S = S + (0.1 - vals[0])*np.identity(size)
    return S

def genRandInv(size, low=0.3, upper=0.6, portion=0.2):
    """Generate inverse covariance matrix."""
    S = np.zeros((size, size))
    for i in range(size):
        for j in range(size):
            if np.random.rand() < portion:
                value = (np.random.randint(2) - 0.5) * 2 * (low + (upper - low) * np.random.rand(1)[0])
                S[i, j] = value
    return S

def make_ticc(num_blocks=5, n_dim_obs=5, n_dim_lat=0, sparsity_inv_matrix=0.5, rand_seed=None, **kwargs):
    if n_dim_lat != 0:
        raise ValueError("not supported")

    np.random.seed(rand_seed)
    size_blocks = n_dim_obs
    block_matrices = {}

    # Generate all the blocks
    for block in range(num_blocks):
        if block == 0:
            block_matrices[block] = genInvCov(
                size=size_blocks,
                portion=sparsity_inv_matrix,
                symmetric=(block == 0),
            )
        else:
            block_matrices[block] = genRandInv(size=size_blocks, portion=sparsity_inv_matrix)

    # Initialize the inverse matrix
    inv_matrix = np.zeros([num_blocks * size_blocks, num_blocks * size_blocks])

    # go through all the blocks
    for block_i in range(num_blocks):
        for block_j in range(num_blocks):
            block_num = np.abs(block_i - block_j)
            inv_matrix[
                block_i * size_blocks : (block_i + 1) * size_blocks,
                block_j * size_blocks : (block_j + 1) * size_blocks,
            ] = (
                block_matrices[block_num] if block_i > block_j else np.transpose(block_matrices[block_num])
            )

    # print out all the eigenvalues
    eigs, _ = np.linalg.eig(inv_matrix)
    lambda_min = min(eigs)

    # Make the matrix positive definite
    inv_matrix += (0.1 + abs(lambda_min)) * np.eye(size_blocks * num_blocks)

    eigs, _ = np.linalg.eig(inv_matrix)
    lambda_min = min(eigs)
    # print("Modified Eigenvalues are:", np.sort(eigs))

    return inv_matrix

def _block_matrix(matrix, r, c):
    S11 = matrix[:r, :c]
    S22 = matrix[r:, c:]
    S21 = matrix[r:, :c]
    S12 = S21.T
    return S11, S22, S21, S12

def make_ticc_dataset(
    clusters=(0, 1, 0),
    n_dim=3,
    w_size=5,
    break_points=None,
    n_samples=200,
    n_dim_lat=0,
    sparsity_inv_matrix=0.5,
    T=9,
    rand_seed=None,
    **kwargs
):
    """Generate data as the TICC method.

    Library implementation of `generate_synthetic_data.py`, original can be
    found at https://github.com/davidhallac/TICC
    """
    if (len(clusters) * n_samples) % T != 0:
        raise ValueError(
            "n_clusters * n_samples should be a multiple of n_times "
            "to avoid having samples in the same time period in different "
            "clusters"
        )

    id_cluster = np.repeat(np.asarray(list(clusters)), n_samples)
    y = np.repeat(np.arange(T), len(clusters) * n_samples // T)

    cluster_mean = np.zeros(n_dim)
    cluster_mean_stack = np.zeros(n_dim * w_size)

    clusters = np.unique(list(clusters))
    # Generate two inverse matrices
    precisions = {}
    covs = {}
    for i, cluster in enumerate(clusters):
        precisions[cluster] = make_ticc(
            rand_seed=i,
            num_blocks=w_size,
            n_dim_obs=n_dim,
            n_dim_lat=n_dim_lat,
            sparsity_inv_matrix=sparsity_inv_matrix,
            **kwargs
        )
        covs[cluster] = linalg.pinvh(precisions[cluster])

    # Data matrix
    X = np.empty((id_cluster.size, n_dim))
    precs = []
    n = n_dim
    for i, label in enumerate(id_cluster):
        # for num in range(old_break_pt, break_pt):
        if i == 0:
            # conditional covariance and mean
            cov_tom = covs[label][:n_dim, :n_dim]
            mean = cluster_mean_stack[n_dim * (w_size - 1) :]

        elif i < w_size:
            cov = covs[label][: (i + 1) * n, : (i + 1) * n]
            Sig11, Sig22, Sig21, Sig12 = _block_matrix(cov, i * n, i * n)
            Sig21Theta11 = Sig21.dot(linalg.pinvh(Sig11))
            cov_tom = Sig22 - Sig21Theta11.dot(Sig12)  # sigma2|1

            mean = cluster_mean + Sig21Theta11.dot(X[:i].flatten() - cluster_mean_stack[: i * n_dim])

        else:
            cov = covs[label][: w_size * n, : w_size * n]
            Sig11, Sig22, Sig21, Sig12 = _block_matrix(cov, (w_size - 1) * n, (w_size - 1) * n)
            Sig21Theta11 = Sig21.dot(linalg.pinvh(Sig11))
            cov_tom = Sig22 - Sig21Theta11.dot(Sig12)  # sigma2|1

            mean = cluster_mean + Sig21Theta11.dot(
                X[i - w_size + 1 : i].flatten() - cluster_mean_stack[: (w_size - 1) * n_dim]
            )

        X[i] = np.random.multivariate_normal(mean, cov_tom)
        precs.append(linalg.pinvh(cov_tom))

    id_cluster_group = []
    for c in np.unique(y):
        idx = np.where(y == c)[0]
        # check samples at same time belong to a single cluster
        assert np.unique(id_cluster[idx]).size == 1
        id_cluster_group.append(id_cluster[idx][0])

    data = Bunch(
        X=X,
        y=y,
        id_cluster=id_cluster,
        covs=covs,
        precs=precs,
        id_cluster_group=np.asarray(id_cluster_group),
    )
    return data

def make_ticc_dataset_new(
    clusters=(0, 1, 0),
    n_dim=3,
    w_size=5,
    break_points=None,
    n_samples=200,
    n_dim_lat=0,
    sparsity_inv_matrix=0.5,
    T=9,
    rand_seed=None,
    **kwargs
):
    """Generate data as the TICC method.

    Library implementation of `generate_synthetic_data.py`, original can be
    found at https://github.com/davidhallac/TICC
    """
    # if (len(clusters) * n_samples) % T != 0:
    #     raise ValueError(
    #         'n_clusters * n_samples should be a multiple of n_times '
    #         'to avoid having samples in the same time period in different '
    #         'clusters')

    id_cluster = np.repeat(np.asarray(list(clusters)), n_samples)
    y = np.repeat(np.arange(T), len(clusters) * n_samples // T)

    size_blocks = n_dim
    num_blocks = n_samples
    bmc = {}

    for rand_seed in clusters:
        np.random.seed(rand_seed)

        bmc[rand_seed] = {
            block: (
                genInvCov(
                    size=size_blocks,
                    portion=sparsity_inv_matrix,
                    symmetric=True,
                )
                if block == 0
                else (genRandInv(size=size_blocks, portion=sparsity_inv_matrix))
                if block < w_size + 1
                else np.zeros((n_dim, n_dim))
            )
            for block in range(num_blocks)
        }

    def get_out_diag(r):
        return np.block(
            [
                [
                    (bmc[r].get(i, np.zeros((n_dim, n_dim))).T if col > j else bmc[r].get(i, np.zeros((n_dim, n_dim))))
                    for col, i in enumerate(range(j, num_blocks + j))
                ]
                for j in range(1, num_blocks + 1)
            ][::-1]
        )

    def get_out_zero(r):
        return np.zeros_like(get_out_diag(r))

    def get_diag(r):
        return np.block(
            [
                [
                    (bmc[r][i] if col > j else bmc[r][i].T)
                    for col, i in enumerate(chain(range(j, 0, -1), range(num_blocks - j)))
                ]
                for j in range(num_blocks)
            ]
        )

    inv = np.block(
        [
            [np.zeros_like(get_out_diag(c))] * max(0, k - 1)
            + ([get_out_diag(clusters[k - 1]).T] if k > 0 else [])
            + [get_diag(c)]
            + ([get_out_diag(c)] if k < len(clusters) - 1 else [])
            + [np.zeros_like(get_out_diag(c))] * max(len(clusters) - 2 - k, 0)
            for k, c in enumerate(clusters)
        ]
    )
    # Make the matrix positive definite
    eigs, _ = np.linalg.eig(inv)
    lambda_min = np.min(eigs)
    inv += (0.1 + abs(lambda_min)) * np.eye(inv.shape[0])
    C = linalg.pinvh(inv)

    # create data
    cluster_mean = np.zeros(n_dim)
    cluster_mean_stack = np.zeros((n_samples * len(clusters), n_dim))
    X = np.empty((id_cluster.size, n_dim))
    precs, sparse_precs = [], []
    for i, label in enumerate(id_cluster):
        cov = C[
            max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
            max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
        ]

        # conditional covariance and mean
        Sig11, Sig22, Sig21, Sig12 = _block_matrix(cov, min(i, w_size) * n_dim, min(i, w_size) * n_dim)
        T11, T22, T21, T12 = _block_matrix(
            inv[
                max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
                max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
            ],
            min(i, w_size) * n_dim,
            min(i, w_size) * n_dim,
        )
        # print(Sig11.shape, Sig22.shape, Sig21.shape, Sig12.shape)

        # when i = 0, Sig11 has shape (0,0), causing an error in pinvh
        Sig21Theta11 = Sig21.dot(linalg.pinvh(Sig11) if Sig11.size > 0 else Sig11)
        cov_tom = Sig22 - Sig21Theta11.dot(Sig12)  # sigma2|1

        mean = cluster_mean + Sig21Theta11.dot(
            X[max(i - w_size, 0) : i].flatten() - cluster_mean_stack[max(i - w_size, 0) : i].flatten()
        )

        X[i] = np.random.multivariate_normal(mean, cov_tom)
        precs.append(linalg.pinvh(cov_tom))
        sparse_precs.append(T22)

    id_cluster_group = []
    for c in np.unique(y):
        idx = np.where(y == c)[0]
        # check samples at same time belong to a single cluster
        assert np.unique(id_cluster[idx]).size == 1
        id_cluster_group.append(id_cluster[idx][0])

    data = Bunch(
        X=X,
        y=y,
        id_cluster=id_cluster,
        precs=precs,
        sparse_precs=sparse_precs,
        id_cluster_group=np.asarray(id_cluster_group),
        inv=inv,
    )
    return data

def make_ticc_dataset_v3(
    clusters=(0, 1, 0),
    n_dim=3,
    w_size=5,
    break_points=None,
    n_samples=200,
    n_dim_lat=0,
    sparsity_inv_matrix=0.5,
    T=9,
    rand_seed=None,
    **kwargs
):
    """Generate data as the TICC method.

    Library implementation of `generate_synthetic_data.py`, original can be
    found at https://github.com/davidhallac/TICC
    """
    # if (len(clusters) * n_samples) % T != 0:
    #     raise ValueError(
    #         'n_clusters * n_samples should be a multiple of n_times '
    #         'to avoid having samples in the same time period in different '
    #         'clusters')

    # id_cluster = np.repeat(np.asarray(list(clusters)), n_samples)
    # y = np.repeat(np.arange(T), len(clusters) * n_samples // T)

    size_blocks = n_dim
    num_blocks = 1
    bmc = {}

    for rand_seed in np.unique(clusters):
        np.random.seed(rand_seed)

        bmc[rand_seed] = {
            block: (
                genInvCov(
                    size=size_blocks,
                    portion=sparsity_inv_matrix,
                    symmetric=True,
                )
                if block == 0
                else (genRandInv(size=size_blocks, portion=sparsity_inv_matrix))
                if block < w_size + 1
                else np.zeros((n_dim, n_dim))
            )
            for block in range(num_blocks)
        }

    def get_out_diag(r):
        return np.block(
            [
                [
                    (bmc[r].get(i, np.zeros((n_dim, n_dim))).T if col > j else bmc[r].get(i, np.zeros((n_dim, n_dim))))
                    for col, i in enumerate(range(j, num_blocks + j))
                ]
                for j in range(1, num_blocks + 1)
            ][::-1]
        )

    def get_out_zero(r):
        return np.zeros_like(get_out_diag(r))

    def get_diag(r):
        return np.block(
            [
                [
                    (bmc[r][i] if col > j else bmc[r][i].T)
                    for col, i in enumerate(chain(range(j, 0, -1), range(num_blocks - j)))
                ]
                for j in range(num_blocks)
            ]
        )

    inv = np.block(
        [
            [np.zeros_like(get_out_diag(c))] * max(0, k - 1)
            + ([get_out_diag(clusters[k - 1]).T] if k > 0 else [])
            + [get_diag(c)]
            + ([get_out_diag(c)] if k < len(clusters) - 1 else [])
            + [np.zeros_like(get_out_diag(c))] * max(len(clusters) - 2 - k, 0)
            for k, c in enumerate(clusters)
        ]
    )
    # Make the matrix positive definite
    eigs, _ = np.linalg.eig(inv)
    lambda_min = np.min(eigs)
    inv += (0.1 + abs(lambda_min)) * np.eye(inv.shape[0])
    C = linalg.pinvh(inv)

    # create data
    cluster_mean = np.zeros(n_dim)
    cluster_mean_stack = np.zeros((n_samples * len(clusters), n_dim))
    X = np.empty((n_samples * len(clusters), n_dim))
    precs, sparse_precs = [], []
    for i, _ in enumerate(clusters):
        cov = C[
            max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
            max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
        ]

        # conditional covariance and mean
        Sig11, Sig22, Sig21, Sig12 = _block_matrix(cov, min(i, w_size) * n_dim, min(i, w_size) * n_dim)
        _, T22, _, _ = _block_matrix(
            inv[
                max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
                max(i - w_size, 0) * n_dim : (i + 1) * n_dim,
            ],
            min(i, w_size) * n_dim,
            min(i, w_size) * n_dim,
        )
        # print(Sig11.shape, Sig22.shape, Sig21.shape, Sig12.shape)

        # when i = 0, Sig11 has shape (0,0), causing an error in pinvh
        Sig21Theta11 = Sig21.dot(linalg.pinvh(Sig11) if Sig11.size > 0 else Sig11)
        cov_tom = Sig22 - Sig21Theta11.dot(Sig12)  # sigma2|1

        mean = cluster_mean + Sig21Theta11.dot(
            X[max(i - w_size, 0) : i].flatten() - cluster_mean_stack[max(i - w_size, 0) : i].flatten()
        )

        X[i * n_samples : (i + 1) * n_samples] = np.random.multivariate_normal(mean, cov_tom, size=n_samples)
        precs.append(linalg.pinvh(cov_tom))
        sparse_precs.append(T22)

    y = np.repeat(np.arange(len(clusters)), n_samples)
    id_cluster = np.repeat(clusters, n_samples)

    data = Bunch(
        X=X,
        y=y,
        id_cluster=id_cluster,
        precs=np.array(precs),
        sparse_precs=np.array(sparse_precs),
        inv=inv,
    )
    return data

def make_cluster_representative(
    n_dim=10,
    degree=2,
    n_clusters=3,
    T=15,
    n_samples=100,
    repetitions=False,
    cluster_series=None,
    shuffle=False,
):
    """Based on the cluster representative, generate similar graphs."""
    import networkx as nx

    cluster_reps = []
    adjacencies = []
    if cluster_series is not None:
        n_clusters = np.unique(cluster_series).size

    for i in range(n_clusters):
        representative = nx.random_regular_graph(d=degree, n=n_dim)
        A = nx.adjacency_matrix(representative).todense().astype(float)
        A[np.where(A != 0)] = np.random.rand(np.where(A != 0)[0].size) * 0.45
        np.fill_diagonal(A, 1)
        adjacencies.append(A)
        cluster_reps.append(representative)

    pos = np.arange(0, T, T // (n_clusters + 1))
    pos = list(pos) + [T - 1]

    if cluster_series is None:
        cluster_series = np.tile(range(n_clusters), (len(pos) // n_clusters) + 1)[: len(pos)]
        if shuffle:
            np.random.shuffle(cluster_series)
        # print(pos)
        # print(cluster_series)
        # pos = np.arange(0, T, T // (clusters + 1))
        # pos = list(pos) + [T - 1]
    else:
        assert len(cluster_series) == len(pos)
    #     a = np.where(cluster_series[:-1] != cluster_series[1:])[0] + 1
    #     T = len(cluster_series) # overwrites T
    #     pos = np.concatenate(([0], a, [T-1]))

    thetas = []
    for i in range(len(pos) - 1):
        # last one is always a representative
        how_many = int(pos[i + 1]) - int(pos[i]) - 1
        new_list = [adjacencies[cluster_series[i]]]
        target = adjacencies[cluster_series[i + 1]]

        for i in range(how_many):
            new = new_list[-1].copy()
            diffs = (new != 0).astype(int) - (target != 0).astype(int)
            diff = np.where(diffs != 0)
            if diff == ():
                break
            if i == 0:
                edges_per_change = int((np.nonzero(diffs)[0].shape[0] / 2) // (how_many + 1))
                if edges_per_change == 0:
                    edges_per_change += 1
            ixs = np.arange(diff[0].shape[0])
            np.random.shuffle(ixs)

            xs = diff[0][ixs[:edges_per_change]]
            ys = diff[1][ixs[:edges_per_change]]
            for j in range(xs.shape[0]):
                if diffs[xs[j], ys[j]] == -1:
                    new[xs[j], ys[j]] = np.random.rand(1) * 0.2
                    new[ys[j], xs[j]] = new[xs[j], ys[j]]
                else:
                    new[xs[j], ys[j]] = 0
                    new[ys[j], xs[j]] = 0
            new_list.append(new)

        thetas += new_list
    thetas.append(target)
    covs = [linalg.pinvh(t) for t in thetas]
    X = np.vstack([np.random.multivariate_normal(np.zeros(n_dim), c, size=n_samples) for c in covs])
    y = np.repeat(np.arange(len(covs)), n_samples)

    distances = squareform([l1_od_norm(t1 - t2) for t1, t2 in combinations(thetas, 2)])
    distances /= np.max(distances)
    labels_pred = AgglomerativeClustering(
        n_clusters=n_clusters, affinity="precomputed", linkage="complete"
    ).fit_predict(distances)

    id_cluster = np.repeat(labels_pred, n_samples)
    data = Bunch(
        X=X,
        y=y,
        id_cluster=id_cluster,
        precs=np.array(thetas),
        thetas=np.array(thetas),
        sparse_precs=np.array(thetas),
        cluster_reps=cluster_reps,
        cluster_series=cluster_series,
    )
    return data


from contextlib import contextmanager

greeting = 'Hello! '

@contextmanager
def many_greetings(msg, count):
    original_msg = msg
    global greeting
    greeting = msg * count
    yield greeting
    greeting = original_msg

print('before with: %s' % greeting)
with many_greetings(greeting, 3) as say:
    print('in with: %s' % say)

print('after with: %s' % greeting)


from skimage import io
import numpy as np
from Filter import Filter

'''We'll implement the avarege filter'''

def getAvaregeKernel(size):
    avarege_kernel = np.zeros((size, size))
    avarege_kernel[:] = 1.0 / (size**2)

    return avarege_kernel

img = io.imread('GUI.jpg')
img = img[:, :, 0]

avaregeKernel = getAvaregeKernel(3)
filter = Filter(img, avaregeKernel, 'Avarege Kernel')
filter.convolveFilter()
filter.plotResult()
filter.plotResult(absolute=True)


import crc16

class GPSSample:
    def __init__(self):
        self.id = None
        self.rssi = 0
        self.remote_sats = 0
        self.remote_pos = [0]*3
        self.remote_vel = [0]*3
        self.base_sats = 0
        self.base_pos = [0]*3
        self.__crc_processor = crc16.CRC16_CCITT_False()

    @property
    def valid(self):
        return self.id is not None

    @property
    def stream_format(self):
        if self.valid:
            self.__crc_processor.reset()
            text = '{:d},'.format(int(self.id))
            text += '{:d},'.format(int(self.rssi))

            # Remote GPS
            text += '{:d},'.format(int(self.remote_sats))
            text += '{:0.9f},{:0.9f},{:0.9f},'.format(*self.remote_pos)
            text += '{:0.9f},{:0.9f},{:0.9f},'.format(*self.remote_vel)

            # Base GPS
            text += '{:d},'.format(int(gauss(self.__base_sats, self.__sats_sd)))
            text += '{:0.9f},{:0.9f},{:0.9f},'.format(*self.remote_pos)

            crc_processor = crc16.CRC16_CCITT_False()
            crc = crc_processor(bytes(text, 'ascii'))
            text += '0x{:04X}\n'.format(crc)
        raise Exception('GPSSample was not valid when trying to create stream format')

    def __str__(self):
        text = 'GPS( {:d} dB'.format(self.rssi)
        text += ', remote sats = {:d}'.format(self.remote_sats)
        text += ', remote xyz = [{:0.3f}, {:0.3f}, {:0.3f}]'.format(*self.remote_pos)
        text += ', remote vel = [{:0.3f}, {:0.3f}, {:0.3f}]'.format(*self.remote_vel)
        text += ', base sats = {:d}'.format(self.base_sats)
        text += ', base xyz = [{:0.3f}, {:0.3f}, {:0.3f}] )'.format(*self.remote_pos)
        return text

class GPSParser:
    class ParserStatistics:
        def __init__(self, debug=False):
            self.reset()
            self.__debug = debug

        def __str__(self):
            return 'Parser Statistics: {} samples, {} segments, {} errors'.format(self.samples, self.segments, self.errors)

        def reset(self):
            self.samples = 0
            self.segments = 0
            self.errors = 0

    def __init__(self, debug=True):
        self.__buffer = ''
        self.__last_gps_sample = GPSSample()
        self.__crc_processor = crc16.CRC16_CCITT_False()
        self.__parser_statistics = GPSParser.ParserStatistics()
        self.__debug = debug

    @property
    def statistics(self):
        return self.__parser_statistics

    def __call__(self, data):
        self.__buffer += data

        done = False
        results = []
        while not done:
            idx = self.__buffer.find('\n')
            if idx < 0:
                done = True
            else:
                segment = self.__buffer[:idx]
                self.__buffer = self.__buffer[idx+1:]  # Copy remainder except for the '\n'
                self.__parser_statistics.segments += 1
                crc_valid = False
                crc_idx = segment.rfind(',')
                if not crc_idx == -1:
                    try:
                        self.__crc_processor.reset()
                        calculated_crc = self.__crc_processor(bytes(segment[:crc_idx+1], 'ascii'))
                        read_crc = int(segment[crc_idx+1:], 16)
                        crc_valid = (calculated_crc == read_crc)
                    except Exception as e:
                        if self.__debug:
                            print('Character Encoding Error')
                if crc_valid:
                    tokens = [t.strip() for t in segment.split(',')]
                    gps_sample = GPSSample()
                    try:
                        gps_sample.id = int(tokens[0])
                        if self.__last_gps_sample.id is None:
                            self.__last_gps_sample.id = gps_sample.id
                        elif gps_sample.id == self.__last_gps_sample.id:
                            if self.__debug:
                                print('Repeated Sample')
                            break
                        elif gps_sample.id > (self.__last_gps_sample.id+1):
                            offset = (gps_sample.id - self.__last_gps_sample.id)
                            if self.__debug:
                                print('Missed {} Frames'.format(offset))
                            self.__parser_statistics.errors += offset

                        gps_sample.rssi = int(tokens[1])
                        gps_sample.remote_sats = int(tokens[2])
                        gps_sample.remote_pos = [float(v) for v in tokens[3:6]]
                        gps_sample.remote_vel = [float(v) for v in tokens[6:9]]
                        gps_sample.base_sats = int(tokens[9])
                        gps_sample.base_pos = [float(v) for v in tokens[10:13]]

                        self.__parser_statistics.samples += 1
                        results.append(gps_sample)
                        self.__last_gps_sample = gps_sample
                    except Exception as e:
                        if self.__debug:
                            print('Parsing Error - {}'.format(e))
                        self.__parser_statistics.errors += 1
                else:
                    if self.__debug:
                        print('CRC Invalid')
                    self.__parser_statistics.errors += 1
        return results


'''
973. K Closest Points to Origin
https://leetcode.com/problems/k-closest-points-to-origin/
'''
from heapq import heappop, heappush, heappushpop
from typing import List

from test_tool import assert_value

class Solution:
    def kClosest(self, points: List[List[int]], k: int) -> List[List[int]]:
        res = []
        for x, y in points:
            dist = -x ** 2 - y ** 2
            if len(res) == k:
                heappushpop(res, (dist, x, y))
            else:
                heappush(res, (dist, x, y))

        return [[x, y] for rank, x, y in res]

    def _kClosest(self, points: List[List[int]], k: int) -> List[List[int]]:
        res = []
        for idx, (x, y) in enumerate(points):
            dist = -x ** 2 - y ** 2
            if len(res) == k:
                if res[0][0] <= dist:
                    heappop(res)
                else:
                    continue
            heappush(res, (dist, idx, x, y))
        return [[x, y] for rank, idx, x, y in sorted(res, key=lambda item: item[1])]

assert_value([[-2, 2]], Solution().kClosest, points=[[1, 3], [-2, 2]], k=1)
assert_value([[3, 3], [-2, 4]], Solution().kClosest, points=[[3, 3], [5, -1], [-2, 4]], k=2)
assert_value([[1, 1]], Solution().kClosest, points=[[2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [1, 1]], k=1)
assert_value([[2, 31], [-27, -38], [-55, -39], [-57, -67], [34, -84]], Solution().kClosest,
             points=[[68, 97], [34, -84], [60, 100], [2, 31], [-27, -38], [-73, -74], [-55, -39], [62, 91], [62, 92],
                     [-57, -67]], k=5)


import numpy as np
import morphology as morph

def fill_background(mask, window_size):
    shape = mask.shape
    mask = mask > 0
    mask = morph.dilate(mask)

    # find empty space moving window
    stack = []
    for x in np.arange(0, shape[0], window_size):
        x2 = max(x + window_size, shape[1])
        if not np.any(mask[x:x2,0]):
            stack.append((x,0))
        if not np.any(mask[x:x2,-1]):
            stack.append((x,-1))

    for y in np.arange(0, shape[1], window_size):
        y2 = max(y + window_size, shape[0])
        if not np.any(mask[0,y:y2]):
            stack.append((0,y))
        if not np.any(mask[-1,y:y2]):
            stack.append((-1,y))

    checked = np.zeros(shape, dtype=bool)
    for t in stack:
        checked[t] = True
    ways = [(0,1), (0,-1), (1,0), (-1,0)]
    while len(stack) > 0:
        c = stack.pop()
        for w in ways:
            d = (c[0] + w[0], c[1] + w[1])
            if d[0] < 0 or d[0] >= shape[0] or d[1] < 0 or d[1] >= shape[1]:
                continue
            if checked[d] or mask[d]:
                continue
            checked[d] = True
            stack.append(d)

    return np.logical_not(checked).astype(dtype=np.uint8) * 255


'''
2 types
for  iterating
while  condition checking

for variablename in array(list,set,tuple,dict,range):
    print(variablename)

'''
a=[1,2,3,4,5,6]
for i in a:
    print(i)


import numpy as np
import matplotlib.pyplot as plt

# Define the derivative function

def differentiable_f(x):
    return x**2

def derivative_f(x):
    return 2 * x

# Create an array of x values
x = np.linspace(-10, 10, 400)
# print(x)

# Calculate the derivative of x
derivative_values = derivative_f(x)

# Calculate the differentiable of x
differentiable_values = differentiable_f(x)

# Create a figure with two subplots side by side
fig, axs = plt.subplots(1, 2, figsize=(12, 6))

# Plot the differentiable function
axs[0].plot(x, differentiable_values, label="f(x) = x ^ 2", color='blue')
axs[0].axhline(0, color='black', linewidth=0.8, linestyle='--')
axs[0].set_title("Plot of the Differentiable Function")
axs[0].set_xlabel("x")
axs[0].set_ylabel("f(x)")
axs[0].legend()
axs[0].grid(True)

# Plot the derivative function
axs[1].plot(x, derivative_values, label="f'(x) = 2x", color='red')
axs[1].axhline(0, color='black', linewidth=0.8, linestyle='--')
axs[1].set_title("Plot of the Derivative Function")
axs[1].set_xlabel("x")
axs[1].set_ylabel("f'(x)")
axs[1].legend()
axs[1].grid(True)

# Adjust layout and display the plots
plt.tight_layout()
plt.show()

# Plot the differentiable function
plt.figure(figsize=(8, 6))
plt.plot(x, differentiable_values, label="f'(x) = 4x^3 + 3", color='blue')
plt.axhline(0, color='black', linewidth=0.8, linestyle='--')
plt.show()
plt.title("Plot of the Differentiable Function")
plt.xlabel("x")
plt.ylabel("f'(x)")
plt.legend()
plt.grid(True)
plt.show()

# Plot the derivative function
plt.figure(figsize=(8, 6))
plt.plot(x, derivative_values, label="f'(x) = 4x^3 + 3", color='blue')
plt.axhline(0, color='black', linewidth=0.8, linestyle='--')
plt.show()
plt.title("Plot of the Derivative Function")
plt.xlabel("x")
plt.ylabel("f'(x)")
plt.legend()
plt.grid(True)
plt.show()


# Weather API Test
from weatherAPI import fetchWeather
def weatherTest(city):
    context = fetchWeather(city)

    return context

def run():
    """
    {'weather': {
        'city': 'Philadelphia',
        'temperature': 78.76,
        'description': 'Broken clouds',
        'icon': 'http://openweathermap.org/img/w/04d.png',
        'humidity': 50, 'pressure': 1005, 'country': 'US',
        'sunrise': datetime.datetime(2022, 6, 9, 5, 31, 54),
        'sunset': datetime.datetime(2022, 6, 9, 20, 28, 10),
        'windspeed': 19.57},
    'alerts': {}}
    """
    ctx = weatherTest('Philadelphia')

    assert ctx['weather'] is not None

    assert ctx['weather']['city'] == 'Philadelphia', \
     f"Expected Philadelphia, got: {ctx['weather']['city']}"

    print(f"SUCCESS")

run()


#!/usr/bin/env python
# _*_ coding:utf-8 _*_
from core import main
class NetMain(main.MainClass):
    def _run(self,host='*'):
        self.host=host
        self.method = 'grains.item'
        self.params = ['ip_interfaces','hwaddr_interfaces']
        result=self._main(self.method,self.params,self.host)
        JG=self.play_data(result)
        return JG

    def play_data(self,result_data):
        JG_dic={}
        for host,value in result_data.items():
            if not value:
                # print(host,'is down')
                continue
            if host not in JG_dic.keys():
                JG_dic[host]={}
            if 'network' not in JG_dic[host].keys():
                JG_dic[host]['network']={}

            for ip_mac,val in value.items():
                if ip_mac=='hwaddr_interfaces':
                    cache_dic=val
                else:
                    for eth,ip in val.items():
                        if eth=='lo':
                            continue
                        JG_dic[host]['network'][eth]=ip
            for eth,mac in cache_dic.items():
                if eth=='lo':
                    continue
                JG_dic[host]['network'][eth].append(mac.strip())
        return JG_dic


"""
Commandline_youtube_downloader
============================================
Coded By: Jackson Kamya
Email: kamyaj89@gmail.com
This script downloads Music from youtube
Pre-Requirements:
=====================
Python 2.7
BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/download/
pytube: www.pypi.python.org/pypi/pytube/ | www.sourceforge.net/projects/pytube/
Features:
=========
1. Download videos one by one as searched at a prompt
2. Download a list of songs as read line by line from file todownload.txt
You are free to copy and modify this script for your own requirements.
"""

import os
import time
from urllib2 import urlopen, Request
import sys

try:
    from bs4 import BeautifulSoup as bs
    from pytube import YouTube
except:
    print('Please install BeautifulSoup and pytube to run this script')
    sys.exit(0)

# creating download folder
base_dir = os.path.expanduser("~")
output_folder = os.path.join(base_dir, "Downloads", "py_cmd_utube_downloads")

if not os.path.exists(output_folder):
    os.mkdir(output_folder)

downloaded_resol = ''

# Getting youtube search term
def get_search_term():
    while True:
        inpt = raw_input('Enter search term\nor <<Enter>> to change mode: > ')
        if not inpt.lower():
            return main('b')
        search = inpt.replace(" ", "+")
        break
    return search

def selected_resol_format(u_obj):
    files = u_obj.videos
    file_list = []
    for f in files:
        ext = f.extension
        resol = f.resolution
        file_list.append((ext, resol))
    count = 0
    print('=' * 50)
    print('Available file formats and resolutions')
    print('=' * 50)
    for item in file_list:
        count += 1
        print("{} >>> {}-{}".format(count, item[0], item[1]))

    while True:
        try:
            choice = int(raw_input("Select from the choices above or enter 0 to exit selection: >"))
            if(choice == 0):
                return main('a')
            elif(choice > count or choice < 1):
                print('Invalid choice, input must be a number in the range 1-{} please try again\n'.format(count))
                continue
            else:
                break
        except Exception as e:
            print('Input must be a number in the range 0-{}, please try again'.format(count))
    return file_list[choice - 1]

# downloading the file
def download(url):
    v = YouTube(url)
    selection = selected_resol_format(v)
    vid = v.get(selection[0], selection[1])
    print('Downloading the video, you will be informed when its done, please hold on')
    vid.download(output_folder)
    global downloaded_resol
    downloaded_resol = selection[1]
    return True

# Selecting a file from the list
def select_video(soup):
    video_list = []
    for anchor in soup.findAll('a', href=True, title=True):
        link = anchor['href']
        title = anchor['title']
        if(link[:6] == "/watch"):
            video_list.append((link, title))

    print('=' * 30)
    print('Search Results')
    print('=' * 30)

    if not video_list:
        print('No results found')
        return main(mode)
    count = 0
    for l, t in video_list:
        count += 1
        print(str(count) + " >>> " + t)
    while True:
        try:
            # fix pep 8 E501
            selection = int(raw_input('Select from the above list or enter 0 to return to main program >'))
            if(selection > count):
                raise ValueError
            elif(selection == 0):
                return main(mode)
            else:
                return video_list[selection - 1]
        except ValueError:
            # fix pep 8 E501
            print('Input must be a number in the range 1-{}, please try again'.format(count))

# Getting the file
def get_video(soup):
        File = select_video(soup)
        title = File[1]
        link = File[0]
        vid_url = 'https://www.youtube.com' + link
        print('\n\nSelected: ' + title)
        print('Gathering available file formats and resolutions, please wait...')
        try:
            start_time = time.time()
            if download(vid_url):
                end_time = time.time()
                mins = (end_time - start_time) / 60
                secs = (end_time - start_time) % 60
                # fix pep 8 E501
                print("Downloaded %s(Resolution: %s) in %.0f minutes, %.0f seconds" % (title, str(downloaded_resol), mins, secs))
            else:
                print("Could not download the video")
        except Exception as e:
            print(e)
        return

def downloader(term):
    # Openning youtube search page
        print("Searching youtube, please wait..")
        try:
            url = 'https://www.youtube.com/results?search_query=' + term
            agent = ("Mozilla/5.0 "
                     "(X11; Ubuntu; Linux x86_64; rv:50.0)"
                     "Gecko/20100101 Firefox/50.0")

            headers = {"User-Agent": agent}
            r = Request(url, headers=headers)
            html = urlopen(r).read()

        except Exception as e:
            print(e)
            return

        # Obtaining page's soup
        try:
            soup = bs(html, "html.parser")
        except Exception as e:
            print(e)
            return

        # Getting the video
        try:
            get_video(soup)
        except Exception as e:
            print(e)
            return

# Selecting mode of operation
def get_mode():
    while True:
        print("A >>> Search songs from the prompt")
        print("B >>> Download songs in the todownload file")
        global mode
        mode = raw_input('Select Mode: A or B > ').lower()
        if len(mode) > 1 or not (mode.startswith('a') or mode.startswith("b")):
            print('Invalid mode, please try again')
            continue
        break
    return mode

# Main function of the program
# None seems unobvious we may pass some default mode
def main(m=None):
    if not m:
        mode = get_mode()
    else:
        mode = m
    if(mode == 'a'):
        while True:
            downloader(get_search_term())
    else:
        input_file = raw_input('Provide input file path e.g. D:/input.txt >')
        if not input_file:
            print("No path provided, please try again")
            return main(mode)
        else:
            if not os.path.exists(input_file):
                print("File does not exist")
                while True:
                    print('A >>> switch mode')
                    print('B >>> Try again')
                    print('C >>> Quit program')
                    choice = raw_input('Select one of the options above >')
                    if choice.lower() not in ('a', 'b', 'c'):
                        print('Invalid choice, please try again')
                        continue
                    if(choice == "a"):
                        return main('a')
                    elif(choice == 'b'):
                        return main(mode)
                    else:
                        sys.exit(0)
        with open(input_file, 'r') as f:
            try:
                files = f.readlines()
            except Exception as e:
                print(e)
                return

        for item in files:
            downloader(item.replace(" ", "+"))

if __name__ == "__main__":
    main()