import fire
import time

from typing import Sequence
from utils import *
from models import *

def main(learning_rate: float = 0.001,
         batch_size: int = 64,
         epochs: int = 10,
         steps_per_epoch: int = 70,
         block_hidden_dims: Sequence = (64, 64),
         n_block: int = 2):
    s_time = time.time()

    configs = {
        'block_hidden_dims': block_hidden_dims,
        'n_block': n_block
    }

    dataset = mnist_dataset(batch_size)

    model = get_model(configs)
    model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate),
                  loss='sparse_categorical_crossentropy',
                  metrics=['acc'])

    model.fit(dataset,
              epochs=epochs,
              steps_per_epoch=steps_per_epoch)

    print(f'Execution time: {time.time() - s_time}')

if __name__ == '__main__':
    fire.Fire(main)


'''Module for lists'''

from main.core.request_manager import RequestsManager as RM
from main.core.utils.api_constants import HttpMethods

class ListsAPI:
    '''Utils for List endpoint'''
    @staticmethod
    def create_list(name, board_id):
        """ Create a list inside a board

        :param name: name of List
        :type name: String
        :param board_id: id of Board where list will create
        :type board_id: String
        """
        body = {
            "name": name,
            "idBoard": board_id
        }
        status_code, json_response = RM.get_instance().do_request(HttpMethods.POST.value,  # pylint: disable=W0612
                                                                  "/lists", body)
        return json_response['id']

    @staticmethod
    def archive_list(list_id, value):
        """ Archive a list

        :param list_id: id of list to archive
        :type list_id: String
        :param value: id of list
        :type value: Boolean
        """
        endpoint = f"/lists/{list_id}/closed"
        body = {
            "value": value
        }
        RM.get_instance().do_request(HttpMethods.PUT.value, endpoint, body)   # pylint: disable=W0612


# -*- coding: utf-8 -*-

"""
ORIGINAL PROGRAM SOURCE CODE:
1: # coding=utf-8
2:
3: __doc__ = "Incorrect types passed on function calls"
4:
5: if __name__ == '__main__':
6:     def functionb(x):
7:         # Type error
8:         return x / 2
9:
10:
11:     def functionb2(x):
12:         # Type error
13:         return x / 2
14:
15:
16:     def functionb3(x):
17:         return x / 2
18:
19:
20:     y = functionb("a")
21:     y = functionb2(range(5))
22:     y = functionb3(4)
23:

"""

# Import the stypy library necessary elements
from stypy.type_inference_programs.type_inference_programs_imports import *

# Create the module type store
module_type_store = Context(None, __file__)

# ################# Begin of the type inference program ##################

# Assigning a Str to a Name (line 3):
str_1 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 3, 10), 'str', 'Incorrect types passed on function calls')
# Assigning a type to the variable '__doc__' (line 3)
module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 3, 0), '__doc__', str_1)

if (__name__ == '__main__'):

    @norecursion
    def functionb(localization, *varargs, **kwargs):
        global module_type_store
        # Assign values to the parameters with defaults
        defaults = []
        # Create a new context for function 'functionb'
        module_type_store = module_type_store.open_function_context('functionb', 6, 4, False)

        # Passed parameters checking function
        functionb.stypy_localization = localization
        functionb.stypy_type_of_self = None
        functionb.stypy_type_store = module_type_store
        functionb.stypy_function_name = 'functionb'
        functionb.stypy_param_names_list = ['x']
        functionb.stypy_varargs_param_name = None
        functionb.stypy_kwargs_param_name = None
        functionb.stypy_call_defaults = defaults
        functionb.stypy_call_varargs = varargs
        functionb.stypy_call_kwargs = kwargs
        arguments = process_argument_values(localization, None, module_type_store, 'functionb', ['x'], None, None, defaults, varargs, kwargs)

        if is_error_type(arguments):
            # Destroy the current context
            module_type_store = module_type_store.close_function_context()
            return arguments

        # Initialize method data
        init_call_information(module_type_store, 'functionb', localization, ['x'], arguments)

        # Default return type storage variable (SSA)
        # Assigning a type to the variable 'stypy_return_type'
        module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 0, 0), 'stypy_return_type', None)

        # ################# Begin of 'functionb(...)' code ##################

        # Getting the type of 'x' (line 8)
        x_2 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 8, 15), 'x')
        int_3 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 8, 19), 'int')
        # Applying the binary operator 'div' (line 8)
        result_div_4 = python_operator(stypy.reporting.localization.Localization(__file__, 8, 15), 'div', x_2, int_3)

        # Assigning a type to the variable 'stypy_return_type' (line 8)
        module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 8, 8), 'stypy_return_type', result_div_4)

        # ################# End of 'functionb(...)' code ##################

        # Teardown call information
        teardown_call_information(localization, arguments)

        # Storing the return type of function 'functionb' in the type store
        # Getting the type of 'stypy_return_type' (line 6)
        stypy_return_type_5 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 6, 4), 'stypy_return_type')
        module_type_store.store_return_type_of_current_context(stypy_return_type_5)

        # Destroy the current context
        module_type_store = module_type_store.close_function_context()

        # Return type of the function 'functionb'
        return stypy_return_type_5

    # Assigning a type to the variable 'functionb' (line 6)
    module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 6, 4), 'functionb', functionb)

    @norecursion
    def functionb2(localization, *varargs, **kwargs):
        global module_type_store
        # Assign values to the parameters with defaults
        defaults = []
        # Create a new context for function 'functionb2'
        module_type_store = module_type_store.open_function_context('functionb2', 11, 4, False)

        # Passed parameters checking function
        functionb2.stypy_localization = localization
        functionb2.stypy_type_of_self = None
        functionb2.stypy_type_store = module_type_store
        functionb2.stypy_function_name = 'functionb2'
        functionb2.stypy_param_names_list = ['x']
        functionb2.stypy_varargs_param_name = None
        functionb2.stypy_kwargs_param_name = None
        functionb2.stypy_call_defaults = defaults
        functionb2.stypy_call_varargs = varargs
        functionb2.stypy_call_kwargs = kwargs
        arguments = process_argument_values(localization, None, module_type_store, 'functionb2', ['x'], None, None, defaults, varargs, kwargs)

        if is_error_type(arguments):
            # Destroy the current context
            module_type_store = module_type_store.close_function_context()
            return arguments

        # Initialize method data
        init_call_information(module_type_store, 'functionb2', localization, ['x'], arguments)

        # Default return type storage variable (SSA)
        # Assigning a type to the variable 'stypy_return_type'
        module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 0, 0), 'stypy_return_type', None)

        # ################# Begin of 'functionb2(...)' code ##################

        # Getting the type of 'x' (line 13)
        x_6 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 13, 15), 'x')
        int_7 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 13, 19), 'int')
        # Applying the binary operator 'div' (line 13)
        result_div_8 = python_operator(stypy.reporting.localization.Localization(__file__, 13, 15), 'div', x_6, int_7)

        # Assigning a type to the variable 'stypy_return_type' (line 13)
        module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 13, 8), 'stypy_return_type', result_div_8)

        # ################# End of 'functionb2(...)' code ##################

        # Teardown call information
        teardown_call_information(localization, arguments)

        # Storing the return type of function 'functionb2' in the type store
        # Getting the type of 'stypy_return_type' (line 11)
        stypy_return_type_9 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 11, 4), 'stypy_return_type')
        module_type_store.store_return_type_of_current_context(stypy_return_type_9)

        # Destroy the current context
        module_type_store = module_type_store.close_function_context()

        # Return type of the function 'functionb2'
        return stypy_return_type_9

    # Assigning a type to the variable 'functionb2' (line 11)
    module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 11, 4), 'functionb2', functionb2)

    @norecursion
    def functionb3(localization, *varargs, **kwargs):
        global module_type_store
        # Assign values to the parameters with defaults
        defaults = []
        # Create a new context for function 'functionb3'
        module_type_store = module_type_store.open_function_context('functionb3', 16, 4, False)

        # Passed parameters checking function
        functionb3.stypy_localization = localization
        functionb3.stypy_type_of_self = None
        functionb3.stypy_type_store = module_type_store
        functionb3.stypy_function_name = 'functionb3'
        functionb3.stypy_param_names_list = ['x']
        functionb3.stypy_varargs_param_name = None
        functionb3.stypy_kwargs_param_name = None
        functionb3.stypy_call_defaults = defaults
        functionb3.stypy_call_varargs = varargs
        functionb3.stypy_call_kwargs = kwargs
        arguments = process_argument_values(localization, None, module_type_store, 'functionb3', ['x'], None, None, defaults, varargs, kwargs)

        if is_error_type(arguments):
            # Destroy the current context
            module_type_store = module_type_store.close_function_context()
            return arguments

        # Initialize method data
        init_call_information(module_type_store, 'functionb3', localization, ['x'], arguments)

        # Default return type storage variable (SSA)
        # Assigning a type to the variable 'stypy_return_type'
        module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 0, 0), 'stypy_return_type', None)

        # ################# Begin of 'functionb3(...)' code ##################

        # Getting the type of 'x' (line 17)
        x_10 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 17, 15), 'x')
        int_11 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 17, 19), 'int')
        # Applying the binary operator 'div' (line 17)
        result_div_12 = python_operator(stypy.reporting.localization.Localization(__file__, 17, 15), 'div', x_10, int_11)

        # Assigning a type to the variable 'stypy_return_type' (line 17)
        module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 17, 8), 'stypy_return_type', result_div_12)

        # ################# End of 'functionb3(...)' code ##################

        # Teardown call information
        teardown_call_information(localization, arguments)

        # Storing the return type of function 'functionb3' in the type store
        # Getting the type of 'stypy_return_type' (line 16)
        stypy_return_type_13 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 16, 4), 'stypy_return_type')
        module_type_store.store_return_type_of_current_context(stypy_return_type_13)

        # Destroy the current context
        module_type_store = module_type_store.close_function_context()

        # Return type of the function 'functionb3'
        return stypy_return_type_13

    # Assigning a type to the variable 'functionb3' (line 16)
    module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 16, 4), 'functionb3', functionb3)

    # Assigning a Call to a Name (line 20):

    # Call to functionb(...): (line 20)
    # Processing the call arguments (line 20)
    str_15 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 20, 18), 'str', 'a')
    # Processing the call keyword arguments (line 20)
    kwargs_16 = {}
    # Getting the type of 'functionb' (line 20)
    functionb_14 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 20, 8), 'functionb', False)
    # Calling functionb(args, kwargs) (line 20)
    functionb_call_result_17 = invoke(stypy.reporting.localization.Localization(__file__, 20, 8), functionb_14, *[str_15], **kwargs_16)

    # Assigning a type to the variable 'y' (line 20)
    module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 20, 4), 'y', functionb_call_result_17)

    # Assigning a Call to a Name (line 21):

    # Call to functionb2(...): (line 21)
    # Processing the call arguments (line 21)

    # Call to range(...): (line 21)
    # Processing the call arguments (line 21)
    int_20 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 21, 25), 'int')
    # Processing the call keyword arguments (line 21)
    kwargs_21 = {}
    # Getting the type of 'range' (line 21)
    range_19 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 21, 19), 'range', False)
    # Calling range(args, kwargs) (line 21)
    range_call_result_22 = invoke(stypy.reporting.localization.Localization(__file__, 21, 19), range_19, *[int_20], **kwargs_21)

    # Processing the call keyword arguments (line 21)
    kwargs_23 = {}
    # Getting the type of 'functionb2' (line 21)
    functionb2_18 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 21, 8), 'functionb2', False)
    # Calling functionb2(args, kwargs) (line 21)
    functionb2_call_result_24 = invoke(stypy.reporting.localization.Localization(__file__, 21, 8), functionb2_18, *[range_call_result_22], **kwargs_23)

    # Assigning a type to the variable 'y' (line 21)
    module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 21, 4), 'y', functionb2_call_result_24)

    # Assigning a Call to a Name (line 22):

    # Call to functionb3(...): (line 22)
    # Processing the call arguments (line 22)
    int_26 = get_builtin_python_type_instance(stypy.reporting.localization.Localization(__file__, 22, 19), 'int')
    # Processing the call keyword arguments (line 22)
    kwargs_27 = {}
    # Getting the type of 'functionb3' (line 22)
    functionb3_25 = module_type_store.get_type_of(stypy.reporting.localization.Localization(__file__, 22, 8), 'functionb3', False)
    # Calling functionb3(args, kwargs) (line 22)
    functionb3_call_result_28 = invoke(stypy.reporting.localization.Localization(__file__, 22, 8), functionb3_25, *[int_26], **kwargs_27)

    # Assigning a type to the variable 'y' (line 22)
    module_type_store.set_type_of(stypy.reporting.localization.Localization(__file__, 22, 4), 'y', functionb3_call_result_28)

# ################# End of the type inference program ##################

module_errors = stypy.errors.type_error.StypyTypeError.get_error_msgs()
module_warnings = stypy.errors.type_warning.TypeWarning.get_warning_msgs()


# -*- coding: utf-8 -*-
"""
NAME:
===============================
Plot (plot.py)

BY:
===============================
Mark Gotham

LICENCE:
===============================
Creative Commons Attribution-ShareAlike 4.0 International License
https://creativecommons.org/licenses/by-sa/4.0/

ABOUT:
===============================
Plot

"""

import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from scipy.optimize import curve_fit
from typing import Callable, Union
from music21 import roman

from . import CODE_FOLDER, data_by_heading, load_json

from .Pitch_profiles import chord_usage

ANTHOLOGY_PATH = CODE_FOLDER.parent / "Anthology"

# ------------------------------------------------------------------------------

# Linear, Polynomials (quadratic + cubic), and more

def linear(x, a, b):
    return a * x + b

def quadratic(x, a, b, c):
    return a * x ** 2 + b * x + c

def cubic(x, a, b, c, d):
    return a * x ** 3 + b * x ** 2 + c * x + d

def arc_tan(x, a, b, c, d):
    return a * np.arctan(b * x + c) + d

def exponential_decay(x, a, b, c):
    return a * np.exp(-b * x) + c

def log_decay(x, a, b, c):
    return -a * np.log(b * x) + c

# ------------------------------------------------------------------------------

def plot_dist(
        values_list_of_lists: list,
        name_list_of_lists: list,
        labels: list | None = None,
        plot_type: str = "bar",  # or "line"
        xlabel: str = "Position",
        ylabel: str = "Weight",
        title: str = "Profile weighting",
        width: float = 0.5,
        savefig: bool = False,
        out_path: Path | str | None = CODE_FOLDER / "test"
) -> plt:
    """
    Plot (bars or line) usage distribution
    whether for a source or a model (or anything else!)

    `values_list_of_lists` and `name_list_of_lists` are
    lists of lists to support any number of value / name pairs.
    There must be at least one sublist to each, and the same number as each other.
    """

    if not labels:
        labels = [str(x) for x in range(len(values_list_of_lists[0]))]
    elif len(labels) != len(values_list_of_lists[0]):
        raise ValueError("The length of variable labels "
                         f"(currently {len(labels)}) "
                         "must be equal to that of the values "
                         f"(currently {len(values_list_of_lists[0])}.")

    for x in values_list_of_lists:
        assert (len(x) == len(values_list_of_lists[0]))  # all the same length

    indexes = range(len(labels))

    # fig, ax = plt.subplots()
    # plt.figure()
    # plt.figure(figsize=(15, 6))

    # plot_type
    if plot_type == "bar":
        f = plt.bar
    elif plot_type == "line":
        f = plt.plot
    else:
        raise ValueError(f"plot_type {plot_type} not supported. Chose `bar` or `line`.")

    for i in range(len(values_list_of_lists)):
        f(indexes, values_list_of_lists[i], width, label=name_list_of_lists[i])

    plt.legend()  # loc="upper left")
    # plt.gcf().subplots_adjust(left=0.15)

    # plt.title(title)
    plt.xlabel(xlabel)  # , fontsize=14)
    plt.ylabel(ylabel)  # , fontsize=14)
    plt.xticks(indexes, labels)

    plt.ylim(0, 0.5)

    plt.tight_layout()

    if not title:
        title = "Plot"
    if savefig:
        plt.savefig(Path(out_path) / f"{title}.pdf", facecolor="w", edgecolor="w", format="pdf")
    else:
        return plt.figure

def plot_hist(
        data_list: list,
        bins: int | None = None,
        xlabel: str = "",
        ylabel: str = "Frequency",
        title: str = "",
        min_max_bin_from_data: bool = False,
        save_fig: bool = True
) -> plt:
    """
    Plotting data in the form of a list of integer values.
    Binning supported within the plot.
    """

    if min_max_bin_from_data:
        mn = min(data_list)
        mx = max(data_list)
        if bins is None:
            bins = int(mx) + 1 - int(mn)
    else:
        mn = 0
        mx = 100
        if bins is None:
            bins = 20

    incr = (mx - mn) / bins

    plt.figure(figsize=(15, 6))

    plt.hist(data_list,
             bins=bins,
             width=incr * 0.8,
             range=(mn, mx),
             align="mid",
             )

    plt.title(title, fontsize=20, family="serif")
    plt.xlabel(xlabel, fontsize=14, family="serif")
    plt.ylabel(ylabel, fontsize=14, family="serif")
    plt.xticks(np.arange(mn, mx, incr))
    plt.yticks(np.arange(0, 10, 1))

    if not title:
        title = "Plot"

    if save_fig:
        plt.savefig(title + ".pdf",
                    facecolor="w",
                    edgecolor="w",
                    format="pdf")
    else:
        return plt.figure

def plot_scatter_best_fit(
        xs: Union[list, np.array],
        ys: Union[list, np.array],
        fit_function: str | Callable = "exponential",
        p0: tuple | None = (2000, .1, 50),
        xscale: str | None = "log",
        xlabel: str = "Length (quarter notes)",
        ylabel: str = "Distance to prototype",
        title: str = "scatter_fit",
        savefig: bool = False
):
    """
    Make a scatter plot of 2D data and include a
    line (e.g. fit_function = "linear"), or
    curve (e.g. fit_function = "exponential"),
    of best fit.

    p0 argument is a tuple with starting estimate for fit function
    """

    if type(xs) == list:
        xs = np.array(xs)
    if type(ys) == list:
        ys = np.array(ys)

    # in case of multiple plots
    fig, ax = plt.subplots()
    ax.scatter(xs, ys, marker="x", linewidths=0.5)
    # plt.plot(xs, ys)  # , ".", label="data")

    if xscale:
        plt.xscale(xscale)

    x_fit = np.linspace(min(xs), max(xs), 10)  # issues with np.array(xs) for some reason

    squared_diffs = None

    if type(fit_function) == str:
        fit_function = fit_function.lower()

    if fit_function in ["lin", "linear", linear]:
        b, a = np.polynomial.polynomial.polyfit(xs, ys, 1)  # NB

        xs = np.array(xs)
        ys = np.array(ys)

        # params, cv = curve_fit(linear, xs, ys, p0)
        # a, b = params

        squared_diffs = np.square(ys - linear(xs, a, b))

        eq = f"{a}x + {b}"
        y_fit = a * x_fit + b

    elif fit_function in ["exp", "exponential", exponential_decay]:

        xs = np.array(xs)
        ys = np.array(ys)

        params, cv = curve_fit(exponential_decay, xs, ys, p0)
        a, b, c = params

        squared_diffs = np.square(ys - exponential_decay(xs, a, b, c))

        eq = f"y = {round(a, 3)}e^(-{round(b, 3)}x) + {round(c, 3)}"
        y_fit = exponential_decay(x_fit, a, b, c)

    elif fit_function in ["log", log_decay]:  # TODO compress

        xs = np.array(xs)
        ys = np.array(ys)

        # Fit
        params, cv = curve_fit(log_decay, xs, ys, p0)
        a, b, c = params

        # r^2 (quality of the fit from min 0 to max fit 1). TODO apply to all
        squared_diffs = np.square(ys - log_decay(xs, a, b, c))

        eq = f"y = -{round(a, 3)} * log({round(b, 3)}x) + {round(c, 3)}"
        y_fit = log_decay(x_fit, a, b, c)

    else:
        raise ValueError("Invalid fit_function")

    if squared_diffs is not None:
        r2_from_squared_diffs(ys, squared_diffs)

    plt.plot(x_fit, y_fit, "--", color="green", label=eq)
    plt.legend()

    plt.title(title, fontsize=16)
    plt.xlabel(xlabel, fontsize=12)
    plt.ylabel(ylabel, fontsize=12)

    if savefig:
        plt.savefig(title + ".pdf", facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()

def r2_from_squared_diffs(
        ys: np.array,
        squared_diffs: np.square,
        print_r2: bool = True
):
    squared_diffs_from_mean = np.square(ys - np.mean(ys))
    r2 = 1 - np.sum(squared_diffs) / np.sum(squared_diffs_from_mean)
    if print_r2:
        print(f"R² = {round(r2 * 100, 3)}%")
    return r2

# ------------------------------------------------------------------------------

def plot_with_best_fit(
        x: list,
        y: list,
        fit_function=exponential_decay,
        a: float = None,  # Initial guesses for the variables
        b: float = None,  # ... as needed for the fx ...
        c: float = None,
        xlabel: str = "Position",
        ylabel: str = "Weight",
        title: str = "Distribution weighting",
        savefig: bool = False
):
    # Check fit_function legit + number of variables

    popt, pcov = curve_fit(fit_function, x, y, p0=(a, b, c))

    # Calculate new x and y values for the curve of best fit
    x_new = np.linspace(x[0], x[-1], 50)  # first, last, step
    # y_new = f(x_new)  # quad
    y_new = fit_function(x_new, *popt)  # exp

    # Plot
    plt.plot(x, y, "o", x_new, y_new)
    plt.xlim(x[0], x[-1])  # min / max
    # plt.plot(x, np.poly1d(np.polyfit(x, nums, 1))(y))
    # plt.plot(x, np.poly1d(np.polyfit(x, y, 1))(y))
    # plt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, 1))(np.unique(x)))  # for unsorted
    plt.title(title, fontsize=16)
    plt.xlabel(xlabel, fontsize=12)
    plt.ylabel(ylabel, fontsize=12)

    if savefig:
        plt.savefig(title + ".pdf", facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()
        return plt

# TODO integrate above ^
def exp_with_fit(
        xs: np.array,
        ys: np.array,
        title: str = "Test"
) -> plt:
    plt.plot(xs, ys, ".")
    plt.title(title)

    # Fit
    p0 = (2000, .1, 50)  # start with plausible values
    params, cv = curve_fit(exponential_decay, xs, ys, p0)
    a, b, c = params

    # r^2 (quality of the fit from min 0 to max fit 1)
    squared_diffs = np.square(ys - exponential_decay(xs, a, b, c))
    squared_diffsFromMean = np.square(ys - np.mean(ys))
    rSquared = 1 - np.sum(squared_diffs) / np.sum(squared_diffsFromMean)
    print(f"R² = {rSquared}")

    eq = f"Y = {round(a, 3)} * e^(-{round(b, 3)} * x) + {round(c, 3)}"

    # plot the results
    plt.plot(xs, ys, ".", label="data")
    plt.plot(xs, exponential_decay(xs, a, b, c), "--", label=f"fit: {eq}")
    plt.legend()

    plt.show()

# ------------------------------------------------------------------------------

def plot_prog_by_positions(
        corpus_name: str = "OpenScore-LiederCorpus",
        what: str = "Quiescenzas"
) -> plt:
    data = data_by_heading(ANTHOLOGY_PATH / corpus_name / (what + ".csv"))
    position_strings = [x["MEASURE"] for x in data]
    positions = []
    for i in range(len(position_strings)):
        x, y = position_strings[i].split("/")
        positions.append(100 * int(x) / int(y))

    plot_hist(
        positions,
        xlabel="Position in work (%, binned)",
        ylabel="Frequency",
        title=f"{what}_{corpus_name}",
    )

def plot_usage(
        corpus_name: str = "OpenScore-LiederCorpus",
        what: str = "Aug6",
        user_list: list | None = ["bII", "bII6", "bII64", "bII7", "bII65", "bII43", "bII2"],
        save_fig: bool = True,
        out_path: Path | None = None
) -> plt:
    """
    Plot the usage of single chords by category.
    E.g., if "what" = "Aug6",
    plots all augmented 6ths in a sub-corpus by mode (maj/min) and
    3 "Nationality" labels ignoring inversion.

    Args:
        corpus_name: one of the sub-corpora
        what: str. What chord type. Currently, "Aug6", "N6", "User" in which last case use ...
        user_list: list. Optional. A user-defined list of figures to plot (in order).
            This is ignored unless "what" is "User".
            The default provides an alternative way of plotting the Neapolitans.
        save_fig: bool. Save a copy
        out_path: Path | None. Path to write to. If None, use ANTHOLOGY_PATH / corpus_name.

    Returns: plt

    """
    minor_data = None
    major_data = None
    all_keys = None

    if what == "Aug6":
        minor_data = chord_usage.get_Aug6s(this_mode="minor", corpus_name=corpus_name)
        major_data = chord_usage.get_Aug6s(this_mode="major", corpus_name=corpus_name)
        all_keys = ("Ger", "It", "Fr")
    elif what == "N6":
        minor_data = chord_usage.get_N6s(this_mode="minor", corpus_name=corpus_name)
        major_data = chord_usage.get_N6s(this_mode="major", corpus_name=corpus_name)
        all_keys = list(minor_data.keys()) + list(major_data.keys())
    elif what == "User":
        chord_usage_dir = CODE_FOLDER / "Resources" / "chord_usage"
        minor_data = load_json(chord_usage_dir / f"minor_{corpus_name}_simple.json")
        major_data = load_json(chord_usage_dir / f"major_{corpus_name}_simple.json")
        all_keys = user_list
        pass

    for k in all_keys:
        if k not in minor_data:
            minor_data[k] = 0
        if k not in major_data:
            major_data[k] = 0

    by_mode = {"minor": tuple([round(minor_data[x], 2) for x in all_keys]),
               "major": tuple([round(major_data[x], 2) for x in all_keys])
               }
    max_val = max(list(by_mode["minor"]) + list(by_mode["major"]))

    x = np.arange(len(all_keys))
    bar_width = 0.25
    count = 0

    fig, ax = plt.subplots(layout='constrained')

    for k, v in by_mode.items():
        offset = bar_width * count
        rects = ax.bar(x + offset, v, bar_width, label=k)
        ax.bar_label(rects, padding=2)
        count += 1

    ax.set_ylabel("Frequency (%)")
    ax.set_xticks(x + bar_width / 2, all_keys)
    ax.legend(loc="upper right", ncols=2)

    ax.set_ylim(0, round(max_val, 2))  # top border = highest value; care if using with title
    ax.set_xlabel("Figure")

    if what == "Aug6":
        ax.set_xlabel('Augmented 6th by so-called "nationality" (any inversion)')

    if not out_path:
        out_path = ANTHOLOGY_PATH / corpus_name

    if save_fig:
        plt.savefig(out_path / (what + ".pdf"), facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()

    return plt

def plot_counts(
        corpora: list | None = None,
        what: list | None = None,
        save_fig: bool = True,
        out_path: Path | None = None,
        title: str | None = None
) -> plt:
    """
    Plot the usage of chords and/or progressions by category.
    E.g., "what" defaults to
    ["ascending_fifths",
    "descending_fifths",
    "aufsteigender_Quintfall",
    "fallender_Quintanstieg"
    ],
    and plots the number of each appearing
    in each of the sub-corpora listed by `corpora`.

    Args:
        corpora: list of which sub-corpora
        what: list of what chord / progressions type.
        save_fig: bool. Save a copy
        out_path: Path | None. Path to write to. If None, use ANTHOLOGY_PATH.

    Returns: plt

    """
    if corpora is None:
        corpora = ["OpenScore-LiederCorpus",
                   "Piano_sonatas",
                   "Keyboard_Other",
                   "Early_Choral"
                   ]
    if what is None:
        what = ["ascending_fifths",
                "descending_fifths",
                "aufsteigender_Quintfall",
                "fallender_Quintanstieg"
                ]

    import csv

    by_topic = {}
    max_val = 0
    for corpus in corpora:
        this_list = []
        for w in what:
            this_path = str(ANTHOLOGY_PATH / corpus / f"{w}.csv")
            with open(this_path) as csvfile:
                data = csv.reader(csvfile)
                row_count = sum(1 for row in data)  # sum() w generator to avoid storing whole file
            this_list.append(row_count - 1)  # header
            if row_count > max_val:
                max_val = row_count
        by_topic[corpus] = this_list

    x = np.arange(len(what))
    c_and_gap = len(corpora) + 1
    bar_width = 1 / c_and_gap  # e.g., 0.25 for 3 corpora
    mid_bars = bar_width * c_and_gap / 2

    fig, ax = plt.subplots(layout='constrained')

    count = 1
    for k, v in by_topic.items():
        offset = bar_width * count
        rects = ax.bar(x + offset, v, bar_width, label=k)
        ax.bar_label(rects, padding=2)
        count += 1

    ax.set_ylabel("Count")
    ax.set_xticks(x + mid_bars,
                  [x.split("_")[0] for x in what]  # shorthand
                  )
    ax.legend(loc="upper right", ncols=1)

    ax.set_ylim(0, round(max_val, 2))

    if not out_path:
        out_path = ANTHOLOGY_PATH
    if not title:
        title = "multi_corpus_plot_count_example.pdf"

    if save_fig:
        plt.savefig(out_path / title, facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()

    return plt

# ------------------------------------------------------------------------------

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument("--plot_prog_by_positions", action="store_true")
    parser.add_argument("--plot_N6_usage", action="store_true")
    parser.add_argument("--plot_Aug6_usage", action="store_true")
    parser.add_argument("--plot_counts", action="store_true")

    parser.add_argument(
        "--corpus",
        type=str,
        required=False,
        default="OpenScore-LiederCorpus",
        help="Process all cases within this sub-corpus."
             "If set to `all` then run on all sub-corpora"
    )

    parser.add_argument(
        "--what",
        type=str,
        required=False,
        default="Quiescenzas",
        help="Which of the anthology cases.")

    args = parser.parse_args()

    if args.plot_prog_by_positions:
        plot_prog_by_positions(
            corpus_name=args.corpus,
            what=args.what
        )

    elif args.plot_N6_usage:
        if args.corpus == "all":
            for child in ANTHOLOGY_PATH.iterdir():
                if child.is_dir():
                    print(str(child.parts[-1]))
                    plot_usage(corpus_name=str(child.parts[-1]), what="N6")
        else:
            plot_usage(corpus_name=args.corpus, what="N6")

    elif args.plot_Aug6_usage:
        if args.corpus == "all":
            for child in ANTHOLOGY_PATH.iterdir():
                if child.is_dir():
                    print(str(child.parts[-1]))
                    plot_usage(corpus_name=str(child.parts[-1]), what="Aug6")
        else:
            plot_usage(corpus_name=args.corpus, what="Aug6")

    elif args.plot_counts:
        plot_counts()

    else:
        parser.print_help()


from User.user_data import User, Item

class table:
    def __init__(self):
        self.users = []
        self.test_id = {} #test_id['user_id'] = 'test_id'
        self.test_user_id = []

    def add_test(self, test_file):
        fi = open(test_file, 'r')
        while True:
            line = fi.readline()
            if not line:
                break
            line = line.split('\t')
            if not int(line[0]) in self.test_id:
                self.test_id[int(line[0])] = []
                self.test_user_id.append(int(line[0]))
            self.test_id[int(line[0])].append(int(line[1]))

    def add_user(self, input_file_name):
        fi = open(input_file_name, 'r')
        items = []
        rank_sum = 0
        now_id = 1
        while True:
            line = fi.readline()  # line: user_id, item_id, rating, timestamp
            if not line:
                break
            line = line.split('\t')
            line[3] = line[3][:len(line[3]) - 2]
            if int(line[0]) > now_id:
                this_user = User(now_id, items, rank_sum / len(items))
                self.users.append(this_user) # (user_id -1) is user list index
                items = []
                rank_sum = 0
                now_id += 1
            this_item = Item(int(line[1]), int(line[2]), int(line[3]))
            items.append(this_item)
            rank_sum += int(line[2])

    def test_fill(self, output_file):
        fo = open(output_file, 'w')
        test_ids = len(self.test_id)
        for i in self.test_user_id:
            test_idx = self.test_id[i]
            user_obj = self.users[i - 1]
            for j in test_idx:
                answer = user_obj.rank_mean
                sim_sum = 0
                predict = 0
                for k in self.users:
                    if k is user_obj:
                        continue
                    item = k.has_item(j)
                    if item is not None:
                        temp = user_obj.similarity(k)
                        temp2 = item.rank - k.rank_mean
                        sim_sum += temp
                        predict += temp*temp2

                if sim_sum > 0:
                    answer += predict/sim_sum
                answer = round(answer)
                if answer < 1:
                    answer = 1
                elif answer > 5:
                    answer = 5
                line = str(user_obj.user_id) + '\t' + str(j) + '\t' + str(answer) + '\n'
                #print(line)
                fo.write(line)


import re

def opening1():
    f = open('in/text_for_exam.txt', 'r', encoding = 'utf-8')
    lines1 = f.readlines()
    f.close()
    lines = []
    for i in lines1:
        i = i.strip('\n')
        lines.append(i)
    return lines

def opening2():
    f = open('examsite.htm', 'r', encoding = 'utf-8')
    html1 = f.readlines()
    f.close()
    html = []
    for i in html1:
        i = i.lower()
        html.append(i)
    return html

def reg(lines, html):
    a1 = set()
    for line in lines:
        req = '(?: |,|\.|!|\?|:|;)' + line + '(?: |,|\.|!|\?|:|;)'
        for i in html:
            a2 = re.findall(req, i)
            if a2!= []:
                a1.add(line)
    return a1

def new_file(a1):
    f = open('wordlist.txt', 'a', encoding = 'utf-8')
    for i in a1:
        f.write(i+'\n')
    f.close()
    return

def final():
    a1 = opening1()
    a2 = opening2()
    a3 = reg(a1, a2)
    a4 = new_file(a3)
    return

if __name__=='__main__':
    final()


from datetime import datetime

from django.core.mail import send_mail

from games.models import Item
from member.models import Member, PaymentMethod
from .models import PurchaseRecord

class Purchase:

    def __init__(self, member: Member, item: Item):
        self.member: Member = member
        self.item: Item = item
        self.price: float = float(item.get_price())
        self.number_of_rewards: int = 0

    def set_number_of_rewards(self, number_of_rewards: int) -> None:
        if number_of_rewards <= self.member.get_number_of_rewards():
            self.number_of_rewards = number_of_rewards
            self.price = (
                float(self.item.get_price()) * (1 - 0.1 * number_of_rewards))

    def get_amount(self) -> float:
        return self.price

    def make_payment(self) -> bool:
        payment = Payment(self.member.get_payment_method())
        successful = payment.pay(self.price)
        if successful:
            self.record_purchase()
            self.member.use_rewards(self.number_of_rewards)
            self.member.accumulate_spending(self.price)
            send_mail(
                'Receipt of your purchase on Fume',
                f'Dear {self.member.nickname},'
                f'\n\n'
                f'Thank you for purchasing on Fume.'
                f' You have purchased the game {self.item.game.title}.'
                f' This is your receipt.'
                f'\n\n'
                f'Best,'
                f'\n'
                f'Fume',
                'noreply@fume.com',
                [self.member.user.email],
                fail_silently=True,
            )
        return successful

    def record_purchase(self) -> None:
        record = PurchaseRecord.create(
            self.member, self.item, datetime.now(), self.number_of_rewards)
        record.save()

class Payment:

    def __init__(self, payment_method: PaymentMethod):
        self.payment_method = payment_method

    def pay(self, amount: float) -> bool:
        return True


import platform
import os
import inspect
import subprocess
from setuptools import setup, find_packages
from distutils.extension import Extension

from Cython.Build import cythonize

def git_version():
    def _minimal_ext_cmd(cmd):
        # construct minimal environment
        env = {}
        for k in ['SYSTEMROOT', 'PATH']:
            v = os.environ.get(k)
            if v is not None:
                env[k] = v
        # LANGUAGE is used on win32
        env['LANGUAGE'] = 'C'
        env['LANG'] = 'C'
        env['LC_ALL'] = 'C'
        out = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
        return out

    try:
        out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD'])
        git_revision = out.strip().decode('ascii')
    except OSError:
        git_revision = "Unknown"

    return git_revision

def get_version_info(version, is_released):
    fullversion = version
    if not is_released:
        git_revision = git_version()
        fullversion += '.dev0+' + git_revision[:7]
    return fullversion

def write_version_py(version, is_released, filename='composites/version.py'):
    fullversion = get_version_info(version, is_released)
    with open("./composites/version.py", "wb") as f:
        f.write(('__version__ = "%s"\n' % fullversion).encode())
    return fullversion

# Utility function to read the README file.
# Used for the long_description.  It's nice, because now 1) we have a top level
# README file and 2) it's easier to type in the README file than to put a raw
# string in below ...
def read(fname):
    setupdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    return open(os.path.join(setupdir, fname)).read()

#_____________________________________________________________________________

install_requires = [
        "numpy",
        ]

CLASSIFIERS = """\

Development Status :: 5 - Production/Stable
Intended Audience :: Science/Research
Intended Audience :: Developers
Intended Audience :: Education
Intended Audience :: End Users/Desktop
Topic :: Scientific/Engineering
Topic :: Education
Topic :: Software Development
Topic :: Software Development :: Libraries :: Python Modules
Operating System :: Microsoft :: Windows
Operating System :: Unix
Operating System :: POSIX :: BSD
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
License :: OSI Approved :: BSD License

"""

is_released = True
version = '0.5.25'

fullversion = write_version_py(version, is_released)

data_files = [('', [
        'README.md',
        'LICENSE',
        'composites/version.py',
        ])]

package_data = {
        'composites': ['*.pxd', '*.pyx'],
        '': ['tests/*.*'],
        }

if platform.system() == 'Windows':
    compile_args = ['/openmp']
    link_args = []
elif platform.system() == 'Linux':
    compile_args = ['-fopenmp', '-static', '-static-libgcc', '-static-libstdc++']
    link_args = ['-fopenmp', '-static-libgcc', '-static-libstdc++']
else: # MAC-OS
    compile_args = []
    link_args = []
include_dirs = [
            ]

extensions = [
    Extension('composites.core',
        sources=[
            './composites/core.pyx',
            ],
        include_dirs=include_dirs,
        extra_compile_args=compile_args,
        extra_link_args=link_args,
        language='c++'),

    ]
ext_modules = cythonize(extensions,
        compiler_directives={'linetrace': True},
        language_level='3',
        )

s = setup(
    name = "composites",
    version = fullversion,
    author = "Saullo G. P. Castro",
    author_email = "S.G.P.Castro@tudelft.nl",
    description = ("Methods to calculate properties of laminated composite materials"),
    long_description = read('README.md'),
    long_description_content_type = 'text/markdown',
    license = "3-Clause BSD",
    keywords = "mechanics composite materials composites shell classical first-order laminated plate theory",
    url = "https://github.com/saullocastro/composites",
    package_data = package_data,
    data_files = data_files,
    classifiers = [_f for _f in CLASSIFIERS.split('\n') if _f],
    install_requires = install_requires,
    ext_modules = ext_modules,
    packages = find_packages(),
)


import Lib.Utils.update_opened_path
from global_imports import *

def OpenDir(parent, title, preferred_dir=None):
    """
    Opens directory and returns directory path or None (if nothing selected/ pop-up cancelled)
    :param parent:
    :param title:
    :param preferred_dir:
    :return:
    """
    options = QtWidgets.QFileDialog.Options()
    options |= QtWidgets.QFileDialog.ShowDirsOnly
    dirName = QtWidgets.QFileDialog.getExistingDirectory(parent, title, directory=preferred_dir, options=options)
    if dirName:
        Lib.Utils.update_opened_path.check_dir_location(dirName)
        return dirName
    else:
        return None


from django import forms

from .models import MailingEmailAddress

class MailingEmailAddressModelForm(forms.ModelForm):
    """Form for creating a mailing email address."""

    class Meta:
        """Meta options for the MailingEmailAddressModelForm."""

        fields = ["email"]
        model = MailingEmailAddress
        widgets = {
            "email": forms.EmailInput(
                attrs={"class": "form-control", "placeholder": "Email address"}
            )
        }


f = open("../src/atari800/compressed-charset.asm", 'r')
o = open("../src/atari800/decompressed-charset.asm", 'w')

bytes = []
labels = {}
while True:
    l = f.readline()
    if not l:
        break
    label = l[:8]
    if label:
        labels[len(bytes)] = label
    bytes += list(filter(None, l[14:45].strip().split(',')))

print(bytes)

def addSequence(cur, i):
    v = bytes[i-2]
    c = int(bytes[i-1][-2:], 16)
    cur += [v] * c

def writeChar(cur, cur_raw, char_count):
    raw_bytes = ".BYTE " + ','.join(cur_raw)
    o.write(raw_bytes)
    o.write(" " * (38 - len(raw_bytes)) + "  ;.BYTE " + ','.join(cur[:8]) + '\n')

    h = ("0" + format(char_count, 'x'))[-2:]
    o.write((" " * 40) + "; CHARACTER $" + h + '\n')

    hex_chars = list(map(lambda x: x.replace("$",""), cur[:8]))
    print(hex_chars)
    bins = [(bin(int(b, 16))[2:].zfill(8)) for b in hex_chars]
    for b in bins:
        o.write((" " * 40) + "; " + b + "   " + b.replace("0", " ").replace("1","*") + '\n')

    del cur[:8]

cur = []
cur_raw = []
tag = "$A1B"
char_count = 0
index = 2
while (index < len(bytes)):
    i = index - 2

    if len(cur) >= 8:
        writeChar(cur, cur_raw, char_count)
        char_count += 1
        cur_raw = []

    cb = bytes[index]
    if cb == tag:
        addSequence(cur, index)
        index += 3
        cur_raw += bytes[i:i+3]
        continue

    if i in labels:
        o.write(labels[i] + '\n')

    b = bytes[i]
    cur_raw += [b]
    cur.append(b)
    index += 1


import os
import random
import numpy as np
import utils
import utils_sgd
import utils_svd as svd

SUBMISSION_FILE = os.path.join(utils.ROOT_DIR, 'data/submission_sf.csv')
SCORE_FILE = os.path.join(utils.ROOT_DIR, 'analysis/sf_scores.csv')
N_EPOCHS = 100
LEARNING_RATE = 0.001
REG_EMB = 0.02
REG_BIAS = 0.05
EPSILON = 0.0001

def learn(data, u_embeddings, z_embeddings, u_bias, z_bias, n_epochs,
          reg_emb, reg_bias):
    # residual_data = np.copy(data)
    training_indices = utils.get_indices_from_file(utils.TRAINING_FILE_NAME)
    total_average = np.mean(data[np.nonzero(data)])
    approximation_rank = u_embeddings.shape[1]
    for feature_index in range(approximation_rank):
        print("Feature %d." % feature_index)
        last_rmse = 5
        for _ in range(n_epochs):
            random.shuffle(training_indices)
            for k, l in training_indices:
                temp_u_emb = u_embeddings[k, feature_index]
                temp_z_emb = z_embeddings[l, feature_index]

                lagrangian = u_bias[k] + z_bias[l] - total_average

                residual = data[k, l] - u_bias[k] - z_bias[l] - np.dot(
                    u_embeddings[k, : feature_index + 1],
                    z_embeddings[l, : feature_index + 1])

                u_embeddings[k, feature_index] *= (1 - LEARNING_RATE * reg_emb)
                u_embeddings[k, feature_index] += LEARNING_RATE * residual * temp_z_emb
                z_embeddings[l, feature_index] *= (1 - LEARNING_RATE * reg_emb)
                z_embeddings[l, feature_index] += LEARNING_RATE * residual * temp_u_emb
                u_bias[k] -= LEARNING_RATE * reg_bias * lagrangian
                u_bias[k] += LEARNING_RATE * residual
                z_bias[l] -= LEARNING_RATE * reg_bias * lagrangian
                z_bias[l] += LEARNING_RATE * residual

            reconstruction = utils_sgd.reconstruct(
                u_embeddings[:, :feature_index + 1],
                z_embeddings[:, :feature_index + 1], u_bias, z_bias)
            # residual_data = data - reconstruction
            rmse = utils.compute_rmse(data, reconstruction, utils.get_observed_indices(data))
            if abs(last_rmse - rmse) < EPSILON:
                break
            last_rmse = rmse
        print("rmse after feature %d: %f" % (feature_index, rmse))
    return reconstruction

# sf stands for Simon Funk.
def predict_by_sf(data, approximation_rank=None, reg_emb=REG_EMB,
                  reg_bias=REG_BIAS, u_embeddings=None, z_embeddings=None,
                  n_epochs=N_EPOCHS):
    np.random.seed(42)
    if u_embeddings is None and z_embeddings is None:
        print("Initialize embeddings.")
        u_embeddings, z_embeddings = utils_sgd.get_initialized_embeddings(
            approximation_rank, data.shape[0], data.shape[1])
    u_bias = np.zeros(u_embeddings.shape[0])
    z_bias = np.zeros(z_embeddings.shape[0])
    reconstruction = learn(
        data, u_embeddings, z_embeddings, u_bias, z_bias, n_epochs, reg_emb,
        reg_bias)
    utils.clip(reconstruction)
    return reconstruction, u_embeddings, z_embeddings, u_bias, z_bias

def main():
    # Initialize hyperparameters.
    ranks = [i for i in range(3, 40)]
    regularizations = [0.005, 0.002, 0.02, 0.05, 0.2, 0.5]
    reg_emb = np.random.choice(regularizations)
    reg_bias = np.random.choice(regularizations)
    rank = np.random.choice(ranks)
    svd_initiliazied = random.choice([True, False])

    all_ratings = utils.load_ratings()
    data = utils.ratings_to_matrix(all_ratings)
    masked_data = utils.mask_validation(data, False)
    if svd_initiliazied:
        initialization_string = 'svd'
        imputed_data = np.copy(masked_data)
        utils.impute_by_variance(imputed_data)
        u_embeddings, z_embeddings = svd.get_embeddings(imputed_data, rank)
        reconstruction, u_embeddings, _, _, _ = predict_by_sf(
            masked_data, rank, reg_emb, reg_bias, u_embeddings, z_embeddings)
    else:
        initialization_string = 'rand'
        reconstruction, u_embeddings, _, _, _ =\
                predict_by_sf(masked_data, rank, reg_emb, reg_bias)

    rmse = utils.compute_rmse(data, reconstruction)
    print('Validation rmse before smoothing: %f' % rmse)
    utils_sgd.write_sgd_score(
        rmse, rank, reg_emb, reg_bias, '!S', initialization_string, SCORE_FILE)
    reconstruction = utils.knn_smoothing(reconstruction, u_embeddings)
    rmse = utils.compute_rmse(data, reconstruction)
    print('Validation RMSE after smoothing: %f' % rmse)
    utils_sgd.write_sgd_score(
        rmse, rank, reg_emb, reg_bias, 'S', initialization_string, SCORE_FILE)
    utils.reconstruction_to_predictions(reconstruction, SUBMISSION_FILE)
    if utils.SAVE_META_PREDICTIONS:
        utils.save_ensembling_predictions(reconstruction, 'sf')

if __name__ == '__main__':
    main()


"""This script is used for controlling the debug console
send command to board to bring up the wifi connection
Version 2: use OOB
"""
import serial
import time
import re

class SerialCom:
    def __init__(self, comport="COM1"):
        self.comport = comport
        self.ser = serial.Serial()
        self.ser.port = self.comport

        # 115200,N,8,1
        self.ser.baudrate = 115200
        self.ser.bytesize = serial.EIGHTBITS  # number of bits per bytes
        self.ser.parity = serial.PARITY_NONE  # set parity check
        self.ser.stopbits = serial.STOPBITS_ONE  # number of stop bits
        self.ser.timeout = 0.5
        self.ser.writeTimeout = 0.5
        self.ser.xonxoff = False  # disable software flow control
        self.ser.rtscts = False  # disable hardware (RTS/CTS) flow control
        self.ser.dsrdtr = False  # disable hardware (DSR/DTR) flow control

    def cmdsend(self, command):
        self.command = command
        self.ser.write(self.command.encode('UTF-8'))

    def cmdsenddly(self, command, secs=0.5):
        self.command = command
        self.secs = secs
        self.ser.write(self.command.encode('UTF-8'))
        time.sleep(self.secs)

    def cmdread(self, waitstring="\r\n"):
        self.waitstring = waitstring
        response = self.ser.read_until(self.waitstring).decode('UTF-8')
        lastline = waitstring
        fmtresponse = response.strip(lastline)
        return fmtresponse

def msg_readuntil(desiremsg, secs = 0.5):
    global serctrl

    testmsg = serctrl.cmdread()
    while desiremsg not in testmsg:
        print(testmsg)
        testmsg = serctrl.cmdread()
        time.sleep(secs)
    print(testmsg)

loginprompt = "login:"
inprompt = "root@"
Target_SSID = "Synology_520HMX_5G"
Wifipasskey = "82273585"
waitstring = "~# "

# iperf test parameter
IperfServ = "10.88.88.147"

serctrl = SerialCom("COM5")

macidRegex = re.compile('([a-fA-F]|\d){12}')
wifiphraseRegex = re.compile('wifi_\w{12}_\w.+_managed_psk')
ipaddrRegex = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")

print("Test starts ...")

try:
    serctrl.ser.open()
except Exception as ex:
    print("open serial port error " + str(ex))
    exit()

if serctrl.ser.isOpen():

    try:
        serctrl.ser.reset_input_buffer()  # flush input buffer
        serctrl.ser.reset_output_buffer()  # flush output buffer

        serdata = ""
        count = 0
        while inprompt not in serdata:
            if count <= 30:
                serdata = serctrl.cmdread()
                if serdata != "":
                    print(serdata)
                else:
                    print(serdata, end="")

            if loginprompt in serdata:
                # serctrl.ser.write('root\n'.encode('UTF-8'))
                serctrl.cmdsend('root\n')
                serdata = serctrl.cmdread(waitstring)
                print("root account is log in")
                # print(serdata)

            time.sleep(0.5)
            count += 1

            if count > 30:
                serctrl.cmdsend('\n')
                serdata = serctrl.cmdread()

        time.sleep(5)
        serctrl.ser.reset_input_buffer()
        serctrl.ser.reset_output_buffer()

        # check if the wlan0 ip is up
        for i in range(3):
            serctrl.cmdsend("ifconfig wlan0 | grep 'inet addr'\n")
            ipinfo = serctrl.cmdread(waitstring)
            time.sleep(5)
            print("%s : check wlan info" %str(i+1))
            if ipaddrRegex.search(ipinfo):
                break
            else:
                pass
                print("wlan info check failed")

        if ipaddrRegex.search(ipinfo):
            print(ipaddrRegex.search(ipinfo).group())
            print("The wlan connection is up")

        else:
            serctrl.cmdsenddly("connmanctl\n", 0.5)
            reponse = serctrl.cmdread(">")
            print(reponse)
            serctrl.cmdsenddly('enable wifi\n', 0.5)
            serctrl.cmdsenddly('scan wifi\n', 2)
            serctrl.cmdsenddly('exit\n', 0.5)
            serctrl.ser.flush()
            reponse = serctrl.cmdread()
            print(reponse)

            # shell
            serctrl.cmdsenddly("connmanctl services > /tmp/wifiservices\n", 0.5)
            serctrl.cmdsend("sync\n")
            serctrl.cmdsenddly("cat /tmp/wifiservices | grep ", 0.5)
            serctrl.cmdsenddly(Target_SSID, 0.5)
            serctrl.cmdsenddly(" | grep 001\n", 0.5)
            wifisvs = serctrl.cmdread(waitstring)
            time.sleep(1)

            # print(wifisvs)
            wifikey = wifiphraseRegex.search(wifisvs).group()
            # print(wifikey)

            # connmanctl shell
            serctrl.cmdsenddly('connmanctl\n', 1)
            # connmanctl shell
            serctrl.cmdsenddly('agent on\n', 1)
            serctrl.cmdsenddly('connect ', 0.5)
            serctrl.cmdsenddly(wifikey, 0.5)
            serctrl.cmdsend('\n')
            # enter Passphrase
            connreact = serctrl.cmdread('>')
            if "Already connected" in connreact or "wlan0: link becomes ready" in connreact:
                pass
            else:
                serctrl.cmdsend(Wifipasskey)
                serctrl.cmdsenddly('\n', 3)
                print(serctrl.cmdread())
                time.sleep(3)

            serctrl.cmdsend('state\n')
            print(serctrl.cmdread())
            time.sleep(1)
            serctrl.cmdsend('exit\n')
            print(serctrl.cmdread(waitstring))
            print("wifi connection completed")
            serctrl.cmdsenddly("\n", 2)

        # iperf test
        print("Iperf TX - Upload test")
        serctrl.cmdsend("iperf3 -c ")
        serctrl.cmdsend(IperfServ)
        serctrl.cmdsenddly(" -t 10 -i 2 -p 5201", 0.5)
        serctrl.cmdsend("\n")

        msg_readuntil("iperf Done.", 2)

        print("Iperf RX - Download test")
        serctrl.cmdsend("iperf3 -c ")
        serctrl.cmdsend(IperfServ)
        serctrl.cmdsenddly(" -t 10 -i 2 -p 5202 -R", 0.5)
        serctrl.cmdsend("\n")

        msg_readuntil("iperf Done.", 2)

        print("Iperf BI-direction test starting ...")
        serctrl.cmdsend("iperf3 -c ")
        serctrl.cmdsend(IperfServ)
        serctrl.cmdsenddly(" -t 10 -i 2 -p 5201&", 0.5)
        serctrl.cmdsend("\n")
        serctrl.cmdsend("iperf3 -c ")
        serctrl.cmdsend(IperfServ)
        serctrl.cmdsenddly(" -t 10 -i 2 -p 5202 -R&", 0.5)
        serctrl.cmdsend("\n")

        msg_readuntil("iperf Done.", 2)

        print("iperf test completed")

        # close console
        serctrl.ser.close()
        print("the comport is closed")

    except Exception as e1:
        print("communicating error , the port is occupied" + str(e1))
        serctrl.ser.close()
        print("the comport is closed")

else:
    print("open serial port error")