""" This module creates a table according to the specifications
The table is entirely scrollable and allows selection of a row
and visibly seeing the selected row using the highlight of grey
color, but the color can be changed easily.
Deleting of a row is supported by this table using the appropriate
functions and methods and once a row is deleted, another is
automatically selected in the place of the deleted row
Scrolling can be done by the mouse wheel or the Scrollbar for both
the vertical and the horizontal

Note:
-----
Python Dictionaries are used for the rows. Each row must be
a python dictionary for this table to work well. Uniform or
constant keys for these dictionaries must be used

Imports:
--------
:tk - The main tkinter module
:Thread - For loading and working in background
:messagebox - For the confirmation of the deletion of a row
:ttk - The themed tkinter

Procedure of How to use the Python Table
----------------------------------------
:- Make a list of all the keys for the python dictionaries. The
keys which represent the individual columns in the row

:- Make a list of dictionaries for the titles. Each title
dictionary is a single column and this dictionary represents
the column header. The dictionary must contain the following
keys:
    :- text:     this is text for the column header
    :- width:    the integer for the width of the column
    :- type:     the type of the widget to use for holding the
        data in the column. The type is represented by a single
        letter in the string data type
        :'l' - for the Label widget
        :'c' - for the Combobox widget

:- Make a list of dictionaries, each dictionary representing the
row. Each row must be having all the keys specified in the list
of keys.

:- Initialize the class called 'Table' passing the neccessary arguments
:- Call 'add_rows()' passing the neccessary arguments

:- For deleting, select a row and call 'delete_row()'
:- You can also get the currently selected row by calling the
method 'get_selected()'
"""

import tkinter as tk
from threading import Thread
from tkinter import messagebox as msg, ttk

def _shade(wl_, color=None):
    """ Shading a given row when selection occurs with the
    given color.
    :type wl_: list
    :param wl_: list of widgets on the row
    :type color: str
    :param color: color to be used. If it is None, then,
        shading occurs.
    :return: None
    """
    if color is None:
        color = ''
    for _w in wl_:
        if isinstance(_w, type(ttk.Label())):
            _w['background'] = color

def _sep_work(cont, lb_list):
    """ Create and Situate separators on a given container between
    widgets
    :type lb_list: list
    :param lb_list: list of widgets to be separated using the
        separators
    :type cont: any container; e.g Frame, LabelFrame, etc
    :param cont: container widget
    :return: None
    """
    _col = 0                            # First column is 0
    # For situating the separators on the container
    for _c in range(len(lb_list) + 1):
        if _c > 0:
            _col = _col + 2
        sep = ttk.Separator(cont, orient='vertical')
        sep.grid(column=_col, row=1, sticky='NS')

        if _c < len(lb_list):
            lb_list[_c].grid(column=(_col + 1), row=1, sticky='WE',
                             padx=2, pady=2)

def check_rows(rows_list, titles, _keys_):
    """ Checking the list of rows to see if they contain any rows.
    If the list is empty, then a row is created to show that
    nothing is found in the list of rows.
    :type rows_list: list
    :param rows_list: list of rows in the dictionary form to
        be put in the table
    :type titles: list
    :param titles: list of dictionaries for the titles which
        represent the column headers.
    :type _keys_: list
    :param _keys_: list of strings for the keys of the titles.
    :return: list - of dictionaries for the rows
    """
    if len(rows_list) == 0:
        no_rows = {}
        for _t in titles:
            if titles.index(_t) == 0:
                no_rows[_keys_[0]] = 'Nothing is Found!'
            else:
                no_rows[_keys_[titles.index(_t)]] = ''
        return [no_rows]
    return rows_list

class Table:
    """ Creates the table form as specified. A container e.g Frame,
    is passed at initialization. Then a method is called to create
    the table.
    """

    def __init__(self, container, _keys_=None, titles=None, width=None,
                 height=None):
        """ Initializes the Table creation
        :type container: any, e.g ttk.Frame(), ttk.LabelFrame(), etc
        :param container: the container to hold the table structure
        :type _keys_: list
        :param _keys_: list of strings for the keys of the titles
        :type titles: list
        :param titles: list of dictionaries for the column headers.
            The headers with their widths and the types of the data
            for the columns, e.g c for the Combobox, l for the Label
        :type width: int
        :param width: integer for the width of the table
        :type height: int
        :param height: integer for the height of the Table
        :return: None
        """
        self.host = ttk.Frame(master=container)
        self.title_pane = ttk.Frame(self.host)

        # the main canvas that is scrollable
        self.list_canvas = tk.Canvas(self.host)
        self._keys_ = _keys_

        # if the keys list id None, then a mock is created
        if self._keys_ is None:
            self._keys_ = ['col1', 'col2', 'col3', 'col4']
        self._width = width
        if self._width is None:
            self._width = 200
        self._height = height
        if self._height is None:
            self._height = 300

        self.titles = titles
        # if the titles list is None, then a mock is created
        if self.titles is None:
            self.titles = [{'text': 'Column 1', 'width': 15, 'type': 'l'},
                           {'text': 'Column 2', 'width': 15, 'type': 'l'},
                           {'text': 'Column 3', 'width': 15, 'type': 'l'},
                           {'text': 'Column 4', 'width': 15, 'type': 'l'}]

        # instance variables
        self.sel_ind = None
        self.col_span = None
        self.rows_list = None
        self.selected_w = None
        self.selected_row = None
        self.mock_rows = []

        # set up mock data in case the table is being run in the
        # stand alone mode
        self.work_on_mock()
        self.rows = []

        # create the main table which involves making the
        # column headers, fixing the width of the columns and the
        # height of the table
        self._create()

    def work_on_mock(self):
        """
        Filling the mock rows with mock data
        :return: None
        """
        for i in range(30):
            self.mock_rows.append({
                    'col1': 'value of col 1 row ' + str(i + 1),
                    'col2': 'value of col 2 row ' + str(i + 1),
                    'col3': 'value of col 3 row ' + str(i + 1),
                    'col4': 'value of col 4 row ' + str(i + 1)})

    def _create(self):
        """ Creates the Table """

        # situating the host on the master
        self.host.grid(column=0, row=0, sticky='NSE')

        # situating the title_pane on the host/container
        self.title_pane.grid(column=0, row=0)
        cols = len(self.titles)
        self.col_span = (cols * 2) + 2

        # adding a separator at the top of the title pane
        titles_top = ttk.Separator(self.title_pane, orient='horizontal')
        titles_top.grid(column=0, row=0, sticky='WE', columnspan=self.col_span)

        # working on the titles
        self._titles_works()

        # adding 2 separators at the bottom of the title pane
        for i in range(2, 4):
            t_btm = ttk.Separator(self.title_pane, orient='horizontal')
            t_btm.grid(column=0, row=i, sticky='WE', columnspan=self.col_span)

        # scroll bars for te vertical and horizontal
        v_scr = tk.Scrollbar(self.host, orient='vertical')

        # situating the canvas and setting it up to be scrollable
        self.list_canvas.grid(column=0, row=1, sticky='WENS',
                              columnspan=(self.col_span - 1))
        self.list_canvas.configure(yscrollcommand=v_scr.set,
                                   width=self._width,
                                   height=self._height)

        v_scr.grid(column=(self.col_span - 1), row=1, sticky='NS', rowspan=3)

        v_scr['command'] = self.list_canvas.yview
        self._mouse_wheel([self.list_canvas, self.host])

    def _mouse_wheel(self, widgets):
        """ Performing the scrolling using the Mouse Wheel
        :type widgets: list
        :param widgets: list of widgets that make up the row
        :return: None
        """

        def _wheel(event):
            """ Movement of the wheel """
            move = int(event.delta / 60)
            self.list_canvas.yview_scroll(move, tk.UNITS)

        for widget in widgets:
            widget.bind('<MouseWheel>', _wheel, True)

    def _titles_works(self):
        """ Positions the titles as column headers and sets the
        column widths
        :return: None
        """
        lb_list = [ttk.Label(self.title_pane, text='S/N', width=5)]

        for title in self.titles:
            _lb = title['text']
            _lb = ttk.Label(self.title_pane, text=title['text'],
                            width=title['width'])
            lb_list.append(_lb)

        _sep_work(cont=self.title_pane, lb_list=lb_list)
        return True

    def add_rows(self, rows_list=None):
        """ Add given rows on to the canvas of the table
        :type rows_list: list
        :param rows_list: list of rows[dictionaries] to be placed on
            the canvas
        :return: None
        """
        self.rows_list = rows_list
        if self.rows_list is None:
            self.rows_list = self.mock_rows

        self.selected_row = None
        self.selected_w = None

        def des():
            for _w in self.list_canvas.winfo_children():
                _w.destroy()
                del _w
        Thread(target=des(), daemon=True).start()

        num_rows = len(self.rows_list)
        scr_v = num_rows * 26

        self.list_canvas['scrollregion'] = (0, 0, 0, scr_v)

        def works():

            y_cord = 0
            for i in range(len(self.rows_list)):
                _llb = 'frame' + str(i)
                _llb = ttk.Frame(self.list_canvas)

                lb_list = []
                lb_sn = ttk.Label(_llb, text=str(i + 1), width=5)
                lb_list.append(lb_sn)
                self._make_row_widgets(_llb, lb_list, i)

                _sep_work(_llb, lb_list)

                sep11 = ttk.Separator(_llb, orient='horizontal')
                sep11.grid(column=0, row=2, sticky='WE', columnspan=self.col_span)

                if i > 0:
                    y_cord = y_cord + 26

                self.list_canvas.create_window(-2, y_cord, window=_llb,
                                               anchor=tk.NW)

                for _ww in _llb.winfo_children():
                    _ww.bind('<ButtonRelease-1>', self._click, True)
                    self._mouse_wheel([_ww])
                self.selected_w = None

        Thread(target=works(), daemon=True).start()

    def _make_row_widgets(self, _llb, lb_list, _i):
        """ Make the widgets for the row. This is done using the keys
        list and the titles dictionaries
        :tyoe _llb: Label
        :param _llb: label, the row main widget
        :type lb_list: list
        :param lb_list: list of widgets. This is where the manufactured
            widgets are put
        :type _i: int
        :param _i: integer for the position or row number being worked
            upon
        :return: None
        """
        for key in self._keys_:
            _text = self.rows_list[_i][key]
            _type = self.titles[self._keys_.index(key)]['type']
            _width = self.titles[self._keys_.index(key)]['width']
            _w = None
            if self.titles[self._keys_.index(key)]['type'] == 'l':
                _w = ttk.Label(_llb, text=_text, width=_width)
            elif _type == 'c':
                _w = ttk.Combobox(_llb, values=_text, width=(_width - 3),
                                  state='readonly')
                if len(_text) > 0:
                    _w.current(0)
            elif _type == 'e':
                _v = tk.StringVar(value=_text)
                _w = ttk.Entry(_llb, textvariable=_v, width=_width,
                               state='readonly')

            lb_list.append(_w)

    def _click(self, event=None):
        """ Performing the Clicking event on a given row
        :type event: event
        :param event: event of button clicking
        :return: None
        """
        parent_name = event.widget.winfo_parent().split('.')

        self.sel_ind = None
        self.selected_row = None

        counts = 0
        _children = self.list_canvas.winfo_children()
        if self.selected_w is not None:
            try:
                self.selected_w.configure(relief='', borderwidth=0)
                _shade(self.selected_w.winfo_children())
                self.selected_w = None
            except Exception:
                pass

        for win_ in _children:
            w_name = win_.winfo_name()
            if parent_name[len(parent_name)-1] == w_name:
                self._select(win_)
                self.sel_ind = counts

            else:
                if counts == len(_children):
                    self.selected_w = None
            counts = counts + 1

    def _select(self, widget):
        """ Selecting the entire row of widgets that make up the row
        of the table. This is the row that has been clicked.
        This is also aimed at deselecting the previous row before
        before the new one is selected
        :type widget: Frame
        :param widget: frame containing the row widgets. The parent
            for the row
        :return: None
        """

        _wch = widget.winfo_children()
        selected_txt = _wch[1]['text']
        if selected_txt != 'Nothing is Found!':
            self.selected_w = widget
            self.selected_w.configure(relief='sunken', borderwidth=1)
            for row in self.rows_list:
                if selected_txt == row[self._keys_[0]]:
                    self.selected_row = row
            _shade(_wch, 'grey')
        else:
            self.selected_row = None
            self.selected_w = None

    def delete_row(self):
        """ Delete a selected row
        :return: None or dict - Previously Selected row
        """
        if self.selected_row is not None and \
            msg.askquestion('Itory: Deletion Confirmation',
                            'Confirm Deletion?') == u'yes':
                self.selected_w.destroy()
                self.rows_list.remove(self.selected_row)
                prev_selected = self.selected_row
                self.add_rows(check_rows(self.rows_list, self.titles,
                                         self._keys_))
                self._select_new_after_delete()
                return prev_selected
        return None

    def _select_new_after_delete(self):
        """ Perform the selection after the deleting action
        :return: None
        """
        counts = 0
        list_ = self.list_canvas.winfo_children()
        for win_ in list_:
            if counts == len(list_):
                self.selected_w = None
                self.selected_row = None
                self.sel_ind = None
                break
            if counts == self.sel_ind:
                self._select(win_)
                self.sel_ind = counts
                break
            counts = counts + 1

    def get_selected(self):
        """ Get the the text on the first widget of the selected row
        :return: dict row
        """
        return self.selected_row

if __name__ == '__main__':
    root = tk.Tk()
    table = Table(root)
    table.add_rows()

    root.mainloop()


from matplotlib import pyplot as plt
from pandasql import sqldf
import pandas as pd
import sqlite3

DB_PATH= r'database/spotify.db'
pysqldf = lambda q: sqldf(q, locals())

#################################################################################################
#                                                                                #
#                                         QUERIES                                               #
#                                                                                #
#################################################################################################

def query_total_entries_per_table(conn:sqlite3.Connection) -> pd.DataFrame:
    """
    Queries the total entries of users, streams, artists, releases and tracks.
    Parameters:
        conn (sqlite3.Connection): Databse connection
    Returns:
        result (pd.DataFrame): Total entries of users, streams, artists, releases and tracks.
    """
    return pd.read_sql_query("""
            SELECT  (
                        SELECT COUNT( DISTINCT user_name)
                        FROM stream
                    ) AS user_count,
                    (
                        SELECT COUNT(1)
                        FROM stream
                    ) AS stream_count,
                    (
                        SELECT COUNT(1)
                        FROM artist
                    ) AS artist_count,
                    (
                        SELECT COUNT(1)
                        FROM release
                    ) AS release_count,
                    (
                        SELECT COUNT(1)
                        FROM track
                    ) AS track_count""",conn)

def query_null_entry_statistic_on_columns(conn:sqlite3.Connection, table:str, columns:set) -> pd.DataFrame:
    """
    Queries the occurrence of null values in the column of a table in percent.
    Parameters:
        conn (sqlite3.Connection): Databse connection
        table (str): Name of the table, that gets queried.
        columns (set): Columns, to query at.
    Returns:
        result (pd.DataFrame):
    """
    frames = []
    for column in columns:
        null_statistic = pd.read_sql_query(f"""
                SELECT
                    ( printf( "%.3f" , 100.0*(COUNT(*)-COUNT({column}))/COUNT(*)) ) AS {column}
                FROM {table}; """, conn)

        frames.append( pd.melt( frame=null_statistic,
                                id_vars=None,
                                value_vars=null_statistic.columns,
                                value_name='null_values_in_percent',
                                var_name='column'))

        result = pd.concat(frames)
        result = result.set_index('column')
    return result

def query_active_users_over_weeks_and_weekdays_by_year(conn:sqlite3.Connection, year: str) -> pd.DataFrame:
    """
    Queries active users distributed over calendar weeks and weekdays based on a defined year.
    Parameters:
        conn (sqlite3.Connection): Databse connection
        year (str): The year for which the data is to be queried
    """
    return pd.read_sql_query(f"""
            SELECT   COUNT( DISTINCT lf.user_name) AS active_user, week, weekday, day_of_week, year
            FROM     stream lf
                        INNER JOIN time t on lf.timestamp_unix_id == t.timestamp_unix_id
            WHERE year = '{year}'
            GROUP BY year, week, day_of_week""", conn)

def query_streams_over_weekdays(conn:sqlite3.Connection) -> pd.DataFrame:
    """
    Queries the average streams over weekdays by year.
    Parameters:
        conn (sqlite3.Connection): Databse connection
    """
    stream_count_weekday_by_week = pd.read_sql_query(f"""
            SELECT year, week, weekday, COUNT(1) AS streams
            FROM stream lf
                    INNER JOIN time t ON lf.timestamp_unix_id == t.timestamp_unix_id
            GROUP BY year, week, weekday""", conn)

    return sqldf("""
            SELECT weekday, AVG(streams ) OVER(PARTITION BY weekday ) AS avg_streams, year
            FROM stream_count_weekday_by_week
            GROUP BY weekday""")

def query_entire_table(conn:sqlite3.Connection, table_name: str) -> pd.DataFrame:
    """
    Queries the entire table, based on table_name.
    Parameters:
        conn (sqlite3.Connection): Databse connection
        table_name (str): Name of the table, that gets queried.
    Returns:
        result (pd.DataFrame): Total entries of users, streams, artists, releases and tracks.
    """
    return pd.read_sql_query(f"SELECT  * from {table_name}",conn)

def weekly_trend_track(conn:sqlite3.Connection, top_n:str = "5") -> pd.DataFrame:
    """
    Queries the weekly top n ranked tracks and the positoin changes to prev week.
    Parameters:
        conn (sqlite3.Connection): Databse connection
        top_n (str): Amount of top tracks per week
    Returns:
        result (pd.DataFrame): dataframe with the query output
    """
    top_track_weekly_rank_table = pd.read_sql_query("""
	        SELECT   year, week, track_name, artist_name, lf.track_msid,
                     DENSE_RANK() OVER(PARTITION by year, week ORDER BY COUNT(lf.track_msid) DESC) as weekly_track_rank,
                     COUNT(lf.track_msid) AS weekly_track
            FROM    stream lf    INNER JOIN track tr ON lf.track_msid == tr.track_msid
                                        INNER JOIN time t   ON lf.timestamp_unix_id == t.timestamp_unix_id
                                        INNER JOIN artist a   ON lf.artist_msid == a.artist_msid
            GROUP BY  lf.track_msid, year, week
            ORDER BY  year, week, weekly_track DESC""", conn)

    top_track_prev_week_rank_table = sqldf("""
            SELECT    year, week, weekly_track_rank, weekly_track, track_name, artist_name,
                            LAG(weekly_track_rank) OVER (partition by track_msid ) AS rank_previous_week
            FROM      top_track_weekly_rank_table
            ORDER BY  year, week, weekly_track_rank""")

    return sqldf(f"""
            SELECT    year, week, weekly_track_rank, weekly_track, (track_name || ' ('|| artist_name|| ')') AS track_name ,
                      (rank_previous_week - weekly_track_rank) as weekly_change_of_ranking
            FROM      top_track_prev_week_rank_table
            WHERE     weekly_track_rank <= {top_n}
            ORDER BY  year DESC, week DESC, weekly_track_rank""")

def weekly_trend_artist(conn:sqlite3.Connection, top_n:str = "5") -> pd.DataFrame:
    """
    Queries the weekly top n ranked artists and the positoin changes to prev week.
    Parameters:
        conn (sqlite3.Connection): Databse connection
        top_n (str): Amount of top artists per week
    Returns:
        result (pd.DataFrame): dataframe with the query output
    """

    top_artist_weekly_rank_table = pd.read_sql_query("""
	        SELECT   year, week, artist_name, lf.artist_msid,
                     DENSE_RANK() OVER(PARTITION by year, week ORDER BY COUNT(lf.artist_msid) DESC) as weekly_artist_rank,
                     COUNT(lf.artist_msid) AS weekly_artist
            FROM     stream lf INNER JOIN artist a ON lf.artist_msid == a.artist_msid
                                INNER JOIN time t   ON lf.timestamp_unix_id == t.timestamp_unix_id

            GROUP BY lf.artist_msid, year, week
            ORDER BY year, week, weekly_artist DESC""", conn)

    top_artist_prev_week_rank_table = sqldf("""
            SELECT   year, week, weekly_artist_rank, weekly_artist, artist_name,
                     LAG(weekly_artist_rank) OVER (partition by artist_msid ) AS rank_previous_week
            FROM     top_artist_weekly_rank_table
            ORDER BY year, week, weekly_artist_rank""")

    return sqldf(f"""
            SELECT   year, week, weekly_artist_rank, weekly_artist, artist_name,
                     (rank_previous_week- weekly_artist_rank) as weekly_change_of_ranking
            FROM     top_artist_prev_week_rank_table
            where    weekly_artist_rank <= {top_n}
            ORDER BY year DESC, week DESC, weekly_artist_rank""")

def weekly_trend_release(conn:sqlite3.Connection, top_n:str = "5") -> pd.DataFrame:
    """
    Queries the weekly top n ranked release and the positoin changes to prev week.
    Parameters:
        conn (sqlite3.Connection): Databse connection
        top_n (str): Amount of top release per week
    Returns:
        result (pd.DataFrame): dataframe with the query output
    """
    top_release_rank_table = pd.read_sql_query("""
	        SELECT   year, week, release_name, artist_name, lf.release_msid,
                     DENSE_RANK() OVER(PARTITION by year, week ORDER BY COUNT(lf.release_msid) DESC) as weekly_release_rank,
                     COUNT(lf.release_msid) AS weekly_release
            FROM     stream lf    INNER JOIN release r ON lf.release_msid == r.release_msid
                                            INNER JOIN time t   ON lf.timestamp_unix_id == t.timestamp_unix_id
                                            INNER JOIN artist a   ON lf.artist_msid == a.artist_msid
            WHERE    lf.release_msid != "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
            GROUP BY lf.release_msid, year, week
            ORDER BY year, week, weekly_release DESC""", conn)

    top_release_prev_rank_table = sqldf("""
            SELECT    year, week, weekly_release_rank, weekly_release, release_name, artist_name,
                      LAG(weekly_release_rank) OVER (partition by release_msid ) AS rank_previous_week
            FROM      top_release_rank_table
            ORDER BY  year, week, weekly_release_rank""")

    return sqldf(f"""
            SELECT    year, week, weekly_release_rank, weekly_release, release_name, artist_name,
                      (rank_previous_week - weekly_release_rank) as weekly_change_of_ranking
            FROM      top_release_prev_rank_table
            where     weekly_release_rank <= {top_n}
            ORDER BY  year DESC, week DESC, weekly_release_rank""")

#################################################################################################
#                                                                                #
#                                         PLOTS                                                 #
#                                                                                #
#################################################################################################

def plot_active_users_over_weeks_and_weekdays_by_year(df:pd.DataFrame, year: str):
    """
    Plots a barchart on active users distributed over calendar weeks and weekdays based on a defined year.
    Parameters:
        df (pd.DataFrame): Dataframe from query_active_users_over_weeks_and_weekdays_by_year
        year (str): The year for which the plot is to be created.
    """
    DAY_ORDER = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday', 'Sunday']

    df['weekday'] = pd.Categorical(df['weekday'], categories=DAY_ORDER,ordered=True)
    df = df.pivot(index=["week"], columns="weekday",values="active_user")

    ax = df.plot(kind='bar',width=0.9, title='Active users distributed over calendar weeks and weekdays')
    ax.set_xlabel(f"Calendar Weeks {year}")
    ax.set_ylabel("Active User")
    plt.legend( title = "Weekdays")

def plot_streams_over_weekdays(df:pd.DataFrame) :
    """
    Plots the average streams over weekdays by year.
    Parameters:
        df (pd.DataFrame): Dataframe, provided by query_streams_over_weekdays
    """
    DAY_ORDER = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday', 'Sunday']

    df['weekday'] = pd.Categorical(df['weekday'], categories=DAY_ORDER,ordered=True)
    df = df.pivot(index="weekday",columns="year", values="avg_streams")

    ax = df.plot(kind='bar',width=0.8,rot=1, title='Average streams over weekdays by year')
    for bar in ax.patches:
        ax.annotate(format(bar.get_height(), '.0f'),
                       (bar.get_x() + bar.get_width() / 2,
                        bar.get_height()*0.9), ha='center', va='center',
                       xytext=(0, 8),textcoords='offset points')

    plt.xlabel('Weekdays')
    plt.ylabel('Average streams')
    plt.xticks(fontsize=8)
    plt.legend( title = "Year")

def plot_chart_table(df:pd.DataFrame,year:int,week:int, ax:plt.axes, subject:str ):
    """
    Combine weekly artist, release and song table plots into one figure.
    Parameters:
        conn (sqlite3.Connection): Databse connection.
        week (int): Week to get the charts
        year (int): Yearto get the charts
    """
    df = df.loc[(df['year']==year) & (df['week']==week)].copy()

    color_vals = df[[f"{subject.lower()}_name", 'weekly_change_of_ranking']].values
    colours = [['white' , 'blue' if row[1]!=row[1] else 'green' if float(row[1])>0 else 'red' if float(row[1])<0 else 'white' ] for row in color_vals]

    correct_rating = lambda x : " ".join((x, "△")) if (float(x) > 0 ) else " ".join((x, "▽")) if (float(x) < 0 ) else " ".join((x, "◁"))
    df["weekly_change_of_ranking"] = df["weekly_change_of_ranking"].astype(str)
    df["weekly_change_of_ranking"] = df["weekly_change_of_ranking"].apply(lambda x: x.replace('.0',''))
    df["weekly_change_of_ranking"] = df["weekly_change_of_ranking"].apply(correct_rating)
    vals = df[[f"{subject.lower()}_name", 'weekly_change_of_ranking']].values

    column_names = [f"{subject}", 'Changed position\ncompared to previous Week']

    ax.set_title(f"Spotify {subject} of the Week\nYear: {str(year)} Week: {str(week)}")
    the_table= ax.table(rowLabels=df[f"weekly_{subject.lower()}_rank"].values, colLabels=column_names,
                        colWidths = [0.25]*vals.shape[1], cellLoc='center',
                        loc='upper center', cellColours=colours, cellText=vals)
    the_table.set_fontsize(15)
    the_table.scale(1,1.5)

#################################################################################################
#                                                                                #
#                                         HELPER                                                #
#                                                                                #
#################################################################################################

def combine_chart_table_plots(conn:sqlite3.Connection, week:int, year:int):
    """
    Combine weekly artist, release and song table plots into one figure.
    Parameters:
        conn (sqlite3.Connection): Databse connection.
        week (int): Week to get the charts
        year (int): Yearto get the charts
    """
    fig, axes = plt.subplots(3, 1, figsize=(10,10))
    [axe.axis('off') for axe in axes]

    df= weekly_trend_artist(conn)
    plot_chart_table(ax=axes[0], subject="Artist", df=df, week=week, year=year)

    df= weekly_trend_track(conn)
    plot_chart_table(ax=axes[1], subject="Track", df=df, week=week, year=year)

    df= weekly_trend_release(conn)
    plot_chart_table(ax=axes[2], subject="Release", df=df, week=week, year=year)

def get_table_statistics(conn:sqlite3.Connection, table_name:str, columns:set) -> pd.DataFrame:
    """
    Combine (pandas) discribe with a null_value statistics based on a table and columns.
    Parameters:
        conn (sqlite3.Connection): Databse connection.
        table_name (str): Databse connection.
        columns (set): Databse connection.
    Returns:
        result (pd.DataFrame): Combination (pandas) discribe and null_value statistics.
    """
    table_null_values= query_null_entry_statistic_on_columns( conn,table=table_name, columns=columns)
    table_descr = query_entire_table(conn, table_name=table_name).describe(include='all')
    table_descr = table_descr[columns].transpose()
    table_descr = table_null_values.join(table_descr)
    table_descr['table'] = table_name
    return table_descr

def combine_table_statistics(conn:sqlite3.Connection) -> pd.DataFrame:
    """
    Combines statistic table data from track, release, time and stream into one df.
    Parameters:
        conn (sqlite3.Connection): Databse connection.
    Returns:
        result (pd.DataFrame): Combination of track, release, time and stream statistic df's.
    """
    #Track table
    track_columns = ['track_number','disc_number', 'track_duration_ms' ]
    track_statistics = get_table_statistics(conn,table_name='track',columns=track_columns)

    #Release table
    release_columns =['total_discs','total_tracks','release_date', 'release_name' ]
    release_statistics = get_table_statistics(conn,table_name='release',columns=release_columns)

    #time table
    time_columns =['day_of_week', 'day', 'week', 'month' , 'year']
    time_statistics = get_table_statistics(conn,table_name='time',columns=time_columns)

    #stream table
    stream_columns =['dedup_tag' ]
    stream_statistics = get_table_statistics(conn,table_name='stream',columns=stream_columns)

    return pd.concat([track_statistics, release_statistics, time_statistics, stream_statistics])

def main():
        conn= sqlite3.connect(DB_PATH)

        print("\n\n1. Overview total user, streams, artists, releases and tracks count")
        print(query_total_entries_per_table(conn))

        print("\n\n2. Statistic over table columns based on pandas describe extended with a custom Null-value percentage column")
        print(combine_table_statistics(conn))

        print("\n\n3. Displays active users, distributed over calendar weeks and weekdays for 2019 as barchart")
        print("See Figure1")
        year="2019"
        df = query_active_users_over_weeks_and_weekdays_by_year(conn,year=year)
        plot_active_users_over_weeks_and_weekdays_by_year(df, year=year)

        print("\n\n4. Displays the average streams over weekdays by year as barchart.")
        print("See Figure2")
        df = query_streams_over_weekdays(conn)
        plot_streams_over_weekdays(df)

        print("\n\n5. Displays weekly Spotify Release, Artist and Track Charts as plotted tables")
        print("See Figure3")
        combine_chart_table_plots(conn, week=12, year=2019)

        plt.show()
        conn.close()

if __name__ == '__main__':
    main()


from xgboost_tweak import *
from feature_engine import encoding, imputation
from sklearn import tree
from sklearn import pipeline
from sklearn import preprocessing
from sklearn import model_selection
from sklearn import dummy
import xgboost as xgb
import matplotlib.pyplot as plt

# reusable function to extract zip files
url = 'https://github.com/mattharrison/datasets/raw/master/data/kaggle-survey-2018.zip'

fname = 'kaggle-survey-2018.zip'
member_name = 'multipleChoiceResponses.csv'

## Create a pipeline
kag_pl = pipeline.Pipeline(
    [('tweak', TweakKagTransformer()),
     ('cat', encoding.OneHotEncoder(top_categories=5, drop_last=True,
           variables=['Q1', 'Q3', 'major'])),
     ('num_impute', imputation.MeanMedianImputer(imputation_method='median',
          variables=['education', 'years_exp']))]
    )

raw = extract_zip(url, fname, member_name)

# run the code
kag_X, kag_y = get_rawX_y(raw, 'Q6')

kag_X_train, kag_X_test, kag_y_train, kag_y_test = \
    model_selection.train_test_split(
        kag_X, kag_y, test_size=.3, random_state=42, stratify=kag_y
    )

X_train = kag_pl.fit_transform(kag_X_train, kag_y_train)
X_test = kag_pl.transform(kag_X_test)

stump_dt = tree.DecisionTreeClassifier(max_depth=1)
X_train = kag_pl.fit_transform(kag_X_train)
stump_dt.fit(X_train, kag_y_train)

fig, ax = plt.subplots(figsize=(8, 4))
features = list(c for c in X_train.columns)
tree.plot_tree(stump_dt, feature_names=features,
               filled=True,
               class_names=list(stump_dt.classes_),
               ax=ax)

X_test = kag_pl.transform(kag_X_test)

X_test

stump_dt.score(X_test, kag_y_test)

# dummy classifier for comparison
dummy_model = dummy.DummyClassifier()
dummy_model.fit(X_train, kag_y_train)

dummy_model.score(X_test, kag_y_test)
## 0.55, so quite a bit worse than our stump

# use xgboost with a single tree
kag_stump = xgb.XGBClassifier(n_estimators=1, max_depth=1)
kag_stump.fit(X_train, kag_y_train)
## throws error! need to handle labels

label_encoder = preprocessing.LabelEncoder()
y_train = label_encoder.fit_transform(kag_y_train)
y_test = label_encoder.transform(kag_y_test)

label_encoder.classes_
## 0 = Data Scientist, 1 = Software Engineer

kag_stump.fit(X_train, y_train)
kag_stump.score(X_test, y_test)
## .55, same as dummy classifier

my_dot_export(kag_stump, num_trees = 0, filename = 'img/xgb_kag_stump.dot', title = 'XGBoost stump')

fig, ax = plt.subplots(figsize=(8, 4))
tree.plot_tree(kag_stump, feature_names=features,
               filled=True,
               class_names=list(label_encoder.classes_),
               ax=ax)

# underfit decision tree
underfit = tree.DecisionTreeClassifier(max_depth=1)

X_train = kag_pl.fit_transform(kag_X_train)

underfit.fit(X_train, kag_y_train)

underfit.score(X_test, kag_y_test)

# overfit decision tree
hi_variance = tree.DecisionTreeClassifier(max_depth=None)

hi_variance.fit(X_train, kag_y_train)

hi_variance.score(X_test, kag_y_test)

fig, ax = plt.subplots(figsize=(8, 4))
features = list(c for c in X_train.columns)
tree.plot_tree(hi_variance, feature_names=features, filled=True)
tree.plot_tree(hi_variance, feature_names=features, filled=True,
                  class_names=list(hi_variance.classes_),
                  max_depth=2, fontsize=6)


import mxnet as mx
from mxnet import gluon, autograd, nd
import os

os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0'
import gluoncv as gcv
import argparse
from mxnet.gluon.data import DataLoader
import logging
import time
from gluoncv import data as gdata
from gluoncv import utils as gutils
from gluoncv.data import batchify
from gluoncv.data.transforms.presets.rcnn import FasterRCNNDefaultTrainTransform
from gluoncv.data.transforms.presets.rcnn import FasterRCNNDefaultValTransform
from gluoncv.utils.metrics.voc_detection import VOC07MApMetric
from gluoncv.utils.metrics.accuracy import Accuracy
from mx_Faster_rcnn import faster_rcnn_resnet50_v1b_voc

def parse_args():
    # create a parser object
    parser = argparse.ArgumentParser(description='This is a script train Faster-RCNN end2end.')
    parser.add_argument('--epochs', type=str, default='',
                        help='Training epochs.')
    parser.add_argument('--gpus', type=str, default='',
                        help='Training with GPUs, you can specify 1,3 for example.')
    parser.add_argument('--resume', type=str, default='',
                        help='Resume from previously saved parameters if not None.'
                             'For example,you can resume from ./faster_rcnn_xxx_0123,params')
    parser.add_argument('--start-epoch', type=int, default=0,
                        help='Starting epoch for resuming,default is 0 for new training')
    parser.add_argument('--lr', type=str, default='',
                        help='set learning rate for training')
    parser.add_argument('--momentum', type=float, default=0.9,
                        help='momentum for SGD training')
    parser.add_argument('--lr-decay', type=float, default=0.1,
                        help='learning rate decay rate')
    parser.add_argument('--lr-decay-epoch', type=str, default='',
                        help='epoches at which learning rate decays. default is 14,20 for voc.')
    parser.add_argument('--wd', type=str, default='',
                        help='Weight decay,default if 5e-4 for voc')
    parser.add_argument('--log-interval', type=int, default=100,
                        help='Logging mini-batch interval,Default is 100')
    parser.add_argument('--save-prefix', type=str, default='',
                        help='saving parameter prefix')
    parser.add_argument('--save-interval', type=int, default=1,
                        help='Saving parameters epoch interval,best model will always be saved. default is 1')
    parser.add_argument('--val-interval', type=int, default=1,
                        help='Validation model epoch interval.Default is 1')
    parser.add_argument('--seed', type=int, default=233,
                        help='Random seed to be fixed')
    parser.add_argument('--verbose', dest='verbose', action='store_true',
                        help='Print helpful debugging info once set')
    args = parser.parse_args()

    args.epochs = int(args.epochs) if args.epochs else 20
    args.lr_decay_epoch = args.lr_decay_epoch if args.lr_decay_epoch else '14,20'
    args.lr = float(args.lr) if args.lr else 0.001
    args.wd = float(args.wd) if args.wd else 5e-4

    return args

class RPNAccMetric(mx.metric.EvalMetric):
    def __init__(self, name='RPNAcc', **kwargs):
        super(RPNAccMetric, self).__init__(name, **kwargs)

    def update(self, labels, preds):
        """Updates the internal evaluation result.

        Parameters
        ----------
        labels : list of `NDArray`  [rpn_label,rpn_weight]
            The labels of the data.

        preds : list of `NDArray`   [rpn_cls_logits]
            Predicted values.
        """
        rpn_label, rpn_weight = labels
        rpn_cls_logits = preds[0]

        num_inst = nd.sum(rpn_weight)
        pred_label = mx.nd.sigmoid(rpn_cls_logits) >= 0.5
        num_acc = mx.nd.sum((pred_label == rpn_label) * rpn_weight)
        # calculate acc
        self.sum_metric += num_acc.asscalar()
        self.num_inst += num_inst.asscalar()

class RPNL1LossMetric(mx.metric.EvalMetric):
    def __init__(self, name='RPNL1Loss', **kwargs):
        super(RPNL1LossMetric, self).__init__(name, **kwargs)

    def update(self, labels, preds):
        """Updates the internal evaluation result.

        Parameters
        ----------
        labels : list of `NDArray`   [rpn_box_target,rpn_box_weight]
            The labels of the data.

        preds : list of `NDArray`   [rpn_box_reg]
            Predicted values.
        """
        rpn_box_target, rpn_box_weight = labels
        rpn_box_reg = preds[0]

        num_inst = nd.sum(rpn_box_weight) / 4

        loss = mx.nd.sum(rpn_box_weight * mx.nd.smooth_l1(rpn_box_reg - rpn_box_target, scalar=3))
        # calculate L1Loss
        self.sum_metric += loss.asscalar()
        self.num_inst += num_inst.asscalar()

class RCNNAccMetric(mx.metric.EvalMetric):
    def __init__(self, name='RCNNAcc', axis=-1, **kwargs):
        super(RCNNAccMetric, self).__init__(name, **kwargs)
        self._axis = axis

    def update(self, labels, preds):
        """Updates the internal evaluation result.

        Parameters
        ----------
        labels : list of `NDArray`   [rcnn_lable,rcnn_weight]
            The labels of the data.

        preds : list of `NDArray`   [rcnn_cls_logits]
            Predicted values.
        """
        rcnn_label, rcnn_weight = labels
        rcnn_cls_logits = preds[0]

        num_inst = nd.sum(rcnn_weight)

        pred_label = nd.argmax(rcnn_cls_logits, axis=self._axis)

        self.sum_metric += ((pred_label == rcnn_label) * rcnn_weight).sum().asscalar()
        self.num_inst += num_inst.asscalar()

class RCNNL1LossMetric(mx.metric.EvalMetric):
    def __init__(self, name='RCNNL1Loss', **kwargs):
        super(RCNNL1LossMetric, self).__init__(name, **kwargs)

    def update(self, labels, preds):
        """Updates the internal evaluation result.

        Parameters
        ----------
        labels : list of `NDArray`   [rcnn_box_target,rcnn_box_weight]
            The labels of the data.

        preds : list of `NDArray`   [rcnn_box_reg]
            Predicted values.
        """
        rcnn_box_target, rcnn_box_weight = labels
        rcnn_box_reg = preds[0]

        num_inst = nd.sum(rcnn_box_weight) / 4

        self.sum_metric += (nd.smooth_l1(rcnn_box_reg - rcnn_box_target, scalar=1) * rcnn_box_weight).sum().asscalar()
        self.num_inst += num_inst.asscalar()

def split_and_load(batch, ctx_list):
    """Split data to 1 batch each device."""
    num_ctx = len(ctx_list)
    new_batch = []
    for i, data in enumerate(batch):
        new_data = [x.as_in_context(ctx) for x, ctx in zip(data, ctx_list)]
        new_batch.append(new_data)
    return new_batch

def get_data_set(train_root='./VOCData/train_val/VOCdevkit', test_root='./VOCData/test/VOCdevkit'):
    """
       get PASCAL-VOC train & val & test dataset
    :return:
        train_dataset :

        val_dataset :
            use test data as valid dataset
    """
    train_dataset = gdata.VOCDetection(root=train_root, splits=[(2007, 'trainval')])
    val_dataset = gdata.VOCDetection(root=test_root, splits=[(2007, 'test')])
    val_metric = VOC07MApMetric(iou_thresh=0.5, class_names=val_dataset.classes)

    return (train_dataset, val_dataset, val_metric)

def get_data_loader(net, train_dataset, val_dataset, batch_size=1):
    """
        get PASCAL-VOC train&val dataloader

    :return:
    """
    train_bfn = batchify.Tuple(*[batchify.Append() for _ in range(5)])

    train_loader = mx.gluon.data.DataLoader(
        train_dataset.transform(FasterRCNNDefaultTrainTransform(net.short, net.max_size, net)),
        batch_size=1, shuffle=True, batchify_fn=train_bfn, last_batch='rollover')
    val_bfn = batchify.Tuple(*[batchify.Append() for _ in range(3)])
    val_loader = mx.gluon.data.DataLoader(
        val_dataset.transform(FasterRCNNDefaultValTransform(net.short, net.max_size)),
        batch_size, False, batchify_fn=val_bfn, last_batch='keep')
    return (train_loader, val_loader)

def validate(net, val_data, ctx, eval_metric):
    """Test on validation dataset."""
    clipper = gcv.nn.bbox.BBoxClipToImage()
    eval_metric.reset()
    for batch in val_data:
        batch = split_and_load(batch, ctx_list=ctx)
        det_bboxes = []
        det_ids = []
        det_scores = []
        gt_bboxes = []
        gt_ids = []
        gt_difficults = []
        for x, y, im_scale in zip(*batch):
            # get prediction results
            ids, scores, bboxes = net(x)
            det_ids.append(ids)
            det_scores.append(scores)
            # clip to image size
            det_bboxes.append(clipper(bboxes, x))
            # rescale to original resolution
            im_scale = im_scale.reshape((-1)).asscalar()
            det_bboxes[-1] *= im_scale
            # split ground truths
            gt_ids.append(y.slice_axis(axis=-1, begin=4, end=5))
            gt_bboxes.append(y.slice_axis(axis=-1, begin=0, end=4))
            gt_bboxes[-1] *= im_scale
            gt_difficults.append(y.slice_axis(axis=-1, begin=5, end=6) if y.shape[-1] > 5 else None)

        # update metric
        for det_bbox, det_id, det_score, gt_bbox, gt_id, gt_diff in zip(det_bboxes, det_ids, det_scores, gt_bboxes,
                                                                        gt_ids, gt_difficults):
            eval_metric.update(det_bbox, det_id, det_score, gt_bbox, gt_id, gt_diff)
    return eval_metric.get()

def save_params(net, logger, best_map, current_map, epoch, save_interval, prefix):
    current_map = float(current_map)
    if current_map > best_map[0]:
        logger.info('[Epoch {}] mAP {} higher than current best {} saving to {}'.format(
            epoch, current_map, best_map, '{:s}_best.params'.format(prefix)))
        best_map[0] = current_map
        net.save_parameters('{:s}_best.params'.format(prefix))
        with open(prefix + '_best_map.log', 'a') as f:
            f.write('{:04d}:\t{:.4f}\n'.format(epoch, current_map))
    if save_interval and (epoch + 1) % save_interval == 0:
        logger.info('[Epoch {}] Saving parameters to {}'.format(
            epoch, '{:s}_{:04d}_{:.4f}.params').format(prefix.epoch, current_map))
        net.save_parameters('{:s}_{:04d}_{:.4f}.params'.format(prefix, epoch, current_map))

def train(net, train_data, val_data, eval_metric, ctx, args):
    """Training pipeline"""
    net.collect_params().setattr('grad_req', 'null')
    net.collect_train_params().setattr('grad_req', 'write')
    """get trainer"""
    trainer = gluon.Trainer(
        net.collect_train_params(),  # fix batchnorm, fix first stage, etc...
        'sgd',
        {'learning_rate': args.lr,
         'wd': args.wd,
         'momentum': args.momentum,
         'clip_gradient': 5})

    rpn_cls_loss = gluon.loss.SigmoidBinaryCrossEntropyLoss(from_sigmoid=False)
    rpn_l1_loss = gluon.loss.HuberLoss(rho=1 / 9)
    rcnn_cls_loss = gluon.loss.SoftmaxCrossEntropyLoss()
    rcnn_l1_loss = gluon.loss.HuberLoss()

    # use to store loss during train
    metrics1 = [mx.metric.Loss('RPN_Conf'),
                mx.metric.Loss('RPN_SmoothL1'),
                mx.metric.Loss('RCNN_CrossEntropy'),
                mx.metric.Loss('RCNN_SmoothL1')]
    # use to calculate acc during train
    rpn_acc_metric = RPNAccMetric()
    rpn_bbox_metric = RPNL1LossMetric()
    rcnn_acc_metric = RCNNAccMetric()
    rcnn_bbox_metric = RCNNL1LossMetric()
    metrics2 = [rpn_acc_metric, rpn_bbox_metric, rcnn_acc_metric, rcnn_bbox_metric]

    lr_steps = sorted([float(ls) for ls in args.lr_decay_epoch.split(',') if ls.strip()])
    lr_decay = float(args.lr_decay)

    # set logging
    logging.basicConfig(level=logging.NOTSET)
    # create a logger object
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    # create a file handler
    log_file_path = args.save_prefix + '_train.log'
    log_dir = os.path.dirname(log_file_path)
    if log_dir and not os.path.exists(log_dir):
        os.mkdir(log_dir)
    fh = logging.FileHandler(log_file_path)
    logger.addHandler(fh)
    logger.info(args)
    if args.verbose:
        logger.info('Trainabel parameters:')
        logger.info(net.collect_train_params().keys())
    logger.info('Start training from [Epoch {}]'.format(args.start_epoch))
    best_map = [0]

    for epoch in range(args.start_epoch, args.epochs):
        btm = time.time()
        ttm = time.time()
        if lr_steps and epoch >= lr_steps[0]:
            current_lr = trainer.learning_rate
            lr_steps.pop(0)
            new_lr = lr_decay * current_lr
            trainer.set_learning_rate(new_lr)
            logger.info('[Epoch {}] Set learning rate to {}'.format(epoch, new_lr))
        """reset metrics"""
        for metric in metrics1:
            metric.reset()

        """get mini-batch traindata"""
        for i, batch in enumerate(train_data):
            batch = split_and_load(batch, ctx_list=ctx)
            batch_size = len(batch[0])
            losses = []
            metric_losses = [[] for _ in metrics1]
            add_losses = [[] for _ in metrics2]
            with autograd.record():
                for data, label, rpn_cls_targets, rpn_box_targets, rpn_box_masks in zip(*batch):
                    gt_label = label[:, :, 4:5]
                    gt_box = label[:, :, :4]
                    # net forward
                    rcnn_cls_preds, rcnn_box_preds, roi, samples, matches, rpn_score, rpn_box, anchors = net(data,
                                                                                gt_box)

                    # generate targets for rcnn
                    rcnn_cls_target, rcnn_box_target, rcnn_box_masks = net.target_generator(roi, samples, matches,
                                                                                gt_label, gt_box)

                    # calculate loss
                    # rpn loss
                    rpn_score = rpn_score.squeeze(axis=-1)
                    num_rpn_pos = (rpn_cls_targets >= 0).sum()
                    rpn_loss1 = rpn_cls_loss(rpn_score, rpn_cls_targets,
                                             (rpn_cls_targets >= 0)) * rpn_cls_targets.size / num_rpn_pos
                    rpn_loss2 = rpn_l1_loss(rpn_box, rpn_box_targets, rpn_box_masks) * rpn_box.size / num_rpn_pos

                    # rcnn loss
                    num_rcnn_pos = (rcnn_cls_target >= 0).sum()
                    rcnn_loss1 = rcnn_cls_loss(rcnn_cls_preds, rcnn_cls_target,
                                               (rcnn_cls_target >= 0)) * rcnn_cls_target.size / num_rcnn_pos
                    rcnn_loss2 = rcnn_l1_loss(rcnn_box_preds, rcnn_box_target,
                                              rcnn_box_masks) * rcnn_box_masks.size / num_rcnn_pos

                    rpn_loss = rpn_loss1 + rpn_loss2
                    rcnn_loss = rcnn_loss1 + rcnn_loss2

                    # autograd.backward([rpn_loss+rcnn_loss])
                    # record metrics
                    losses.append(rpn_loss.sum() + rcnn_loss.sum())
                    metric_losses[0].append(rpn_loss1.sum())
                    metric_losses[1].append(rpn_loss2.sum())
                    metric_losses[2].append(rcnn_loss1.sum())
                    metric_losses[3].append(rcnn_loss2.sum())

                    add_losses[0].append([[rpn_cls_targets, rpn_cls_targets >= 0], [rpn_score]])
                    add_losses[1].append([[rpn_box_targets, rpn_box_masks], [rpn_box]])
                    add_losses[2].append([[rcnn_cls_target, rcnn_cls_target >= 0], [rcnn_cls_preds]])
                    add_losses[3].append([[rcnn_box_target, rcnn_box_masks], [rcnn_box_preds]])
                # backward
                autograd.backward(losses)
                # update matrics
                for metric, record in zip(metrics1, metric_losses):
                    metric.update(0, record)
                for metric, records in zip(metrics2, add_losses):
                    for pred in records:
                        metric.update(pred[0], pred[1])
            # optimize
            trainer.step(batch_size=batch_size)
            if args.log_interval and not (i + 1) % args.log_interval:
                msg = ','.join(['{}={:3f}'.format(*metric.get()) for metric in metrics1 + metrics2])
                # logging
                log = '[Epoch {}][Batch {}], Speed: {:.3f} samples/sec , {}'.format(
                    epoch, i, args.log_interval * batch_size / (time.time() - btm), msg)
                logger.info(log)
                btm = time.time()
        # end of one epoch
        # logging
        msg = ','.join(['{}={.3f}'.format(*metric.get()) for metric in metrics1])
        log = '[Epoch {}] Training cost :{:.3f},{}'.format(
            epoch, (time.time() - ttm), msg)
        logger.info(log)

        if not (epoch + 1) % args.val_interval:
            # validation the model
            # TODO(Qinghong Zeng)
            map_name, mean_ap = validate(net, val_data, ctx, eval_metric)
            val_msg = '\n'.join(['{}={}'.format(k, v) for k, v in zip(map_name, mean_ap)])
            logger.info('[Epoch {}] Validation: \n{}'.format(epoch, val_msg))
            current_map = float(mean_ap[-1])
        else:
            current_map = 0.
        save_params(net, logger, best_map, current_map, epoch, args.save_interval, args.save_prefix)

if __name__ == "__main__":
    # parse arguments
    args = parse_args()
    gutils.random.seed(args.seed)

    # training contexts
    ctx = [mx.gpu(int(i)) for i in args.gpus.split(',') if i.strip()]
    ctx = ctx if ctx else [mx.cpu()]
    args.batch_size = len(ctx)  # 1 batch per device

    # training contexts
    net_name = 'faster_rccn_voc'
    args.save_prefix += net_name
    # network

    net = faster_rcnn_resnet50_v1b_voc(pretrained=False, pretrained_base=True)

    if args.resume.strip():
        net.load_parameters(filename=args.resume.strip())
    else:
        for param in net.collect_params().values():
            if param._data is not None:
                continue
            param.initialize()

    net.collect_params().reset_ctx(ctx)
    # train data
    train_dataset, val_dataset, val_metric = get_data_set()
    # train loader
    train_loader, test_loader = get_data_loader(net, train_dataset, val_dataset, batch_size=1)

    # train
    train(net, train_loader, test_loader, val_metric, ctx, args)


# Import libraries and packages
import config, main, bitfinex
import telegram

from telegram import * #Update, ForceReply, InlineKeyboardButton, InlineKeyboardMarkup, CallbackQuery
from telegram.ext import * #Updater, CommandHandler, MessageHandler, Filters

# Main bot function
button_list = [[InlineKeyboardButton("BTC/USD", callback_data='btcusd')],
                [InlineKeyboardButton("ETH/USD", callback_data='ethusd'),
                InlineKeyboardButton("ETP/USD", callback_data='etpeth'),
                InlineKeyboardButton("XMR/USD", callback_data='xmrusd')],
                [InlineKeyboardButton("MENU", callback_data='menu')]]
button_markup = InlineKeyboardMarkup(button_list)

menu_list = [[InlineKeyboardButton("HELP", callback_data='help'),
             InlineKeyboardButton("BACK", callback_data='back'),]]
menu_markup = InlineKeyboardMarkup(menu_list)
# Required start function
def start(bot, update):
    message = update.message
    chat_id = message.chat_id

    bot.send_message(chat_id = chat_id, text = "Hello", reply_markup=button_markup)

def callback(bot, update):
    call = update.callback_query
    message = call.message
    chat_id = message.chat_id

    if call.data == 'menu':
        bot.edit_message_text(chat_id = chat_id, message_id=message.message_id, text = 'MENU', reply_markup=menu_markup)
    elif call.data == 'back':
        bot.edit_message_text(chat_id = chat_id, message_id=message.message_id, text = 'PRICE - ___', reply_markup=button_markup)
    else:
        x = bitfinex.BitFinex(call.data)
        mid = x.mid()
        bot.edit_message_text(chat_id = chat_id, message_id=message.message_id, text = 'PRICE - '+mid, reply_markup=button_markup)
def crypto(bot, update):
    message = update.message
    chat_id = message.chat_id
    text = message.text

    x = bitfinex.BitFinex(text)
    mid = x.mid()
    bot.send_message(chat_id = chat_id, text = mid , reply_markup = crypto_markup)


# Discord
import discord
from discord.ext import commands
from discord import app_commands

# Other
import asyncio
from typing import Optional

# My import
from classes.discordbot import DiscordBot
from classes.utils import COLORS

class Info(commands.GroupCog, name="info", group_name="info", group_description="Pour gérer les votes !"):
    def __init__(self, bot: DiscordBot):
        self.bot = bot

    @app_commands.command(name="server", description="Les informations du serveur.")
    @app_commands.guild_only()
    async def server_info(self, interaction: discord.Interaction):
        await interaction.response.defer(thinking=True)
        await asyncio.sleep(1)

        server_name = interaction.guild.name

        nbr_members = interaction.guild.member_count

        nbr_roles = len(interaction.guild.roles) - 1

        channels = interaction.guild.channels
        nbr_txt_channels = 0
        nbr_voc_channels = 0
        for channel in channels:
            if isinstance(channel, discord.TextChannel):
                nbr_txt_channels += 1
            if isinstance(channel, discord.VoiceChannel):
                nbr_voc_channels += 1

        nbr_roles = len(interaction.guild.roles)

        owner = interaction.guild.owner.mention

        afk_channel = interaction.guild.afk_channel
        if isinstance(afk_channel, discord.VoiceChannel):
            afk_channel = afk_channel.mention
        else:
            afk_channel = '`non défini`'

        date_creation = interaction.guild.created_at

        url_server_image = interaction.guild.icon.url

        # Generate Embed
        dict_embed = {
            'title': f'Information Serveur',
            'description': f':white_small_square: Nom du serveur : **{server_name}**\n:white_small_square: **{nbr_members}** membres \n:white_small_square: **{nbr_txt_channels}** salons textuels \n:white_small_square: **{nbr_voc_channels}** salons vocaux \n:white_small_square: **{nbr_roles}** roles \n:white_small_square: {owner} est le propriétaire \n:white_small_square: Créé le **{date_creation.day}/{date_creation.month}/{date_creation.year}** \n:white_small_square: Salon AFK : {afk_channel}',
            'color': COLORS['white'],
            'thumbnail': {
                'url': url_server_image,
            }
        }
        embed = discord.Embed.from_dict(dict_embed)

        await interaction.followup.send(embed=embed)

    @app_commands.command(name="member", description="Les informations d'un membre.")
    @app_commands.describe(member='Les informations de qui ?')
    @app_commands.guild_only()
    async def member_info(self, interaction: discord.Interaction, member: Optional[discord.Member] = None):
        await interaction.response.defer(thinking=True)
        await asyncio.sleep(1)

        if member == None:
            member = interaction.user

        avatar_url = member.avatar.url
        discriminator = member.discriminator
        if discriminator == 0:
            discriminator = '`aucun`'
        else:
            discriminator = f'**{discriminator}**'

        flags = member.public_flags.all()
        badge = ''
        if len(flags) > 0:
            badge = ""
            for flag in flags:
                name = str(flag).split('.')[1]
                badge += f'`{name}` '
        else:
            badge = '`aucun`'

        # Generate Embed
        dict_embed = {
            'title': 'Information Membre',
            'description': f':white_small_square: Nom du membre : **{member.mention}**\n:white_small_square: Tag : {discriminator} \n\n:white_small_square: Badge(s) : {badge}',
            'color': COLORS['white'],
            'thumbnail': {
                'url': avatar_url,
            }
        }
        embed = discord.Embed.from_dict(dict_embed)

        await interaction.followup.send(embed=embed)

async def setup(bot: DiscordBot):
    await bot.add_cog(Info(bot))


from heapq import heappop
from heapq import heappush

def solution(book_times):
    book_times = [[book_time[0].split(':'), book_time[1].split(':')] for book_time in book_times]
    books = []
    for book_time in book_times:
        book = []
        book.append(int(book_time[0][0]) * 60 + int(book_time[0][1]))
        book.append(int(book_time[1][0]) * 60 + int(book_time[1][1]))
        books.append(book)
    books.sort()

    first_book = books.pop(0)
    rooms = [first_book[1]]

    for book in books:
        fast_room = heappop(rooms)
        if fast_room + 10 <= book[0]:
            fast_room = book[1]
        else:
            heappush(rooms, book[1])
        heappush(rooms, fast_room)

    return len(rooms)


#
# @lc app=leetcode id=133 lang=python3
#
# [133] Clone Graph
#

# @lc code=start

# Definition for a Node.
class Node:
    def __init__(self, val = 0, neighbors = []):
        self.val = val
        self.neighbors = neighbors

class Solution:
    def cloneGraph(self, node):
        root = self.dfs(node, {})
        return root

    def dfs(self, node, d):
        if not node:
            return None
        if node in d:
            return d[node]
        node_copy = Node(node.val, [])
        d[node] = node_copy
        for item in node.neighbors:
            next_copy = self.dfs(item, d)
            if next_copy:
                node_copy.neighbors.append(next_copy)
        return node_copy
# @lc code=end


import time
import random

# standing posture generator

# GRIPE = Gestural Reduced Instruction Pose Expression
# PRI[S]PE = Postural Reduced Instruction [Standing] Pose Expression

directions = ['down', 'left', 'right', 'front', 'back', 'up']  # make sure 'down' remains only the first and 'up' remains only the last
# foot up and foot front are the same?

options = ['grab', 'torsion']  # grab is for hands, torsion is for the core/torso

lower_joints = ['right knee', 'right foot', 'right foot', 'left knee', 'left foot', 'left foot']  # feet cannot go down; punta di piedi?
core_joints = ['head']
left_upper_joints = ['head', 'left elbow', 'left hand', 'left hand', 'left hand']  # by head we mean the head top, no sacrum
right_upper_joints = ['head', 'right elbow', 'right hand', 'right hand', 'right hand']
# what about shoulders?

def RandomDirectionFor(joint_name):
    if 'foot' in joint_name or 'knee' in joint_name or 'elbow' in joint_name:
        return int(random.uniform(1, len(directions) - 0.001))  # feet cannot go down; elbow down does not make sense in most cases; hands down is the default, but could be useful in the sequence case: relax, go back to default, if previously not in default position
    if 'head' in joint_name:
        return int(random.uniform(0, len(directions) - 1.001))  # head up is the default, but could be useful in the sequence case: relax, go back to default, if previously not in default position
    return int(random.uniform(0, len(directions) - 0.001))

def DirectionName(direction):
    return directions[direction]

def JointName(joint_names, joint):
    return joint_names[joint]

def RandomJointWithDirection(joints):
    joint = int(random.uniform(0, len(joints) - 0.001))
    joint_name = JointName(joints, joint)
    direction = RandomDirectionFor(joint_name)
    direction_name = DirectionName(direction)
    return [joint, joint_name, direction, direction_name]

def PrintJointAndDirection(joint_and_direction):
    print(joint_and_direction[1] + '  >>>  ' + joint_and_direction[3])

def GenerateNewPosture(previous):
    random.seed(time.time())
    if previous == None:
        previous_lower_side = 'none'
    else:
        previous_lower_side = previous[0][1].partition(' ')[0]
    posture = []
    while True:
        lower = RandomJointWithDirection(lower_joints)
        if not previous_lower_side in lower[1]:  # alternate between left and right leg
            break
    posture.append(lower)
    while True:
        left = RandomJointWithDirection(left_upper_joints)
        if not left[2] == lower[2]:  # to avoid too many limbs in the same direction
            break
    posture.append(left)
    if True:
        while True:
            right = RandomJointWithDirection(right_upper_joints)
            if not right[1] == left[1] and not right[2] == lower[2]:  # to avoid duplicates
                break
        posture.append(right)
    for p in posture:
        PrintJointAndDirection(p)
    print('')
    return posture

def ChangeLower(previous):
    if previous == None:
        previous_lower_side = 'none'
    else:
        previous_lower_side = previous[0][1].partition(' ')[0]
    posture = previous
    while True:
        lower = RandomJointWithDirection(lower_joints)
        if not previous_lower_side in lower[1]:  # alternate between left and right leg
            break
    posture[0] = lower
    PrintJointAndDirection(lower)
    print('')
    return posture

def ChangeUpper(previous, change_left):
    posture = previous
    lower = previous[0]
    if True:  # change only one side
        posture = previous
        left = posture[1]
        right = posture[2]
        while True:
            if change_left:
                left = RandomJointWithDirection(left_upper_joints)
            else:
                right = RandomJointWithDirection(right_upper_joints)
            if not right[1] == left[1] and ((not right[2] == lower[2] and not change_left and not right[2] == previous[2][2]) or (not left[2] == previous[1][2] and change_left and not left[2] == lower[2])):  # to avoid duplicates
                break
        if change_left:
            posture[1] = left
        else:
            posture[2] = right
        if change_left:
            PrintJointAndDirection(left)
        else:
            PrintJointAndDirection(right)
    print('')
    return posture

last_pose_change = -1

def ChangeSomething(previous_posture):
    global last_pose_change
    last_pose_change = (last_pose_change + 1) % 3
    if last_pose_change == 0:
        return last_pose_change,ChangeLower(previous_posture)
    if last_pose_change == 1:
        return last_pose_change,ChangeUpper(previous_posture, True)
    return last_pose_change,ChangeUpper(previous_posture, False)

# mancano due gambe distese dritte verso terra


#!/usr/bin/python
import spidev
import time
import os
import RPi.GPIO as GPIO

GPIO.setmode(GPIO.BOARD)
GPIO.setwarnings(False)

# Open SPI bus
spi = spidev.SpiDev()
spi.open(0, 0)

# Define GPIO to motor
DC_motor_a = 7
DC_motor_b = 11

GPIO.setup(DC_motor_a, GPIO.OUT)
GPIO.setup(DC_motor_b, GPIO.OUT)

# Define sensor channels
temp_channel = 0

# Timing constants
E_PULSE = 0.0005
E_DELAY = 0.0005
delay = 1

# Function to read SPI data from MCP3008 chip
# Channel must be an integer 0-7
def ReadChannel(channel):
    adc = spi.xfer2([1, (8 + channel) << 4, 0])
    data = ((adc[1] & 3) << 8) + adc[2]
    return data

while (1):
    light_level = ReadChannel(temp_channel)
    if (light_level < 100):
        lcd_string("Store off", LCD_LINE_2)
        GPIO.output(DC_motor_a, True)
        GPIO.output(DC_motor_b, FALSE)
        time.sleep(1)
    else:
        GPIO.output(DC_motor_b, TRUE)
        GPIO.output(DC_motor_a, FALSE)
        time.sleep(1)


# XParty - A Framework for Building Tools to Support Social Learning in Synchronous Environments
# Authors: Ben Bederson - www.cs.umd.edu/~bederson
#          Alex Quinn - www.cs.umd.edu/~aq
#          Anne Rose - www.cs.umd.edu/hcil/members/~arose
#          University of Maryland, Human-Computer Interaction Lab - www.cs.umd.edu/hcil
# Date: Originally created July 2011
# License: Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0

from XPartyView import XPartyView
from server import model_access

class MainPage(XPartyView):
    def get(self):
        self.init_user_context()
        if  model_access.is_teacher_logged_in(self.user):
            self.redirect("/teacher_dashboard")

        elif model_access.is_student_logged_in(self.user):
            self.redirect("/student")

        else:
            self.write_response_with_template("index.html")


''' Method 1 - Topdown
f(i, j) = min path sum from (0, 0) to (i, j)

[Base case]
f(0, 0) = grid[i][j]
f(i < 0 or j < 0) = 0

[General case]
f(i, j) = grid[i][j] + min(f(i - 1, j), f(i, j - 1))
'''
class Solution:
    def minPathSum(self, grid: List[List[int]]) -> int:
        m, n = len(grid), len(grid[0])
        self.dp = [[-1 for _ in range(n)] for _ in range(m)]
        return self.topdown(grid, m - 1, n - 1)

    def topdown(self, grid, i, j):
        if i == 0 and j == 0:
            return grid[i][j]

        if i < 0 or j < 0:
            return inf

        if self.dp[i][j] != -1:
            return self.dp[i][j]

        self.dp[i][j] = grid[i][j] + min(self.topdown(grid, i - 1, j), self.topdown(grid, i, j - 1))

        return self.dp[i][j]

''' Method 2 - Bottom up
'''
class Solution:
    def minPathSum(self, grid: List[List[int]]) -> int:
        m, n = len(grid), len(grid[0])
        dp = [[inf for _ in range(n + 1)] for _ in range(m + 1)]
        dp[1][1] = grid[0][0]
        for i in range(1, m + 1):
            for j in range(1, n + 1):
                if i == 1 and j == 1:
                    continue
                dp[i][j] = grid[i - 1][j - 1] + min(dp[i - 1][j], dp[i][j - 1])

        return dp[m][n]

Dynamic Programming/#64. Minimum Path Sum.py


import csv
import os
import argparse
import librosa
import soundfile as sf
import tqdm

# Set Inputs
csvpath = r"/media/ali/G2/Data/pl_PL/by_book/male/piotr_nater/lalka/metadata.csv"
outputpath = "PolandTTS/Prepared"
outputfilename = "metadata.txt"
audiopath = "/media/ali/PrjPrgm/Projects/CoguiTTS/PolandTTS/Prepared/wavs"

# format the data and save it in the meta_data.txt for using Congui-TTS
def format_data():

     with open(os.path.join(outputpath, outputfilename), 'w') as fwrite:

        with open (csvpath) as fd:
            rd = csv.reader ( fd, delimiter="\t", quotechar='"' )
            idx = 0
            for row in rd:
                try:
                    audio_name = row[ 0 ].split ( '|' )[ 0 ]
                    transcript = row[ 0 ].split ( '|' )[ 1 ]

                    print ( f"Audio file name is {audio_name} &"
                            f" Its Transcription is {transcript}" )
                    if os.path.exists (os.path.join ( audiopath, audio_name ) +".wav"):
                        #audio, sr = sf.read ( os.path.join ( audiopath, audio_name )+".wav" )
                        #sf.write ( file=os.path.join ( outputpath, "wavs2",  audio_name )+".wav", data=audio, samplerate=22050 )
                        #if sr == 22050:
                        fwrite.write(audio_name)
                        fwrite.write("||")
                        fwrite.write ("<"+transcript+">" )
                        fwrite.write("\n")

                        idx += 1

                        if idx == 1000:
                             break
                        # else:
                        #     os.remove(os.path.join ( audiopath, audio_name ) +".wav")
                        #     print(os.path.join ( audiopath, audio_name ) +".wav" + "is removed")

                except:
                    continue
     pass

format_data()


# Turtle Graphic Game
import time
import turtle
import math
import random
import os
import simpleaudio as sa
import wave

# Sounds
bounce_sound = sa.WaveObject.from_wave_file('Bounce.wav')
collision_sound = sa.WaveObject.from_wave_file('Collision.wav')

# Set up screen
wn = turtle.getscreen()
wn.clear()
wn.bgcolor('lightgreen')
#wn.bgpic('TGGame.gif')
wn.tracer(3)

# Draw border
mypen = turtle.Turtle()
mypen.penup()
mypen.setposition(-330, -330)
mypen.pendown()
mypen.pensize(3)
for side in range(4):
    mypen.forward(660)
    mypen.left(90)
mypen.hideturtle()

# Create player Turtle
player = turtle.Turtle()
player.color('blue')
player.shape('circle')
player.shapesize(2, 2, 1)
player.penup()  # delete trace
player.speed(0)

# Create multiple goals
max_goals = 25
goals = []
for count in range(max_goals):
    goals.append(turtle.Turtle())
    goals[count].color('dark red')
    goals[count].shape('circle')
    goals[count].penup()  # delete trace
    goals[count].speed(2)
    goals[count].setposition(random.randint(-300, 300), random.randint(-300, 300))

# Set speed variable
speed = 1

# Create the score of the game
game_score = 0

# Define functions
def turn_left():
    player.left(30)

def turn_right():
    player.right(30)

def increase_speed():
    global speed
    speed += 1

def decrease_speed():
    global speed
    speed -= 1

def is_collision(t1, t2):
    distance = math.sqrt(math.pow(t1.xcor() - t2.xcor(), 2) + math.pow(t1.ycor() - t2.ycor(), 2))
    if distance < 20:
        return True
    else:
        return False

# Set keyboard binding
turtle.listen()
turtle.onkey(turn_left, "Left")
turtle.onkey(turn_right, "Right")
turtle.onkey(increase_speed, "Up")
turtle.onkey(decrease_speed, "Down")

while True:
    player.forward(speed)

    # Boundary Checking for player
    if player.xcor() > 330 or player.xcor() < -330:
        player.right(random.randint(160, 200))
        play_bounce = bounce_sound.play()
        play_bounce.stop()

    if player.ycor() > 330 or player.ycor() < -330:
        player.right(random.randint(160, 200))
        play_bounce = bounce_sound.play()
        play_bounce.stop()

    # Move the goal
    for count in range(max_goals):
        goals[count].forward(0.3)

        # Collision checking
        if is_collision(player, goals[count]):
            goals[count].setposition(random.randint(-300, 300), random.randint(-300, 300))
            goals[count].right(random.randint(0, 360))
            game_score += 1
            # Draw  the score on the screen
            mypen.undo()
            mypen.penup()
            mypen.hideturtle()
            mypen.setposition (-300, 330)
            score_string = 'Score: %s' %game_score
            mypen.write(score_string, False, align='left', font=('Arial', 14, 'normal'))
            play_collision = collision_sound.play()
            play_collision.stop()

        # Boundary Checking for goal
        if goals[count].xcor() > 320 or goals[count].xcor() < -320:
            goals[count].right(random.randint(110, 250))

        if goals[count].ycor() > 320 or goals[count].ycor() < -320:
            goals[count].right(random.randint(110, 250))