# -*- coding: utf-8 -*-
"""
Created on Fri Oct 11 14:40:05 2019

@author: Fede
"""
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import numpy as np
import matplotlib.pyplot as plt
import os

def plot_results(vector_mean,vector_mean1,train_episode,negative_r1,negative_r2,pos_r,pos_r2,null_r,loss_mean,title,LOSSES,success_episodes):
    bound=[]
    for i in range(int(train_episode/100)):
        bound.append(1)
    print (bound)
    episodes=[]
    for i in range(int(train_episode/10)):
        episodes.append(i*100)
    base=[]
    base1=[]
    base2=[]
    base3=[]
    base4=[]

    for i in range(int(len(negative_r1))):
        base.append(negative_r1[i]*100/(negative_r1[i]+negative_r2[i]+pos_r[i]+pos_r2[i]+null_r[i]))
        base1.append(negative_r2[i]*100/(negative_r1[i]+negative_r2[i]+pos_r[i]+pos_r2[i]+null_r[i]))
        base2.append(pos_r[i]*100/(negative_r1[i]+negative_r2[i]+pos_r[i]+pos_r2[i]+null_r[i]))
        base3.append(pos_r2[i]*100/(negative_r1[i]+negative_r2[i]+pos_r[i]+pos_r2[i]+null_r[i]))
        base4.append(null_r[i]*100/(negative_r1[i]+negative_r2[i]+pos_r[i]+pos_r2[i]+null_r[i]))

    fig = make_subplots(rows=2,cols=2,subplot_titles=("Rewards distributions","Times both drones reach their targets [%]","Mean reward per episode per step","Mean loss"))

    fig.add_trace(go.Bar(x=episodes, y=base,
                        marker_color='yellow',
                        name='Same position'),row=1,col=1)
    fig.add_trace(go.Bar(x=episodes, y=base2,
                        marker_color='darkgreen',
                        name='Optimal position'
                        ),row=1,col=1)
    fig.add_trace(go.Bar(x=episodes, y=base3,
                        marker_color='lightgreen',
                        name='Only 1 drone correct'
                        ),row=1,col=1)
    fig.add_trace(go.Bar(x=episodes, y=base1,
                        marker_color='crimson',
                        name='Outside map position'
                        ),row=1,col=1)
    fig.add_trace(go.Bar(x=episodes, y=base4,
                        marker_color='grey',
                        name='Null rewards'
                        ),row=1,col=1)

    min_v=[]
    max_v=[]
    mean_v=[]
    min_v1=[]
    max_v1=[]
    mean_v1=[]
    min_loss=[]
    max_loss=[]
    mean_loss=[]

    for i in range(len(vector_mean[0])):
        temp=[]
        temp1=[]
        for j in range(len(vector_mean)):
            temp.append(vector_mean[j][i])
            temp1.append(vector_mean1[j][i])
        min_v.append(min(temp))
        max_v.append(max(temp))
        mean_v.append(np.mean(temp))
        min_v1.append(np.mean(temp1)-min(temp1))
        max_v1.append(max(temp1)-np.mean(temp1))
        mean_v1.append(np.mean(temp1))
    for i in range(len(loss_mean[0])):
        temp_loss=[]
        for j in range(len(loss_mean)):
            temp_loss.append(loss_mean[j][i])
        min_loss.append(min(temp_loss))
        max_loss.append(max(temp_loss))
        mean_loss.append(np.mean(temp_loss))

    fig.add_trace(go.Bar(x=episodes, y=mean_v1,
                        marker_color='orange',error_y=dict(
                                            symmetric=False,
                                            array=max_v1,
                                            arrayminus=min_v1),
                        name='Perfect episodes [%]'
                        ),row=1,col=2)

    fig.add_trace(go.Scatter(y=np.asarray(min_v),line=dict(width=0),mode='lines',name='Min reward'),row=2,col=1)
    fig.add_trace(go.Scatter(y=np.asarray(mean_v),line=dict(color='rgb(31, 119, 180)'),
                             fillcolor='rgba(68, 68, 68, 0.3)',fill='tonexty',name='Mean reward'),row=2,col=1)
    fig.add_trace(go.Scatter(y=np.asarray(max_v),line=dict(width=0),fillcolor='rgba(68, 68, 68, 0.3)',
                             fill='tonexty',name='Upper reward'),row=2,col=1)
    fig.add_trace(go.Scatter(y=bound,line=go.scatter.Line(color="crimson"),name='Bound reward'),
                  row=2,col=1)

    fig.add_trace(go.Scatter(y=np.asarray(min_loss),line=dict(width=0),mode='lines',name='Min reward'),row=2,col=2)
    fig.add_trace(go.Scatter(y=np.asarray(mean_loss),line=dict(color='rgb(68, 0, 0)'),
                             fillcolor='rgba(68, 0, 0, 0.3)',fill='tonexty',name='Mean reward'),row=2,col=2)
    fig.add_trace(go.Scatter(y=np.asarray(max_loss),line=dict(width=0),fillcolor='rgba(68, 0, 0, 0.3)',
                             fill='tonexty',name='Upper reward'),row=2,col=2)

    fig.update_xaxes(tickfont=dict(size=30))
    fig.update_yaxes(tickfont=dict(size=30))
    fig.update_layout(barmode='relative',
                          title=go.layout.Title(
                          text="Results"),
                          title_font_size=30,
                          legend=go.layout.Legend(
                          font=dict(
                          family="sans-serif",
                          size=30,
                          color="black"
                          )),
                          width=1920,height=1080)

    fig.write_image('General_Results'+str(title)+'.pdf')

    fig = go.Figure(data=[
            go.Bar(x=episodes, y=base1,
                        marker_color='crimson',
                        name='Outside map position'),
            go.Bar(x=episodes, y=base,
                        marker_color='yellow',
                        name='Same position'),
            go.Bar(x=episodes, y=base4,
                        marker_color='grey',
                        name='Null rewards'
                        ),
            go.Bar(x=episodes, y=base3,
                        marker_color='lightgreen',
                        name='Only 1 drone correct'),
            go.Bar(x=episodes, y=base2,
                        marker_color='darkgreen',
                        name='Optimal position')
                   ])

    fig.update_layout(barmode='stack',
                          title=go.layout.Title(
                          text="Greedy rewards"),
                          yaxis=dict(title='Reward distributions (%)'),
                          xaxis=dict(title='Trained episodes',tick0 = 0,dtick = 50))
    fig.write_image('Greedy_rewards_'+str(title)+'.pdf')

    fig = go.Figure(data=[
            go.Bar(x=episodes, y=mean_v1,
                        marker_color='orange',error_y=dict(
                                            symmetric=False,
                                            array=max_v1,
                                            arrayminus=min_v1),
                        name='Perfect episodes [%]'
                        )])

    fig.update_layout(barmode='stack',
                          title=go.layout.Title(
                          text="Correct episodes"),
                          yaxis=dict(title='Correct episodes (%)'),
                          xaxis=dict(title='Trained episodes',tick0 = 0,dtick = 50))
    fig.write_image('Correct_episodes_'+str(title)+'.pdf')

    fig = go.Figure(data=[
            go.Scatter(y=np.asarray(min_loss),line=dict(width=0.5),mode='lines',name='Min reward'),
            go.Scatter(y=np.asarray(mean_loss),line=dict(color='rgb(68, 0, 0)'),
                             fillcolor='rgba(68, 0, 0, 0.3)',fill='tonexty',name='Mean reward'),
            go.Scatter(y=np.asarray(max_loss),line=dict(width=0.5),fillcolor='rgba(68, 0, 0, 0.3)',
                             fill='tonexty',name='Upper reward')])

    fig.update_layout(title=go.layout.Title(
                          text="Loss"),
                          yaxis=dict(title='MSE'),
                          xaxis=dict(title='Times NN trained',tick0 = 0,dtick = 50))
    fig.write_image('Loss_'+str(title)+'.pdf')

    fig = go.Figure(data=[
            go.Scatter(y=np.asarray(min_v),line=dict(width=0.5),mode='lines',name='Min reward'),
            go.Scatter(y=np.asarray(mean_v),line=dict(color='rgb(31, 119, 180)'),
                             fillcolor='rgba(68, 68, 68, 0.3)',fill='tonexty',name='Mean reward'),
            go.Scatter(y=np.asarray(max_v),line=dict(width=0.5),fillcolor='rgba(68, 68, 68, 0.3)',
                             fill='tonexty',name='Upper reward'),
            go.Scatter(y=bound,line=go.scatter.Line(color="crimson"),name='Bound reward'),
                  ])

    fig.update_layout(barmode='stack',
                          title=go.layout.Title(
                          text="Mean reward per episodes per step"),
                          yaxis=dict(title='Reward'),
                          xaxis=dict(title='Trained episodes',tick0 = 0,dtick = 50))
    fig.write_image('Mean_reward_'+str(title)+'.pdf')

def plot_memory_replay(SMR,GSMR,TMR,GTMR,Map,title):
    fig = make_subplots(rows=2,cols=2,
                        subplot_titles=("Pre-training","Available gaussians in the map in the initial MR","Available gaussians in the map in the end in the MR","Map heterogeneity"),
                        specs=[[{"type": "domain"}, {"type": "domain"}],
                               [{"type": "domain"},{"type": "heatmap"}]])
    labels=["Outside map","Same position","0 reward", "1 Correct","All correct"]
    labels1=["0 available gaussians","1 available gaussians","2 available gaussians"]
    total=[0,0,0]
    for i in range(len(GTMR)):
        total[0]+=GTMR[i][0]
        total[1]+=GTMR[i][1]
        total[2]+=GTMR[i][2]
    fig.add_trace(go.Pie(values=GSMR[0],labels=labels),row=1,col=1)
    fig.add_trace(go.Pie(values=GTMR[0],labels=labels1),row=1,col=2)
    fig.add_trace(go.Pie(values=total,labels=labels1),row=2,col=1)
    maximum=max(max(Map[0]),max(Map[1]),max(Map[2]),max(Map[3]),max(Map[4]),max(Map[5]),max(Map[6]))
    fig.add_trace(go.Heatmap(z=[Map[0],Map[1],Map[2],Map[3],Map[4],Map[5],Map[6]],zmax=0,zmid=maximum/2,zmin=maximum,reversescale=False,showscale=False,colorscale='jet'),row=2,col=2)
    fig.update_xaxes(tickfont=dict(size=30))
    fig.update_yaxes(tickfont=dict(size=30))
    fig.update_layout(barmode='stack',
                          title=go.layout.Title(
                          text="Dataset"),
                          title_font_size=30,
                          legend=go.layout.Legend(
                          font=dict(
                          family="sans-serif",
                          size=30,
                          color="black"
                          )),
                          width=1920,height=1080)

    fig.write_image('Dataset'+str(title)+'.pdf')

def plot_qvalues(VAR,MIN,MAX,MEAN,title):
    fig = make_subplots(rows=2,cols=2,
                        subplot_titles=("Mean qvalues variance",
                                        "Mean qvalues minimum",
                                        "Mean qvalues maximum",
                                        "Mean qvalues mean"),
                        specs=[[{"type": "xy"}, {"type": "xy"}],
                               [{"type": "xy"},{"type": "xy"}]])
    VAR1=[]
    MEAN1=[]
    MINIMUM=[]
    MAXIMUM=[]
    for i in range(len(VAR)):
        VAR1.append(VAR[i][0])
        MEAN1.append(MEAN[i][0])
        MINIMUM.append(MIN[i][0])
        MAXIMUM.append(MAX[i][0])
    fig.add_trace(go.Scatter(y=VAR1,line=go.scatter.Line(color="red"),name='Variance'),row=1,col=1)
    fig.add_trace(go.Scatter(y=MINIMUM,line=go.scatter.Line(color="orange"),name='Minimum'),row=1,col=2)
    fig.add_trace(go.Scatter(y=MAXIMUM,line=go.scatter.Line(color="green"),name='Maximum'),row=2,col=1)
    fig.add_trace(go.Scatter(y=MEAN1,line=go.scatter.Line(color="darkgreen"),name='Mean'),row=2,col=2)
    fig.update_xaxes(tickfont=dict(size=30))
    fig.update_yaxes(tickfont=dict(size=30))
    fig.update_layout(title=go.layout.Title(
                          text="Dataset"),
                          title_font_size=30,
                          legend=go.layout.Legend(
                          font=dict(
                          family="sans-serif",
                          size=30,
                          color="black"
                          )),
                          width=1920,height=1080)

    fig.write_image('Qvalues'+str(title)+'.pdf')

def plot_IAHOS(y,ogp,ogp2,tgp,tgp2,model):
    """fig = make_subplots(rows=2, cols=2,subplot_titles=("Mean train. accuracy first round",
                                                   "Mean valid accuracy first round",
                                                  "Mean train accuracy last round",
                                                  "Mean valid accuracy last round"))"""

    fig = go.Figure()
    x = np.linspace(0,len(tgp[0])-1,len(tgp[0]))
    Colorscale = [[0, '#FF0000'],[0.5, '#F1C40F'], [1, '#00FF00']]
    """fig.add_trace(go.Heatmap(y=[y[i] for i in range(len(y))],
                       x=[0,1],
                       z=ogp2, colorscale = Colorscale),row=1,col=1)
    fig.add_trace(go.Heatmap(y=[y[i] for i in range(len(y))],
                       x=[0,1],
                       z=ogp,colorscale=Colorscale),row=1,col=2)
    fig.add_trace(go.Heatmap(y=[y[i] for i in range(len(y))],
                       x=x,
                       z=tgp2, colorscale = Colorscale),row=2,col=1)"""
    fig.add_trace(go.Heatmap(y=[y[i] for i in range(len(y))],
                       x=x,
                       z=tgp,colorscale=Colorscale))
    fig.update_layout(height=600, width=800,title=dict(text='IAHOS results'))
    if not os.path.exists("images"):
        os.mkdir("images")
    fig.write_image('images/IAHOS_'+str(model)+'.pdf')

def plot_pretrain(performances,y,params,model):
    fig = make_subplots(rows=2, cols=2,subplot_titles=("All performances",
                                                   "Performances w.r.t. Gamma",
                                                  "Performances w.r.t. malus same position",
                                                  ""))
    indeces=[]
    for i in range(len(performances)):
        indeces.append(performances[i][-1])
    index1=np.argmax(indeces)
    index2=np.argmin(indeces)

    bound=[]
    for i in range(len(performances[0])):
        bound.append(1)
    for i in range(len(performances)):
        if i!=index1 and i!=index2:
            fig.add_trace(go.Scatter(y=performances[i],line=go.scatter.Line(color="grey"),showlegend=False),row=1,col=1)
    fig.add_trace(go.Scatter(y=performances[index1],line=go.scatter.Line(color="green",width=4),name='Best performance'),row=1,col=1)
    fig.add_trace(go.Scatter(y=performances[index2],line=go.scatter.Line(color="red",width=4),name='Worst performance'),row=1,col=1)
    fig.add_trace(go.Scatter(y=bound,line=go.scatter.Line(color="green",dash='dashdot',width=4),name='Bound'),row=1,col=1)
    fig.add_trace(go.Bar(x=params[0], y=y[0]),row=1,col=2)
    fig.add_trace(go.Bar(x=params[1], y=y[1]),row=2,col=1)
    #fig.add_trace(go.Bar(x=params[2], y=y[2]),row=2,col=2)

    fig.update_layout(title=go.layout.Title(text="Results"),title_font_size=30,width=1920,height=1080)

    fig.write_image('Results_pretraining_'+str(model)+'.pdf')

def plot_training(VARIATIONS):
    VAR1=[]
    VAR2=[]
    VAR3=[]
    VAR4=[]
    for i in range(len(VARIATIONS)):
        VAR1.append(VARIATIONS[i][0]*100)
        VAR2.append(VARIATIONS[i][1]*100)
        VAR3.append(VARIATIONS[i][2]*100)
        VAR4.append(VARIATIONS[i][3]*100)
    fig = make_subplots(rows=2, cols=2,subplot_titles=("Outside map","Same position","Null rewards","+1 reward"))
    fig.add_trace(go.Scatter(y=VAR1,marker_color='crimson',name='Outside map position'),row=1,col=1)

    fig.add_trace(go.Scatter(y=VAR2,marker_color='orange', name='Same position'),row=1,col=2)
    fig.add_trace(go.Scatter(y=VAR3,marker_color='grey',name='Null rewards'),row=2,col=1)
    fig.add_trace(go.Scatter(y=VAR4,marker_color='lightgreen',name='+1 reward'),row=2,col=2)

    fig.update_layout(title=go.layout.Title(
                          text="Argmax q-values variations during training"),
                          yaxis=dict(title='Variation (%)'),
                          xaxis=dict(title='Trained episodes',tick0 = 0,dtick = 50)
                          ,width=1920,height=1080)
    fig.write_image('Argmax q-values variations during training.pdf')


from ....enums import LocationType
from ....models.definitions.distribution_event import DistributionEvent
from ....models.definitions.location import Location

def get_base_distribution_events(*, base_id, states):
    """Return distribution events for base that match given states (default: all)."""
    state_filter = DistributionEvent.state << states if states else True
    return (
        DistributionEvent.select()
        .join(Location)
        .where(
            Location.base_id == base_id,
            Location.type == LocationType.DistributionSpot,
            state_filter,
        )
    )


from aiogram import Bot
from aiogram.types import BotCommand, BotCommandScopeDefault
from config import _

async def get_commands(bot: Bot):
    commands = [
        BotCommand(
            command='start',
            description=_('Главное меню')
        ),
    ]
    await bot.set_my_commands(commands, BotCommandScopeDefault())


from flask import Flask
from flask import jsonify

app = Flask(__name__)

@app.route("/")
def index():
  return 'hello devops, this is your home.
click me'

@app.route("/endpoint")
def endpoint():
  devops = {
    'name': 'Mehmet Taskiner',
    'github': 'http://github.com/mehmettaskiner'
  }
  return jsonify(devops)

if __name__ == "__main__":
  app.run()


#!/usr/bin/env python3.5
import unittest

# palindrome code, this is just a test project for pdb (there's a bug here)
#

def digits(x):
    """Convert an integer into a list of digits.

    Args:
        x: The number whose digits we want.

    Returns: A list of the digits, in order of ``x``.
    >>> digits(4586378)
    [4, 5, 8, 6, 3, 7, 8]
    """
    # NOTE: import is done in the method
    # import pdb
    # pdb.set_trace()

    digs = []
    while x != 0:
        div, mod = divmod(x, 10)
        digs.append(mod)
        # x = mod  # THIS IS THE BUG
        x = div
    return digs

def is_palindrome(x):
    """Determine if an integer is a palindrome.

    Args:
        x: The number to check for the palidrom-o-city.

    Returns: True if the digits of ``x`` are a palindrome, else false.

    >>> is_palindrom(1234)
    False
    >>> is_palindrom(2468642)
    True
    """
    digs = digits(x)
    for f, r in zip(digs, reversed(digs)):
        if f != r:
            return False
    return True

class Tests(unittest.TestCase):
    """ Test for the ```is_palindrome()``` function."""

    def test_negative(self):
        "Check that it returns false correctly."
        self.assertFalse(is_palindrome(1234))

    def test_positive(self):
        "Check that it returns true correctly."
        self.assertTrue(is_palindrome(1230321))

    def test_single_digit(self):
        "Check that it works for single digit numbers."
        for i in range(10):
            self.assertTrue(is_palindrome(i))

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
easymail

an abstraction layer on top of the standard email package.
It aims to hide all the troublesome mime stuff and sets some sane defaults.
All to make your life a little bit easier and make sending emails easier.

>>> e = Email('sender@foo.com', 'recipient@foo.com')
>>> e.subject = 'urgent!!!1'
>>> e.body = 'this is very important.'
>>> e.attachments.append(Attachment('/path/to/audiofile.ogg'))

>>> e.sender
'sender@foo.com'
>>> str(e.subject)
'urgent!!!1'
>>> e.recipients
['recipient@foo.com']
"""

import os
import logging
from mimetypes import guess_type
from email import encoders
from email.header import Header
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate

__all__ = ['Attachment', 'Email']
logger = logging.getLogger(__file__)

class Attachment(object):
    def __init__(self, path, mimetype=None, filename=None):
        """create an attachment

        :param path: path to the file on disk
        :param mimetype: (optional, will be guessed by fileextension
        :param filename: filename in the email,
                         if None the name from path will be used
        """

        #: full path to the attachment
        self.path = path

        #: mimetype of the attachment
        self.mimetype = mimetype or (guess_type(path)[0] or
                                     'application/octet-stream')
        #: filename of the attachment in the email
        self.filename = filename

    def as_msg(self):
        """return the mime msg.

        the results concrete type depends on the attachments mimetype.
        """
        maintype, subtype = self.mimetype.split('/')
        with open(self.path, 'rb') as fp:
            if maintype == 'image':
                msg = MIMEImage(fp.read(), _subtype=subtype)
            elif maintype == 'audio':
                msg = MIMEAudio(fp.read(), _subtype=subtype)
            else:
                msg = MIMEBase(maintype, subtype)
                msg.set_payload(fp.read())
                encoders.encode_base64(msg)

        msg.add_header('Content-Disposition',
                       'attachment',
                       filename=self.filename or os.path.basename(self.path))
        return msg

class Email(object):
    def __init__(self, sender, recipients, subject='', body=''):
        """

        :param sender: email of the sender. E.g. 'me@mydomain.com'
                       or in the format 'My Name <me@mydomain.com>'
        :param recipients: a :class:`list` of recipients.
                           ['one@recipient.com', 'Two <two@recipient.com>']
        :param subject: email subject
        :param body: email body.
                     If it contains HTML `body_is_html` needs to be set to True
        """
        self.sender = sender

        if isinstance(recipients, str):
            self.recipients = [recipients]
        else:
            self.recipients = recipients

        self.subject = subject
        self.attachments = []
        self.reply_to = ''
        self.cc = []
        self.bcc = []

        #: Mail body
        self.body = body

        self._body_is_html = False

    def __repr__(self):
        return '<Email>'

    def __str__(self):
        return self.get_msg()

    @property
    def body_is_html(self):
        """If set to `True` the email will be sent as HTML
        Otherwise a regular plain/text email will be sent.

        :returns: :class:`bool`
        """
        return self._body_is_html

    @body_is_html.setter
    def body_is_html(self, value):
        self._body_is_html = value

    @property
    def all_recipients(self):
        """recipients including cc and bcc"""
        return self.recipients + self.cc + self.bcc

    @property
    def subject(self):
        """subject of the email"""
        return str(self._subject)

    @subject.setter
    def subject(self, value):
        if isinstance(value, str):
            self._subject = Header(value, 'utf-8')
        else:
            self._subject = value
        assert isinstance(self._subject, Header)

    @property
    def args(self):
        """ *e.args can be used for :func:`smtplib.SMTP.sendemail`

        >>> e = Email('Sender <sender@mydomain.com>',
        ...           ['rec1@dom.com', 'rec2@dom.com'],
        ...           'subject',
        ...           'body')
        >>> e.args  # doctest:+ELLIPSIS
        ('Sender <sender@mydomain.com>', ['rec1@dom.com', 'rec2@dom.com'], ...

        :returns: :class:`tuple`
        """
        return (self.sender, self.all_recipients, self.get_msg())

    def get_msg(self):
        # python 3.3 doesn't like the utf-8 charset if the body is empty.
        charset = self.body != '' and 'utf-8' or 'us-ascii'

        if self.attachments or self.body_is_html:
            msg = MIMEMultipart()
        else:
            msg = MIMEText(self.body, _charset=charset)

        msg['Subject'] = self._subject
        msg['From'] = self.sender
        msg['To'] = ', '.join(self.recipients)
        if self.reply_to:
            msg['Reply-To'] = self.reply_to
        msg['Date'] = formatdate(localtime=True)

        if self.body_is_html:
            body = MIMEText(self.body, 'html', charset)
            msg.attach(body)

        if self.attachments:
            if not self.body_is_html:
                msg.attach(MIMEText(self.body, _charset=charset))

            for attachment in self.attachments:
                msg.attach(attachment.as_msg())

        result = str(msg)
        logger.debug(result)
        return result

if __name__ == "__main__":
    import doctest
    doctest.testmod()


def next_number(li):
    if len(li) == 0:
        return 1
    for j in range(1, len(li) + 1):
        if j not in li:
            return j
    return len(li)

def reverse(lst):
    n = len(lst)
    for i in range(int(n / 2)):
        temp = lst[i]
        lst[i] = lst[n - i - 1]
        lst[n - i - 1] = temp

def pretty_print_matrix(m):
    for i in range(len(m)):
        k = m[i]
        for j in range(len(k)):
            print(str(m[i][j]), " ", end=" ")
        print()
    print("-------------------------------")

def create_matrix(k):
    rez = []
    for i in range(k):
        rez.append([])
        for j in range(k):
            rez[i].append(1)
    return rez

def create_list():
    # lst = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    lst = [x for x in range(1, 11)]
    # lst = [x for x in range(1, 5)] + [5, 6, 7, 8, 9, 10]
    return lst

def row_column_equal(lst_2d):
    for i in range(len(lst_2d)):
        for j in range(len(lst_2d)):
            n = 0
            for k in range(len(lst_2d)):
                if lst_2d[i][k] == lst_2d[k][j]:
                    n = n + 1
                else:
                    break
            if n == len(lst_2d):
                return True
    return False

def tic_tac_toe(ttt):
    for i in range(len(ttt)):
        px = 0
        p0 = 0
        px2 = 0
        p02 = 0
        px3 = 0
        p03 = 0
        px4 = 0
        p04 = 0
        for j in range(len(ttt)):
            if ttt[i][j] == 'X':
                px = px + 1
            elif ttt[i][j] == '0':
                p0 = p0 + 1
            if ttt[j][i] == 'X':
                px2 = px2 + 1
            elif ttt[j][i] == '0':
                p02 = p02 + 1
            if ttt[j][j] == 'X':
                px3 = px3 + 1
            elif ttt[j][j] == '0':
                p03 = p03 + 1
            if ttt[j][len(ttt) - j - 1] == 'X':
                px4 = px4 + 1
            elif ttt[j][len(ttt) - j - 1] == '0':
                p04 = p04 + 1
        if px == len(ttt) | px2 == len(ttt) | px3 == len(ttt) | px4 == len(ttt):
            return 'X'
        elif p0 == len(ttt) | p02 == len(ttt) | p03 == len(ttt) | p04 == len(ttt):
            return '0'
        else:
            return ' '

def main():
    ttt = [['0', '0', 'X'],
           ['X', 'X', 'X'],
           ['X', ' ', '0']]
    print(tic_tac_toe(ttt))

if __name__ == '__main__':
    main()


# Faça um programa que calcule o mostre a média aritmética de N notas

n = int(input("Digite a quantidade de notas que deseja calcular: "))
qtd = 0
soma = 0

while n > 0:
    nota = float(input("Digite a nota: "))
    soma += nota
    qtd += 1
    n -= 1
media = soma/qtd

print("Média = %.2f" % media)

Lista de exercícios pybrasil/Repetição/24.py


from drf_spectacular.utils import OpenApiResponse, extend_schema

from api.docs.base_docs import custom_extend_schema, fields_to_md
from api.examples import (
    audio_complain_201_example,
    audio_complain_curl,
    audio_detail_200_example,
    audio_detail_404_example,
    audio_detail_curl,
    audio_related_200_example,
    audio_related_404_example,
    audio_related_curl,
    audio_search_200_example,
    audio_search_400_example,
    audio_search_list_curl,
    audio_stats_200_example,
    audio_stats_curl,
    audio_waveform_200_example,
    audio_waveform_404_example,
    audio_waveform_curl,
)
from api.serializers.audio_serializers import (
    AudioReportRequestSerializer,
    AudioSearchRequestSerializer,
    AudioSerializer,
    AudioWaveformSerializer,
)
from api.serializers.error_serializers import (
    InputErrorSerializer,
    NotFoundErrorSerializer,
)
from api.serializers.media_serializers import MediaThumbnailRequestSerializer
from api.serializers.provider_serializers import ProviderSerializer

search = custom_extend_schema(
    desc=f"""
        Search audio files using a query string.

        By using this endpoint, you can obtain search results based on specified
        query and optionally filter results by
        {fields_to_md(AudioSearchRequestSerializer.fields_names)}.

        Results are ranked in order of relevance and paginated on the basis of the
        `page` param. The `page_size` param controls the total number of pages.

        Although there may be millions of relevant records, only the most relevant
        several thousand records can be viewed. This is by design: the search
        endpoint should be used to find the top 10,000 most relevant results, not
        for exhaustive search or bulk download of every barely relevant result. As
        such, the caller should not try to access pages beyond `page_count`, or else
        the server will reject the query.""",
    params=AudioSearchRequestSerializer,
    res={
        200: (AudioSerializer, audio_search_200_example),
        400: (InputErrorSerializer, audio_search_400_example),
    },
    eg=[audio_search_list_curl],
    external_docs={
        "description": "Openverse Syntax Guide",
        "url": "https://openverse.org/search-help",
    },
)

stats = custom_extend_schema(
    desc=f"""
        Get a list of all content providers and their respective number of
        audio files in the Openverse catalog.

        By using this endpoint, you can obtain info about content providers such
        as {fields_to_md(ProviderSerializer.Meta.fields)}.""",
    res={200: (ProviderSerializer, audio_stats_200_example)},
    eg=[audio_stats_curl],
)

detail = custom_extend_schema(
    desc=f"""
        Get the details of a specified audio track.

        By using this endpoint, you can obtain info about audio files such as
        {fields_to_md(AudioSerializer.Meta.fields)}""",
    res={
        200: (AudioSerializer, audio_detail_200_example),
        404: (NotFoundErrorSerializer, audio_detail_404_example),
    },
    eg=[audio_detail_curl],
)

related = custom_extend_schema(
    desc=f"""
        Get related audio files for a specified audio track.

        By using this endpoint, you can get the details of related audio such as
        {fields_to_md(AudioSerializer.Meta.fields)}.""",
    res={
        200: (AudioSerializer(many=True), audio_related_200_example),
        404: (NotFoundErrorSerializer, audio_related_404_example),
    },
    eg=[audio_related_curl],
)

report = custom_extend_schema(
    res={201: (AudioReportRequestSerializer, audio_complain_201_example)},
    eg=[audio_complain_curl],
)

thumbnail = extend_schema(
    parameters=[MediaThumbnailRequestSerializer],
    responses={200: OpenApiResponse(description="Thumbnail image")},
)

waveform = custom_extend_schema(
    res={
        200: (AudioWaveformSerializer, audio_waveform_200_example),
        404: (NotFoundErrorSerializer, audio_waveform_404_example),
    },
    eg=[audio_waveform_curl],
)


from random import randint, random

def sorteia(lista):
  for cont in range(0, 5):
    n = randint(1, 10)
    lista.append(n)
  print(lista)

def somar(lista):
  soma = 0
  for i in lista:
    if i % 2 == 0:
      soma+= i
  print(f'SOMA DOS VALORES PARES: {soma}')

numero = list()
sorteia(numero)
somar(numero)


import codecs
import torch
import numpy as np
from nltk.corpus import wordnet as wn
import pickle as pkl

def get_trimmed_w2v_vectors(filename):
    """
    Args:
        filename: path to the npz file
    Returns:
        matrix of embeddings (np array)
    """
    with np.load(filename) as data:
        return data['embeddings']

def load_vocab(filename):
    """
    Args:
        filename: file with a word per line
    Returns:
        d: dict[word] = index
    """
    d = dict()
    with open(filename) as f:
        for idx, word in enumerate(f):
            word = word.strip()
            d[word] = idx + 1  # preserve idx 0 for pad_tok
    return d

def load_vocab_utf8(filename):
    """
    Args:
        filename: file with a word per line
    Returns:
        d: dict[word] = index
    """
    d = dict()
    with codecs.open(filename, encoding='utf-8') as f:
        for idx, word in enumerate(f):
            word = word.strip()
            d[word] = idx + 1  # preserve idx 0 for pad_tok
    return d

def load_pkl(path):
    with open(path, 'rb') as file:
        return pkl.load(file)

def dump_pkl(obj, path):
    with open(path, 'wb') as file:
        pkl.dump(obj, file)

def build_label_for_df(df):
    '''
    Build label for dataframe
    '''
    for i in range(len(df)):
        label = torch.tensor([df.iloc[i]['label']]).type(torch.FloatTensor)
        if i == 0:
            labels = label
        else:
            labels = torch.vstack((labels, label))

    return labels

def get_lookup(path):
    '''
    Get lookup table from file
    '''
    with open(path, 'r') as file:
        f = file.read().split('\n')
    return {f[i]: i + 1 for i in range(len(f))}

def lookup(word, dct):
    '''
    Get index of word in lookup table
    '''
    try:
        idx = dct[word]
    except:
        idx = 0
    return idx

def get_idx(sent, vocab_lookup, tag_lookup, direction_lookup, edge_lookup):
    '''
    Get index of features of tokens in sentence
    '''
    if sent == None:
        return None
    to_return = list()
    i = 0
    for dp in sent:
        word1_idx = lookup(dp[0][0], vocab_lookup)
        word2_idx = lookup(dp[2][0], vocab_lookup)
        tag1_idx = lookup(dp[0][1], tag_lookup)
        tag2_idx = lookup(dp[2][1], tag_lookup)
        direction_idx = lookup(dp[1][0], direction_lookup)
        edge_idx = lookup(dp[1][1], edge_lookup)
        pos1 = dp[0][2]
        pos2 = dp[2][2]
        v = torch.tensor([word1_idx, tag1_idx, direction_idx, edge_idx, word2_idx, tag2_idx])
        v = torch.hstack((v[:2], pos1, v[2:6], pos2))
        if i == 0:
            to_return = v.view(1, -1)
        else:
            to_return = torch.vstack((to_return, v))
        i += 1
    return to_return

def get_idx_dataset(data,
                    vocab_lookup,
                    tag_lookup,
                    direction_lookup,
                    edge_lookup):
    tmp = list()
    for i in data:
        tmp.append(get_idx(i, vocab_lookup, tag_lookup, direction_lookup, edge_lookup))
    return tmp

def prepare_dict_simul_paths(lookup, X, y):
    '''
    Get training tensors from lookup table and X for simultaneous paths approach
    Args:
        lookup: lookup table
        X: list of tensors
    Returns:
        list of tensors
    '''
    to_return_dict = dict()

    for k, v in lookup.items():
        tmp = []
        for i in v:
            if X[i] != None:
                tmp.append(X[i])
            label = y[i]
        if tmp != []:
            to_return_dict[k] = (tmp, label)

    return to_return_dict
def get_tensors_simul_paths(lookup: dict,
                            select_option: str,
                            top_k: int):
    '''
    Get top k paths and convert to tensors for simultaneous paths approach
    Args:
        lookup: lookup table
        select_option: select method, could be "random", "shortest", "longest"
        top_k: number of paths to select
    Returns:
        X: dataset of tensors
        y: labels
    '''
    from tqdm import tqdm

    X = list()
    y = list()
    for k, v in tqdm(lookup.items()):
        X_tmp = v[0]
        y_tmp = v[1]
        if select_option == "random":
            if top_k < len(X_tmp):
                X_tmp = X_tmp[:top_k]
        X.append(X_tmp)
        y.append(y_tmp)

    return X, y


import csv
from datetime import datetime, timedelta

# Define the file path
file_path = 'hw.csv'

# Function to check if an employee worked for 7 consecutive days
def has_worked_for_7_consecutive_days(dates):
    dates = sorted(dates)
    for i in range(len(dates) - 6):
        if all((dates[i + j] - dates[i + j - 1]).days == 1 for j in range(1, 7)):
            return True
    return False

# Function to check if an employee has less than 10 hours between shifts
def has_less_than_10_hours_between_shifts(dates):
    for i in range(1, len(dates)):
        if (dates[i] - dates[i - 1]).seconds < 36000 and (dates[i] - dates[i - 1]).seconds > 3600:
            return True
    return False

# Function to check if an employee has worked for more than 14 hours in a single shift
def has_worked_more_than_14_hours_in_a_single_shift(hours_worked):
    return max(hours_worked) > 14

# Open and read the CSV file
with open(file_path, 'r') as file:
    csv_reader = csv.DictReader(file)

    # Create dictionaries to store data for each employee
    employees = {}

    for row in csv_reader:
        name = row['Employee Name']
        position = row['Position ID']
        time_value = row['Time']

        # Check if the 'Time' value is empty or missing
        if not time_value:
            continue  # Skip this row and continue with the next

        # Adjust the datetime format for your date and time format
        date = datetime.strptime(time_value, '%m/%d/%Y %I:%M %p')

        # Parse 'Timecard Hours (as Time)' column with format 'HH:MM'
        timecard_hours = row['Timecard Hours (as Time)'].split(':')
        hours_worked = float(timecard_hours[0]) + float(timecard_hours[1]) / 60.0

        # Add the data to the employee's record
        if name in employees:
            employees[name]['dates'].append(date)
            employees[name]['hours_worked'].append(hours_worked)
        else:
            employees[name] = {'position': position, 'dates': [date], 'hours_worked': [hours_worked]}

# Iterate through the employee records and apply the criteria
for name, data in employees.items():
    if has_worked_for_7_consecutive_days(data['dates']):
        print(f"{name} (Position ID: {data['position']}) worked for 7 consecutive days.")
    if has_less_than_10_hours_between_shifts(data['dates']):
        print(f"{name} (Position ID: {data['position']}) has less than 10 hours between shifts.")
    if has_worked_more_than_14_hours_in_a_single_shift(data['hours_worked']):
        print(f"{name} (Position ID: {data['position']}) worked for more than 14 hours in a single shift.")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, unicode_literals, absolute_import

import connection as co
import readline, string, os, sys
import argparse

from pydoc import pager

try:
    input = raw_input
except:
    pass

FNAME = os.path.expanduser('~/.revterm_history')

WARNING = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
FAIL = '\033[91m'
ENDC = '\033[0m'

codes = {
    0: OKGREEN,
    1: OKGREEN,
    2: OKGREEN,
    3: OKGREEN,
    4: WARNING
}

class Prompt(object):
    def __init__(self):
        self.database = '<none>'
        self.count = 0
        self.format = string.Template('$database [$count] $prompt_char ')
        self.prompt_char = '>'

    def increment(self):
        self.count += 1

    def in_copy(self):
        self.prompt_char = '|'

    def out_copy(self):
        self.prompt_char = '>'

    def __str__(self):
        return OKBLUE + self.format.safe_substitute(self.__dict__) + ENDC

def present_prompt(p):
    try:
        value = input(str(p))
    except EOFError:
        result = 'k'

        while len(result) >= 1 and result[0] != 'y' and result[0] != 'n':
            try:
                result = input("\nDo you really want to exit? ([y]/n) ")
            except EOFError:
                result = 'y'

        if len(result) == 0 or result[0] == 'y':
            sys.exit(0)
        else:
            return present_prompt(p)
    except KeyboardInterrupt:
        print('')
        return present_prompt(p)
    return value

def generate_output(value):
    output = ''

    if isinstance(value.message, str):
        output += value.message
    else:
        for item in value.message:
            output += '\t' + '\t'.join(item) + '\n'

    return output

def send(con, msg):
    try:
        con.send(msg)
        return con.recv()
    except Exception as e:
        print('Client encountered fatal error. Exiting...')
        print(e)
        sys.exit(1)

def send_direct(con, msg):
    try:
        con.send(msg)
    except Exception as e:
        print('Client encountered fatal error. Exiting...')
        print(e)
        sys.exit(1)

def build_parser():
    parser = argparse.ArgumentParser(description='CLI Interface for Revori')

    parser.add_argument('-hs', '--host',  type=str, default='localhost')
    parser.add_argument('-p', '--port', type=int, default=8017)
    parser.add_argument('-n', '--lines', type=int, default=25,
        help='Maximum number of lines to display before paging.')

    return parser

def parse_args(args):
    parser = build_parser()

    return parser.parse_args(args)

def config_readline():
    with open(FNAME, 'a') as f:
        os.utime(FNAME, None)

    readline.set_history_length(500)
    readline.read_history_file(FNAME)

def printout(max_lines, result):
    output = generate_output(result)
    if max_lines > 0 and output.count('\n') < max_lines:
        output = codes.get(result.code, ENDC) + output + ENDC
        print(output)
    else:
        pager(output)

def main():
    args = parse_args(sys.argv[1:])

    config_readline()

    p = Prompt()
    c = co.RevoriConnection(args.host, args.port)

    copying = False

    print("Welcome to the Revori SQL client interface." + ENDC)
    print('Type "help" to get started.\n' + ENDC);
    try:
        while True:
            value = present_prompt(p)

            if value == 'exit' or value == 'quit':
                sys.exit(0)

            value = value.strip()

            if len(value) > 0:
                if not copying:
                    p.increment()

                    result = send(c, value)

                    printout(args.lines, result)

                    if hasattr(result, 'database'):
                        p.database = result.database

                    copying = result.copy_context

                    if copying:
                        p.in_copy()
                else:
                    send_direct(c, value)

                    if value == '\\.':
                        copying = False
                        p.out_copy()

                        result = c.recv()
                        output = generate_output(result)
                        printout(args.lines, result)

    finally:
        c.close()
        print(ENDC)

        readline.write_history_file(FNAME)

if __name__ == '__main__':
    main()


#!/bin/python3

#for typing.Dict
import typing
#for function empty, add_ingot, get_ingot
import functional_purse
#for random if booty not divided by 3
import random

#using Arbitrary Arguments, *args for receive any number
#of purses, check for negatives

# summing all split_ingots
# then dividing them into equal 3 parts
# dividing mod randomly part if it exists
# creating tuple of 3 dicts with respective values of list

#def split_booty(*booty):
#    assert all(map(lambda x: type(x["gold_ingots"]) == int and x["gold_ingots"] >= 0, list(filter(lambda x: "gold_ingots" in x.keys(), booty)))), "One of booty gold_ingots is negative or non-integer!"
#    #sum all gold_ingots
#    split_ingots = sum([x["gold_ingots"] for x in list(filter(lambda x: "gold_ingots" in x.keys(), booty))])
#    # sum is divided by 3 eq parts
#    res = [split_ingots // 3 for i in range(3)]
#    # if mod not zero do random choice
#    if split_ingots % 3:
#        lucky = random.randint(0, 2)
#        if split_ingots % 3 == 1:
#            l = map(lambda x: int(x == lucky), range(3))
#        else:
#            l = map(lambda x: int(x != lucky), range(3))
#        res = [x + y for x, y in zip(l, res)]
#    return tuple([dict({"gold_ingots" : x}) for x in res])

# Work with every purse of booty
# Get ingots from purse while it has ingots
# Every ingot is given to current of 3 men, then to next
# more accurately who will get ingot depend on the number of ingot
# we consider
# Total amount of ingots on this moment (or number of current ingots) is counted in split_ingots

def split_booty(*booty: typing.Dict[str, int]):
    def correct_input(*booty: typing.Dict[str, int]):
        def correct_ingot(x: typing.Dict[str, int]):
            return type(x["gold_ingots"]) == int and x["gold_ingots"] >= 0
        item_ingots = list(filter(lambda x: "gold_ingots" in x.keys(), booty))
        Error_message = "One of booty gold_ingots is negative or non-integer!"
        assert all(map(correct_ingot, item_ingots)), Error_message

    correct_input(*booty)
    Bert = functional_purse.empty()
    Tom = functional_purse.empty()
    Will = functional_purse.empty()
    res = (Bert, Tom, Will)
    split_ingots = 0
    for purse in booty:
        if not "gold_ingots" in purse.keys():
            continue
        while purse["gold_ingots"] > 0:
            purse = functional_purse.get_ingot(purse)
            cur_purse = res[split_ingots % len(res)]
            cur_purse = functional_purse.add_ingot(cur_purse)
            split_ingots += 1
    return res

#test_cases
if __name__ == "__main__":
    print(split_booty({"gold_ingots":3}, {"gold_ingots":2}, {"apples":10}))
    print(split_booty({"gold_ingots":2}, {"gold_ingots":2}, {"apples":2}))
    print(split_booty({"gold_ingots":3}, {"gold_ingots":2}, {"apples":10}))
    print(split_booty({"gold_ingots":201}, {"gold_ingots":21}, {"apples":203}))
    #error case split_ingot
    print(split_booty({"gold_ingots": "srhydsfxh"}, {"gold_ingots":21}, {"apples":203}))


import Tkinter
import time
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg,NavigationToolbar2TkAgg
from matplotlib.figure import Figure
import numpy as np
import socket
import rospy
from std_msgs.msg import String

#Visualization application for the static rtk case
class App:
	def __init__(self,master):

		#This was a final test to demonstrate that the software can be extended
		#To include the GUI directly within the positioning node
		rospy.init_node('Bamn')

		#Initialize the publisher
		self.bamn_pub = rospy.Publisher('Bamn', String, queue_size = 30)

		#The Reference Rover and Base coordinates were hard coded in order to
		#avoid "over engineering" as the intention is to eventually have this
		#information input in an "elegant" fasion
		self.ref = np.array([4641949.35634791,1393045.6421954,4133287.64031007])
		self.base = np.array([4641952.63326322,1393062.95794337,4133278.25700797])

		#Store a reference to the GUI within the obhject
		self.master = master

		#Create the network socket and bind to the "static_rtk" port
		self.sock = socket.socket()
		host = socket.gethostname()
		port = 5802
		self.sock.connect((host,port))

		#Create 2 buttons and place them at the top of the GUI. These buttons
		#change the view between baseline mode and rover mode
		buttonFrame = Tkinter.Frame(master)

		self.roverButton = Tkinter.Button(buttonFrame,text='Rover Mode',command=self.roverMode)
		self.roverButton.pack(side="left")

		self.baselineButton = Tkinter.Button(buttonFrame,text='Baseline Mode',command=self.baselineMode)
		self.baselineButton.pack(side='left')

		buttonFrame.pack()

		#Initialize the application to rover mode
		self.mode = 'rover'

		self.frame = Tkinter.Frame(master)
		self.frame.pack()

		#Create a text box to display the rover coordinates
		self.text = Tkinter.Text(master, height=5, width=20)
		output = 'x = 0\n\ny = 0\n\nz = 0'
		self.text.insert(Tkinter.END, output)
		self.text.pack(side='left')

		#Create the data structures to store the epoch-to-epoch results
		self.pos = [[],[],[]]
		self.dPos = [[],[],[]]

		self.bl = [[],[],[]]
		self.dBL = [[],[],[]]

		self.Cx = [[],[],[],[],[],[]]

		self.gpst = []
		self.ftest = []
		self.delay = []
		self.gdop = []

		#Create blank plots and store them as references within the object
		self.fig = Figure()

		self.plt = []

		for i in range(231,237):

			self.plt.append(self.fig.add_subplot(i))

		#Add the plots to the application and add the toolbar at the bottom
		#In order to interact with the plots (e.g. zoom and translate the plots)
		self.canvas = FigureCanvasTkAgg(self.fig,master=self.master)
		self.canvas.get_tk_widget().pack(side='top', fill='both', expand=1)
		self.toolbar = NavigationToolbar2TkAgg( self.canvas, master )
		self.toolbar.update()
		self.canvas._tkcanvas.pack(side=Tkinter.TOP, fill=Tkinter.BOTH, expand=1)

	#This method switches the application's mode to rover mode, which is then
	#interpreted by the rtkPlot function to plot the rover state details
	def roverMode(self):

		self.mode = 'rover'

	#This method switches the application's mode to baseline mode, which is
	#then interpreted by the rtkPlot function to plot the baseline state details
	def baselineMode(self):

		self.mode = 'baseline'

	#At each measurement epoch, get the output from RTKRCV, parse it, and use it
	#To compute the different quantities that are displayed
	def getData(self):

		#WGS84 Parameters
		e2 = 6.69437999014e-3
		a = 6378137.0

		#Receive the message from RTKRCV
		msgStr = self.sock.recv(1024)

		#Parse the message
		msg = msgStr.split()

		#Compute the radius of curvature
		N = 1.0*a/np.sqrt(1-e2*(np.sin(float(msg[2])*np.pi/180.0)**2))

		#ECEF XYZ Pose
		self.pos[0].append((N+float(msg[4]))*np.cos(float(msg[2])*np.pi/180.0)*np.cos(float(msg[3])*np.pi/180.0))
		self.pos[1].append((N+float(msg[4]))*np.cos(float(msg[2])*np.pi/180.0)*np.sin(float(msg[3])*np.pi/180.0))
		self.pos[2].append((N*(1-e2)+float(msg[4]))*np.sin(float(msg[2])*np.pi/180.0))

		#Reference Coordinate Discrepancies
		self.dPos[0].append(self.pos[0][-1] - self.ref[0])
		self.dPos[1].append(self.pos[1][-1] - self.ref[1])
		self.dPos[2].append(self.pos[2][-1] - self.ref[2])

		#Baseline and baseline discrepancy (same as reference coord discrepancy)
		self.bl[0].append(self.pos[0][-1] - self.base[0])
		self.bl[1].append(self.pos[1][-1] - self.base[1])
		self.bl[2].append(self.pos[2][-1] - self.base[2])
		self.dBL[0].append(self.bl[0][-1] - (self.ref[0] - self.base[0]))
		self.dBL[1].append(self.bl[1][-1] - (self.ref[1] - self.base[1]))
		self.dBL[2].append(self.bl[2][-1] - (self.ref[2] - self.base[2]))

		#Covariance matrix output from RTKRCV
		self.Cx[0].append(float(msg[7]))
		self.Cx[1].append(float(msg[8]))
		self.Cx[2].append(float(msg[9]))
		self.Cx[3].append(float(msg[10]))
		self.Cx[4].append(float(msg[11]))
		self.Cx[5].append(float(msg[12]))

		#Norm of the covariance matrix diagonal elements
		self.gdop.append(np.sqrt(self.Cx[0][-1]*self.Cx[0][-1] + self.Cx[1][-1]*self.Cx[1][-1] + self.Cx[2][-1]*self.Cx[2][-1]))

		#Store other GNSS relevant data
		self.gpst.append(float(msg[1]))
		self.ftest.append(float(msg[14]))
		self.delay.append(float(msg[13]))

		#Test out publishing over a rostopic without using a ROS based loop
		self.bamn_pub.publish("BAMN")

		#Call the function again after 1000 ms
		self.master.after(1000,self.getData)

	#At each measurement epoch, replot the requested quantities with the newly
	#acquired values
	def rtkPlot(self):

		#If Rover mode is selected
		if self.mode == 'rover':

			#Datasets to be plotted and their labels
			datasets = [self.dPos[0], self.dPos[1], self.dPos[2], self.gdop, self.ftest, self.delay]
			titles = ['dx', 'dy', 'dz', 'GDOP', 'ftest', 'delay']

			#Plot each dataset
			for i in range(len(datasets)):

				self.plt[i].cla()
				self.plt[i].plot(range(len(datasets[i])),datasets[i])
				self.plt[i].set_title(titles[i])

			#Display rover coordinates
			output = 'x = ' + str(self.pos[0][-1]) + '\n\ny = ' + str(self.pos[1][-1]) + '\n\nz = ' + str(self.pos[2][-1])
			self.text.delete(1.0,Tkinter.END)
			self.text.insert(Tkinter.END, output)
			self.text.pack(side='left')

		#If baseline mode is selected
		if self.mode == 'baseline':

			#Datasets to be plotted and their labels
			datasets = [self.bl[0], self.bl[1], self.bl[2], self.dBL[0], self.dBL[1], self.dBL[2]]
			titles = ['DX', 'DY', 'DZ', 'dDX', 'dDY', 'dDZ']

			#Plot each dataset
			for i in range(len(datasets)):

				self.plt[i].cla()
				self.plt[i].plot(range(len(datasets[i])),datasets[i])
				self.plt[i].set_title(titles[i])

			#Display the baseline info
			output = 'x = ' + str(self.pos[0][-1]) + '\n\ny = ' + str(self.pos[1][-1]) + '\n\nz = ' + str(self.pos[2][-1])
			self.text.delete(1.0,Tkinter.END)
			self.text.insert(Tkinter.END, output)
			self.text.pack(side='left')

		#Draw the new plots
		self.canvas.draw()

		#Call the plotting function every seconds
		self.master.after(1000,self.rtkPlot)

#Main function
root = Tkinter.Tk()
root.wm_title("RTK Data Analyzer")
app = App(root)
root.after(1000,app.getData)
root.after(1000,app.rtkPlot)
root.mainloop()


import os
from requests import get
from io import BytesIO
from PIL import Image
from typing import Optional

class ImageDownloader:
    def __init__(self, link_dict: dict):
        self.links = link_dict

    def _switch_directory(self, folder_name: Optional[str] = None):
        """
        Checks the file path. If it is in the wrong directory,
        it comes back and enters the correct directory.
        If it is in the main directory(images), it enters the specified directory

        images: -> Main Folder
            icon_folder -> Subfolder
                icon.png
            badge_folder -> Subfolder
                badge.png
        """
        curr_path = os.getcwd().split('/')[-1]

        if (folder_name and folder_name == curr_path) or (not folder_name and curr_path == "images"):
            return
        elif folder_name and curr_path != "images":
            target_directory = '../' + folder_name
        elif not folder_name and curr_path != "images":
            target_directory = '..'
        else:
            target_directory = folder_name

        os.chdir(target_directory)

    def restore_directory(self):
        curr_path = os.getcwd().split('/')
        if curr_path[-1] == "images":
            old_directory = "../../../"
        else:
            old_directory = "../../../../"

        os.chdir(old_directory)

    async def start_download(self):
        """Download images in database to specified directories"""
        # Set directory
        os.chdir("cogs/assets")
        os.makedirs("images", mode=0o777, exist_ok=True)
        os.chdir("images")
        os.makedirs("expense_icons", mode=0o777, exist_ok=True)
        os.makedirs("badges", mode=0o777, exist_ok=True)

        try:
            for key, value in self.links.items():
                response = get(value)
                img = Image.open(BytesIO(response.content)).convert("RGBA")

                # Change directory
                if key[:4] == "icon":
                    self._switch_directory("expense_icons")
                elif key[:5] == "badge":
                    self._switch_directory("badges")
                else:
                    self._switch_directory()

                img.save(f'{key}.png')
            self.restore_directory()
            return True
        except Exception as err:
            print(err)
            return False


"""
pyaircraft.io.read_p3_flight
=========================

This is a grouping of scripts designed to process NOAA P-3
 flight level data recorded during flights and put into NetCDF
 format by NOAA AOC.

Created by Nick Guy.

"""
# NOTES:: This has only been tested with DYNAMO data files, versions
#         may change and another function may be needed.
# HISTORY::
#   8 Jan 2014 - Nick Guy.   NRC, NOAA/NSSL (nick.guy@noaa.gov)
#                Converted NCL functions below to Python
# FUNCTIONS::
#  flight_level_variable - Read in a variable from flight level NetCDF
#  flight_track - Read in data to for flight track
#-------------------------------------------------------------------
# Load the needed packages
from scipy.io import netcdf
from netCDF4 import Dataset,num2date
import json
import numpy as np
import pytz
#-------------------------------------------------------------------
# Define various constants that may be used for calculations
#===============================================================
# BEGIN FUNCTIONS
#===============================================================
def flight_level_variable(fname,Rec):
    """Read in data from NetCDF file containing P3 flight level data created
  by NOAA AOC.  The NetCDF should be read in the main program and passed
  to this function.
  A call such as this can be used in the main program:
      FltncID=addfile(FlightFileStringName,"r")
 INPUT::
  fname           = Filename [string]
  Rec             = Variable name to be pulled out [string]
 OUTPUT::
  VarOut          = Masked array containing variable data
 USAGE::
  Lat = read_flight_level_dynamo('P3.nc','LatGPS.3')
 NOTES::
 Data file structure::
  Available variables (not full list) :
  LonGPS.3      = Novatel GPS Longitude
  LatGPS.3      = Novatel GPS Latitude
  AltGPS.3      = Novatel GPS Altitude [m]
  THdgI-GPS.1   = True heading [deg]
  TRK.1         = Track [deg]
  AltPaADDU.1   = Pressure altitude [m]
  WSZ_DPJ       = Vertical wind via D Jorgensen calculation [m/s]
  TA.1          = Ambient Temperature [C]
  TD.1          = Dewpoint Temperature [C]
  TVIRT.1       = Virtual Temperature [K]
  THETA.1       = Potential Temperature [K]
  THETAE.1      = Equivalent Potential Temperature [K]
  THETAV.1      = Virtual Potential Temperature [K]
  WS.1          = Wind Speed [m/s]
  WD.1          = Wind Direction [deg]
  HUM_REL.1     = Relative Humidity [%]
  HUM_SPEC.1    = Specific Humidity [g/kg]
  MR.1          = Mixing ratio [g] [g/g?]
  EE.1          = Vapor Pressure [hPa]
  EW.1          = Saturated Vapor Pressure [hPa]
      """
# HISTORY::
#  25 Jul 2013 - Nick Guy NOAA/NSSL/WRDD, NRC
#   8 Jan 2014 - With change to python, the procedure of setting the bad
#                 data by user is no longer used, but the numpy masking
#                 technique is instead used to nuke "bad" occurences
#---------------------------------------------------
    # Read the NetCDF
    ncFile = netcdf.netcdf_file(fname,'r')

    # Get the variable of interest
    VarOut = ncFile.variables['Rec'][:]

    # Mask any "_FillValue" or some missing_data type attribute
    #try:
    #    VarOut = np.ma.masked_values(VarOut, VarOut.missing_value)
    #except:
    #    pass
    #try:
    #    VarOut = np.ma.masked_values(VarOut, VarOut.Missing_Value)
    #except:
    #    pass
    #try:
    #    VarOut = np.ma.masked_values(VarOut, VarOut._FillValue)
    #except:
    #    pass

    # Mask any NaN values
    #VarOut = np.ma.masked_values(VarOut, np.isnan(VarOut)

    return VarOut
#**====================================================
def flight_track(fname):
    """Read in data from NetCDF file containing P3 flight level data created
  by NOAA AOC.  Pull out the needed variables for flight track info.
 INPUT::
  fname           = Filename [string]
 OUTPUT::
  Lat             = Aircraft latitude
  Lon             = Aircraft longitude
  Alt             = Aircraft altitude
  PAlt            = Aircraft pressure altitude
  Time            = Aircraft time array
 USAGE::
  Lat,Lon,Alt,PAlt = flight_track(fname)
    """
# HISTORY::
#   7 Mar 2014 - Nick Guy NOAA/NSSL/WRDD, NRC
#---------------------------------------------------
    # Read the NetCDF
    ncFile = Dataset(fname,'r')

    # Pull out each variable
    Lat = ncFile.variables['LatGPS.3'][:]
    Lon = ncFile.variables['LonGPS.3'][:]
    Alt = ncFile.variables['AltGPS.3'][:]
    PAlt = ncFile.variables['AltPaADDU.1'][:]

    # Pull out the start time
    StartTime = ncFile.StartTime

    # Create a time array
    TimeSec = np.linspace(StartTime,StartTime + len(Lat), len(Lat))

    Time_unaware = num2date(TimeSec,'seconds since 1970-01-01 00:00:00+0:00')
    Time = Time_unaware#.replace(tzinfo=pytz.UTC)

    # Now mask missing values
    np.ma.masked_invalid(Lat)
    np.ma.masked_invalid(Lon)
    np.ma.masked_invalid(Alt)
    np.ma.masked_invalid(PAlt)

    return Lat,Lon,Alt,PAlt,Time


# -------------------------------------------------------------------------------------------------------------------
# Reducing the Dimensionality of the Data
# Defeating the curse of dimensionality: for a given sample size, there exists an optimal number of features for best
# classification performance

# -------------------------------------------------------------------------------------------------------------------
# Implementing Principal Component Analysis (PCA) in OpenCV

# PCA rotates all data points they align with the two axes that explains the spread of the data. PCA aims to transform
# the data to a new coordinate system by means of an orthogonal linear transformation.
# Project the data onto the new coordinate system such that the first coordinate has the greatest variance.
# This is called the first principal component

# Here is some random data drawn from a multivariate Gaussian:
import numpy as np
from sklearn import decomposition

mean = [20, 20]
cov = [[5, 0], [25, 25]]
np.random.seed(42)
x, y = np.random.multivariate_normal(mean, cov, 1000).T

# Plot this data using Matplotlib. Look at the spread:
import matplotlib.pyplot as plt
# plt.style.use('ggplot')
plt.figure(figsize=(10, 6))
plt.plot(x, y, 'o', zorder=1)
plt.axis([0, 40, 0, 40])
plt.xlabel('feature 1')
plt.ylabel('feature 2')
# plt.show()

# Format the data by stacking the `x` and `y` coordinates as a feature matrix `X`:
X = np.vstack((x, y)).T

# Create an empty array `np.array([])` as a mask, telling OpenCV to use all data points in the feature matrix.
# Compute PCA on the feature matrix `X`
import cv2
mu, eig = cv2.PCACompute(X, np.array([]))
# print(eig)

# Note the following looks complicated but is simply for demonstration purposes (showing directional arrows of PC.
# Plot the eigenvectors of the decomposition on top of the data:
plt.figure(figsize=(10, 6))
plt.plot(x, y, 'o', zorder=1)
plt.quiver(mean[0], mean[1], eig[:, 0], eig[:, 1], zorder=3, scale=0.2, units='xy')
plt.text(mean[0] + 5 * eig[0, 0], mean[1] + 5 * eig[0, 1], 'u1', zorder=5,
         fontsize=16, bbox=dict(facecolor='white', alpha=0.6))
plt.text(mean[0] + 7 * eig[1, 0], mean[1] + 4 * eig[1, 1], 'u2', zorder=5,
         fontsize=16, bbox=dict(facecolor='white', alpha=0.6))
plt.axis([0, 40, 0, 40])
plt.xlabel('feature 1')
plt.ylabel('feature 2')
# plt.show()

# PCA rotates the data so that the two axes (`x` and `y`) are aligned with the first two principal components.

# In OpenCV the PCAProject function both initiates and fits the transformed matrix to X2:
# X2 = cv2.PCAProject(X, mu, eig)

# Scikit: first instantiate using the `decomposition` module:
pca = decomposition.PCA(n_components=2)

# Now use the `fit_transform` method:
X2 = pca.fit_transform(X)

# The blob of data is rotated so that the most spread is along the `x` axis:
# Note the more even spread
plt.figure(figsize=(10, 6))
plt.plot(X2[:, 0], X2[:, 1], 'o')
plt.xlabel('first principal component')
plt.ylabel('second principal component')
plt.axis([-20, 20, -10, 10])
# plt.show()

# Implementing Independent Component Analysis (ICA) performs the same mathematical
# steps as PCA, but it chooses the components of the decomposition to be as independent as possible from each other.

# Again, first instantiate, then use the `fit_transform` method:
ica = decomposition.FastICA()
X2 = ica.fit_transform(X)

# Plot the projected data on the first two independent components:
plt.figure(figsize=(10, 6))
plt.plot(X2[:, 0], X2[:, 1], 'o')
plt.xlabel('first independent component')
plt.ylabel('second independent component')
plt.axis([-0.2, 0.2, -0.2, 0.2])
plt.show()


# all model classes

import joblib
import numpy as np
import os
import random
import time
import datetime

from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (precision_score, recall_score, f1_score, roc_curve)
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from xgboost import XGBClassifier

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from transformers import BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup

MODEL_DIR = 'models/'
MODEL_DIR_MAP = {
    "lr": 'logistic_regression',
    "svm": 'svm',
    "rf": 'random_forest',
    "xgb": 'xgboost',
    "bert": 'bert',
    "cnn": '1d_cnn'
}

class Classifier():

    def __init__(self, model_name, random_state):
        super().__init__()
        self.model_name = model_name
        self.model = None
        self.model_params = None
        self.random_state = random_state
        self.batch_size = None

    def fit(self, X_train, Y_train, X_test=None, word_embeddings=None, tune=False, save=True):
        if self.model_name not in ['bert', 'cnn']:
            model_file = os.path.join(MODEL_DIR + MODEL_DIR_MAP[self.model_name], 'tfidf_0.005/clf.joblib')
        else:
            model_file = os.path.join(MODEL_DIR + MODEL_DIR_MAP[self.model_name], 'saved_model')

        if self.model_name == 'lr':
            if tune:
                lr = LogisticRegression(random_state=self.random_state)
                parameters = {
                    "penalty": ['l2'],
                    "C": [0.5, 0.7, 0.8, 1.0],
                    }

                clf = GridSearchCV(lr, parameters, scoring='f1_macro', cv=5, n_jobs=-1, verbose=2)
                clf.fit(X_train, Y_train)
                if save:
                    joblib.dump(clf, model_file)

            else:
                clf = joblib.load(model_file)

            self.model = clf.best_estimator_
            self.model_params = clf.best_params_

        elif self.model_name == 'rf':
            if tune:
                rf = RandomForestClassifier(random_state=self.random_state)
                parameters = {
                    "n_estimators": [100, 200, 500],
                    "max_depth": [5, 7, 11, 17, 19, 23],
                    "max_features": ['auto', 'sqrt', 'log2'],
                    "criterion": ['gini', 'entropy']
                    }

                clf = GridSearchCV(rf, parameters, scoring='f1_macro', cv=5, n_jobs=-1, verbose=2)
                clf.fit(X_train, Y_train)
                if save:
                    joblib.dump(clf, model_file)

            else:
                clf = joblib.load(model_file)

            self.model = clf.best_estimator_
            self.model_params = clf.best_params_

        elif self.model_name == 'svm':
            if tune:
                svc = SVC(random_state=self.random_state)
                parameters = {
                    "C": [0.5, 0.8, 1.0],
                    "kernel": ['poly', 'rbf', 'sigmoid'],
                    "degree": [3, 5, 7, 9],
                    "gamma": ['scale', 'auto']
                    }

                clf = GridSearchCV(svc, parameters, scoring='f1_macro', cv=5, n_jobs=-1, verbose=2)
                clf.fit(X_train, Y_train)
                if save:
                    joblib.dump(clf, model_file)

            else:
                clf = joblib.load(model_file)

            self.model = clf.best_estimator_
            self.model_params = clf.best_params_

        elif self.model_name == 'xgb':
            if tune:
                xgb_classifier = XGBClassifier(random_state=self.random_state, n_jobs=-1, use_label_encoder=False)
                parameters = {
                    "n_estimators": [100, 200, 500],
                    "max_depth": [5, 9, 13, 23, 47],
                    "subsample": [0.8, 1.0],
                    "colsample_bytree": [0.5, 0.8, 1.0],
                }

                clf = GridSearchCV(xgb_classifier, parameters, scoring='f1_macro', cv=5, n_jobs=-1, verbose=2)
                clf.fit(X_train, Y_train)
                if save:
                    joblib.dump(clf, model_file)

            else:
                clf = joblib.load(model_file)

            self.model = clf.best_estimator_
            self.model_params = clf.best_params_

        elif self.model_name == 'cnn':
            model_file = os.path.join(model_file, 'model.pt')
            self.model = Classifier_1D_CNN(
                    embedding=word_embeddings,
                    dropout_size=0.2,
                    n_grams=[2, 3, 4],
                    out_channels=[50, 50, 50],
                    output_size_dense_layer=10,
                    classes=3,
                    num_epochs=30, lr=0.001, batch_size=32)
            if tune:
                self.model.fit(X_train, Y_train)

                if save:
                    torch.save(self.model.state_dict(), model_file)
            else:
                self.model.load_state_dict(torch.load(model_file))

        elif self.model_name == 'bert':
            self.model = CustomBERT()
            self.batch_size = 32
            if tune:
                self.model.finetune(
                    train_dataset=X_train, test_dataset=X_test, num_epochs=4, batch_size=self.batch_size, random_state=self.random_state)

                if save:
                    self.model.save(model_file)
            else:
                self.model.load(model_file)

        else:
            raise ValueError(f'Unsupported model type: {self.model_name}')

    def predict(self, X_test):

        if self.model_name != 'bert':
            y_pred = self.model.predict(X_test)
        else:
            y_pred = self.model.predict(X_test, batch_size=self.batch_size)

        return y_pred

    def interpret_results(self):
        pass

    def classification_results(self, true, pred, pred_prob=None, result_label='test'):

        result_dict = {"acc": (true==pred).mean(),
                    "precision": precision_score(true, pred, average='macro'),
                    "recall": recall_score(true, pred, average='macro'),
                    "f1": f1_score(true, pred, average='macro')}

        if pred_prob is not None:
            result_dict.update({"roc_auc": get_roc_auc(true, pred_prob)})

        return pd.DataFrame(result_dict, index=[result_label]).T

class Classifier_1D_CNN(nn.Module):

    def __init__(self, embedding, n_grams, out_channels, output_size_dense_layer, classes, batch_size, lr, num_epochs, dropout_size):

        super(Classifier_1D_CNN, self).__init__()

        self.lr = lr
        self.batch_size = batch_size
        self.num_epochs = num_epochs
        self.vocab_size, self.embed_dim = embedding.shape
        self.embedding = nn.Embedding.from_pretrained(embedding, freeze=True)
        self.out_channels = out_channels
        self.n_grams = n_grams
        self.conv1d_list = nn.ModuleList([
                                    nn.Conv1d(
                                        in_channels=self.embed_dim,
                                        out_channels=out_channels[i],
                                        kernel_size=n_grams[i])
                                    for i in range(len(self.n_grams))
                                    ]
                                    )

        self.dropout = nn.Dropout(dropout_size)
        self.dense1 = nn.Linear(np.sum(out_channels), output_size_dense_layer)
        self.output_dense = nn.Linear(output_size_dense_layer, classes)

    def forward(self, x):

        x = self.embedding(x).float().permute(0,2,1)
        x_conv_list = [F.relu(layer(x)) for layer in self.conv1d_list]

        x_pool_list = [F.max_pool1d(x_conv, kernel_size=x_conv.shape[2]) for x_conv in x_conv_list]

        x = torch.cat([x_pool.squeeze(dim=2) for x_pool in x_pool_list], dim=1)

        x = self.dense1(x)
        x = self.dropout(x)
        output = self.output_dense(x)
        output = F.softmax(output)

        return output

    def fit(self, train_data, train_labels):

        dataset = TensorDataset(train_data, torch.Tensor(train_labels))
        dataloader = DataLoader(dataset, batch_size=self.batch_size, shuffle=True)

        loss_fn = nn.CrossEntropyLoss()
        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr)

        self.train()

        for epoch in range(self.num_epochs):
            for id_batch, (x_batch, y_batch) in enumerate(dataloader):
                optimizer.zero_grad()
                y_pred = self(x_batch)
                loss = loss_fn(y_pred, y_batch)
                loss.backward()
                optimizer.step()

            print(f"Epoch {epoch}: training loss: {loss.item()}")

    def predict(self, test_data, prob=False):

        self.eval()

        pred = self(test_data).detach()

        if not prob:
            pred = torch.max(pred, dim=1)[-1]

        return pred.numpy()

class Classifier_NN(nn.Module):

    def __init__(self, input_size, output_size, hidden_layer_dims, dropouts):
        super().__init__()

        self.input_size = input_size
        self.output_size = output_size

        self.all_layer_dims = [self.input_size] + hidden_layer_dims + [self.output_size]
        self.linear_layers = nn.ModuleList([nn.Linear(self.all_layer_dims[i], self.all_layer_dims[i+1]) for i in range(len(self.all_layer_dims)-1)])

        self.dropouts = dropouts
        self.classes_ = None

    def forward(self, x):
        for layer in range(len(self.linear_layers)-1):
            x = F.relu(F.dropout(self.linear_layers[layer](x), p=self.dropouts[layer]))

        x = self.linear_layers[-1](x)
        x = F.softmax(x)

        return x

    def fit(self, train_data, train_labels, batch_size, lr, num_epochs):

        dataset = TensorDataset(torch.Tensor(train_data), torch.Tensor(train_labels))
        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

        loss_fn = nn.CrossEntropyLoss()
        optimizer = torch.optim.Adam(self.parameters(), lr=lr)

        self.train()

        for epoch in range(num_epochs):
            for id_batch, (x_batch, y_batch) in enumerate(dataloader):
                optimizer.zero_grad()
                y_pred = self(torch.Tensor(x_batch))
                loss = loss_fn(y_pred, y_batch)
                loss.backward()
                optimizer.step()

        print(f"Epoch {epoch}: traing loss: {loss.item()}")

    def predict(self, test_data, prob=False):
        self.eval()

        pred = self(torch.Tensor(test_data)).detach()

        if not prob:
            pred = torch.max(pred, dim=1)[-1]

        return pred.numpy()

class CustomBERT():

    def __init__(self, model_type="bert-base-uncased", to_cuda=True):

        self.model = BertForSequenceClassification.from_pretrained(
            model_type, num_labels=3, output_attentions=False, output_hidden_states=False)
        self.device = torch.device("cpu")
        if to_cuda:
            self.device = torch.device("cuda")
            self.model = self.model.cuda()

    def finetune(self, train_dataset, test_dataset, num_epochs, batch_size, random_state):

        train_loader = DataLoader(
            train_dataset, sampler=RandomSampler(train_dataset), batch_size=batch_size)

        optimizer = AdamW(self.model.parameters(), lr=2e-5, eps=1e-8)
        epochs = num_epochs
        total_steps = len(train_loader) * epochs
        scheduler = get_linear_schedule_with_warmup(
            optimizer, num_warmup_steps=0, num_training_steps=total_steps)

        random.seed(random_state)
        np.random.seed(random_state)
        torch.manual_seed(random_state)
        torch.cuda.manual_seed_all(random_state)

        training_stats = []
        total_t0 = time.time()
        for epoch_i in range(0, epochs):
            print("")
            print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
            print('Training...')

            t0 = time.time()
            total_train_loss = 0

            self.model.train()

            for step, batch in enumerate(train_loader):

                if step % 40 == 0 and not step == 0:
                    elapsed = format_time(time.time() - t0)
                    print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_loader), elapsed))

                b_input_ids = batch[0].to(self.device)
                b_input_mask = batch[1].to(self.device)
                b_labels = batch[2].to(self.device)

                self.model.zero_grad()

                result = self.model(
                    b_input_ids, token_type_ids=None, attention_mask=b_input_mask,
                    labels=b_labels, return_dict=True)

                loss = result.loss
                logits = result.logits
                total_train_loss += loss.item()

                loss.backward()

                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)

                optimizer.step()
                scheduler.step()

            avg_train_loss = total_train_loss / len(train_loader)
            training_time = format_time(time.time() - t0)
            print("")
            print("  Average training loss: {0:.2f}".format(avg_train_loss))
            print("  Training epcoh took: {:}".format(training_time))

            print("")
            print("Running on Test Data...")
            t0 = time.time()

            test_loader = DataLoader(
                test_dataset, sampler=SequentialSampler(test_dataset), batch_size=batch_size)

            self.model.eval()

            total_eval_accuracy = 0
            total_eval_loss = 0
            nb_eval_steps = 0
            for batch in test_loader:
                b_input_ids = batch[0].to(self.device)
                b_input_mask = batch[1].to(self.device)
                b_labels = batch[2].to(self.device)

                with torch.no_grad():
                    result = self.model(
                        b_input_ids, token_type_ids=None, attention_mask=b_input_mask,
                        labels=b_labels, return_dict=True)

                loss = result.loss
                logits = result.logits
                total_eval_loss += loss.item()

                logits = logits.detach().cpu().numpy()
                label_ids = b_labels.to('cpu').numpy()
                total_eval_accuracy += self.flat_accuracy(logits, label_ids)

            avg_test_accuracy = total_eval_accuracy / len(test_loader)
            print("  Accuracy: {0:.2f}".format(avg_test_accuracy))

            avg_test_loss = total_eval_loss / len(test_loader)
            test_time = format_time(time.time() - t0)

            print("  Test Loss: {0:.2f}".format(avg_test_loss))
            print("  Run on Test Data took: {:}".format(test_time))

            training_stats.append(
                {
                    'epoch': epoch_i + 1,
                    'Training Loss': avg_train_loss,
                    'Test Loss': avg_test_loss,
                    'Test Accuracy': avg_test_accuracy,
                    'Training Time': training_time,
                    'Test Time': test_time
                    }
                    )

        print("")
        print("Training complete!")
        print("Total training took {:} (h:mm:ss)".format(format_time(time.time()-total_t0)))

    def predict(self, test_dataset, batch_size):

        test_loader = DataLoader(
            test_dataset, sampler=SequentialSampler(test_dataset), batch_size=batch_size)

        self.model.eval()

        predictions = []
        for batch in test_loader:
            b_input_ids = batch[0].to(self.device)
            b_input_mask = batch[1].to(self.device)
            b_labels = batch[2].to(self.device)

            with torch.no_grad():
                result = self.model(
                    b_input_ids, token_type_ids=None, attention_mask=b_input_mask, return_dict=True)

            logits = result.logits
            logits = logits.detach().cpu().numpy()
            predictions.append(logits)

        predictions = np.concatenate(predictions, axis=0)
        predictions = np.argmax(predictions, axis=1).flatten()

        return predictions

    def save(self, file_path):

        self.model.save_pretrained(file_path)

    def load(self, file_path):

        self.model = BertForSequenceClassification.from_pretrained(file_path)

        if self.device == torch.device("cuda"):
            self.model = self.model.cuda()

    def flat_accuracy(self, preds, labels):

        pred_flat = np.argmax(preds, axis=1).flatten()
        labels_flat = labels.flatten()

        return np.sum(pred_flat == labels_flat) / len(labels_flat)

def format_time(elapsed):

    elapsed_rounded = int(round((elapsed)))

    return str(datetime.timedelta(seconds=elapsed_rounded))


import enum
from typing import Any, Dict, List, Optional, Union

import attr

"""
This module holds representations for request and results for the metrics API
endpoint provided by the Snap Store.

The relevant API documentation is available at:
https://dashboard.snapcraft.io/docs/api/snap.html?#fetch-metrics-for-snaps
"""

MetricsStatus = enum.Enum(  # type: ignore
    value="MetricsStatus",
    names=[("OK", "OK"), ("FAIL", "FAIL"), ("NO DATA", "NO DATA")],
)

class MetricsNames(enum.Enum):
    DAILY_DEVICE_CHANGE = "daily_device_change"
    INSTALLED_BASE_BY_CHANNEL = "installed_base_by_channel"
    INSTALLED_BASE_BY_COUNTRY = "installed_base_by_country"
    INSTALLED_BASE_BY_OPERATING_SYSTEM = "installed_base_by_operating_system"
    INSTALLED_BASE_BY_VERSION = "installed_base_by_version"
    WEEKLY_DEVICE_CHANGE = "weekly_device_change"
    WEEKLY_INSTALLED_BASE_BY_CHANNEL = "weekly_installed_base_by_channel"
    WEEKLY_INSTALLED_BASE_BY_COUNTRY = "weekly_installed_base_by_country"
    WEEKLY_INSTALLED_BASE_BY_OPERATING_SYSTEM = (
        "weekly_installed_base_by_operating_system"
    )
    WEEKLY_INSTALLED_BASE_BY_VERSION = "weekly_installed_base_by_version"

@attr.s(auto_attribs=True)
class MetricsFilter:
    snap_id: str
    metric_name: str
    start: str
    end: str

    def marshal(self) -> Dict[str, str]:
        return attr.asdict(self)

@attr.s(auto_attribs=True)
class Series:
    name: str
    values: List[Union[str, int]]
    currently_released: Optional[bool] = None

    def marshal(self) -> Dict[str, str]:
        obj = attr.asdict(self)

        if self.currently_released is None:
            obj.pop("currently_released")

        return obj

    @classmethod
    def unmarshal(cls, payload: Dict[str, Any]) -> "Series":
        name = payload.get("name")
        if not isinstance(name, str):
            raise ValueError(f"Invalid metric name: {name!r}")

        values = payload.get("values")
        if not isinstance(values, list) or not all(
            isinstance(b, str) or isinstance(b, int) or b is None for b in values
        ):
            raise ValueError(f"Invalid metric values: {values!r}")

        currently_released = payload.get("currently_released")
        if currently_released is not None and not isinstance(currently_released, bool):
            raise ValueError(
                f"Invalid metric currently_released: {currently_released!r}"
            )

        return cls(
            name=name,
            values=values,
            currently_released=currently_released,
        )

@attr.s(auto_attribs=True)
class MetricResults:
    status: MetricsStatus
    snap_id: str
    metric_name: str
    buckets: List[str]
    series: List[Series]

    def marshal(self) -> Dict[str, Any]:
        return {
            "status": self.status.name,
            "snap_id": self.snap_id,
            "metric_name": self.metric_name,
            "buckets": self.buckets,
            "series": [s.marshal() for s in self.series],
        }

    @classmethod
    def unmarshal(cls, payload: Dict[str, Any]) -> "MetricResults":
        status_string = payload.get("status", "")
        try:
            status = MetricsStatus[status_string]
        except (KeyError, TypeError):
            raise ValueError(f"Invalid metric status: {status_string!r}")

        snap_id = payload.get("snap_id")
        if not isinstance(snap_id, str):
            raise ValueError(f"Invalid metric snap id: {snap_id!r}")

        metric_name = payload.get("metric_name")
        if not isinstance(metric_name, str):
            raise ValueError(f"Invalid metric name: {metric_name!r}")

        buckets = payload.get("buckets")
        if not isinstance(buckets, list) or any(
            [not isinstance(b, str) for b in buckets]
        ):
            raise ValueError(f"Invalid metric buckets: {buckets!r}")

        series = payload.get("series")
        if not isinstance(series, list):
            raise ValueError(f"Invalid metric series: {series!r}")

        return cls(
            status=status,
            snap_id=snap_id,
            metric_name=metric_name,
            buckets=buckets,
            series=[Series.unmarshal(s) for s in series],
        )

@attr.s(auto_attribs=True)
class MetricsResults:
    metrics: List[MetricResults]

    def marshal(self) -> Dict[str, Any]:
        return {"metrics": [m.marshal() for m in self.metrics]}

    @classmethod
    def unmarshal(cls, payload: Dict[str, Any]) -> "MetricsResults":
        if not isinstance(payload, dict):
            raise ValueError(f"Invalid metrics results: {payload!r}")

        metrics = payload.get("metrics")
        if not isinstance(metrics, list):
            raise ValueError(f"Invalid metrics: {metrics!r}")

        return cls(metrics=[MetricResults.unmarshal(m) for m in metrics])


from .DoublyPositionList import PositionalList

class FavoritesList:
    """List of elements ordered from most frequently accessed to least"""

    # --------------NESTED ITEM CLASS----------

    class _item:
        __slots__ = '_value', '_count'

        def __init__(self, e):
            self._value = e
            self._count = 0

    # --------------NON PUBLIC UTILITIES--------

    def _find_position(self, e):
        """search for element e and return its Position (or none if not found)"""
        walk = self._data.first()
        while walk is not None and walk.element()._value != e:
            walk = self._data.after(walk)
        return walk

    def _move_up(self, p):
        """move an item at position p earlier in the list based on access count"""
        if p != self._data.first():
            cnt = p.element()._count
            walk = self._data.before(p)
            if cnt > walk.element()._count:
                while (walk != self._data.first() and
                       cnt > self._data.before(walk).element()._count()):
                    walk = self._data.before(walk)
                self._data.add_before(walk, self._data.delete(p))

    #---------PUBLIC METHODS------------
    def __init__(self):
        """Create an empty list of favorites"""
        self._data = PositionalList()

    def __len__(self):
        return len(self._data)

    def is_empty(self):
        return len(self._data) == 0

    def access(self, e):
        """access element e therby increasing its access counts"""
        p = self._find_position(e)
        if p is None:
            p = self._data.add_last(self._item(e))
        p.element()._count += 1
        self._move_up(p)

    def remove(self, e):
        """Remove element e foem the list of favorites"""
        p = self._find_position(e)              #find the position of the element
        if p is not None:
            self._data.delete(p)                #delete if found

    def top(self, k):
        """Generate a sequence of top k elements in terms of access count"""
        if not  1 <= k <=len(self):
            raise ValueError('Illegal value for k')
        walk = self._data.first()
        for j in range(k):
            item  = walk.element()
            yield item._value
            walk = self._data.after(walk)

class FavoritesListMTF(FavoritesList):
    """List of elements ordered with move-to-front heurisitc"""

    #----override _move_up to provide move to front semantics
    def _move_up(self, p):
        """Move accessed item to the front of the list"""
        if p != self._data.first():
            self._data.add_first(self._data.delete(p))

    # we override top as the list is no longer sorted
    def top(self, k):
        """Generate sequence of top k elements in terms of access counts"""
        if not 1 <= k <= len(self):
            raise ValueError("Illegal value of k")

        # we begin by making a copy of the list
        temp = PositionalList()
        for item in self._data:
            temp.add_last(item)

        # we repeatedly find report and remove element with largest count
        for j in range(k):
            highPos = temp.first()
            walk = temp.after(highPos)
            while walk is not None:
                if walk.element()._count > highPos.element()._count:
                    highPos = walk
                walk = temp.after(walk)
            #we have found the element with the highest count
            yield highPos.element()._value
            temp.delete(highPos)


from flask_script import Manager
from flask_migrate import Migrate, MigrateCommand
from api import create_app
from api.models import db

app = create_app()

manager = Manager(app)
migrate = Migrate(app, db)

manager.add_command("db", MigrateCommand)

@manager.command
def runserver():
    app.run(debug=True, host="0.0.0.0", port=5000)

@manager.command
def runworker():
    app.run(debug=False)

@manager.command
def recreate_db():
    """
    Recreates a local database. You probably should not use this on
    production.
    """
    db.drop_all()
    db.create_all()
    db.session.commit()

if __name__ == "__main__":
    manager.run()


# -*- coding: utf-8 -*-

from uuid import uuid1

from flask import Flask, jsonify
from flask_cors import CORS
from pre_request import pre, Rule

from code import ResponseCode, ResponseMessage
from log import logger
from utils import base64_to_img, create_date_dir

# 创建一个服务
app = Flask(__name__)
CORS(app, supports_credentials=True)

"""
# 方法功能说明
"""
@app.route(rule='/api/moduleName/methodName', methods=['POST'])
def methodName():

    # 参数校验
    rule = {
        "img": Rule(type=str, required=True),
        "file_name": Rule(type=str, required=False)
    }
    try:
        params = pre.parse(rule=rule)
    except Exception as e:
        logger.error(e)
        fail_response = dict(code=ResponseCode.RARAM_FAIL, msg=ResponseMessage.RARAM_FAIL, data=None)
        logger.error(fail_response)
        return jsonify(fail_response)

    # 获取参数
    image_b64 = params.get("img")
    file_name = params.get("file_name")
    if file_name is None:
        file_name = '{}.jpg'.format(uuid1())

    # 将base64字符串解析成图片保存
    img_root_path = "./img"
    img_base_path = create_date_dir(img_root_path)
    img_path = img_base_path + file_name
    try:
        base64_to_img(image_b64, img_path)
    except Exception as e:
        logger.error(e)
        fail_response = dict(code=ResponseCode.RARAM_FAIL, msg=ResponseMessage.RARAM_FAIL, data=None)
        logger.error(fail_response)
        return jsonify(fail_response)

    # 下面对保存的图片进行若干处理
    result = image_b64
    logger.info("测试日志记录")

    # 处理完成后删除生成的图片文件
    # os.remove(img_path)

    # 成功的结果返回
    success_response = dict(code=ResponseCode.SUCCESS, msg=ResponseMessage.SUCCESS, data=result)
    logger.info(success_response)
    return jsonify(success_response)

if __name__ == '__main__':
    # 解决中文乱码问题
    app.config['JSON_AS_ASCII'] = False
    # 启动服务 指定主机和端口
    app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)


# -*- coding: utf-8 -*-
# __author__ = "gaohao"

from biocluster.agent import Agent
from biocluster.tool import Tool
import os,re
import string
import types
import subprocess
from biocluster.core.exceptions import OptionError
from mbio.packages.statistical.normalization import Normalization
import pandas as pd

class RocAgent(Agent):
    """
    计算roc，需要roc_plus_gh.pl
    author: gaohao
    last_modifued:2018.09.29
    """

    def __init__(self, parent):
        super(RocAgent, self).__init__(parent)
        options = [
            {"name": "mode", "type": "int", "default": 1},# 1.丰度表的前n物种计算模式;2.根据物种名提取特殊物种丰度计算模式（目前页面不存在2模式）
            {"name": "abu_table", "type": "infile", "format": "meta.otu.otu_table, meta.otu.tax_summary_dir"},
            {"name": "group_table", "type": "infile", "format": "meta.otu.group_table"},
            {"name": "method", "type": "string", "default": "sum"},
            {"name": "confidence_interval", "type": "string", "default":"0.95"},
            {"name": "top_n", "type": "int", "default": 100},
            {"name": "norm_method", "type": "string", "default": ""},
            {"name": "env_labs", "type": "string", "default": ''},
        ]
        self.add_option(options)

    def check_options(self):
        if not self.option('abu_table') and self.option('mode') in [1]:
            raise OptionError('abu_table file must be provided !', code="34101501")
        if self.option('mode') in [1]:
            self.option('abu_table').get_info()
            if self.option('abu_table').prop['sample_num'] < 2:
                raise OptionError('The number of samples in the abu_table table is less than 2, and ROC calculation is not possible', code="34101502")
            if not self.option('group_table').is_set:
                raise OptionError("group_table file must be provided !", code="34101503")
            if self.option('method') != '':
                if self.option('method') not in ['sum', 'average', 'median']:
                    raise OptionError("Abundance calculation method can only choose one of sum, average, median", code="34101504")

    def set_resource(self):
        """
        设置内存和CPU
        """
        self._cpu = 10
        self._memory = '15G'

    def end(self):
        result_dir = self.add_upload_dir(self.output_dir)
        result_dir.add_relpath_rules([
            [".", "", "ROC分析结果目录"],
            ["./roc_curve.xls", "xls", "ROC受试者工作特征曲线数据"],
            ["./roc_auc.xls", "xls", "ROC受试者工作特征曲线-AUC VALUE"],
            ["./roc_auc_smooth.xls", "xls", "ROC受试者工作特征曲线-AUC VALUE"],
            ["./roc_curve_smooth.xls", "xls", "ROC受试者工作特征曲线平滑处理后数据"],
            ["./roc_interval.xls", "xls", "ROC受试者工作特征曲线的可信区间"],
            ["./best_loc.xls", "xls", "ROC受试者工作特征曲线最佳cut off点"]
        ])
        super(RocAgent, self).end()

class RocTool(Tool):
    def __init__(self, config):
        super(RocTool, self).__init__(config)
        self._version = '1.0.1'
        self.gcc = self.config.SOFTWARE_DIR + '/gcc/5.1.0/bin'
        self.gcc_lib = self.config.SOFTWARE_DIR + '/gcc/5.1.0/lib64'
        self.set_environ(PATH=self.gcc, LD_LIBRARY_PATH=self.gcc_lib)

    def run(self):
        """
        运行
        """
        super(RocTool, self).run()
        self.run_roc()
        self.set_output()
        self.end()

    def data_process(self):
        '''
        当没有top_n==-1及不设置top时 top_n = 丰度表的所有行
        当配置设置了top_n且参数中有 env_labs时，先去除包含env_labs的行；
           再取top_n，然后和env_labs的行合并，最后top_n = top_n + len(env_labs)
        '''
        import commands
        abu_table = self.option('abu_table').path
        if self.option('top_n') == -1:
            cmd = '/usr/bin/wc -l ' + self.option('abu_table').path
            status, output = commands.getstatusoutput(cmd)
            lines = int(output.split()[0]) - 1
            self.option('top_n', lines)
        elif self.option('env_labs'):
            top_n = self.option('top_n')
            env_labs = self.option('env_labs').split(',')
            tb = pd.read_csv(abu_table, sep='\t')
            first = tb.columns[0]
            envs = tb[tb[first].isin(env_labs)]
            otu = tb[tb[first].isin(env_labs).isin([False])]
            if self.option('method') == 'sum':
                filters = otu.sum(axis=1).sort_values(ascending=False)
            elif self.option('method') == 'average':
                filters = otu.mean(axis=1).sort_values(ascending=False)
            elif self.option('method') == 'median':
                filters = otu.median(axis=1).sort_values(ascending=False)
            otu = otu.head(top_n)
            tb = pd.concat([otu, envs])
            print(top_n)
            print(len(otu.index))
            print(len(envs.index))
            abu_table = os.path.join(self.work_dir, 'new_abu.xls')
            tb.to_csv(abu_table, sep='\t', index=False)
            self.option('top_n', top_n + len(env_labs))

        if self.option('norm_method'):
            new_abu = os.path.join(self.work_dir, 'normalized.xls')
            Normalization(abu_table, norm_meth=self.option('norm_method'), out=new_abu).run()
            abu_table = new_abu
        return abu_table

    def run_roc(self):
        """
        运行roc_plus_gh.pl
        """
        abu_table = self.data_process()
        cmd = self.config.SOFTWARE_DIR + '/miniconda2/bin/perl ' + self.config.PACKAGE_DIR + '/metagenomic/scripts/roc_plus_gh.pl '
        cmd += '-o %s ' % (self.work_dir + '/ROC/')
        if not os.path.exists(self.work_dir + '/ROC/'):
            os.mkdir(self.work_dir + '/ROC/')
        if self.option('mode') in [1]:
            cmd += '-i %s ' % (abu_table)
        cmd += '-mode %d ' % (self.option('mode'))
        cmd += '-group %s ' % (self.option('group_table').prop['path'])
        if self.option('method'):
            cmd += '-method %s ' % (self.option('method'))
        if self.option('mode') == 1:
            cmd += '-n %d ' % (self.option('top_n'))
        if self.option('confidence_interval'):
            cmd += '-conf_level %s ' % (self.option('confidence_interval'))
        self.logger.info('开始运行roc_plus_gh.pl计算ROC相关数据')
        try:
            subprocess.check_output(cmd, shell=True)
            self.logger.info('生成 roc.cmd.r 成功')
        except subprocess.CalledProcessError:
            self.logger.info('生成 roc.cmd.r 失败')
            self.set_error('Unable to generate roc.cmd.r file', code="34101501")
        try:
            subprocess.check_output(self.config.SOFTWARE_DIR +
                                    '/program/R-3.3.1/bin/R --restore --no-save < %s/roc.cmd.r' % (self.work_dir + '/ROC'), shell=True)
            self.logger.info('ROC计算成功')
        except subprocess.CalledProcessError:
            self.logger.info('ROC计算失败')
            self.set_error('R running calculation ROC failed', code="34101502")
        self.logger.info('运行calc_roc.pl程序进行ROC计算完成')

    def set_output(self):
        files =os.listdir(self.work_dir + '/ROC')
        for file in files:
            if re.search(r'.xls',file):
                if os.path.exists(self.output_dir + '/' +file):
                    os.remove(self.output_dir + '/' +file)
                os.link(self.work_dir + '/ROC/' + file,self.output_dir + '/' +file)