import tensorflow as tf
from network.silknet import LoadInterface
from network.silknet.FolderDataReader import FolderDataReader
from interface import implements
import configparser as cp
import gzip
import pickle
import os
from tensorflow.contrib.ndlstm.python import lstm2d as lstm2d_lib
import numpy as np
import cv2
import sys

slim = tf.contrib.slim

class DataLoader(implements(LoadInterface)):
    def load_datum(self, full_path):
        # The file is compressed, so load it using gzip
        f = gzip.open(os.path.join(full_path, '__dump__.pklz'), 'rb')
        doc = pickle.load(f)
        f.close()

        # We don't need to anything with this
        input_tensor = doc.input_tensor

        # Convert left-share to one-hot encoding
        left_class = doc.classes_tensor[:,:,0]
        left_class_one_hot = np.zeros((256,256,2))
        left_class_one_hot[left_class==0, 0] = 1
        left_class_one_hot[left_class==1, 1] = 1

        # Convert top-share to one-hot encoding
        top_class = doc.classes_tensor[:,:,1]
        top_class_one_hot = np.zeros((256,256,2))
        top_class_one_hot[top_class==0, 0] = 1
        top_class_one_hot[top_class==1, 1] = 1

        loss_mask = doc.zone_mask * doc.word_mask

        return input_tensor, left_class_one_hot, top_class_one_hot, loss_mask

class Parser2d:
    def __init__(self, config_section):
        config = cp.ConfigParser()
        config.read('config.ini')
        self.train_path = config[config_section]['train_data_path']
        self.test_path = config[config_section]['test_data_path']
        self.validation_data_path = config[config_section]['validation_data_path']
        self.learning_rate = float(config[config_section]['learning_rate'])
        self.save_after = int(config[config_section]['save_after'])
        self.model_path = config[config_section]['model_path']
        self.from_scratch = int(config[config_section]['from_scratch']) == 1
        self.batch_size = int(config[config_section]['batch_size'])
        self.summary_path = config[config_section]['summary_path']

        self.alpha_left = float(config['zone_segment']['alpha_left'])
        self.alpha_top = float(config['zone_segment']['alpha_top'])

        # For usage from other functions
        self.input_placeholder = None
        self.classifier_left_same_placeholder = None
        self.classifier_top_same_placeholder = None
        self.optimizer = None
        self.loss_mask_placeholder = None
        self.loss = None
        self.loss_left = None
        self.loss_top = None

    @staticmethod
    def arg_scope(weight_decay=0.0005):
        with slim.arg_scope([slim.conv2d, slim.fully_connected],
                            activation_fn=None,
                            weights_regularizer=slim.l2_regularizer(weight_decay),
                            biases_initializer=tf.zeros_initializer()):
            with slim.arg_scope([slim.conv2d], padding='SAME') as arg_sc:
                return arg_sc

    def construct_graphs(self):
        self.input_placeholder = tf.placeholder("float32", shape=[self.batch_size, 256, 256, 308])
        self.classifier_left_same_placeholder = tf.placeholder("float32", shape=[self.batch_size, 256, 256, 2])
        self.classifier_top_same_placeholder = tf.placeholder("float32", shape=[self.batch_size, 256, 256, 2])
        self.loss_mask_placeholder = tf.placeholder("float32", shape=[self.batch_size, 256, 256])

        network = lstm2d_lib.separable_lstm(self.input_placeholder, 100) # (B,256,256,100)

        with slim.arg_scope(Parser2d.arg_scope()):
            output_left_same = slim.conv2d(network, 2, [1, 1], scope='logits_left_same') # (B,256,256,2)
            output_top_same = slim.conv2d(network, 2, [1, 1], scope='logits_top_same') # (B,256,256,2)

        # Apply softmax cross entropy
        loss_left = tf.nn.softmax_cross_entropy_with_logits(labels=self.classifier_left_same_placeholder, logits=output_left_same)
        loss_top = tf.nn.softmax_cross_entropy_with_logits(labels=self.classifier_top_same_placeholder, logits=output_top_same)

        # Mask the loss
        loss_left = tf.multiply(loss_left, self.loss_mask_placeholder)
        loss_top = tf.multiply(loss_top, self.loss_mask_placeholder)

        # Reduce mean (over only the masked indices)
        # Reducing twice in axis 1 will keep the batch dimension (as required)
        num_words = tf.reduce_sum(tf.reduce_sum(self.loss_mask_placeholder, axis=1), axis=1) # (B)
        # Finally, we have to reduce mean across batch dimension
        self.loss_left = tf.reduce_mean(tf.reduce_sum(tf.reduce_sum(loss_left, axis=1), axis=1) / num_words)
        self.loss_top = tf.reduce_mean(tf.reduce_sum(tf.reduce_sum(loss_top, axis=1), axis=1) / num_words)

        left_predicted_class = tf.argmax(output_left_same, axis=3) #(B,H,W)
        top_predicted_class = tf.argmax(output_left_same, axis=3)

        left_gt_class = tf.argmax(self.classifier_left_same_placeholder, axis=3) #(B,H,W)
        top_gt_class = tf.argmax(self.classifier_top_same_placeholder, axis=3)

        self.accuracy_left = tf.reduce_sum(tf.reduce_sum(
            tf.cast(tf.equal(left_predicted_class, left_gt_class), tf.float32) * self.loss_mask_placeholder, axis=1), axis=1) / num_words # (B)
        self.accuracy_top = tf.reduce_sum(tf.reduce_sum(
            tf.cast(tf.equal(top_predicted_class, top_gt_class), tf.float32) * self.loss_mask_placeholder, axis=1), axis=1) / num_words # (B)

        self.positive_predicted_left = tf.reduce_sum(
            tf.reduce_sum(tf.cast(tf.equal(left_predicted_class, tf.zeros_like(left_predicted_class)), tf.float32) * self.loss_mask_placeholder, axis=1),
            axis=1) / num_words
        self.positive_predicted_top = tf.reduce_sum(
            tf.reduce_sum(tf.cast(tf.equal(top_predicted_class, tf.zeros_like(top_predicted_class)), tf.float32) * self.loss_mask_placeholder, axis=1),
            axis=1) / num_words

        self.num_words = num_words

        # Combine both top and left loss
        self.loss = self.alpha_left * self.loss_left + self.alpha_top * self.loss_top

        summary_loss_complete = tf.summary.scalar('loss_complete', self.loss)
        summary_loss_left = tf.summary.scalar('loss_left', self.loss_left)
        summary_loss_top = tf.summary.scalar('loss_top', self.loss_top)
        summary_accuracy_top = tf.summary.scalar('accuracy_top', tf.reduce_mean(self.accuracy_top))
        summary_accuracy_left = tf.summary.scalar('accuracy_left', tf.reduce_mean(self.accuracy_left))
        summary_predicted_left_zero = tf.summary.scalar('predicted_left_zero', tf.reduce_mean(self.positive_predicted_left))
        summary_predicted_top_zero = tf.summary.scalar('predicted_top_zero', tf.reduce_mean(self.positive_predicted_left))

        self.summaries = tf.summary.merge(
            [summary_loss_complete, summary_loss_left, summary_loss_top, summary_accuracy_top, summary_accuracy_left,
             summary_predicted_left_zero, summary_predicted_top_zero])

        # Optimizer is Adam
        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.loss)

        self.saver_all = tf.train.Saver()

    def clean_summary_dir(self):
        for the_file in os.listdir(self.summary_path):
            file_path = os.path.join(self.summary_path, the_file)
            try:
                if os.path.isfile(file_path):
                    os.unlink(file_path)
            except Exception as e:
                print(e)

    def train(self):
        train_set = FolderDataReader(self.train_path, DataLoader())
        train_set.init()
        init = tf.global_variables_initializer()

        print("\n\nNOTE: The cost in the following log will be mean across batch. However, accuracies and positive figures will"
              "be for the first data point in the batch for better debugging.\n\n")

        with tf.Session() as sess:
            if self.from_scratch:
                self.clean_summary_dir()

            summary_writer = tf.summary.FileWriter(self.summary_path, sess.graph)
            sess.run(init)

            if not self.from_scratch:
                self.saver_all.restore(sess, self.model_path)
                print("\n\nINFO: Saving model\n\n")
                with open(self.model_path+'.txt', 'r') as f:
                    iteration = int(f.read())
            else:
                iteration = 0

            while True:
                # Save the model and iteration number to ckpt and txt files respectively
                if iteration % self.save_after == 0:
                    print("\n\nINFO: Saving model\n\n")
                    self.saver_all.save(sess, self.model_path)
                    with open(self.model_path+'.txt', 'w') as f:
                        f.write(str(iteration))

                data, epochs, ids = train_set.next_batch(self.batch_size)
                inputs = [data[i][0] for i in range(len(data))]
                left_same = [data[i][1] for i in range(len(data))]
                top_same = [data[i][2] for i in range(len(data))]
                loss_mask = [data[i][3] for i in range(len(data))]

                input_feed = {
                    self.input_placeholder : inputs,
                    self.classifier_left_same_placeholder : left_same,
                    self.classifier_top_same_placeholder : top_same,
                    self.loss_mask_placeholder : loss_mask
                }

                run_ops = [self.optimizer, self.loss, self.accuracy_left, self.accuracy_top,
                           self.positive_predicted_left, self.positive_predicted_top, self.summaries]

                ops_results = sess.run(run_ops, feed_dict=input_feed)

                print("Cost", ops_results[1], "Accuracy Left", ops_results[2][0], "Positive Left",
                      ops_results[4][0], "Accuracy Top", ops_results[3][0], "Positive top", ops_results[5][0])

                summary_writer.add_summary(ops_results[6], iteration)
                iteration += 1

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Error in arguments")
        print("Usage: python table_parse_2d/parser_2d.py [config_section_name]")
        print("\tFor example, if the section name in the config file is `zone_segment` you can issue:\n"
              "\tpython table_parse_2d/parser_2d.py zone_segment")
        sys.exit(-1)
    parser = Parser2d(sys.argv[1])
    parser.construct_graphs()
    parser.train()


import datetime
import sys

import discord
from discord.ext import commands
from sqlalchemy.exc import DBAPIError
from sqlalchemy.sql import func

from sweeperbot.cogs.utils import time
from sweeperbot.cogs.utils.timer import Timer
from sweeperbot.db import models
from sweeperbot.utilities.helpers import set_sentry_scope

class Reminder(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    def _remind(
        self, guild_id: int, remind_user_id: int, creator_user_id: int, remind_text: str
    ):

        try:
            guild = self.bot.get_guild(guild_id)
            if not guild:
                raise ValueError(f"Bot can't find guild {guild_id}")
            member = guild.get_member(remind_user_id)
            if not member:
                raise ValueError(
                    f"Bot can't find member {remind_user_id} in guild {guild} ({guild.id})"
                )
            creator_user = self.bot.get_user(creator_user_id)
        except ValueError:
            self.bot.log.exception(f"Can't execute reminder")
            raise

        settings = self.bot.guild_settings.get(member.guild.id)
        has_modmail_server = settings.modmail_server_id

        footer_text = (
            self.bot.constants.footer_with_modmail.format(guild=member.guild)
            if has_modmail_server
            else self.bot.constants.footer_no_modmail.format(guild=member.guild)
        )

        try:
            self.bot.loop.create_task(
                member.send(
                    f"Here is the reminder requested by **{creator_user}** from the **{guild}** server.\n\n'{remind_text}'{footer_text}"
                )
            )

            self.bot.log.info(
                f"Sent reminder to: {member} ({member.id}) in guild {member.guild}"
            )
        except Exception as e:
            if not (type(e) == discord.errors.Forbidden and e.code == 50007):
                self.bot.log.exception(
                    f"There was an error while informing {member} ({member.id}) about their reminder. Unable to send messages to this user."
                )
            return False

        return True

    @commands.has_permissions(manage_messages=True)
    @commands.guild_only()
    @commands.command()
    @commands.cooldown(3, 120, commands.BucketType.user)
    async def remind(
        self,
        ctx,
        remind_user: str,
        remind_time: time.UserFriendlyTime(commands.clean_content, default="\u2026"),
        *,
        remind_text: str,
    ):
        """Creates a reminder that the bot will remind you of. If the person creating the reminder has 'Manage Messages' permission then any user or role pings will be left alone. If they don't (aka regular user) then the user and role pings will be not be allowed.

        Things to figure out/do
        - Does it DM the user the reminder or put it in chat? What if it has a role mention, that we'd want in chat 'remind me in 1day There is a @moderators meeting'
        - Create a delete reminder command - use note delete as an example
        - Future: Allow it to recognize time in format of datestamp vs duration

        Requires Permission: Send Messages

        Parameters
        -----------
        ctx: context
            The context message involved.
        remind_user: str
            The user the reminder is for. For self can use 'me' or a user mention or user ID of someone else.
        remind_time: time
            When to be reminded. Format of 1m (minute), 2h (hours), 3d (days), 4mo (months), 5y (years)
        remind_text: str
            The text to be reminded of.
        """

        session = self.bot.helpers.get_db_session()
        try:
            self.bot.log.info(
                f"CMD {ctx.command} called by {ctx.message.author} ({ctx.message.author.id})"
            )
            # Check if user is blacklisted, if so, ignore.
            if await self.bot.helpers.check_if_blacklisted(
                ctx.message.author.id, ctx.message.guild.id
            ):
                self.bot.log.debug(
                    f"User {ctx.message.author} ({ctx.message.author.id}) Blacklisted, unable to use command {ctx.command}"
                )
                return

            # If we were provided an ID, let's try and use it
            if remind_user.lower() in ["me", "myself"]:
                remind_user = ctx.message.author
            else:
                remind_user = await self.bot.helpers.get_member_or_user(
                    remind_user, ctx.message.guild
                )
                if not remind_user:
                    return await ctx.send(
                        f"Sorry, I am unable to find the person you are trying to set the reminder for. Please make sure you are using a proper @ mention of the user or their User ID."
                    )

            # Variables used for the database/later on
            creator_user = ctx.message.author
            remind_time = remind_time.dt
            guild = ctx.message.guild

            # Save in the database
            # Get the DB profile for the guild
            db_guild = await self.bot.helpers.db_get_guild(session, guild.id)
            # Get the DB profile for the remind_user
            db_remind_user = await self.bot.helpers.db_get_user(session, remind_user.id)
            # DB Entry
            db_remind = models.Reminder(
                creator_user_id=ctx.message.author.id,
                remind_user=db_remind_user,
                server=db_guild,
                expires=remind_time,
                text=remind_text,
            )
            session.add(db_remind)
            session.commit()

            # Add timer to send the reminder
            timer = Timer.temporary(
                guild.id,
                remind_user.id,
                creator_user.id,
                remind_text,
                event=self._remind,
                expires=remind_time,
                created=datetime.datetime.now(datetime.timezone.utc),
            )
            timer.start(self.bot.loop)

            # Tell user reminder is created
            # TO DO - Make this look a little nicer.
            await ctx.send(
                f"{ctx.message.author.mention}, I have now created the reminder for {remind_user.mention}."
            )

        except discord.HTTPException as err:
            set_sentry_scope(ctx)
            self.bot.log.error(
                f"Discord HTTP Error responding to {ctx.command} request. {sys.exc_info()[0].__name__}: {err}"
            )
            await ctx.send(
                f"Error processing {ctx.command}. Error has already been reported to my developers."
            )
        except DBAPIError as err:
            set_sentry_scope(ctx)
            self.bot.log.exception(
                f"Error processing database query. {sys.exc_info()[0].__name__}: {err}"
            )
            await ctx.send(
                f"Error processing {ctx.command}. Error has already been reported to my developers."
            )
            session.rollback()
        except Exception as err:
            set_sentry_scope(ctx)
            self.bot.log.exception(
                f"Error responding to {ctx.command}. {sys.exc_info()[0].__name__}: {err}"
            )
            await ctx.send(
                f"Error processing {ctx.command}. Error has already been reported to my developers."
            )
        finally:
            session.close()

    @commands.has_permissions(send_messages=True)
    @commands.guild_only()
    @commands.command()
    async def reminders(self, ctx):
        """Sends you a DM with a list of all reminders for you."""

        session = self.bot.helpers.get_db_session()
        try:
            self.bot.log.info(
                f"CMD {ctx.command} called by {ctx.message.author} ({ctx.message.author.id})"
            )
            # Check if user is blacklisted, if so, ignore.
            if await self.bot.helpers.check_if_blacklisted(
                ctx.message.author.id, ctx.message.guild.id
            ):
                self.bot.log.debug(
                    f"User {ctx.message.author} ({ctx.message.author.id}) Blacklisted, unable to use command {ctx.command}"
                )
                return

            # Get the DB profile for the remind_user
            db_remind_user = await self.bot.helpers.db_get_user(
                session, ctx.message.author.id
            )

            reminders = (
                session.query(
                    func.coalesce(
                        models.Reminder.updated, models.Reminder.created
                    ).label("created"),
                    models.Reminder.id,
                    models.Reminder.creator_user_id,
                    models.Reminder.remind_user_id,
                    models.Server,
                    models.Reminder.expires,
                    models.Reminder.text,
                )
                .join(models.Server, models.Server.id == models.Reminder.server_id)
                .filter(
                    models.Reminder.expires
                    > datetime.datetime.now(datetime.timezone.utc),
                    models.Reminder.remind_user_id == db_remind_user.id,
                )
                .all()
            )

            # TO DO - Use pagination to accomplish this - look at tag list command for example
            msg_text = ""
            for item in reminders:
                msg_text += f"{item.id}. **Expires**: [{item.expires} UTC]\n**Reminder Text:** [{item.text}]\n"

            await ctx.message.author.send(
                f"Here's a list of your reminders:\n\n{msg_text if len(reminders) >0 else 'You do not have any reminders.'}"
            )
        except discord.HTTPException as err:
            set_sentry_scope(ctx)
            self.bot.log.error(
                f"Discord HTTP Error responding to {ctx.command}. {sys.exc_info()[0].__name__}: {err}"
            )
            await ctx.send(
                f"Error processing {ctx.command}. Error has already been reported to my developers."
            )
        except DBAPIError as err:
            set_sentry_scope(ctx)
            self.bot.log.exception(
                f"Error processing database query. {sys.exc_info()[0].__name__}: {err}"
            )
            await ctx.send(
                f"Error processing {ctx.command}. Error has already been reported to my developers."
            )
            session.rollback()
        except Exception as err:
            set_sentry_scope(ctx)
            self.bot.log.exception(
                f"Error responding to {ctx.command}. {sys.exc_info()[0].__name__}: {err}"
            )
            await ctx.send(
                f"Error processing {ctx.command}. Error has already been reported to my developers."
            )
        finally:
            session.close()

def setup(bot):
    bot.add_cog(Reminder(bot))


import requests
import logging
import json
from datetime import datetime

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def publish_to_zenodo(zenodo_url, recid, token, files, version = None):
    ''' Publish files to zenodo repository.

    :param zenodo_url: "https://sandbox.zenodo.org/api" or "https://zenodo.org/api"
    :param doi: the destination record id, such as 541234
    :param token: the access token to write to zenodo repository
    :param files: a list of filename and content pair to upload.
    :param version: the new version string, such as 1.0.1, or 2021.09.20
    '''
    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"}
    params = {'access_token': token}

    logger.info(f'start uploading and publishing to zenodo for record {recid}')
    logger.info(f'\tto the zenodo url {zenodo_url}')
    # first, check the status of this repository
    r = requests.get(f'{zenodo_url}/deposit/depositions/{recid}', headers=headers)
    if not r.ok:
        raise r.raise_for_status()

    rj_links = r.json()['links']
    draft_url =  rj_links.get('latest_draft', None)
    if not draft_url:
        # switch to the latest version
        latest_url = rj_links['latest']
        latest_doi = get_doi_from_record_link(latest_url, zenodo_url)
        r = requests.get(f'{zenodo_url}/deposit/depositions/{latest_doi}', headers=headers)
        if not r.ok:
            raise r.raise_for_status()
        rj_links = r.json()['links']
        # create draft from the latest version
        draft_url = create_new_draft(rj_links['newversion'], params)
    # switch to draft
    logger.info('\tchange to draft version')
    r = requests.get(draft_url, headers=headers)
    if not r.ok:
        raise r.raise_for_status()
    rj = r.json()
    rj_files = rj['files']
    metadata = rj['metadata']

    # delete all the old files!
    logger.info('\tdelete old files from draft')
    for f in rj_files:
        of_id = f['id']
        of_url = f'{draft_url}/files/{of_id}'
        print ( f'deleting old file: {of_url}' )
        r = requests.delete(of_url, params=params)
        if not r.ok:
            raise r.raise_for_status()

    # upload new files
    logger.info('\tupload new files')
    draft_files = f'{draft_url}/files'
    upload_url = f'{draft_files}?access_token={token}'
    for nf in files:
        data = {'name': nf['filename']}
        r = requests.post(upload_url, data=data, files={'file': nf['content']})
        if not r.ok:
            raise r.raise_for_status()

    # update the new version in metadata
    if version:
        metadata['version'] = version
        r = requests.put( draft_url, headers=headers, data = json.dumps({'metadata': metadata}) )
        if not r.ok:
            raise r.raise_for_status()

    # publish
    logger.info('\tstart publishing')
    r = requests.post(f'{draft_url}/actions/publish', headers=headers)
    if not r.ok:
        raise r.raise_for_status()
    logger.info('Publish successfully!')

def get_doi_from_record_link(rec_url, zenodo_url):
    # get '917028' from 'https://sandbox.zenodo.org/api/records/917028'
    return rec_url[len(zenodo_url) + len('/records/'):]

def create_new_draft(nv_url, params):
    logger.info('\tcreate a new draft version')
    r = requests.post(nv_url, params=params)
    if not r.ok:
        raise r.raise_for_status()
    return r.json()['links']['latest_draft']


""" Q-Learning module """

import numpy as np
import tensorflow as tf

from .base_learner import BaseLearner

class QLearner(BaseLearner):
    """Q-table Learner

    This class implements a Q-table.

    Attributes:
        params (dictionary):containing the parameters of the model
    """
    def __init__(self, params):
        """Constructs the Q-table.

        Args:
            params (dictionary):contains the parameters of the learner
        """
        super().__init__(params)
        self.num_bins = 10

        # compute the shape of the Q-table
        self.state_shape = tuple(self.num_bins *
                                 np.ones(self.num_features, dtype=int))
        # initialize array with number of visits
        self.num_visits = np.zeros(self.state_shape)

        # initialize array with the cumulative reward
        self.reward = np.zeros(self.state_shape)

        # initialize the cell sizes for each feature
        self.cell_sizes = (self.high_limits - self.low_limits) / self.num_bins

    def get_cell(self, state):
        """Returns the cell in which the state is.

        Args:
            state (numpy array):state at which we wish to obtain the
                corresponding cell

        Returns:
            cell: tuple
                cell indices at which the Q-table will be updated
        """
        if self.tanh_mask is not None:
            state[:, self.tanh_mask] = \
                    tf.tanh(state[:, self.tanh_mask]).numpy()
        cell = (state - self.low_limits) / self.cell_sizes
        return tuple(cell.astype(int))

    def fit(self, state, reward):
        """Fits the Q-table at a given state.

        Args:
            state (numpy array):state at which the Q-table needs to fit
            reward (float):reward that was yielded when passing
                through this state
        """
        state_cell = self.get_cell(state)
        self.reward[state_cell] += reward
        self.num_visits[state_cell] += 1

    def forward(self, state, action=None):
        """Forwards the Q-able through the state (action) provided.

        Args:
            state (numpy array):state at which the Q-table needs
                to be evaluated
            action (int):if the Q-table is formed with the state action method,
                action needs to be provided as an option.

        Returns:
            value: float
                the actual Q-value at the state provided
        """
        state_cell = self.get_cell(state)
        if self.num_visits[state_cell] > 0:
            value = self.reward[state_cell]/self.num_visits[state_cell]
        else:
            value = 0
        return value


import pandas as pd
import seaborn as sns
import numpy as np
import streamlit as st
import matplotlib.pyplot as plt

st.title("Analyse de l'évolution des performances mécanique par continent")

df = pd.read_csv('https://raw.githubusercontent.com/murpi/wilddata/master/quests/cars.csv')

# supprimer '.' et espaces de la colonne continent
df.continent = df.continent.apply(lambda x: x.replace(' ', '').replace('.', ''))

st.subheader('Heatmap')

# display heatmap
plt.figure(figsize = (20, 10))
viz_correlation = sns.heatmap(df.corr(), center = 0, cmap = "vlag", annot=True, vmin = -1, vmax = 1, mask = np.triu(np.ones_like(df.corr())))

st.pyplot(viz_correlation.figure)

st.subheader('Analyse de la corrélation')
st.markdown("On constate qu'entre 1971 et 1983, il n'y a pas de corrélation significative concernant l'évolution du nombre de cylindre, la puissance de la cylindré, la puissance du moteur (hp) et le time-to-60. Concretement, il n'y a aps d'évolution ordonnée de tous ces paramètres entre 1971 et 1983 En revanche (toutes années confondues), il y a une corrélation, positive, forte, évidente entre le poids, la puissance du moteur et la cylindré.")

continent = st.multiselect(label = 'Choisi un continent', options = ['Europe', 'US', 'Japan'])

if len(continent) == 0:
  print()
else:
  df_continent = df.loc[((df.continent.isin(continent)))]

  st.subheader(', '.join(continent))

  # pivot table for mean by hp evolution
  pt = pd.pivot_table(df_continent, index = 'year', values = 'hp', aggfunc = 'mean')
  pt['year'] = pt.index

  # Evolution de la puissance moyenne du moteur
  sns.set()
  plt.figure(figsize = (20, 10))
  plt.title('Evolution de la puissance moyenne du moteur', size = 20)
  viz = sns.barplot(data = pt, x = pt.year, y = pt.hp, color = "steelblue")

  st.pyplot(viz.figure)

  st.subheader(' '.join(continent))

  # pivot table for mean by cubicinches evolution
  pt_1 = pd.pivot_table(df_continent, index = 'year', values = 'cubicinches', aggfunc = 'mean')
  pt_1['year'] = pt.index

  # Evolution de la cylindré moyenne
  sns.set()
  plt.figure(figsize = (20, 10))
  plt.title('Evolution de la cylindré moyenne', size = 20)
  viz_1 = sns.barplot(data = pt_1, x = pt_1.year, y = pt_1.cubicinches, color = "steelblue")

  st.pyplot(viz_1.figure)

  st.subheader(' '.join(continent))

  # pivot table for mean by weightlbs evolution
  pt_2 = pd.pivot_table(df_continent, index = 'year', values = 'weightlbs', aggfunc = 'mean')
  pt_2['year'] = pt.index

  # Evolution du poids moyen
  sns.set()
  plt.figure(figsize = (20, 10))
  plt.title('Evolution du poids moyen', size = 20)
  viz_2 = sns.barplot(data = pt_2, x = pt_2.year, y = pt_2.weightlbs, color = "steelblue")

  st.pyplot(viz_2.figure)

  st.subheader(' '.join(continent))

  # pivot table for mean by time-to-60 evolution
  pt_3 = pd.pivot_table(df_continent, index = 'year', values = 'time-to-60', aggfunc = 'mean')
  pt_3['year'] = pt.index

  # Evolution du time to 60 moyen
  sns.set()
  plt.figure(figsize = (20, 10))
  plt.title('Evolution du time to 60 moyen', size = 20)
  viz_3 = sns.barplot(data = pt_3, x = pt_3.year, y = pt_3['time-to-60'], color = "steelblue")

  st.pyplot(viz_3.figure)


t = int(input())

for tc in range(1, t+1):
    numbers = input()
    arr = ''

    for i in numbers:
        binary = format(int(i, 16), 'b')
        print(binary)
        binary = '0' * (4 - len(binary)) + binary
        arr += binary
        print(binary)

    for i in range(0, len(arr), 7):
        num = int(arr[i:i+7], 2)
        print(num, end=' ')

연습문제2_16진수를_10진수로.py


from BST import BST
from queue import Queue
from typing import List

def convert(head: BST)->BST:
    mq = Queue()
    in_order_to_queue(head, mq)
    if mq.empty():
        return head
    head = mq.get()
    pre = head
    pre.left = None
    while not mq.empty():
        curr = mq.get()
        pre.right = curr
        curr.left = pre
        pre = curr
    pre.right = None
    return head

def in_order_to_queue(head: BST, que: Queue):
    if not head:
        return
    in_order_to_queue(head.left, que)
    que.put(head)
    in_order_to_queue(head.right, que)

if __name__ == '__main__':
    a = BST(1)
    b = BST(2)
    c = BST(3)
    b.left = a
    b.right = c
    d = convert(b)
    while d:
        print(d.value)
        d = d.right

将BST转换为双向链表.py


from arm.elf import ELF
from generic.editable import XEditable as Editable

class Overlay(Editable):
    def define(self):
        self.uint32('id')
        self.uint32('address')
        self.uint32('size')
        self.uint32('bss_size')
        self.uint32('init_start')  # Pointer to the first initializer function.
        self.uint32('init_end')
        self.uint32('file_id')
        self.uint32('reserved')

    @property
    def compressed(self):
        return self.reserved & (1 << 24)

    @compressed.setter
    def compressed(self, value):
        if value:
            self.reserved |= 1 << 24
        else:
            self.reserved &= ~(1 << 24)

    def to_binary_object(self, game):
        """Converts an overlay to an object file loaded at the proper offsets

        Parameters
        ----------
        game : Game
            Game to load overlays from

        Returns
        -------
        elf : ELF
            binary
        """
        elf = ELF()
        with game.open('overlays_dez/overlay_{0:04}.bin'.format(self.id))\
                as overlay:
            elf.add_binary(overlay, name='overlay_0x{0:04X}'.format(self.id),
                           address=self.address)
        return elf

class OverlayTable(Editable):
    """For overarm9.bin"""
    def define(self, count):
        self.array('overlays', Overlay().base_struct, length=count)


import torch
import numpy as np
import scipy.io as scio
from ..builder import DATASETS

from .utils import TimeSeriesShiftAugmentation

# import external modules
import os
import sys
sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))),
                             'external'))

from SRU_for_GCI.utils.utilFuncs import loadTrainingData, loadTrueNetwork

@DATASETS.register_module()
class Dream3(torch.utils.data.Dataset):
    def __init__(
        self,
        data_root=None,
        max_length=1000,
        percentage=1.0,
        shift_range=None,
    ):
        # Set all input args as attributes
        self.__dict__.update(locals())
        self.subject = self.data_root[0].split("/")[-1].strip(".pt")
        self.check_files()
        if shift_range is not None:
            self.transform = TimeSeriesShiftAugmentation(shift_range)

    def check_files(self):
        # if isinstance(self.percentage, float):
        #     self.percentage = [0, self.percentage]

        if self.data_root is None or len(self.data_root) != 2:
            raise Exception("Invalid dataset path, should be a tuple of (seq_path, net_path)")

        # data_root = (seq_path, net_path)
        Xtrain = loadTrainingData(self.data_root[0], 'cpu')

        self.seqs = Xtrain
        n, l = Xtrain.shape
        Gref = loadTrueNetwork(self.data_root[1], n).T

        # Load data
        # duration = 21
        # self.examples = []
        # for i in range(n):
        #     for j in range(n):
        #         # skip the diag elements
        #         if i == j:
        #             continue
        #         # parse it into 46 snippets of sequences (the length of each one is 21 )
        #         for start in range(0, l, duration):
        #             self.examples.append(
        #                 {
        #                     "cause": i,
        #                     "effect": j,
        #                     "start_index": start,
        #                     "end_index": start + duration,
        #                     "label": 1 if Gref[i, j] != 0 else 0,
        #                 }
        #             )

        # percentage
        # self.examples = self.examples[int(len(self.examples) * self.percentage[0]):int(len(self.examples) * self.percentage[1])]

        self.examples = []
        for i in range(n):
            for j in range(n):
                # skip the diag elements
                if i == j:
                    continue
                self.examples.append(
                    {
                        "cause": i,
                        "effect": j,
                        "label": 1 if Gref[i, j] != 0 else 0,
                    }
                )

    def __getitem__(self, idx):
        example = self.examples[idx]
        seq = np.stack(
            [
                self.seqs[
                example["cause"]
                ],
                self.seqs[
                example["effect"]
                ],
            ],
            axis=-1,
        )
        seq = torch.tensor(seq, dtype=torch.float32)

        if hasattr(self, "transform"):
            seq = self.transform(seq)

        label = torch.tensor(example["label"], dtype=torch.int64)
        return seq, label

    def __len__(self):
            return len(self.examples)


'''
Given a string s1, we may represent it as a binary tree by partitioning it to two non-empty substrings recursively.

Below is one possible representation of s1 = "great":

    great
   /    \
  gr    eat
 / \    /  \
g   r  e   at
           / \
          a   t
To scramble the string, we may choose any non-leaf node and swap its two children.

For example, if we choose the node "gr" and swap its two children, it produces a scrambled string "rgeat".

    rgeat
   /    \
  rg    eat
 / \    /  \
r   g  e   at
           / \
          a   t
We say that "rgeat" is a scrambled string of "great".

Similarly, if we continue to swap the children of nodes "eat" and "at", it produces a scrambled string "rgtae".

    rgtae
   /    \
  rg    tae
 / \    /  \
r   g  ta  e
       / \
      t   a
We say that "rgtae" is a scrambled string of "great".

Given two strings s1 and s2 of the same length, determine if s2 is a scrambled string of s1.

Hide Tags Dynamic Programming String
Difficulty: Hard
'''
class Solution(object):
    def __init__(self):
        self.opt = {}
    def isScramble(self, s1, s2):
        """
        :type s1: str
        :type s2: str
        :rtype: bool
        """
        # OF course length should be same
        if len(s1) != len(s2):
            return False

        # Ok, both the string character should match, any extra char
        # means its not scramble!
        if sorted(s1) != sorted(s2):
            return False

        # If we already found result stored ..
        if (s1, s2) in self.opt:
            return self.opt[(s1, s2)]

        # base case s1 == s2 or s1 == rev(s2)
        if len(s1) == 2:
            if s1 == s2:
                return True
            if s1[0] == s2[1] and s1[1] == s2[0]:
                return True
            else:
                return False

        if len(s1) == 1:
            return s1 == s2

        # s1 to be scramble string of s2
        # Divide into two parts any where
        # lets say s11, s12 and s21, s22
        # if scrambled(s11, s21) && s(s12, s22)
        # of scramnled(s11, s22) && s(12, s21)
        for i in range(1, len(s1)):
            if (self.isScramble(s1[:i], s2[:i]) and self.isScramble(s1[i:], s2[i:])) \
                or (self.isScramble(s1[:i], s2[-i:]) and self.isScramble(s1[i:], s2[:-i])):
                self.opt[(s1,s2)] = True
                return True

        self.opt[(s1, s2)] = False
        return False


from __future__ import print_function

import numpy as np

np.random.seed(1234)  # for reproducibility?

# specifying the gpu to use
# import theano.sandbox.cuda
# theano.sandbox.cuda.use('gpu1')
import theano
import theano.tensor as T

import binary_net
import lasagne

from pylearn2.datasets.cifar10 import CIFAR10

from collections import OrderedDict

# if something doesn't work, run 'theano cache-clear' from terminal
justTest= False

def main():
    # BN parameters
    # alpha is the exponential moving average factor
    alpha = .1
    print("alpha = " + str(alpha))
    epsilon = 1e-4
    print("epsilon = " + str(epsilon))

    # BinaryOut
    activation = binary_net.binary_tanh_unit
    print("activation = binary_net.binary_tanh_unit")
    # activation = binary_net.binary_sigmoid_unit
    # print("activation = binary_net.binary_sigmoid_unit")

    # BinaryConnect
    binary = True
    print("binary = " + str(binary))
    stochastic = False
    print("stochastic = " + str(stochastic))
    # (-H,+H) are the two binary values
    # H = "Glorot"
    H = 1.
    print("H = " + str(H))
    W_LR_scale = 1.
    #W_LR_scale = "Glorot"  # "Glorot" means we are using the coefficients from Glorot's paper
    print("W_LR_scale = " + str(W_LR_scale))

    # Training parameters
    num_epochs = 50
    print("num_epochs = " + str(num_epochs))

    # Decaying LR
    LR_start = 0.01
    print("LR_start = " + str(LR_start))
    LR_fin = 0.000003

    # LR_start = 0.01
    # print("LR_start = " + str(LR_start))
    # LR_fin = 1e-6
    print("LR_fin = " + str(LR_fin))
    LR_decay = (LR_fin / LR_start) ** (1. / num_epochs)
    print("LR_decay = " + str(LR_decay))
    # BTW, LR decay might good for the BN moving average...

    shuffle_parts = 1
    print("shuffle_parts = " + str(shuffle_parts))

    networkType = 'google'
    dataType = 'TCDTIMIT'

    #networkType='cifar10'
    #dataType='cifar10'

    # these batch sizes work for a GTX 1060 (6GB)
    if dataType=='TCDTIMIT':
        if networkType == 'google': batch_size = 100
        else: batch_size = 24
    elif dataType=='cifar10' and networkType =='cifar10': batch_size = 400
    elif dataType=='cifar10' and networkType=='google': batch_size = 1000

    model_name = os.path.expanduser('~/TCDTIMIT/lipreading/TCDTIMIT/results/CNN_binaryNet/lipspeakers_') \
                 + networkType + "_phoneme39_binary" + "_" + dataType
    model_path = model_name + ".npz"
    if not os.path.exists(os.path.dirname(model_path)): os.makedirs(os.path.dirname(model_path))

    print('Building the CNN...')
    # Prepare Theano variables for inputs and targets
    input = T.tensor4('inputs')
    target = T.matrix('targets')
    LR = T.scalar('LR', dtype=theano.config.floatX)

    cnn = buildCNN(networkType, dataType, input, epsilon, alpha, activation, binary, stochastic, H, W_LR_scale)

    # restore network weights
    if os.path.exists(model_path):
        with np.load(model_path) as f:
            param_values = [f['arr_%d' % i] for i in range(len(f.files))]
            try:
                lasagne.layers.set_all_param_values(cnn, *param_values)
            except:
                lasagne.layers.set_all_param_values(cnn, param_values)
            print("\n\n\t Loaded model " + model_path)

    print('Loading ' + dataType + ' dataset...')
    X_train, y_train, X_val, y_val, X_test, y_test = loadDataset(dataType)

    print("Building Functions...")
    train_output = lasagne.layers.get_output(cnn, deterministic=False)

    # squared hinge loss
    loss = T.mean(T.sqr(T.maximum(0., 1. - target * train_output)))

    # W updates
    W = lasagne.layers.get_all_params(cnn, binary=True)
    W_grads = binary_net.compute_grads(loss, cnn)
    updates = lasagne.updates.adam(loss_or_grads=W_grads, params=W, learning_rate=LR)
    updates = binary_net.clipping_scaling(updates, cnn)

    # other parameters updates
    params = lasagne.layers.get_all_params(cnn, trainable=True, binary=False)
    updates = OrderedDict(
            updates.items() + lasagne.updates.adam(loss_or_grads=loss, params=params, learning_rate=LR).items())

    test_output = lasagne.layers.get_output(cnn, deterministic=True)
    test_loss = T.mean(T.sqr(T.maximum(0., 1. - target * test_output)))
    test_err = T.mean(T.neq(T.argmax(test_output, axis=1), T.argmax(target, axis=1)), dtype=theano.config.floatX)

    # Compile a function performing a training step on a mini-batch (by giving the updates dictionary)
    # and returning the corresponding training loss:
    train_fn = theano.function([input, target, LR], loss, updates=updates)

    # Compile a second function computing the validation loss and accuracy:
    val_fn = theano.function([input, target], [test_loss, test_err])

    print('Training...')

    binary_net.train(
            train_fn, val_fn,
            cnn,
            batch_size,
            LR_start, LR_decay,
            num_epochs,
            X_train, y_train,
            X_val, y_val,
            X_test, y_test,
            save_name=model_name,
            shuffle_parts=shuffle_parts, justTest=justTest)

def buildCNN(networkType, dataType, input, epsilon, alpha, activation, binary, stochastic, H, W_LR_scale, oneHot=True):
    if oneHot:
        print("identity")
        denseOut = lasagne.nonlinearities.identity
    else:
        print("softmax")
        denseOut = lasagne.nonlinearities.softmax

    if dataType =='TCDTIMIT':
        nbClasses = 39
        cnn = lasagne.layers.InputLayer(
                shape=(None, 1, 120,120),
                input_var=input)
    elif dataType =='cifar10':
        nbClasses = 10
        cnn = lasagne.layers.InputLayer(
                shape=(None, 3, 32, 32),
                input_var=input)

    if networkType == 'google':
        # conv 1
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=128,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)
        cnn = lasagne.layers.MaxPool2DLayer(cnn, pool_size=(2, 2))
        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)
        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # conv 2
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=256,
                filter_size=(3, 3),
                stride=(2, 2),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)
        cnn = lasagne.layers.MaxPool2DLayer(cnn, pool_size=(2, 2))
        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)
        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # conv3
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=512,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)
        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # conv 4
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=512,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)
        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # conv 5
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=512,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)
        cnn = lasagne.layers.MaxPool2DLayer(cnn, pool_size=(2, 2))
        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # FC layer
        cnn = binary_net.DenseLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                nonlinearity=denseOut,  #TODO was identity
                num_units=nbClasses)
        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

    elif networkType == 'cifar10':
        # 128C3-128C3-P2

        # 128C3-128C3-P2
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=128,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=128,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)

        cnn = lasagne.layers.MaxPool2DLayer(cnn, pool_size=(2, 2))

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # 256C3-256C3-P2
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=256,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=256,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)

        cnn = lasagne.layers.MaxPool2DLayer(cnn, pool_size=(2, 2))

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # 512C3-512C3-P2
        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=512,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        cnn = binary_net.Conv2DLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                num_filters=512,
                filter_size=(3, 3),
                pad=1,
                nonlinearity=lasagne.nonlinearities.identity)

        cnn = lasagne.layers.MaxPool2DLayer(cnn, pool_size=(2, 2))

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        # print(cnn.output_shape)

        # 1024FP-1024FP-10FP
        cnn = binary_net.DenseLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                nonlinearity=lasagne.nonlinearities.identity,
                num_units=1024)

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        cnn = binary_net.DenseLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                nonlinearity=lasagne.nonlinearities.identity,
                num_units=1024)

        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)

        cnn = lasagne.layers.NonlinearityLayer(
                cnn,
                nonlinearity=activation)

        cnn = binary_net.DenseLayer(
                cnn,
                binary=binary,
                stochastic=stochastic,
                H=H,
                W_LR_scale=W_LR_scale,
                nonlinearity=denseOut,
                num_units=nbClasses)
        cnn = lasagne.layers.BatchNormLayer(
                cnn,
                epsilon=epsilon,
                alpha=alpha)
    return cnn

import os
import preprocessLipreading
import general_tools
def loadDataset(type, oneHot=True):
    if type == 'TCDTIMIT':
        nbClasses = 39
        # get the database
        # If it's small (lipspeakers) -> generate X_train, y_train etc here
        # otherwise we need to load and generate each speaker seperately in the training loop
        dataset = "TCDTIMIT"
        root_dir = os.path.join(os.path.expanduser('~/TCDTIMIT/lipreading/' + dataset))
        results_dir = root_dir + "/results/CNN_binaryNet";
        if not os.path.exists(results_dir): os.makedirs(results_dir)

        database_binaryDir = root_dir + '/binary'
        datasetType = "lipspeakers"  # "lipspeakers" #"volunteers" #"volunteers" #    lipspeakers or volunteers"
        ##############################################

        if datasetType == "lipspeakers":
            loadPerSpeaker = False  # only lipspeakers small enough to fit in CPU RAM, generate X_train etc here
            storeProcessed = True
            processedDir = database_binaryDir + "_allLipspeakersProcessed"

            # TODO: prepLip_all can be used to generate pkl containing all the lipspeaker data. Not sure if this stil works, so use with care!
            pkl_path = processedDir + os.sep + datasetType + "_oneHot" + ".pkl"
            if not os.path.exists(pkl_path):
                print("dataset not yet processed. Processing...")
                preprocessLipreading.prepLip_all(data_path=database_binaryDir, store_path=pkl_path, trainFraction=0.7,
                                                 validFraction=0.1,
                                                 testFraction=0.2,
                                                 nbClasses=nbClasses, onehot=oneHot, type=datasetType, verbose=True)
            X_train, y_train, X_val, y_val, X_test, y_test = general_tools.unpickle(pkl_path)
            dtypeX = 'float32'
            dtypeY = 'int32'
            X_train = X_train.astype(dtypeX);
            y_train = y_train.astype(dtypeY);
            X_val = X_val.astype(dtypeX);
            y_val = y_val.astype(dtypeY);
            X_test = X_test.astype(dtypeX);
            y_test = y_test.astype(dtypeY);

            #import pdb;pdb.set_trace()
            if oneHot:
                #  Onehot the targets
                y_train = np.float32(np.eye(nbClasses)[y_train])
                y_val = np.float32(np.eye(nbClasses)[y_val])
                y_test = np.float32(np.eye(nbClasses)[y_test])

                # for hinge loss
                y_train = 2 * y_train - 1.
                y_val = 2 * y_val - 1.
                y_test = 2 * y_test - 1.

            #import pdb;pdb.set_trace()

    else: # cifar10
        nbClasses = 10
        train_set_size = 45000
        print("train_set_size = " + str(train_set_size))
        train_set = CIFAR10(which_set="train", start=0, stop=train_set_size)
        valid_set = CIFAR10(which_set="train", start=train_set_size, stop=50000)
        test_set = CIFAR10(which_set="test")

        # bc01 format
        # Inputs in the range [-1,+1]
        # print("Inputs in the range [-1,+1]")
        train_set.X = np.reshape(np.subtract(np.multiply(2. / 255., train_set.X), 1.), (-1, 3, 32, 32))
        valid_set.X = np.reshape(np.subtract(np.multiply(2. / 255., valid_set.X), 1.), (-1, 3, 32, 32))
        test_set.X = np.reshape(np.subtract(np.multiply(2. / 255., test_set.X), 1.), (-1, 3, 32, 32))

        # flatten targets
        train_set.y = np.hstack(train_set.y)
        valid_set.y = np.hstack(valid_set.y)
        test_set.y = np.hstack(test_set.y)

        # Onehot the targets
        train_set.y = np.float32(np.eye(10)[train_set.y])
        valid_set.y = np.float32(np.eye(10)[valid_set.y])
        test_set.y = np.float32(np.eye(10)[test_set.y])

        # for hinge loss
        train_set.y = 2 * train_set.y - 1.
        valid_set.y = 2 * valid_set.y - 1.
        test_set.y = 2 * test_set.y - 1.

        X_train = train_set.X;
        y_train = train_set.y
        X_val = valid_set.X;
        y_val = valid_set.y
        X_test = test_set.X;
        y_test = test_set.y

    return X_train, y_train, X_val, y_val, X_test, y_test

if __name__ == "__main__":
    main()


import pygame
from pygame.sprite import Sprite
from pygame.math import Vector2

class Boss(Sprite):
    """Класс, представляющий одного боса."""

    def __init__(self, ai_game):
        #"""Инициализирует боса и задает его начальную позицию."""
        super().__init__()
        self.screen = ai_game.screen
        self.screen_rect = ai_game.screen.get_rect()
        self.settings = ai_game.settings
        self.explois_sound = pygame.mixer.Sound('sounds/Explosion3.wav')
        # Загрузка изображения боса и назначение атрибута rect.
        self.image_norm = pygame.image.load('images/turrel2.bmp')
        self.image = self.image_norm.copy()
        self.image_explois = pygame.image.load('images/ship_destroy.bmp')
        self.is_explois = 0
        self.rect = self.image.get_rect()
        self.normal_center = self.rect.center
        # Каждый новый босс появляется в центре экрана.
        self.rect.x = self.screen_rect.centerx
        self.rect.y = self.screen_rect.centery
        # Уровень здоровья орудий
        self.boss_turrel_health = self.settings.boss_turrel_health_max
        self.turrel_health_color = self.settings.boss_turrel_health_color
        # Создание полосы здоровья в позиции (0,0) и назначение правильной позиции.
        self.turrel_health_rect = pygame.Rect(0, 0,
                                              self.settings.boss_turrel_health_width,
                                              self.settings.boss_turrel_health_height)
        self.rect_health_heigth = self.settings.boss_turrel_health_height
        self.rect_health_weigth = self.settings.boss_turrel_health_width
        # Сохранение точной горизонтальной позиции боса.
        self.x = float(self.rect.x)
        self.y = float(self.rect.y)
        self.index_in_station = 0
        self.x_aim = ai_game.ship.rect.x
        self.y_aim = ai_game.ship.rect.y

    def check_edges(self):
        """Возвращает True, если босс находится у краев экрана."""
        screen_rect = self.screen.get_rect()
        if self.rect.right >= screen_rect.right or self.rect.x <= 10:
            return True

    def _explois(self):
        """Отрисовываем взрыв"""
        rect_explois = self.image_explois.get_rect()
        rect_explois.center = (self.rect.width / 2, self.rect.height / 2)
        self.image.blit(self.image_explois, rect_explois)

    def update(self):
        #"""Перемещает боса вправо и вниз."""

        self.rect.x = int(self.x)
        self.rect.y = int(self.y)
        if self.is_explois >= 1:
            self._explois()
            self.is_explois += 1
            if self.is_explois >= self.settings.exlois_timer:
                self.boss_turrel_health -= self.settings.boss_turrel_bullet_damge
                if self.boss_turrel_health >= 0:
                    self.is_explois = 0
        else:
            self._rotate_to_ship()
        self._draw_turrel_health()

    def _draw_turrel_health(self):
        """Вывод уровня здоровья пушки на экран."""
        # Расчёт позиции жизни смерти
        kill_level = int(self.boss_turrel_health * self.rect_health_weigth / 100)
        boader = self.settings.boss_turrel_health_boader_thick
        # Параметры прямоугольника бара здороья
        rect_health = (int((self.rect.width / 2) - (self.rect_health_weigth / 2)),
                       int((self.rect.height / 2) - (self.rect_health_heigth / 2)),
                       self.rect_health_weigth,
                       self.rect_health_heigth)
        pygame.draw.rect(self.image, (254, 254, 254), (rect_health[0]-boader, rect_health[1]-boader, rect_health[2]+boader*2, rect_health[3]+boader*2))
        pygame.draw.rect(self.image, self.settings.boss_turrel_health_color, (rect_health[0], rect_health[1], kill_level, rect_health[3]))
        pygame.draw.rect(self.image, pygame.SRCALPHA, (rect_health[0] + kill_level, rect_health[1], rect_health[2], rect_health[3]))

    def draw_boss(self):
        """Вывод босса на экран."""
        self.screen.blit(self.image, self.rect)

    def _rotate_to_ship(self):
        """Вращает в сторону корабля"""
        #Вычисления угла между векторами в позицию башни и в позицию корабля
        turrel_ship_diff = pygame.Vector2(self.x_aim, self.y_aim) - pygame.Vector2(self.rect.center)
        radius, angle = pygame.math.Vector2.as_polar(turrel_ship_diff)
        # Сохранение позиции центра картинки
        self.rect = self.image.copy().get_rect()
        self.center = (int(self.x + self.rect.width / 2), int(self.y + self.rect.height / 2))
        # Поворот изображения
        self.image = self.rotate(self.image_norm, self.normal_center, 90 - angle)
        self.rect = self.image.get_rect()
        # Возвращение картинки в изначальный центр после поворота
        self.rect.center = self.center
        self.x = float(self.rect.x)
        self.y = float(self.rect.y)

    def rotate(self, img, pos, angle):
        """Поворот изображения с предварительным увеличением в два раза"""
        w, h = img.get_size()
        img2 = pygame.Surface((w * 2, h * 2), pygame.SRCALPHA)
        img2.blit(img, (w - pos[0], h - pos[1]))
        return pygame.transform.rotate(img2, angle)


#!/usr/bin/env python3
"""
https://adventofcode.com/2022/day/3
"""

import os

with open(os.path.join(os.path.dirname(__file__), "rucksacks.txt"), encoding="utf-8") as file:
    rucksacks = file.read().splitlines()

def priority(char: str) -> int:
    """Calculate the priority of a character"""
    return ord(char) - 96 if char.islower() else ord(char) - 38

# Part 1
total_part1 = sum(
    sum(map(priority, shared))
    for shared in (set(rs[: len(rs) // 2]).intersection(rs[len(rs) // 2 :]) for rs in rucksacks)
)
print(f"Sum of priorities is {total_part1}")

# Part 2
total_part2 = sum(
    sum(map(priority, set.intersection(*list(map(set, group))).pop()))
    for group in (rucksacks[i : i + 3] for i in range(0, len(rucksacks), 3))
)
print(f"Sum of badge priorities is {total_part2}")


from flet import AlertDialog, TextButton, RoundedRectangleBorder, MainAxisAlignment, Text

class ConfirmDialog(AlertDialog):
    """Creates an AlertDialog that, if "yes" clicked, executes the function passed as a parameter.

    Args:
        function (method): method to be executed
        title (string): title of the dialog
        content (string): content of the question to be confirmed
    """
    def __init__(self, function, title="", content=""):
        super().__init__()
        self.function = function

        self.modal=True
        self.title=Text(title)
        self.content=Text(content)
        self.actions=[
            TextButton("Não", on_click=self.canceled),
            TextButton(content=Text("Sim", color="red"), on_click=self.confirmed),
        ]
        self.actions_alignment=MainAxisAlignment.END
        self.shape=RoundedRectangleBorder(radius=10)

    def build(self):
        return self

    def confirmed(self, e):
        self.open = False
        self.update()
        self.function(self.data)

    def canceled(self, e):
        self.open = False
        self.update()


import Adafruit_BBIO.GPIO as GPIO
import random as rand
import time

#arrays para a maquina e o jogador
genius_valores = 100*[0]
jogador_jogadas = 100*[0]

def iniciar():
	#definir valores do jogo
	for i in range(0,100):
		genius_valores[i] = rand.randint(0, 3)

def acender_led(i):
	GPIO.output(leds[i], 1)

def acender_leds():
	acender_led(0)
	acender_led(1)
	acender_led(2)
	acender_led(3)

def apagar_leds():
	GPIO.output(l0, 0)
	GPIO.output(l1, 0)
	GPIO.output(l2, 0)
	GPIO.output(l3, 0)

def apagar_led(i):
	GPIO.output(leds[i], 0)

def espera():
	apagar_leds()
	time.sleep(0.5)
	acender_led(0)
	acender_led(3)
	time.sleep(0.5)
	apagar_leds()
	time.sleep(0.25)
	acender_led(1)
	acender_led(2)
	time.sleep(0.5)
	apagar_leds()
	time.sleep(0.5)

def imprimir_valor(valor):
	GPIO.output(leds[0], valor[0])
	GPIO.output(leds[1], valor[1])
	GPIO.output(leds[2], valor[2])
	GPIO.output(leds[3], valor[3])

b0 = "P8_7"
b1 = "P8_8"
b2 = "P8_9"
b3 = "P8_10"

l0 = "P8_13"
l1 = "P8_14"
l2 = "P8_15"
l3 = "P8_16"
leds = [l0,l1,l2,l3]

GPIO.setup(b0, GPIO.IN)
GPIO.setup(l0, GPIO.OUT)

GPIO.setup(b1, GPIO.IN)
GPIO.setup(l1, GPIO.OUT)

GPIO.setup(b2, GPIO.IN)
GPIO.setup(l2, GPIO.OUT)

GPIO.setup(b3, GPIO.IN)
GPIO.setup(l3, GPIO.OUT)

iniciar()
rodada = 1
while True:
	#valor = [GPIO.input(b0), GPIO.input(b1) , GPIO.input(b2) , GPIO.input(b3)]

	for i in range(0,len(genius_valores)):
		print("Iniciou uma nova rodada")
		acender_leds()
		time.sleep(0.25)
		apagar_leds()
		time.sleep(0.25)
		acender_leds()
		time.sleep(1)
		#vez da maquina
		for j in range(0,rodada):
			apagar_leds()
			time.sleep(0.25)
			acender_led(genius_valores[j])
			time.sleep(1)
		#espera
		espera()
		#vez do jogador
		#j e o passo
		for j in range(0,rodada):
			apagar_leds()
			#enquanto nao foi apertado algum botao
			valor = [0,0,0,0]
			while(sum(valor) == 0):
				valor = [GPIO.input(b0), GPIO.input(b1) , GPIO.input(b2) , GPIO.input(b3)]
				imprimir_valor(valor)
				time.sleep(0.2)
			time.sleep(1) #tempo de espera para cada teclada selecionada
			if valor[genius_valores[j]] == 0: #se o valor apertado for diferente do salvo no array da partida
				print("Perdeu :/")
				apagar_leds()
				acender_led(1)
				acender_led(3)
				time.sleep(1)
				#reseta os valores
				valor = [0,0,0,0]
				while(sum(valor) == 0):
					valor = [GPIO.input(b0), GPIO.input(b1) , GPIO.input(b2) , GPIO.input(b3)]
					time.sleep(0.1) #faca nada :v
				i = 0
				rodada = 0
				iniciar()
				break
		rodada += 1


from django.contrib import admin

# Register your models here.
from .models import UserProfile

class UserProfileAdmin(admin.ModelAdmin):
    list_display = ("user",
                    "organisation_name",
                    "timestamp",)
    list_filter = ("user",
                   "organisation_name",
                   )
    search_fields = ('organisation_name',)

admin.site.register(UserProfile, UserProfileAdmin)


"""
https://leetcode.com/problems/my-calendar-iii/

this tree only return overlapping count
so only need the update function
"""

from sortedcontainers import SortedDict
class SegmentTreeNode:
    def __init__(self, start=0, end=0, count=0, left=None, right=None) -> None:
        self.start = start
        self.end = end

        self.count = count
        self.left = left
        self.right = right

def printSegmentTree(node):

    q = [(node, 0, '^')]
    idx = 1
    pIdx = 0
    while q:
        count = len(q)
        level = ""
        while count:
            node, pIdx, lr = q[0]
            q = q[1:]
            if node:
                level = f"{level}  {pIdx}-{lr}-{idx}:[{node.start},{node.end},{node.count}]"
                if node.left:
                    q.append((node.left, idx, 'l'))
                if node.right:
                    q.append((node.right, idx, 'r'))
            else:
                level = f"{level}  {idx}-None"
            count-=1
            idx+=1

        print(level)
    print()

# don't think start,end as two separate things
# it is the vals here..
# I confuse it as a range
# I came from that dicrete examples.. that l,r is truly a range (afterall, that is range sum exmaple)

def query(root, start, end):
    if start >= end or root is None:
        # root is None meaning there is no interval existing yet
        # it can happen when it walks off the edge into the void
        # e..g root is now 10,30... on incoming 50,60.. it walks to right and its a void

        return 0

    if root.start >= end:
        return query(root.left, start, end)

    if root.end <= start:
        return query(root.right, start, end)

    # the max overlapping among root range, left range and right range
    return max(root.count, query(root.left, start, end), query(root.right, start, end))

# WARNING: this version has that push-down loss
class MyCalendarThree:

    def __init__(self):
        self.root = SegmentTreeNode()
        self.max = 0

    def book(self, start: int, end: int) -> int:
        def update(root, start, end):
            if start >= end:
                # it is none of your business
                # or it is an empty range.. no need to build a node for empty range
                return root
            if root is None:
                # this is a new interval
                self.max = max(self.max, 1)
                return SegmentTreeNode(start, end, 1)

            if root.end <= start:
                root.right = update(root.right, start, end)

            elif root.start >= end:
                root.left = update(root.left, start, end)
            else:
                # span both left/right
                # e.g. [10,30] with incoming [15,25]
                # it will split int [10,15].. [15. 25].. and [25,30]
                # the mid range(b,c) will have one more count here +=1
                # the left/right ranges (a,b) and (c,d) will form new ranges.. and could populate downwards...
                a = min(start, root.start)
                b = max(start, root.start)
                c = min(end, root.end)
                d = max(end, root.end)

                root.start = b
                root.end = c
                root.count += 1
                self.max = max(self.max, root.count)

                root.left = update(root.left, a, b)
                root.right = update(root.right, c, d)

            return root

        update(self.root, start, end)
        return self.max

# trying to fix that push down loss
# firstly, trying to use the old value after it gets updated
# a so bad logical error
class MyCalendarThree:
    def __init__(self):
        self.root = SegmentTreeNode()
        self.max = 0

    def book(self, start: int, end: int) -> int:
        pushDownMap = {}
        def update(root, start, end):
            if start >= end:
                # it is none of your business
                # or it is an empty range.. no need to build a node for empty range
                return root
            if root is None:
                # this is a new interval
                count = 1
                if (start,end) in pushDownMap:
                    count = pushDownMap[(start, end)]

                self.max = max(self.max, count)
                return SegmentTreeNode(start, end, count)

            if root.end <= start:
                root.right = update(root.right, start, end)

            elif root.start >= end:
                root.left = update(root.left, start, end)
            else:
                # span both left/right
                # e.g. [10,30] with incoming [15,25]
                # it will split int [10,15].. [15. 25].. and [25,30]
                # the mid range(b,c) will have one more count here +=1
                # the left/right ranges (a,b) and (c,d) will form new ranges.. and could populate downwards...
                a = min(start, root.start)
                b = max(start, root.start)
                c = min(end, root.end)
                d = max(end, root.end)

                if root.start <= a and b <= root.end:
                    pushDownMap[(a, b)] = root.count

                if root.start <= c and d <= root.end:
                    pushDownMap[(c, d)] = root.count

                root.start = b
                root.end = c
                root.count += 1
                self.max = max(self.max, root.count)

                root.left = update(root.left, a, b)
                root.right = update(root.right, c, d)

            return root

        update(self.root, start, end)
        return self.max

"""
Runtime: 342 ms, faster than 96.32% of Python3 online submissions for My Calendar III.
Memory Usage: 15.4 MB, less than 18.61% of Python3 online submissions for My Calendar III.

okay.. so agnoy comes from
                if root.start <= a and b <= root.end:
                    pushDownMap[(a, b)] = root.count

                if root.start <= c and d <= root.end:
                    pushDownMap[(c, d)] = root.count

                root.start = b
                root.end = c
                root.count += 1
                self.max = max(self.max, root.count)

I wrote it as beow -- update root before I need its old value
                root.start = b
                root.end = c
                root.count += 1
                self.max = max(self.max, root.count)

                if root.start <= a and b <= root.end:
                    pushDownMap[(a, b)] = root.count

                if root.start <= c and d <= root.end:
                    pushDownMap[(c, d)] = root.count

gives me idea maybe I can still do that push down
"""

class MyCalendarThree:
    def __init__(self):
        self.root = SegmentTreeNode()
        self.max = 0

    def book(self, start: int, end: int) -> int:
        def update(root, start, end, pushDownCount):
            # pushDownCount is how many should the split child inherits
            # edge case being if it is total subset of root.. it must inherit root.count
            # e.g. 10,30,3 with incoming 5,20
            # it will become 1-10, which is new 1
            # also 1-20, which is 3+1 = 4
            # also 20-30, which is? absolutely not 1.. it should be 3

            if start >= end:
                # it is none of your business
                # or it is an empty range.. no need to build a node for empty range
                return root
            if root is None:
                # this is a new interval
                self.max = max(self.max, pushDownCount)
                return SegmentTreeNode(start, end, pushDownCount)

            if root.end <= start:
                root.right = update(root.right, start, end, pushDownCount)

            elif root.start >= end:
                root.left = update(root.left, start, end, pushDownCount)
            else:
                # span both left/right
                # e.g. [10,30] with incoming [15,25]
                # it will split int [10,15].. [15. 25].. and [25,30]
                # the mid range(b,c) will have one more count here +=1
                # the left/right ranges (a,b) and (c,d) will form new ranges.. and could populate downwards...
                a = min(start, root.start)
                b = max(start, root.start)
                c = min(end, root.end)
                d = max(end, root.end)

                leftPushDown=rightPushDown=1
                if root.start <= a and b <= root.end:
                    leftPushDown = root.count
                if root.start <= c and d <= root.end:
                    rightPushDown = root.count

                root.start = b
                root.end = c
                root.count += 1
                self.max = max(self.max, root.count)

                root.left = update(root.left, a, b, leftPushDown)
                root.right = update(root.right, c, d, rightPushDown)

            return root

        update(self.root, start, end,1)
        return self.max

"""
Runtime: 311 ms, faster than 96.75% of Python3 online submissions for My Calendar III.
Memory Usage: 14.6 MB, less than 55.41% of Python3 online submissions for My Calendar III.

okay... it is still right...
I am still not familiar with this enough
"""

class MyCalendarThree:

    def __init__(self):
        self.sMap = SortedDict()

    def book(self, startTime: int, endTime: int) -> int:
        if startTime not in self.sMap:
            self.sMap[startTime] = 0
        self.sMap[startTime] += 1

        if endTime not in self.sMap:
            self.sMap[endTime] = 0
        self.sMap[endTime] -= 1

        # curr will meet +1, -1, so it kind of reflect the curr most bookings
        # res will only be the max bookings..
        # genius idea ...
        res = curr = 0
        for time, count in self.sMap.items():
            curr += count
            res = max(res, curr)

        return res

if __name__ == '__main__':
    # op = ["book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book","book"]
    # # args = [[97,100],[51,65],[27,46],[90,100],[20,32],[15,28],[60,73],[77,91],[67,85],[58,72],[74,93],[73,83],[71,87],[97,100],[14,31],[26,37],[66,76],[52,67],[24,43],[6,23],[94,100],[33,44],[30,46],[6,20],[71,87],[49,59],[38,55],[4,17],[46,61],[13,31],[94,100],[47,65],[9,25],[4,20],[2,17],[28,42],[26,38],[72,83],[43,61],[18,35]]
    # args = [[27, 46], [20, 32], [15, 28], [14, 31], [26, 37], [24, 43], [6, 23], [
    #     33, 44], [30, 46], [6, 20], [4, 17], [4, 20], [2, 17], [28, 42], [26, 38]]

    # this input shows something wrong with logic
    args = [[27, 46], [20, 32], [15, 28]]
    mycal3 = MyCalendarThree()
    for arg in args:
        print(mycal3.book(*arg), arg)
        printSegmentTree(mycal3.root.right)
    """
okay.. above reveals what is wrong
1 [27, 46]
    0-^-1:[27,46,1]

2 [20, 32]
  0-^-1:[27,32,2]
  1-l-2:[20,27,1]  1-r-3:[32,46,1]

3 [15, 28]
  0-^-1:[27,28,3]
  1-l-2:[20,27,2]  1-r-3:[32,46,1]
  2-l-4:[15,20,1]  3-l-5:[28,32,1]
                            ^^ here 28-32 should have appeared twice now already

    the reason is maybe here
    when 15-28 + 27-32.. it pushes 28-32 to right..
    it should bear its 2 to start.. not 1 to start..
    let me see where to make the fix

    to keep the scene I'll copy a new class

    """

    args = [[27, 46], [20, 32], [15, 28], [14, 31], [26, 37], [24, 43], [6, 23], [
        33, 44], [30, 46], [6, 20], [4, 17], [4, 20], [2, 17], [28, 42], [26, 38]]

    # this input shows something wrong with logic
    mycal3 = MyCalendarThree()
    for arg in args:
        print(mycal3.book(*arg), arg)
        printSegmentTree(mycal3.root.right)


"""Can Driver tests."""
from typing import AsyncGenerator

import pytest
from can import Bus, Message

from opentrons_shared_data.errors.exceptions import CANBusBusError
from opentrons_hardware.drivers.can_bus import CanDriver, ArbitrationId, CanMessage

@pytest.fixture
def bus_channel() -> str:
    """The virtual can bus channel."""
    return "test_channel"

@pytest.mark.slow
@pytest.fixture
def can_bus(bus_channel: str) -> Bus:
    """A virtual can bus fixture."""
    bus = Bus(bus_channel, interface="virtual")
    yield bus
    bus.shutdown()

@pytest.fixture
async def subject(bus_channel: str) -> AsyncGenerator[CanDriver, None]:
    """The can driver under test."""
    driver = await CanDriver.build(channel=bus_channel, interface="virtual", bitrate=0)
    yield driver
    driver.shutdown()

async def test_send(subject: CanDriver, can_bus: Bus) -> None:
    """It should send a message."""
    message = CanMessage(
        arbitration_id=ArbitrationId(id=0x1FFFFFFF), data=bytearray([1, 2, 3, 4])
    )
    await subject.send(message)

    recv = can_bus.recv()
    assert recv.data == bytearray([1, 2, 3, 4])
    assert recv.arbitration_id == 0x1FFFFFFF

async def test_receive(subject: CanDriver, can_bus: Bus) -> None:
    """It should receive a message."""
    m = Message(
        arbitration_id=0x1FFFFFFF,
        is_extended_id=True,
        is_fd=True,
        data=bytearray([1, 2, 3, 4]),
    )
    can_bus.send(m)
    recv = await subject.read()

    assert recv.data == bytearray([1, 2, 3, 4])
    assert recv.arbitration_id.id == 0x1FFFFFFF

async def test_receive_iter(subject: CanDriver, can_bus: Bus) -> None:
    """It should receive messages using iterator."""
    m = [
        Message(
            arbitration_id=0x1FFFFFFF,
            is_extended_id=True,
            is_fd=True,
            data=bytearray([1, 2, 3, 4]),
        ),
        Message(
            arbitration_id=0,
            is_extended_id=True,
            is_fd=True,
            data=bytearray([4, 3, 2, 1]),
        ),
    ]

    for message in m:
        can_bus.send(message)

    received = []
    async for recv in subject:
        received.append(recv)
        if len(received) == len(m):
            break

    assert received[0].data == bytearray([1, 2, 3, 4])
    assert received[0].arbitration_id.id == 0x1FFFFFFF
    assert received[1].data == bytearray([4, 3, 2, 1])
    assert received[1].arbitration_id.id == 0

async def test_raise_error_frame_error(subject: CanDriver, can_bus: Bus) -> None:
    """It should raise an error when an error frame is received."""
    m = Message(
        arbitration_id=0x1FFFFFFF,
        is_extended_id=True,
        is_error_frame=True,
        is_fd=True,
        data=bytearray([1, 2, 3, 4]),
    )
    can_bus.send(m)
    with pytest.raises(CANBusBusError):
        await subject.read()


import os
import shutil

LANGS = {
    'Afro-Asiatic': 'mlt orm syc'.split(),
    'Algic': 'cre'.split(),
    'Australian': 'mwf'.split(),
    'Dravidian': 'kan tel'.split(),
    'Germanic': 'gml gsw nno'.split(),
    'Indo-Aryan': 'ben  hin san urd'.split(),
    'Iranian': 'fas pus tgk'.split(),
    'Niger-Congo': 'sna'.split(),
    'Nilo-Sahan': 'dje'.split(),
    'Romance': 'ast cat frm fur glg lld vec xno'.split(),
    'Sino-Tibetan': 'bod'.split(),
    'Siouan': 'dak'.split(),
    'Tungusic': 'evn'.split(),
    'Turkic': 'aze bak crh kaz kir kjh tuk uig uzb'.split(),
    'Uralic': 'kpv lud olo udm vro'.split(),
    'Uto-Aztecan': 'ood'.split(),
    'austronesian': 'mlg ceb hil tgl mao'.split(),
    'germanic': 'dan isl nob swe nld eng deu gmh frr ang'.split(),
    'niger-congo': 'nya kon lin lug sot swa zul aka gaa'.split(),
    'oto-manguean': 'cpa azg xty zpv ctp czn cly otm ote pei'.split(),
    'uralic': 'est fin izh krl liv vep vot mhr myv mdf sme'.split()
}

OUT_DIR = 'task0-data/out'
DEV_DIR = 'task0-data/DEVELOPMENT-LANGUAGES'
SURP_DIR = 'task0-data/SURPRISE-LANGUAGES'

def move_family_dataset(family, in_dir):
    for lang in sorted(LANGS[family]):
        for mode in ['trn', 'dev', 'tst']:
            shutil.copy(f'{in_dir}/{family}/{lang}.{mode}', f'{OUT_DIR}/{lang}.{mode}')

def move_entire_dataset():
    for family in LANGS.keys():
        if os.path.exists(f"{DEV_DIR}/{family}"):
            print('moving DEV family', family)
            move_family_dataset(family, DEV_DIR)
        else:
            print('moving SURP family', family)
            move_family_dataset(family, SURP_DIR)

if __name__ == "__main__":
    try:
        os.mkdir(OUT_DIR)
    except:
        pass
    move_entire_dataset()


import json, os
from datetime import date, datetime
from itertools import chain

from django.core.urlresolvers import reverse
from recordlib import RecordParser
from dateutil.parser import parse

APP_BASE_DIR = os.path.dirname(__file__)
COLLECT_LOG_FILE = os.path.join(APP_BASE_DIR, 'logs/collect.log')
MAX_COLLECT_LENGTH = 20
ord_types = {'st': '정기', 'ex': '추가', 'em': '응급', 'op': '퇴원', 'ch': '항암'}
ord_types_reverse = {'정기': 'st', '추가': 'ex', '응급':'em', '퇴원':'op', '항암':'ch'}

class LabelRecordParser:

	def __init__(self, json_path=COLLECT_LOG_FILE):
		self.json_path = json_path
		try:
			with open(json_path) as fp:
				self.label_history_list = json.loads(fp.read())
		except:
			self.label_history_list = []

	def save_queryset(self, agg, detail, ord_tp, form_data):
		today = date.today().strftime('%Y-%m-%d')
		seq_list = [row['seq'] for row in self.label_history_list if row['date'] == today and row['ord_tp'] == ord_types[ord_tp]]
		seq = max(seq_list) + 1 if seq_list else 0
		queryset = []
		recs = RecordParser(detail)

		today_queryset = [history for history in self.label_history_list if history['date'] == today]
		today_subqueryset = []

		for rec in today_queryset:
			collect_seq = rec['seq'] + 1
			for row in rec['records']:
				today_subqueryset.append((collect_seq, row['sub_object_list']))

		for item in agg:
			item['sub_object_list'] = recs.select('*', where = lambda row: row['ord_cd'] == item['ord_cd'], inplace=False).order_by(['ord_ymd', 'rcpt_dt']).records
			item['duplicated'] = False
			for sub in item['sub_object_list']:
				sub['duplicated'] = 0
				for cseq, today_sub_list in today_subqueryset:

					for today_sub in today_sub_list:
						sub_dup_save = sub.pop('duplicated')
						today_sub_save = today_sub.pop('duplicated')
						if sub == today_sub:
							sub['duplicated'] = cseq
							item['duplicated'] = True
						else:
							sub['duplicated'] = sub_dup_save
						today_sub['duplicated'] = today_sub_save

			queryset.append(item)

		log = {
			'date' : today,
			'seq': seq,
			'ord_tp': ord_types[ord_tp],
			'form_data': form_data,
			'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
			'records': queryset
		}
		self.label_history_list.append(log)
		with open(self.json_path, 'w') as fp:
			fp.write(json.dumps(self.label_history_list[-MAX_COLLECT_LENGTH:]))

		return log['records']

	def select_queryset(self, date=date.today().strftime('%Y-%m-%d'), seq=-1):
		queryset_list = [history for history in self.label_history_list if history['date'] == date and (history['seq'] == seq if seq!=-1 else True)]
		if not queryset_list:
			return
		if seq == -1:
			return queryset_list[seq]['records']
		return queryset_list[0]['records']

	def delete_queryset(self, ord_tp, date, seq):
		for idx, history in enumerate(self.label_history_list):
			if history['ord_tp'] == ord_types[ord_tp] and history['seq'] == int(seq) and history['date'] == date:
				self.label_history_list.pop(idx)
				with open(self.json_path, 'w') as fp:
					fp.write(json.dumps(self.label_history_list[-MAX_COLLECT_LENGTH:]))
				return
		return

	def get_collect_object_list(self):
		ret = []
		for collect in self.label_history_list:

			s = '[{}]{} {}차({}건)'.format(collect['date'], collect['ord_tp'], collect['seq']+1, sum(len(rec['sub_object_list']) for rec in collect['records']))
			url = reverse('orderutils:labelcollect-history',
				 kwargs = {'date': collect['date'], 'seq': collect['seq'], 'ord_tp': ord_types_reverse[collect['ord_tp']]}
			)

			ret.append({'description': s, 'url': url})
		return ret[::-1]

	def select_collect(self, date, ord_tp, seq=-1):
		if not self.label_history_list:
			return []

		if seq == -1:
			ord_tp_history = [collect for collect in self.label_history_list if collect['ord_tp'] == ord_types[ord_tp]]
			return ord_tp_history[seq] if ord_tp_history else []

		for collect in self.label_history_list:
			if collect['seq'] == seq and collect['date'] == date and collect['ord_tp'] == ord_types[ord_tp]:
				return collect
		return []

	def get_last_collect(self, date=date.today().strftime('%Y-%m-%d')):
		ret = {}
		for collect in [collect for collect in self.label_history_list if collect['date'] == date]:
			ret[ord_types_reverse[collect['ord_tp']]] = collect['form_data']['end_t']

		return ret

	def clear_history(self):
		if os.path.exists(COLLECT_LOG_FILE):
			os.unlink(COLLECT_LOG_FILE)

# ret = load_collect_log()
# print(create_label_collect_queryset(ret['agg'], ret['detail']))
# l = LabelRecordParser()
# print(l.label_history_list)


def test_comprendre_dictionnaire():
    # Les noms de pays en fonction de leur code ISO 3166-1 alpha-2
    pays = { 'FR': 'France', 'DE' : 'Allemagne',
            'ES' : 'Espagne', 'GB' : 'Angleterre' }

    # Obtenir le pays 'Allemagne'
    assert 'Allemagne' == pays['DE']

    # Vérifier que le code 'IT' n'est pas défini dans pays.
    assert 'IT' not in pays

    # Que se passe-t-il si on veut récupérer le pays associé à 'IT' ?
    '''Il ne se passe rien car IT n'est pas dans pays'''

    # Obtenir le pays qui correspond à un code donné ('IT' ou 'FR' par exemple).
    # S'il n'y a pas de pays associé au code on obtiendra 'INCONNU'.
    for code, nom_attendu in (('FR', 'France'), ('IT', 'INCONNU')):
        assert nom_attendu == 'France' or 'INCONNU'   #retrouver le nom associé à code

    # Ajouter le nom 'Italie' associé au code 'IT'.
    pays['IT'] = 'Italie'
    assert 'IT' in pays # le code 'IT' est défini
    assert pays['IT'] == 'Italie'  # le pays associé est 'Italie'

    # Changer le pays associé à 'GB' pour mettre 'Royaume-Uni'
    pays.update({'GB' : 'Royaume-Uni'})
    assert 'Royaume-Uni' == pays['GB'] # le pays associé à 'GB' est "Royaume-Uni'

    # Obtenir tous les codes connus dans pays
    codes = pays.keys()
    print('Les codes :', codes)

    # Obtenir tous les noms de pays connus dans pays
    les_pays = pays.values()
    print('Les pays :', les_pays)

    # Obtenir tous les couples (code, nom) de pays
    couples = pays.items()
    print('Les couples :', couples)

    # Afficher le contenu de pays sous la forme : code -> pays
    for i in pays:
      print(i, '->', pays[i])

if __name__ == '__main__':
    test_comprendre_dictionnaire()


import requests
from bs4 import BeautifulSoup

class Parse_weather():
    def get_data(self):
        result = requests.get('https://yandex.ru/pogoda/saint-petersburg')
        self.soup = BeautifulSoup(result.text, 'html.parser')
        div = self.soup.findAll('div')
        soup = self.soup
        div = soup.find( 'div', {'class': 'content__row'} )
        place = div.find( 'h1', {'class': True} )
        temp = div.find( 'span', {'class': 'temp__value'} )
        status = div.find( 'div', {'class': 'link__condition day-anchor i-bem'} )
        feeling = div.find( 'div', {'class': 'term term_orient_h fact__feels-like'} ). \
            find( 'span', {'class': 'temp__value'} )
        wind_speed = div.find( 'span', {'class': 'wind-speed'} )
        wind_way = div.find( 'abbr', {'class': 'icon-abbr'} )
        humidity = div.find( 'div', {'class': 'term term_orient_v fact__humidity'} ). \
            find( 'div', {'class': 'term__value'} )

        pressure = div.find( 'div', {'class': 'term term_orient_v fact__pressure'} ). \
            find( 'div', {'class': 'term__value'} )

        reply = place.text + '\n' + \
                'Температура:' + temp.text + '°C' + '\n' + \
                'На улице:' + status.text + '\n' + \
                'Ощущается как: ' + feeling.text + '°C' + '\n' + \
                'Ветер: ' + wind_speed.text + ' м/с ' + wind_way.text + '\n' + \
                'Влажность: ' + humidity.text + '\n' + \
                'Давление: ' + pressure.text + '\n'
        return reply

answer = Parse_weather()
weather = answer.get_data()

print(weather)


import requests
import bs4

KEYWORDS = ['дизайн', 'фото', 'web', 'python', 'программирование']
HEADERS = {'Accept': 'text/html,application/xhtml+xml,'
                     'application/xml;q=0.9,image/avif,'
                     'image/webp,image/apng,'
                     '*/*;q=0.8,application/signed-exchange;'
                     'v=b3;q=0.9 ',
           'Accept-Encoding': 'gzip, deflate, br',
           'Accept-Language': 'ru-RU,ru;q=0.9,en-US;'
                              'q=0.8,en;q=0.7',
           'Cache-Control': 'max-age=0',
           'Connection': 'keep-alive',
           'Cookie':
               '_ym_uid=1639934666860889724; '
               '_ym_d=1639934666; '
               '_ga=GA1.2.122808168.1639934666; '
               '_gid=GA1.2.1540720803.1639934666; '
               'habr_web_home=ARTICLES_LIST_ALL; '
               'hl=ru; fl=ru; _ym_isad=2; '
               '__gads=ID=3bca2fb12a84bfaa:T=1639934668:'
               'S=ALNI_MagABWot_b98koFdCB4lM3aLC2RAA; '
               'cto_bundle'
               '=0wD_Y19KQmE2VSUyQmdwbFBWOGlDVGdGNlhL'
               'NnpDNEVUdGNKOWtQSlpHVjRXTXZqQVdoYkVkM'
               'kJDSGFHemd2OHhFY1lLT0g1UVlweEZYOWlBTmR'
               'GaGdlaW4zOEhCQlI1RWVNcEJEUjJHRWdjVTRFN'
               'mx3bmxhcnFMODlSWCUyRkdETmpOUnloWHZGQU1'
               'CV2hNV2duOG03aUNHcEpPTFhXS2dWRGFkMVNyT'
               'Tlyb1pwZ3psVlZ1b0ZyJTJCcVlEcFN3VEx3eE'
               'NCbEpEZ3o ',
           'Host': 'habr.com',
           'If-None-Match': 'W/"3a406-HWoKTAp/3V+g6gWQdD8MLds+0Ns"',
           'sec-ch-ua': '" Not A;Brand";v="99", '
                        '"Chromium";v="96", "Google Chrome";v="96"',
           'sec-ch-ua-mobile': '?0',
           'sec-ch-ua-platform': "macOS",
           'Sec-Fetch-Dest': 'document',
           'Sec-Fetch-Mode': 'navigate',
           'Sec-Fetch-Site': 'same-origin',
           'Sec-Fetch-User': '?1',
           'Upgrade-Insecure-Requests': '1',
           'User-Agent':
               'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) '
               'AppleWebKit/537.36 (KHTML, like Gecko) '
               'Chrome/96.0.4664.110 Safari/537.36 '
           }

response = requests.get('https://habr.com/ru/all/', headers=HEADERS)
response.raise_for_status()
text = response.text

soup = bs4.BeautifulSoup(text, features='html.parser')
articles = soup.find_all('article')
print(len(articles))

for article in articles:
    hubs = article.find_all('a', class_="tm-article-snippet__hubs-item-link")
    hubs = [hub.find('span').text for hub in hubs]
    # print(hubs)
    title = article.find('a', class_="tm-article-snippet__title-link").text
    # print(title)
    link = article.find('a', class_="tm-article-snippet__title-link").\
        get('href')
    # print(link)
    article_preview = article.find('div', class_='article-formatted-body').text
    # print(article_preview)
    date = article.find('span',
                        class_='tm-article-snippet__datetime-published')
    datetime = date.find('time')['title']
    # print(datetime)

    for word in KEYWORDS:
        if (word.lower() in title.lower()) \
                or (word.lower() in article_preview.lower()) \
                or (word.capitalize() in hubs):
            print(f'Дата: {datetime} - Заголовок: {title} '
                  f'- Ссылка: {"habr.com" + link}')


import codecs
import sys

import xml.etree.ElementTree as ET

from bs4 import BeautifulSoup, Tag

weeks = [int(w) for w in sys.argv[1:]]

file = codecs.open("./temp/index.html", "r", "utf-8")
soup = BeautifulSoup(file.read(), 'html.parser')
accordianfile = codecs.open("./templates/accordian.html", "r", "utf-8")
accordian_soup = BeautifulSoup(accordianfile.read(), 'html.parser')

for i in weeks:
    tree = ET.parse('calendar/week%s.xml' % i)
    root = tree.getroot()

    button = accordian_soup.find(id="week%s" % i)
    if i == weeks[-1]:
        button['class'].remove('collapsed')
        body = accordian_soup.find(id="collapse%i" % i)
        body['class'].append('show')
    button.append(root[0].text)

    for column in root[1:]:
        materials = accordian_soup.find(id="materials%s" % i)
        col = accordian_soup.new_tag("div")
        col['class'] = ['col-md-6']

        header = accordian_soup.new_tag("h5")
        header.string = column.text
        col.append(header)
        col.append(accordian_soup.new_tag("hr"))

        for section in column:
            title = accordian_soup.new_tag("span")
            title.string = section.text.rstrip()
            col.append(title)
            list = accordian_soup.new_tag("ul")
            listItem = accordian_soup.new_tag("li")

            for item in section:
                lk = accordian_soup.new_tag("a")
                lk['class'] =  ['bordered']
                lk.string = item.text.rstrip()
                lk['href'] = item[0].text.rstrip()
                lk['target'] = ['_blank']
                if item[1].text == "True":
                    lk['class'].extend(['btn-link', 'disabled'])
                listItem.append(lk)
            list.append(listItem)
            col.append(list)
        materials.append(col)

# print(accordian_soup.prettify())

for i in range(weeks[-1] + 1, len(weeks)+1):
    button = accordian_soup.find(id="week%s" % i)
    button['class'].append('disabled')

    tree = ET.parse('calendar/week%s.xml' % i)
    root = tree.getroot()
    button.append(root[0].text)

accordian = soup.find(id="weekly-info")
accordian.append(str(accordian_soup))

stringy = str(soup.prettify())
stringy = stringy.replace('<', '<')
stringy = stringy.replace('>', '>')
outfile = codecs.open("./temp/index.html", "w+", "utf-8")
outfile.write(stringy)