"""
Author(s):      Nihal Dhamani (nihaldhamani@gmail.com),
                Carson Schubert (carson.schubert14@gmail.com)
Date Created:   12/06/2019

Core command group and commands for TF Bounding Box plugin.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from comet_ml import Experiment
import os
import click
import io
import sys
import shutil
import yaml
import importlib
import re
import glob
import json
import traceback
import time
import tensorflow as tf
import rmltraintfbbox.validation.utils as utils
from contextlib import ExitStack
from pathlib import Path
from datetime import datetime
from ravenml.train.options import pass_train
from ravenml.train.interfaces import TrainInput, TrainOutput
from ravenml.utils.question import cli_spinner, user_selects, user_input
from ravenml.utils.plugins import raise_parameter_error
from rmltraintfbbox.utils.helpers import prepare_for_training, download_model_arch
from rmltraintfbbox.utils.exporter import export_inference_graph
from rmltraintfbbox.validation.stats import BoundingBoxEvaluator
from google.protobuf import text_format
from matplotlib import pyplot as plt
from object_detection import model_lib, model_lib_v2, inputs, protos
from object_detection.builders import optimizer_builder, model_builder
from object_detection.utils import label_map_util, visualization_utils, config_util

# regex to ignore 0 indexed checkpoints
checkpoint_regex = re.compile(r'model.ckpt-[1-9][0-9]*.[a-zA-Z0-9_-]+')

### OPTIONS ###

### COMMANDS ###
@click.group(help='TensorFlow2 Object Detection with bounding boxes.')
@click.pass_context
def tf_bbox(ctx):
    pass

@tf_bbox.command(help='Train a model.')
@pass_train
@click.pass_context
def train(ctx: click.Context, train: TrainInput):
    # If the context has a TrainInput already, it is passed as "train"
    # If it does not, the constructor is called AUTOMATICALLY
    # by Click because the @pass_train decorator is set to ensure
    # object creation, after which execution will fail as this means
    # the user did not pass a config. see ravenml core file train/commands.py for more detail

    # NOTE: after training, you must create an instance of TrainOutput and return it

    ## SET UP CONFIG ##
    config = train.plugin_config
    metadata = train.plugin_metadata
    comet = config.get('comet')

    if config.get('verbose'):
        tf.autograph.set_verbosity(level=10, alsologtostdout=True)
    else:
        tf.autograph.set_verbosity(level=0, alsologtostdout=True)

    # set base directory for model artifacts
    base_dir = train.artifact_path

    # load model choices from YAML
    models = {}
    models_path = os.path.dirname(__file__) / Path('utils') / Path('model_info.yml')
    with open(models_path, 'r') as stream:
        try:
            models = yaml.safe_load(stream)
        except yaml.YAMLError as exc:
            print(exc)

    # prompt for model selection if not in config
    model_name = config.get('model')
    model_name = model_name if model_name else user_selects('Choose model', models.keys())
    # grab fields and add to metadata
    try:
        model = models[model_name]
    except KeyError as e:
        hint = 'model name, model is not supported by this plugin.'
        raise_parameter_error(model_name, hint)

    # extract information and add to metadata
    model_type = model['type']
    model_url = model['url']
    metadata['architecture'] = model_name

    # download model arch
    arch_path = download_model_arch(model_url, train.plugin_cache)

    # prepare directory for training/prompt for hyperparams
    if not prepare_for_training(train.plugin_cache, train.artifact_path, train.dataset.path,
                                arch_path, model_type, metadata, train.plugin_config):
        ctx.exit('Training cancelled.')

    model_dir = os.path.join(base_dir, 'models/model')
    train_dir = os.path.join(model_dir, 'train')
    eval_dir = os.path.join(model_dir, 'eval')
    pipeline_config_path = os.path.join(model_dir, 'pipeline.config')

    experiment = None
    if comet:
        experiment = Experiment(workspace='seeker-rd', project_name='bounding-box')
        experiment.set_name(comet)
        experiment.log_parameters(metadata['hyperparameters'])
        experiment.set_git_metadata()
        experiment.set_os_packages()
        experiment.set_pip_packages()
        experiment.log_asset(pipeline_config_path)

    # get number of training steps
    num_train_steps = int(metadata['hyperparameters']['train_steps'])

    configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
    model_config = configs['model']
    train_config = configs['train_config']
    train_input_config = configs['train_input_config']
    eval_input_config = configs['eval_input_config']
    eval_config = configs['eval_config']

    detection_model = model_builder.build(model_config=model_config, is_training=True)

    # create tf.data.Dataset()
    train_input = inputs.train_input(train_config, train_input_config, model_config, model=detection_model)
    eval_input = inputs.eval_input(eval_config, eval_input_config, model_config, model=detection_model)

    train_input_iterator = iter(train_input)

    global_step = tf.Variable(
        0, trainable=False, dtype=tf.int64, name='global_step',
        aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA)
    optimizer, (learning_rate,) = optimizer_builder.build(
        train_config.optimizer, global_step=global_step)

    if callable(learning_rate):
        learning_rate_fn = learning_rate
    else:
        learning_rate_fn = lambda: learning_rate

    # restore from checkpoint
    load_fine_tune_checkpoint(detection_model, train_config.fine_tune_checkpoint,
                                    train_config.fine_tune_checkpoint_type,
                                    train_config.fine_tune_checkpoint_version,
                                    train_input,
                                    train_config.unpad_groundtruth_tensors)

    @tf.function
    def train_step(detection_model, train_input_iterator, optimizer, learning_rate_fn, global_step):

        features, labels = train_input_iterator.next()
        loss = model_lib_v2.eager_train_step(detection_model, features,
                            labels, train_config.unpad_groundtruth_tensors,
                            optimizer, learning_rate_fn(),
                            add_regularization_loss=True,
                            clip_gradients_value=None,
                            global_step=global_step)
        global_step.assign_add(1)
        return loss

    # @tf.function
    def evaluate(detection_model, configs, eval_input, global_step):

        text_trap = io.StringIO()
        sys.stdout = text_trap
        sys.stderr = text_trap

        metrics = model_lib_v2.eager_eval_loop(detection_model, configs, eval_input, global_step=global_step)

        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__

        click.echo(f'Evaluation loss: {metrics["Loss/total_loss"]}, Evaluation mAP: {metrics["DetectionBoxes_Precision/mAP"]}')

        return metrics

    checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=detection_model)
    manager = tf.train.CheckpointManager(checkpoint, directory=model_dir, max_to_keep=5)

    with ExitStack() as stack:
        if comet:
            stack.enter_context(experiment.train())
        click.echo('Training model...')

        start = time.time()
        # main training loop
        losses = []
        for step in range(1, num_train_steps+1):

            losses.append(train_step(detection_model, train_input_iterator, optimizer, learning_rate_fn, global_step))

            if step % config.get('log_train_every') == 0:
                avg_loss = sum(losses) / len(losses)
                print(f'Avg train loss at step {step}: {avg_loss}')
                losses = []
                if comet:
                    experiment.log_metric('avg_loss', avg_loss)

            if step % config.get('log_eval_every') == 0:
                manager.save()
                eval_metrics = evaluate(detection_model, configs, eval_input, global_step)
                if comet:
                    stack.enter_context(experiment.validate())
                    experiment.log_metrics(eval_metrics, step=step)
                    stack.enter_context(experiment.train())

        training_time = time.time() - start

        click.echo(f'Training complete. Took {training_time} seconds.')

    # final metadata and return of TrainOutput object
    datetime_finished = datetime.utcnow().isoformat() + "Z"
    metadata['date_completed_at'] = datetime_finished

    # get extra config files
    extra_files = _get_paths_for_extra_files(base_dir)

    if config.get('evaluate'):
        click.echo("Evaluating model...")
        with ExitStack() as stack:
            if comet:
                stack.enter_context(experiment.test())
            # path to label_map.pbtxt
            label_path = str(train.dataset.path / 'label_map.pbtxt')
            test_path = str(train.dataset.path / 'test')
            output_path = str(base_dir / 'validation')
            os.mkdir(output_path)

            image_dataset = utils.get_image_dataset(test_path)
            truth_data = list(utils.gen_truth_data(test_path))

            category_index = label_map_util.create_category_index_from_labelmap(label_path)
            evaluator = BoundingBoxEvaluator(category_index)

            for (i, (bbox, centroid, z)), image in zip(enumerate(truth_data), image_dataset):
                true_shape = tf.expand_dims(tf.convert_to_tensor(image.shape), axis=0)
                start = time.time()
                output = detection_model.call(tf.expand_dims(image, axis=0))
                inference_time = time.time() - start
                evaluator.add_single_result(output, true_shape, inference_time, bbox, centroid)
                drawn_img = visualization_utils.draw_bounding_boxes_on_image_tensors(
                                                tf.cast(tf.expand_dims(image, axis=0), dtype=tf.uint8),
                                                output['detection_boxes'],
                                                tf.cast(output['detection_classes'] + 1, dtype=tf.int32),
                                                output['detection_scores'],
                                                category_index,
                                                max_boxes_to_draw=1,
                                                min_score_thresh=0)
                tf.keras.preprocessing.image.save_img(output_path+f'/img{i}.png', drawn_img[0])

            evaluator.dump(os.path.join(output_path, 'validation_results.pickle'))
            if comet:
                experiment.log_asset(os.path.join(output_path, 'validation_results.pickle'))
            evaluator.calculate_default_and_save(output_path)

            extra_files.append(os.path.join(output_path, 'stats.json'))
            extra_files.append(os.path.join(output_path, 'validation_results.pickle'))
            extra_files += glob.glob(os.path.join(output_path, '*_curve_*.png'))
            if comet:
                experiment.log_asset(os.path.join(output_path, 'stats.json'))
                for img in glob.glob(os.path.join(output_path, '*_curve_*.png')):
                    experiment.log_image(img)

    # export detection_model as SavedModel
    saved_model_dir = os.path.join(model_dir, 'export')
    configproto = config_util.create_pipeline_proto_from_configs(configs)
    export_inference_graph('image_tensor', configproto, model_dir, saved_model_dir)

    #zip files in export directory and add to extra_files
    shutil.make_archive(os.path.join(saved_model_dir, 'export'), 'zip', model_dir, 'export')
    extra_files.append(os.path.join(saved_model_dir, 'export.zip'))
    saved_model_path = os.path.join(saved_model_dir, 'saved_model', 'saved_model.pb')

    if comet:
        experiment.log_parameter('training_title', comet)
        experiment.log_asset_data(train.metadata, file_name="metadata.json")
        experiment.log_parameter('dataset_name', train.config['dataset'])

    # export metadata locally
    with open(base_dir / 'metadata.json', 'w') as f:
        json.dump(train.metadata, f, indent=2)

    extra_files = [Path(f) for f in extra_files]
    result = TrainOutput(Path(saved_model_path), extra_files)
    return result

### HELPERS ###

def load_fine_tune_checkpoint(
        model, checkpoint_path, checkpoint_type, checkpoint_version, input_dataset,
        unpad_groundtruth_tensors):

    # NOTE: Moved this function from OD API to add 'expect_partial' to the restore
    """ Load a fine tuning classification or detection checkpoint.
        To make sure the model variables are all built, this method first executes
        the model by computing a dummy loss. (Models might not have built their
        variables before their first execution)
        It then loads an object-based classification or detection checkpoint.
        This method updates the model in-place and does not return a value.
    Args:
        model: A DetectionModel (based on Keras) to load a fine-tuning
            checkpoint for.
        checkpoint_path: Directory with checkpoints file or path to checkpoint.
        checkpoint_type: Whether to restore from a full detection
            checkpoint (with compatible variable names) or to restore from a
            classification checkpoint for initialization prior to training.
            Valid values: `detection`, `classification`.
        checkpoint_version: train_pb2.CheckpointVersion.V1 or V2 enum indicating
            whether to load checkpoints in V1 style or V2 style.  In this binary
            we only support V2 style (object-based) checkpoints.
        input_dataset: The tf.data Dataset the model is being trained on. Needed
            to get the shapes for the dummy loss computation.
        unpad_groundtruth_tensors: A parameter passed to unstack_batch.
    Raises:
        IOError: if `checkpoint_path` does not point at a valid object-based
            checkpoint
        ValueError: if `checkpoint_version` is not train_pb2.CheckpointVersion.V2
    """
    if not model_lib_v2.is_object_based_checkpoint(checkpoint_path):
        raise IOError('Checkpoint is expected to be an object-based checkpoint.')
    if checkpoint_version == protos.train_pb2.CheckpointVersion.V1:
        raise ValueError('Checkpoint version should be V2')

    features, labels = iter(input_dataset).next()

    @tf.function
    def _dummy_computation_fn(features, labels):
        model._is_training = False  # pylint: disable=protected-access
        tf.keras.backend.set_learning_phase(False)
        labels = model_lib.unstack_batch(
            labels, unpad_groundtruth_tensors=unpad_groundtruth_tensors)

        return model_lib_v2._compute_losses_and_predictions_dicts(
            model,
            features,
            labels)

    strategy = tf.compat.v2.distribute.get_strategy()
    if hasattr(tf.distribute.Strategy, 'run'):
        strategy.run(
            _dummy_computation_fn, args=(
                features,
                labels,
            ))
    else:
        strategy.experimental_run_v2(
            _dummy_computation_fn, args=(
                features,
                labels,
            ))

    restore_from_objects_dict = model.restore_from_objects(
        fine_tune_checkpoint_type=checkpoint_type)
    ckpt = tf.train.Checkpoint(**restore_from_objects_dict)
    ckpt.restore(checkpoint_path).expect_partial()

def _get_paths_for_extra_files(artifact_path: Path):
    """Returns the filepaths for all checkpoint, config, and pbtxt (label)
    files in the artifact directory. Gets filepath for the exported inference
    graph.

    Args:
        artifact_path (Path): path to training artifacts

    Returns:
        list: list of Paths that point to files
    """
    extras = []
    # get checkpoints
    extras_path = artifact_path / 'models' / 'model'
    files = os.listdir(extras_path)

    # path to label map
    labels_path = artifact_path / 'label_map.pbtxt'

    checkpoints = [f for f in files if checkpoint_regex.match(f)]

    # calculate the max checkpoint
    max_checkpoint = 0
    for checkpoint in checkpoints:
        checkpoint_num = int(checkpoint.split('-')[1].split('.')[0])
        if checkpoint_num > max_checkpoint:
            max_checkpoint = checkpoint_num

    ckpt_prefix = 'model.ckpt-' + str(max_checkpoint)
    checkpoint_path = extras_path / ckpt_prefix
    pipeline_path = extras_path / 'pipeline.config'

    # append files to include in extras directory
    extras = [extras_path / f for f in checkpoints]
    extras.append(pipeline_path)

    # append event checkpoints for tensorboard
    for f in os.listdir(extras_path):
        if f.startswith('events.out'):
            extras.append(extras_path / f)

    extras.append(labels_path)
    return extras


import logging
import os
import sys
import traceback

from PySide6.QtCore import QCoreApplication
from PySide6.QtWidgets import QMessageBox

# noinspection PyArgumentList
def exceptionHandler(exctype, value, trace):
    """
        This exception handler prevents quitting to the command line when there is
        an unhandled exception while processing a Qt signal.

        The script/application willing to use it should implement code similar to:

        .. code-block:: python

            if __name__ == "__main__":
                sys.excepthook = exceptionHandler

        """

    if exctype == KeyboardInterrupt:
        sys.__excepthook__(exctype, value, trace)
        sys.exit()

    if trace is not None:
        msg = f"{exctype.__name__} in {trace.tb_frame.f_code.co_name}"
    else:
        msg = exctype.__name__
    logger = logging.getLogger()
    logger.error(msg, exc_info=(exctype, value, trace))
    msg = QMessageBox(QCoreApplication.instance().activeWindow())
    msg.setWindowTitle("Unhandled exception")
    msg.setIcon(QMessageBox.Icon.Warning)
    msg.setText(("It seems you have found a bug in {}. Please report details.\n"
                 "You should restart the application now.").format(QCoreApplication.applicationName()))
    msg.setInformativeText(str(value))
    msg.setDetailedText(''.join(traceback.format_exception(exctype, value, trace)))
    bt_restart = msg.addButton("Restart now", QMessageBox.ButtonRole.ResetRole)
    msg.addButton(QMessageBox.StandardButton.Ignore)
    msg.raise_()
    msg.exec_()
    if msg.clickedButton() == bt_restart:  # Restart application
        os.execv(sys.executable, [sys.executable] + sys.argv)


# -*- coding: utf-8 -*-
# <nbformat>3.0</nbformat>

# <markdowncell>

# # Converting Pinboard.in to CSV
#
# Tony has saved tons of stuff on Pinboard.in, let's convert this to a csv so we can all categorize stuff using a spreadsheet.

# <codecell>

import urllib2, lxml.etree,re

# <codecell>

u=urllib2.urlopen("https://feeds.pinboard.in/rss/t:scodamd/")
data=u.read()
u.close()

# <codecell>

root=lxml.etree.fromstring(data)

# <codecell>

ns=[("rss","http://purl.org/rss/1.0/"),("rdf","http://www.w3.org/1999/02/22-rdf-syntax-ns#"),("taxo","http://purl.org/rss/1.0/modules/taxonomy/")]

# <codecell>

items=root.xpath("//rss:item",namespaces=ns)

# <codecell>

items

# <codecell>

def tagdict(x,y):
    x[y[0]]=x.get(y[0],"")+y[1]+" "
    return x

def process_item(i):
    link=i.xpath("./rss:link",namespaces=ns)[0].text
    title=i.xpath("./rss:title",namespaces=ns)[0].text.split("|")[0].strip().encode("utf-8")
    tags=[re.search("/t:([a-zA-Z0-9:]+)$",n.attrib["{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource"]).group(1) for n in i.xpath("./taxo:topics/rdf:Bag/rdf:li",namespaces=ns)]
    r=reduce(tagdict,[(j[0],j[1]) for j in filter(lambda x: len(x)>1, [i.split(":") for i in tags])],{})
    r["link"]=link
    r["title"]=title
    return r

# <codecell>

courses=[process_item(i) for i in items]

# <codecell>

import csv
f=open("tagged-courses.csv","wb")
w=csv.DictWriter(f,delimiter=",",fieldnames=["title","link","course","theme","level","skill","skilll","tool","topic","media","audience","meta"])
for c in courses:
    w.writerow(c)
f.close()

# <codecell>

# <codecell>

courses

# <codecell>


from twisted.internet import reactor
from twisted.internet.protocol import Factory, connectionDone
from twisted.protocols.basic import LineReceiver

class MyProtocol(LineReceiver):
    def rawDataReceived(self, data):
        pass

    def __init__(self):
        self.name = None
        self.curFunc = self.handle_hello
        self.loggedIn = False

    def end(self, reason):
        self.transport.write("closing connection.\r\n")
        self.transport.write("reason is [" + reason + "]\r\n")
        self.transport.loseConnection()
        if self.loggedIn:
            self.factory.num_users -= 1
            self.loggedIn = False

    def handle_hello(self, line):
        if line == "hello":
            self.transport.write("hello\r\n")
            self.curFunc = self.handle_auth
        else:
            self.end("did not get hello")

    def handle_auth(self, line):
        (auth, name, password) = line.split(",")
        if auth == "auth":
            if name * 2 == password:
                self.transport.write("auth ok\r\n")
                self.curFunc = self.handle_status
                self.factory.num_users += 1
                self.name = name
                self.loggedIn = True
            else:
                self.end("authentication problem")
        else:
            self.end("did not get auth line")

    def handle_status(self, line):
        if line == "status":
            self.transport.write(
                "num_users is " + str(self.factory.num_users) + "\r\n")
            self.transport.write("your name is " + self.name + "\r\n")
            self.curFunc = self.handle_bye
        else:
            self.end("did not get status")

    def handle_bye(self, line):
        if line == "bye":
            self.transport.write("bye " + self.name + "\r\n")
            self.end("conversation ended")
        else:
            self.end("did not get bye")

    def connectionMade(self):
        print("connection made")
        self.transport.write("Welcome to MyServer\r\n")

    def lineReceived(self, line):
        print("got line ", line)
        self.curFunc(line)

    def connectionLost(self, reason=connectionDone):
        print("connection was lost")
        if self.loggedIn:
            self.factory.num_users -= 1

class MyFactory(Factory):
    protocol = MyProtocol

    def __init__(self):
        self.num_users = 0
        self.port = 8007

myFactory = MyFactory()
# pylint: disable=no-member
reactor.listenTCP(myFactory.port, myFactory)  # type: ignore
reactor.run()  # type: ignore


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import RPi.GPIO as GPIO
from time import sleep

def flash_led(color):
    if color=="blue":
        LED=16          #青がGPIO16
    elif color=="green":
        LED=20          #緑がGPIO20
    elif color=="red":
        LED=21          #赤がGPIO21

    GPIO.setmode(GPIO.BCM)
    GPIO.setup(LED, GPIO.OUT)

    #5回点滅させる
    for i in range(5):
        GPIO.output(LED, GPIO.HIGH)
        sleep(0.5)
        GPIO.output(LED, GPIO.LOW)
        sleep(0.5)
    else:
        GPIO.cleanup()


from django.forms import ValidationError
from rest_framework import generics, mixins, status
from rest_framework.response import Response

from rest_framework.decorators import api_view
from django.views.decorators.csrf import ensure_csrf_cookie
from django.core.files import File

from api.models import DataSet
from api.serializers import DataSetSerializer, DataSetFormSerializer
from wsgiref.util import FileWrapper
from django.http import HttpResponse
import os

class DataSetGenericApiView(
        generics.GenericAPIView,
        mixins.ListModelMixin,
        mixins.RetrieveModelMixin,
        mixins.CreateModelMixin,
        mixins.UpdateModelMixin,
        mixins.DestroyModelMixin
        ):

    queryset = DataSet.objects
    serializer_class = DataSetSerializer

    def get(self, request, pk=None):
        if pk:
            return Response(
                    self.retrieve(request).data,
                    status = status.HTTP_200_OK
                    )
        else:
            return Response(
                    self.list(request).data,
                    status = status.HTTP_200_OK
                )

    def post(self,request):
        try:
            response = self.create(request).data,
        except ValidationError as error:
            return Response(
                    error.message_dict,
                    status = status.HTTP_400_BAD_REQUEST
                )
        return Response(
                response,
                status = status.HTTP_201_CREATED
            )

    def delete(self, request, pk = None):
        if not pk:
            return Response(
                    {"error":"Data set id is required"},
                    status = status.HTTP_404_NOT_FOUND
                )
        data_set = DataSet.objects.filter(id = pk).first()
        os.remove(data_set.graph.path)
        os.remove(data_set.file.path)

        self.destroy(request)
        return Response(
                {"message":f'Data set with id {pk} was deleted!'},
                status = status.HTTP_204_NO_CONTENT
                )

@api_view(['GET'])
def get_data_set_form_data(request):
    data = DataSet.objects.all()
    serialier = DataSetFormSerializer(data, many = True)
    return Response(
            {'data_set_options': serialier.data},
            status = status.HTTP_200_OK
            )


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.13.6
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# %% [markdown]
# ---
#
# #  Iterables
#
# ---
#
#
#
# ---

# %% [markdown]
# Iterable types in Python contain sets of elements. There are
# + **string** - ordered sequence of characters
# + **set** - sets of unique objects
# + **tuple** - immutable ordered sets of objects
# + **list** - mutable ordered sets of objects
# + **dictionary** - mutable sets of key/value pairs.
#
# We will study each of these, but first we will look at built-in functions that apply to iterables in general.

# %% [markdown]
# ## Checking membership:  `in` and `not in`
#
# `a in A` evaluates to `True` if `a` is an element of the iterable `A`, and `False` otherwise.
#
# You can also use `a not in A` to get the opposite behavior.
#
# In the following example, `A` is a list.

# %%
A = [ 1/2, 0.4, 'hi', [0,1] ]

print(A)

# %% [markdown]
# ## Unpacking
#
# Unpacking is a shorthand syntax for assigning each of the elements of an iterable to different scalar variables.

# %%

# %% [markdown]
# **Question**: What do you think happens if the number of variable names does not match the number of elements in the iterable?

# %% [markdown]
# ---
# #   Indexing iterables
# ---
#
# + Strings, tuples, and lists are **ordered iterables**.
#
# + Indexing is **0-based** and uses square brackets `[]`.
#
# + Sets and dictionaries are not ordered. Therefore they cannot be indexed.

# %% [markdown]
# **Example** : Index a string

# %%
astring = 'Food is an important part of a balanced diet - Fran Lebowitz'

# %% [markdown]
# **Example** : Index a list

# %%
alist = [4.45, 'hello', 4+5j]

# %% [markdown]
# **Example** : Index a tuple

# %%
atuple = (4.45, 'hello', 4+5j)

# %% [markdown]
# **Lists are mutable, tuples and strings are not**

# %%

# %% [markdown]
# ## Slice indexing
# + You can get a 'slice' of an iterable using the colon symbol `:`
# + This returns a new iterable that is a portion of the original object.
# + `astring[a:b]` is the substring starting at index `a` and ending at `b-1`.

# %%

# %% [markdown]
# + Leaving out the start index means 'from the beginning'
# + Leaving out the end index means 'to the end'

# %%

# %% [markdown]
# ## Negative indices
# Negative indices count backward from the end.

# %%

# %% [markdown]
# ## Skipping values
#
# You can specify a "skip" value after a second colon: `A[start:end:skip]`

# %%
A = [0,1,2,3,4,5,6,7,8,9]

# What will this return?
A[0:-2:3]

# %% [markdown]
# ---
# #  [Strings methods](https://docs.python.org/3/library/stdtypes.html#string-methods)
# ---
#
# Things we can do with strings,
# + create them,
# + change cases in various ways,
# + search for a substring,
# + split strings,
# + etc.

# %%
strA = 'ABC'
strB = '123'

# %% [markdown]
# ## String `+`

# %%
print(strA + strB)

# %% [markdown]
# ## Formatting strings - [`.format()`](https://docs.python.org/3/library/stdtypes.html#str.format)
#
# The `format()` method is especially useful for building strings that involve numerical or other variables. The function is powerful, and you can find advanced examples [here](https://docs.python.org/3/library/string.html\#format-examples). However the simple form works most of the time. Here is an example.
#
# ### Example: "Johann Sebastian Bach was born in 1685, and died in 1750."

# %%
name = 'Johann Sebastian Bach'
birth_year = 1685
death_year = 1750

# using the 'format' method
str1 = '{0} was born in {1}, and died in {2}.'.format(name, birth_year, death_year)

print(str1)

# "f-string" method
str2 = f'{name} was born in {birth_year}, and died in {death_year}.'

print(str2)

# %% [markdown]
# ### `.split()`

# %%
a = '1,phone,None,-4.5,'
print(a)
b = a.split(',')
print(b)

# %% [markdown]
# ---
# ##  >> 5-minute challenge <<
# ---
# Use `split` to extract the name of the author in `astring`.

# %%
print(astring)

# %% [markdown]
# ---
# #  Lists: `[]`
# ---
# A **list** is a sequence of objects that is:
# + **ordered**: They can be indexed with `[]`.
# + **inhomogeneous**: They can contain a variety of object types.
# + **mutable**: You can modify them by adding and/or deleting items after they are created.

# %%

# %% [markdown]
# ## [List methods](https://docs.python.org/3/tutorial/datastructures.html#more-on-lists)
#
# The methods attached to list objects provide functionality for adding, removing, and changing the order of the elements in a list.

# %% [markdown]
# ## Building lists
#
# The `append`, `extend`, and `insert` methods can be used to populate a list. Later we will learn about "list comprehensions" which give us a compact syntax for building large lists (as well as sets and dictionaries).

# %% [markdown]
# ### `.append()` Puts a value at the end of the list.

# %% [markdown]
# ### `.extend()` Appends each value of an iterable to the list.

# %% [markdown]
# ### `.insert` Inserts an element at a given location in the list.

# %% [markdown]
# ## Removing items from lists
#
# ### `.remove()` Remove the first instance of a given object.

# %% [markdown]
# ### `.pop()` Extract the item at a given index and return it.

# %% [markdown]
# ### `del`
# Remove an item at a given index.

# %%
a=[4,1,9]
del a[1]
print(a)

# %% [markdown]
# ### `.clear()` Remove all items from the list.

# %%
a=[4,1,2,1,9]
a.clear()
print(a)

# %% [markdown]
# ---
# ##  >> 5-minute challenge <<
# ---
# 1. Create this list: [4,1,9]
# 2. Use list object methods to put it in reverse order: [9,4,1]
#
# **HINT**: `help(a.sort)` and`help(a.reverse)`

# %% [markdown]
# ---
# #  [Tuples](https://docs.python.org/3/tutorial/datastructures.html#tuples-and-sequences): `()`
# ---
#
# A **tuple** is a sequence of objects that is:
# + **ordered**
# + **inhomogeneous**
# + **mutable**
#
# Tuples offer only 2 methods: `count()` and `index()`.
#
# ### Why use a *tuple* instead of a *list*?
#
# + return values from functions
# + keys in dictionaries

# %%
a = (1,'asdf',4.3)

print(a)

a.index(4.3)

# %% [markdown]
#
# ---
# #  [Sets](https://docs.python.org/3/tutorial/datastructures.html#sets): `{}`
# ---
#
# A **set** is a sequence of objects that is:
# + **inhomogeneous**
# + **not ordered**
# + **mutable**
# + **contains no duplicates**
#

# %%
a = {4,6,3,3}

# %% [markdown]
# ---
# ##  >> 5-minute challenge <<
# ---
#
# Use a `set` to count the number of unique words in the following paragraph.
#
# X = "Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python’s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available in source or binary form for all major platforms from the Python Web site, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation."

# %%
X = "Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python’s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available in source or binary form for all major platforms from the Python Web site, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation."

words = X.split(' ')
unique_words = set(words)

print(len(words))

# %% [markdown]
# ---
# #  [Dictionaries](https://docs.python.org/3/tutorial/datastructures.html#dictionaries): `{:}`
# ---
# A **dictionary** is a mapping from a set of *keys* to a set of *values*.
# + The keys in a dictionary must be **immutable** (scalars, strings, tuples).
# + The values in a dictionary can be **anything**.
# + Dictionaries are created with **curly brackets** and **colons**: { a1:b1 , a2:b2 }

# %%
JSB = { 'name' : 'Johann Sebastian Bach' ,
        'birth_year' : 1685 ,
        'death_year' : 1750 }

print(JSB)

# %% [markdown]
# ### Querying the dictionary: square brackets

# %%
JSB['name']

# %% [markdown]
# ### Get the set of keys

# %%
type(set(JSB.keys()))

# %% [markdown]
# ### Change a value

# %%
JSB['death_year'] = 2030
print(JSB)

# %% [markdown]
# ### Add a new key-value pair

# %%


import rclpy
from rclpy.node import Node
from sensor_msgs.msg import Imu
import random

class IMUSubscriber(Node):

    def __init__(self):
        super().__init__('imuSubscriber')
        self.subscription = self.create_subscription(
            Imu,
            'imu',
            self.listener_callback,
            10)
        self.subscription  # prevent unused variable warning

    def listener_callback(self, msg):
        self.get_logger().info('nanosec: %d' %(msg.header.stamp.nanosec))

def main(args=None):
    rclpy.init(args=args)

    imuSubscriber = IMUSubscriber()

    rclpy.spin(imuSubscriber)

    # Destroy the node explicitly
    # (optional - otherwise it will be done automatically
    # when the garbage collector destroys the node object)
    imuSubscriber.destroy_node()
    rclpy.shutdown()

if __name__ == '__main__':
    main()


# ======================================================================
# Category Six
#   Advent of Code 2019 Day 23 -- Eric Wastl -- https://adventofcode.com
#
# Computer simulation by Dr. Dean Earl Wright III
# ======================================================================

# ======================================================================
#                           n e t w o r k . p y
# ======================================================================
"Network for the Category Six problem for AoC 2019 Day 23"

# ----------------------------------------------------------------------
#                                                                 import
# ----------------------------------------------------------------------
import intcode

# ----------------------------------------------------------------------
#                                                              constants
# ----------------------------------------------------------------------
NUMBER_OF_COMPUTERS = 50
MAX_CHANNELS = 256
NO_INPUT = [-1]

PACKET_LENGTH = 3
PACKET_DEST = 0
PACKET_X = 1
PACKET_Y = 2

NAT_RECV = 255
NAT_SEND = 0

# ======================================================================
#                                                                Network
# ======================================================================

class Network():
    """Object representing a network of intcode computer"""

    def __init__(self, text=None, number=NUMBER_OF_COMPUTERS):

        # 1. Set the initial values
        self.nic_software = text
        self.number = number
        self.first_message_to = [None for _ in range(MAX_CHANNELS)]
        self.first_message_from = [None for _ in range(MAX_CHANNELS)]
        self.last_message_to = [None for _ in range(MAX_CHANNELS)]
        self.last_message_from = [None for _ in range(MAX_CHANNELS)]
        self.cur_output = [[] for _ in range(MAX_CHANNELS)]
        self.input_queue = [[] for _ in range(MAX_CHANNELS)]
        self.computers = [intcode.IntCode(text=text) for _ in range(number)]
        self.last_nat_msg = None
        self.prev_nat_msg = None
        self.nat_msg = None

    def boot_up(self, watch=False):
        "Start up all the network computers"

        # 1. Assume that the computer will want more input
        halted = intcode.STOP_INP

        # 2. Loop for all of the computers
        for cnum, comp in enumerate(self.computers):

            # 3. Start up each of the computers
            halted = comp.run(inp=[cnum])
            if watch:
                print("Started up number %d, status = %d" % (cnum, halted))
            if halted != intcode.STOP_INP:
                break

        # 3. Return boot up status
        return halted

    def run_until_msg_to(self, channel, watch=False, maxtime=0):
        "Run the network computer until one of them sends a message to 255"

        # 1. Assume that the computer will want more input
        halted = intcode.STOP_INP
        steps = 0

        # 1. Loop until someone sends a message to the specified channal
        while self.first_message_to[channel] is None and halted == intcode.STOP_INP:

            # 2. Run all the network computers
            halted = self.run_all(watch=watch)

            # 3. See if we have spent too much time
            steps += 1
            if steps > maxtime > 0:
                halted = intcode.STOP_STP

        # 4. Return the reason we stopped
        return halted

    def run_until_nat_repeat(self, watch=False, maxtime=0):
        "Run the network until NAT sends to computer 0 the same message twice in a row"

        # 1. Assume that the computer will want more input
        halted = intcode.STOP_INP
        steps = 0

        # 1. Loop until nat sends the same message twice
        while halted == intcode.STOP_INP and (self.last_nat_msg == None or
                                              (self.last_nat_msg != self.prev_nat_msg)):

            # 2. Run all the network computers
            halted = self.run_all(watch=watch, nat=True)

            # 3. See if we have spent too much time
            steps += 1
            if steps > maxtime > 0:
                halted = intcode.STOP_STP

        # 4. Return the reason we stopped
        return halted

    def run_all(self, watch=False, nat=False):
        "Run each computer until it requests input"

        # 1. Assume we will want input
        halted = intcode.STOP_INP
        inp_knt = 0
        out_knt = 0

        # 2.  Loop for every computer in the network
        for cnum, comp in enumerate(self.computers):

            # 3. Is there input for this computer?
            if self.input_queue[cnum]:

                # 3a. Yes, take it from the queue and give it to the computer
                comp.add_inp(self.input_queue[cnum])
                self.input_queue[cnum] = []
                inp_knt += 1

            # 4. If the computer has no input waiting, give something
            else:
                if not comp.any_inp():
                    comp.add_inp(NO_INPUT)
                else:
                    inp_knt += 1

            # 5. Run the computer until it stops:
            halted = comp.run()
            if halted != intcode.STOP_INP:
                if watch:
                    print("Ran number %d, status = %d" % (cnum, halted))
                break

            # 6. If the computer has any output, add it to the output collected already
            outputs = comp.outputs()
            if outputs:
                out_knt += 1
                for out in outputs:
                    self.cur_output[cnum].append(out)

            # 7. Process any collected messages in the output of this computer
            snd_knt = self.process_messages(cnum, watch=watch)

        # 8. See if the NAT has something to do
        if nat:
            self.process_nat(inp_knt, out_knt, snd_knt, watch=watch)

        # 8. Return why we stopped (STOP_INP if we completed a loop without problems)
        return halted

    def process_messages(self, cnum, watch=False):
        "Process messages from the specified computer"

        # 1. Assume not messages sent
        result = 0

        # 2. Loop while we have enough output for a message
        while len(self.cur_output[cnum]) >= PACKET_LENGTH:

            # 3. Take off the first packet from the output
            packet = self.cur_output[cnum][0:PACKET_LENGTH]
            del self.cur_output[cnum][:PACKET_LENGTH]
            if watch:
                print("Packet from %d to %d: X=%d, Y=%d" %
                      (cnum, packet[0], packet[1], packet[2]))

            # 4. Add to first and last messages
            msg_from = cnum
            msg_to = packet[PACKET_DEST]
            if self.first_message_from[msg_from] is None:
                if watch:
                    print("First packet from %d is to %d: X=%d, Y=%d" %
                          (msg_from, msg_to, packet[1], packet[2]))
                self.first_message_from[msg_from] = packet
            if self.first_message_to[msg_to] is None:
                if watch:
                    print("First packet to %d is from %d: X=%d, Y=%d" %
                          (msg_to, msg_from, packet[1], packet[2]))
                self.first_message_to[msg_to] = packet
            self.last_message_from[msg_from] = packet
            self.last_message_to[msg_to] = packet

            # 5. Put the message on the queue for the appropiate computer
            self.input_queue[msg_to].append(packet[PACKET_X])
            self.input_queue[msg_to].append(packet[PACKET_Y])

            # 6. Keep track of number sent
            result += 1

        # 7. Return number of messages sent
        return result

    def get_first_msg_to(self, channel):
        "Returns the first message to the channel"
        return self.first_message_to[channel]

    def get_last_msg_to(self, channel):
        "Returns the last message to the channel"
        return self.last_message_to[channel]

    def process_nat(self, inp_knt, out_knt, snd_knt, watch=False):
        "Handle nat transmission"

        # 1. Output counts (if requested)
        if watch:
            print("inp=%d, out=%d, snd=%d, to_nat=%s, prev=%s, last=%s" %
                  (inp_knt, out_knt, snd_knt,
                   self.get_last_msg_to(NAT_RECV),
                   self.prev_nat_msg,
                   self.last_nat_msg))

        # 2. If traffic, no need for nat to send
        if inp_knt + snd_knt > 0:
            return

        # 3. If nothing to send, nothing to to
        last_nat = self.get_last_msg_to(NAT_RECV)
        if last_nat is None:
            return

        # 4. Have nat send a message
        self.input_queue[NAT_SEND].append(last_nat[PACKET_X])
        self.input_queue[NAT_SEND].append(last_nat[PACKET_Y])

        # 5. Record it
        self.prev_nat_msg = self.last_nat_msg
        self.last_nat_msg = [last_nat[PACKET_X], last_nat[PACKET_Y]]
        if watch:
            print("Sent NAT msg X=%d Y=%d from %d to %s" %
                  (last_nat[PACKET_X], last_nat[PACKET_Y], NAT_RECV, NAT_SEND))

    def get_last_nat_msg(self):
        "Returns the last message that the nat sent"
        return self.last_nat_msg

# ----------------------------------------------------------------------
#                                                  module initialization
# ----------------------------------------------------------------------
if __name__ == '__main__':
    pass

# ======================================================================
# end                           a i i . p y                          end
# ======================================================================


import xml.etree.ElementTree as ET

CK_file = ("C://Users//Ved Prakash Shukla//Downloads//Python_Scripts//new_ckk31.xml")
HH_file = ("C://Users//Ved Prakash Shukla//Downloads//Python_Scripts//new_hhh31.xml")
Allheart_file = ("C://Users//Ved Prakash Shukla//Downloads//Python_Scripts//allheartt31.xml")

def update_allheart(CK_file, HH_file, Allheart_file):
    # parse xml files
    CK_tree = ET.parse(CK_file)
    HH_tree = ET.parse(HH_file)
    Allheart_tree = ET.parse(Allheart_file)

    # create a set to store the login ids
    login_ids = set()

    # get the root element for each xml file
    CK_root = CK_tree.getroot()
    HH_root = HH_tree.getroot()
    Allheart_root = Allheart_tree.getroot()

    # create a namespace dictionary for easy access to elements
    ns = {'ns': 'http://www.demandware.com/xml/impex/customer/2006-10-31'}

    # loop through all customers in the CK xml file
    for customer in CK_root.findall("./ns:customer", ns):
        # get the login id
        login = customer.find("./ns:credentials/ns:login", ns).text

        # if the login id is not already in the set of login ids
        if login not in login_ids:
            # add the customer to the Allheart xml file
            Allheart_root.append(customer)
            # add the login id to the set of login ids
            login_ids.add(login)

    # loop through all customers in the HH xml file
    for customer in HH_root.findall("./ns:customer", ns):
        # get the login id
        login = customer.find("./ns:credentials/ns:login", ns).text

        # if the login id is not already in the set of login ids
        if login not in login_ids:
            # add the customer to the Allheart xml file
            Allheart_root.append(customer)
            # add the login id to the set of login ids
            login_ids.add(login)

    # write the updated Allheart xml file
    Allheart_tree.write("Allheartt31.xml")

# example usage
update_allheart("new_ckk31.xml", "new_hhh31.xml", "Allheartt31.xml")


import sys
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QWidget

from PyQt5.QtCore import pyqtSlot, pyqtSignal
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *

from Features import Ui_Form

class  Features_window(QWidget):
    signal = pyqtSignal(list)

    def __init__(self, parent=None):
        super(Features_window,self).__init__(parent)
        self.ui = Ui_Form()
        self.ui.setupUi(self)
        self.setWindowFlags(QtCore.Qt.Window | QtCore.Qt.FramelessWindowHint)
        self.check_list = []
        self.checkBox_list = [self.ui.checkBox_11, self.ui.checkBox_13, self.ui.checkBox_14, self.ui.checkBox_15,
                              self.ui.checkBox_16,
                              self.ui.checkBox_17, self.ui.checkBox_18, self.ui.checkBox_19, self.ui.checkBox_20,
                              self.ui.checkBox_21, self.ui.checkBox_5, self.ui.checkBox_6, self.ui.checkBox_7,
                              self.ui.checkBox_8, self.ui.checkBox_9,
                              self.ui.checkBox_11,
                              self.ui.checkBox_22]
        self.ui.checkBox_12.clicked.connect(lambda: self.set_all())

        # Set Main background to transparent
        self.setAttribute(QtCore.Qt.WA_TranslucentBackground)  # ХЗ, что это

        self.ui.pushButton.clicked.connect(lambda: self.clk())
        self.ui.pushButton_2.clicked.connect(lambda: self.close())

        def MoveWindow(e):
            if self.isMaximized() == False:
                if e.buttons() == Qt.LeftButton:
                    self.move(self.pos() + e.globalPos() - self.clickPosition)
                    self.clickPosition = e.globalPos()
                    e.accept()

        self.ui.frame_2.mouseMoveEvent = MoveWindow

    def mousePressEvent(self, event):
        self.clickPosition = event.globalPos()

    def set_all(self):

        if self.ui.checkBox_12.isChecked():
            for i in self.checkBox_list:
                i.setChecked(True)
        else:
            for i in self.checkBox_list:
                i.setChecked(False)

    @pyqtSlot()
    def clk(self):
        for i in self.checkBox_list:
            if i.isChecked():
                self.check_list.append(i.text())
        print(self.check_list)
        self.signal.emit(self.check_list)
        self.close()

if __name__ == "__main__":
    app = QApplication(sys.argv)
    window = Features_window(0)
    window.show()
    sys.exit(app.exec_())


import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, roc_auc_score
from retention_utils import get_data, get_ourvoice, get_2019_data, get_current_data
import pickle

def evaluate(model, test_feat, test_lab):
    print('L=', test_lab)
    predictions = model.predict(test_feat)

    errors = abs(predictions - test_lab)

    mape = 100 * np.mean(errors / test_lab)
    accuracy = 100 - mape
    print('Model Performance')
    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
    print('Accuracy = {:0.2f}%.'.format(accuracy))

    return accuracy

region = 'OURVOICE'
train_features, \
    train_labels, \
    test_features, \
    test_labels, \
    new_test_features, \
    new_test_labels, \
    features,\
    new_test_ids = get_data(region)

# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start=1000, stop=2000, num=2)]
# Number of features to consider at every split
max_features = ['auto', 'sqrt']
# Maximum number of levels in tree
max_depth = [int(x) for x in np.linspace(10, 100, num=3)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]
# Method of selecting samples for training each tree
bootstrap = [True, False]

# Create the random grid
random_grid = {'n_estimators': [2000],  # n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               # 'random_state': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
               'bootstrap': bootstrap}
print(random_grid)

# best_values = {'n_estimators': [1000], 'min_samples_split': [2], 'min_samples_leaf': [1],
# 'max_features': ['auto'], 'max_depth': [50], 'bootstrap': [False]}

# Use the random grid to search for best hyper parameters
# First create the base model to tune
rf = RandomForestClassifier()
# Random search of parameters, using 3 fold cross validation,
# search across 100 different combinations, and use all available cores
rf_random = RandomizedSearchCV(estimator=rf, param_distributions=random_grid, cv=3, verbose=2,
                               n_jobs=-1, n_iter=10,
                               scoring='precision_weighted'
                               # scoring='f1_weighted'
                               )
# Fit the random search model
rf_random.fit(train_features, train_labels)

print(rf_random.best_params_)

base_model = RandomForestClassifier(n_estimators=10, random_state=42)
base_model.fit(train_features, train_labels)

base_accuracy = evaluate(base_model, test_features, test_labels)
print(base_accuracy)
best_random = rf_random.best_estimator_
# best_random = base_model
random_accuracy = evaluate(best_random, test_features, test_labels)
print(random_accuracy)

print('Improvement of {:0.2f}%.'.format(100 * (random_accuracy - base_accuracy) / base_accuracy))

importances = best_random.feature_importances_
std = np.std([tree.feature_importances_ for tree in best_random.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")
for f in range(train_features.shape[1]):
    print("%d. feature %s (%f)" % (f + 1, features[indices[f]], importances[indices[f]]))

y_true, y_pred = new_test_labels, best_random.predict_proba(new_test_features)
print(len(y_true), len(y_pred), len(new_test_ids))
pickle_name = 'parrot_tree_ourvoice_test.pkl'
with open(pickle_name, 'wb') as f:
    pickle.dump([new_test_ids, y_pred, y_true], f)
# y_true2, y_pred2 = y_resigned_new3, best_random.predict_proba(X_resigned_new3)
# print(y_pred2)
# print(list(y_true2))

# y_true = [1 for yy in y_resigned_new2]
# y_true2 = [-1 for yy in y_resigned_new3]

active = [x[1] for x, y in zip(y_pred, y_true) if y == -1]
resigned = [x[1] for x, y in zip(y_pred, y_true)if y == 1]
# active = [x[1] for x, y in zip(y_pred2, y_true2)]
# resigned = [x[1] for x, y in zip(y_pred, y_true)]

a1 = [x for x in active if x > 0.5]
a2 = [x for x in active if x < 0.5]
r1 = [x for x in resigned if x > 0.5]
r2 = [x for x in resigned if x < 0.5]

print('False positive=', len(a1))
print('True negative=', len(a2))
print('True positive=', len(r1))
print('False negative=', len(r2))

# y_true2 = [-1 for yy in y_resigned_new3]
print(accuracy_score(y_true, best_random.predict(new_test_features)))
# print(accuracy_score(y_true2, best_random.predict(X_resigned_new3)))

# print(list(base_model.predict(X_resigned_new3)))
# print(sum(base_model.predict(X_resigned_new3)))

# print(roc_auc_score(np.concatenate((y_true, y_true2), axis=0), np.concatenate((y_pred[:, 1], y_pred2[:, 1]),
#                                                                              axis=0)))

print(roc_auc_score(y_true, (y_pred[:, 1])))

# print(confusion_matrix(np.concatenate((y_true, y_true2), axis=0),
#                       np.concatenate((best_random.predict(X_resigned_new2), best_random.predict(X_resigned_new3)),
#                                      axis=0)))

print(confusion_matrix(y_true, best_random.predict(new_test_features)))

fpr, tpr, _ = roc_curve(y_true, (y_pred[:, 1]))

plt.clf()
plt.plot(fpr, tpr)
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.title('ROC curve')
# plt.show()

# sys.exit()

fig, ax = plt.subplots()
ax.hist(active, 40, density=1, label='Active', alpha=0.5, color="blue")
ax.hist(active, bins=40, density=True, histtype='step', cumulative=1,
        label='')
# ax.hist(resigned, 40, density=1, label='Resigned', alpha=0.5, color="red")
plt.xlim(0.0, 1.0)
ax.legend(loc='best')
plt.xlabel("Resignation Probability")

fig, ax = plt.subplots()
# ax.hist(active, 40, density=1, label='Active', alpha=0.5, color="blue")
ax.hist(resigned, 40, density=1, label='Resigned', alpha=0.5, color="red")
ax.hist(resigned, bins=40, density=True, histtype='step', cumulative=-1,
        label='')
plt.xlim(0.0, 1.0)
ax.legend(loc='best')
plt.xlabel("Resignation Probability")

y_true, y_pred = new_test_labels, best_random.predict(new_test_features)
print('Results on the test set:')
print(classification_report(y_true, y_pred, target_names=['Active', 'Resigned']))

y_true, y_pred = new_test_labels, best_random.predict_proba(new_test_features)
# print('Results on the test set:')
# print(classification_report(y_true, y_pred, target_names=['Active', 'Resigned']))

active = [x[1] for x, y in zip(y_pred, y_true) if y == -1]
resigned = [x[1] for x, y in zip(y_pred, y_true) if y == 1]
xs = [i/10 for i in range(1, 10)]
ps = []
rs = []
fs = []
for i in xs:
    a1 = [x for x in active if x > i]
    a2 = [x for x in active if x < i]
    r1 = [x for x in resigned if x > i]
    r2 = [x for x in resigned if x < i]
    tp = len(r1)
    fp = len(a1)
    tn = len(a2)
    fn = len(r2)
    print('x=%f, tp=%d, fp=%d, tn=%d, fn=%d' % (i, tp, fp, tn, fn))
    if tp+fp > 0:
        precision = tp/(tp+fp)
    else:
        precision = -0.01
    if tp+fn > 0:
        recall = tp/(tp+fn)
    else:
        recall = -0.01
    ps.append(precision)
    rs.append(recall)
    if precision + recall > 0:
        fs.append(2*precision*recall/(precision + recall))
    else:
        fs.append(-0.01)

print(xs)
print(ps)
print(rs)
print(fs)
x = np.linspace(0, 10)
x = x/10
y = x/np.inf + 0.5
_ = plt.figure()
plt.plot(np.multiply(xs, 100), np.multiply(ps, 100), color='red', label='Precision')
plt.plot(np.multiply(xs, 100), np.multiply(rs, 100), color='blue', label='Recall')
plt.plot(np.multiply(xs, 100), np.multiply(fs, 100), color='green', label='F1 Score')
plt.plot(np.multiply(x, 100), y*100, color='black', linestyle=':')
plt.xlabel('Probability Threshold [%]')
plt.ylabel('Percentage [%]')
plt.legend()
plt.show()

if 'OURVOICE' in region:
    test_features_2017, test_features_2019, id_2017, id_2019 = get_ourvoice()
    predicted_labels_2017 = best_random.predict_proba(test_features_2017)
    pickle_name = 'parrot_tree_ourvoice_2017.pkl'
    with open(pickle_name, 'wb') as f:
        pickle.dump([id_2017, predicted_labels_2017], f)

    predicted_labels_2019 = best_random.predict_proba(test_features_2019)
    pickle_name = 'parrot_tree_ourvoice_2019.pkl'
    with open(pickle_name, 'wb') as f:
        pickle.dump([id_2019, predicted_labels_2019], f)

if 'CHINA' not in region and 'SEA' not in region:
    plt.show()

new_test_features, new_test_wwids, new_test_labels2 = get_2019_data(region)

predicted_labels = best_random.predict_proba(new_test_features)

y_true, y_pred = new_test_labels2, [x[0] for x in predicted_labels]

active = [x for x, y in zip(y_pred, y_true) if y == 0]
resigned = [x for x, y in zip(y_pred, y_true) if y == 1]

# fpr, tpr, _ = roc_curve(y_true, y_pred)
# plt.clf()
# plt.plot(fpr, tpr)
# plt.xlabel('FPR')
# plt.ylabel('TPR')
# plt.title('ROC curve')
# plt.show()

# sys.exit()

fig, ax = plt.subplots()
ax.hist(active, 40, density=1, label='Active', alpha=0.5, color="blue")
ax.hist(active, bins=40, density=True, histtype='step', cumulative=1,
        label='')
# ax.hist(resigned, 40, density=1, label='Resigned', alpha=0.5, color="red")
plt.xlim(0.0, 1.0)
ax.legend(loc='best')
plt.xlabel("Resignation Probability")

fig, ax = plt.subplots()
# ax.hist(active, 40, density=1, label='Active', alpha=0.5, color="blue")
ax.hist(resigned, 40, density=1, label='Resigned', alpha=0.5, color="red")
ax.hist(resigned, bins=40, density=True, histtype='step', cumulative=-1,
        label='')
plt.xlim(0.0, 1.0)
ax.legend(loc='best')
plt.xlabel("Resignation Probability")

# cm = confusion_matrix(new_test_labels2, np.round(predicted_labels))
cm = confusion_matrix(new_test_labels2, [[1] if x[0] > 0.2 else [0] for x in predicted_labels])

plt.matshow(cm, alpha=0)
plt.title('Confusion matrix')
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

for (i, j), z in np.ndenumerate(cm):
    plt.text(j, i, str(z), ha='center', va='center')

xs = [i/10 for i in range(1, 10)]
ps = []
rs = []
fs = []
for i in xs:
    a1 = [x for x in active if x > i]
    a2 = [x for x in active if x < i]
    r1 = [x for x in resigned if x > i]
    r2 = [x for x in resigned if x < i]
    tp = len(r1)
    fp = len(a1)
    tn = len(a2)
    fn = len(r2)
    precision = tp / (tp + fp) if (tp + fp) else 0
    recall = tp / (tp + fn) if (tp + fn) else 0
    ps.append(precision)
    rs.append(recall)
    if precision + recall > 0:
        fs.append(2 * precision * recall / (precision + recall))
    else:
        fs.append(0)

print(xs)
print(ps)
print(rs)
print(fs)
x = np.linspace(0, 10)
x = x/10
y = x/np.inf + 0.5
_ = plt.figure()
plt.plot(np.multiply(xs, 100), np.multiply(ps, 100), color='red', label='Precision')
plt.plot(np.multiply(xs, 100), np.multiply(rs, 100), color='blue', label='Recall')
plt.plot(np.multiply(xs, 100), np.multiply(fs, 100), color='green', label='F1 Score')
plt.plot(np.multiply(x, 100), y*100, color='black', linestyle=':')
plt.xlabel('Probability Threshold [%]')
plt.ylabel('Percentage [%]')
plt.legend()

plt.show()

new_test_features, new_test_wwids = get_current_data(region)


import csv

def challenge1():
    with open('./data/2005_data.csv', 'r') as f:
        data = f.read()
    print(data)

def challenge2():
    with open('./data/answer.txt', 'w', encoding='utf-8') as f:
        qa = input('好きな食べ物は？')
        f.write(qa)

def challenge3():
    data = [
        ['Top Gun', 'Risky Business', 'Minority Report'],
        ['Titanic', 'The Revenant', 'Inception'],
        ['Training Day', 'Man on Fire', 'Flight']
    ]
    with open('./data/movies.csv', 'w') as f:
        writer = csv.writer(f, delimiter=',')
        for row in data:
            writer.writerow(row)

def challenge4():
    data = [
        ['トップガン', '卒業白書', 'マイノリティ・レポート'],
        ['タイタニック', 'レヴィナント', 'インセプション'],
        ['トレーニングデイ', 'マイ・ボディガード', 'フライト']
    ]
    with open('./data/movies_ja.csv', 'w', encoding='utf-8') as f:
        writer = csv.writer(f, delimiter=',')
        for row in data:
            writer.writerow(row)


# -*- coding: utf-8 -*-
"""The business logic"""
import json
import logging
from django.conf import settings
from django.contrib.auth.tokens import default_token_generator
from django.core.cache import cache
from django.core.mail import send_mail
from django.http import JsonResponse
from django.shortcuts import get_object_or_404
from django.template.loader import render_to_string
from http import HTTPStatus
from numbers import Number
from datetime import timedelta, datetime

from .models import (
    FAR_FUTURE,
    CalculatedHoliday,
    Company,
    Employee,
    Person,
    Ward,
    DifferentDay,
    Planning,
    ChangeLogging,
    Department,
    process_change,
)
from .utils import get_first_of_month, time_since

def get_plan_data(
    company_id,
    department_ids,
    month="",
    day="",
    is_editor=False,
    is_dep_lead=False,
    is_company_admin=False,
):
    """Produce data for views.plan

    month is '' or 'YYYYMM'
    day is '' or 'YYYYMMDD' or None (for path "/tag")
    """
    if month == "" and day:
        month = day[:6]
    # Get all Persons who work here currently
    first_of_month = get_first_of_month(month)
    # start_of_data should be one month earlier
    start_of_data = (first_of_month - timedelta(28)).replace(day=1)
    persons_qs = (
        Person.objects.filter(company_id=company_id)
        .order_by("position", "name")
        .prefetch_related("functions", "departments")
    )
    wards = Ward.objects.filter(
        departments__id__in=department_ids
    ).prefetch_related("after_this", "not_with_this")
    if len(persons_qs) == 0 or len(wards) == 0:
        return None
    different_days = DifferentDay.objects.filter(
        ward__departments__id__in=department_ids, day__gte=start_of_data
    ).select_related("ward")
    plannings = Planning.objects.filter(
        ward__in=wards,
        ward__active=True,
        end__gte=start_of_data,
        superseded_by=None,
    ).select_related("ward")

    if not is_editor:
        plannings = [
            p
            for p in plannings
            if not p.ward.approved or p.start <= p.ward.approved
        ]
    holidays = get_holidays_for_company(company_id)
    departments = dict(
        (d.id, d.name)
        for d in Department.objects.filter(id__in=department_ids)
    )
    data = {
        "persons": [
            p.toJson() for p in persons_qs if p.end_date >= start_of_data
        ],
        "wards": [w.toJson() for w in wards if w.active],
        "different_days": [
            (
                dd.ward.shortname,
                dd.day.strftime("%Y%m%d"),
                "+" if dd.added else "-",
            )
            for dd in different_days
        ],
        "plannings": [p.toJson() for p in plannings],
        "is_editor": is_editor,
        "is_dep_lead": is_dep_lead,
        "is_company_admin": is_company_admin,
        "data_year": start_of_data.year,
        "data_month": start_of_data.month - 1,
        "holidays": [h.toJson() for h in holidays],
        "departments": departments,
    }
    last_change = (
        ChangeLogging.objects.filter(
            company_id=company_id,
        )
        .values("pk", "change_time")
        .order_by("pk")
        .last()
    )
    if last_change is not None:
        data["last_change_pk"] = last_change["pk"]
        data["last_change_time"] = int(
            time_since(last_change["change_time"]).total_seconds()
        )

    return {
        "data": json.dumps(data),
        "former_persons": [
            p for p in persons_qs if p.end_date < start_of_data
        ],
        "inactive_wards": [w for w in wards if not w.active],
    }

def get_for_company(klass, request=None, company_id="", **kwargs):
    """Return a model for this company

    Raises Http404 if not found
    """
    if request is None:
        return get_object_or_404(klass, company__id=company_id, **kwargs)
    return get_object_or_404(
        klass, company__id=request.session["company_id"], **kwargs
    )

def get_holidays_for_company(company_id):
    return CalculatedHoliday.objects.filter(regions__companies__id=company_id)

def apply_changes(user, company_id, day, ward_id, continued, persons):
    """Apply changes for this day and ward.
    Return a list of dicts of effective changes to be returned to the client
    'persons' is a list of dicts like
    {'id': <id>,
     'action': ‘add'|'remove'}
    """
    ward = get_for_company(Ward, company_id=company_id, id=ward_id)
    known_persons = {
        person.id: person
        for person in Person.objects.filter(
            company__id=company_id, id__in=(p["id"] for p in persons)
        )
    }
    data = dict(
        company_id=company_id,
        user=user,
        ward=ward,
        day=datetime.strptime(day, "%Y%m%d").date(),
        continued=continued,
        until=None,
    )
    if isinstance(continued, str):
        data["until"] = datetime.strptime(continued, "%Y%m%d").date()
        data["continued"] = True
    cls = []
    for p in persons:
        p_id = int(p["id"])
        # TODO: log error
        assert p_id in known_persons, f"{p_id} is not in the persons database"
        cl = ChangeLogging.objects.create(
            person=known_persons[p_id], added=p["action"] == "add", **data
        )
        cl_dict = process_change(cl)
        if cl_dict:
            cls.append(cl_dict)
    if len(cls):
        set_cached_last_change_pk(max(cl["pk"] for cl in cls), company_id)
    return cls

def set_approved(wards, approved, department_ids):
    """
    wards is [<ward.shortname>, ...],
    approved is False|<YYYYMMDD>, (False means unlimited approval)
    company_id is <company.id>

    Only the wards of the given departments are approved
    """
    to_approve = Ward.objects.filter(
        departments__id__in=department_ids, shortname__in=wards
    )
    approval = (
        datetime.strptime(approved, "%Y%m%d").date()
        if approved
        else FAR_FUTURE
    )
    to_approve.update(approved=approval)
    to_approve_sn = [w.shortname for w in to_approve]
    return {
        "wards": to_approve_sn,
        "approved": approved,
        "not approved wards": [
            ward for ward in wards if ward not in to_approve_sn
        ],
    }

def get_cached_last_change_pk(company_id):
    """Returns the pk of the last changelogging or None if not set"""
    return cache.get(f"last_change_pk-{company_id}")

def set_cached_last_change_pk(last_change_pk, company_id):
    """Sets the pk of the last changelogging"""
    return cache.set(f"last_change_pk-{company_id}", last_change_pk)

def get_last_change_response(company_id, last_change_pk):
    """Return a JsonResponse with the changes since last_change_pk
    and pk and elapsed time of the last change.
    """
    assert isinstance(last_change_pk, Number)
    _lc_pk = get_cached_last_change_pk(company_id)
    if _lc_pk and (_lc_pk == last_change_pk):
        # Nothing changed
        return JsonResponse({}, status=HTTPStatus.NOT_MODIFIED)

    # get all ChangeLoggings since and including last_change_pk
    cls = list(
        ChangeLogging.objects.filter(
            company_id=company_id, pk__gte=last_change_pk
        ).order_by("pk")
    )
    if len(cls) == 0:
        # Only older ChangeLoggings, so last_change_pk is wrong
        cls = [ChangeLogging.objects.order_by("pk").last()]
        if len(cls) == 0:
            # No ChangeLoggings
            logging.debug("No ChangeLoggings found")
            return JsonResponse({}, status=HTTPStatus.NOT_MODIFIED)
    if cls[0].pk == last_change_pk:
        if len(cls) == 1:
            # No newer ChangeLoggings
            return JsonResponse({}, status=HTTPStatus.NOT_MODIFIED)
        cls = cls[1:]
    last_cl = cls[-1]
    time_diff = time_since(last_cl.change_time)
    if _lc_pk is None:
        # Set cache
        set_cached_last_change_pk(last_cl.pk, company_id)
    return JsonResponse(
        {
            "cls": [
                json.loads(cl.json) if cl.json else cl.toJson() for cl in cls
            ],
            "last_change": {
                "pk": last_cl.pk,
                "time": time_diff.days * 86400 + time_diff.seconds,
            },
        }
    )

def send_activation_mail(user):
    "send activation mail in the signup process"
    # don't send mails out of playwright tests
    if user.email.startswith("_pwt_"):
        return
    mail_subject = "Benutzerkonto für Stationsplan.de aktivieren"
    message = render_to_string(
        "sp_app/signup/activation_mail.txt",
        {
            "username": user.get_full_name() or user.get_username(),
            "uid": user.pk,
            "token": default_token_generator.make_token(user),
        },
    )
    send_mail(mail_subject, message, settings.SERVER_EMAIL, [user.email])

# Tests ------------------------------------------------------


from _dbConnector import *
from _api import *
# from dateutil.parser import parse
import datetime
from _hosts import host_locations

matcher = {
    "Shell 00": "c-piscine-shell-00",
    "Shell 01": "c-piscine-shell-01",

    "C 00": "c-piscine-c-00",
    "C 01": "c-piscine-c-01",
    "C 02": "c-piscine-c-02",
    "C 03": "c-piscine-c-03",
    "C 04": "c-piscine-c-04",
    "C 05": "c-piscine-c-05",
    "C 06": "c-piscine-c-06",
    "C 07": "c-piscine-c-07",
    "C 08": "c-piscine-c-08",
    "C 09": "c-piscine-c-09",
    "C 10": "c-piscine-c-10",
    "C 11": "c-piscine-c-11",
    "C 12": "c-piscine-c-12",
    "C 13": "c-piscine-c-13",

    "Rush 00": "c-piscine-rush-00",
    "Rush 01": "c-piscine-rush-01",
    "Rush 02": "c-piscine-rush-02",
    "BSQ": "c-piscine-bsq",

    "Your very first own library": "42cursus-libft",
    "Get Next Line": "42cursus-get_next_line",
    "printf": "42cursus-ft_printf",
    "Born2beRoot": "born2beroot",

    "Push_swap": "42cursus-push_swap",
    "Minitalk": "minitalk",
    "Pipex": "pipex",
    "FDF": "42cursus-fdf",
    "fract’ol": "42cursus-fract-ol",
    "So Long": "so_long",

    "Philosophers": "42cursus-philosophers",
    "Minishell": "42cursus-minishell",

    "miniRT": "minirt",
    "cub3D": "cub3d",
    "Net_Practice": "netpractice",
    "C++ - Module 00": "cpp-module-00",
    "C++ - Module 01": "cpp-module-01",
    "C++ Pool - Module 01": "cpp-module-01",
    "C++ - Module 02": "cpp-module-02",
    "C++ - Module 03": "cpp-module-03",
    "C++ - Module 04": "cpp-module-04",
    "C++ - Module 05": "cpp-module-05",
    "C++ - Module 06": "cpp-module-06",
    "C++ - Module 07": "cpp-module-07",
    "C++ - Module 08": "cpp-module-08",
    "C++ - Module 09": "cpp-module-09",
    "Inception\nS": "inception",
    "Webserv": "webserv",
    "IRC": "ft_irc",
    "ft_containers": "ft_containers",
    "ft_transcendence": "ft_transcendence",
}

def process_pdf(update_all = True):
    global local_projects

    from _utils_mylogger import mylogger, LOGGER_DEBUG, LOGGER_INFO, LOGGER_WARNING, LOGGER_ERROR, LOGGER_ALERT

    mylogger("Start pdf processor", LOGGER_ALERT)

    subject_headers = executeQuerySelect("SELECT id, title FROM subject_hashmaps")

    for subject_header in subject_headers:

        # mylogger(f"Look {subject_header['id']}", LOGGER_DEBUG)
        flag = False

        for search in matcher.keys():

            if ("Inception" in subject_header['title']):
                mylogger(f"Inc ___{subject_header['title']}___ {search}", LOGGER_DEBUG)

            if search in subject_header['title'] and local_projects.get(matcher[search]) != None:

                mylogger(f"Match {subject_header['id']} {search} {local_projects.get(matcher[search])}", LOGGER_INFO)
                flag = True

                executeQueryAction("""UPDATE subject_hashmaps
                                SET project_id = %(project_id)s
                                WHERE id = %(id)s""", {
                    "project_id": local_projects.get(matcher[search]),
                    "id": subject_header['id'],
                })

                break

            if (flag == False):
                executeQueryAction("""UPDATE subject_hashmaps
                                SET project_id = NULL
                                WHERE id = %(id)s""", {
                    "project_id": local_projects.get(matcher[search]),
                    "id": subject_header['id'],
                })

    mylogger("End pdf processor", LOGGER_ALERT)

if __name__ == "__main__":
    global local_projects

    local_projects = executeQuerySelect("SELECT id, slug FROM projects")
    local_projects = {one['slug']: one['id'] for one in local_projects}

    process_pdf(True)


from airflow.operators.http_operator import SimpleHttpOperator

from plugins.hooks.mattermost_webhook_hook import MattermostWebhookHook

class MattermostOperator(SimpleHttpOperator):

    """
    Operator that allows sending messages to Mattermost. For more details on the parameters, please check
    `Incoming webhooks <https://developers.mattermost.com/integrate/webhooks/incoming/#parameters>_` documentation.
    :param mattermost_conn_id: :ref:`http connection<howto/connection:http>` that has the base
        url i.e https://mattermost.example.com/ and webhook's id under password.
    :param text: The message to send.
    :param channel:	Overrides the channel the message posts in. Use the channel’s name and not the display name, e.g.
        use town-square, not Town Square.
    :param username: Overrides the username the message posts as.
    :param type: Sets the post type, mainly for use by plugins.
    :param props: Sets the post props, a JSON property bag for storing extra or meta data on the post.
    """

    template_fields = ('text', 'channel', 'username', 'type', 'props', 'attachments')

    def __init__(
        self,
        *,
        mattermost_conn_id,
        text=None,
        channel=None,
        username=None,
        icon_url=None,
        icon_emoji=None,
        type=None,
        props=None,
        attachments=None,
        **kwargs,
    ) -> None:
        super().__init__(endpoint=None, **kwargs)
        self.mattermost_conn_id = mattermost_conn_id
        self.text = text
        self.channel = channel
        self.username = username
        self.icon_url = icon_url
        self.icon_emoji = icon_emoji
        self.type = type
        self.props = props
        self.attachments = attachments

    def hook(self):
        return MattermostWebhookHook(
            mattermost_conn_id=self.mattermost_conn_id,
            text=self.text,
            channel=self.channel,
            username=self.username,
            icon_url=self.icon_url,
            icon_emoji=self.icon_emoji,
            type=self.type,
            props=self.props,
            attachments=self.attachments,
        )

    def execute(self, context):
        self.hook().execute()


#!/usr/bin/env python2.7
import mysql.connector
from mysql.connector import Error
import sys
import os, shutil

ff = sys.argv[1]

def write_file(data, filename):
    with open(filename, 'wb') as file:
        a = file.write(data)
    return a

try:
    connection = mysql.connector.connect(host='localhost',
                                         database='SENTINELX',
                                         user='root',
                                         password='',
                                         buffered=True,
                                         )

    cursor1 = connection.cursor()
    cursor1.execute("SELECT f_key FROM dl_event_tab WHERE seq_no=%s" %ff)
    value =cursor1.fetchone()
    name = value[0]
    names = name.split('_')
    c = names[0]
    ti = names[1]

    cursor = connection.cursor()
    sql_select_query = "SELECT * FROM image_tab where unixtime>=%s and unixtime<=%s and channel_id=%s"
    ta = int(ti)-60
    tb = int(ti)+60
    d = int(ta)
    e = int(tb)
    t = (d, e, c, )
    cursor.execute(sql_select_query, t)
    record = cursor.fetchall()

    count = 0
    folder = "/home/asus/Desktop/POKA/a/"
    for filename in os.listdir(folder):
        file_path = os.path.join(folder, filename)
        try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)
        except Exception as e:
            print('Failed to delete %s. Reason: %s' % (file_path, e))

    for r in record:
        Seqno = r[0]
        channel_id = r[1]
        Image = r[2]
        unixtime = r[3]

        photo = "/home/asus/Desktop/POKA/a/" + str(count) + ".jpg"
        write_file(Image,photo)
        count += 1

except Error as e:
    print("Error while connecting to MySQL", e)
finally:
    connection.close()

os.system("python3 /home/asus/Desktop/POKA/video.py")


# basic imports
import asyncio as aio
import logging

# Since this is now a python module please copy this example file to the
# AIOSerial parent directory before executing (cp Example.py ../Example.py)

# all the asyncio-serial port related stuff
from AIOSerial.AIOSerial import (AIOSerial, AIOSerialClosedException,
    AIOSerialErrorException, AIOSerialNotOpenException)

# AIOSerial now logs!
logging.basicConfig(level=logging.DEBUG)

# port usage example
async def example():
    # try to open the serial port and do some basic communication
    try:
        # open the port with following settings, use the line mode (return full
        #  text lines from the read method)
        async with AIOSerial('COM3', baudrate=115200, line_mode=True) as aios:
            # this is the way of accessing the serial port internals directly if
            # you are in need of altering the baudrate, etc..
            aios.sp.baudrate = 230400

            # send some data, here we use the AT protocol as it is one of the
            # most popular ways for the communication over the serial port
            await aios.write(b"AT\r\n")

            # uncomment this to get the AIOSerialClosedException
            #
            # await aios.close()

            # if you use virtual com port (like in case of usb-usart dongles)
            # you can test the AIOSerialErrorException by disconnecting the
            # dongle during following sleep period. You will still get all the
            # data that was received before the disconnection, but you won't be
            # able to send any (obviously)
            #
            # await aio.sleep(10)

            # read may fail if the peer device does not understand the AT
            # command
            try:
                while True:
                    # read with timeout
                    rcvd = await aio.wait_for(aios.read(), timeout=1.0)
                    # print the data received
                    print(f"data received: {rcvd}")
            # read timeout
            except aio.TimeoutError:
                print("reception timed out ;-(")
    # unable to open port
    except AIOSerialNotOpenException:
        print("Unable to open the port!")
    # port fatal error
    except AIOSerialErrorException:
        print("Port error!")
    # port already closed
    except AIOSerialClosedException:
        print("Serial port is closed!")

# run the example
if __name__ == '__main__':
    aio.run(example())


import re
import shutil
from pathlib import Path

import pytest
import pytorch_lightning as pl
from omegaconf.errors import OmegaConfBaseException

from nemo.constants import NEMO_ENV_VARNAME_VERSION
from nemo.utils.exp_manager import (
    CheckpointMisconfigurationError,
    LoggerMisconfigurationError,
    NotFoundError,
    exp_manager,
)

@pytest.fixture
def cleanup_local_folder():
    # Asserts in fixture are not recommended, but I'd rather stop users from deleting expensive training runs
    assert not Path("./lightning_logs").exists()
    assert not Path("./NeMo_experiments").exists()
    assert not Path("./nemo_experiments").exists()

    yield

    if Path("./lightning_logs").exists():
        shutil.rmtree('./lightning_logs')
    if Path("./NeMo_experiments").exists():
        shutil.rmtree('./NeMo_experiments')
    if Path("./nemo_experiments").exists():
        shutil.rmtree('./nemo_experiments')

class TestExpManager:
    @pytest.mark.unit
    def test_omegaconf(self):
        """Ensure omegaconf raises an error when an unexcepted argument is passed"""
        with pytest.raises(OmegaConfBaseException):
            exp_manager(None, {"unused": 1})

    @pytest.mark.unit
    def test_trainer_loggers(self, cleanup_local_folder, tmp_path):
        """ Test that a trainer with logger errors out with a number of arguments. Test that it works with
        create_tensorboard_logger set to False
        """
        test_trainer = pl.Trainer()  # Should create logger and modelcheckpoint

        with pytest.raises(LoggerMisconfigurationError):  # Fails because exp_manager defaults to trainer
            exp_manager(test_trainer, {"exp_dir": str(tmp_path)})
        with pytest.raises(LoggerMisconfigurationError):  # Fails because exp_manager defaults to trainer
            exp_manager(test_trainer, {"explicit_log_dir": str(tmp_path)})
        with pytest.raises(LoggerMisconfigurationError):  # Fails because exp_manager defaults to trainer
            exp_manager(test_trainer, {"resume_if_exists": True})

        # Check that exp_manager uses trainer.logger, it's exp_dir, name, and version
        log_dir = exp_manager(test_trainer, {"create_tensorboard_logger": False, "create_checkpoint_callback": False})
        assert log_dir.resolve() == Path("./lightning_logs/version_0").resolve()
        assert Path("./lightning_logs").exists()
        assert Path("./lightning_logs/version_0").exists()

        # Check that a trainer without a logger gets a logger attached to it
        test_trainer = pl.Trainer(logger=False)
        log_dir = exp_manager(
            test_trainer,
            {"create_tensorboard_logger": True, "create_checkpoint_callback": False, "exp_dir": str(tmp_path)},
        )
        assert isinstance(test_trainer.logger, pl.loggers.TensorBoardLogger)

        test_trainer = pl.Trainer(logger=False)
        # Check that a create_wandb_logger=True errors out unless wandb_logger_kwargs is passed.
        with pytest.raises(ValueError):
            log_dir = exp_manager(
                test_trainer,
                {
                    "create_tensorboard_logger": False,
                    "create_checkpoint_callback": False,
                    "exp_dir": str(tmp_path),
                    "create_wandb_logger": True,
                },
            )
        # Check that a WandbLogger is attached to logger if create_wandb_logger=True and wandb_logger_kwargs has name
        # and project
        log_dir = exp_manager(
            test_trainer,
            {
                "create_tensorboard_logger": False,
                "create_checkpoint_callback": False,
                "exp_dir": str(tmp_path),
                "create_wandb_logger": True,
                "wandb_logger_kwargs": {"name": "", "project": ""},
            },
        )
        assert isinstance(test_trainer.logger, pl.loggers.WandbLogger)

    @pytest.mark.unit
    def test_checkpoint_configurations(self, cleanup_local_folder):
        """ Test that trainer creating modelcheckpoint and asking exp_manager to do it too results in errors, but
        is error free if only one is asked to do so.
        """
        disable_tb_logger = {"create_tensorboard_logger": False}
        test_trainer = pl.Trainer()  # Should create logger and modelcheckpoint
        with pytest.raises(CheckpointMisconfigurationError):  # Fails because both try to create modelcheckpoint
            exp_manager(test_trainer, disable_tb_logger)

        # Should succeed without error
        exp_manager(test_trainer, {"create_checkpoint_callback": False, "create_tensorboard_logger": False})

        test_trainer_2 = pl.Trainer(checkpoint_callback=False)
        exp_manager(test_trainer_2, disable_tb_logger)  # Should succeed without error

    @pytest.mark.unit
    def test_default_log_dir(self, cleanup_local_folder):
        """Check the default of ./nemo_experiments/default/datetime works as intended"""
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)

        log_dir = exp_manager(test_trainer, {"create_tensorboard_logger": False, "create_checkpoint_callback": False})
        assert (log_dir / "..").resolve() == Path("./nemo_experiments/default/").resolve()
        assert Path("./nemo_experiments").exists()
        assert Path("./nemo_experiments/default/").exists()
        sub_dirs = [x for x in Path("./nemo_experiments/default/").iterdir() if x.is_dir()]
        assert len(sub_dirs) == 1
        assert re.match(r"[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}", sub_dirs[0].name)

    @pytest.mark.unit
    def test_log_dir_overrides(self, monkeypatch, tmp_path):
        """Check a variety of trainer options with exp_manager"""
        # Checks that explicit_log_dir ignores exp_dir, name, and version
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)
        log_dir = exp_manager(test_trainer, {"explicit_log_dir": str(tmp_path / "test_log_dir_overrides")})
        assert log_dir.resolve() == (tmp_path / "test_log_dir_overrides").resolve()
        assert Path(tmp_path).exists()
        assert Path(tmp_path / "test_log_dir_overrides").exists()

        # Checks that exp_manager uses exp_dir, default name, and explicit version
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)
        log_dir = exp_manager(test_trainer, {"exp_dir": str(tmp_path / "test_no_name"), "version": 957})
        assert log_dir.resolve() == (tmp_path / "test_no_name" / "default" / "957").resolve()
        assert Path(tmp_path).exists()
        assert Path(tmp_path / "test_no_name" / "default" / "957").exists()

        monkeypatch.delenv(NEMO_ENV_VARNAME_VERSION)
        # Checks that use_datetime_version False toggle works
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)
        log_dir = exp_manager(test_trainer, {"exp_dir": str(tmp_path / "test_no_name"), "use_datetime_version": False})
        assert log_dir.resolve() == (tmp_path / "test_no_name" / "default" / "version_0").resolve()
        assert Path(tmp_path).exists()
        assert Path(tmp_path / "test_no_name" / "default" / "version_0").exists()

        monkeypatch.delenv(NEMO_ENV_VARNAME_VERSION)
        # Checks that use_datetime_version False toggle works and version increments
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)
        log_dir = exp_manager(test_trainer, {"exp_dir": str(tmp_path / "test_no_name"), "use_datetime_version": False})
        assert log_dir.resolve() == (tmp_path / "test_no_name" / "default" / "version_1").resolve()
        assert Path(tmp_path).exists()
        assert Path(tmp_path / "test_no_name" / "default" / "version_1").exists()

    @pytest.mark.unit
    def test_resume(self, tmp_path):
        """ Tests the resume capabilities of exp_manager"""
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)

        # Error because explicit_log_dir does not exist
        with pytest.raises(NotFoundError):
            exp_manager(
                test_trainer,
                {
                    "exp_dir": str(tmp_path / "test_resume"),
                    "resume_if_exists": True,
                    "explicit_log_dir": "Does_not_exist",
                },
            )

        # Error because checkpoints folder does not exist
        with pytest.raises(NotFoundError):
            exp_manager(test_trainer, {"resume_if_exists": True, "exp_dir": str(tmp_path / "test_resume")})

        # No error because we tell exp_manager to ignore notfounderror
        exp_manager(
            test_trainer,
            {
                "resume_if_exists": True,
                "exp_dir": str(tmp_path / "test_resume_2"),
                "resume_ignore_no_checkpoint": True,
            },
        )

        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)
        Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints").mkdir(parents=True)
        # Error because checkpoints do not exist in folder
        with pytest.raises(NotFoundError):
            exp_manager(
                test_trainer,
                {
                    "resume_if_exists": True,
                    "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0"),
                },
            )

        Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--end.ckpt").touch()
        # Error because *end.ckpt is in folder indicating that training has already finished
        with pytest.raises(ValueError):
            exp_manager(
                test_trainer,
                {
                    "resume_if_exists": True,
                    "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0"),
                },
            )

        Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--end.ckpt").unlink()
        Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--last.ckpt").touch()
        Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel2--last.ckpt").touch()
        # Error because multiple *last.ckpt is in folder. If more than one, don't know which to restore
        with pytest.raises(ValueError):
            exp_manager(
                test_trainer,
                {
                    "resume_if_exists": True,
                    "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0"),
                },
            )

        # Finally succeed
        Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel2--last.ckpt").unlink()
        log_dir = exp_manager(
            test_trainer,
            {"resume_if_exists": True, "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0")},
        )
        checkpoint = Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--last.ckpt")
        assert Path(test_trainer.resume_from_checkpoint).resolve() == checkpoint.resolve()

        # Succeed again and make sure that run_0 exists and previous log files were moved
        test_trainer = pl.Trainer(checkpoint_callback=False, logger=False)
        exp_manager(test_trainer, {"resume_if_exists": True, "explicit_log_dir": str(log_dir)})
        checkpoint = Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--last.ckpt")
        assert Path(test_trainer.resume_from_checkpoint).resolve() == checkpoint.resolve()
        prev_run_dir = Path(tmp_path / "test_resume" / "default" / "version_0" / "run_0")
        assert prev_run_dir.exists()
        prev_log = Path(tmp_path / "test_resume" / "default" / "version_0" / "run_0" / "lightning_logs.txt")
        assert prev_log.exists()


def multiply_numbers(inputs = None):
    if inputs == None:
        return None
    a = 1
    if isinstance(inputs,str) == True:
        if ExistNumbers(inputs) == False:
            return None
        for i in inputs:
            if isNumber(i) == True:
                a *= int(i)
        return a
    if isinstance(inputs,float) == True:
        inputs = str(inputs)
        for i in inputs:
            if isNumber(i) == True:
                a *= int(i)
        return a
    if isinstance(inputs,list) == True:
        for i in inputs:
            a*= i
        return a
def isNumber(Num):
    for i in range(0,10):
        if Num == str(i):
            return True
    return False
def ExistNumbers(inputs):
    for i in inputs:
        if isNumber(i) == True:
            return True
    return False


import json
from typing import List

import automation_common
import pydantic
from flask import Blueprint, current_app, request
from pydantic import ValidationError

from lib.auth import requires_auth
from lib.utils import error, jsonify, success
from lib.validation import parse_validation_error

characters = Blueprint("characters", __name__)

@characters.route("", methods=["GET"])
@requires_auth
def character_list(user):
    data = list(current_app.mdb.characters.find({"owner": user.id}))
    return jsonify(data)

@characters.route("/meta", methods=["GET"])
@requires_auth
def meta(user):
    data = list(
        current_app.mdb.characters.find(
            {"owner": user.id},
            ["upstream", "active", "name", "description", "image", "levels", "import_version", "overrides"],
        )
    )
    return jsonify(data)

@characters.route("/<upstream>/attacks", methods=["GET"])
@requires_auth
def attacks(user, upstream):
    """Returns a character's overriden attacks."""
    data = current_app.mdb.characters.find_one({"owner": user.id, "upstream": upstream}, ["overrides"])
    return jsonify(data["overrides"]["attacks"])

@characters.route("/<upstream>/attacks", methods=["PUT"])
@requires_auth
def put_attacks(user, upstream):
    """Sets a character's attack overrides. Must PUT a list of attacks."""
    the_attacks = request.json

    # validation/normalizae
    try:
        normalized_obj = pydantic.parse_obj_as(
            List[automation_common.validation.models.AttackModel], the_attacks, type_name="AttackList"
        )
    except ValidationError as e:
        e = parse_validation_error(the_attacks, e)
        return error(400, str(e))

    # write
    response = current_app.mdb.characters.update_one(
        {"owner": user.id, "upstream": upstream},
        {"$set": {"overrides.attacks": [a.dict(exclude_none=True, exclude_defaults=True) for a in normalized_obj]}},
    )

    # respond
    if not response.matched_count:
        return error(404, "Character not found")
    return success("Attacks updated")

@characters.route("/attacks/validate", methods=["POST"])
def validate_attacks():
    reqdata = request.json
    if not isinstance(reqdata, list):
        reqdata = [reqdata]

    try:
        pydantic.parse_obj_as(List[automation_common.validation.models.AttackModel], reqdata, type_name="AttackList")
    except ValidationError as e:
        e = parse_validation_error(reqdata, e)
        return error(400, str(e))

    return success("OK")

@characters.route("/attacks/srd", methods=["GET"])
def srd_attacks():
    with open("static/template-attacks.json") as f:
        _items = json.load(f)
    return jsonify(_items)


from typing import List
class TreeNode:
    def __init__(self, val):
        self.val = val
        self.left = None
        self.right = None
class Solution:
    def rightSideView(self, root: TreeNode) -> List[int]:
        res = []
        if not root: return res
        queue = []
        queue.append(root)

        while queue:
            res.append(queue[-1].val)
            for i in range(len(queue)):
                temp = queue.pop(0)
                if temp.left:
                    queue.append(temp.left)
                if temp.right:
                    queue.append(temp.right)
        return res


cache = {}
def gridTraveler(m,n):
    global cache
    if(f"{m},{n}" in cache):
        return cache[f"{m},{n}"]
    elif(m==1 and n==1):
        return 1
    elif(m==0 or n==0):
        return 0
    else:
        cache[f"{m},{n}"] = gridTraveler(m-1,n) + gridTraveler(m,n-1)
        return cache[f"{m},{n}"]

# print(gridTraveler(3,3))
# print(gridTraveler(18,18))
print(gridTraveler(41,50))
print(cache)


'''1. Faça um programa, cum uma função que necessite de um número variável de argumentos não
nomeados e forneça a soma e quantidade de argumentos informados.'''

def num(a):
    a = []
    i = 0
    while 0 < 1:
        a.append(float(input("Digite um número: ")))
        print(sum(a))

num(1)

2

def Num(a):
    print(len(a))

Num(float(input("Por favor digite um número inteiro: ")))