"""
All the functions to properly build the object hierarchies.
"""
import collections.abc
import enum
import warnings
from typing import Any, Iterable, Iterator, Mapping, MutableMapping, Optional, Union, cast

from kopf._cogs.helpers import thirdparty
from kopf._cogs.structs import bodies, dicts
from kopf._core.actions import execution
from kopf._core.intents import causes

K8sObject = Union[MutableMapping[Any, Any], thirdparty.PykubeObject, thirdparty.KubernetesModel]
K8sObjects = Union[K8sObject, Iterable[K8sObject]]

class _UNSET(enum.Enum):
    token = enum.auto()

def append_owner_reference(
        objs: K8sObjects,
        owner: Optional[bodies.Body] = None,
        *,
        controller: Optional[bool] = True,
        block_owner_deletion: Optional[bool] = True,
) -> None:
    """
    Append an owner reference to the resource(s), if it is not yet there.

    Note: the owned objects are usually not the one being processed,
    so the whole body can be modified, no patches are needed.
    """
    real_owner = _guess_owner(owner)
    owner_ref = bodies.build_owner_reference(
        real_owner, controller=controller, block_owner_deletion=block_owner_deletion
    )
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            refs = obj.setdefault('metadata', {}).setdefault('ownerReferences', [])
            if not any(ref.get('uid') == owner_ref['uid'] for ref in refs):
                refs.append(owner_ref)
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if obj.metadata.owner_references is None:
                obj.metadata.owner_references = []
            refs = obj.metadata.owner_references
            if not any(ref.uid == owner_ref['uid'] for ref in refs):
                refs.append(thirdparty.V1OwnerReference(
                    api_version=owner_ref['apiVersion'],
                    kind=owner_ref['kind'],
                    name=owner_ref['name'],
                    uid=owner_ref['uid'],
                    controller=owner_ref['controller'],
                    block_owner_deletion=owner_ref['blockOwnerDeletion'],
                ))
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def remove_owner_reference(
        objs: K8sObjects,
        owner: Optional[bodies.Body] = None,
) -> None:
    """
    Remove an owner reference to the resource(s), if it is there.

    Note: the owned objects are usually not the one being processed,
    so the whole body can be modified, no patches are needed.
    """
    real_owner = _guess_owner(owner)
    owner_ref = bodies.build_owner_reference(real_owner)
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            refs = obj.setdefault('metadata', {}).setdefault('ownerReferences', [])
            if any(ref.get('uid') == owner_ref['uid'] for ref in refs):
                refs[:] = [ref for ref in refs if ref.get('uid') != owner_ref['uid']]
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if obj.metadata.owner_references is None:
                obj.metadata.owner_references = []
            refs = obj.metadata.owner_references
            if any(ref.uid == owner_ref['uid'] for ref in refs):
                refs[:] = [ref for ref in refs if ref.uid != owner_ref['uid']]
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def label(
        objs: K8sObjects,
        labels: Union[Mapping[str, Union[None, str]], _UNSET] = _UNSET.token,
        *,
        forced: bool = False,
        nested: Optional[Union[str, Iterable[dicts.FieldSpec]]] = None,
        force: Optional[bool] = None,  # deprecated
) -> None:
    """
    Apply the labels to the object(s).
    """
    nested = [nested] if isinstance(nested, str) else nested
    if force is not None:
        warnings.warn("force= is deprecated in kopf.label(); use forced=...", DeprecationWarning)
        forced = force

    # Try to use the current object being handled if possible.
    if isinstance(labels, _UNSET):
        real_owner = _guess_owner(None)
        labels = real_owner.get('metadata', {}).get('labels', {})
    if isinstance(labels, _UNSET):
        raise RuntimeError("Impossible error: labels are not resolved.")  # for type-checking

    # Set labels based on the explicitly specified or guessed ones.
    for obj in cast(Iterator[K8sObject], dicts.walk(objs, nested=nested)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            obj_labels = obj.setdefault('metadata', {}).setdefault('labels', {})
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if obj.metadata.labels is None:
                obj.metadata.labels = {}
            obj_labels = obj.metadata.labels
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

        for key, val in labels.items():
            if forced:
                obj_labels[key] = val
            else:
                obj_labels.setdefault(key, val)

def harmonize_naming(
        objs: K8sObjects,
        name: Union[None, str, _UNSET] = _UNSET.token,
        *,
        forced: bool = False,
        strict: bool = False,
) -> None:
    """
    Adjust the names or prefixes of the objects.

    In strict mode, the provided name is used as is. It can be helpful
    if the object is referred by that name in other objects.

    In non-strict mode (the default), the object uses the provided name
    as a prefix, while the suffix is added by Kubernetes remotely.
    The actual name should be taken from Kubernetes response
    (this is the recommended scenario).

    If the objects already have their own names, auto-naming is not applied,
    and the existing names are used as is.
    """

    # Try to use the current object being handled if possible.
    if isinstance(name, _UNSET):
        real_owner = _guess_owner(None)
        name = real_owner.get('metadata', {}).get('name', None)
    if isinstance(name, _UNSET):
        raise RuntimeError("Impossible error: the name is not resolved.")  # for type-checking
    if name is None:
        raise LookupError("Name must be set explicitly: couldn't find it automatically.")

    # Set name/prefix based on the explicitly specified or guessed name.
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            noname = 'metadata' not in obj or not set(obj['metadata']) & {'name', 'generateName'}
            if forced or noname:
                if strict:
                    obj.setdefault('metadata', {})['name'] = name
                    if 'generateName' in obj['metadata']:
                        del obj['metadata']['generateName']
                else:
                    obj.setdefault('metadata', {})['generateName'] = f'{name}-'
                    if 'name' in obj['metadata']:
                        del obj['metadata']['name']
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            noname = obj.metadata.name is None and obj.metadata.generate_name is None
            if forced or noname:
                if strict:
                    obj.metadata.name = name
                    if obj.metadata.generate_name is not None:
                        obj.metadata.generate_name = None
                else:
                    obj.metadata.generate_name = f'{name}-'
                    if obj.metadata.name is not None:
                        obj.metadata.name = None
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def adjust_namespace(
        objs: K8sObjects,
        namespace: Union[None, str, _UNSET] = _UNSET.token,
        *,
        forced: bool = False,
) -> None:
    """
    Adjust the namespace of the objects.

    If the objects already have the namespace set, it will be preserved.

    It is a common practice to keep the children objects in the same
    namespace as their owner, unless explicitly overridden at time of creation.
    """

    # Try to use the current object being handled if possible.
    if isinstance(namespace, _UNSET):
        real_owner = _guess_owner(None)
        namespace = real_owner.get('metadata', {}).get('namespace', None)
    if isinstance(namespace, _UNSET):
        raise RuntimeError("Impossible error: the namespace is not resolved.")  # for type-checking

    # Set namespace based on the explicitly specified or guessed namespace.
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            if forced or obj.get('metadata', {}).get('namespace') is None:
                obj.setdefault('metadata', {})['namespace'] = namespace
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if forced or obj.metadata.namespace is None:
                obj.metadata.namespace = namespace
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def adopt(
        objs: K8sObjects,
        owner: Optional[bodies.Body] = None,
        *,
        forced: bool = False,
        strict: bool = False,
        nested: Optional[Union[str, Iterable[dicts.FieldSpec]]] = None,
) -> None:
    """
    The children should be in the same namespace, named after their parent, and owned by it.
    """
    real_owner = _guess_owner(owner)
    real_owner_name = real_owner.get('metadata', {}).get('name', None)
    real_owner_namespace = real_owner.get('metadata', {}).get('namespace', None)
    real_owner_labels = real_owner.get('metadata', {}).get('labels', {})
    append_owner_reference(objs, owner=real_owner)
    harmonize_naming(objs, forced=forced, strict=strict, name=real_owner_name)
    adjust_namespace(objs, forced=forced, namespace=real_owner_namespace)
    label(objs, forced=forced, nested=nested, labels=real_owner_labels)

def _guess_owner(
        owner: Optional[bodies.Body],
) -> bodies.Body:
    if owner is not None:
        return owner

    try:
        cause = execution.cause_var.get()
    except LookupError:
        pass
    else:
        if cause is not None and isinstance(cause, causes.ResourceCause):
            return cause.body

    raise LookupError("Owner must be set explicitly, since running outside of a handler.")


import json
from collections import namedtuple

from scorum.graphenebase.amount import Amount
from scorum.utils.time import fmt_time_from_now

from automation.account import Account

GenesisAccount = namedtuple("GenesisAccount", ["account", "amount"])

class Genesis(object):
    def __init__(self):
        self.genesis_accounts = []  # list(GenesisAccount))
        self.witness_candidates = []  # list(Account)
        self.founders = set()  # set((str, float))
        self.reg_committee = set()  # set(str)
        self.dev_committee = set()  # set(str)
        self.steemit_bounty_accounts = set()  # set(str)

        self.accounts_supply = Amount()
        self.steemit_bounty_supply = Amount('0 SP')
        self.params = {}
        self._set_timestamp()
        self["lock_withdraw_sp_until_timestamp"] = "2018-07-01T00:00:00"
        self["total_supply"] = "700.000000000 SCR"
        self["registration_supply"] = "100.000000000 SCR"
        self["registration_bonus"] = "0.000000100 SCR"
        self["accounts_supply"] = "100.000000000 SCR"
        self["rewards_supply"] = "100.000000000 SCR"
        self["founders_supply"] = "100.000000000 SP"
        self["steemit_bounty_accounts_supply"] = "100.000000000 SP"
        self["development_sp_supply"] = "100.000000000 SP"
        self["development_scr_supply"] = "100.000000000 SCR"
        self["accounts"] = []
        self["founders"] = []
        self["steemit_bounty_accounts"] = []
        self["witness_candidates"] = []
        self["registration_schedule"] = [
            {"stage": 1, "users": 24, "bonus_percent": 100},
            {"stage": 2, "users": 1, "bonus_percent": 75},
            {"stage": 3, "users": 1, "bonus_percent": 50},
            {"stage": 4, "users": 1, "bonus_percent": 25}
        ]
        self["registration_committee"] = []
        self["development_committee"] = []

    def __getitem__(self, item):
        return self.params[item]

    def __setitem__(self, key, value):
        self.params[key] = value

    def add_account(self, name, balance="100.000000000 SCR"):
        amount = Amount(balance)
        account = self.get_account(name)
        if not account:
            account = name if type(name) is Account else Account(name)
            self.genesis_accounts.append(GenesisAccount(account, amount))

    def get_account(self, name):
        account = name if type(name) is Account else Account(name)
        for ga in self.genesis_accounts:
            if account == ga.account:
                return ga.account

    def add_witness_acc(self, name):
        account = self.get_account(name)
        if account:
            account.set_signing_key()
            self.witness_candidates.append(account)

    def add_founder_acc(self, name, percent: float):
        account = self.get_account(name)
        if account:
            self.founders.add((account.name, percent))

    def add_dev_committee_acc(self, name):
        account = self.get_account(name)
        if account:
            self.dev_committee.add(account.name)

    def add_reg_committee_acc(self, name):
        account = self.get_account(name)
        if account:
            self.reg_committee.add(account.name)

    def add_steemit_bounty_acc(self, name):
        account = self.get_account(name)
        if account:
            self.steemit_bounty_accounts.add(account.name)

    def calculate_supplies(self):
        # calculate accounts supply
        for ga in self.genesis_accounts:
            self.accounts_supply += ga.amount

            account = {'name': ga.account.name,
                       'scr_amount': str(ga.amount),
                       'public_key': ga.account.get_owner_public()}
            self['accounts'].append(account)
        self['accounts_supply'] = str(self.accounts_supply)
        # collect witnesses
        self['witness_candidates'] = [
            {'name': acc.name, 'block_signing_key': acc.get_signing_public()}
            for acc in self.witness_candidates
        ]
        # collect founders
        self['founders'] = [
            {'name': name, 'sp_percent': percent}
            for name, percent in self.founders
        ]
        # collect dev_committee
        self['development_committee'] = list(self.dev_committee)
        # collect reg_committee
        self['registration_committee'] = list(self.reg_committee)
        # calculate steemit_bounty supply
        bounty_per_acc = Amount(
            self['steemit_bounty_accounts_supply']
        ) / len(self.steemit_bounty_accounts)

        self['steemit_bounty_accounts'] = [
            {'name': acc, 'sp_amount': str(bounty_per_acc)}
            for acc in self.steemit_bounty_accounts
        ]
        self.steemit_bounty_supply = bounty_per_acc * len(
            self.steemit_bounty_accounts
        )
        self['steemit_bounty_accounts_supply'] = str(self.steemit_bounty_supply)
        # calculate total supply
        supplies = [
            'accounts_supply', 'rewards_supply', 'registration_supply',
            'founders_supply', 'steemit_bounty_accounts_supply',
            'development_sp_supply', 'development_scr_supply'
        ]
        total = Amount()
        for supply in supplies:
            total += Amount(self[supply])
        self['total_supply'] = str(total)

    def dump(self):
        return json.dumps(self.params)

    def _set_timestamp(self):
        self['initial_timestamp'] = fmt_time_from_now(1)

    def get_accounts(self):
        return [ga.account for ga in self.genesis_accounts]


import logging

from utils import messages
import helpers

from descriptors import EndpointAddress
from task import Task
import zmq

logger = logging.getLogger('zmqpipeline.distributor')

class Distributor(object):
    """
    Responsible for distributing (pushing) tasks to workers. What gets distributed is determined
    by Tasks, which are user-implementations that configure how the distributor works.

    The pipeline pattern assumes there is only one distributor with one or more registered tasks.
    """
    task_classes = {}

    def __init__(self, collector_endpoint, collector_ack_endpoint,
                 receive_metadata = False, metadata_endpoint = None):
        """
        Instantiate a distributor.

        :param EndpointAddress collector_endpoint: The endpoint of the collector
        :param EndpointAddress collector_ack_endpoint: The endpoint of the collector for receiving ACKs
        :param bool receive_metadata: When true, the distributor will wait for meta data from a meta data worker before distributing tasks to clients
        :param EndpointAddress metadata_endpoint: The endpoint from which to receive meta data from a meta data worker.
        :return: A Distributor object
        """
        logger.info('Initializing distributor')
        self.context = zmq.Context()
        self.metadata = {}

        if not isinstance(collector_endpoint, EndpointAddress):
            raise TypeError('collector_endpoint must be an EndpointAddress instance')
        if not isinstance(collector_ack_endpoint, EndpointAddress):
            raise TypeError('collector_ack_endpoint must be an EndpointAddress instance')

        logger.info('Connecting to collector endpoint %s', collector_endpoint)
        self.sink = self.context.socket(zmq.PUSH)
        self.sink.connect(collector_endpoint)

        logger.info('Connecting to collector ACK endpoint %s', collector_ack_endpoint)
        self.sink_ack = self.context.socket(zmq.PULL)
        self.sink_ack.connect(collector_ack_endpoint)

        if receive_metadata:
            logger.debug('Setting up receiving metadata')
            if not metadata_endpoint:
                raise AttributeError('metadata_endpoint is required to receive metadata')
            if not isinstance(metadata_endpoint, EndpointAddress):
                raise TypeError('metadata_endpoint must be an EndpointAddress instance')

            logger.info('Connecting to meta data worker endpoint %s', metadata_endpoint)
            self.metadata_client = self.context.socket(zmq.REP)
            self.metadata_client.bind(metadata_endpoint)
        else:
            self.metadata_client = None

        self.poller = zmq.Poller()
        self.poller.register(self.sink_ack, zmq.POLLIN)

        self.client_addresses = {}

        # keyed on worker ids
        self.worker_initialized = {}
        self.clients = {}
        self.tasks = {}

        for taskcls in self.registered_task_classes:
            self.register_task_instance(taskcls)

        self.receive_metadata = receive_metadata
        if receive_metadata:
            self.wait_for_metadata()

        # data received from collector
        self.ack_data = {}

    @property
    def registered_task_classes(self):
        return self.task_classes.values()

    @classmethod
    def register_task(cls, task):
        if task.task_type not in cls.task_classes:
            cls.task_classes[task.task_type] = task

    def wait_for_metadata(self):
        logger.info('Waiting to receive metadata')
        msg = self.metadata_client.recv()
        data, task, msgtype = messages.get(msg)
        assert msgtype == messages.MESSAGE_TYPE_META_DATA
        assert task == ''

        logger.debug('Storing metadata: %s', data)

        if data:
            self.metadata = data

        logger.info('Meta data received - sending success response')
        self.metadata_client.send(messages.create_success())

        logger.info('Sending meta data to collector')
        self.sink.send(messages.create_metadata(self.metadata))

    def process_sink_ack(self):
        msg = self.sink_ack.recv()
        data, tt, msgtype = messages.get(msg)

        self.tasks[tt].n_acks += 1
        ack_id = data.get('_id', 0)

        logger.debug('Received ACK ID %d - %d acks from task type %s', ack_id, self.tasks[tt].n_acks, tt)
        # logger.info('Received ACK ID %d - %d acks from task type %s - msgtype: %s', ack_id, self.tasks[tt].n_acks, tt, msgtype)
        if data:
            self.ack_data = data
        else:
            self.ack_data = {}

    def register_task_instance(self, taskcls):
        task = taskcls()
        logger.debug('Registering Task with type %s', task.task_type)

        logger.info('Connecting task type %s to endpoint %s', task.task_type, task.endpoint)
        s = self.context.socket(zmq.ROUTER)
        s.bind(helpers.endpoint_binding(task.endpoint))

        task.client = s
        self.clients[task.task_type] = s
        self.tasks[task.task_type] = task

        logger.debug('Registering task %s', task.task_type)
        Task.register_task_instance(task)
        self.poller.register(s, zmq.POLLIN)

    def add_client_address(self, task_type, addr):
        if task_type not in self.client_addresses:
            self.client_addresses[task_type] = set()
        if addr not in self.client_addresses[task_type]:
            logger.info('Adding new client address: %s for task type: %s', addr, task_type)

        self.client_addresses[task_type].add(addr)

    def run(self):
        """
        Runs the distributor and blocks. Call this immediately after instantiating the distributor.

        Since this method will block the calling program, it should be the last thing the calling code
        does before exiting. Run() will automatically shutdown the distributor gracefully when finished.

        :return: None
        """
        logger.info('Distributor is running (main loop processing)')

        while True:
            try:
                socks = dict(self.poller.poll())
            except KeyboardInterrupt:
                break

            if self.sink_ack in socks:
                self.process_sink_ack()

            alltasks = self.tasks.values()
            if all([t.is_complete for t in alltasks]):
                "exit main loop if all registered tasks are complete"
                logger.info('All tasks complete - exiting distributor main loop')
                break

            for task_type, task in self.tasks.items():
                if task.is_complete:
                    # task finished - skip
                    continue

                if task.is_task_ready_for_initialization() or task.received_init_signal:
                    # task either needs to be initialized, or has been initialized and is ready for processing

                    if task.received_init_signal and not task.is_available_for_handling(self.ack_data):
                        # only consult the task about handling availability if the init signal has been received
                        continue

                    address, empty, msg = task.client.recv_multipart()
                    self.add_client_address(task_type, address)

                    if not self.worker_initialized.get(address):
                        logger.debug('Initial signal for task %s from worker %s recieved', task_type, address)
                        self.worker_initialized[address] = True
                        task.received_init_signal = True

                        if hasattr(task, 'initialize'):
                            logger.info('Initializing task type %s', task.task_type)
                            task.initialize(metadata = self.metadata)

                        logger.debug('Sending success reply to worker address: %s', address)
                        task.client.send_multipart([
                            address, b'', messages.create_metadata(self.metadata)
                        ])

                    else:
                        data, tt, msgtype = messages.get(msg)
                        data = data or {}

                        if msgtype == messages.MESSAGE_TYPE_READY:
                            # invoke registered task's handler
                            logger.debug('Invoking task.handle for task type: %s, client address %s - passing data: %s', task.task_type, address, data)
                            sdata = task.handle(data, address, msgtype, self.ack_data) or {}
                            if not isinstance(sdata, dict) and sdata is not None:
                                raise TypeError('Task handler must return a dictionary or nothing')

                            sdata.update(data)

                            logger.debug('Sending to collector - task type: %s - data: %s', tt, sdata)
                            task.client.send_multipart([
                                address, b'', messages.create_data(tt, sdata)
                            ])

        # shutdown collector + workers
        self.shutdown()

    def shutdown(self):
        """
        Shuts down the distributor. This is automatically called when run() is complete and the distributor
        exits gracefully. Client code should only invoke this method directly on exiting prematurely,
        for example on a KeyboardInterruptException

        :return: None
        """
        logger.info('Shutting down collector and workers')

        logger.debug('Sending END signal to collector')
        self.sink.send(messages.create_end())

        for task_type, task in self.tasks.items():
            client_addrs = self.client_addresses[task_type]
            n_clients = len(client_addrs)

            logger.debug('Shutting down %d clients for task type %s', n_clients, task_type)
            for _ in client_addrs:
                address, empty, msg = task.client.recv_multipart()

                logger.debug('Sending END signal to worker address %s - task type %s', address, task_type)
                task.client.send_multipart([
                    address, b'', messages.create_end(task = task_type)
                ])


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from django.core.management.base import BaseCommand

from crashstats.cron import JOBS, DEFAULT_FREQUENCY
from crashstats.cron.models import Job

class Command(BaseCommand):
    help = "List available cron jobs."

    def handle(self, **options):
        for job_spec in JOBS:
            self.stdout.write(job_spec["cmd"])

            cmdline = [job_spec["cmd"]]
            if job_spec.get("cmd_args"):
                cmdline.extend(job_spec["cmd_args"])
            if job_spec.get("backfill"):
                cmdline.append("--run-time=RUNTIME")
            if job_spec.get("last_success"):
                cmdline.append("--last-success=LASTSUCCESS")
            cmdline = " ".join(cmdline)
            self.stdout.write("    cmdline:      %s" % cmdline)

            schedule = []
            schedule.append("every " + job_spec.get("frequency", DEFAULT_FREQUENCY))
            if job_spec.get("time"):
                schedule.append(job_spec["time"] + " UTC")
            schedule = " @ ".join(schedule)
            self.stdout.write("    schedule:     %s" % schedule)

            try:
                job = Job.objects.get(app_name=job_spec["cmd"])
                self.stdout.write("    last run:     %s" % job.last_run)
                self.stdout.write("    last_success: %s" % job.last_success)
                self.stdout.write("    next run:     %s" % job.next_run)
                if job.last_error:
                    self.stdout.write("    " + job.last_error)
            except Job.DoesNotExist:
                self.stdout.write("    Never run.")


import sys, os
from pprint import pprint
from JMTTools import *
from JMTROOTTools import *
set_style()

run = run_from_argv()
run_dir = run_dir(run)
in_fn = os.path.join(run_dir, 'POHBias.root')
if not os.path.isfile(in_fn):
    raise RuntimeError('no file at %s' % in_fn)
out_dir = os.path.join(run_dir, 'dump_pohbias')
os.system('mkdir -p %s' % out_dir)

f = ROOT.TFile(in_fn)

gains = range(4)
feds = range(1294, 1298)

# JMT need ROOT os.walk...
all_graphs = defaultdict(dict)

for gainkey in f.GetListOfKeys():
    gainname = gainkey.GetName()
    gain = int(gainname.replace('gain', ''))
    for fedkey in f.Get(gainname).GetListOfKeys():
        fedname = fedkey.GetName()
        fed = int(fedname.replace('FED', ''))
        for graphkey in f.Get(gainname + '/' + fedname).GetListOfKeys():
            graphname = graphkey.GetName()
            fiber = int(graphname.split('_')[-1].replace('fiber', ''))
            graph = graphkey.ReadObj()
            all_graphs[(fed, fiber)][gain] = graph

for fedfiber, graphs in all_graphs.iteritems():
    c = ROOT.TCanvas('c', '', 1000, 800)
    leg = ROOT.TLegend()
    for gain in range(4):
        g = graphs[gain]
        g.SetTitle('FED %i fiber %i;' % fedfiber + g.GetTitle().split(';', 1)[-1])
        color = gain + 1
        g.SetLineColor(color)
        g.SetMarkerColor(color)
        g.GetYaxis().SetRangeUser(0, 0.5)
        #g.GetFunction("fit_to_rssi_response").Delete() #SetBit(ROOT.TF1.kNotDraw)
        if gain == 0:
            g.Draw('AP')
        else:
            g.Draw('P')
    c.SaveAs(os.path.join(out_dir, 'FED%i_fiber%i.png' % fedfiber))
    del c


from callset import GCNVCallset, TruthCallset
from interval_collection import IntervalCollection
from evaluator import PerEventEvaluator, PerBinEvaluator
from evaluation_result import PerEventEvaluationResult, PerBinEvaluationResult
import pandas as pd
import numpy as np

pd.set_option("display.max_rows", None, "display.max_columns", None)

# analyzed intervals
ANALYZED_INTERVALS = "./test_files/analyzed_intervals.interval_list"

# gCNV test callset resources
GCNV_CALLSET_TEST_VCF_LIST = ["./test_files/GCNV_SAMPLE_0.vcf", "./test_files/GCNV_SAMPLE_1.vcf",
                              "./test_files/GCNV_SAMPLE_2.vcf", "./test_files/GCNV_SAMPLE_3.vcf"]

# Truth test callset resources
TRUTH_CALLSET_TEST_BED = "./test_files/truth.bed"

# Per event evaluator expected results
PER_EVENT_EVALUATION_RESULT_EXPECTED = PerEventEvaluationResult()
PER_EVENT_EVALUATION_RESULT_EXPECTED.precision_size_to_tp[1] = 3
PER_EVENT_EVALUATION_RESULT_EXPECTED.precision_size_to_fp[0] = 2
PER_EVENT_EVALUATION_RESULT_EXPECTED.precision_size_to_fp[1] = 1

PER_EVENT_EVALUATION_RESULT_EXPECTED.recall_size_to_tp[2] = 3
PER_EVENT_EVALUATION_RESULT_EXPECTED.recall_size_to_fn[0] = 3
PER_EVENT_EVALUATION_RESULT_EXPECTED.recall_size_to_fn[1] = 3

# Per bin evaluator expected results
PER_BIN_EVALUATION_RESULT_EXPECTED = PerBinEvaluationResult(np.zeros(1), 1)  # initialize with fake values
PER_BIN_EVALUATION_RESULT_EXPECTED.quality_thresholds = [0.0, 20.0, 60.0, 100.0]
PER_BIN_EVALUATION_RESULT_EXPECTED.true_positives = {0.0: 7, 20.0: 7, 60.0: 7, 100.0: 1}
PER_BIN_EVALUATION_RESULT_EXPECTED.false_positives = {0.0: 3, 20.0: 2, 60.0: 0, 100.0: 0}
PER_BIN_EVALUATION_RESULT_EXPECTED.false_negatives = {0.0: 8, 20.0: 8, 60.0: 10, 100.0: 16}

def test_event_evaluator():
    interval_collection = IntervalCollection.read_interval_list(ANALYZED_INTERVALS)
    gcnv_callset = GCNVCallset.read_in_callset(gcnv_segment_vcfs=GCNV_CALLSET_TEST_VCF_LIST,
                                               interval_collection=interval_collection)
    truth_callset = TruthCallset.read_in_callset(truth_callset_bed_file=TRUTH_CALLSET_TEST_BED,
                                                 interval_collection=interval_collection,
                                                 samples_to_keep=gcnv_callset.sample_set)
    truth_callset.filter_out_uncovered_events_from_joint_callset(interval_collection, min_overlap_fraction=0.3)
    evaluator = PerEventEvaluator(truth_callset=truth_callset, callset=gcnv_callset)
    evaluation_result_actual = evaluator.evaluate_callset_against_the_truth(minimum_overlap=0.4)
    assert evaluation_result_actual == PER_EVENT_EVALUATION_RESULT_EXPECTED

def test_bin_evaluator():
    interval_collection = IntervalCollection.read_interval_list(ANALYZED_INTERVALS)
    gcnv_callset = GCNVCallset.read_in_callset(gcnv_segment_vcfs=GCNV_CALLSET_TEST_VCF_LIST)
    truth_callset = TruthCallset.read_in_callset(truth_callset_bed_file=TRUTH_CALLSET_TEST_BED,
                                                 interval_collection=interval_collection,
                                                 samples_to_keep=gcnv_callset.sample_set)
    evaluator = PerBinEvaluator(truth_callset=truth_callset, gcnv_callset=gcnv_callset, interval_collection=interval_collection)
    evaluation_result_actual = evaluator.evaluate_callset_against_the_truth(gcnv_callset, 4)
    assert evaluation_result_actual == PER_BIN_EVALUATION_RESULT_EXPECTED

def main():
    test_event_evaluator()
    #test_bin_evaluator()

if __name__ == '__main__':
    main()


import json
import os

import geopandas as gpd
import numpy
from geomet import wkt

from noise_api.noise_analysis.road_info import RoadInfo

cwd = os.path.dirname(os.path.abspath(__file__))

# road_type_ids from IffStar NoiseModdeling
road_types_iffstar_noise_modelling = {
    "boulevard": 56,  # in boulevard 70km/h
    "street": 53,  # extra boulevard Street 50km/h
    "alley": 54,  # extra boulevard Street <50km/,
    "railroad": 99,  # railroad
}

all_roads = []

def reset_all_roads():
    global all_roads
    all_roads = []

# opens a json from path
def open_geojson(path):
    with open(path) as f:
        return json.load(f)

# extract traffic data from road properties
def get_car_traffic_data(road_properties):
    #  https://d-nb.info/97917323X/34
    # p. 68, assuming a mix of type a) and type b) for car traffic around grasbrook (strong morning peak)
    # most roads in hamburg belong to type a) or b)
    # assuming percentage of daily traffic in peak hour = 11%
    # truck traffic: the percentage of daily traffic in peak hour seems to be around 8%
    # source. https://bast.opus.hbz-nrw.de/opus45-bast/frontdoor/deliver/index/docId/1921/file/SVZHeft29.pdf

    # Sources daily traffic: HafenCity GmbH , Standortanalyse, S. 120

    if road_properties["road_type"] == "railroad":
        return None, None, None

    car_traffic = int(int(road_properties["car_traffic_daily"]) * 0.11)
    truck_traffic = int(int(road_properties["truck_traffic_daily"]) * 0.08)
    max_speed = int(road_properties["max_speed"])

    return max_speed, car_traffic, truck_traffic

# source for train track data = http://laermkartierung1.eisenbahn-bundesamt.de/mb3/app.php/application/eba
def get_train_track_data(road_properties):
    if not road_properties["road_type"] == "railroad":
        return None, None, None, None

    train_speed = road_properties["train_speed"]
    train_per_hour = road_properties["trains_per_hour"]
    ground_type = road_properties["ground_type"]
    has_anti_vibration = road_properties["has_anti_vibration"]

    return train_speed, train_per_hour, ground_type, has_anti_vibration

def apply_traffic_settings_to_design_roads(design_roads, traffic_settings):
    max_speed = traffic_settings["max_speed"]
    traffic_quota = traffic_settings["traffic_quota"]

    for road in design_roads["features"]:
        # only adjust traffic settings of manipulatable roads
        if (
            "traffic_settings_adjustable" in list(road["properties"].keys())
            and road["properties"]["traffic_settings_adjustable"]
        ):
            road["properties"]["max_speed"] = max_speed
            road["properties"]["truck_traffic_daily"] = (
                road["properties"]["truck_traffic_daily"] * traffic_quota
            )
            road["properties"]["car_traffic_daily"] = (
                road["properties"]["car_traffic_daily"] * traffic_quota
            )

    return design_roads

def get_road_queries(traffic_settings, roads_gdf):
    roads_geojson = json.loads(roads_gdf.to_json())
    roads_geojson = apply_traffic_settings_to_design_roads(
        roads_geojson, traffic_settings
    )
    road_features = roads_geojson["features"]

    for feature in road_features:
        id = feature["properties"]["id"]
        road_type = get_road_type(feature["properties"])
        coordinates = feature["geometry"]["coordinates"]

        # input road type might not be defined. road is not imported # TODO consider using a fallback
        if road_type == 0:
            continue
        if feature["geometry"]["type"] == "MultiLineString":
            # beginning point of the road
            start_point = coordinates[0][0]
            # end point of the road
            end_point = coordinates[-1][-1]
        else:
            # beginning point of the road
            start_point = coordinates[0]
            # end point of the road
            end_point = coordinates[1]
            # build string containing all coordinates

        geom = wkt.dumps(feature["geometry"], decimals=0)

        max_speed, car_traffic, truck_traffic = get_car_traffic_data(
            feature["properties"]
        )
        (
            train_speed,
            train_per_hour,
            ground_type,
            has_anti_vibration,
        ) = get_train_track_data(feature["properties"])

        # init new RoadInfo object
        road_info = RoadInfo(
            id,
            geom,
            road_type,
            start_point,
            end_point,
            max_speed,
            car_traffic,
            truck_traffic,
            train_speed,
            train_per_hour,
            ground_type,
            has_anti_vibration,
        )
        all_roads.append(road_info)

    nodes = create_nodes(all_roads)
    sql_insert_strings = []
    for road in all_roads:
        sql_insert_string = get_insert_query_for_road(road, nodes)
        sql_insert_strings.append(sql_insert_string)

    return sql_insert_strings

# returns sql queries for the traffic table,
def get_traffic_queries():
    sql_insert_strings_noisy_roads = []
    nodes = create_nodes(all_roads)
    for road in all_roads:
        node_from = get_node_for_point(road.get_start_point(), nodes)
        node_to = get_node_for_point(road.get_end_point(), nodes)

        if (
            road.get_road_type_for_query()
            == road_types_iffstar_noise_modelling["railroad"]
        ):
            # train traffic
            train_speed = road.get_train_speed()
            trains_per_hour = road.get_train_per_hour()
            ground_type = road.get_ground_type_train_track()
            has_anti_vibration = road.is_anti_vibration()

            sql_insert_string = (
                "INSERT INTO roads_traffic (node_from,node_to, train_speed, "
                "trains_per_hour, ground_type, has_anti_vibration) "
                "VALUES ({0},{1},{2},{3},{4},{5});".format(
                    node_from,
                    node_to,
                    train_speed,
                    trains_per_hour,
                    ground_type,
                    has_anti_vibration,
                )
            )
        else:
            # car traffic
            traffic_cars = road.get_car_traffic()
            traffic_trucks = road.get_truck_traffic()
            max_speed = road.get_max_speed()
            load_speed = max_speed * 0.9
            junction_speed = max_speed * 0.85

            sql_insert_string = (
                "INSERT INTO roads_traffic (node_from,node_to,load_speed,junction_speed,max_speed,"
                "lightVehicleCount,heavyVehicleCount) "
                "VALUES ({0},{1},{2},{3},{4},{5},{6});".format(
                    node_from,
                    node_to,
                    load_speed,
                    junction_speed,
                    max_speed,
                    traffic_cars,
                    traffic_trucks,
                )
            )
        sql_insert_strings_noisy_roads.append(sql_insert_string)

    return sql_insert_strings_noisy_roads

# returns a wkt string for a multipolygon containing all buildings
def get_buildings_geom_as_wkt(buildings_gdf: gpd.GeoDataFrame) -> str:
    # simplify complex geometries to speed up calculation and avoid hickups with spatial db.
    buildings_gdf.geometry = buildings_gdf.geometry.simplify(0.1)

    # calculation of noise results is 5sec. faster if building geometries are provided as single multipolygon
    return f"'{buildings_gdf.geometry.unary_union}'"

# create nodes for all roads - nodes are connection points of roads
def create_nodes(all_roads):
    nodes = []
    for road in all_roads:
        coordinates_start_point = road.get_start_point()
        nodes.append(coordinates_start_point)
        coordinates_end_point = road.get_end_point()
        nodes.append(coordinates_end_point)
    unique_nodes = []

    # filter for duplicates
    for node in nodes:
        if not any(
            numpy.array_equal(node, unique_node) for unique_node in unique_nodes
        ):
            unique_nodes.append(node)
    return unique_nodes

def get_node_for_point(point, nodes):
    dict_of_nodes = {i: nodes[i] for i in range(0, len(nodes))}
    for node_id, node in dict_of_nodes.items():
        if point == node:
            return node_id
    print("could not find node for point", point)
    exit()

def get_road_type(road_properties):
    # if not in road types continue
    for output_road_type in road_types_iffstar_noise_modelling.keys():
        if road_properties["road_type"] == output_road_type:
            return road_types_iffstar_noise_modelling[output_road_type]

    print("no matching noise road_type_found for", road_properties["road_type"])
    return 0

def get_insert_query_for_road(road, nodes):
    node_from = get_node_for_point(road.get_start_point(), nodes)
    node_to = get_node_for_point(road.get_end_point(), nodes)

    sql_insert_string = (
        "INSERT INTO roads_geom (the_geom,NUM,node_from,node_to,road_type) "
        "VALUES (ST_GeomFromText('{0}'),{1},{2},{3},{4});".format(
            road.get_geom(),
            road.get_road_id(),
            node_from,
            node_to,
            road.get_road_type_for_query(),
        )
    )

    return sql_insert_string


from django import forms

class PanelistForm(forms.Form):
    email = forms.EmailField(label='Email address')
    name = forms.CharField(label='Name', max_length=280)
    returning = forms.BooleanField(
        label='Returning Panelist',
        help_text="Have you been a panelist at ConFusion before?",
        required=False)
    bio = forms.CharField(
        label='Bio',
        widget=forms.Textarea(attrs={'rows':3}),
        help_text="New panelist? Tell us a little about yourself. (This is for our edification only and won't be published).",
        required=False)

class PanelSubmissionForm(forms.Form):
    email = forms.EmailField(
        label='Contact Email',
        max_length=280,
        help_text="If you'd like to be on or moderate this panel, please use the same email you used on the Panelist Signup Form.")
    title = forms.CharField(
        label='Panel Title',
        max_length=280,
        help_text="Panel titles and descriptions may be edited for style, clarity, or fun.")
    description = forms.CharField(
        label='Panel Description',
        widget=forms.Textarea(attrs={'rows':4}),
        help_text="This should be a draft of the panel description as it would appear in the program book.")
    notes = forms.CharField(
        label='Notes for ConFusion Planners',
        widget=forms.Textarea(attrs={'rows':3}),
        required=False,
        help_text="Does this panel require a/v? Did you pre-arrange it with other panelists? Any other notes for us?"
        )


"""Inventory Category Python model

"""

__author__ = "Lujain Battiki, Rawan Amro"
__copyright__ = "Copyright 2020, POSRocket"
__credits__ = ["Lujain Battiki, Rawan Amro"]
__license__ = "GPL"
__version__ = "1.0.0"
__maintainer__ = "Lujain Battiki, Rawan Amro"
__email__ = "Lanuchpad@posrocket.com"
__status__ = "Beta"

class InventoryCategoryModel:
    """mapper class for Inventory Category object from Json Dict

    """
    id: str
    name: str
    trackables_count: float

    def __init__(self, id=None, name=None, trackables_count=None, **kwargs):
        """map a dict to Inventory Category object

        :param kwargs: Inventory Category json dict
        """
        self.id = id
        self.name = name
        self.trackables_count = trackables_count

    def __str__(self) -> str:
        """ String representation for the Inventory Category model

        :return: Inventory Category name
        """
        return f'{self.name}'


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.1
#   kernelspec:
#     display_name: Python 3.8.10 ('ml4t')
#     language: python
#     name: python3
# ---

# + [markdown] id="sNw45BR0m62X" pycharm={"name": "#%% md\n"}
# # Importing dependencies
# We used the following libraries in the first task:
# - `pandas` to manipulate the data
# - `scikit-learn` for imputing and scaling the data
# - `seaborn` and `matplotlib` for visualization.

# + id="4CRS48qom62Z" pycharm={"name": "#%%\n"}
import pandas as pd
from sklearn import preprocessing
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns

# + [markdown] id="XyY6wwS-m62a" pycharm={"name": "#%% md\n"}
# # Data exploration
# Test
#
# Let's load the training dataset from the corresponding .csv file.
# Since we know that the columns represent the mean/standard deviation of the positions and angles of the 60 points, respectively, followed by the label name and code, let us rename the columns accordingly to allow for easier reading.

# + id="KNoDyk_fm62a" pycharm={"name": "#%%\n"}
def load_data():
  training_data = pd.read_csv('train-final.csv', header=None)

  name_mappings = {
      # Feature columns
      **{i:f'positions_mean_{i}' for i in range(60)},
      **{i:f'positions_std_{i}' for i in range(60, 120)},
      **{i:f'angles_mean_{i}' for i in range(120, 180)},
      **{i:f'angles_std_{i}' for i in range(180, 240)},
      # Label columns
      **{240: 'label_name', 241: 'label_code'},
  }

  training_data.rename(name_mappings, axis=1, inplace=True)
  training_feature_columns = training_data.columns[:-2]

  training_features = training_data[training_feature_columns]
  training_labels = training_data.label_name

  return training_features, training_labels

training_features, training_labels = load_data()

# + [markdown] id="_aYlx2ckm62b" pycharm={"name": "#%% md\n"}
# Let's show some of the data

# + colab={"base_uri": "https://localhost:8080/", "height": 300} id="paURPIp3m62b" outputId="4b22de09-f76e-4f25-8e2d-9d356e7ca70d" pycharm={"name": "#%%\n"}
training_features.head()

# + [markdown] id="DYPTNtFmm62c" pycharm={"name": "#%% md\n"}
# How many different labels do we have in the training dataset?

# + colab={"base_uri": "https://localhost:8080/"} id="l8hDw1bgm62c" outputId="7f333633-1d1c-4483-8fbc-0c1f9acf96a4" pycharm={"name": "#%%\n"}
number_of_classes = training_labels.nunique()
number_of_classes

# + [markdown] id="2NwxuXvqm62c" pycharm={"name": "#%% md\n"}
# Now let's take a look at how many occurrences we have of each label.

# + colab={"base_uri": "https://localhost:8080/", "height": 322} id="_O-CkrFfm62d" outputId="ad9afcd5-4836-4ad1-fa9b-7d7a5aa5954e" pycharm={"name": "#%%\n"}
training_labels.value_counts().plot(kind='bar', figsize=(10, 4))

# + [markdown] id="YlISrYJpm62d" pycharm={"name": "#%% md\n"}
# We can see that `child` is the most common label in the training dataset and that `go` is the least common label.

# + [markdown] id="AhOTmkwRm62d" pycharm={"name": "#%% md\n"}
# Now let's look for columns that have missing values. The missing values are in the following columns (along with the missing value count):
#

# + colab={"base_uri": "https://localhost:8080/"} id="rTfhsEE8m62f" outputId="62eb3157-0b72-4f9c-a1bf-91cf110ced34" pycharm={"name": "#%%\n"}
# Look for columns that have missing values
columns_null_sum = training_features.isnull().sum()
columns_with_nulls = columns_null_sum[columns_null_sum > 0]

print(
    "Total amount of missing values in the dataframe:",
    training_features.isnull().sum().sum()
)
print(
    "Missing values in the following column indexes (and missing value count):"
)
print(columns_with_nulls)

# + [markdown] id="yprFNiVkm62j" pycharm={"name": "#%% md\n"}
# Some classifiers are more sensitive to the range, mean & outliers of the features, such as linear regression models, for example.
# In order to be able to train a wide range of classifiers and compare them, we will need to preprocess the data for scaling and outlier treatment.

# + [markdown] id="t9yB47dLm62k" pycharm={"name": "#%% md\n"}
# Let's see if the dataset also contains outliers. There are quite a few way to detect outliers (Source:
# [Outlier detection methods in Scikit-Learn](https://scikit-learn.org/stable/modules/outlier_detection.html)):
# - Isolation forest
# - Local outlier factor
# - One-class support vector machine (SVM)
# - Elliptic envelope
#
# We start by doing a boxplot for all features to get a visual indication of the outlier situation.

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="r77J-ZiOm62k" outputId="0bddf0cf-5a7c-4708-823f-a73359d67ac3" pycharm={"name": "#%%\n"}
training_features.boxplot(figsize=(18,7))
plt.xticks([1], [''])

# + [markdown] id="m4tCX_XRnYRS" pycharm={"name": "#%% md\n"}
# Based on the boxplot, there appears to be many columns with outliers. Many classifiers, e.g. linear classifiers like Logistic Regression will not handle outliers well, so we need to find a way to handle also outliers.

# + [markdown] id="J1sVekRXoT4Q" pycharm={"name": "#%% md\n"}
# # Methods

# + [markdown] id="Kq-ne6mOolOy" pycharm={"name": "#%% md\n"}
# ## Outliers
#
# As we saw in the boxplot above, there are many columns with outliers. And while there are many methods to detect outliers,let's begin with just identifying the values that are farthest from the mean.
#
# A simple approach is to identify the values that lie outside of 3$\sigma$ (that is, three times the standard deviation) as outliers, and drop the rows that have at least one outlier. Let's give it a try.

# + id="sOaxwVV_m62k" pycharm={"name": "#%%\n"}
#training_features_outliers_marked = training_features[abs(training_features) <= 3]
from scipy import stats

training_features_outliers_marked = training_features[
    np.abs(stats.zscore(training_features.fillna(training_features.mean()))) < 3
]

# + colab={"base_uri": "https://localhost:8080/", "height": 300} id="gGFZoxbwpqaE" outputId="e4bbb462-72f6-46b3-e83a-1d5e98fe2561" pycharm={"name": "#%%\n"}
training_features_outliers_marked.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 281} id="j2OIbvJTm62l" outputId="9170fed3-adf5-4aab-e724-f6e342249732" pycharm={"name": "#%%\n"}
training_features_outliers_removed = training_features_outliers_marked.dropna()
training_features_outliers_removed.boxplot(figsize=(18,7))
plt.xticks([1], [''])
print("Number of rows left", training_features_outliers_removed.shape[0])

# + [markdown] id="r8fsvlJZm62l" pycharm={"name": "#%% md\n"}
# The boxplot now looks better, except for the second part (columns 60 to 120), which is `positions_std_i`.
# We can also see that if we remove all the rows with at least one detected outlier, we are left with less than half of the original data! This is due to the large number of features.
#
# We need another method for this dataset, let's instead cap the outliers to 3 sigma.

# + id="-artO4WUm62l" pycharm={"name": "#%%\n"}
def pipeline_outliers(df, std_cap=3):
  df = df.copy()

  for column in df.columns:

    mean = df[column].mean(skipna = True)
    std = df[column].std(skipna = True)

    df[column] = np.clip(df[column], -(mean + std_cap*std), mean + std_cap*std)

  return df

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="oDGrLk2Jm62l" outputId="3145fc38-ed0e-4bd3-f389-77c2a98ec641" pycharm={"name": "#%%\n"}
df = pipeline_outliers(training_features)
df.boxplot(figsize=(18,7))
plt.xticks([1], [''])

# + [markdown] id="QHeOA5iqm62g" pycharm={"name": "#%% md\n"}
# ## Missing data

# + [markdown] id="T-HJFZs8m62g" pycharm={"name": "#%% md\n"}
# As we saw above, there are 6 columns that have missing values (3 or 4 missing values each). Many classifiers do not handle missing values directly, such as Logistic Regression and SVM, for example. As such we need to find a way to manage the missing values.

# + [markdown] id="ENEICm2Om62g" pycharm={"name": "#%% md\n"}
# There are many different ways of handling missing values and we will explore a few of them here. To get started, let's examine the features/columns that contain missing data. The two visualizations chosen for each of the features/columns are:
#
# - *Histogram* - This will give a good indication of the distribution, for example if it appears to be normal.
# - *Boxplot* - We get some additional information from the boxplot showing the median, quartiles as well as outliers.
#
# Let's plot the distributions for the columns with missing data:

# + colab={"base_uri": "https://localhost:8080/", "height": 332} id="3-dLLDaPm62g" outputId="66d2de3f-72c3-4e08-80cc-832bca38404b" pycharm={"name": "#%%\n"}
def plot_distributions_for_columns(dataframe, columns):
    # Plot distributions for each of the columns that have missing values
    figure, axes = plt.subplots(2, len(columns), figsize=(12, 6))

    for index, column in enumerate(columns):
        # plot a histogram of the column for the first row
        dataframe[column].plot(
            kind='hist', ax=axes[0, index], title=column, bins=15
        )
        # Do a box plot as well
        sns.boxplot(y=dataframe[column], ax=axes[1, index]).set_title(column)

    plt.tight_layout()

plot_distributions_for_columns(training_features, columns_with_nulls.index)

# + [markdown] id="TBDoZnucm62h" pycharm={"name": "#%% md\n"}
# The distributions are quite different, which mean we may need different imputation techniques for each column.
# For example, the columns `positions_mean_9` and `positions_mean_15` are clearly not normal distributions and replacing missing values with the mean would likely not be ideal.
# For instance in the case of `positions_mean_9` the mean is close to 0 where few other samples are, moreover this feature may even need to be split into two separate features as it appears to be the combination of two gaussian distributions.
#
# Also, we do not know if the missing values themselves have a significance, i.e. we might want to create a separate column to indicate that a missing value is present or not. There are relatively few rows that have missing values, though, that may limit the usefulness of this technique and usefulness will also depend on the classifier used in the end.
#
# For now, we will implement support for the following imputation strategies:
# - Replacing with **mean**
# - **Drop the rows** containing at least one missing value
# - **K-Nearest Neighbour (KNN)** imputation, i.e. use the mean value of the K nearest neighbours

# + id="AjTqPq0Qm62h" pycharm={"name": "#%%\n"}
from sklearn.impute import KNNImputer, SimpleImputer

def impute(dataframe, imputer_class, **kwargs):
    imputer = imputer_class(**kwargs)
    dataframe = dataframe.copy()
    dataframe[dataframe.columns] = imputer.fit_transform(dataframe.values)
    return dataframe

# KNN imputation
def impute_knn(dataframe):
    return impute(dataframe, KNNImputer, n_neighbors=2, weights='uniform')

# Drop rows
def impute_drop_rows(dataframe):
    return dataframe.dropna()

# Mean imputation
def impute_mean(dataframe):
    return impute(
        dataframe, SimpleImputer, missing_values=np.nan, strategy='mean'
    )

# + [markdown] id="o09ETbgLm62h" pycharm={"name": "#%% md\n"}
# Let's compare the mean and KNN imputation methods.

# + id="C8MiBZ8Jm62h" pycharm={"name": "#%%\n"}
columns_null_sum = training_features.isnull().sum()
columns_with_nulls = columns_null_sum[columns_null_sum > 0]

# + id="PUSYg7HNm62h" pycharm={"name": "#%%\n"}
#training_features = training_data[training_feature_columns]
all_na_values = training_features.isna()

# + [markdown] id="FTGyuLvum62h" pycharm={"name": "#%% md\n"}
# First let's execute both imputations individually.
# Starting with the mean imputation, the imputed values are the following:

# + colab={"base_uri": "https://localhost:8080/"} id="v14LUk-Km62i" outputId="3a6b15cc-155c-4759-bd5f-40c0ce506048" pycharm={"name": "#%%\n"}
training_data_mean_imputed = impute_mean(training_features)
training_data_mean_imputed.values[all_na_values]

# + [markdown] id="lGKhqpOsm62i" pycharm={"name": "#%% md\n"}
# While the KNN-imputed values look like this:

# + colab={"base_uri": "https://localhost:8080/"} id="FscTkmDLm62i" outputId="c888a8ae-11c4-450b-d89f-b553060a65ea" pycharm={"name": "#%%\n"}
training_data_knn_imputed = impute_knn(training_features)
training_data_knn_imputed.values[all_na_values]

# + [markdown] id="EtCVnW3um62i" pycharm={"name": "#%% md\n"}
# ### Comparison of mean and KNN
# Now it is time to compare the results and visualize the differences on a histogram

# + colab={"base_uri": "https://localhost:8080/", "height": 265} id="Ne7NYuHXm62i" outputId="53fa39df-ee95-4aac-a6d6-c46eef07f36a" pycharm={"name": "#%%\n"}
differences = (
    training_data_mean_imputed.values[all_na_values]
    - training_data_knn_imputed.values[all_na_values]
)
plt.hist(differences);

# + [markdown] id="rQTrrd-qm62i" pycharm={"name": "#%% md\n"}
# We can conclude that the differences between the methods are not significant, the mode is close to zero, for most of the missing values the two methods give very similar imputed values - more than half of the values are in the range [-0.1,0.1]
#
# Now let's look at how the difference is distributed for each column using a boxplot and a swarm plot.
#

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="WGeg5UJDm62j" outputId="7b4edc51-7010-4610-f78f-8f5fcdde7b0b" pycharm={"name": "#%%\n"}
na_rows, na_columns = np.where(all_na_values)
dataframe_differences_columns = pd.DataFrame(
    {'diff': differences, 'column': training_features.columns[na_columns]}
)

fig, axs = plt.subplots(ncols=2)

axis = sns.boxplot(
    x='column', data=dataframe_differences_columns, y='diff', ax=axs[0]
)
axis.set_xticklabels(axis.get_xticklabels(), rotation=90);

axis = sns.swarmplot(
    x='column', data=dataframe_differences_columns, y='diff', ax=axs[1]
)
axis.set_xticklabels(axis.get_xticklabels(), rotation=90);

fig.tight_layout()

# + [markdown] id="BeH5QKZUm62j" pycharm={"name": "#%% md\n"}
# When comparing the imputed values between the 'mean' and 'KNN' approaches, we see that they produce very similar values for columns `positions_mean_7` and `positions_mean_16` but larger differences by varying degrees for the other columns. The largest (absolute) deviance is for `positions_mean_9`.
#
#
# Based on this it is likely that we may need to employ different imputation techniques depending on the column/feature.

# + [markdown] id="7EvO3QTJm62j" pycharm={"name": "#%% md\n"}
# For now, however, let's use KNN imputation and make sure that after the imputation there are no more missing values in our dataset. We will bring these three different methods into Task 2 of the project.
#

# + id="cJCnEdNqok3O" pycharm={"name": "#%%\n"}
# Let's create a pipeline function that uses the KNN imputation
def pipeline_missing_values(df):
  df = df.copy()

  columns_null_sum = df.isnull().sum()
  columns_with_nulls = columns_null_sum[columns_null_sum > 0]

  df[columns_with_nulls.index] = impute_knn(df[columns_with_nulls.index])
  assert df.isnull().sum().sum() == 0

  return df

# + [markdown] id="UIZibV1-m62j" pycharm={"name": "#%% md\n"}
# ## Scaling
# First we will take a look at how the data looks by feature/column. While the dataset contains a lot of features, we can use a boxplot to get an overview understanding of how the different columns compare.

# + [markdown] id="1UU_s9ajm62j" pycharm={"name": "#%% md\n"}
# Let's make a boxplot for every feature to get an overview of how they all relate in terms of range & centre. No need to have labels for the feature names, we just want to show all of them in one plot.

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="zZ8mc_GWm62k" outputId="2af47191-5b38-4fc8-f431-1002b2a0cc98" pycharm={"name": "#%%\n"}
training_features.boxplot(figsize=(18, 7))
plt.xticks([1], [''])

# + [markdown] id="IabX4HxSm62k" pycharm={"name": "#%% md\n"}
# The columns are clearly not all scaled to the same range and they have different means. As such a scaler that centers the mean and normalizes (scale to the variance) may be suitable. Let's use scikit's `StandardScaler` for this.
#
# The `StandardScaler`normalizes the data so that the mean becomes zero, and the variance one, i.e. the scaled dataset follows a *standard* normal distribution.

# + colab={"base_uri": "https://localhost:8080/", "height": 300} id="aeXO-9R9m62k" outputId="3e3b1dda-14ed-4bb7-f572-190ac361ba85" pycharm={"name": "#%%\n"}
def pipeline_scale(dataframe):
    scaler = preprocessing.StandardScaler()
    scaled_values = scaler.fit_transform(dataframe.values)
    return pd.DataFrame(scaled_values)

training_features_scaled = pipeline_scale(training_features)
training_features_scaled.head()

# + [markdown] id="z3ewuEEFm62k" pycharm={"name": "#%% md\n"}
# Looks good, i.e. the mean is about 0 and the standard deviation is around 1. All the columns have now been scaled. Let's rerun the boxplot.

# + colab={"base_uri": "https://localhost:8080/", "height": 335} id="W1uf0D3QsqPH" outputId="8c9271bc-c2aa-4a2c-e1fd-10d410f6655d" pycharm={"name": "#%%\n"}
training_features_scaled.boxplot(figsize=(18, 7))
plt.xticks([1], [''])

# + [markdown] id="Unie-y19uJpv" pycharm={"name": "#%% md\n"}
# # Pipeline

# + [markdown] id="wTBLe3qVvs2f" pycharm={"name": "#%% md\n"}
# In the Methods section above we have defined the pipeline methods needed for handling outliers, missing data and scaling. Let's put them all together.

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="1kLsdpktv7pS" outputId="ca6ad9cb-0932-4d61-d04c-84b137dc5174" pycharm={"name": "#%%\n"}
# Load the data
training_features, training_labels = load_data()

# Plot before pipeline
training_features.boxplot(figsize=(18, 7))
plt.xticks([1], [''])

# + [markdown] id="NqTkoxZQv7EW" pycharm={"name": "#%% md\n"}
# Run the pipeline and rerun the boxplot on the resulting dataset

# + colab={"base_uri": "https://localhost:8080/", "height": 281} id="8TDqvcNTzgQ4" outputId="9b00742b-403e-41bd-a19f-badf2f8cffb3" pycharm={"name": "#%%\n"}
df = pipeline_outliers(training_features, std_cap=3)
df = pipeline_missing_values(df)
df = pipeline_scale(df)

df.boxplot(figsize=(18, 7))
plt.xticks([1], [''])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Mar  7 21:08:18 2021

@author: ztzhou
Merge sort the PDFs to combine the 20 seeds(realizations) outputs together
and then take the average.
pdf_simu['pdf6'][seed3]: pdf 6 for seed 3;
pdf_simu['pdf6']['mean']: the combined pdf6 from all seeds;
"""

import numpy as np
import os
import pickle as pkl
import matplotlib.pyplot as plt
import natsort
import time
from tqdm import tqdm
import scipy.interpolate
import matplotlib.ticker as mticker

def sort_merge(pdf1, pdf2):
    '''pdf1,2: (x, 2) arrays representing pdfs, this algorithm merge pdf1 and
    pdf2 by combining 2 arrays with x sorted.'''
    new_pdf = []
    i_1 = 0
    i_2 = 0
    while i_1 <len(pdf1) and i_2 < len(pdf2):
        temp_1, temp_2 = pdf1[i_1][0], pdf2[i_2][0]
        if temp_1 < temp_2:
            if len(new_pdf)>0 and temp_1 == new_pdf[-1][0]:
                new_pdf[-1][1] += pdf1[i_1][1]
            else:
                new_pdf.append(pdf1[i_1])
            i_1 += 1
        elif temp_1 == temp_2:
            if len(new_pdf)>0 and temp_1 == new_pdf[-1][0]:
                new_pdf[-1][1] += pdf1[i_1][1] + pdf2[i_2][1]
            else:
                new_pdf.append([pdf1[i_1][0], pdf1[i_1][1] + pdf2[i_2][1]])
            i_1 += 1
            i_2 += 1
        else:
            if len(new_pdf)>0 and temp_2 == new_pdf[-1][0]:
                new_pdf[-1][1] += pdf2[i_2][1]
            else:
                new_pdf.append(pdf2[i_2])
            i_2 += 1
    while i_1 < len(pdf1):
        if len(new_pdf)>0 and pdf1[i_1][0] == new_pdf[-1][0]:
            new_pdf[-1][1] += pdf1[i_1][1]
        else:
            new_pdf.append(pdf1[i_1])
        i_1 += 1
    while i_2 < len(pdf2):
        if len(new_pdf)>0 and pdf2[i_2][0] == new_pdf[-1][0]:
            new_pdf[-1][1] += pdf2[i_2][1]
        else:
            new_pdf.append(pdf2[i_2])
        i_2 += 1
    return new_pdf

def pdf2cdf(pdf):
    '''Normalize the pdf to sum to 1, then turn into cdf'''
    cdf = pdf.copy()
    cdf[:,1] = cdf[:,1]/sum(cdf[:,1])
    for i in range(len(cdf)-1):
        cdf[i+1,1] += cdf[i, 1]

    return cdf

def normalize(pdf):
    norm_pdf = pdf.copy()
    norm_pdf[:, 1] = norm_pdf[:,1]/np.sum(norm_pdf[:,1])
    return norm_pdf

def merge_pdf(dir='', filename='combined_seeds', seed_n=20,):
    pdf_simu = {'pdf'+str(i):{} for i in range(1,10001)}
    os.chdir(dir)
    seeds_list = natsort.natsorted(os.listdir(dir+'/')) #sort seeds by the number
    seeds_list = [i for i in seeds_list if i.startswith('OutputSimu')]
    for f_i in tqdm(range(len(seeds_list)), unit="seeds"):
        seeds_folder = seeds_list[f_i]
        seed_ind = seeds_folder[-2:]
        ##pickle load all pdfs for 1 seed, with the index referring to the CD pair.
        with open (seeds_folder+'/ind_pdf.pkl', 'rb') as file:
            [index, pdf] = pkl.load(file)
        ##update pdf_simu[pdfn][seedsm] for n in index
        for i,ind in enumerate(index):
            pdf_simu['pdf'+str(ind)]['seed'+str(seed_ind)] = pdf[i]
            if 'mean' not in pdf_simu['pdf'+str(ind)].keys():
                pdf_simu['pdf'+str(ind)]['mean']= []
            pdf_simu['pdf'+str(ind)]['mean'] = sort_merge(
                pdf_simu['pdf'+str(ind)]['mean'], pdf[i]
                )
    for pdf_ind in pdf_simu.keys():
        pdf_simu[pdf_ind]['mean'] = normalize(np.array(
            pdf_simu[pdf_ind]['mean'])
            )
    with open(filename, 'wb') as file:
        pkl.dump([pdf_simu], file)

    return pdf_simu

def pdf2ICDF(pdf, x=np.round(np.linspace(0.01, 0.99, 50), 2)):
    '''Convert pdf array to inverse cdf(ICDF)'''
    pdf = np.array([p for p in pdf if p[1]!=0])
    cdf = pdf2cdf(pdf)
    y_interp = scipy.interpolate.interp1d(
        cdf[:,1], cdf[:,0], fill_value="extrapolate"
        )
    y = y_interp(x)
    return y

if __name__ == '__main__':
    '''merge pdf for each simulation'''
    dirs = [
        '/Users/zitongzhou/Desktop/2021-04-16.tmp/Uncorrelated_100part',
        '/Users/zitongzhou/Desktop/2021-04-16.tmp/Uncorrelated_1000part']
    filenames = [
        # 'corr_100_20pdf.pkl',
        # 'corr_1000_20pdf.pkl',
        'uncorr_100_20pdf.pkl',
        'uncorr_1000_20pdf.pkl',
        ]
    # start = time.time()
    # for dir_simu, filename in zip(dirs, filenames):
    #     pdf_simu = merge_pdf(dir=dir_simu, filename=filename, )
    # end = time.time()
    # print('Combining seeds for {} folders took '.format(len(dirs)), end-start, ' seconds.')
    # Combining seeds for 4 folders took  689.9622631072998  seconds.
    '''pickle C,D and ICDF for uncorrelated simulations, 100 particles'''
    input_filename = '/Users/zitongzhou/Desktop/FRACTURE/seeds20/C_d.txt'
    input_CD = np.loadtxt(fname = input_filename)

    ##compute the minumal number of connected cases for each file
    n_conn = {}

    '''Randomly sample 6 simulations,
    from uncorrelated 100 and 1000 dataset for further comparison'''
    pdf_samples = {}
    np.random.seed(88)
    rand_i = np.random.randint(1, 10001, 6)
    for ind,f in enumerate(filenames):
        os.chdir(dirs[ind])
        pdf_samples[f] = {}
        with open(f, 'rb') as file:
            [pdf_simu] = pkl.load(file)
            n_conn[f] = {i: len(pdf_simu[i].keys())-1 for i in pdf_simu.keys()}
        for i in rand_i:
            assert pdf_simu['pdf'+str(i)]['mean'] is not None, \
                '{} is not in {}'.format(i, f)
            pdf_samples[f][str(i)] = pdf_simu['pdf'+str(i)]

    # with open('uncorr_100_20pdf_conn.pkl', 'wb') as file:
    #     pkl.dump([C_conn, D_conn, num_conn], file)
    for i in n_conn.keys():
        print(
            i+':', min(list(n_conn[i].values()))
        )

    ''' plot the histogram of the connected cases for each simulation'''
    os.chdir('/Users/zitongzhou/Desktop/FRACTURE/images')
    SMALL_SIZE = 12
    MEDIUM_SIZE = 14
    BIGGER_SIZE = 16
    plt.rc('font',**{'family':'serif','serif':['Times']})
    plt.rcParams['text.usetex'] = True
    plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
    plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title
    plt.rc('axes', labelsize=SMALL_SIZE)    # fontsize of the x and y labels
    plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
    plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
    plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
    plt.rc('figure', titlesize=BIGGER_SIZE)

    markerstyles = ['.', '1', '*', 'x','d']
    colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd']

    ''' plot the histogram of the connected cases for each simulation'''
    figsize = (3.5, 4)
    cols = 1
    rows = 1
    fig_cdf, ax_cdf = plt.subplots(rows, cols, figsize=figsize,
                              constrained_layout=True,
                             # sharey=True,
                             )

    figsize = (7, 4)
    cols = 2
    rows = 1
    fig, axs = plt.subplots(rows, cols, figsize=figsize,
                              constrained_layout=True,
                             # sharey=True,
                             )

    # fig2, axs2 = plt.subplots(rows, cols, figsize=figsize,
    #                           # constrained_layout=True,
    #                           # sharey=True,
    #                           )
    # axs2_cdf = axs2.twinx()
    axs_all = [axs.flat[0], axs.flat[1]]
               # , axs2.flat[0], axs2.flat[1]]
    titles = ['100 particles', '1000 particles', ]
              # '100 particles', '1000 particles']!
    suptitles = ['corr_simu_hist', 'uncorr_simu_hist']
    linestyles = ['-', '-.', ':', '--', '(3, 1, 1, 1, 1, 1)', 'dashdot']
    labels = ['100', '1000']
    for i, filename in enumerate(filenames):
        data = np.array(list(n_conn[filenames[i]].values()))
        d = np.diff(np.unique(data)).min()
        left_of_first_bin = data.min() - float(d)/2
        right_of_last_bin = data.max() + float(d)/2
        bins = np.arange(left_of_first_bin, right_of_last_bin + d, d)
        n, bins, patches = axs_all[i].hist(
            data,
            bins=bins,
            ec='k',
        )

        # axs_cdf = axs_all[i].twinx()
        cdf_data = n.cumsum()
        cdf_data = cdf_data/cdf_data.max()
        cdf_bin = (bins[:-1]+bins[1:])//2
        # axs_cdf.plot(cdf_bin, cdf_data, color=colors[1])
        ax_cdf.plot(cdf_bin, cdf_data, linestyle = linestyles[i],
                    color=colors[i], label=labels[i])
        axs_all[i].set_xlabel('connected cases in 20')
        axs_all[i].set_ylabel('count')
        # axs_cdf.set_ylabel('CDF')
        axs_all[i].locator_params(axis='x', integer=True)
        axs_all[i].set_ylim(bottom=-100, top=n.max()+1000)
        axs_all[i].set_xlim(left=left_of_first_bin)
        axs_all[i].set_xticks(np.arange(data.min(), data.max()+1))
        axs_all[i].set_title(titles[i])
        for m in range(len(n)):
            axs_all[i].text(bins[m]+0.2,n[m]+100,str(int(n[m])),
                            rotation='vertical',)
    # fig.savefig(suptitles[0]+'.pdf')
    ax_cdf.legend(title='nb\_part')
    # fig_cdf.savefig('hist_cdf.pdf')
    # fig.savefig(suptitles[1]+'.pdf')

    ## distribution of connected cases
    fig, axs = plt.subplots(1, 2, figsize=(7, 4),  sharey='row', sharex='col')

    for f, ax in zip(filenames, axs.flat):
        C_conn = []
        D_conn = []
        num_conn = []
        for key in n_conn[f].keys():
            ind = int(''.join([i for i in key if i.isnumeric()]))
            C_conn.append(input_CD[ind-1,0])
            D_conn.append(input_CD[ind-1, 1])
            num_conn.append(n_conn[f][key])
        C_conn = np.array(C_conn)
        D_conn = np.array(D_conn)
        num_conn = np.array(num_conn)
        c01map = ax.scatter(C_conn, D_conn, c=num_conn, cmap='jet', marker=",", s = 0.5,
                            # interpolation='nearest',
                  # extent=[C_d[:,0].min(), C_d[:,0].max(), C_d[:,1].min(), C_d[:,1].max()],
                  vmin=num_conn.min(), vmax = num_conn.max(),
                  # origin='lower',aspect='auto'
                  )
        ax.set_xlim(2.5, 6.5)
        ax.set_xticks(np.linspace(2.5, 6.5, 5, endpoint=True))
        ax.set_ylim(1.0, 1.3)
        ax.set_xlabel('C [-]')
        ax.set_ylabel('D [-]')
        v1 = np.arange(np.min(num_conn), np.max(num_conn)+1,)
        fig.colorbar(c01map, ax=ax, fraction=0.05, pad=0.04,ticks=v1,)

        # ax1.set_ylim([0, 12])
        # ax.set_xlabel('C [-]')
        # ax.set_ylabel('D [-]')
    plt.tight_layout()
    fig.savefig('conn_density.pdf', format='pdf',bbox_inches='tight')
    plt.show()
    # end2 = time.time()

    '''plot 2 figures, 1st for comparing 20 seeds for 6 simulations for 100
    particles, 2nd for comparing models with 100 particles and 1000 particles,
    3rd for comparing 20 seeds for 6 simulations for 1000 particles'''
    figsize = (10, 8)
    cols = 3
    rows = 2
    fig1, axs = plt.subplots(rows, cols, figsize=figsize,
                             constrained_layout=True,
                             # sharey='row',
                             # sharex='col',
                             )
    axs = axs.flat

    fig2, axs2 = plt.subplots(rows, cols, figsize=figsize,
                             constrained_layout=True,
                             # sharey='row',
                             # sharex='col',
                             )
    axs2 = axs2.flat

    fig3, axs3 = plt.subplots(rows, cols, figsize=figsize,
                             constrained_layout=True,
                             # sharey='row',
                             # sharex='col',
                             )
    axs3 = axs3.flat

    # filename1 = filenames[2]
    # filename2 = filenames[3]
    filename1 = filenames[0]
    filename2 = filenames[1]

    for ind, ax, ax2, ax3 in zip(rand_i, axs, axs2, axs3):
        f1 = pdf_samples[filename1][str(ind)]
        f2 = pdf_samples[filename2][str(ind)]
        for key in f1.keys():
            if key != 'mean':
                pdf = f1[key]
                pdf = np.array([p for p in pdf if p[1]!=0])
                ax.plot(
                    np.log(pdf[:,0]), pdf2cdf(pdf)[:,1],
                    label=''.join([i for i in key if i.isnumeric()]).zfill(2)
                    )

            else:
                pdf1 = f1[key]
                pdf2 = f2[key]
                ax2.plot(
                    np.log(pdf2[:,0]), pdf2cdf(pdf2)[:,1],
                    '-', color=colors[0],linewidth=2.5,
                    label='1000'
                    )
                ax2.plot(
                    np.log(pdf1[:,0]), pdf2cdf(pdf1)[:,1],
                    '-.', color=colors[1],linewidth=2.5,
                    label='100'
                    )
        for key in f2.keys():
            if key != 'mean':
                pdf1000 = f2[key]
                pdf1000 = np.array([p for p in pdf1000 if p[1]!=0])
                ax3.plot(
                    np.log(pdf1000[:,0]), pdf2cdf(pdf1000)[:,1],
                    label=''.join([i for i in key if i.isnumeric()]).zfill(2)
                    )
        # ax.legend(title='seed', loc='lower right', ncol=2)#, mode="expand")
        # ax3.legend(title='seed', loc='lower right', ncol=2)
        # ax2.legend(loc="lower right")#title='nb\_part=')
        ax2.legend().get_frame().set_edgecolor('b')
        ax2.legend().get_frame().set_linewidth(0.0)
        ax2.legend(loc="lower right")

        ax.set_title(
            '(C,D)=({:.2f}, {:.2f})'.format(input_CD[ind-1,0], input_CD[ind-1, 1])
            )
        ax3.set_title(
            '(C,D)=({:.2f}, {:.2f})'.format(input_CD[ind-1,0], input_CD[ind-1, 1])
            )
        ax2.set_title(
            '(C,D)=({:.2f}, {:.2f})'.format(input_CD[ind-1,0], input_CD[ind-1, 1])
            )
        # ax.set_xlabel('log(particle arrivial time)')
        # ax.set_ylabel('CDF')
        ax.xaxis.set_major_locator(mticker.LinearLocator(6))
        ax.xaxis.set_major_formatter('{x:.00f}')
        ax.set_xlim([15,30])
        # ax2.set_xlabel('log(particle arrivial time)')
        # ax2.set_ylabel('CDF')
        ax2.xaxis.set_major_locator(mticker.LinearLocator(6))
        ax2.xaxis.set_major_formatter('{x:.00f}')
        ax2.set_xlim([15,38])
        # ax3.set_xlabel('log(particle arrivial time)')
        # ax3.set_ylabel('CDF')
        ax3.xaxis.set_major_locator(mticker.LinearLocator(6))
        ax3.xaxis.set_major_formatter('{x:.00f}')
        ax3.set_xlim([15,30])
    # fig1.savefig('uncorr20seeds.pdf')
    fig2.savefig('uncorr100_1000par.pdf')
    # fig3.savefig('uncorr20seeds_1000.pdf')

    '''plot a figure of mean cdf for 6 simulations, and the ICDF'''
    figsize = (7, 4)
    cols = 2
    rows = 1
    fig, axs = plt.subplots(rows, cols, figsize=figsize,
                            constrained_layout=True)
    axs = axs.flat
    x = np.round(np.linspace(0.01, 0.99, 50), 2)
    colors = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red',
              'tab:purple', 'tab:brown','tab:pink',
              'tab:gray', 'tab:olive', 'tab:cyan']
    dashes_choices = [
        [],[1,1],[3,5,1, 5],[5, 10], [5, 1], [3, 1, 1, 1, 1, 1]
        ]
    for i,ind in enumerate(rand_i):
        f1 = pdf_samples[filename1][str(ind)]
        pdf = f1['mean']
        pdf = np.array([p for p in pdf if p[1]!=0])
        axs[0].plot(
            np.log(pdf[:,0]), pdf2cdf(pdf)[:,1],
            color=colors[i],
            linewidth = 2,
            dashes=dashes_choices[i],
            # linestyle = linestyles[i],
            label=
            '({:.2f}, {:.2f})'.format(input_CD[ind-1,0], input_CD[ind-1, 1])
            )

        axs[1].plot(
            x, np.log(pdf2ICDF(pdf,x)),
            color=colors[i],
            linewidth = 2,
            dashes=dashes_choices[i],
            # linestyle = linestyles[i],
            label=
            '({:.2f}, {:.2f})'.format(input_CD[ind-1,0], input_CD[ind-1, 1])
            )
    axs[0].set_xlabel('log(particle arrivial time)')
    axs[0].set_ylabel('TCDF')
    axs[0].xaxis.set_major_locator(mticker.LinearLocator(6))
    axs[0].xaxis.set_major_formatter('{x:.00f}')
    axs[1].set_ylabel('log(particle arrivial time)')
    axs[1].set_xlabel('TCDF')
    axs[1].yaxis.set_major_locator(mticker.LinearLocator(6))
    axs[1].yaxis.set_major_formatter('{x:.00f}')
    # axs[0].legend(title='(C, D)=')
    axs[1].legend(title='(C, D)=')
    fig.savefig('uncorr6simu.pdf')


''' fungsi dengan kembalian'''

# template fungdi dengan kembalian
# def nama_fungsi(argumen):
#       badan fungsi
#       return output

# fungsi kuadrat

def kuadrat(input_angka):
    '''fungsi kuadrat'''
    output_angka = input_angka**2
    return output_angka

y = kuadrat(5)
print(y)

print(kuadrat(6))

z = 10 + kuadrat(7)
print(z)

# fungsi tambah

def fungsi_tambah(angka1,angka2):
    '''fungsi return dengan multi input'''
    return angka1 + angka2

a = fungsi_tambah(10,8)
print(a)

# fungsi dengan return banyak

def operasi_mtk(angka1,angka2):
    tambah = angka1 + angka2
    kurang = angka1 - angka2
    kali = angka1 * angka2
    bagi = angka1 / angka2

    return tambah,kurang,kali,bagi

k,l,m,n = operasi_mtk(9,5)

print(f"hasil = {k}")
print(f"hasil = {l}")
print(f"hasil = {m}")
print(f"hasil = {n}")


#!/usr/bin/env python
import sys
import copy
import logging
import argparse
from BCBio import GFF
# from gff3 import feature_lambda, feature_test_type
logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)

def feature_lambda(feature_list, test, test_kwargs, subfeatures=True, parent=None, invert=False, recurse=True):
    # Either the top level set of [features] or the subfeature attribute
    for feature in feature_list:
        feature._parent = parent
        test_result = test(feature, **test_kwargs)
        # if (not invert and test_result) or (invert and not test_result):
        # print feature.type, test_kwargs, test_result, invert ^ test_result
        if invert ^ test_result:
            if not subfeatures:
                feature_copy = copy.deepcopy(feature)
                feature_copy.sub_features = []
                yield feature_copy
            else:
                yield feature

        if recurse and hasattr(feature, 'sub_features'):
            for x in feature_lambda(feature.sub_features, test, test_kwargs, subfeatures=subfeatures, parent=feature, invert=invert, recurse=recurse):
                yield x

def feature_test_type(feature, **kwargs):
    if 'type' in kwargs:
        return feature.type == kwargs['type']
    elif 'types' in kwargs:
        return feature.type in kwargs['types']
    raise Exception("Incorrect feature_test_type call, need type or types")

coloring = {
    'Petty': {
        '13': '#FFFF00',
        '15': '#FFFF00',
        '16': '#FFFF00',
        '17': '#FFFF00',
        '18': '#FFFF00',
        '22': '#FFFF00',
        '23': '#FFFF00',

        '24': '#FFA500',
        '25': '#FFA500',
        '26': '#FFA500',

        '30': '#87CEFA',
        '31': '#87CEFA',
        '32': '#87CEFA',
        '34': '#87CEFA',
        '35': '#87CEFA',
        '36': '#87CEFA',
        '37': '#87CEFA',
        '38': '#87CEFA',
        '39': '#87CEFA',

        '40': '#FF00FF',
        '41': '#FF00FF',

        '42': '#71BC78',
        '43': '#71BC78',
    },
    'Abp1': {
        '20': '#FFFF00',
        '23': '#FFFF00',
        '24': '#FFFF00',
        '25': '#FFFF00',
        '26': '#FFFF00',
        '29': '#FFFF00',
        '30': '#FFFF00',

        '34': '#FFA500',

        '37': '#87CEFA',
        '39': '#87CEFA',
        '42': '#87CEFA',
        '43': '#87CEFA',
        '46': '#87CEFA',

        '52': '#71BC78',
    },
    'phiAB1': {
        '15': '#FFFF00',
        '16': '#FFFF00',
        '17': '#FFFF00',
        '18': '#FFFF00',
        '19': '#FFFF00',
        '20': '#FFFF00',
        '21': '#FFFF00',
        '23': '#FFFF00',
        '24': '#FFFF00',

        '27': '#FFA500',
        '28': '#FFA500',

        '30': '#87CEFA',
        '31': '#87CEFA',
        '32': '#87CEFA',
        '33': '#87CEFA',
        '36': '#87CEFA',
        '37': '#87CEFA',
        '38': '#87CEFA',
        '39': '#87CEFA',
        '40': '#87CEFA',
        '41': '#87CEFA',

        '42': '#FF00FF',
        '43': '#FF00FF',

        '44': '#71BC78',
        '45': '#71BC78',
    },
    'Acibel007': {
        '16': '#FFFF00',
        '18': '#FFFF00',
        '20': '#FFFF00',
        '22': '#FFFF00',
        '26': '#FFFF00',
        '28': '#FFFF00',

        '31': '#FFA500',
        '32': '#FFA500',

        '3': '#87CEFA',
        '4': '#87CEFA',
        '5': '#87CEFA',
        '24': '#87CEFA',
        '35': '#87CEFA',
        '36': '#87CEFA',
        '37': '#87CEFA',
        '38': '#87CEFA',
        '41': '#87CEFA',
        '42': '#87CEFA',
        '43': '#87CEFA',
        '44': '#87CEFA',
        '45': '#87CEFA',
        '46': '#87CEFA',

        '47': '#FF00FF',
        '48': '#FF00FF',

        '50': '#71BC78',
        '51': '#71BC78',
    },
    'Fri1': {
        '17': '#FFFF00',
        '21': '#FFFF00',
        '23': '#FFFF00',
        '24': '#FFFF00',
        '25': '#FFFF00',
        '26': '#FFFF00',
        '30': '#FFFF00',
        '31': '#FFFF00',
        '32': '#FFFF00',

        '33': '#FFA500',
        '35': '#FFA500',
        '36': '#FFA500',

        '39': '#87CEFA',
        '40': '#87CEFA',
        '41': '#87CEFA',
        '44': '#87CEFA',
        '45': '#87CEFA',
        '46': '#87CEFA',
        '47': '#87CEFA',
        '48': '#87CEFA',
        '49': '#87CEFA',

        '50': '#FF00FF',
        '51': '#FF00FF',

        '52': '#71BC78',
        '53': '#71BC78',
    },
    'phiAB6': {
        '12': '#FFFF00',
        '13': '#FFFF00',
        '16': '#FFFF00',
        '17': '#FFFF00',
        '19': '#FFFF00',
        '20': '#FFFF00',
        '22': '#FFFF00',
        '23': '#FFFF00',
        '24': '#FFFF00',

        '26': '#FFA500',
        '27': '#FFA500',

        '29': '#87CEFA',
        '30': '#87CEFA',
        '31': '#87CEFA',
        '32': '#87CEFA',
        '35': '#87CEFA',
        '36': '#87CEFA',
        '37': '#87CEFA',
        '38': '#87CEFA',
        '39': '#87CEFA',
        '40': '#87CEFA',

        '41': '#FF00FF',
        '42': '#FF00FF',

        '43': '#71BC78',
        '44': '#71BC78',
    }
}

def color(gff3):
    for rec in GFF.parse(gff3):
        genes = list(feature_lambda(rec.features, feature_test_type, {'type': 'gene'}, subfeatures=True))
        cdss_sorted = []
        for gene in genes:
            cdss = sorted(list(feature_lambda(gene.sub_features, feature_test_type, {'type': 'CDS'}, subfeatures=True)), key=lambda x: x.location.start)
            for cds in cdss:
                cdss_sorted.append(cds)

        for num, c in enumerate(sorted(cdss_sorted, key=lambda x: x.location.start), 1):
            color = "black"
            if str(num) in coloring[rec.id]:
                color = coloring[rec.id][str(num)]
            c.qualifiers.update({'color': [color]})
            c.qualifiers.update({'num': [num]})
        GFF.write([rec], sys.stdout)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='add color to CDSs')
    parser.add_argument('gff3', type=argparse.FileType("r"), help='GFF3 annotations')
    args = parser.parse_args()
    color(**vars(args))


# coding: utf-8

"""
    Collibra Data Governance Center Core API

The Core REST API allows you to create your own integrations with Collibra Data Governance Center.

Create custom applications to help users get access to the right data.
  # noqa: E501

    The version of the OpenAPI document: 2.0
    Generated by: https://openapi-generator.tech
"""

import pprint
import re  # noqa: F401

import six

from collibra_core.configuration import Configuration

class AssignedCharacteristicType(object):
    """NOTE: This class is auto generated by OpenAPI Generator.
    Ref: https://openapi-generator.tech

    Do not edit the class manually.
    """

    """
    Attributes:
      openapi_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    openapi_types = {
        'minimum_occurrences': 'int',
        'maximum_occurrences': 'int',
        'assigned_resource_id': 'str',
        'system': 'bool',
        'read_only': 'bool',
        'id': 'str'
    }

    attribute_map = {
        'minimum_occurrences': 'minimumOccurrences',
        'maximum_occurrences': 'maximumOccurrences',
        'assigned_resource_id': 'assignedResourceId',
        'system': 'system',
        'read_only': 'readOnly',
        'id': 'id'
    }

    def __init__(self, minimum_occurrences=None, maximum_occurrences=None, assigned_resource_id=None, system=None, read_only=None, id=None, local_vars_configuration=None):  # noqa: E501
        """AssignedCharacteristicType - a model defined in OpenAPI"""  # noqa: E501
        if local_vars_configuration is None:
            local_vars_configuration = Configuration()
        self.local_vars_configuration = local_vars_configuration

        self._minimum_occurrences = None
        self._maximum_occurrences = None
        self._assigned_resource_id = None
        self._system = None
        self._read_only = None
        self._id = None
        self.discriminator = None

        if minimum_occurrences is not None:
            self.minimum_occurrences = minimum_occurrences
        if maximum_occurrences is not None:
            self.maximum_occurrences = maximum_occurrences
        if assigned_resource_id is not None:
            self.assigned_resource_id = assigned_resource_id
        if system is not None:
            self.system = system
        if read_only is not None:
            self.read_only = read_only
        if id is not None:
            self.id = id

    @property
    def minimum_occurrences(self):
        """Gets the minimum_occurrences of this AssignedCharacteristicType.  # noqa: E501

        :return: The minimum_occurrences of this AssignedCharacteristicType.  # noqa: E501
        :rtype: int
        """
        return self._minimum_occurrences

    @minimum_occurrences.setter
    def minimum_occurrences(self, minimum_occurrences):
        """Sets the minimum_occurrences of this AssignedCharacteristicType.

        :param minimum_occurrences: The minimum_occurrences of this AssignedCharacteristicType.  # noqa: E501
        :type: int
        """

        self._minimum_occurrences = minimum_occurrences

    @property
    def maximum_occurrences(self):
        """Gets the maximum_occurrences of this AssignedCharacteristicType.  # noqa: E501

        :return: The maximum_occurrences of this AssignedCharacteristicType.  # noqa: E501
        :rtype: int
        """
        return self._maximum_occurrences

    @maximum_occurrences.setter
    def maximum_occurrences(self, maximum_occurrences):
        """Sets the maximum_occurrences of this AssignedCharacteristicType.

        :param maximum_occurrences: The maximum_occurrences of this AssignedCharacteristicType.  # noqa: E501
        :type: int
        """

        self._maximum_occurrences = maximum_occurrences

    @property
    def assigned_resource_id(self):
        """Gets the assigned_resource_id of this AssignedCharacteristicType.  # noqa: E501

        :return: The assigned_resource_id of this AssignedCharacteristicType.  # noqa: E501
        :rtype: str
        """
        return self._assigned_resource_id

    @assigned_resource_id.setter
    def assigned_resource_id(self, assigned_resource_id):
        """Sets the assigned_resource_id of this AssignedCharacteristicType.

        :param assigned_resource_id: The assigned_resource_id of this AssignedCharacteristicType.  # noqa: E501
        :type: str
        """

        self._assigned_resource_id = assigned_resource_id

    @property
    def system(self):
        """Gets the system of this AssignedCharacteristicType.  # noqa: E501

        :return: The system of this AssignedCharacteristicType.  # noqa: E501
        :rtype: bool
        """
        return self._system

    @system.setter
    def system(self, system):
        """Sets the system of this AssignedCharacteristicType.

        :param system: The system of this AssignedCharacteristicType.  # noqa: E501
        :type: bool
        """

        self._system = system

    @property
    def read_only(self):
        """Gets the read_only of this AssignedCharacteristicType.  # noqa: E501

        :return: The read_only of this AssignedCharacteristicType.  # noqa: E501
        :rtype: bool
        """
        return self._read_only

    @read_only.setter
    def read_only(self, read_only):
        """Sets the read_only of this AssignedCharacteristicType.

        :param read_only: The read_only of this AssignedCharacteristicType.  # noqa: E501
        :type: bool
        """

        self._read_only = read_only

    @property
    def id(self):
        """Gets the id of this AssignedCharacteristicType.  # noqa: E501

        :return: The id of this AssignedCharacteristicType.  # noqa: E501
        :rtype: str
        """
        return self._id

    @id.setter
    def id(self, id):
        """Sets the id of this AssignedCharacteristicType.

        :param id: The id of this AssignedCharacteristicType.  # noqa: E501
        :type: str
        """

        self._id = id

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.openapi_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, AssignedCharacteristicType):
            return False

        return self.to_dict() == other.to_dict()

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        if not isinstance(other, AssignedCharacteristicType):
            return True

        return self.to_dict() != other.to_dict()


import json

from app.common.constants import ModuleName
from app.userpanel.models import Calculation
from tests.integration.constants import URL

def test_get_hardy_weinberg_ok(test_client):
    response = test_client.get(URL.HARDY_WEINBERG_GET)
    assert response.status_code == 200
    assert b'Hardy-Weinberg equilibrium' in response.data

def test_post_hardy_weinberg_ok(test_client):
    data = dict()
    data["ho"] = 4
    data["he"] = 3
    data["rho"] = 2
    data["alfa"] = 0.05

    response = test_client.post(URL.HARDY_WEINBERG_POST, data=json.dumps(data), content_type='application/json')
    calculation = Calculation.query.first()

    assert response.status_code == 200
    assert ModuleName.HARDY_WEINBERG in calculation.module_name


def main():
	a =int(0)
	Alcool=int(0)
	Gasolina=int(0)
	Diesel=int(0)
	while (a!=4):
		a = int(input())

		while a>4 or a<=0:

			a = int(input())

		if (a == 1):
			Alcool=Alcool+1

		if (a == 2):
			Gasolina = Gasolina+1
		if (a == 3):
			Diesel=Diesel+1

	print("MUITO OBRIGADO")
	print("Alcool: " +str(Alcool))
	print("Gasolina: " +str(Gasolina))
	print("Diesel: " +str(Diesel))

main()


from functions.utils import get_data, adfuller_test, difference
from functions.MDT_functions import MDT
import matplotlib.pyplot as plt
import tensorly as tl
import pandas as pd

def stationarity_tests(dataset_name, ds, rs, Ns):
    X, _, _ = get_data(dataset_name, Ns)

    '''Check if Series are Stationary'''
    counter = 0
    indices = []
    index = 0
    df_train = pd.DataFrame(X.T)
    for name, column in df_train.iteritems():
        c = adfuller_test(column, name=column.name)
        if c==0:
            indices.append(index)
        index += 1
        counter += c
        #print('\n')
    print("Original Time Series")
    print(counter, "/", X.shape[0], "are Stationary")

    print("\nHankelized Original Time Series")
    for r in rs:
        X_hat, _ = MDT(X, r)
        X_hat_vec = tl.unfold(X_hat, -1).T
        df_hat = pd.DataFrame(X_hat_vec.T)

        counter = 0
        indices = []
        index = 0
        for name, column in df_hat.iteritems():
            c = adfuller_test(column, name=column.name)
            if c==0:
                indices.append(index)
            index += 1
            counter += c
            #print('\n')

        print("r:", r, "~", counter, "/", X_hat_vec.shape[0], "Series are Stationary")

    for d in ds:
        X, _, _ = get_data(dataset_name,  Ns)
        #print("X shape:", X.shape)

        '''Check differenced data for Stationarity'''
        X_d, inv = difference(X, d)
        df_differenced = pd.DataFrame(X_d.T)
        counter = 0
        indices = []
        index = 0
        for name, column in df_differenced.iteritems():
            c = adfuller_test(column, name=column.name)
            if c==0:
                indices.append(index)
            index += 1
            counter += c
            #print('\n')
        print("\nOriginal Time Series after",d,"-order Differencing")
        print(counter, "/", X.shape[0], "are Stationary")

        '''Check various MDT orders for Stationarity'''
        X_d, inv = difference(X, d)
        print("\nHankelized Time Series on", d, "-order differenced data")
        for r in rs:
            X_hat, _ = MDT(X_d, r)
            X_hat_vec = tl.unfold(X_hat, -1).T
            df_hat = pd.DataFrame(X_hat_vec.T)

            counter = 0
            indices = []
            index = 0
            for name, column in df_hat.iteritems():
                c = adfuller_test(column, name=column.name)
                if c==0:
                    indices.append(index)
                index += 1
                counter += c
            print("r:", r, "~", counter, "/", X_hat_vec.shape[0], "Series are Stationary")

#                             Index  Var x Time
datasets = ['book1', #__________0     3 x sum(Ns) # DONE
            'stackloss', #______1     4 x 21      # DONE
            'macro', #__________2     12 x 203    # DONE
            'elnino', #_________3     12 x 61     # DONE
            'ozone', #__________4      8 x 203    # DONE
            'nightvisitors', #__5      8 x 56     # DONE
            'nasdaq', #_________6     82 x 40560  #
            'yahoo'] #__________7     5 x 2469    #

Ns = [80, 5, 5]
data_name = datasets[6]

X_train, _, _ = get_data(data_name, Ns)

# plt.figure(figsize = (12,5))
# plt.plot(X.T)
# plt.show()
fig = plt.figure(figsize = (12,9))
gs = fig.add_gridspec(4, hspace=0)
axs = gs.subplots(sharex=True, sharey=False)
# fig.suptitle('Sharing both axes')
axs[0].plot(X_train.T)
axs[1].plot(X_train[0, :].T, 'tab:blue')
axs[2].plot(X_train[1, :].T, 'tab:orange')
axs[3].plot(X_train[2, :].T, 'tab:green')
plt.show()

ds = [i for i in range(1, 9)]
rs = [i for i in range(2, 11)]
print("Dataset: " + data_name + " \n")
stationarity_tests(data_name, ds, rs, Ns)