"""
Created on Wed Nov  7 13:06:08 2018

@author: david
"""
import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PolynomialFeatures
import pandas as pd

#from astropy.convolution import Gaussian2DKernel,convolve
#import scipy.linalg as linalg
#from mpl_toolkits.mplot3d import Axes3D
DATA=pd.read_csv("SIGCOL_RUN.txt",delimiter=" ",header=None)

x=DATA[2].values
y=DATA[3].values
names=DATA[0].values

#plt.scatter(x,y)

ii=x>np.amin(x)

x=x[ii]
y=y[ii]

n_bins=15
tam_x=np.amax(x)-np.amin(x)
l_bin=tam_x/n_bins

n_x=np.zeros(n_bins)
n_y=np.zeros(n_bins)

for i in range(n_bins):
    n_x[i]=np.amin(x)+i*l_bin
    ii=np.logical_and(x>=n_x[i],x<n_x[i]+l_bin)
    n_y[i]=0
    if(len(y[ii])!=0):
        n_y[i]=np.mean(y[ii])

deg=5
poly = PolynomialFeatures(degree=deg)
X = poly.fit_transform(n_x.reshape(-1,1))
pred_x=poly.fit_transform(x.reshape(-1,1))
act_y=poly.fit_transform(y.reshape(-1,1))
Y=poly.fit_transform(n_y.reshape(-1,1))
clf = linear_model.LinearRegression()
clf.fit(X,Y)
pred_y=clf.predict(pred_x)

TH=2
rev=1

ii=TH*pred_y[:,1]<=act_y[:,1]
jj=np.logical_not(ii)

indexes=np.where(ii)[0]
#variabs=names[ii]

fileout=open("VAR_"+str(deg)+".txt","w")
for i in indexes:
    fileout.write(names[i]+" "+str(x[i])+" " +str(y[i]))
    fileout.write("\n")
fileout.close()

fileout=open("VAR_"+str(deg)+"_names.txt","w")
for i in indexes:
    fileout.write(names[i])
    fileout.write("\n")
fileout.close()

plt.figure(figsize=[10,10])
plt.title("Numero de candidatos = "+str(len(x[ii])))
plt.scatter(x[ii],y[ii],c="g",alpha=0.2)
print(len(x[ii]), " estrellas candidatas.")
plt.scatter(x[jj],y[jj],c="r",alpha=0.2)

t="Y = "
for i in range (len(clf.coef_[rev])):
    t=t+str(round(clf.coef_[rev][i]*10**1,2))+"x $10^{-10}$ "+"$X^{"+str(i+1)+"}$  +  "
t=t+str(round(clf.intercept_[rev]*10**1,2))+"x $10^{-10}$ "
plt.title(t,fontsize=int(65/len(clf.coef_[rev])))

plt.scatter(x,TH*pred_y[:,rev],s=2)
plt.savefig("FIT_"+str(deg)+"sigc.png")

#plt.scatter(x,y)


from __future__ import print_function

import argparse
import collections
import fnmatch
import json
import logging
import os
import re
import tempfile

import genpy
import rosbag
import roslib

# See the following sources for many of the genpy/rosbag/roslib patterns used
# here:
#   http://docs.ros.org/en/melodic/api/rosbag/html/python/rosbag.migration-pysrc.html
#   https://github.com/gavanderhoorn/rosbag_fixer

INTERNAL_MESSAGE_DEF_REGEX = re.compile(r"^MSG: ", re.MULTILINE)

def dosys(cmd):
    logging.info(cmd)
    ret = os.system(cmd)
    if ret != 0:
        logging.warning("Command failed with return value %s\n", ret)
    return ret

def get_rule_entry(rule):
    key = (rule["old_type"], rule["old_type_md5sum"])

    new_type = rule["new_type"]
    new_pytype = genpy.message.get_message_class(new_type)
    if rule["new_type_md5sum"] != new_pytype._md5sum:
        logging.warning("warning: md5sum mismatch for: %s", new_type)
        logging.warning("  migration rule md5sum: %s", rule["new_type_md5sum"])
        logging.warning("  latest md5sum: %s", new_pytype._md5sum)
        logging.warning("  [a changed message definition could cause data corruption]")

    val = {
        "type": new_type,
        "md5sum": rule["new_type_md5sum"],
        "message_definition": new_pytype._full_text,
        "pytype": new_pytype,
    }

    return (key, val)

def get_rules(rules_files):
    fix_topic_patterns = []
    rename_lookup = {}
    for rules_file in rules_files:
        with open(rules_file, "r") as rules_stream:
            rules_info = json.load(rules_stream)

        for k, rules in rules_info.items():
            if k == "fix_message_definition_topic_patterns":
                fix_topic_patterns += rules
            elif k == "rename_types":
                rename_lookup.update(dict((get_rule_entry(r) for r in rules)))

    return (fix_topic_patterns, rename_lookup)

def topic_matcher(topic, topic_patterns):
    return any((fnmatch.fnmatch(topic, p) for p in topic_patterns))

def reindex(inbag_path):
    tmp_folder = tempfile.mkdtemp()
    dosys("rosbag reindex --output-dir=%s %s " % (tmp_folder, inbag_path))
    reindexed_path = os.path.join(tmp_folder, os.path.basename(inbag_path))
    return reindexed_path, tmp_folder

def reindex_if_needed(inbag_path):
    try:
        with rosbag.Bag(inbag_path, "r") as inbag:
            pass
        return inbag_path, None
    except rosbag.bag.ROSBagUnindexedException:
        logging.warning(
            "WARNING: input bag %s was unindexed, reindexing automatically", inbag_path
        )
        return reindex(inbag_path)

def fix_message_definitions(inbag, fix_topic_patterns, verbose=False):
    """
    Modifies inbag metadata (in memory, not on disk). Changes will take
    effect when inbag's messages are subsequently written to outbag.
    """

    if not fix_topic_patterns:
        return

    fixed_topics = set()
    for conn in inbag._get_connections():
        if not topic_matcher(conn.topic, fix_topic_patterns):
            continue
        if INTERNAL_MESSAGE_DEF_REGEX.search(conn.msg_def):
            logging.info(
                "Connection's message definition would be fixed based on message topic '%s', but leaving it alone because it already appears to properly incorporate internal message definitions."
                % conn.topic
            )
            # Note that "fixing" a message definition means forcing it to be
            # deserialized according to the current schema for that message
            # type, which could be wrong if the schema has changed since the
            # message was recorded. The usual problem with rosjava message
            # definitions is that they don't incorporate the definitions of
            # referenced field types. So we do a superficial check for
            # that problem and bail out if it comes up negative.
            #
            # A real-world example of where this check can be important: the
            # /mgt/cpu_monitor/state message can be published on both the MLP
            # and the HLP (different publishers). If on the HLP, the message
            # definition will be broken and we need to fix it. If on the MLP,
            # the message definition will not be broken. The schema for that
            # message has also changed, so fixing non-broken MLP messages from
            # before the schema change could break them!
            continue
        pytype = roslib.message.get_message_class(conn.datatype)
        if pytype is None:
            raise ValueError("Message class '%s' not found." % conn.datatype)
        conn.header["message_definition"] = pytype._full_text
        conn.msg_def = pytype._full_text
        fixed_topics.add(conn.topic)

    if verbose:
        logging.info("Fixed message definitions for topics:")
        if fixed_topics:
            for topic in sorted(fixed_topics):
                logging.info("  %s", topic)
        else:
            logging.info("  [none]")

def rename_types(inbag, outbag_path, rename_lookup, verbose=False):
    # rewrite connection headers
    renamed_types = set()
    for conn in inbag._get_connections():
        hdr = conn.header
        rename_key = (hdr["type"].decode("utf-8"), hdr["md5sum"].decode("utf-8"))
        tgt = rename_lookup.get(rename_key)
        if tgt is not None:
            renamed_types.add(rename_key)
            # rewrite the necessary fields of the connection header
            for k in ("type", "md5sum", "message_definition"):
                hdr[k] = tgt[k].encode("utf-8")

    # summarize rewrite rules that were applied
    if verbose:
        # summary of renamed messages
        migrated = sorted(renamed_types)
        logging.info("Renamed types in bag:")
        if not migrated:
            logging.info("  [no matching types found in bag]")
        for key in migrated:
            old_type, old_type_md5sum = key
            tgt = rename_lookup[key]
            logging.info(
                "  %s %s -> %s %s",
                old_type,
                old_type_md5sum[:8],
                tgt["type"],
                tgt["md5sum"][:8],
            )

    # while copying messages we need to rename types so they are mapped to the right connection
    do_meter = False  # unhelpful in CI test logs
    with rosbag.Bag(outbag_path, "w", options=inbag.options) as outbag:
        if do_meter:
            meter = rosbag.rosbag_main.ProgressMeter(
                outbag_path, inbag._uncompressed_size
            )
            total_bytes = 0

        for topic, raw_msg, t in inbag.read_messages(raw=True):
            msg_type, serialized_bytes, md5sum, pos, pytype = raw_msg

            rename_key = (msg_type, md5sum)
            tgt = rename_lookup.get(rename_key)
            if tgt is not None:
                out_msg = (
                    tgt["type"],
                    serialized_bytes,
                    tgt["md5sum"],
                    pos,
                    tgt["pytype"],
                )
            else:
                out_msg = raw_msg

            outbag.write(topic, out_msg, t, raw=True)

            if do_meter:
                total_bytes += len(serialized_bytes)
                meter.step(total_bytes)

        if do_meter:
            meter.finish()

def rewrite_msg_types1(
    inbag_path_in, outbag_path, rules, verbose=False, no_reindex=False
):
    inbag_path, tmp_folder = reindex_if_needed(inbag_path_in)

    fix_topic_patterns, rename_lookup = rules
    with rosbag.Bag(inbag_path, "r") as inbag:
        fix_message_definitions(inbag, fix_topic_patterns, verbose)
        rename_types(inbag, outbag_path, rename_lookup, verbose)

    if tmp_folder:
        dosys("rm -r %s" % tmp_folder)

    cmd = "rosbag reindex -q %s" % outbag_path
    if no_reindex:
        logging.info("%s probably needs to be re-indexed. Run: %s\n", outbag_path, cmd)
    else:
        dosys(cmd)
        dosys("rm %s" % (os.path.splitext(outbag_path)[0] + ".orig.bag"))

def rewrite_msg_types(
    inbag_paths, outbag_path_pattern, rules_files, verbose=False, no_reindex=False
):
    rules = get_rules(rules_files)

    for inbag_path in inbag_paths:
        inbag_name = os.path.splitext(inbag_path)[0]
        outbag_path = outbag_path_pattern.format(inbag=inbag_name)
        if os.path.exists(outbag_path):
            logging.error("Not overwriting existing file %s", outbag_path)
            continue
        rewrite_msg_types1(
            inbag_path, outbag_path, rules, verbose=verbose, no_reindex=no_reindex
        )

class CustomFormatter(
    argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter
):
    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=CustomFormatter
    )
    parser.add_argument(
        "-v", "--verbose", help="print debug info", default=False, action="store_true"
    )
    parser.add_argument(
        "-o",
        "--output",
        help="path for output bag",
        default="{inbag}.rewrite_types.bag",
    )
    parser.add_argument(
        "-r",
        "--rules",
        nargs="?",
        help="path to input rules file (specify multiple times for multiple files)",
        default=[],
        action="append",
    )
    parser.add_argument(
        "-n",
        "--no-reindex",
        action="store_true",
        help="suppress 'rosbag reindex' call",
        default=False,
    )
    parser.add_argument("inbag", nargs="+", help="input bag")

    args = parser.parse_args()

    level = logging.DEBUG if args.verbose else logging.WARN
    logging.basicConfig(level=level, format="%(message)s")

    if not args.rules:
        parser.error("-r argument is required")

    rewrite_msg_types(
        args.inbag,
        args.output,
        args.rules,
        verbose=args.verbose,
        no_reindex=args.no_reindex,
    )

    # suppress confusing ROS message at exit
    logging.getLogger().setLevel(logging.WARN)


class FiniteAutomaton:
    def __init__(self):
        self.states = set()
        self.alphabet = set()
        self.transitions = {}
        self.start_state = None
        self.final_states = set()

def grammar_to_finite_automaton(grammar):
    fa = FiniteAutomaton()
    fa.states.add('q0')
    fa.start_state = 'q0'
    for symbol in grammar.VT:
        fa.alphabet.add(symbol)
    for variable in grammar.VN:
        for production in grammar.P[variable]:
            source_state = 'q' + str(hash(production))
            fa.states.add(source_state)
            if variable == grammar.start:
                fa.transitions[(fa.start_state, '')] = source_state
            for i in range(len(production)):
                if production[i] in grammar.VT:
                    fa.transitions[(source_state, production[i])] = 'q' + str(hash(production[:i] + '.' + production[i+1:]))
            if production[-1] in grammar.VN:
                fa.final_states.add(source_state)
    return fa

def accepts(fa, input_string):
    current_state = fa.start_state
    for symbol in input_string:
        if (current_state, symbol) in fa.transitions:
            current_state = fa.transitions[(current_state, symbol)]
        else:
            return False
    return current_state in fa.final_states


from __future__ import print_function
import nltk
from nltk.translate.bleu_score import SmoothingFunction, sentence_bleu

def BLUE_score(prediction, ground_truth, smooth=False, weights = None):
    """
    For a single prediction, given a list of reference ground true captions
    by directly calling this function, it will give you a blue score for that prediction
    """
    cc = SmoothingFunction()
    if smooth:
        return nltk.translate.bleu_score.sentence_bleu(ground_truth, prediction, smoothing_function=cc.method4)
    elif weights:
        return nltk.translate.bleu_score.sentence_bleu(ground_truth, prediction, weights)
    else:
        return nltk.translate.bleu_score.sentence_bleu(ground_truth, prediction)

if __name__ == '__main__':
    prediction = ['It', 'is', 'a', 'cat', 'at', 'room']
    ground_truth = [['It', 'is', 'a', 'cat', 'inside', 'the', 'room']]
    print(BLUE_score(prediction, ground_truth))


# coding: utf-8
"""
    cellfetch.py
    ~~~~~~~~~~~~

    cell towers location spider.

    Data From: cellocation.com
    Coord Type: gcj02
    see link <http://baike.baidu.com/link?url=Q_CyKgQIOdKwdF0FOMIuGY2UbrlYtClUrYny73GNM4lcGQuDGw6GaBzU5ja_qti_pqZFRE2mFiyNosqDwLZasK>
    for more details.
    Author: Vito
    Date: 2016-04-20 20:51
"""

import os
import re
import sys
import csv
import json
import time
import argparse
from datetime import datetime
from urllib.request import Request, urlopen
from urllib.error import URLError
from queue import Queue
from pyextend.core import log
from pyextend.core.itertools import merge

log.set_logger(filename=os.path.join(os.path.curdir, './logs/cellfetch.log'), with_filehandler=True)

PY_VERSION = sys.version_info[0]

_default_outfile = os.path.join(os.path.curdir, 'results/cell_towers_{}.csv'.format(datetime.now().strftime('%Y%m%d')))

parser = argparse.ArgumentParser(usage='cellfetch <options>',
                                 description='a web spider used for fetching cell towers coordinates')

normal_options = parser.add_argument_group('normal options')

normal_options.add_argument('-f', '--file', type=str,
                            help="source csv file with 'mcc','mnc','lac','cid' columns. ")

normal_options.add_argument('--line', type=int, default=-1, help="a integer number; read [--file] begin at [--line].")

normal_options.add_argument('-o', '--out', type=str,
                            default=_default_outfile,
                            help="output file;\
                            result csv file will with 'mcc','mnc','lac','cid','lat','lon' and other columns.\
                            (default '{}')".format(_default_outfile))

normal_options.add_argument('--key', action='store_true',
                            default=False,
                            help="can generate key field at output file ")

normal_options.add_argument('-s', '--sleep-time', type=int, default=1*60,
                            help="a integer number; when api response '403',\
                            cellfetch process will sleep some seconds (default 60s).")

check_options = parser.add_argument_group('check options')

check_options.add_argument('-c', '--check-line', type=int, default=0,
                           help="a integer number; used for checking [-c] line's position at the [--file].")

args, remaining = parser.parse_known_args(sys.argv)

class SpiderCursor(object):
    def __init__(self):
        if not os.path.exists('spider.lock'):
            open('spider.lock', mode='w').close()
        self.f = open('spider.lock', mode='r+')

    def read(self):
        self.f.close()
        self.f = open('spider.lock', mode='r+')

        lines = self.f.readlines()
        lines.extend(['0' for _ in range(2 - len(lines))])
        if PY_VERSION >= 3:
            line_no, pos, *other = lines
        else:
            line_no = lines[0]
            pos = lines[1]
        return int(line_no or 0), int(pos or 0)

    def write(self, ln, pos):
        self.f.seek(0)
        self.f.truncate(0)
        self.f.writelines([str(ln), '\n', str(pos)])  # TODO need clear file body then write
        self.f.flush()

    def close(self):
        self.f.close()

_headers = {
    'User-Agent': 'mozilla/5.0 (compatible; baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
    # 'cookie': cookies_str
}

COORD = 'wgs84'
OUT_PUT = 'json'  # or 'csv' or 'xml

# usual conditions we use 'BASE_URL' , and 'BASE_URL2' for back up
# the 'BASE_URL2' param-> 96 (default) is ASU or dBm
# 坐标类型(wgs84/gcj02/bd09)，默认wgs84
BASE_URL = 'http://api.cellocation.com/cell/?coord=%s&output=%s&mcc={}&mnc={}&lac={}&ci={}' % (COORD, OUT_PUT)
BASE_URL2 = 'http://api.cellocation.com/loc/?coord=%s&output=%s&cl={},{},{},{},96' % (COORD, OUT_PUT)

ERR_CODES = {
    10000: '查询参数错误',
    10001: '无基站数据',
    403: '每日查询超限',
}

ENCODE_STR = '%03x%04x%08x%08x'
DECODE_REG = re.compile('^([0-9a-f]{3})([0-9a-f]{4})([0-9a-f]{8})([0-9a-f]{8})$')

_is_gen_key = False
_sleep_time = 1 * 60
_source_file = None
# '/home/mi/tmpdata/mongodb/cell_key_china.csv'
#  '/home/mi/tmpdata/mongodb/cell_key_china.csv' test.csv
_target_file = os.path.join(os.path.curdir,  # '../',
                            'results/cell_towers_{}.csv'.format(datetime.now().strftime('%Y%m%d')))

_cursor = SpiderCursor()

def spider(cell_code, headers=_headers, **kwargs):

    def gen_url(code):
        def get_code(name):
            return code[name] or ''
        url_fmt = kwargs.get('url') or BASE_URL
        try:
            return url_fmt % (get_code('mcc'), get_code('mnc'), get_code('lac'), get_code('cid'))
        except TypeError:
            return url_fmt.format(get_code('mcc'), get_code('mnc'), get_code('lac'), get_code('cid'))

    def verify_data(url, s):
        if OUT_PUT == 'json':
            data = json.loads(s, encoding='utf-8')
        elif OUT_PUT == 'csv':
            result = s.strip().split(',')
            data = dict()
            data['errcode'] = int(result[0])
            data['lat'] = result[1]
            data['lon'] = result[2]
            data['radius'] = result[3]
            data['address'] = result[4]

        # if 'radio' not in data.keys():
        #     data['radio'] = ''

        try:
            errcode = data.pop('errcode')
        except KeyError:
            return data
        if errcode != 0 and errcode in ERR_CODES.keys():
            # print('\n!!!!!!DataError: ', 'url={}, errcode={}, errmsg={}'.format(url, errcode, ERR_CODES[errcode]))
            log.error('!!!!!!DataError: url={}, errcode={}, errmsg={}'.format(url, errcode, ERR_CODES[errcode]))
            return None
        return data

    # def merge(result, code):
    #     return dict(result, **code)

    url = gen_url(cell_code)
    req = Request(url, headers=headers)
    try:
        response = urlopen(req)
    except URLError as e:
        if hasattr(e, 'code'):
            if e.code in ERR_CODES.keys():
                log.error('!!!!!!RequestError: status={}, errmsg={}'.format(e.code, ERR_CODES[e.code]))
                return e.code
            else:
                log.error('!!!!!!RequestError: The server couldn\'t fulfill the request. status={}'.format(e.code))
        elif hasattr(e, 'reason'):
            log.error('We failed to reach a server. reason={}'.format(e.reason))
    else:
        data = response.read().decode("utf8")
        data = verify_data(url, data)
        return merge(data, cell_code) if data is not None else None

class ParseCellKeyError(AttributeError):
    pass

def cell_pk(**kwargs):
    return ENCODE_STR % (int(kwargs['mcc']), int(kwargs['mnc']), int(kwargs['lac']), int(kwargs['cid']))

def parse_cell_pk(string):
    try:
        mcc, mnc, lac, cid = DECODE_REG.match(string).groups()
    except AttributeError:
        raise ParseCellKeyError('Wrong cell key format: {0}'.format(string))
    mcc = int(mcc, base=16)
    mnc = int(mnc, base=16)
    lac = int(lac, base=16)
    cid = int(cid, base=16)
    return mcc, mnc, lac, cid

def test():
    cell_codes = [
        {"cid": 5465, "mcc": 460, "lac": 2, "mnc": 0},
        {"cid": 4198, "mcc": 460, "lac": 12, "mnc": 0},
        {"cid": 12175, "mcc": 460, "lac": 12, "mnc": 0},
        {"cid": 10, "mcc": 460, "lac": 17, "mnc": 0},
        {"cid": 8, "mcc": 460, "lac": 18, "mnc": 0},
        {"cid": 108186371, "mcc": 460, "lac": 23, "mnc": 0},
    ]

    q = Queue()

    for code in cell_codes:
        q.put(code)
        # print('qsize={}'.format(q.qsize()))
        while q.qsize() > 0:
            current_code = q.get()
            print('>>>>>>current_code={}'.format(current_code))
            d = spider(current_code)
            if isinstance(d, dict):
                print(d)
            elif d == 403 and d in ERR_CODES.keys():
                q.put(current_code)
                print('sleep {}s'.format(_sleep_time))
                time.sleep(_sleep_time)

def csv_reader(filename):
    ln, pos = _cursor.read()
    tmp_ln = ln

    with open(filename, mode='r+') as f:
        if pos > 0:
            f.seek(pos)
        log.info('begin line_no=%s, position=%s' % (ln, pos))

        next_row_len = len(f.readline())
        f.seek(pos)
        reader = csv.DictReader(f, fieldnames=['mcc', 'mnc', 'lac', 'cid'])
        for line in reader:
            tmp_ln = (ln-1) + reader.line_num
            if tmp_ln > 0:
                d = dict()
                d['mcc'] = line['mcc']
                d['mnc'] = line['mnc']
                d['lac'] = line['lac']
                d['cid'] = line['cid']
                yield tmp_ln, d
            pos += next_row_len
            tmp_ln += 1
            log.info('next line=%s, position=%s' % (tmp_ln, pos))
            _cursor.write(tmp_ln, pos)
            next_row_len = len(f.readline())
            f.seek(pos)
        log.info('end pos %s' % f.tell())

def csv_writer(filename):
    if not os.path.exists(os.path.dirname(filename)):
        os.mkdir(os.path.dirname(filename))
    f = open(filename, mode='a+')
    writer = csv.DictWriter(f, fieldnames=['key', 'mcc', 'mnc', 'lac', 'cid', 'lat', 'lon', 'address', 'radius'])
    if f.tell() == 0:
        writer.writeheader()
    return f, writer

def run_fetching():
    log.info('source file: %s' % _source_file)
    log.info('output file: %s' % _target_file)
    log.info('sleep time: %ss' % _sleep_time)

    q = Queue()
    reader = csv_reader(_source_file)
    f_w, writer = csv_writer(_target_file)

    def write_data(data, is_gen_key):
        if is_gen_key:
            key = cell_pk(**data)
            data['key'] = key
        log.debug(data)
        writer.writerow(data)
        f_w.flush()

    for idx, code in reader:
        # print(code)
        q.put(code)
        # print('qsize={}'.format(q.qsize()))
        while q.qsize() > 0:
            current_code = q.get()
            log.info('>>>>>>NO.{} current_code={}'.format(idx, current_code))
            d = spider(current_code)
            if isinstance(d, dict):
                write_data(d, is_gen_key=_is_gen_key)
                continue
            elif d == 403 and d in ERR_CODES.keys():
                # try backup url
                d = spider(current_code, url=BASE_URL2)
                if isinstance(d, dict):
                    write_data(d, is_gen_key=_is_gen_key)
                    continue
                q.put(current_code)
                log.warning('sleep {}s'.format(_sleep_time))
                time.sleep(_sleep_time)
    log.info('Done')
    f_w.close()

def get_position(line):
    """Returns a integer; the position of source file at specified line"""
    ln = 0
    with open(_source_file, mode='r') as f:
        while ln < line:
            f.readline()  # if used next(f) then cannot tell() it
            ln += 1
        return f.tell()

def run_checking(check_line):
    log.debug('>>>>>>checking source file: %s' % _source_file)
    log.debug('>>>>>>checking line: %s' % check_line)
    log.info('line: %s ---> position: %s' % (check_line, get_position(check_line)))

def run():
    # print(args)
    global _source_file, _target_file, _sleep_time, _is_gen_key

    _source_file = args.file
    _target_file = args.out
    _sleep_time = args.sleep_time

    begin_line = args.line
    check_line = args.check_line

    _is_gen_key = args.key

    if _source_file is None and check_line == 0:
        parser.print_help()
        exit(0)

    if check_line != 0:
        if check_line < 0:
            parser.print_help()
            parser.error("[--check-line] need > 0.")  # exit with code 2
        run_checking(check_line)
    else:
        if begin_line >= 0:
            pos = get_position(begin_line)
            _cursor.write(begin_line, pos)
        elif begin_line != -1:
            parser.print_help()
            parser.error("[--line] need >= 0.")  # exit with code 2

        run_fetching()

if __name__ == '__main__':
    run()


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import os, random
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

random.seed(0)

def read_data(file_path):
    column_names = ['timestamp', 'acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z', 'mag_x', 'mag_y', 'mag_z']
    data = pd.read_csv(file_path, sep=' ', header = None, names = column_names)
    return data

def load_data(dataset, user_id, window_length=100, axis_num=9, data_path='./harbox_sensys21_dataset/'):
    class_set = ['Call','Hop','typing','Walk','Wave']
    class_num = len(class_set)

    for class_id in range(class_num):
        read_path = data_path +  str(user_id) + '/' + str(class_set[class_id]) + '_train' + '.txt'
        if os.path.exists(read_path):
            tmp_original_data = read_data(read_path)
            tmp_coll = tmp_original_data.iloc[:, 1:axis_num+1].to_numpy().reshape(-1, window_length, axis_num)
            if class_id not in dataset.keys():
                dataset[class_id]=[]
            dataset[class_id].extend(tmp_coll)
    return dataset

def split_dataset(dataset={}, test_size=0.2, window_length=100):
    for user_id in range(1,121):
        load_data(dataset, user_id, window_length=window_length)

    x_dataset = []
    y_dataset = []

    for key in dataset.keys():
        x_dataset.extend(dataset[key])
        y_dataset.extend(key*np.ones(len(dataset[key])).astype(int))

    x_train, x_test, y_train, y_test = train_test_split(x_dataset, y_dataset, test_size = test_size, random_state=42)
    x_train, x_test, y_train, y_test = np.array(x_train), np.array(x_test), np.array(y_train), np.array(y_test)

    return x_train, x_test, y_train, y_test

x_train, x_test, y_train, y_test = split_dataset(test_size = 0.25, window_length=100)

class Autoencoder(nn.Module):
    def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0):
        super(Autoencoder,self).__init__()
        self.in_channel = in_channel
        self.out_channel = out_channel
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        # self.encoder = nn.Sequential(nn.Conv2d(in_channel, out_channel, kernel_size, stride, padding), nn.BatchNorm2d(out_channel)) # not good effect
        self.encoder = nn.Sequential(nn.Conv2d(in_channel, out_channel, kernel_size, stride, padding))
        self.decoder = nn.Sequential(nn.ConvTranspose2d(out_channel, in_channel, kernel_size, stride, padding))
    def forward(self,x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x
    def extract_feature_1(self, data, batch_size, window_length, device):
        result = []
        for idx in range(0, len(data)-batch_size, batch_size):
            sample = torch.tensor(data[idx:idx+batch_size, :, :]).reshape(batch_size, 1, window_length, -1).float()
            sample = sample.to(device)
            tmp_x = self.encoder(sample)
            result.append(tmp_x)
        return result
    def extract_feature_2(self, data):
        result = []
        for idx in range(len(data)):
            tmp_x = self.encoder(data[idx])
            result.append(tmp_x)
        return result

def autoencoder_train(model, loss_function, optimizer, x_train, y_train, window_length=100, batch_size=32, epochs=1, device='cpu'):
    model.train()
    for epoch in range(epochs+1):
        # correct = 0
        # batch_cnt=0
        for idx in range(0, len(x_train)-batch_size, batch_size):
            sample = torch.tensor(x_train[idx:idx+batch_size, :, :]).reshape(batch_size, 1, window_length, -1).float()
            # true_scores = torch.tensor([y_train[idx:idx+batch_size]], dtype=torch.long).reshape(batch_size)
            sample = sample.to(device)
            # true_scores = true_scores.to(device)
            model.zero_grad()
            recon_sample = model(sample)
            loss = loss_function(recon_sample, sample)
            loss.backward()
            optimizer.step()

            # pred = tag_scores.argmax(dim=1, keepdim=True)
            # correct += pred.eq(true_scores.view_as(pred)).sum().item()
            # batch_cnt+=1
        clear_output(wait=True)
        print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, loss.item()/batch_size))
    return model

def decoder_train(decoder_model, optimizer, data, raw_data, epochs=1, device='cpu'):
    model.train()
    for epoch in range(epochs+1):
        for idx in range(len(data)):
            sample = data[idx]
            sample = sample.to(device)
            model.zero_grad()
            recon_sample = decoder_model(sample)
            loss = nn.MSELoss()(recon_sample, raw_data) # grad_penality
            loss.backward(retain_graph=True)
            optimizer.step()
        clear_output(wait=True)
        print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, loss.item()/batch_size))
    return model

def autoencoder_train_2(model, loss_function, optimizer, data, window_length=100, batch_size=32, epochs=1, device='cpu'):
    model.train()
    for epoch in range(epochs+1):
        for idx in range(len(data)):
            sample = data[idx]
            sample = sample.to(device)
            model.zero_grad()
            recon_sample = model(sample)
            loss = loss_function(recon_sample, sample)
            loss.backward(retain_graph=True)
            optimizer.step()
        clear_output(wait=True)
        print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, loss.item()/batch_size))
    return model

# generator
class Decoder(nn.Module):
    def __init__(self, in_channel, out_channel, kernel_size_1, kernel_size_2, stride=1, padding=0):
        super(Decoder, self).__init__()
        self.conv_trans1 = nn.ConvTranspose2d(in_channel, out_channel, kernel_size_1, stride, padding)
        self.conv_trans2 = nn.ConvTranspose2d(out_channel, 1, kernel_size_2, stride, padding)
    def forward(self, x):
        x = self.conv_trans1(x)
        x = self.conv_trans2(x)
        return x

# Discriminator
class D_net_gauss(nn.Module):
    def __init__(self):
        super(D_net_gauss, self).__init__()
        self.lin1 = nn.Linear(z_dim, N)
        self.lin2 = nn.Linear(N, N)
        self.lin3 = nn.Linear(N, 1)
    def forward(self, x):
        x = F.dropout(self.lin1(x), p=0.2, training=self.training)
        x = F.relu(x)
        x = F.dropout(self.lin2(x), p=0.2, training=self.training)
        x = F.relu(x)
        return F.sigmoid(self.lin3(x))

if __name__ == "__main__":

    axis_num=9
    batch_size=32
    epochs=150
    window_length=100

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = Autoencoder(1, 16, 3).to(device)
    # optimizer = optim.Adam(model.parameters(), lr=1e-3)
    optimizer = optim.SGD(model.parameters(), lr=5*1e-5)
    loss_function = nn.MSELoss()

    trained_model = autoencoder_train(model, loss_function, optimizer, x_train, y_train, window_length, batch_size, epochs, device)

    batch_size = 1
    sample = x_test[0:batch_size, :, :]
    sample_reshape = torch.tensor(sample).reshape(batch_size, 1, window_length, -1).float().to(device)
    with torch.no_grad():
        recon_sample = trained_model(sample_reshape)
    recon_sample_reshape = recon_sample.reshape(batch_size, window_length, -1)

    sample_one = sample[0]
    recon_sample_reshape_one = recon_sample_reshape[0].cpu().numpy()

    import matplotlib.pyplot as plt
    plt.rcParams["figure.figsize"] = (20,10)

    from scipy.interpolate import interp1d

    # def inter_func(idx):
    #     def scale(val):
    #         return (val - min(recon_sample_reshape_one[:, idx])) / (max(recon_sample_reshape_one[:, idx])\
    #                                                                 -min(recon_sample_reshape_one[:,idx])) * (max(sample_one[:, idx])-min(sample_one[:,idx])) + min(sample_one[:,idx])
    #     return scale

    def inter_func(idx):
        return interp1d([min(recon_sample_reshape_one[:, idx]), max(recon_sample_reshape_one[:, idx])], [min(sample_one[:, idx]), max(sample_one[:, idx])], kind = 'slinear', fill_value = 'extrapolate')

    x = np.arange(0, window_length)
    plt.subplot(3, 3, 1)
    plt.plot(x, sample_one[:, 0], '.-')
    plt.ylabel('X acceleration')

    tt = recon_sample_reshape_one[:, 0]
    # tt = inter_func(0)(recon_sample_reshape_one[:, 0])
    plt.plot(x, tt, 'c.-')
    plt.ylabel('X acceleration')

    plt.subplot(3, 3, 2)
    plt.plot(x, sample_one[:, 1], '.-')
    plt.ylabel('Y acceleration')

    tt = recon_sample_reshape_one[:, 1]
    # tt = inter_func(1)(recon_sample_reshape_one[:, 1])
    plt.plot(x, tt, 'c.-')
    plt.ylabel('Y acceleration')

    plt.subplot(3, 3, 3)
    plt.plot(x, sample_one[:, 2], '.-')
    plt.ylabel('Z acceleration')

    tt = recon_sample_reshape_one[:, 2]
    # tt = inter_func(2)(recon_sample_reshape_one[:, 2])
    plt.plot(x, tt, 'c.-')
    plt.ylabel('Z acceleration')

    plt.subplot(3, 3, 4)
    plt.plot(x, sample_one[:, 3], '.-')
    plt.ylabel('X acceleration')

    tt = recon_sample_reshape_one[:, 3]
    # tt = inter_func(3)(recon_sample_reshape_one[:, 3])
    plt.plot(x, tt, 'g.-')
    plt.ylabel('X acceleration')

    plt.subplot(3, 3, 5)
    plt.plot(x, sample_one[:, 4], '.-')
    plt.ylabel('Y acceleration')

    tt = recon_sample_reshape_one[:, 4]
    # tt = inter_func(4)(recon_sample_reshape_one[:, 4])
    plt.plot(x, tt, 'g.-')
    plt.ylabel('Y acceleration')

    plt.subplot(3, 3, 6)
    plt.plot(x, sample_one[:, 5], '.-')
    plt.ylabel('Z acceleration')

    tt = recon_sample_reshape_one[:, 5]
    # tt = inter_func(5)(recon_sample_reshape_one[:, 5])
    plt.plot(x, tt, 'g.-')
    plt.ylabel('Z acceleration')

    plt.subplot(3, 3, 7)
    plt.plot(x, sample_one[:, 6], '.-')
    plt.ylabel('X acceleration')

    tt = recon_sample_reshape_one[:, 6]
    plt.plot(x, tt, 'r.-')
    plt.ylabel('X-rec acceleration')

    plt.subplot(3, 3, 8)
    plt.plot(x, sample_one[:, 7], '.-')
    plt.ylabel('Y acceleration')

    tt = recon_sample_reshape_one[:, 7]
    # tt = inter_func(7)(recon_sample_reshape_one[:, 7])
    plt.plot(x, tt, 'r.-')
    plt.ylabel('Y acceleration')

    plt.subplot(3, 3, 9)
    plt.plot(x, sample_one[:, 8], '.-')
    plt.ylabel('Z acceleration')

    tt = recon_sample_reshape_one[:, 8]
    # tt = inter_func(8)(recon_sample_reshape_one[:, 8])
    plt.plot(x, tt, 'r.-')
    plt.ylabel('Z acceleration')

    plt.show()

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model_1 = Autoencoder(1, 16, 3).to(device)
    optimizer = optim.SGD(model_1.parameters(), lr=5*1e-5)
    loss_function = nn.MSELoss()

    trained_model_1 = autoencoder_train(model_1, loss_function, optimizer, x_train, y_train, window_length, batch_size, epochs, device)
    layer_1 = trained_model_1.extract_feature_1(x_train, batch_size, window_length, device)
    print(np.shape(layer_1[0]))

    model_2 = Autoencoder(16, 32, 3).to(device)
    optimizer_2 = optim.SGD(model_2.parameters(), lr=5*1e-5)
    trained_model_2 = autoencoder_train_2(model_2, loss_function, optimizer_2, layer_1, window_length, batch_size, epochs, device)

    layer_2 = trained_model_2.extract_feature_2(layer_1)
    print(np.shape(layer_2[0]))

    model_3 = Autoencoder(32, 64, 3).to(device)
    optimizer_3 = optim.SGD(model_3.parameters(), lr=5*1e-5)
    trained_model_3 = autoencoder_train_2(model_3, loss_function, optimizer_3, layer_2, window_length, batch_size, epochs, device)

    layer_3 = trained_model_3.extract_feature_2(layer_2)
    print(np.shape(layer_3[0]))

    decoder = Decoder(64, 32, kernel_size_1=3, kernel_size_2=5, stride=1).to(device)
    print(np.shape(layer_3[0]))
    decode_output = decoder(layer_3[0])
    print(np.shape(decode_output))


# Inspired by https://twitter.com/ryotakob/status/1319998482283806720?s=20
#つぶやきProcessing
a=TWO_PI/6
r=rotate
t=translate
size(600,600)
clear()
stroke(-1)
s="A";d={"A":"B-A-B","B":"A+B+A"}
t(50,50)
for i in range(10):
 b=""
 for c in s:b+=d[c]if c in d else c
 s=b
for c in s:
 if c in"AB":line(0,0,2,0);t(2,0)
 if c=="+":r(a)
 if c=="-":r(-a)

saveFrame("sketch_2020_10_24a.png")


import time
import socket
import sys
import os

s = socket.socket()
host = socket.gethostname()
port = 8080
s.bind(('',port))

s.listen()
conn, addr = s.accept()
print(addr, "Conectado! ")

command = input(str("Entre com o comando: "))
conn.send(command.encode())
print("Comando enviado com sucesso.")
data = conn.recv(1024)

if data:
    print("Comando recebido e executado com sucesso.")


import abc
import argparse
import logging
import os
import pytorch_lightning as pl
import torch
import torch.utils.data
import torch_geometric

from torch_geometric.data import Data
from torch_geometric.transforms import Cartesian
from typing import Callable, List, Optional, Tuple

from aegnn.utils.bounding_box import crop_to_frame
from .event_ds import EventDataset

class EventDataModule(pl.LightningDataModule):

    def __init__(self, img_shape: Tuple[int, int], batch_size: int, shuffle: bool, num_workers: int,
                 pin_memory: bool, transform: Optional[Callable[[Data], Data]] = None):
        super(EventDataModule, self).__init__(dims=img_shape)

        self.num_workers = num_workers
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.pin_memory = pin_memory

        self.train_dataset = None
        self.val_dataset = None
        self.transform = transform

    def prepare_data(self) -> None:
        logging.info("Preparing datasets for loading")
        self._prepare_dataset("training")
        self._prepare_dataset("validation")

    def setup(self, stage: Optional[str] = None):
        logging.debug("Load and set up datasets")
        self.train_dataset = self._load_dataset("training")
        self.val_dataset = self._load_dataset("validation")
        if len(self.train_dataset) == 0 or len(self.val_dataset) == 0:
            raise UserWarning("No data found, check AEGNN_DATA_DIR environment variable!")

    #########################################################################################################
    # Data Loaders ##########################################################################################
    #########################################################################################################
    def train_dataloader(self) -> torch.utils.data.DataLoader:
        return torch.utils.data.DataLoader(self.train_dataset, batch_size=self.batch_size,
                                           num_workers=self.num_workers, collate_fn=self.collate_fn,
                                           shuffle=self.shuffle, pin_memory=self.pin_memory)

    def val_dataloader(self, num_workers: int = 2) -> torch.utils.data.DataLoader:
        return torch.utils.data.DataLoader(self.val_dataset, self.batch_size, num_workers=num_workers,
                                           collate_fn=self.collate_fn, shuffle=False)

    #########################################################################################################
    # Processing ############################################################################################
    #########################################################################################################
    @abc.abstractmethod
    def _prepare_dataset(self, mode: str):
        raise NotImplementedError

    @abc.abstractmethod
    def _add_edge_attributes(self, data: Data) -> Data:
        max_value = self.hparams.get("preprocessing", {}).get("r", None)
        edge_attr = Cartesian(norm=True, cat=False, max_value=max_value)
        return edge_attr(data)

    @abc.abstractmethod
    def raw_files(self, mode: str) -> List[str]:
        raise NotImplementedError

    @abc.abstractmethod
    def processed_files(self, mode: str) -> List[str]:
        raise NotImplementedError

    #########################################################################################################
    # Data Loading ##########################################################################################
    #########################################################################################################
    def _load_dataset(self, mode: str):
        processed_files = self.processed_files(mode)
        logging.debug(f"Loaded dataset with {len(processed_files)} processed files")
        return EventDataset(processed_files, load_func=self.load_processed_file)

    def load_processed_file(self, f_path: str) -> Data:
        data = self._load_processed_file(f_path)

        # Post-Processing on loaded data before giving to data loader. Crop and index the bounding boxes
        # and apply the transform if it is defined.
        if hasattr(data, 'bbox'):
            data.bbox = data.bbox.view(-1, 5)
            data.bbox = crop_to_frame(data.bbox, image_shape=self.dims)
        if self.transform is not None:
            data = self.transform(data)

        # Add a default edge attribute, if the data does not have them already.
        if not hasattr(data, 'edge_attr') or data.edge_attr is None:
            data = self._add_edge_attributes(data)

        # Checking the loaded data for the sake of assuring shape consistency.
        assert data.pos.shape[0] == data.x.shape[0], "x and pos not matching in length"
        assert data.pos.shape[-1] >= 2
        assert data.x.shape[-1] == 1
        assert data.edge_attr.shape[0] == data.edge_index.shape[1], "edges index and attribute not matching"
        assert data.edge_attr.shape[-1] >= 2, "wrong edge attribute dimension"
        if hasattr(data, 'bbox'):
            assert len(data.bbox.shape) == 2 and data.bbox.shape[1] == 5
            assert len(data.y) == data.bbox.shape[0], "annotations not matching"

        return data

    @staticmethod
    def collate_fn(data_list: List[Data]) -> torch_geometric.data.Batch:
        batch = torch_geometric.data.Batch.from_data_list(data_list)
        if hasattr(data_list[0], 'bbox'):
            batch_bbox = sum([[i] * len(data.y) for i, data in enumerate(data_list)], [])
            batch.batch_bbox = torch.tensor(batch_bbox, dtype=torch.long)
        return batch

    @abc.abstractmethod
    def _load_processed_file(self, f_path: str) -> Data:
        """Load pre-processed file to Data object.

        The pre-processed file is loaded into a torch-geometric Data object. With N the number of events,
        L the number of annotations (e.g., bounding boxes in the sample) and P the number of edges, the
        output object should minimally be as shown below.

        :param f_path: input (absolute) file path of preprocessed file.
        :returns Data(x=[N] (torch.float()), pos=[N, 2] (torch.float()), bbox=[L, 5] (torch.long()), file_id,
                      y=[L] (torch.long()), label=[L] (list), edge_index=[2, P] (torch.long())
        """
        raise NotImplementedError

    #########################################################################################################
    # Dataset Properties ####################################################################################
    #########################################################################################################
    @classmethod
    def add_argparse_args(cls, parent_parser: argparse.ArgumentParser, **kwargs) -> argparse.ArgumentParser:
        parent_parser.add_argument("--dataset", action="store", type=str, required=True)

        group = parent_parser.add_argument_group("Data")
        group.add_argument("--batch-size", action="store", default=8, type=int)
        group.add_argument("--num-workers", action="store", default=8, type=int)
        group.add_argument("--pin-memory", action="store_true")
        return parent_parser

    @property
    def root(self) -> str:
        return os.path.join(os.environ["AEGNN_DATA_DIR"], self.__class__.__name__.lower())

    @property
    def name(self) -> str:
        return self.__class__.__name__.lower()

    @property
    def classes(self) -> List[str]:
        raise NotImplementedError

    @property
    def num_classes(self) -> int:
        return len(self.classes)

    def __repr__(self):
        train_desc = self.train_dataset.__repr__()
        val_desc = self.val_dataset.__repr__()
        return f"{self.__class__.__name__}[Train: {train_desc}\nValidation: {val_desc}]"


from flask import Flask, render_template, redirect, send_file, make_response

from file import get_file_entries, retrieve_file
from logger import apply_log_config, log_exception
from path import PathInfo

apply_log_config()
app = Flask(__name__)

@app.route("/")
def home():
    # Hard-coded, current mechanism forces full URL to be displayed
    # because the website is ran under localhost,
    # but actually served through IIS, which filter the subdomain
    return redirect("http://nas.raenonx.cc/list")

@app.route("/error")
def error():
    raise Exception("For testing purpose")

@app.route("/list", defaults={"file_path": "/"})
@app.route("/list/<path:file_path>")
def get_file_list(file_path: str):
    try:
        file_entries = get_file_entries(file_path)
        return render_template("file_list.html", current_path=PathInfo(file_path), file_entries=file_entries)
    except FileNotFoundError:
        log_exception()
        return "Directory not found.", 404
    except NotADirectoryError:
        log_exception()
        return "Path is not a directory.", 406
    except Exception as ex:
        log_exception(repr(ex))
        return f"Error occurred when viewing the directory. ({ex})", 500

@app.route("/download/<path:file_path>")
def download(file_path: str):
    try:
        file_stream = retrieve_file(PathInfo(file_path))
    except FileNotFoundError:
        log_exception()
        return "Path not found.", 404
    except IsADirectoryError:
        log_exception()
        return "Path to download the file is a directory.", 406

    resp = make_response(send_file(file_stream.stream, as_attachment=True, attachment_filename=file_stream.file_name))
    resp.headers["Content-Length"] = file_stream.file_size

    return resp

if __name__ == "__main__":
    app.run()


from random import choice
from string import ascii_lowercase, digits

from django.utils import timezone
from django.db import models
from django.conf import settings
from django.contrib.auth import get_user_model

from django.db.models.signals import post_delete, post_save
from django.dispatch import receiver

import jwt

from onadata.apps.fieldsight.models import Project, Site

def generate_random_username():
    chars = ascii_lowercase + digits
    length = 16
    username = ''.join([choice(chars) for _ in range(length)])
    User = get_user_model()

    try:
        User.objects.get(username=username)
        return generate_random_username()
    except User.DoesNotExist:
        return username

class RemoteApp(models.Model):
    title = models.CharField(max_length=255)
    projects = models.ManyToManyField(
        Project,
        through='ConnectedProject',
        blank=True
    )
    auth_user = models.OneToOneField(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
    )
    token = models.TextField(blank=True)

    def __str__(self):
        return self.title

    def clean(self):
        if not hasattr(self, 'auth_user') or not self.auth_user:
            User = get_user_model()
            user = User.objects.create_user(
                username=generate_random_username()
            )
            self.auth_user = user

        if not hasattr(self, 'token') or not self.token:
            payload = {'userId': self.auth_user.id}
            self.token = jwt.encode(
                payload,
                settings.SECRET_KEY,
                algorithm='HS256',
            )

class ConnectedProject(models.Model):
    key = models.CharField(max_length=255)
    project = models.ForeignKey(Project)
    app = models.ForeignKey(RemoteApp)
    updated_at = models.DateTimeField(default=None, blank=True, null=True)

    def __str__(self):
        return str(self.project)

class ConnectedDomain(models.Model):
    app = models.ForeignKey(RemoteApp)
    domain = models.CharField(max_length=255)

    def __str__(self):
        return self.domain

@receiver(post_delete, sender=RemoteApp)
def handle_remote_app_delete(sender, instance, **kwargs):
    user = instance.auth_user
    user.delete()

@receiver(post_save, sender=Project)
def handle_project_save(sender, instance, **kwargs):
    project = ConnectedProject.objects.filter(
        project=instance
    ).first()
    if project:
        project.updated_at = timezone.now()
        project.save()

@receiver(post_save, sender=Site)
def handle_site_save(sender, instance, **kwargs):
    project = ConnectedProject.objects.filter(
        project=instance.project
    ).first()
    if project:
        project.updated_at = timezone.now()
        project.save()


import logging
import sys

LOGGING_FORMAT = (
    "%(asctime)s | %(levelname)-8s | Process: %(process)d | %(name)s:%("
    "funcName)s:%(lineno)d - %(message)s"
)

def configure_logger(level: int) -> None:
    """
    Define logs level and formats.

    Args:
        verbose: logging level
    """
    logging.basicConfig(
        stream=sys.stdout,
        format=LOGGING_FORMAT,
        level=level,
    )


import sys
import random

def main():

    if len(sys.argv) < 2:
        print("Usage: python shuffler.py <input file> <optional output file>")
        sys.exit(1)

    TARGET_FILE = sys.argv[1]
    output_file = TARGET_FILE

    # Specify output file if provided
    if len(sys.argv) == 3:
        output_file = sys.argv[2]

    with open(TARGET_FILE, 'r') as f:
        lines = f.readlines()

    # Handle newline chars
    lines = [line.strip() for line in lines]
    lines = [line + "\n" for line in lines]

    # Shuffle lines
    random.shuffle(lines)

    # Remove newline from last line
    lines[-1] = lines[-1].strip()

    with open(output_file, 'w') as f:
        f.writelines(lines)

if __name__ == '__main__':
    main()


#Inheritance is considered as reusabilty.
# A child class is inherites the property of base class.

class Publication:
    def __init__(self, title, price):
        self.title = title
        self.price = price

class Periodical(Publication):
    def __init__(self, title, price,  period, publisher):
        super().__init__(title, price)
        self.publisher = publisher
        self.period = period

class Book(Publication):
    def __init__(self, title, author, pages, price):
        super().__init__(title, price)
        self.pages = pages
        self.author = author

class Magzin(Periodical):
    def __init__(self, title, publisher, period, price):
        super().__init__(title, price,period,publisher)

class Newspaper(Periodical):
    def __init__(self, title, publisher, period, price):
        super().__init__(title, price,period,publisher)

b1 = Book("New World", "pike", 311, 29.0)
n1 = Newspaper("NY Time", "New York Times", "Daily", 6.0)
m1 = Magzin("New Magzine", "Times of india", "Monthly", 59.0)

print(b1.author)
print(n1.publisher)
print(b1.price, m1.price, n1.price)


#Faça um programa que leia a largura e a altura de uma parede em metros,
# calcule sua área e a quantidade de tinta necessária para pinta-la, sabendo que cada litro de tinta pinta uma área de 2m².

nome = str(input("Digite seu nome\n"))
alt = float(input("Digite o valor da altura\n"))
larg = float(input("Digite o valor da largura\n"))
area = alt*larg
tinta = area / 2

print("{}, serão necessários {} litros de tinta para pintar {} m² de área.".format(nome, tinta, area))


import sys

start = 'A'

grammar = [x.strip().replace(':==', '=').replace(' ', '').replace('ε', '') for x in sys.stdin.readlines()]
rules = {rule.split('=')[0] : rule.split('=')[1].split('|') for rule in grammar}

# Compute first set
first = {nt:set() for nt in rules.keys()}
for nt, prod in rules.items():
    for e in prod:
        if len(e) == 0: # epsilon
            first[nt].add('')
        elif e[0].islower(): # term
            first[nt].add(e[0])
        else: #nonterm
            for c in e: # continue adding firsts as long as we have nullable nonterms
                if c.islower():
                    first[nt].add(c)
                    break
                first[nt].add('('+c)
                if not '' in rules[c]:
                    break
for nt, first_set in first.items():
    new_set = first_set.copy()
    while any([x[0]=='(' for x in new_set if len(x)>0]):
        old_set = new_set.copy()
        for to_merge in [x for x in new_set if len(x)>0 and x[0]=='(']:
            to_merge = first[to_merge[1]].copy()
            if '' in to_merge:
                to_merge.remove('')
            new_set |= to_merge
        if new_set == old_set:
            break
    new_set = {x for x in new_set if len(x)==0 or x[0]!='('}
    if len(new_set) == 0:
        new_set.add('')
    first[nt] = new_set

print('first set:', first)

# Compute follow set
follow = {nt:set() for nt in rules.keys()}
follow[start].add('$')
for nt, productions in rules.items():
    for prod in productions:
        for i in range(len(prod)):
            if prod[i].isupper():
                if i == len(prod) - 1:
                    follow[prod[i]].add(')'+nt)
                elif prod[i+1].islower():
                    follow[prod[i]].add(prod[i+1])
                else:
                    j = i+1
                    while j < len(prod):
                        if prod[j].islower():
                            follow[prod[i]].add(prod[j])
                            break
                        if prod[j].isupper():
                            follow[prod[i]].add('('+prod[j])
                        if '' not in rules[prod[j]]:
                            break
                        j += 1
follow = {nt:{x for x in fset if x != ')'+nt} for nt, fset in follow.items()}
for nt, follow_set in follow.items():
    new_set = follow_set.copy()
    while any([x[0]==')' for x in new_set if len(x)>0]):
        old_set = new_set.copy()
        for to_merge in [x for x in new_set if len(x)>0 and x[0]==')']:
            to_merge = follow[to_merge[1]].copy()
            new_set |= to_merge
        if new_set == old_set:
            break
    new_set = {x for x in new_set if len(x)==0 or x[0]!=')'}
    while any([x[0]=='(' for x in new_set if len(x)>0]):
        to_merge = [x for x in new_set if len(x)>0 and x[0]=='('][0]
        new_set.remove(to_merge)
        to_merge = first[to_merge[1]].copy()
        if '' in to_merge:
            to_merge.remove('')
        new_set |= to_merge
    follow[nt] = new_set

print('follow set:', follow)

# Compute parsing table
terms = {x for z in rules.values() for y in z for x in y if x.islower()}
terms.add('$')
table = {nt:{t:set() for t in terms} for nt in rules.keys()}
for nt, productions in rules.items():
    for prod in productions:
        if prod == '':
            for x in follow[nt]:
                table[nt][x].add(prod)
        elif prod[0].islower():
            table[nt][prod[0]].add(prod)
        else:
            first_set = set()
            for i in range(len(prod)):
                if prod[i].islower():
                    first_set.add(prod[i])
                    break
                first_set |= first[prod[i]]
                if '' not in rules[prod[i]]:
                    break
                if i == len(prod)-1:
                    first_set.add('$')
            for x in first_set:
                if x != '':
                    table[nt][x].add(prod)

print('table:', table)

# Check LL(1)
is_ll1 = all([len(y)<2 for x in [x.values() for x in table.values()] for y in x])

print('is ll1?', is_ll1)


#!/usr/bin/env python
'''code description'''
# pylint: disable = I0011, E0401, C0103, C0321

class Solution(object):
    '''Solution description'''
    def largestPalindrome(self, n):
        if n==1: return 9
        if n==2: return 987
        for a in range(2, 9*10**(n-1)):
            hi=(10**n)-a
            lo=int(str(hi)[::-1])
            if a**2-4*lo < 0: continue
            if (a**2-4*lo)**.5 == int((a**2-4*lo)**.5):
                return (lo+10**n*(10**n-a))%1337

def main():
    '''main function'''
    _solution = Solution()
    inp = [1, 2, 3, 4, 5, 6, 7, 8]
    for i in inp:
        print(_solution.largestPalindrome(i))

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
#
# pyping.py, a parallel ip pinging utility.
#
# Dan Liebster
#
# Currently supported on Linux, MacOS, and Windows. AIX SHOULD have worked with the default ping arguements, but as anticipated...
#
# IP input is either range of class Cs, e.g 10.100.11.1-50, or defaults to looking for a local file named "ip_list" expecting an IP per line..
#
# Usage examples:  ./pyping.py 10.100.11.1-120     ||     ./pyping.py (will look in local dir for file named ip_list)
#
# Current version: 0.1 @ 12-13-2016.

import os
import sys
import subprocess
import platform
import re

os_type = platform.system()
dev_null = open(os.devnull, 'w')
good_ip = bad_ip = 0
in_data = "none"
ips = []
ip_list = []
# The following is regex to check if an ip address was supplied as an arg.
ip_pattern = re.compile(
    "^([2][0-5][0-5]|^[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})$")

while in_data == "none":
    if len(sys.argv) > 1:

        in_data = "cli"
        user_args = sys.argv[1]
        ip_match = re.search(ip_pattern, user_args.split("-")[0])

        if ip_match:

            ip_range = user_args.split(".")
            a = ip_range[0]
            b = ip_range[1]
            c = ip_range[2]
            d = ip_range[3]

            ip_range_d = d.split("-")
            d_start = int(ip_range_d[0])
            d_end = int(ip_range_d[1])

            index = d_start

            while index <= d_end:
                item_ip = a + "." + b + "." + c + "." + str(index)
                ips.append(item_ip)
                index += 1
    else:
        in_data = "filelist"
        try:
            ip_file = open('ip_list', 'r')
        except IOError as e:
            print(
                f"\n\n Expecting to find file named \"ip_list\" given the lack of subnet range argument. ({e.errno}): {e.strerror}\n\n")
            exit()
        except:
            print("Unexpected error:", sys.exc_info()[0])
            raise

        for item in ip_file:
            ips.append(item)

p = {}
for ip in ips:
    if not ip.isspace():
        if os_type == "Darwin":
            p[ip] = subprocess.Popen(['ping', '-n', '-W2', '-c2', ip], stdout=dev_null, stderr=subprocess.STDOUT)
        elif os_type == "Windows":
            p[ip] = subprocess.Popen(['ping', '-w', '1000', '-n', '2', ip], stdout=dev_null, stderr=subprocess.STDOUT)
        else:
            p[ip] = subprocess.Popen(['ping', '-n', '-w2', '-c2', ip], stdout=dev_null, stderr=subprocess.STDOUT)

while p:
    for ip, proc in p.items():
        if proc.poll() is not None:
            del p[ip]
            retcode = proc.returncode
            if proc.returncode == 0:
                print('{0} active'.format(ip.rstrip()))
                good_ip += 1
            elif proc.returncode == 1:
                print('{0} no response'.format(ip.rstrip()))
                bad_ip += 1
            else:
                print('{0} returned error {1}'.format(ip.rstrip(), retcode))
                bad_ip += 1
            break

print('\nSummary:\n{} IPs responded.'.format(good_ip))
print('{} did not.'.format(bad_ip))


import argparse

from notification_service.high_availability import DbHighAvailabilityStorage, SimpleNotificationServerHaManager
from notification_service.master import NotificationMaster
from notification_service.service import NotificationService, HighAvailableNotificationService
from notification_service.event_storage import DbEventStorage
from notification_service.util.utils import get_ip_addr

def start_notification_service(port: int = 50052,
                               db_conn: str = None,
                               enable_ha: bool = False,
                               advertised_uri: str = None,
                               create_table_if_not_exists: bool = True):
    if db_conn:
        storage = DbEventStorage(db_conn, create_table_if_not_exists)
    else:
        raise Exception('Failed to start notification service without database connection info.')

    if enable_ha:
        server_uri = advertised_uri if advertised_uri is not None else get_ip_addr() + ':' + str(port)
        ha_storage = DbHighAvailabilityStorage(db_conn=db_conn)
        ha_manager = SimpleNotificationServerHaManager()
        service = HighAvailableNotificationService(
            storage,
            ha_manager,
            server_uri,
            ha_storage,
            5000)
        master = NotificationMaster(service=service,
                                    port=int(port))
    else:
        master = NotificationMaster(service=NotificationService(storage),
                                    port=port)

    master.run(is_block=True)

def _prepare_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--port', type=int, default=50052,
                        help='The port on which to run notification service，default is 50052.')
    parser.add_argument('--database-conn', type=str, default=None,
                        help='Database connection info')
    parser.add_argument('--enable-ha', type=bool, default=False,
                        help='Whether to start a notification service with HA enabled, default is False')
    parser.add_argument('--advertised-uri', type=str, default=None,
                        help='Hostname and port the server will advertise to clients when HA enabled. '
                             'If not set, it will use the local ip and configured port')
    return parser.parse_args()

if __name__ == '__main__':

    args = _prepare_args()
    ns_port = args.port
    database_conn = args.database_conn
    enable_ha = args.enable_ha
    advertised_uri = args.advertised_uri

    start_notification_service(port=ns_port,
                               db_conn=database_conn,
                               enable_ha=enable_ha,
                               advertised_uri=advertised_uri)


from datetime import datetime
from ecinema.tools.validation import validate_year
from ecinema.models.Price import Price
from string import ascii_uppercase

def clean_phone(phone: str) -> str:
    num = ""
    for c in phone:
        if not (c == ' ' or c == '-' or
                c == '(' or c == ')' or c == '+'):
            num = num + c

    return num

def clean_expiration(month: str, year: str):
    exp_date = datetime.now()
    error = None
    if validate_year(year):
        exp_date = datetime(int(year),
                            int(month[0:2]),
                            1, 1, 1)
    else:
        error = "Invalid year"

    return exp_date, error

def create_datetime_from_sql(stime):
    print("stime!!")
    print(stime)
    year = int(stime[0:4])
    month = int(stime[5:7])
    day = int(stime[8: 10])

    hour = int(stime[11:13])
    minute = int(stime[14:16])

    return datetime(year, month, day, hour, minute, 0)

def clean_tickets(a_tickets):
    tickets = []
    ticket = {}
    subtotal = 0
    tid = 0
    for t in a_tickets:
        ticket['tid'] = tid
        tid = tid + 1
        ticket['seat'] = t[0]
        ticket['type'] = t[1]

        # fetch a price object for that type
        # use get price to get the actual price
        price = Price()
        price.fetch(ticket['type'])
        price_amt = price.get_price()
        ticket['price'] = "${0:.2f} USD".format(price_amt)
        ticket['num_price'] = price_amt
        subtotal = subtotal + price_amt

        tickets.append(dict(ticket))
    return tickets, subtotal

def format_price(string) -> str:
    return "${0:.2f} USD".format(string)

def format_seat(seat) -> str:
    letter_index = int(int(seat) / 8)
    letter = ascii_uppercase[letter_index]

    last_num = (int(seat) % 8) + 1

    return letter + str(last_num)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 27 2018

@author: chke (Christian Kehl)
"""
from argparse import ArgumentParser
import os
import re
import numpy
import sys
import itertools
import time
import math
from scipy import io
import h5py
import glob
from scipy import ndimage

TRAIN_RATIO = 0.85
VALID_RATIO = 0.1
TEST_RATIO = 0.05
#SCALE_FACTOR = 1.70703125
SCALE_FACTOR = 1.706666667

if __name__ == '__main__':
    optionParser = ArgumentParser(description="tool for fusing separate input files into one concise output field")
    optionParser.add_argument("-i","--inputDir",action="store",dest="inputDir",default="",help="directory with input data files")
    optionParser.add_argument("-o","--outputDir",action="store",dest="outputDir",default="",help="directory for output files")
    optionParser.add_argument("-n","--name",action="store",dest="datasetName",default="",help="name of the experiment")
    optionParser.add_argument("-I","--InputDirs",action="store",nargs='*',dest="inputDirs",help="multiple directories with input data files")
    optionParser.add_argument("-O","--OutputDirs",action="store",nargs='*',dest="outputDirs",help="multiple directories for each train, test and validation data")
    options = optionParser.parse_args()

    argDict = vars(options)
    inputFileArray = []
    scatterFileArray = []
    observationFileArray = []
    inputFileArrayTemp = []
    scatterFileArrayTemp = []
    observationFileArrayTemp = []

    if("inputDirs" in argDict) and (argDict["inputDirs"]!=None):
        dirArray = []
        for entry in argDict["inputDirs"]:
            dirArray.append(entry)
        for entry in dirArray:
            for name in glob.glob(os.path.join(entry,'*X*.h5')):
                inputFileArrayTemp.append(name)
            for name in glob.glob(os.path.join(entry,'*Y*.h5')):
                scatterFileArrayTemp.append(name)
            for name in glob.glob(os.path.join(entry,'*Z*.h5')):
                observationFileArrayTemp.append(name)
    else:
        for name in glob.glob(os.path.join(options.inputDir,'*X*.h5')):
            inputFileArrayTemp.append(name)
        for name in glob.glob(os.path.join(options.inputDir,'*Y*.h5')):
            scatterFileArrayTemp.append(name)
        for name in glob.glob(os.path.join(options.inputDir,'*Z*.h5')):
            observationFileArrayTemp.append(name)

    outputDir = options.outputDir

    dumpDataFile = h5py.File(inputFileArrayTemp[0], 'r')
    dumpData = numpy.array(dumpDataFile['data']['value'])
    dumpDataFile.close()

    #==========================================================================#
    #===   S T A R T   O F   M I S S I N G   F I L E   T R E A T M E N T    ===#
    #==========================================================================#
    # sort the names
    digits = re.compile(r'(\d+)')
    def tokenize(filename):
        return tuple(int(token) if match else token for token, match in ((fragment, digits.search(fragment)) for fragment in digits.split(filename)))
    # Now you can sort your file names like so:
    inputFileArrayTemp.sort(key=tokenize)
    scatterFileArrayTemp.sort(key=tokenize)
    observationFileArrayTemp.sort(key=tokenize)

    lenX = len(inputFileArrayTemp)
    lenY = len(scatterFileArrayTemp)
    lenZ = len(observationFileArrayTemp)
    if lenX != lenY or lenX != lenZ or lenY != lenZ:
        print("Number of arrays do not match - input: %d, scatterMap: %d, observation: %d" % (lenX,lenY,lenZ))
        exit()
    numSamples = 0
    missingSamples = 0
    for i in itertools.islice(itertools.count(), 0, lenX):
        inName = inputFileArrayTemp[i]
        scatterName = scatterFileArrayTemp[i]
        outName = observationFileArrayTemp[i]
        #print(name)
        bname = os.path.basename(inName)
        fname = bname.split(".")[0]
        numIn = int(fname.split("_")[1])
        bname = os.path.basename(scatterName)
        fname = bname.split(".")[0]
        numScatter = int(fname.split("_")[1])
        bname = os.path.basename(outName)
        fname = bname.split(".")[0]
        numOut = int(fname.split("_")[1])
        if numIn == numOut == numScatter:
            inputFileArray.append(inName)
            scatterFileArray.append(scatterName)
            observationFileArray.append(outName)
            numSamples+=1

    del inputFileArrayTemp[:]
    del scatterFileArrayTemp[:]
    del observationFileArrayTemp[:]
    print("Given: %d samples, with with %d missing files." % (numSamples,missingSamples))
    #==========================================================================#
    #===     E N D   O F   M I S S I N G   F I L E   T R E A T M E N T     ====#
    #==========================================================================#

    lenX = len(inputFileArray)
    lenY = len(scatterFileArray)
    lenZ = len(observationFileArray)
    if lenX != lenY or lenX != lenZ or lenY != lenZ:
        print("Number of arrays do not match - input: %d, scatterMap: %d, observation: %d" % (lenX,lenY,lenZ))

    global_data_index = 0
    prtsz = True
    print("=== data - training ===")
    for i in itertools.islice(itertools.count(), 0, lenX):
        inName = inputFileArray[i]
        scatterName = scatterFileArray[i]
        observeName = observationFileArray[i]

        f = h5py.File(inName, 'r')
        img = numpy.array(f['data']['value'])
        f.close()
        #== SECT reduction
        #img = numpy.squeeze(numpy.sum(img, 2))
        img = numpy.squeeze(numpy.mean(img, 2))
        #== Note: to be done on data load to reduce load-from-disk time ==#
        #img = ndimage.zoom(img, [SCALE_FACTOR,SCALE_FACTOR,1], order=2)           # for new data with margin: zoom factor = 1.70703125 (256 -> 437); then clip to 256 in preproc.
        #== Note: normalisation - temporarily exempted ==#
        #minVal = numpy.min(img)
        #maxVal = numpy.max(img)
        #img = (img-minVal)/(maxVal-minVal)
        #== Note: for over-sized dataset: crop here ==#
        if img.shape[0] > 150:
            if prtsz == True:
                print(img.shape)
            im_center = numpy.array([(img.shape[0]-1)/2, (img.shape[1]-1)/2], dtype=numpy.int32)
            left = im_center[0]-74
            right = left+150
            top = im_center[1]-74
            bottom = top+150
            img = img[left:right,top:bottom,:]
            if prtsz == True:
                print(img.shape)
                prtsz = False

        fname = "%s_%05d_X" % (options.datasetName,global_data_index)
        fOut = h5py.File(os.path.join(outputDir,fname+".h5"),'w')
        outGroup = fOut.create_group('data')
        #outGroup.create_dataset('value', data=img.astype(numpy.float32));
        outGroup.create_dataset('value', data=img);
        fOut.close()

        #== Note: normalisation - temporarily exempted ==#
        #rangeOut = open(os.path.join(trainOutputDir,fname+"_norm"+".txt"),"w")
        #rangeOut.write("min_global: %g\n" % minVal)
        #rangeOut.write("max_global: %g" % maxVal)
        #rangeOut.close()
        del img

        f = h5py.File(scatterName, 'r')
        #img = numpy.array(f['dataY'], order='F').transpose()
        img = numpy.array(f['data']['value'])
        f.close()
        #== SECT reduction
        #img = numpy.squeeze(numpy.sum(img, 2))
        img = numpy.squeeze(numpy.mean(img, 2))
        #== Note: to be done on data load to reduce load-from-disk time ==#
        #img = ndimage.zoom(img, [SCALE_FACTOR,SCALE_FACTOR,1], order=2)           # for new data with margin: zoom factor = 1.70703125 (256 -> 437); then clip to 256 in preproc.
        #== Note: normalisation - temporarily exempted ==#
        #minVal = numpy.min(img)
        #maxVal = numpy.max(img)
        #img = (img-minVal)/(maxVal-minVal)
        #== Note: for over-sized dataset: crop here ==#
        if img.shape[0] > 150:
            if prtsz == True:
                print(img.shape)
            im_center = numpy.array([(img.shape[0]-1)/2, (img.shape[1]-1)/2], dtype=numpy.int32)
            left = im_center[0]-74
            right = left+150
            top = im_center[1]-74
            bottom = top+150
            img = img[left:right,top:bottom,:]
            if prtsz == True:
                print(img.shape)
                prtsz = False

        fname = "%s_%05d_Y" % (options.datasetName,global_data_index)
        fOut = h5py.File(os.path.join(outputDir,fname+".h5"),'w')
        outGroup = fOut.create_group('data')
        #outGroup.create_dataset('value', data=img.astype(numpy.float32));
        outGroup.create_dataset('value', data=img);
        fOut.close()

        #== Note: normalisation - temporarily exempted ==#
        #rangeOut = open(os.path.join(trainOutputDir,fname+"_norm"+".txt"),"w")
        #rangeOut.write("min_global: %g\n" % minVal)
        #rangeOut.write("max_global: %g" % maxVal)
        #rangeOut.close()
        del img

        f = h5py.File(observeName, 'r')
        img = numpy.array(f['data']['value'])
        f.close()
        #== SECT reduction
        #img = numpy.squeeze(numpy.sum(img, 2))
        img = numpy.squeeze(numpy.mean(img, 2))
        #== Note: to be done on data load to reduce load-from-disk time ==#
        #img = ndimage.zoom(img, [SCALE_FACTOR,SCALE_FACTOR,1], order=2)
        #== Note: normalisation - temporarily exempted ==#
        #minVal = numpy.min(img)
        #maxVal = numpy.max(img)
        #img = (img-minVal)/(maxVal-minVal)
        #== Note: for over-sized dataset: crop here ==#
        if img.shape[0] > 150:
            if prtsz == True:
                print(img.shape)
            im_center = numpy.array([(img.shape[0]-1)/2, (img.shape[1]-1)/2], dtype=numpy.int32)
            left = im_center[0]-74
            right = left+150
            top = im_center[1]-74
            bottom = top+150
            img = img[left:right,top:bottom,:]
            if prtsz == True:
                print(img.shape)
                prtsz = False

        fname = "%s_%05d_Z" % (options.datasetName,global_data_index)
        fOut = h5py.File(os.path.join(outputDir,fname+".h5"),'w')
        outGroup = fOut.create_group('data')
        #outGroup.create_dataset('value', data=img.astype(numpy.float32));
        outGroup.create_dataset('value', data=img);
        fOut.close()

        #== Note: normalisation - temporarily exempted ==#
        #rangeOut = open(os.path.join(trainOutputDir,fname+"_norm"+".txt"),"w")
        #rangeOut.write("min_global: %g\n" % minVal)
        #rangeOut.write("max_global: %g" % maxVal)
        #rangeOut.close()
        del img

        if ((global_data_index%100) == 0) and (global_data_index!=0):
            print("progress: %02f %%" % (global_data_index*100/lenX))
        global_data_index+=1


import pytest
from src.quantcerebro.event import GraphEventEmitter, GraphEvent

def test_emit():
    e = GraphEventEmitter()

    e.create_event("a")

    e.add_listener("a",lambda x,y,z: (x,y,z))

    e.emit("a", "msg")
    assert e.events["a"].value() == "msg"

    e.emit("a" , "msg1" , "msg2" , "msg3")
    assert e.events["a"].value() == ("msg1","msg2","msg3")

def test_create_event():
    e = GraphEventEmitter()
    e.create_event("a")
    assert "a" in e.events.keys()

def test_set_event():
    e = GraphEventEmitter()
    a = GraphEvent("a")
    e.set_event("a",a)
    assert e.events["a"] == a

def test_set_source():
    e = GraphEventEmitter()

    e.create_event("a")
    e.create_event("b")

    e.set_source("b", "a")

    e.events[ "b" ] += lambda x: x
    e.events["a"].emit("msg")

    assert e.events["a"].value() == "msg"


"""
Created on Mon Apr 23 17:16:39 2018

@author: Hao Wang
@email: wangronin@gmail.com

"""
from pdb import set_trace

from typing import Callable, Any, Tuple
import os, sys, dill, functools, logging, time

import numpy as np
import json, copy, re
from copy import copy
from joblib import Parallel, delayed

from . import InfillCriteria
from .base import baseBO
from .Solution import Solution
from .SearchSpace import SearchSpace
from .Surrogate import SurrogateAggregation

class BO(baseBO):
    def _create_acquisition(self, fun=None, par={}, return_dx=False):
        fun = fun if fun is not None else self._acquisition_fun
        if hasattr(getattr(InfillCriteria, fun), 'plugin'):
            if 'plugin' not in par:
                par.update({'plugin': self.fopt})

        return super()._create_acquisition(fun, par, return_dx)

    def pre_eval_check(self, X):
        """Check for the duplicated solutions as it is not allowed in noiseless cases
        """
        if not isinstance(X, Solution):
            X = Solution(X, var_name=self.var_names)

        N = X.N
        if hasattr(self, 'data'):
            X = X + self.data

        _ = []
        for i in range(N):
            x = X[i]
            idx = np.arange(len(X)) != i
            CON = np.all(np.isclose(np.asarray(X[idx][:, self.r_index], dtype='float'),
                                    np.asarray(x[self.r_index], dtype='float')), axis=1)
            INT = np.all(X[idx][:, self.i_index] == x[self.i_index], axis=1)
            CAT = np.all(X[idx][:, self.d_index] == x[self.d_index], axis=1)
            if not any(CON & INT & CAT):
                _ += [i]

        return X[_]

class ParallelBO(BO):
    # TODO: add other Parallelization options:
    # 1) niching-based approach (my EVOLVE paper),
    # 2) bi-objective Pareto-front (PI vs. EI) (my WCCI '16 paper), and
    # 3) maybe QEI?
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        assert self.n_point > 1

        if self._acquisition_fun == 'MGFI':
            self._par_name = 't'
            # Log-normal distribution for `t` supported on [0, \infty)
            self._sampler = lambda x: np.exp(np.log(x['t']) + 0.5 * np.random.randn())
        elif self._acquisition_fun == 'UCB':
            self._par_name = 'alpha'
            # Logit-normal distribution for `alpha` supported on [0, 1]
            self._sampler = lambda x: 1 / (1 + np.exp((x['alpha'] * 4 - 2) \
                + 0.6 * np.random.randn()))
        elif self._acquisition_fun == 'EpsilonPI':
            self._par_name = 'epsilon'
            self._sampler = None # TODO: implement this!
        else:
            raise NotImplementedError

        _criterion = getattr(InfillCriteria, self._acquisition_fun)()
        if self._par_name not in self._acquisition_par:
            self._acquisition_par[self._par_name] = getattr(_criterion, self._par_name)

    def _batch_arg_max_acquisition(self, n_point, return_dx):
        criteria = []
        for _ in range(n_point):
            _par = self._sampler(self._acquisition_par)
            _acquisition_par = copy(self._acquisition_par)
            _acquisition_par.update({self._par_name : _par})
            criteria.append(
                self._create_acquisition(par=_acquisition_par, return_dx=return_dx)
            )

        if self.n_job > 1:
            __ = Parallel(n_jobs=self.n_job)(
                delayed(self._argmax_restart)(c, logger=self._logger) for c in criteria
            )
        else:
            __ = [list(self._argmax_restart(_, logger=self._logger)) for _ in criteria]

        return tuple(zip(*__))

class AnnealingBO(ParallelBO):
    def __init__(self, t0=2, tf=1e-1, schedule='exp', *argv, **kwargs):
        super().__init__(*argv, **kwargs)
        self.t0 = t0
        self.tf = tf
        self.schedule = schedule
        self._acquisition_par['t'] = t0

        # TODO: add supports for UCB and epsilon-PI
        max_iter = (self.max_FEs - self._DoE_size) / self.n_point
        if self.schedule == 'exp':                          # exponential
            self.alpha = (self.tf / t0) ** (1. / max_iter)
            self._anealer = lambda t: t * self.alpha
        elif self.schedule == 'linear':
            self.eta = (t0 - self.tf) / max_iter            # linear
            self._anealer = lambda t: t - self.eta
        elif self.schedule == 'log':
            self.c = self.tf * np.log(max_iter + 1)         # logarithmic
            self._anealer = lambda t: t * self.c / np.log(self.iter_count + 2)
        else:
            raise NotImplementedError

        def callback():
            self._acquisition_par['t'] = self._anealer(self._acquisition_par['t'])
        self._acquisition_callbacks += [callback]

class SelfAdaptiveBO(ParallelBO):
    def __init__(self, *argv, **kwargs):
        super.__init__(*argv, **kwargs)
        assert self.n_point > 1

    def _batch_arg_max_acquisition(self, n_point, return_dx):
        criteria = []
        _t_list = []
        N = int(n_point / 2)

        for _ in range(n_point):
            _t = np.exp(self._acquisition_par['t'] * np.random.randn())
            _t_list.append(_t)
            _acquisition_par = copy(self._acquisition_par)
            _acquisition_par.update({'t' : _t})
            criteria.append(
                self._create_acquisition(par=_acquisition_par, return_dx=return_dx)
            )

        if self.n_job > 1:
            __ = Parallel(n_jobs=self.n_job)(
                delayed(self._argmax_restart)(c, logger=self._logger) for c in criteria
            )
        else:
            __ = [list(self._argmax_restart(_, logger=self._logger)) for _ in criteria]

        # NOTE: this adaptation is different from what I did in the LeGO paper..
        idx = np.argsort(__[1])[::-1][:N]
        self._acquisition_par['t'] = np.mean([_t_list[i] for i in idx])
        return tuple(zip(*__))

class NoisyBO(ParallelBO):
    def pre_eval_check(self, X):
        if not isinstance(X, Solution):
            X = Solution(X, var_name=self.var_names)
        return X

    def _create_acquisition(self, fun=None, par={}, return_dx=False):
        if hasattr(getattr(InfillCriteria, self._acquisition_fun), 'plugin'):
            # use the model prediction to determine the plugin under noisy scenarios
            # TODO: add more options for determining the plugin value
            y_ = self.model.predict(self.data)
            plugin = np.min(y_) if self.minimize else np.max(y_)
            par.update({'plugin' : plugin})

        return super()._create_acquisition(par=par, return_dx=return_dx)

class PCABO(ParallelBO):
    def __init__(self):
        pass


# Import Library
from machine import ADC , Pin

def temp():
    """
    Gets the onboard temperature data from the pico
    """
    ad_read = ADC(4)
    temp_value = ad_read.read_u16()
    temp_resolution = 3.3/65535
    temp_voltage = temp_value * temp_resolution

    temp = 27 - (temp_voltage - 0.706)/0.001721

    return temp


import sys
import io
import folium
from PyQt5 import QtGui
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout
from PyQt5.QtWebEngineWidgets import QWebEngineView

class MyApp(QWidget):
    def __init__(self):
        super().__init__()
        self.setWindowTitle('Белгородские достопримечательности')
        self.window_width, self.window_height = 1310, 670
        self.setMinimumSize(self.window_width, self.window_height)
        self.setWindowIcon(QtGui.QIcon('icon.ico'))
        self.setStyleSheet('background-color: rgb(30, 30, 30); font-size: 35px')
        #self.setStyleSheet('font-size: 35px')
        self.setWindowFlags(self.windowFlags())

        layout = QVBoxLayout()
        self.setLayout(layout)

        # задаем карту / setting map
        coordinate = (50.595694, 36.587375) # задаем координаты стартовой точки
        mapboxtoken = 'API TOKEN HERE'
        mapboxurl = 'https://api.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}@2x.png?access_token=' + mapboxtoken

        # задаем саму карту
        m = folium.Map(
            location=coordinate,
            zoom_start=95,
            tiles=mapboxurl,
            attr='Mapbox'
        )

        tooltip = "Кликните для информации"

        # соборная площадь
        marker = folium.Marker(
            location=[50.595694, 36.587375],
            popup="<stong>Соборная площадь</stong>",
            tooltip=tooltip )

        marker.add_to(m)

        # музей курская битва
        marker = folium.Marker(
            location=[50.59139291354616, 36.587886771355244],
            popup='<stong>Музей-диорама\n "Курская Битва. Белгородское Направление"</stong>',
            tooltip=tooltip )

        marker.add_to(m)

        # памятник владмиру великому
        marker = folium.Marker(
            location=[50.580255103847705, 36.58382132424922],
            popup='<stong>Памятник князю Владимиру</stong>',
            tooltip=tooltip )

        marker.add_to(m)

        # Центральный парк имени В. И. Ленина
        marker = folium.Marker(
            location=[50.604452900159345, 36.58727489834365],
            popup='<stong>Центральный парк имени В. И. Ленина</stong>',
            tooltip=tooltip )

        marker.add_to(m)

        # сохранение инфы / saving data
        data = io.BytesIO()
        m.save(data, close_file=False)

        # вывод карты в qt / output map in qt app
        webView = QWebEngineView()
        webView.setHtml(data.getvalue().decode())
        layout.addWidget(webView)

if __name__ == '__main__':
    app = QApplication(sys.argv)

    myApp = MyApp()
    myApp.show()

    try:
        sys.exit(app.exec_())
    except SystemExit:
        print('Закрываем приложение...')