import sys
import argparse
import os
import json
import yaml
#import xmltodict
import re
import traceback
import time
import random
import string
from functools import partial
from .tblfmt import SimpleTable
from .commandline import commandline
from .json2table import prepare_table
from json2html import JsonConverter
from collections import (deque,defaultdict)
from types import FunctionType
from pygments import highlight
from pygments.lexers import (JsonLexer,YamlLexer,XmlLexer,IniLexer,HtmlLexer,guess_lexer)
from pygments.lexers.python import PythonLexer
from pygments.formatters import Terminal256Formatter
from prompt_toolkit import PromptSession
from prompt_toolkit.completion import WordCompleter
from prompt_toolkit.lexers import PygmentsLexer

sys.path.append(os.getcwd())

print = partial(print,flush=True)

DSQ_DOT="_yx_dsq_dot_yx_"

def _json(ds) :
    return json.dumps(ds,indent=2,sort_keys=True)
#def _xml(ds) :
#    import xml.dom.minidom as dom
#    import dicttoxml
#    xmlstr = dicttoxml.dicttoxml(ds).decode()
#    xml = dom.parseString(xmlstr)
#    return xml.toprettyxml()
def _yaml(ds) :
    return yaml.dump(ds,default_flow_style=False,explicit_start=True, explicit_end=False)
def _l(ds,brk="\n") :
    return brk.join([str(i) for i in ds])
def _h(ds) :
    return JsonConverter(ds).json2html()
def _zh(ds) :
    return JsonConverter(ds,recursive=True,collapseandexpand=True).json2html()
def _l2t(xjson,header=None,maxcolwidth=100) :
    data,header=prepare_table(xjson,header)
    return SimpleTable(data=data,header=header,maxwidth=maxcolwidth)
def _l2pt(xjson,header=None) :
    data,header=prepare_table(xjson,header)
    return SimpleTable(data=data,header=header).repr_pivot()
def _t(data=list(),header=None,dataonly=False,maxcolwidth=100) :
    return SimpleTable(data=data,header=header,noheader=dataonly,maxwidth=maxcolwidth)
def _pt(data=list(),header=None,dataonly=False) :
    return SimpleTable(data=data,header=header,noheader=dataonly).repr_pivot()
def _flatlist(ds,ret=None) :
    if ret is None :
        ret = list()
    if type(ds) is list :
        for i in ds :
            _flatlist(i,ret)
    else :
        ret.append(ds)
    return ret
def _qx(cmd) :
    try :
        ret,out,err = commandline.qx(cmd)
        print(out)
        if err :
            print(err)
    except :
        traceback.print_exc()
def _rawstr(ds,last="_"):
    res = ""
    if type(ds) is dict :
        for k,v in ds.items() :
            if type(v) in [dict,list] :
                res += _rawstr(v,last+"."+k)
            else :
                if type(v) is str :
                    vstr = '"{}"'.format(v.replace('"','\\"'))
                else :
                    vstr = str(v)
                res += last + "." + k +"="+ vstr  + "\n"
    if type(ds) is list :
        for i,v in enumerate(ds) :
            if type(v) in [dict,list] :
                res += _rawstr(v,last+"["+str(i)+"]")
            else :
                if type(v) is str :
                    vstr = '"{}"'.format(v.replace('"','\"'))
                else :
                    vstr = str(v)
                res += last+"["+str(i)+"]=" + str(v) + "\n"
    return res

_fl=_flatlist
_j=_json
#_x=_xml
_y=_yaml

def dsq_main():
    parser = argparse.ArgumentParser()
    parser.add_argument(dest="code", nargs='?', default="_", help="code to compile. may be a file.")
    parser.add_argument("-f", "--infile", dest="infile", help="input file")
    parser.add_argument("-t", "--srctype", dest="srctype", default="JSON", help="JSON,YAML")
    parser.add_argument("-i", "--indent", dest="indent", default=4, help="how many spaces for indent. default 4.")
    parser.add_argument("-l", "--rows", dest="rows",type=int, default=2**30, help="use this to shrink each list included. useful for DS including too many records.")
    parser.add_argument("-o", "--output", dest="outfile",default=None,help="write as well as console. ansi color kept.")
    parser.add_argument("-m", "--modules", dest="modules", help="import modules.")
    parser.add_argument("-K", "--keys", dest="keys_included", help="only keep these keys.")
    parser.add_argument("-E", "--nokeys", dest="keys_excluded", help="these keys should be excluded.")
    parser.add_argument("-F", "--functionize", dest="func", action="store_true", default=False, help="wrap code into an internal function",)
    parser.add_argument("-s", "--rawstr", dest="rawstr", action="store_true", default=False, help="output raw stings for easy grep",)
    parser.add_argument("-n", "--origin", dest="origin", action="store_true", default=False, help="no change to the input stream and take it as is. no expanding.")
    parser.add_argument("-I", "--interactive", dest="interactive", action="store_true", default=False, help="interactive mode",)
    parser.add_argument("-p", "--plain", dest="plain", action="store_true", default=False, help="force no color code",)
    parser.add_argument("-c", "--compact", dest="compact", action="store_true", default=False, help="dump data structure in compact mode",)
    parser.add_argument("-C", "--keepcolor", dest="keepcolor", action="store_true", default=False, help="do not remove ANSI color code from input stream.",)
    parser.add_argument("-X", "--debug", dest="debug", action="count", default=False, help="debug mode",)
    _x_args = parser.parse_args()

    class bcolors:
        HEADER = '\033[95m'
        OKBLUE = '\033[94m'
        OKCYAN = '\033[96m'
        OKGREEN = '\033[92m'
        WARNING = '\033[93m'
        FAIL = '\033[91m'
        ENDC = '\033[0m'
        BOLD = '\033[1m'
        UNDERLINE = '\033[4m'

    def warn(s) :
        return bcolors.WARNING + s + bcolors.ENDC
    def notify(s) :
        return bcolors.OKCYAN + s + bcolors.ENDC
    def critical(s) :
        return bcolors.FAIL + s + bcolors.ENDC
    def print_err(s,lvl=0) :
        if lvl == 1 :
            print(notify(s),file=sys.stderr)
            return
        if lvl == 2 :
            print(warn(s),file=sys.stderr)
            return
        if lvl == 3 :
            print(critical(s),file=sys.stderr)
            return
        print(s,file=sys.stderr)

    if _x_args.debug >=2 :
        print_err("# args = ",lvl=1)
        strargs = "\n".join([str(v)+"="+str(getattr(_x_args,v)) for v in vars(_x_args)])
        print_err(strargs)

    def supports_color():
        plat = sys.platform
        supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ)
        is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
        m = re.search(r"^xterm",os.environ.get("TERM","n/a"),re.IGNORECASE)
        try :
            ret,out,err = commandline.qx("uname -s")
        except :
            return False
        iscygwin = re.search(r"cygwin",out,re.IGNORECASE)
        return (is_a_tty or iscygwin) and (m or supported_platform)
    if not _x_args.plain :
        colored = True
        if not supports_color() :
            colored = False
        if os.environ.get("force_ansicolor","") in ["true","yes","y","1"] :
            colored = True
        _x_args.plain = colored
    if _x_args.keys_included :
        _xdt_keys_included = dict()
        for w in  _x_args.keys_included.split(",") :
            recursive = 0
            if re.search(r"(\*|\/|\+)+$",w) :
                recursive = 1
                w = re.sub(r"(\*|\/|\+)+$","",w)
            if w :
                _xdt_keys_included[w.lower()] = recursive
    else :
        _xdt_keys_included = None
    if _x_args.keys_excluded :
        _xset_keys_excluded = set([w.lower() for w in _x_args.keys_excluded.split(",") if w])
    else :
        _xset_keys_excluded = None

    def cutkeys_helper(ds,inkeys,outkeys) :
        if not inkeys and not outkeys :
            return ds
        if type(ds) is dict :
            ds = {k:v for k, v in ds.items() if (not inkeys or k.lower() in inkeys) and (not outkeys or k.lower() not in outkeys)}
            for k,v in ds.items() :
                if not inkeys or (inkeys and k.lower() in inkeys and inkeys[k.lower()] != 1) :
                    ds[k] = cutkeys_helper(v,inkeys,outkeys)
        if type(ds) is list :
            for i,v in enumerate(ds) :
                if type(v) in [dict,list] :
                    ds[i] = cutkeys_helper(v,inkeys,outkeys)
        return ds
    def cutkeys(ds,inkeys=_xdt_keys_included,outkeys=_xset_keys_excluded) :
        ds = cutkeys_helper(ds,inkeys,outkeys)
        if _xset_keys_excluded :
            ds = cutkeys_helper(ds,inkeys=dict(),outkeys=outkeys)
        return ds

    def shrink_list(ds,pos="_") :
        if type(ds) is dict :
            for k,v in ds.items() :
                if type(v) in [dict,list] :
                    ds[k] = shrink_list(v,pos+".{}".format(k))
        if type(ds) is list :
            oldlen = len(ds)
            if _x_args.rows < oldlen :
                print_err("# {}[] {} -> {}".format(pos,oldlen,_x_args.rows),lvl=1)
            ds = ds[:_x_args.rows]
            for i, v in enumerate(ds) :
                if type(v) in [dict,list] :
                    ds[i] = shrink_list(v,pos+"[{}]".format(i))
        return ds

    def collect_keys(ds, dt) :
        if type(ds) is dict :
            for k, v in ds.items() :
                dt[k.lower()].add(k)
                collect_keys(v,dt)
        elif type(ds) in [list,tuple] :
            for i in ds :
                collect_keys(i,dt)
        else :
            pass

    def show_result(res) :
        def print_tee(s) :
            if _x_args.outfile :
                with open(_x_args.outfile,"a") as fw :
                    fw.write(s)
            print(s)
        if _x_args.outfile :
            with open(_x_args.outfile,"w") as fw :
                pass
            xprint = print_tee
        else :
            xprint = print

        if not res :
           return
        if type(res) is str :
            if _x_args.plain :
                xprint(res)
                return
            try :
                if _x_args.rawstr :
                    xprint(highlight(res,IniLexer(),Terminal256Formatter()))
                    return
            except :
                pass
            try :
                xprint(highlight(res,HtmlLexer(),Terminal256Formatter()))
                return
            except :
                pass
            try :
                json.loads(res)
                if _x_args.rows != 2**30 :
                    res = json.loads(res)
                    res = shrink_list(res)
                    res = json.dumps(res,indent=2,sort_keys=True)
                xprint(highlight(res,JsonLexer(),Terminal256Formatter()))
                return
            except :
                pass
            try :
                yaml.safe_load(res)
                if _x_args.rows != 2**30 :
                    res = yaml.safe_load(res)
                    res = shrink_list(res)
                    res = yaml.dump(res,default_flow_style=False,explicit_start=True, explicit_end=False)
                xprint(highlight(res,YamlLexer(),Terminal256Formatter()))
                return
            except :
                pass
            #try :
            #    xmltodict.parse(res)
            #    xprint(highlight(res,XmlLexer(),Terminal256Formatter()))
            #    return
            #except :
            #    pass
            xprint(highlight(res,guess_lexer(res),Terminal256Formatter()))
        else :
            res = cutkeys(res)
            res = shrink_list(res)
            if _x_args.rawstr :
                if _x_args.plain :
                    xprint(_rawstr(res))
                else :
                    xprint(highlight(_rawstr(res),IniLexer(),Terminal256Formatter()))
            else :
                try :
                    if _x_args.plain :
                        if _x_args.compact :
                            xprint(json.dumps(res))
                        else :
                            xprint(json.dumps(res,indent=2))
                    else :
                        if _x_args.compact :
                            xprint(highlight(json.dumps(res),JsonLexer(),Terminal256Formatter()))
                        else :
                            xprint(highlight(json.dumps(res,indent=2),JsonLexer(),Terminal256Formatter()))
                except :
                    xprint(str(res))

    def removespaces(code) :
        if not code or "{" not in code or "}" not in code :
            return code
        xcode = code
        m = re.search(r"\{.*?\s+.*?\}",xcode)
        while m :
            before = code[:m.start()]
            end = code[m.end():]
            chain = m.group(0)
            newchain = re.sub(r"\s+","",chain)
            xcode = before + newchain + end
            m = re.search(r"\{.*?\s+.*?\}",xcode)
        return xcode

    def log_special(code) :
        ret = dict()
        if not code :
            return (code,ret)
        if not ("<" in code and ">" in code) :
            return (code,ret)
        m = re.search(r"\<\s*([^\<\>]+?)\s*\>",code)
        while m :
            full = m.group(0)
            tokeep = m.group(1)
            special = "".join([random.choice(string.ascii_lowercase) for _ in range(10)])
            code = code.replace(full,special)
            ret[special] = tokeep
            if _x_args.debug > 2:
                print("# [passwd] {} -> {}".format(tokeep,special))
            m = re.search(r"\<\s*([^\<\>]+?)\s*\>",code)
        return (code, ret)

    def choiceexpand(code) :
        if not code :
            return code
        m = re.search(r"((\w|\-|\d|\.|\[|\]|\(|\))+?)\.\{\s*(\S+?\s*(,\s*\S+?)*)\s*\}",code)
        if not m :
            return code
        if code.count("{") > 1 :
            print_err("# recusive choices are not supported.",lvl=2)
            return code
        before = code[:m.start()]
        end = code[m.end():]
        dt = m.group(1)
        keys = m.group(3)
        #print(keys)
        res = before + "{" + ",".join(["'{}'".format(k.replace(".",DSQ_DOT))  + ":" + "{}.{}".format(dt,k) for k in keys.split(",")]) + "}" + end
        if _x_args.debug > 2:
            print("# [choice] before = ",code)
            print("# [choice] after  = ",res)
        return res

    def insert_spaces(code) :
        return code.replace("(","( ").replace(")"," )")

    def listexpand(code) :
        def hasslice(code) :
            if re.search(r"\[(\d|\:|\-|\s)*\]",code) and not re.search(r"\[(\s|\d|\-)+\]",code) :
                return True
            return False
        if not code or not hasslice(code) :
            return code
        def listexpand_helper(code) :
            if not hasslice(code) :
                return code
            xcode = code
            tvar = "_" + "".join([random.choice(string.ascii_lowercase) for _ in range(3)])
            #l,x = xcode.split("[]",1)
            m = re.match(r"(.*?)(\[(\d|\:|\s|\-)*\])(.*)",code)
            b,i,a = m.group(1),m.group(2),m.group(4)
            if re.sub(r"\s+","",i) in [ "[]", "[:]" ] :
                i=""
            return "[ {} for {} in {} ]".format(listexpand_helper(tvar+a),tvar,b+i)
        res = ""
        xcode = insert_spaces(code)
        _x_m = re.search(r"\S*\[(\d|\:|\-|\s)*\]\S*",xcode,re.DOTALL)
        while _x_m :
            before = xcode[:_x_m.start()]
            end = xcode[_x_m.end():]
            chain = _x_m.group(0)
            res += before + listexpand_helper(chain)
            xcode = end
            _x_m = re.search(r"\S*\[(\d|\:|\-|\s)*\]\S*",xcode,re.DOTALL)
        res += xcode
        if _x_args.debug > 2:
            print("# [ list ] before = ",code)
            print("# [ list ] after  = ",res)
        return res

    def dotexpand(code,keydt=None) :
        chgsin = code
        _x_m = re.search(r"\{(\w|\.)+\}|(\w+|\]|\))(\.\w+|\.\+\w+)+",chgsin,re.DOTALL)
        res=""
        while _x_m :
            before = chgsin[:_x_m.start()]
            end = chgsin[_x_m.end():]
            chain = _x_m.group(0)
            newchain = ""
            words = chain.split(".")
            for i,w in enumerate(words) :
                if w.startswith("+") or (i==len(words)-1 and end.startswith("(")) :
                    nw = w.lstrip("+")
                    newchain += "." + nw
                else :
                    try :
                        if newchain == "" :
                            stest = w
                        else :
                            stest = newchain + "." + w
                        eval(stest)
                        newchain = stest
                        continue
                    except :
                        pass
                    nw = w.lower()
                    if nw in keydt and len(keydt[nw]) == 1 :
                        nw = list(keydt[nw])[0]
                        if _x_args.debug and w != nw :
                            print_err("# keyword replcement : {} -> {}".format(w,nw),lvl=1)
                    if newchain :
                        newchain += ".get('"+ nw + "','')"
                    else :
                        newchain += nw
            res += before + newchain
            chgsin = end
            _x_m = re.search(r"\{(\w|\.)+\}|(\w+|\]|\))(\.\w+|\.\+\w+)+",chgsin,re.DOTALL)
        res = res+chgsin
        if _x_args.debug > 2:
            print("# [ dot  ] before = ",code)
            print("# [ dot  ] after  = ",res)
        return res

    def runcode(code,data=None) :
        _ = data
        if not code :
            return
        if not _x_args.origin :
            code = code.replace("\\n","\n")
            code = removespaces(code)
            code,passbook = log_special(code)
            code = listexpand(code)
            code = choiceexpand(code)
            code = dotexpand(code,_x_key_dict)
            code = code.replace(DSQ_DOT,".")
            if passbook :
                for k,v in passbook.items() :
                    code = code.replace(k,v)
        if _x_args.debug:
            print_err("# run : ",lvl=1)
            print_err(code)
        attempts=0
        err=""
        while True :
            try :
                if attempts == 0 :
                    if re.search("^\w+\s*=\S+",code) or len(code.splitlines())>1 or re.search(r"^(for|while)\s+",code) :
                        attempts += 1
                        continue
                    if _x_args.debug >= 2:
                        print_err("# eval : ",lvl=1)
                        print_err(code)
                    res = eval(code)
                    show_result(res)
                    return
                elif attempts == 1 :
                    if _x_args.debug >= 2:
                        print_err("# exec :",lvl=1)
                        print_err(code)
                    exec(code)
                    return
                else :
                    print_err("# expanded code :",lvl=2)
                    print_err("{}".format(code),lvl=2)
                    print_err("# {}".format(err.splitlines()[-1]),lvl=2)
                    #if _x_args.debug :
                    #    print_err(err,lvl=2)
                    return
            except :
                err += traceback.format_exc()
                attempts += 1
                continue

    INPUT = None
    if _x_args.infile:
        if not os.path.isfile(_x_args.infile):
            print_err("# {} not exists.".format(_x_args.infile),lvl=2)
        with open(_x_args.infile, "r") as f:
            INPUT = f.read()
            INPUT = INPUT.strip()
    else :
        if not _x_args.interactive :
            import signal
            TIMEOUT = 10
            def interrupted(signal, frame):
                print("# timeout/no input from STDIN detected.",
                      file=sys.stderr,
                      flush=True)
                parser.print_help()
                sys.exit(0)

            try :
                signal.signal(signal.SIGALRM, interrupted)
                signal.alarm(TIMEOUT)
            except :
                pass
            INPUT = sys.stdin.read()
            signal.alarm(0)
            INPUT = INPUT.strip()
    if not INPUT :
        INPUT = json.dumps({})

    if _x_args.debug >= 3 :
        print_err("# INPUT :",lvl=1)
        print_err(INPUT)

    if not _x_args.keepcolor :
        INPUT = re.sub(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]',"",INPUT)

    try :
       if _x_args.srctype.upper() == "JSON" :
            _ = json.loads(INPUT)
       elif _x_args.srctype.upper() == "YAML" :
            _ = yaml.safe_load(INPUT)
       #elif _x_args.srctype.upper() == "XML" :
       #     _ = xmltodict.parse(INPUT)
       else :
        print_err("# unsupported file type.",lvl=2)
        return -1
    except :
        print_err("# invalid JSON/YAML/XML.",lvl=2)
        traceback.print_exc()
        return -1

    if _x_args.debug >= 2 :
        print_err("# data loaded :",lvl=1)
        print_err(json.dumps(_,indent=2))

    _x_key_dict = defaultdict(set)
    collect_keys(_,dt=_x_key_dict)
    if _x_args.debug >= 2 :
        print_err("# keys collected : {}".format(str(_x_key_dict)),lvl=1)
    keys_extra= "_,_t,_x,_y,_j,_l,_tbl,_l2t,_l2pt,_pt,_qx,_rawstr,_flatlist,_fl".split(",")
    _x_word_completer = WordCompleter(sorted(list(set([x for x in _x_key_dict.keys()]+keys_extra))))

    if _x_args.modules :
        _x_args.modules = os.path.expanduser(_x_args.modules)
        _x_args.modules = os.path.expandvars(_x_args.modules)
        for m in _x_args.modules.split(",")  :
            if m :
                m.strip()
                if m.startswith("from ") :
                    exec(m)
                elif os.path.isfile(m) :
                    mpath = os.path.abspath(os.path.dirname(m))
                    mfile = os.path.basename(m)
                    sys.path.append(mpath)
                    mn = mfile.replace(".py","")
                    import importlib
                    globals()[mn] = importlib.import_module(mn)
                elif os.path.isdir(m) :
                    mpath = os.path.abspath(m)
                    sys.path.append(m)
                else :
                    import importlib
                    globals()[m] = importlib.import_module(m)

    def run_as_func(code,_=_) :
        fname = "".join([random.choice(string.ascii_letters) for _ in range(20)])
        newcode = ""
        if _x_args.modules :
            for m in _x_args.modules.split(",")  :
                if m :
                    m.strip()
                    if m.startswith("from ") :
                        newcode += m + "\n"
                    else :
                        newcode += "import " + m + "\n"
        newcode += "def {}(_) :\n".format(fname)
        code = code.replace("\\n","\n")
        for ln in code.splitlines() :
            ln = ln.rstrip()
            newcode += " "*(int(_x_args.indent)) + ln + "\n"
        code = listexpand(newcode)
        code = choiceexpand(newcode)
        code = dotexpand(newcode,_x_key_dict)
        code = code.replace(DSQ_DOT,".")
        if _x_args.debug :
            print_err("# code to compile : ",lvl=1)
            print_err(code)
        xcode = compile(code,"<string>","exec")
        cobj = None
        for c in xcode.co_consts :
            if c and type(c) not in [str,int,tuple] :
                cobj = c
                break
        if not cobj :
            print_err("# no code object found.",lvl=2)
            sys.exit(-1)
        xfunc = FunctionType(cobj, globals())
        print(xfunc(_))

    if os.path.isfile(_x_args.code) :
        code = open(_x_args.code,"r").read()
    else :
        code = _x_args.code
    if re.match(r"^\s*\_\w+\s*$",code) :
        code = code + "(_)"

    xcode = code
    xcode = xcode.replace("\\n","\n")
    if any([re.match(r"^\s*return.*",ln) for ln in xcode.splitlines()]) \
       or any([re.match(r"^\s*return.*",ln) for ln in xcode.split(";")])  :
        _x_args.func = True

    if _x_args.code and not _x_args.func :
        if not (_x_args.interactive and _x_args.code == "_") :
            runcode(code,data=_)

    if _x_args.code and _x_args.func :
        x = run_as_func(code, _)
        if x :
            print(x)

    if _x_args.interactive :
        try :
            _x_session = PromptSession(lexer=PygmentsLexer(PythonLexer), completer=_x_word_completer)
        except :
            print_err("[qic] $ WARN: word completion disabled on combination of {}/{}".format(sys.platform,os.environ.get("TERM")),lvl=2)
            _x_session = None
        _x_dotkey = True
        _x_history = deque(maxlen=200)
        _x_cmd0=""
        _x_oldsin=""
        _x_block=False
        while True :
            _x_doeval=True
            res=""
            err=""
            try :
                if _x_session :
                    _x_sin = _x_session.prompt('[qic] $ ')
                else :
                    _x_sin = input('[qic] $ ')
            except KeyboardInterrupt:
                continue
            except EOFError :
                print_err("# IO Error. Pipeline intput is not supported in interactive mode, pls use (-f).",lvl=3)
                break
            if not _x_sin :
                continue
            if re.match(r"^\s*\_\w+\s*$",_x_sin) :
                _x_sin = _x_sin + "(_)"
            if _x_sin.startswith("'''")  :
                if _x_block :
                    _x_sin = _x_oldsin
                    _x_oldsin=""
                    _x_block=False
                    _x_doeval = False
                else :
                    _x_block = True
                    continue
            if _x_block :
                if _x_oldsin :
                    _x_oldsin += "\n" + _x_sin
                else :
                    _x_oldsin += _x_sin
                continue
            lastc =[int(ord(c)) for c in _x_sin][-1]
            if lastc and lastc in [65,66,67,68] :
                _x_sin = "\\hist"
            if _x_sin in ["quit()","\\q"] :
                break
            if _x_sin in ["dotkey"] :
                _x_dotkey = True
                continue
            if _x_sin in ["nodotkey","no dotkey"] :
                _x_dotkey = False
                continue
            mqx = re.match(r"!\s*(\S.*)",_x_sin)
            if mqx :
                res = _qx(mqx.group(1))
                show_result(res)
                continue
            if _x_sin in ["\\hist","\\history"] :
                if not _x_history :
                    print_err("# no history found.",lvl=2)
                for i,cmd in enumerate(_x_history) :
                    print_err("# {:3} : {}".format(i,cmd),lvl=1)
                continue
            _x_m1 = re.match(r"\\r (\d+)",_x_sin)
            _x_m2 = re.match(r"\\(\d+)",_x_sin)
            if _x_m1 or _x_m2:
                if _x_m1 :
                    ix = int(_x_m1.group(1))
                if _x_m2 :
                    ix = int(_x_m2.group(1))
                if ix < len(_x_history) :
                    _x_sin = _x_history[ix]
                else :
                    continue

            if re.search(r"\breturn\b",_x_sin) :
                x=run_as_func(_x_sin,_)
                if x :
                    print(x)
                continue

            _x_cmd0 = None
            if _x_dotkey and "." in _x_sin :
                _x_cmd0 = _x_sin
            try :
                if _x_cmd0 :
                    _x_history.append(_x_cmd0)
                elif not _x_sin.startswith("\\") :
                    _x_history.append(_x_sin)
                if _x_sin.startswith("\\") :
                    res = ""
                    err = "# command not recognized."
                else :
                    runcode(_x_sin,data=_)
            except :
                traceback.print_last()
                continue
            time.sleep(0.1)
        return 0

if __name__ == "__main__" :
    dsq_main()


import numpy as np

def loadData(fileName):
    try:
        lines=None
        with open(fileName) as f:
            lines=f.readlines()
        rows=len(lines)
        if rows<=0:
            return
        cols=len(lines[0].strip().split("\t"))
        mat=[]
        labels=[]
        index=0
        for line in lines:
            line=line.strip()
            items=line.split("\t")
            mat.append([float(item) for item in items[0:-1]])
            labels.append(float(items[-1]))
            index+=1
        return mat,labels
    except Exception as ex:
        print(ex)


'''

Module Main:

    description: main module, features returnRecipe method where
                 cook-time optimization algorithm is performed

    Variables (not all global):

        numTasks           : Number of tasks in this recipe [int]
        recipeName         : Name of the recipe being optimized
                             [String]
        recipeCompleted    : Are we done with the optimized
                             recipe (ready to output?) [bool]
        recipe_out         : List of TaskNode objects in the
                             optimized order [list]
        PQ                 : Max Priority Queue for considering
                             TaskNodes, compared by
                             background time [PriorityQueue]
        completeCtr        : Counter to determine if all TaskNodes
                             were completed [int]

    methods:

        initializeRecipe(input file):
            -Makes TaskNode objects from file
            -Create Directed graph of all TaskNodes to complete a
            certain recipe from an input file
            -Returns the DiGraph as a sparse adjancency matrix

        optimizeRecipe(DiGraph):
            -Takes in the DiGraph from initializeRecipe, and runs
            our optimization algorithm.
            -Returns the final list of objects in optimized order,
            i.e. recipe_out

        execute(TaskNode):
            -change state in "inProgress"
            -while (PQ.peek.back_time + elap_time) < back_time
            increment elap_time

        main():
            -Tester client

'''

from Queue import PriorityQueue
import numpy as np
from tnode import TaskNode
import taskNodeGeneratorUtils as tnUtils
from taskNodeGeneratorUtils import recipeLibrary
#from scipy.sparse import csr_matrix
#from scipy.sparse.csgraph import breadth_first_order
#from scipy.sparse.csgraph import connected_components

#Target recipe name
#target = ['add_salt','boil_water','dice_tomatoes',\
#          'drain_pasta','pasta_in_water','pot_on_stove',\
#          'prepare_drink','tomatoes_on_pasta']

target = ['Piggy Wiggy', 'Cocktail Sausages', 'Roll Dough',\
           'Wrap Sausages', 'Turn on Oven', 'Preheat Oven',\
            ]

#Create recipe library
#recipe_lib = recipeLibrary('./recipe_files/')
#recipe_lib = recipeLibrary('./test_recipes/')
recipe_lib = recipeLibrary('./more_recipes/')

#Node List
node_list = recipe_lib.extract_list(target)

'''
for i in range(0, len(node_list)):
    print(node_list[i].id)
    print(node_list[i].depends)
    print(node_list[i].state)
    print(node_list[i].task_str)
    print("================")
'''

#Initialize Digraph adjacency matrix to be of dimension
#(numTasks) on a side.
numTasks =(len(node_list),len(node_list))
DG_dense = np.zeros(numTasks)

recipe_out = []

#Queue of background tasks in progress
InProgQ = []

#Priority queue of tasks
PQ = PriorityQueue(numTasks[0])

#Global time (for assigning beginning times)
global_time = 0.0

#Use a dictionary to maintain connectivity between
#TaskNode data and id
dct = dict()

def initializeRecipe():
    global DG_dense
    #global numTasks

    #Populate dictionary and adjacency matrix from node_list
    for i in range(0,numTasks[0]):
        dct[i] = node_list[i]
        #print(dct[i].depends)
        for k in range(0,len(dct[i].depends)):
           j = dct[i].depends[k]
           DG_dense[i][j] = 1

    DG_dense = DG_dense.transpose()
    #print(DG_dense)

    #Rearrange dependencies in the dictionary to
    #reflect reversed Digraph
    for j in range(0, numTasks[0]):
        newDep = []
        for i in range(0, numTasks[0]):
            if DG_dense[j][i] == 1:
                newDep.append(i)
        dct[j].depends = newDep

    dct[0].state = "complete"

    print("+++++++++++++++++++++++++")
    for i in range(0, len(node_list)):
        print(dct[i].id)
        print(dct[i].depends)
        print(dct[i].state)
        print(dct[i].task_str)
        print("================")

def getNewlyAccesibleTasks(tnode):
    for i in tnode.depends:
        canPut = True
        print("Current possible dependencies are : ")
        print(tnode.depends)
        for j in range(0,numTasks[0]):
            if DG_dense[j][i] == 1:
                print("i is : " + str(i))
                print("j is : " + str(j))
                print(dct[j].state != "complete")
                if dct[j].state != "complete":
                    if j != tnode.id:
                        #print("Self-Match pruned")
                        canPut = False
                        break
        if canPut == True:
            PQ.put(dct[i])
            print("Just added " + dct[i].task_str)

def optimizeRecipe():
    initializeRecipe()
    cTask = dct[0]
    while cTask.task_str != "done":
        getNewlyAccesibleTasks(cTask)
        cTask = PQ.get()
        execute(cTask)
    return recipe_out

def execute(tnode):
    global global_time
    print("Executing " + tnode.task_str)
    recipe_out.append(tnode)
    tnode.beg_time = global_time
    if tnode.act_time > 0:
        completeTask(tnode)
        global_time = global_time + tnode.act_time
        for i in InProgQ:
            i.elap_time = i.elap_time + tnode.act_time
            #print("Elapsed time for task :" + str(i.elap_time))
            #print("Task : " + str(i.task_desc))
            #print("PQ is empty returns : " + str(PQ.empty() == True))
            #temp = PQ.get()
            #print("Just popped from PQ : " + temp.task_desc)
            #PQ.put(temp)
            if PQ.empty() == True:
                #print("Nothing on PQ")
                long_on_ProgQ = 0
                for j in InProgQ:
                    print(j.task_str)
                    if j.back_time > long_on_ProgQ:
                        long_on_ProgQ = j.back_time
                    completeTask(j)
                global_time = global_time + long_on_ProgQ

            if inProgF(i) == False and self.PQ.empty() != True:
                 completeTask(i)

    elif tnode.back_time > 0:
        tnode.state = "inProgress"
        InProgQ.append(tnode)

def inProgF(tnode):
    assert(tnode.act_time == 0)
    if tnode.elap_time < tnode.back_time:
        return True
    else:
        return  False

def completeTask(tnode):
    print("completed " + tnode.task_str)
    tnode.state = "complete"
    if tnode.back_time > 0:
        for i in InProgQ:
            print("test : " + i.task_str)
        InProgQ.remove(tnode)
        #getNewlyAccesibleTasks(tnode)
    #print(tnode.task_desc)

def main():
    optimizeRecipe()
    for i in recipe_out:
        print(i.task_desc)
        print(i.beg_time)

if __name__ == '__main__':
    main()


from PyQt5.QtWidgets import QDialog, QTreeWidget, QPushButton, QStatusBar, QVBoxLayout, QLabel, QFileDialog, QMessageBox, QTreeWidgetItem, QLineEdit, QComboBox, QFormLayout
from PyQt5.QtCore import QDateTime, QCoreApplication, QStandardPaths
import time
from .constants import langcodes
from .dictionary import getDictsForLang, getFreqlistsForLang, getAudioDictsForLang
from .dictformats import supported_dict_formats, dictinfo
import json
from .local_dictionary import LocalDictionary

class DictManager(QDialog):
    def __init__(self, parent) -> None:
        super().__init__(parent)
        self.settings = parent.settings
        self.setWindowTitle("Manage Local Dictionaries")
        self.parent = parent # type: ignore
        self.dictdb: LocalDictionary = parent.dictdb
        self.resize(500, 400)
        self.initWidgets()
        self.setupWidgets()
        self.refresh()
        self.showStats()

    def initWidgets(self):
        self.tview = QTreeWidget()
        self.tview.setColumnCount(4)
        self.tview.setHeaderLabels(["Name", "Type", "Language", "Headwords"])
        self.add_dict = QPushButton("Import dictionary, frequency list, or cognate data...")
        self.add_dict.clicked.connect(self.onAdd)
        self.add_audio = QPushButton(
            "Import GoldenDict/LinguaLibre audio library...")
        self.add_audio.clicked.connect(self.onAddAudio)
        self.remove = QPushButton("Remove")
        self.remove.clicked.connect(self.onRemove)
        self.rebuild = QPushButton("Rebuild dictionary database")
        self.rebuild.setToolTip("""\
This will regenerate the database containing dictionary entries.
This program stores all dictionary entries in a single database to
improve performance during lookups. The files must be in their original location
to be reimported, otherwise this operation will fail.\
        """)
        self.rebuild.clicked.connect(self.rebuildDB)
        self.bar = QStatusBar()

    def setupWidgets(self):
        self._layout = QVBoxLayout(self)
        self._layout.addWidget(
            QLabel(
                "Note: "
                "Do not delete any files after importing them!
"
                "VocabSieve does not store a copy of these files; it only indexes and caches them.
"
                "If you delete the files, your dictionaries will disappear when the database is rebuilt."))
        self._layout.addWidget(self.tview)
        self._layout.addWidget(self.add_dict)
        self._layout.addWidget(self.add_audio)
        self._layout.addWidget(self.remove)
        self._layout.addWidget(self.rebuild)
        self._layout.addWidget(self.bar)

    def rebuildDB(self):
        start = time.time()
        dicts = json.loads(self.settings.value("custom_dicts", '[]'))
        self.dictdb.purge()
        n_dicts = len(dicts)
        failed_reads = []
        failed_errors = []
        for (i, item) in enumerate(dicts):
            try:
                self.status(f"Rebuilding database: dictionary ({i+1}/{n_dicts})"
                            ".. this can take a while.")
                QCoreApplication.processEvents()
                self.dictdb.dictimport(item['path'], item['type'], item['lang'], item['name'])
            except Exception as e:
                # Delete dictionary if read fails
                failed_reads.append(item['name'])
                failed_errors.append("\tError:" + repr(e))
                del dicts[i]
                self.settings.setValue("custom_dicts", json.dumps(dicts))
        self.dictdb.makeIndex()
        failures = [name + ": Error: " + error for name, error in zip(failed_reads, failed_errors)]
        failed_msg = ("\nThe following dictionaries could not be imported, and have been removed: \n"
                      + "\n\t".join(failures) if failures else "")

        QMessageBox.information(self, "Database rebuilt",
                                f"Database rebuilt in {format(time.time()-start, '.3f')} seconds.{failed_msg}")
        self.refresh()
        self.showStats()

    def onAdd(self):
        fdialog = QFileDialog()
        fdialog.setFileMode(QFileDialog.ExistingFile)
        fdialog.setNameFilter("Dictionary files (*.json *.ifo *.mdx *.dsl *.dsl.dz *.csv *.tsv *.json.xz *.json.bz2 *.json.gz)")
        fdialog.exec()
        if fdialog.selectedFiles() == []:
            return
        else:
            fname = fdialog.selectedFiles()[0]
        dialog = AddDictDialog(self, fname)
        dialog.exec()
        self.showStats()

    def onAddAudio(self):
        folder = QFileDialog.getExistingDirectory(
            self, "Select sound library", QStandardPaths.writableLocation(
                QStandardPaths.HomeLocation), QFileDialog.ShowDirsOnly)
        if not folder:
            return
        dialog = AddDictDialog(self, folder, True)
        dialog.exec()
        self.showStats()

    def onRemove(self):
        index = self.tview.indexFromItem(self.tview.currentItem())
        dicts = json.loads(self.settings.value("custom_dicts", '[]'))
        if dicts == []:
            return
        self.dictdb.dictdelete(dicts[index.row()]['name'])
        del dicts[index.row()]
        self.settings.setValue("custom_dicts", json.dumps(dicts))
        self.refresh()
        self.showStats()

    def refresh(self):
        dicts = json.loads(self.settings.value("custom_dicts", '[]'))
        self.tview.clear()
        for item in dicts:
            treeitem = QTreeWidgetItem(
                [
                    item['name'],
                    supported_dict_formats[item['type']],
                    langcodes[item['lang']],
                    str(self.dictdb.countEntriesDict(item['name']))
                ]
            )
            self.tview.addTopLevelItem(treeitem)
        for i in range(4):
            self.tview.resizeColumnToContents(i)

    def status(self, msg, t=4000):
        self.bar.showMessage(self.time() + " " + msg, t)

    def time(self):
        return QDateTime.currentDateTime().toString('[hh:mm:ss]')

    def closeEvent(self, event):
        self.parent.loadDictionaries()
        self.parent.loadFreqSources()
        event.accept()

    def showStats(self):
        n_dicts = self.dictdb.countDicts()
        n_entries = self.dictdb.countEntries()
        self.status(f"Total: {n_dicts} dictionaries, {n_entries} entries.", t=0)
        # t=0 means it will not disappear

class AddDictDialog(QDialog):
    def __init__(self, parent, fname, audiolib=False):
        super().__init__(parent)
        self.settings = parent.settings
        self.parent = parent
        self.resize(250, 150)
        self.fname = fname
        self.audiolib = audiolib
        if audiolib:
            self.setWindowTitle("Add dictionary or frequency list")
        else:
            self.setWindowTitle("Add sound library")
            try:
                dictinfo(self.fname)
            except NotImplementedError:
                self.warn("Unsupported format")
                self.close()
            except IOError:
                self.warn("Failed to read file")
                self.close()

        self.parent.status("Reading " + self.fname)
        info = dictinfo(self.fname)
        self.parent.status("Reading done.")
        self.dicttype = info['type']
        self.path = info['path']
        self.basename = info['basename']
        self.initWidgets()
        self.setupWidgets()

    def initWidgets(self):
        self.name = QLineEdit()
        self.name.setText(self.basename)
        self.type = QComboBox()
        self.type.addItems(supported_dict_formats.inverse.keys())
        self.type.setCurrentText(supported_dict_formats[self.dicttype])
        self.lang = QComboBox()
        self.lang.addItems(langcodes.values())
        if self.dicttype == "cognates":
            self.lang.setCurrentText(langcodes["<all>"])
            self.lang.setEnabled(False)
            self.type.setEnabled(False)
            self.name.setText("cognates")
            self.name.setReadOnly(True)
        else:
            self.lang.setCurrentText(
                langcodes[self.settings.value("target_language", 'en')])
        self.commit_button = QPushButton("Add")
        self.commit_button.clicked.connect(self.commit)

    def setupWidgets(self):
        self._layout = QFormLayout(self)
        self._layout.addRow(QLabel("Name"), self.name)
        self._layout.addRow(QLabel("Type"), self.type)
        self._layout.addRow(QLabel("Language"), self.lang)
        self._layout.addRow(self.commit_button)

    def commit(self):
        "Give it a name, then add dictionary"
        name = self.name.text()
        dicts = json.loads(self.settings.value("custom_dicts", '[]'))
        lang = langcodes.inverse[self.lang.currentText()]
        existing_names = getDictsForLang(lang, dicts)\
            + getFreqlistsForLang(lang, dicts)\
            + getAudioDictsForLang(lang, dicts)\
            + ['wikt-en', 'gtrans', '####METAINFO']
        if name.lower() in [n.lower() for n in existing_names]:
            # Name conflict!!
            QMessageBox.critical(
                self,
                "Name conflict",
                f"A dictionary with name '{name}' already exists. "
                + "Please choose a different name",
            )
            return

        self.parent.dictdb.dictimport(
            self.path,
            supported_dict_formats.inverse[self.type.currentText()],
            lang,
            self.name.text())
        dicts.append({"name": self.name.text(),
                      "type": supported_dict_formats.inverse[self.type.currentText()],
                      "path": self.path,
                      "lang": langcodes.inverse[self.lang.currentText()],
                      })
        self.settings.setValue("custom_dicts", json.dumps(dicts))
        self.parent.status(f"Importing {self.name.text()} to database..")
        self.parent.refresh()
        self.parent.status("Importing done.")
        self.parent.showStats()
        self.close()

    def warn(self, text):
        msg = QMessageBox()
        msg.setIcon(QMessageBox.Warning)
        msg.setText(text)
        msg.exec()


import numpy

from collatable.extras.indexer import TokenIndexer
from collatable.fields.list_field import ListField
from collatable.fields.span_field import SpanField
from collatable.fields.text_field import TextField

def test_span_field() -> None:
    PAD_TOKEN = "<PAD>"
    indexer = TokenIndexer[str](specials=(PAD_TOKEN,))
    with indexer.context(train=True):
        a = TextField(
            ["embassy", "of", "japan", "in", "the", "united", "states"],
            indexer=indexer,
            padding_value=indexer[PAD_TOKEN],
        )
        b = TextField(
            ["nara", "institute", "of", "science", "and", "technology"],
            indexer=indexer,
            padding_value=indexer[PAD_TOKEN],
        )
        fields = [
            ListField([SpanField(2, 3, a), SpanField(4, 7, a), SpanField(0, 7, a)]),
            ListField([SpanField(0, 1, b), SpanField(0, 6, b)]),
        ]
    output = fields[0].collate(fields)

    assert isinstance(output, numpy.ndarray)
    assert output.shape == (2, 3, 2)

    span_lengths = (output.max(2) >= 0).sum(1)
    assert span_lengths.tolist() == [3, 2]


# pi = 3.14159...
# по формуле Лейбница следующий бесконечный ряд сходится к pi
# pi = 4/1 - 4/3 + 4/5 - 4/7 + 4/9 - 4/11 ...

# простое решение
# чем больше проходов, тем выше точность
def calculate_pi(n_terms: int) -> float:
    numenator: float = 4.0
    denominator: float = 1.0
    operation: float = 1.0
    pi: float = 0.0

    for _ in range(n_terms):
        pi += (numenator / denominator) * operation
        denominator += 2
        operation *= -1

    return pi

if __name__ == '__main__':
    print(calculate_pi(10000000))


from email.policy import default
from problem.models import Problem
from account.decorators import login_required
from forumPost.models import ForumComment, ForumPost
from forumPost.serializers import CreateOrEditForumCommentSerializer, CreateOrEditForumPostSerializer, ForumPostSerializer, ForumCommentSerializer
from utils.api.api import APIView, validate_serializer

class ForumPostApi(APIView):
    @validate_serializer(CreateOrEditForumPostSerializer)
    @login_required
    def post(self, request):
        problemId = request.GET.get("problemId")
        data = request.data
        if not problemId:
            return self.error("Invalid parameter, problemId is required")
        try:
            data["problem"] = Problem.objects.get(id=problemId)
        except Problem.DoesNotExist:
            return self.error("Couldn't find a problem with a provided id")
        data["author"] = request.user
        data["edited"] = False
        try:
            forumPost = ForumPost.objects.create(**data)
        except Exception:
            return self.error("Couldn't create a new forum post")
        return self.success(ForumPostSerializer(forumPost).data)

    @validate_serializer(CreateOrEditForumPostSerializer)
    @login_required
    def put(self, request):
        data = request.data
        id = request.GET.get("forumPostId")
        if not id:
            return self.error("Invalid parameter, forumPostId is required")

        try:
            post = ForumPost.objects.get(id=id)
        except ForumPost.DoesNotExist:
            return self.error("Couldn't find post")

        setattr(post, "content", data["content"])
        setattr(post, "edited", True)

        try:
            post.save()
        except Exception:
            return self.error("Couldn't update the forum post")
        return self.success()

    @login_required
    def get(self, request):
        id = request.GET.get("forumPostId")
        problemId = request.GET.get("problemId")
        if not id and not problemId:
            return self.error("Invalid parameter, forumPostId or problemId is required")
        try:
            db_data = ForumPost.objects.get(id=id) if id else ForumPost.objects.filter(problem_id=problemId)
        except ForumPost.DoesNotExist:
            return self.error("Couldn't find post")

        if id:
            postWithComments = ForumPostSerializer(db_data).data
            postWithComments["comments"] = ForumCommentSerializer(ForumComment.objects.select_related("post").filter(post__id=id), many=True, allow_empty=True, default=[]).data

            return self.success(postWithComments)
        else:
            posts = []
            for post in db_data:
                postWithComments = ForumPostSerializer(post).data
                postWithComments["comments"] = ForumCommentSerializer(ForumComment.objects.select_related(
                    "post").filter(post__id=post.id), many=True, allow_empty=True, default=[]).data
                posts.append(postWithComments)
            return self.success(posts)

class ForumCommentApi(APIView):
    @validate_serializer(CreateOrEditForumCommentSerializer)
    @login_required
    def post(self, request):
        postId = request.GET.get("forumPostId")
        data = request.data
        if not postId:
            return self.error("Invalid parameter, forumPostId is required")
        try:
            data["post"] = ForumPost.objects.get(id=postId)
        except Problem.DoesNotExist:
            return self.error("Couldn't find a forum post with a provided id")
        data["author"] = request.user
        data["edited"] = False
        try:
            forumComment = ForumComment.objects.create(**data)
        except Exception:
            return self.error("Couldn't create a new comment")
        return self.success(ForumCommentSerializer(forumComment).data)

    @validate_serializer(CreateOrEditForumCommentSerializer)
    @login_required
    def put(self, request):
        commentId = request.GET.get("commentId")
        data = request.data
        if not commentId:
            return self.error("Invalid parameter, commentId is required")
        try:
            comment = ForumComment.objects.get(id=commentId)
        except Problem.DoesNotExist:
            return self.error("Couldn't find a forum comment with a provided id")

        setattr(comment, "content", data["content"])
        setattr(comment, "edited", True)

        try:
            comment.save()
        except Exception:
            return self.error("Couldn't update the comment")
        return self.success()


import datetime
import os
import json
import re
import pickle
import requests
from bs4 import BeautifulSoup
from dateutil import parser
from traceback import format_exc
from retry import retry
from project.server.main.utils_swift import upload_object, download_object
from project.server.main.logger import get_logger

logger = get_logger(__name__)

@retry(delay=60, tries=5)
def get_idref_list_in_referentiel():
    NB_ROWS = 10000
    start = 0

    idref_to_download = []
    keep_going = True

    while keep_going:
        url = f'https://www.idref.fr/Sru/Solr?q=recordtype_z:b&version=2.2&start={start}&rows={NB_ROWS}&wt=json&fl=ppn_z'
        res = requests.get(url).json()['response']
        idref_to_download += [p['ppn_z'] for p in res['docs']]
        keep_going = len(res['docs']) > 0
        start += NB_ROWS
        logger.debug(f"{len(idref_to_download)} sur {res['numFound']}")
    return idref_to_download

def get_referentiel(collection_name):
    idref_to_download = get_idref_list_in_referentiel()

    chunk_index = 0
    data_notice, data_parsed = [], []
    MAX_DATA_SIZE = 5000
    #MAX_DATA_SIZE = 100 #fast
    nb_idref = len(idref_to_download)
    logger.debug(f'{nb_idref} idref struct to download and parse')
    for ix, idref in enumerate(idref_to_download):
        if ix % 1000 == 0:
            logger.debug(f'idref struct {ix}')
        try:
            notice = download_referentiel_notice(idref)
            elt_notice = {'idref': idref, 'notice': notice}
            data_notice.append(elt_notice)
            elt_parsed = parse_idref(idref, notice)
            data_parsed.append(elt_parsed)
        except:
            logger.debug(f'error in downloading notice for idref {idref}')
            continue
        if (len(data_notice) > MAX_DATA_SIZE) or (ix == nb_idref - 1):
            if data_notice:
                source = f'idref_struct_raw_{chunk_index}.json'
                target = f'{collection_name}/{source}'
                save_data_referentiel(data_notice, source, target)
                data_notice = []
                chunk_index += 1
    return data_parsed

def save_data_referentiel(data, source, target):
    current_file = source
    json.dump(data, open(current_file, 'w'))
    os.system(f'gzip {current_file}')
    upload_object('theses', f'{source}.gz', f'{target}.gz')
    os.system(f'rm -rf {source}.gz')

def parse_idref(idref, notice):
    soup = BeautifulSoup(notice, 'lxml')
    elt = {}
    idref_elt = soup.find('controlfield', {"tag": "001"})
    try:
        assert(idref_elt.text == idref)
    except Exception as e:
        print("idref not matching in notice")

    elt['idref'] = idref
    alias_idref = []
    aliases = []

    name_elt = soup.find("datafield", {"tag": "210"})
    if name_elt and name_elt.find("subfield", {'code': 'a'}):
        elt['name'] = name_elt.find("subfield", {'code': 'a'}).text

    for f in soup.find_all("datafield", {"tag": "410"}) + soup.find_all("datafield", {"tag": "911"}):
        if f.find("subfield", {'code': 'a'}):
            aliases.append(f.find("subfield", {'code': 'a'}).text)
    elt['aliases'] = list(set(aliases))

    addresses, websites = [], []
    comments = []
    for f in soup.find_all("datafield", {"tag": "340"}):
        if f.find("subfield", {'code': 'a'}):
            txt = f.find("subfield", {'code': 'a'}).text
            if 'adresse' in txt.lower() and len(txt.split(':'))>1:
                addresses.append(':'.join(txt.split(':')[1:]).strip())
            elif 'site internet' in txt.lower() and len(txt.split(':'))>1:
                websites.append(':'.join(txt.split(':')[1:]).strip())
            else:
                comments.append(txt.strip())

    if addresses:
        elt['addresses'] = addresses
    if websites:
        elt['websites'] = websites
    if comments:
        elt['comments'] = comments

    for f in soup.find_all("datafield", {"tag": "010"}):
        if f.find("subfield", {'code': 2}) and f.find("subfield", {'code': 2}).text=='ISNI':
            if f.find("subfield", {'code': 'a'}):
                elt['ISNI'] = f.find("subfield", {'code': 'a'}).text

    for f in soup.find_all("datafield", {"tag": "033"}):
        if f.find("subfield", {'code': 2}) and f.find("subfield", {'code': 2}).text=='BNF':
            if f.find("subfield", {'code': 'a'}):
                elt['BNF'] = f.find("subfield", {'code': 'a'}).text

    for f in soup.find_all("datafield", {"tag": "035"}):
        if f.find("subfield", {'code': 9}) and f.find("subfield", {'code': 9}).text=='sudoc':
            if f.find("subfield", {'code': 'a'}):
                alias_idref.append(f.find("subfield", {'code': 'a'}).text)

        if f.find("subfield", {'code': 2}) and f.find("subfield", {'code': 2}).text=='HAL':
            if f.find("subfield", {'code': 'a'}):
                elt['docid'] = f.find("subfield", {'code': 'a'}).text

        if f.find("subfield", {'code': 2}) and f.find("subfield", {'code': 2}).text=='RNSR':
            if f.find("subfield", {'code': 'a'}):
                elt['rnsr'] = f.find("subfield", {'code': 'a'}).text

        if f.find("subfield", {'code': 2}) and f.find("subfield", {'code': 2}).text=='VIAF':
            if f.find("subfield", {'code': 'a'}):
                elt['viaf'] = f.find("subfield", {'code': 'a'}).text

    elt['alias_idref'] = alias_idref
    return elt

@retry(delay=60, tries=5)
def download_referentiel_notice(idref):
    r = requests.get(f'http://www.idref.fr/{idref}.xml')
    if r.text[0:5] == '<?xml':
        return r.text
    return None

def create_idref_map(data):
    idref_map = {}
    parsed_data = []
    for d in data:
        idrefs = [d['idref']]
        parsed_elt = d
        idrefs += parsed_elt.get('alias_idref', [])
        idrefs = list(set(idrefs))
        parsed_data.append(parsed_elt)
        for idref in idrefs:
            idref_map[str(idref)] = parsed_elt
    logger.debug(f'{len(data)} elts and {len(idref_map)} idrefs (structure) in map')
    return parsed_data, idref_map

def harvest_and_save_idref(collection_name):
    # raw data
    data = get_referentiel(collection_name)

    #parsed data
    parsed_data, idref_map = create_idref_map(data)

    source = f'idref_struct.json'
    target = f'{collection_name}/{source}'
    save_data_referentiel(parsed_data, source, target)

    # idref mapping
    source = f'idref_struct_dict.json'
    target = f'{collection_name}/{source}'
    save_data_referentiel(idref_map, source, target)


from rest_framework.views import APIView
from events.models import Events
from events.serializers import EventSerializers
from rest_framework.permissions import IsAuthenticated
from services.pagination import CustomPagination
from rest_framework.authentication import TokenAuthentication
from rest_framework.response import Response
from rest_framework.status import HTTP_404_NOT_FOUND, HTTP_200_OK
from django.utils.translation import gettext as _
from django.utils.translation import activate, get_language
from django.utils import timezone

class ListEventsAPI(APIView):
    # apply pagination and authentication in Get Events
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAuthenticated]
    pagination_class = CustomPagination

    def get(self, request):
        paginator = self.pagination_class()

        # Display latest Events first
        now = timezone.now()
        events = Events.objects.filter(event_date__lte=now).order_by(
            "-event_date"
        )
        paginated_events = paginator.paginate_queryset(events, request)
        serializer = EventSerializers(
            paginated_events, many=True, context={"user": request.user.id}
        )

        # Filter and include upcoming events
        upcoming_events = Events.objects.filter(event_date__gt=now).order_by(
            "event_date"
        )
        upcoming_serializer = EventSerializers(
            upcoming_events, many=True, context={"user": request.user.id}
        )

        # Get counts for both latest events and upcoming events
        latest_events_count = Events.objects.filter(event_date__lte=now).count()
        upcoming_events_count = Events.objects.filter(event_date__gt=now).count()

        return paginator.get_paginated_response(
            {
                "latest_events_count": latest_events_count,
                "upcoming_events_count": upcoming_events_count,
                "latest_events": serializer.data,
                "upcoming_events": upcoming_serializer.data,
            }
        )

    def patch(self, request, event_id):
        # Activate the language based on the request header
        activate(request.META.get("HTTP_ACCEPT_LANGUAGE"))
        # Get the current language code
        language_code = get_language()

        try:
            event = Events.objects.get(id=event_id)
        except Events.DoesNotExist:
            return Response(
                {"message": _("Event not found"), "data": None},
                status=HTTP_404_NOT_FOUND,
                headers={"Content-Language": language_code},
            )
        user = request.user
        is_liked = request.data.get("is_liked")
        is_bookmarked = request.data.get("is_bookmarked")

        if is_liked:
            event.liked_by.add(user)
        else:
            event.liked_by.remove(user)

        if is_bookmarked:
            event.bookmarked_by.add(user)
        else:
            event.bookmarked_by.remove(user)

        event.save()

        serializer = EventSerializers(event, context={"user": request.user.id})
        return Response(
            {"message": _("Event updated successfully"), "data": serializer.data},
            status=HTTP_200_OK,
            headers={"Content-Language": language_code},
        )

class EventsSearchAPIView(APIView):
    # apply pagination and authentication in Get News
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAuthenticated]
    pagination_class = CustomPagination

    def get(self, request):
        query = request.query_params.get("event", "")
        if query:
            # Perform search based on the query parameter
            results = Events.objects.filter(event_name__icontains=query)
        else:
            # Return all events if no query is provided
            results = Events.objects.all()

        paginator = self.pagination_class()

        paginated_news = paginator.paginate_queryset(results, request)
        serializer = EventSerializers(paginated_news, many=True)
        return paginator.get_paginated_response(serializer.data)


import sys

K, N = map(int, sys.stdin.readline().split())

if N == 1:
    print(-1)
else:
    ans = (K * N) // (N - 1)
    if (K * N) % (N - 1):
        ans += 1
    print(ans)


# -*- coding: utf-8 -*-

deciphers = {}
with open("442-assignment-3-ordered.txt", "r") as file:
    for line in file.readlines():
        line = line.split(":")
        deciphers[line[0].strip()] = line[1].strip()

def count_words(plaintext, word_list):
    count = 0
    for word in word_list:
        count += plaintext.count(word) * len(word)
    return count

word_list = []

MAX_NUM = 10

num = 0

user_input = input("search word: ")
while len(user_input) > 1:
    num = 0
    for item in sorted(deciphers.items(), key = lambda t: count_words(t[1],[user_input]), reverse=True):
        print("Keyword: ", item[0])
        print("Plaintext: ", item[1])
        num += 1
        if num > MAX_NUM:
            break
    user_input = input("search word: ")


load("//bazel:common.bzl", "files_exist")

_TENSORRT_INSTALL_PATH = "TENSORRT_INSTALL_PATH"

def _cc_import_myelin():
    return """
cc_import(
    name = "myelin_compiler_static",
    static_library = "lib/libmyelin_compiler_static.a",
)

cc_import(
    name = "myelin_executor_static",
    static_library = "lib/libmyelin_executor_static.a",
)

cc_import(
    name = "myelin_pattern_library_static",
    static_library = "lib/libmyelin_pattern_library_static.a",
)

cc_import(
    name = "myelin_pattern_runtime_static",
    static_library = "lib/libmyelin_pattern_runtime_static.a",
)

cc_library(
    name = "myelin_static",
    deps = [
        ":myelin_compiler_static",
        ":myelin_executor_static",
        ":myelin_pattern_library_static",
        ":myelin_pattern_runtime_static",
    ]
)

"""

def warn(msg):
    print("{red}{msg}{nc}".format(red = "\033[0;31m", msg = msg, nc = "\033[0m"))

def _impl(repo_ctx):
    tensorrt_path = repo_ctx.os.environ.get(_TENSORRT_INSTALL_PATH, None)
    if tensorrt_path == None:
        warn("Please set the customize tensorrt library path via env var: {}".format(_TENSORRT_INSTALL_PATH))
        tensorrt_path = "/usr/local/TensorRT/"

    repo_ctx.symlink(tensorrt_path + "/include", "include")
    repo_ctx.symlink(tensorrt_path + "/lib", "lib")

    if_has_myelin = all(files_exist(
        repo_ctx,
        [
            "lib/libmyelin_compiler_static.a",
            "lib/libmyelin_executor_static.a",
            "lib/libmyelin_pattern_library_static.a",
            "lib/libmyelin_pattern_runtime_static.a",
        ],
    ))

    repo_ctx.template("BUILD", Label("//bazel/tensorrt:trt.BUILD.tpl"), {
        "%{myelin_static_rule}": _cc_import_myelin() if if_has_myelin else "",
    })
    repo_ctx.template("build_defs.bzl", Label("//bazel/tensorrt:build_defs.bzl.tpl"), {
        "%{IF_HAS_MYELIN}": "True" if if_has_myelin else "False",
    })

tensorrt_configure = repository_rule(
    implementation = _impl,
    local = True,
    environ = [_TENSORRT_INSTALL_PATH],
)


import pygame
from .tile import Tile

# Creates the Tiles of the chessboard
class Tiles:
    # 2d Map where each element is a Tile
    tile_map = None

    # Dictionary of the tiles as a key of the location tuple and a value of the tile object
    tile_dict = {}

    # All the initial tiles where the pieces will be placed
    pieces_on_tiles = None

    # Constructor for tiles that specifically makes all of the tiles that reside inside of tile map
    def __init__(self, window):
        color_black = False
        self.tile_map = pygame.sprite.Group()
        self.pieces_on_tiles = pygame.sprite.Group()
        for row in range(8):
            row_sprites = pygame.sprite.Group()
            for column in range(8):
                tile_square = Tile((255, 248, 220), (column, row), window) if color_black else Tile((139, 69, 19), (column, row), window)
                color_black = not color_black
                self.tile_dict[tile_square.parent_link] = tile_square
                row_sprites.add(tile_square)
            color_black = not color_black
            self.tile_map.add(row_sprites)
            if row < 2 or row > 5:
                self.pieces_on_tiles.add(row_sprites)


from sklearn.linear_model import LogisticRegression

from sklearn.compose import make_column_transformer
from sklearn.compose import make_column_selector as selector

from imblearn.pipeline import make_pipeline as make_pipeline_with_sampler

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, PolynomialFeatures
from sklearn.pipeline import make_pipeline

# pipe = make_pipeline(StandardScaler(), LogisticRegression())
# impute missing values: numeric values with their mean and boolean values (reported symptoms) with False
num_pipe = make_pipeline(
    StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)
)

cat_pipe = make_pipeline(
    SimpleImputer(strategy="constant", fill_value=False),
    OneHotEncoder(),
)

preprocessor_linear = make_column_transformer(
    [num_pipe, selector(dtype_include="number")],
    [cat_pipe, selector(dtype_include="object")],
    n_jobs=2,
)

# use polynomial features to enable the model to fit different sets of parameters for different virus variants
# Also use class_weight='balanced' in logistic regression to correct for imbalance between positive and negative test results.

def create_model():
    return make_pipeline_with_sampler(
        preprocessor_linear,
        PolynomialFeatures(interaction_only=True),
        LogisticRegression(max_iter=1000, class_weight="balanced"),
    )

model = make_pipeline_with_sampler(
    preprocessor_linear,
    PolynomialFeatures(interaction_only=True),
    LogisticRegression(max_iter=1000, class_weight="balanced"),
)


from bayesian_optimization.plotter.plot import Plot
from bayesian_optimization.datahandler.data_table import COLUMNS
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

class RegretPlot(Plot):

    def __init__(self, figsize, title, out_path, df):
        super().__init__(figsize, title, out_path)
        self.df = df
        self.legend = None

    def with_axis_range(self, index:int, ylim, visible=True):
        self._add_twinx_if_needed(index)
        self.axis[index-1].set_ylim(ylim)
        self.axis[index-1].patch.set_alpha(0.1)
        if not visible:
            self.axis[index-1].tick_params(
                axis='y',
                which='both',
                bottom=False,
                top=False,
                left=False,
                right=False,
                labelright=False
            )
        return self

    def with_log_y(self):
        axs = plt.gca()
        axs.set(yscale="log")
        return self

    def set_y_lim(self, lim):
        axs = plt.gca()
        axs.set_ylim(lim)
        return self

    def draw_line_with_mean(self, ax=1, x=COLUMNS.STEP, y=COLUMNS.REGRET, hue=COLUMNS.METHODS, key_list=None, value_list=None, ci=None, linestyle="-", **kwargs):
        return self.draw_line(ax=ax, x=x, y=y, hue=hue, key_list=key_list, value_list=value_list, ci=ci, estimator=np.mean, linestyle=linestyle, **kwargs)

    def draw_line_with_median(self, ax=1, x=COLUMNS.STEP, y=COLUMNS.REGRET, hue=COLUMNS.METHODS, key_list=None, value_list=None, ci=None,linestyle="-", **kwargs):
        return self.draw_line(ax=ax, x=x, y=y, hue=hue, key_list=key_list, value_list=value_list, ci=ci, estimator=np.median, linestyle=linestyle, **kwargs)

    def draw_line(
            self,
            ax=1,
            x=COLUMNS.STEP,
            y=COLUMNS.REGRET,
            hue=COLUMNS.METHODS,
            key_list=None,
            value_list=None,
            ci=None,
            estimator=np.mean,
            linestyle="-",
            **kwargs
    ):
        if key_list is None:  #
            key_list = [COLUMNS.METHODS]
        if value_list is None:  #
            value_list = [("GP",), ("NOMU",), ("DE",), ("DO",)]
        data = self.df[self.df[key_list].apply(tuple, axis=1).isin(value_list)]
        dashes = [(1, 0) for i in data[hue].unique()]
        if linestyle == "-":
            dashes = [(1,0) for i in data[hue].unique()]
        if linestyle == "--":
            dashes = [(2,1) for i in data[hue].unique()]
        splot = sns.lineplot(data=data, x=x, y=y, hue=hue, estimator=estimator, ci=ci, dashes=dashes, style=hue, **kwargs)
        return self


import os

import tensorflow as tf

from tensor2tensor import problems
from tensor2tensor.bin import t2t_decoder  # To register the hparams set
from tensor2tensor.utils import registry
from tensor2tensor.utils import trainer_lib
from tensor2tensor.visualization import attention
from tensor2tensor.visualization import visualization
from IPython.display import Image, display

import numpy as np
from scipy import ndimage
import matplotlib.pyplot as plt
import skimage.transform
import pickle
from PIL import Image
from subprocess import call
import matplotlib.patches as patches

import base64
import csv
import sys

csv.field_size_limit(sys.maxsize)
FIELDNAMES = ['image_id', 'image_w','image_h','num_boxes', 'boxes', 'features']

# Saving the objects:
def save_objects(obj):
    with open('data/objs.pkl', 'w') as f:  # Python 3: open(..., 'wb')
        pickle.dump(obj, f)

# Getting back the objects:
def restore_objects():
    with open('data/objs.pkl') as f:  # Python 3: open(..., 'rb')
        obj = pickle.load(f)
    return obj

# PUT THE MODEL YOU WANT TO LOAD HERE!
CHECKPOINT = os.path.expanduser('data/t2t_train/image_ms_coco_tokens32k_object_position/transformer-transformer_base_single_gpu')

# HParams
problem_name = 'image_ms_coco_tokens32k'
data_dir = os.path.expanduser('data/t2t_data_caption_bottom')
model_name = "transformer"
hparams_set = "transformer_base_single_gpu"

# ## Visualization
use_bottom_up_features = True
visualizer = visualization.AttentionVisualizer(hparams_set, use_bottom_up_features, model_name, data_dir, problem_name, beam_size=4)

tf.Variable(0, dtype=tf.int64, trainable=False, name='global_step')

sess = tf.train.MonitoredTrainingSession(
    checkpoint_dir=CHECKPOINT,
    save_summaries_secs=0,
)

image_file_name = 'COCO_val2014_000000076619'
image_file_parts = image_file_name.split('_')
image_id = int(image_file_parts[2])
image_path = os.path.expanduser('data/visualize/'+image_file_name+'.jpg')
bottom_up_feature_filepath = os.path.expanduser('data/tmp/t2t_datagen_caption/bottom_up_features/trainval/karpathy_val_resnet101_faster_rcnn_genome.tsv')

if use_bottom_up_features:
    with open(bottom_up_feature_filepath, "r+b") as tsv_in_file:
        reader = csv.DictReader(tsv_in_file, delimiter='\t', fieldnames=FIELDNAMES)
        for item in reader:
            item['image_id'] = int(item['image_id'])
            item['image_h'] = int(item['image_h'])
            item['image_w'] = int(item['image_w'])
            item['num_boxes'] = int(item['num_boxes'])
            for field in ['boxes', 'features']:
                item[field] = np.frombuffer(base64.decodestring(item[field]),
                                            dtype=np.float32).tolist()
            if image_id == item['image_id']:
                print('Found Matching object features')
                break

output_string, inp_text, out_text, att_mats = visualizer.get_vis_data_from_image(sess, image_id, image_path, item['features'], item['boxes'], use_bottom_up_features)
save_objects([output_string, inp_text, out_text, att_mats])

output_string, inp_text, out_text, att_mats = restore_objects()
print(np.shape(att_mats[0]))
print(np.shape(att_mats[1]))
print(np.shape(att_mats[2]))
print(output_string)

if use_bottom_up_features:
    img = ndimage.imread(image_path, mode='RGB')
    for l in range(4,6):
        for a in range(8):
            # Plot original image
            # img = ndimage.imread(image_path)
            plt.subplot(4, 4, 1)
            plt.imshow(img)
            plt.axis('off')

            # Plot images with attention weights
            words = out_text
            for t in range(len(words)):
                plt.subplot(4, 4, t + 2)
                words[t] = words[t].replace('_', '')
                plt.text(0, 1, '%s' % words[t], color='black', backgroundcolor='white', fontsize=8)
                plt.imshow(img)
                input_output_attn = att_mats[2]
                alp_curr = np.zeros([item['image_w'], item['image_h']])
                num_boxes = item['num_boxes']
                boxes = np.asarray(item['boxes']).reshape(-1, 4)
                for box_index in range(num_boxes):
                    for x in range(int(boxes[box_index][0]), int(boxes[box_index][2])):
                        for y in range(int(boxes[box_index][1]), int(boxes[box_index][3])):
                            alp_curr[x][y] += input_output_attn[l][0][a][t][box_index]
                alp_img = alp_curr

                #alp_img = skimage.transform.pyramid_expand(alp_curr, upscale=16, sigma=20)
                plt.imshow(np.transpose(alp_img), alpha=0.5)
                plt.axis('off')
            # plt.show()
            attn_file = 'data/visualize' + str(image_file_name) \
                        + '/input_output_' + 'layer_' + str(l) + 'head' + str(a)

            directory = os.path.dirname(attn_file)
            try:
                os.stat(directory)
            except:
                os.mkdir(directory)
            plt.savefig(attn_file, format='png', dpi=1000)
    exit(0)

img = inp_text.astype(int)
for l in range(6):
    for a in range(8):
        # Plot original image
        #img = ndimage.imread(image_path)
        plt.subplot(4, 4, 1)
        plt.imshow(img)
        plt.axis('off')

        # Plot images with attention weights
        words = out_text
        for t in range(len(words)):
            plt.subplot(4, 4, t+2)
            words[t] =words[t].replace('_', '')
            plt.text(0, 1, '%s' % words[t], color='black', backgroundcolor='white', fontsize=8)
            plt.imshow(img)
            input_output_attn = att_mats[2]
            alp_curr = input_output_attn[l][0][a][t].reshape(14, 14)
            alp_img = alp_curr
            alp_img = skimage.transform.pyramid_expand(alp_curr, upscale=16, sigma=20)
            plt.imshow(alp_img, alpha=0.85)
            plt.axis('off')
        #plt.show()
        attn_file = 'data/visualize/'+ str(image_file_name) \
                    + '/input_output_' + 'layer_' + str(l) + 'head' + str(a)

        directory = os.path.dirname(attn_file)
        try:
            os.stat(directory)
        except:
            os.mkdir(directory)
        plt.savefig(attn_file, format='png', dpi=1000)


#6485. 삼성시의 버스 노선

T = int(input())

for test_case in range(1, T + 1):
    n = int(input())
    ai_list = [0] * n
    bi_list = [0] * n
    for i in range(n):
        a, b = map(int, input().split())
        ai_list[i] = a
        bi_list[i] = b
    p = int(input())
    counter = [0] * p
    for i in range(p):
        counted = 0
        st = int(input())
        for j in range(n):
            if st >= ai_list[j] and st<=bi_list[j]:
                counted+=1
        counter[i] = counted
    print(f'#{test_case}', end = ' ')
    for i in range(p):
        print(counter[i], end = ' ')
    print()


from ListaDoblementeEnlazada import ListaDoblementeEnlazada #Importación del TDA Lista Doble
from ListaSimple import ListaSimple                         #Importación del TDA Lista Simpre
import xml.etree.ElementTree as ET                          #Importacion de el módulo ElementTree
from xml import etree                                       #Importacion del módulo para leer archivos xml

#  C:\Users\Squery\Desktop\Proyecto 1 IPC2\Matriz.xml
#  C:\Users\Squery\Desktop\Proyecto 1 IPC2\Matriz2.xml

ruta = ""
lista_enlazada = ListaDoblementeEnlazada()

#Obtiene la ruta del archivo que se desea leer
def cargar_archivo():
    global ruta
    ruta = input()

#Este método lee el archivo y lo ingresa en los tda listas
def procesar_archivo(ruta):
    global lista_enlazada
    try:
        #Se obtiene todo el archivo leído
        tree = ET.parse(ruta)
        #Se obtiene la raíz del archivo
        root = tree.getroot()
        #Con este for se recorre toda la raíz para buscar los diferentes elementos obtenidos dentro de ella
        for elemento in root:
            #Contador para comparar si las dimensiones de la matriz leída coinciden con los elementos del archivo
            contador = 0
            #Lista simple temporal
            lista_simple = ListaSimple()
            #Con este for se ingresan todos los datos de una matriz en la lista simple
            for subelemento in elemento:
                lista_simple.insertar(int(subelemento.attrib['x']), int(subelemento.attrib['y']), int(subelemento.text))
                #Por cada elemento ingresado en la matriz temporal se aumenta en 1 el contador
                contador += 1
                #Si los tamaños de n y m multiplicados coinciden con el numero del contador se ingresan
                #los datos a la lista doble
            if int(elemento.attrib['n'])*int(elemento.attrib['m']) == contador and lista_enlazada.comprobar_nombre(elemento.attrib["nombre"]) != True:
                #matriz_patrones(lista_simple)
                lista_enlazada.insertar_final(elemento.attrib['nombre'], int(elemento.attrib['n']), int(elemento.attrib['m']), lista_simple)

                #Se verifica que si un nombre de la matriz ya existe, no se deje ingresar otra matriz con
                #el mismo nombre y se procede a cerrar el programa
            elif lista_enlazada.comprobar_nombre(elemento.attrib["nombre"]) == True:
                print("Matrices con nombres repetidos!")
                exit()
            else:
                #Si los tamaños de la n y m multiplicados no coinciden con el contador, no se deja
                #ingresar la matriz y se procede a cerrar el programa
                print("Los tamaños de n, m, no coinciden con los datos leídos del archivo")
                exit()
        print('archivo procesado con exito\n')
    except IOError:
        #Si el archivo posee algun tipo de error muestra un mensaje de que no se pudo leer
        print("El archivo no se pudo leer", IOError)

def crear_archivo():

    lista_enlazada.mostrar()
    lista_enlazada.obtener_longitud()

#Variable de tipo booleano para mantener el bucle while del menú siempre y cuando no se seleccione
#la opción de salir
x = True
while x:
    print('\n\nMenú principal')
    print('      1. Cargar archivo')
    print('      2. Procesar archivo')
    print('      3. Escribir archivo de salida')
    print('      4. Mostrar datos del estudiante')
    print('      5. Generar gráfica')
    print('      6. Salida')

    entrada = input('Ingrese una opción\n')

    if entrada == '1':
        print('Ingrese la ruta del archivo: ')
        cargar_archivo()

    elif entrada == '2':
        print("Procesando archivo")
        procesar_archivo(ruta)
        #lista_enlazada.mostrar()

    elif entrada == '3':
        crear_archivo()

    elif entrada == '4':
        print('>Diego Abraham Robles Meza\n>201901429\n>Introducción a la programación y computación 2 Sección D\n>Ingeniería en ciencias y sistemas\n>5to Semestre')

    elif entrada == '5':
        print("ingrese el nombre de la matriz a graficar")
        nombre = input()
        lista_enlazada.grafos(nombre)

    elif entrada == '6':
        print('Saliento del programa')
        exit()
    elif int(entrada) > 6 or int(entrada) < 0:
        print('Opción inválida')


import os
import re
import copy
import json
import tensorflow as tf
import numpy as np
import time
from classes.keyboard_structure import KeyboardStructure
from classes.characters_placement import CharactersPlacement

start_time = time.time()

random_seed=961
np.random.seed(random_seed)
tf.random.set_seed(random_seed)

punct = True
number_of_characters_placements = 10000

def _preprocess_line(line):
    _regex = re.compile('[^%s]' % ''.join(sorted(set(initial_characters_placement))))
    return _regex.sub('', line)

def build_corpus():
    corpus_path = "./data_dir/TED2013 v1.1.txt"
    maximum_line_length = 500
    searching_corpus_size = 10000

    corpus = open(corpus_path, 'r', encoding='utf-8').read().split('\n')
    corpus = [line for line in corpus if len(line) <= maximum_line_length]

    rng = np.random.RandomState(random_seed)
    rng.shuffle(corpus)

    searching_corpus = [_preprocess_line(line) for line in corpus[:searching_corpus_size]]

    if len(searching_corpus) < searching_corpus_size:
        print('Searching corpus size didn\'t reach %s, its current size is %s' %
                    (searching_corpus_size, len(searching_corpus)))

    searching_corpus_dict = dict()
    for line in searching_corpus:
        for char in line.strip():
            try:
                searching_corpus_dict[char] += 1
            except:
                searching_corpus_dict[char] = 1

    return searching_corpus_dict

def buildCarpalxInput(keyboard):
    carpalx_file_name = "carpalx-0.12\keren\keren.conf"
    with open(carpalx_file_name, 'w') as carpalx_file:
        carpalx_file.write(
            "<keyboard>\n<row 1>\nkeys    = `~ 1! 2@ 3\\# 4$ 5% 6^ 7& 8* 9( 0) -_ =+\nfingers =  0  1  1   2  3  3  3  6 7   7  8  9  9\n</row>\n<row 2>\nkeys    =")

        letters = 0
        for button, character in zip(keyboard_structure.buttons, keyboard):
            if character in my_letters:
                letters += 1
                carpalx_file.write(" " + character)

                if letters == 9:
                    carpalx_file.write(" ;: [{ ]} \\|\nfingers = 0 1 2 3 3 6 6 7 8 9  9  9  9\n</row>\n<row 3>\nkeys    =")

                if letters == 19:
                    carpalx_file.write(" '\"\nfingers = 0 1 2 3 3 6 6 7 8  9  9\n</row>\n<row 4>\nkeys    =")

                if letters == 26:
                    carpalx_file.write(" ,< .> /?\nfingers = 0 1 2 3 3 6 6  7  8  9\n</row>\n</keyboard>")
                    break

        carpalx_file.close()

def buildCarpalxInput_punct(keyboard):
    punct_map = {"-":"-_", "+":"=+", "{":"[{", "}":"]}", ";":";:", "'":"'\"", ",":",<", ".":".>", "?":"/?"}

    carpalx_file_name = "carpalx-0.12\keren\keren.conf"
    with open(carpalx_file_name, 'w') as carpalx_file:
        carpalx_file.write("<keyboard>\n<row 1>\nkeys    = `~ 1! 2@ 3\\# 4$ 5% 6^ 7& 8* 9( 0)")

        letters = 0
        for button, character in zip(keyboard_structure.buttons, keyboard):
            if character in my_letters:
                letters += 1

                if character in punct_map.keys():
                    carpalx_file.write(" " + punct_map[character])
                elif character>='a' and character<='z':
                    carpalx_file.write(" " + character)
                else:
                    print("wrong character: ", character)
                    exit(1)

                if letters == 2:
                    carpalx_file.write("\nfingers =  0  1  1   2  3  3  3  6 7   7  8  9  9\n</row>\n<row 2>\nkeys    =")

                if letters == 14:
                    carpalx_file.write(" \\|\nfingers = 0 1 2 3 3 6 6 7 8 9  9  9  9\n</row>\n<row 3>\nkeys    =")

                if letters == 25:
                    carpalx_file.write(" \nfingers = 0 1 2 3 3 6 6 7 8  9  9\n</row>\n<row 4>\nkeys    =")

                if letters == 35:
                    carpalx_file.write(" \nfingers = 0 1 2 3 3 6 6  7  8  9\n</row>\n</keyboard>")
                    break

        carpalx_file.close()

#open genetic_config
if(punct):
    path_str = "and_punctuations"
else:
    path_str = "only"

genetic_config_path = "searched_characters_placements\letters_" + path_str + "_dir\genetic_config.json"
with open(genetic_config_path, 'r') as file:
    genetic_config = json.load(file)
initial_characters_placement = CharactersPlacement(characters_set=genetic_config['characters_set'])
keyboard_structure = KeyboardStructure(
    name=genetic_config['keyboard_structure']['name'],
    width=genetic_config['keyboard_structure']['width'],
    height=genetic_config['keyboard_structure']['height'],
    buttons=genetic_config['keyboard_structure']['buttons'],
    hands=genetic_config['hands']
)

#my buttons ids
my_letters = []
my_buttons = [keyboard_structure.buttons[i].id for i in range(len(keyboard_structure.buttons))]
for c in initial_characters_placement.characters_set:
    if(c.button_id is not None):
        my_buttons.remove(c.button_id)
    else:
        my_letters.append(c.character)
number_of_characters = len(my_letters)

print("generating labeled keyboards...")
searching_corpus_dict = build_corpus()
characters_placements = list()
labels = np.zeros([number_of_characters_placements], dtype=float)
for i in range(number_of_characters_placements):
    characters_placements.append(copy.deepcopy(initial_characters_placement))
    characters_placements[-1].randomize()

    if(punct):
        buildCarpalxInput_punct(characters_placements[-1])
    else:
        buildCarpalxInput(characters_placements[-1])
    my_cmd = "perl carpalx-0.12/keren/carpalx_keren -conf carpalx-0.12/etc/tutorial-00.conf"
    my_cmd_output = os.popen(my_cmd)
    for line in my_cmd_output:
        labels[i] = float(line.rstrip())

#one-hot from bzq
features = np.zeros([number_of_characters_placements,number_of_characters,number_of_characters], dtype=int)
for cp in range(len(characters_placements)):
    for l in range(len(my_letters)):
        for button, character in zip(keyboard_structure.buttons, characters_placements[cp]):
            if character == my_letters[l]:
                #if (button.id==22):
                #    print(cp)
                #    for button, character in zip(keyboard_structure.buttons, characters_placements[cp]):
                #        print(button.id, character)
                #    exit()
                position = my_buttons.index(button.id)
                features[cp][l][position] = 1
                break

#qwerty
#qpath = "predefined_characters_placements\qwerty_dir\genetic_config.json"
#with open(qpath, 'r') as file:
#    qgenetic_config = json.load(file)
#qwerty_placement = CharactersPlacement(characters_set=qgenetic_config['characters_set'])
#qwerty_onehot = np.zeros([1,number_of_characters,number_of_characters], dtype=int)
#for l in range(len(my_letters)):
#    for button, character in zip(keyboard_structure.buttons, qwerty_placement):
#        if character == my_letters[l]:
#            position = my_buttons.index(button.id)
#            qwerty_onehot[0][l][position] = 1
#            break

#qwerty_placement.calculate_fitness(keyboard_structure, searching_corpus_dict)
#qwerty_label = qwerty_placement.fitness

#save generated data
path_str = ""
if(punct):
    path_str = "_punct"

data_file_path = "carpalx-0.12\keren\data" + path_str + ".txt"
with open(data_file_path, 'w') as data_file:
    for f in range(len(features)):
        for i in range(len(my_letters)):
            for j in range(len(my_letters)):
                data_file.write(str(features[f][i][j]) + " ")
        data_file.write(str(labels[f]) + "\n")

data_file.close()

print("time: ", time.time() - start_time)


import nexusproto.NexusContent_pb2 as nexusproto
from nexusproto.serialization import from_shaped_array, to_shaped_array
from springxd.tcpstream import start_server, LengthHeaderTcpProcessor

def transform(self, tile_data):
    """
    This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180

    :param self:
    :param tile_data: The tile data
    :return: Tile data with altered longitude values
    """
    nexus_tile = nexusproto.NexusTile.FromString(tile_data)

    the_tile_type = nexus_tile.tile.WhichOneof("tile_type")

    the_tile_data = getattr(nexus_tile.tile, the_tile_type)

    longitudes = from_shaped_array(the_tile_data.longitude)

    # Only subtract 360 if the longitude is greater than 180
    longitudes[longitudes > 180] -= 360

    the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes))

    yield nexus_tile.SerializeToString()

def start():
    start_server(transform, LengthHeaderTcpProcessor)

if __name__ == "__main__":
    start()


#person_2.py  22Dec2020  crs
"""
A bit more in example of a class
Embodies a person's information
"""
class Person:
    """
    Embodies a person's information
    """

    # Class instance (object) setup function
    def __init__(self,      # Reference to us
                 name):     # Person's name: required
                            #Saving setup info
        self.name = name

    def __str__(self):      # Used when object accessed
                            # as string
        """ Give object's string representation
        """
        ret = f"Person: {self.name}"
        return ret

# Selftest
if __name__ == "__main__":
    per = Person("Ray")
    print(per.name,per)

    per = Person("Arlene")
    print(per.name,per)

    per = Person("Rich")
    print(per.name, per)


import tensorflow as tf

def unique_with_counts(tensor_1d: tf.Tensor):
    """
    Same as `tf.unique_with_counts` function, but compatible with TPU
    (`tf.unique_with_counts` function does not support TPU)
    """
    unique, indices = tf.unique(tensor_1d)
    counts = tf.map_fn(lambda v: tf.cast(tf.math.count_nonzero(v == tensor_1d), unique.dtype), unique)

    return unique, indices, counts

def get_rank(data: tf.Tensor) -> tf.Tensor:
    """
    Get averaged rank of input data

    Example
    data: [ 1,  2,  5, 12,  3,  4,  5,  3,  1,  3]
    return: [ 1.5,  3. ,  8.5, 10. ,  5. ,  7. ,  8.5,  5. ,  1.5,  5. ]

    :param data: Input data shaped [BatchSize]
    :returns: averaged rank of each item.
    """
    tf.debugging.assert_rank(data, 1)

    _, index, counts = unique_with_counts(tf.sort(data))
    counts = tf.cast(counts, tf.float32)
    end_numbers = tf.scan(lambda x, y: x + y, counts)
    unique_ranks = end_numbers - (counts - 1) / 2
    average_ranks = tf.map_fn(lambda i: unique_ranks[i], index, dtype=tf.float32)

    increasing_rank = tf.argsort(tf.argsort(data))
    average_ranks_of_input = tf.map_fn(lambda i: average_ranks[i], increasing_rank, dtype=tf.float32)
    return average_ranks_of_input

def pearson_correlation_coefficient(x: tf.Tensor, y: tf.Tensor) -> tf.Tensor:
    """
    Calculate Pearson correlation coefficients

    :param x: Input Tensor shaped [BatchSize]
    :param y: Input Tensor shaped [BatchSize]
    :returns: pearson correlation scalar tensor
    """
    x = tf.cast(x, tf.float32)
    y = tf.cast(y, tf.float32)

    tf.debugging.assert_rank(x, 1)
    tf.debugging.assert_rank(y, 1)

    x_deviation = x - tf.reduce_mean(x)
    y_deviation = y - tf.reduce_mean(y)

    pearson_corr = tf.reduce_sum(x_deviation * y_deviation) / (tf.norm(x_deviation) * tf.norm(y_deviation))
    return pearson_corr

def spearman_correlation_coefficient(x: tf.Tensor, y: tf.Tensor) -> tf.Tensor:
    """
    Calculate Spearman correlation coefficients

    :param x: Input Tensor shaped [BatchSize]
    :param y: Input Tensor shaped [BatchSize]
    :returns: spearman correlation scalar tensor
    """
    x_rank = get_rank(x)
    y_rank = get_rank(y)
    return pearson_correlation_coefficient(x_rank, y_rank)

class SparseCategoricalAccuracy(tf.keras.metrics.Metric):
    """Normal sparse categorical accuracy with ignore index"""

    def __init__(self, ignore_index: int = 0, name="accuracy"):
        super().__init__(name=name)

        self.ignore_index = ignore_index
        self.total_sum = self.add_weight(name="total_sum", initializer="zeros")
        self.total_count = self.add_weight(name="total_count", initializer="zeros")

    def update_state(self, y_true, y_pred, sample_weight=None):
        accuracy = tf.keras.metrics.sparse_categorical_accuracy(y_true, y_pred)
        accuracy = tf.boolean_mask(accuracy, y_true != self.ignore_index)
        if sample_weight is not None:
            accuracy = tf.multiply(accuracy, sample_weight)

        self.total_sum.assign_add(tf.reduce_sum(accuracy))
        self.total_count.assign_add(tf.cast(tf.shape(accuracy)[0], tf.float32))

        return accuracy

    def result(self):
        return self.total_sum / self.total_count

class BinaryF1Score(tf.keras.metrics.Metric):
    """Binary F1 Score"""

    def __init__(self, threshold: float = 0.5, name="f1_score"):
        super().__init__(name=name)

        self.threshold = threshold
        self.true_positive = self.add_weight(name="true_positive", initializer="zeros")
        self.true_ground_truth = self.add_weight(name="true_ground_truth", initializer="zeros")
        self.true_predicted = self.add_weight(name="true_predicted", initializer="zeros")
        self.eps = 1e-8

    def update_state(self, y_true, y_pred, sample_weight=None):
        y_true = tf.cast(y_true > self.threshold, dtype=tf.float32)
        y_pred = tf.cast(y_pred > self.threshold, dtype=tf.float32)

        true_positive = tf.reduce_sum(y_true * y_pred, axis=-1)
        true_ground_truth = tf.reduce_sum(y_true, axis=-1)
        true_predicted = tf.reduce_sum(y_pred, axis=-1)

        self.true_positive.assign_add(true_positive)
        self.true_ground_truth.assign_add(true_ground_truth)
        self.true_predicted.assign_add(true_predicted)

    def result(self):
        precision = self.true_positive / (self.true_predicted + self.eps)
        recall = self.true_positive / (self.true_ground_truth + self.eps)
        return (2 * precision * recall) / (precision + recall + self.eps)

class PearsonCorrelationMetric(tf.keras.metrics.Metric):
    """Pearson correlation coefficient metric"""

    def __init__(self, name="pearson_coef"):
        super().__init__(name=name)

        self.coef_sum = self.add_weight(name="coef_sum", initializer="zeros")
        self.total_count = self.add_weight(name="total_count", initializer="zeros")

    def update_state(self, y_true: tf.Tensor, y_pred: tf.Tensor, sample_weight=None) -> tf.Tensor:
        """
        Update inner metric value using y_ture and y_pred

        :param y_true: true label tensor shaped [BatchSize]
        :param y_pred: pred label tensor shaped [BatchSize]
        :return: pearson correlation coefficient metric value of inputs
        """
        pearson_coef = pearson_correlation_coefficient(y_true, y_pred)

        count = tf.cast(tf.shape(y_true)[0], tf.float32)
        self.coef_sum.assign_add(pearson_coef * count)
        self.total_count.assign_add(count)
        return pearson_coef

    def result(self):
        return self.coef_sum / self.total_count

class SpearmanCorrelationMetric(tf.keras.metrics.Metric):
    """Spearman correlation coefficient metric"""

    def __init__(self, name="spearman_coef"):
        super().__init__(name=name)

        self.coef_sum = self.add_weight(name="coef_sum", initializer="zeros")
        self.total_count = self.add_weight(name="total_count", initializer="zeros")

    def update_state(self, y_true: tf.Tensor, y_pred: tf.Tensor, sample_weight=None) -> tf.Tensor:
        """
        Update inner metric value using y_ture and y_pred

        :param y_true: true label tensor shaped [BatchSize]
        :param y_pred: pred label tensor shaped [BatchSize]
        :return: spearman correlation coefficient metric value of inputs
        """
        spearman_coef = spearman_correlation_coefficient(y_true, y_pred)

        count = tf.cast(tf.shape(y_true)[0], tf.float32)
        self.coef_sum.assign_add(spearman_coef * count)
        self.total_count.assign_add(count)
        return spearman_coef

    def result(self):
        return self.coef_sum / self.total_count


from flask import Flask
from flask.helpers import send_from_directory
import os

app = Flask(__name__, static_url_path='', static_folder='.')
app.config['CORS_HEADERS'] = 'Content-Type'

# CORS section

@app.after_request
def after_request_func(response):
    response.headers.add("Access-Control-Allow-Origin", "*")
    response.headers.add('Access-Control-Allow-Headers', "*")
    response.headers.add('Access-Control-Allow-Methods', "*")
    return response
# end CORS section

import error_handles

# Add your API endpoints here
from routes import mail
# from routes import cars
# ...

@app.route('/')
def get_endpoint_function():
    try:
        res = "
FLASK API HOME
"
        return res

    except Exception as e:
        print(e)

# Setting Favicon
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                               'favicon.ico', mimetype='image/vnd.microsoft.icon')

if __name__ == "__main__":
    app.run(debug=True)