#!/usr/bin/python

# -*- coding: utf-8 -*-

"""
PyCOMPSs Testbench Tasks
========================
"""

# Imports
import unittest

from pycompss.api.task import task
from pycompss.api.parameter import *

class testFiles(unittest.TestCase):

    @task(fin=FILE_IN, returns=str)
    def checkName(self, fin, name):
        print("TEST FILE NAME")
        if fin.endswith(name):
            print("The file name is OK: " + str(fin))
        else:
            raise Exception("FILENAME NOT AS EXPECTED: " + str(fin) + " != " + str(name))
        return name

    @task(fin=FILE, returns=str)
    def fileIn(self, fin):
        print("TEST FILE IN")
        # Open the file and read the content
        fin_d = open(fin, 'r')
        content = fin_d.read()
        print("- In file content:\n", content)
        # Close and return the content
        fin_d.close()
        return content

    @task(finout=FILE_INOUT, returns=str)
    def fileInOut(self, finout):
        print("TEST FILE INOUT")
        # Open the file and read the content
        finout_d = open(finout, 'r+')
        content = finout_d.read()
        print("- Inout file content:\n", content)
        # Add some content
        content += "\n===> INOUT FILE ADDED CONTENT"
        finout_d.write("\n===> INOUT FILE ADDED CONTENT")
        print("- Inout file content after modification:\n", content)
        # Close and return with the modification
        finout_d.close()
        return content

    @task(fout=FILE_OUT, returns=str)
    def fileOut(self, fout, content):
        print("TEST FILE OUT")
        # Open the file for writting and write some content
        with open(fout, 'w') as fout_d:
            fout_d.write(content)
        print("- Out file content added:\n", content)
        return content

    @task(returns=FILE)
    def returnFile(self, filename, content):
        print("TEST RETURN FILE")
        # Open the file for writting and write some content
        fout_d = open(filename, 'w')
        fout_d.write(content)
        print("- Out file name: ", filename)
        print("- Out file content added: ", content)
        # Close and return the content written
        fout_d.close()
        return filename

    @task(returns=(FILE, FILE))
    def multireturnFile(self, content1, content2):
        print("TEST MULTIRETURN FILE")
        filename1 = 'retFile1'
        filename2 = 'retFile2'
        # Open the file for writting and write some content
        fout_d1 = open(filename1, 'w')
        fout_d1.write(content1)
        print("- Out file name: ", filename1)
        print("- Out file content added: ", content1)
        fout_d2 = open(filename2, 'w')
        fout_d2.write(content2)
        print("- Out file name: ", filename2)
        print("- Out file content added: ", content2)
        # Close and return the content written
        fout_d1.close()
        fout_d2.close()
        return filename1, filename2

    @task(returns=bool, first=FILE_IN, second=FILE_INOUT, third=FILE_OUT)
    def fileNone(self, first=None, second=None, third=None):
        if first is None and \
           second is None and \
           third is None:
            return True
        else:
            return False

    def testFileNAME(self):
        """ Test FILE NAME """
        from pycompss.api.api import compss_wait_on
        fin = "complex_name.txt"
        content = "FILE CONTENT"
        with open(fin, 'w') as f:
            f.write(content)
        res = self.checkName(fin, fin)
        res = compss_wait_on(res)
        self.assertEqual(res, fin, "File names are not equal: {}, {}".format(res, fin))

    def testFileIN(self):
        """ Test FILE_IN """
        from pycompss.api.api import compss_wait_on
        fin = "infile"
        content = "IN FILE CONTENT"
        with open(fin, 'w') as f:
            f.write(content)
        res = self.fileIn(fin)
        res = compss_wait_on(res)
        self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content))

    def testFileOUT(self):
        """ Test FILE_OUT """
        from pycompss.api.api import compss_wait_on
        from pycompss.api.api import compss_open
        fout = "outfile"
        content = "OUT FILE CONTENT"
        res = self.fileOut(fout, content)
        res = compss_wait_on(res)
        with compss_open(fout, 'r') as fout_r:
            content_r = fout_r.read()
        # The final file is only stored after the execution.
        # During the execution, you have to use the compss_open, which will
        # provide the real file where the output file is.
        # fileInFolder = os.path.exists(fout)
        # self.assertTrue(fileInFolder, "FILE_OUT is not in the final location")
        self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content))
        self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))

    def testFileINOUT(self):
        """ Test FILE_INOUT """
        from pycompss.api.api import compss_wait_on
        from pycompss.api.api import compss_open
        finout = "inoutfile"
        content = "INOUT FILE CONTENT"
        with open(finout, 'w') as f:
            f.write(content)
        res = self.fileInOut(finout)
        res = compss_wait_on(res)
        with compss_open(finout, 'r') as finout_r:
            content_r = finout_r.read()
        content += "\n===> INOUT FILE ADDED CONTENT"
        self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content))
        self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))

    @unittest.skip("not supported skipping")
    def testReturnFile(self):
        """ Test return FILE """
        from pycompss.api.api import compss_wait_on
        fret = "fret"
        content = "RETURN FILE CONTENT"
        res = self.returnFile(fret, content)
        res = compss_wait_on(res)
        with compss_open(res, 'r') as f:
            content_r = f.read()
        self.assertEqual(res, fret, "strings are not equal: {}, {}".format(res, content))
        self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))

    @unittest.skip("not supported - skipping")
    def testMultiReturnFile(self):
        """ Test multireturn FILE """
        fretout1 = "retFile1"
        fretout2 = "retFile2"
        content1 = "RETURN FILE CONTENTS A"
        content2 = "RETURN FILE CONTENTS B"
        retResA, retResB = self.multireturnFile(fretout1, fretout2, content1, content2)
        retResA = compss_wait_on(retResA)
        retResB = compss_wait_on(retResB)
        with compss_open(retResA, 'r') as f:
            content1_r = f.read()
        with compss_open(retResB, 'r') as f:
            content2_r = f.read()
        self.assertEqual(res, fret, "strings are not equal: {}, {}".format(content2_r, content2))
        self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content1_r, content1))

    def testWorkflowFiles(self):
        """ Test a workflow with FILES """
        from pycompss.api.api import compss_open
        fin = "inwork"
        fout = "outwork"
        content = "Before the task "
        with open(fin, 'w') as f:
            f.write(content)
        res = self.fileIn(fin)
        f2 = self.fileOut(fout, res)
        f3 = self.fileInOut(fout)
        with compss_open(fout, 'r') as f:
            content_r = f.read()
        content += "\n===> INOUT FILE ADDED CONTENT"
        self.assertEqual(content, content_r, "strings are not equal: {}, {}".format(content_r, content))

    def testFilesWithNone(self):
        """ Test a workflow with FILES using None"""
        from pycompss.api.api import compss_wait_on
        fin = None
        finout = None
        fout = None
        res = self.fileNone(fin, finout, fout)
        res = compss_wait_on(res)
        self.assertEqual(res, True, "A parameter was not None in the fileNone task.")
        res = self.fileNone()
        res = compss_wait_on(res)
        self.assertEqual(res, True, "A parameter was not None in the fileNone task using default parameters.")
        # makes sense to wait on a None file? No


from tkinter import*
import tkinter
from tkinter.constants import LEFT
from tkinter.ttk import Button, Frame, Label
from random import randint
root=tkinter.Tk()
root.geometry("1270x627+0+0")
root.title("WEL COME TO KBC")

root.config(bg="black")

leftframe=Frame(root)
leftframe.grid(row=0,column=0)

topframe=Frame(leftframe)
topframe.grid()

buttonframe=Frame(leftframe)
buttonframe.grid(row=2,column=0)

rightframe=Frame(root)
rightframe.grid(row=0,column=1)

img=PhotoImage(file="/home/priya/Downloads/KBC2.png")

lifeline50=Button(topframe,image=img)
lifeline50.grid(row=0,column=0)

rightframe=Frame(root)
rightframe.grid(row=0,column=1)

img=PhotoImage(file="271.4 kB (2,71,361 bytes)/call")

life=Button(topframe,image=img)
life.grid(row=0,column=1)

root.mainloop()


import pytest

from colibris.middleware.errors import handle_errors_json
from colibris.schemas import ModelSchema

from colibris import persist
from colibris.views.filtering import fields, operators
from colibris.views.filtering.base import ModelFilter
from colibris.views.generic import ListCreateModelView

class Item(persist.Model):
    name = persist.CharField()
    count = persist.IntegerField()
    info = persist.TextField()

class ItemSchema(ModelSchema):
    class Meta:
        model = Item

class ExplicitFieldsFilter(ModelFilter):
    name = fields.String(field='name', operation=operators.EQ)
    name__regexp = fields.String(field='name', operation=operators.REGEXP)
    name__not = fields.String(field='name', operation=operators.NOT)
    name__ilike = fields.String(field='name', operation=operators.ILIKE)
    count__gt = fields.String(field='count', operation=operators.GT)
    count__ge = fields.String(field='count', operation=operators.GE)
    count__lt = fields.String(field='count', operation=operators.LT)
    count__le = fields.String(field='count', operation=operators.LE)

    class Meta:
        model = Item

class GeneratedFieldsFilter(ModelFilter):
    class Meta:
        model = Item
        fields = {
            'name': (operators.EQ, operators.REGEXP, operators.NOT, operators.ILIKE),
            'count': (operators.GT, operators.GE, operators.LT, operators.LE)
        }

class ExplicitFieldsFilterItemsView(ListCreateModelView):
    body_schema_class = ItemSchema
    model = Item
    filter_class = ExplicitFieldsFilter

class GeneratedFieldsFilterItemsView(ListCreateModelView):
    body_schema_class = ItemSchema
    model = Item
    filter_class = GeneratedFieldsFilter

MODELS = [Item]

def setup_db(models):
    settings = {
        'backend': 'colibris.persist.backends.SQLiteBackend',
        'database': ':memory:',
        'regexp_function': True
    }

    persist.DatabaseBackend.configure(settings)
    db = persist.get_database()
    db.connect()
    persist.models.set_database(db)
    db.create_tables(models)

def setup_module(module):
    setup_db(MODELS)

    Item.create(name='Justo', count=1, info='Ridiculus Risus')
    Item.create(name='Justo', count=2, info='Ligula Dolor')
    Item.create(name='Vestibulum', count=3, info='Justo Dolor')
    Item.create(name='Sit', count=4, info='Tortor Sollicitudin')
    Item.create(name='Porta', count=5, info='Etiam Consectetur')
    Item.create(name='Mattis', count=6, info='Parturient Vulputate')

@pytest.fixture
async def http_client(http_client_maker):
    return await http_client_maker(middlewares=[handle_errors_json],
                                   routes=[
                                       ('/some-items', ExplicitFieldsFilterItemsView),
                                       ('/same-items', GeneratedFieldsFilterItemsView), ]
                                   )

class TestFilterExplicitFields:
    @classmethod
    def setup_class(cls):
        cls.url = '/some-items'

    async def test_single_filter_eq(self, http_client):
        response = await http_client.get(self.url, params={'name': 'Justo'})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 2
        assert data[0]['name'] == 'Justo'
        assert data[1]['name'] == 'Justo'

    async def test_single_filter_regexp(self, http_client):
        response = await http_client.get(self.url, params={'name__regexp': '.*J.*u.*s.*t.*o.*'})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 2
        assert data[0]['name'] == 'Justo'
        assert data[1]['name'] == 'Justo'

    async def test_single_filter_not(self, http_client):
        response = await http_client.get(self.url, params={'name__not': 'Justo'})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 4

    async def test_single_filter_like(self, http_client):
        response = await http_client.get(self.url, params={'name__ilike': '%%sto%'})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 2
        assert data[0]['name'] == 'Justo'
        assert data[1]['name'] == 'Justo'

    async def test_single_filter_gt(self, http_client):
        response = await http_client.get(self.url, params={'count__gt': 3})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 3

    async def test_single_filter_ge(self, http_client):
        response = await http_client.get(self.url, params={'count__ge': 3})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 4

    async def test_single_filter_lt(self, http_client):
        response = await http_client.get(self.url, params={'count__lt': 6})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 5

    async def test_single_filter_le(self, http_client):
        response = await http_client.get(self.url, params={'count__le': 6})
        assert response.status == 200

        data = await response.json()
        assert isinstance(data, list)
        assert len(data) == 6

class TestFilterGeneratedFields(TestFilterExplicitFields):
    @classmethod
    def setup_class(cls):
        cls.url = '/same-items'


#!/usr/bin/python
#coding=utf8
"""
# Author: andy
# Created Time : 2017-10-12 17:50:43

# File Name: make_vocabulary.py
# Description:
# 做训练用的基础词表

"""
import sys
import re
import jieba
import common_segment

if __name__ == '__main__':
    if len(sys.argv) != 4:
        print("need input_file & seg_output_file & tfdf_output_file")
        exit(0)
    usr_dict = '../data/jieba_user.dict.org.fil'
    jieba.load_userdict(usr_dict)

    # 把事件相关词加入分词词典
    rfile = open('../data/event_rel')
    word_dict = {}
    for line in rfile.readlines():
        line = line.decode('utf-8')
        line = line.rstrip('\n')
        tlist = line.split('\t')
        # 事件相关词
        for w in tlist[1:]:
            if w not in word_dict:
                word_dict[w] = {}
                jieba.add_word(w, 100)
    rfile.close()

    p_hanzi = re.compile(u'[\u3400-\u9fa5]')
    p_eng = re.compile('[a-zA-Z]')

    tf_dict = {}
    df_dict = {}
    ifile = open(sys.argv[1])
    oseg_file = open(sys.argv[2], 'w')
    otfdf_file = open(sys.argv[3], 'w')
    for line in ifile.readlines():
        line = line.decode('utf-8')
        line = line.rstrip('\n')
        tlist = line.split('\t')
        title = tlist[2]
        content = tlist[3]
        terms = common_segment.segment(title + '#&#' + content)
        oseg_file.write((' '.join(terms) + '\n').encode('utf-8'))

        tmp_dict = {}
        for term in terms:
            if len(term) == 1:
                continue
            if p_hanzi.search(term) is None and p_eng.match(term) is None:
                #print(term.encode('utf-8'))
                continue
            tf_dict[term] = tf_dict.setdefault(term, 0) + 1
            tmp_dict[term] = 1
        for term in tmp_dict:
            df_dict[term] = df_dict.setdefault(term, 0) + 1
    ifile.close()
    oseg_file.close()
    for term in tf_dict:
        otfdf_file.write(("%s\t%d\t%d\n" % (term, tf_dict[term], df_dict.get(term, 0))).encode('utf-8'))
    otfdf_file.close()


"""
implement some more feature than pyorient
@author:yuantaixing
"""
import logging

from pyorient.ogm import Graph
from pyorient.ogm.query import Query

from ngpyorient import ng_node, ng_relationship

class NgGraph(Graph):
    """extends Graph,but has more feature than graph --next generation graph"""

    def out(self, from_, *edge_classes, **kwargs):
        """Get adjacent outgoing vertexes from vertex or class.

        :param from_: Vertex id, class, or class name
        :param edge_classes: Filter by these edges
        :param **kwargs: conditons
        """

        condition_str = self._get_condition_str(**kwargs)
        if condition_str:
            sql_string = 'SELECT EXPAND( out({0}) ) FROM {1} {2}'.format(
                ','.join(Graph.coerce_class_names_to_quoted(edge_classes))
                , self.coerce_class_names(from_), condition_str)
        else:
            sql_string = 'SELECT EXPAND( out({0}) ) FROM {1}'.format(
                ','.join(Graph.coerce_class_names_to_quoted(edge_classes))
                , self.coerce_class_names(from_))
        records = self.client.query(sql_string, -1)

        return [self.vertex_from_record(v) for v in records] \
            if records else []

    def in_(self, to, *edge_classes, **kwargs):
        """Get adjacent incoming vertexes to vertex or class.

        :param to: Vertex id, class, or class name
        :param edge_classes: Filter by these edges
        :param **kwargs: conditons
        """

        condition_str = self._get_condition_str(**kwargs)
        if condition_str:
            sql_string = 'SELECT EXPAND( in({0}) ) FROM {1}{2}'.format(
                ','.join(Graph.coerce_class_names_to_quoted(edge_classes))
                , self.coerce_class_names(to), condition_str)
        else:
            sql_string = 'SELECT EXPAND( in({0}) ) FROM {1}'.format(
                ','.join(Graph.coerce_class_names_to_quoted(edge_classes))
                , self.coerce_class_names(to))
        records = self.client.query(sql_string, -1)
        return [self.vertex_from_record(v) for v in records] \
            if records else []

    def bothE(self, from_to, *edge_classes, **kwargs):
        """Get outgoing/incoming edges from/to vertex or class.

        :param from_to: Vertex id, class, or class name
        :param edge_classes: Filter by these edges
        :param **kwargs: conditons
        """
        condition_str = self._get_condition_str(**kwargs)
        if condition_str:
            sql_string = 'SELECT EXPAND( bothE({0}) ) FROM {1}{2}'.format(
                ','.join(Graph.coerce_class_names_to_quoted(edge_classes))
                , self.coerce_class_names(from_to), condition_str)
        else:
            sql_string = 'SELECT EXPAND( bothE({0}) ) FROM {1}'.format(
                ','.join(Graph.coerce_class_names_to_quoted(edge_classes))
                , self.coerce_class_names(from_to))
        records = self.client.query(sql_string, -1)
        return [self.edge_from_record(r) for r in records] \
            if records else []

    def _get_condition_str(self, **kwargs):

        def add_quote(item):
            """add quote to string"""
            if type(item) == str:
                return "\'" + item + "\'"
            else:
                return item

        condition_string = None
        # add condition
        if kwargs:
            header = " where "
            condition_string = header + "".join(["{}={} and " for i in kwargs.items()]).rstrip("and ")
            list_new = [[key, add_quote(value)] for key, value in kwargs.items()]
            key_values = [item for temp in list_new for item in temp]

            condition_string = condition_string.format(*key_values)
        return condition_string

    def delete_records(self, records):
        """delete records"""
        if not records or not len(records):
            raise Exception("records should not be null")
        elif type(records) == Query:
            for record in records:
                cluster_id, position_id = [int(id.lstrip("#")) for id in record._id.split(":")]
                self.client.record_delete(cluster_id, position_id)
        else:
            for record in records:
                cluster_id, position_id = [int(id.lstrip("#")) for id in record._rid.split(":")]
                self.client.record_delete(cluster_id, position_id)

    def update_records(self, records, data):
        """update records -->actually batch update but without transaction"""
        if not records or not len(records):
            raise Exception("records should not be null")
        elif type(records) != list:
            raise Exception(
                """only support query from raw sql by client. eg:graph.client.query("select from person where name='yuantaixing'")""")
        else:
            for record in records:
                cluster_id, position_id = [int(id.lstrip("#")) for id in record._rid.split(":")]
                self.client.record_update(cluster_id, position_id, data, record._version)

    def close(self):
        if self.client is not None:
            self.client.close()
            self.client = None

    def update_records_with_transaction(self, records, data, partial=True):
        """
        update with transaction
        fixme:have bug here:will override the relationship(edge) of vertex
        """
        tx = self.client.tx_commit()

        # Begin Transaction
        tx.begin()

        try:
            # Create Records
            for record in records:
                record.oRecordData.update(data)
                cluster_id, position_id = [int(id.lstrip("#")) for id in record._rid.split(":")]
                new = self.client.record_update(cluster_id, position_id, record.oRecordData, record._version)
                tx.attach(new)
            tx.commit()
        except Exception as e:
            logging.error("update failed because {}".format(e))
            tx.rollback()

    def drop_class_simple(self, cls):
        """"""
        if ng_node.NgNode in cls.__mro__:
            sql = "delete vertex {}".format(cls.__name__)
            ""
        elif ng_relationship.NgRelationship in cls.__mro__:
            sql = "delete edge {}".format(cls.__name__)
        else:
            raise Exception("the class {} is either vertex nor edge".format(cls))
        self.client.command(sql)
        self.client.command("drop class {}".format(cls.__name__))

    def drop_class_simple_by_class_name(self, cls_name, type="vertex"):
        """"""
        if type == "vertex":
            sql = "delete vertex {}".format(cls_name)
            ""
        elif type == "edge":
            sql = "delete edge {}".format(cls_name)
        else:
            raise Exception("the class {} is either vertex nor edge".format(cls_name))
        self.client.command(sql)
        self.client.command("drop class {}".format(cls_name))


import os

datasets = ["caltech101", "dtd", "eurosat", "fgvc_aircraft", "food101", "oxford_flowers", "oxford_pets", "stanford_cars", "sun397", "ucf101"]
mc       = [20,40,20,10,20,10,40,20,10,5]
epochs   = [20,10,60,10,20,40,20,40,10,20]
seeds    = [1]
shots    = [16]
GPOUIDS  = "0,1,2,3,4,5,6,7"
backbones = ['RN50', 'RN101']

for dataset, l, epoch in zip(datasets, mc, epochs):
    for backbone in backbones:
        os.system(f"bash base2new_train_arch.sh {dataset} 1 {GPOUIDS} 16 4 {l} {epoch} {backbone}")
        os.system(f"bash base2new_test_arch.sh {dataset} 1 {GPOUIDS} 16 4 {l} {epoch} {backbone}")


import datetime
import os

File = os.path.basename(__file__)

EPS = 1e-8

def make_orders(order_books, pair, current_balance, need_base, need_quote):
    """
    makes orders to rebalance assets
    :param order_books: see example at the beginning of this file
    :param pair: pair or currencies to balance assets for
    :param need_base: desired amount base currency
    :param need_quote: desired amount of quote currency
    :return: our_orders
    """
    our_orders = dict()
    # current_balance = copy.deepcopy(current_balance)

    currencies = pair.split('_')
    base_cur = currencies[0]  # base currency of current pair (BTC for BTC/USD)
    quote_cur = currencies[1]  # quote currency of current pair (USD for BTC/USD)
    # print(pair, base_cur, quote_cur)

    ax = 0
    bx = 0
    asks = order_books[pair]['asks']
    bids = order_books[pair]['bids']
    ask_count = len(asks)
    bid_count = len(bids)

    base_amount = 0  # required amount of base currency
    quote_amount = 0  # required amount of quote currency

    sell_orders = {}  # all sell_orders for current pair
    buy_orders = {}  # all buy_orders for current pair

    if need_base != 0.0: # BUY
        while ax < ask_count and need_base > EPS:
            ask_price = asks[ax][0]
            if ask_price < EPS:
                ax += 1
                continue

            ask_vol = asks[ax][1]
            if ask_vol < EPS:
                ax += 1
                continue

            ask_price_real = asks[ax][2]

            ask_exch = asks[ax][3]  # BID: base -> quote

            ask_bal = current_balance[ask_exch][quote_cur]
            if ask_bal < EPS:
                ax += 1
                continue

            try:
                m = min(ask_vol, ask_bal / ask_price_real, need_base)  # current micro-trade volume
                if m < EPS:
                    ax += 1
                    continue
                need_base -= m
                asks[ax][1] -= m
                current_balance[ask_exch][quote_cur] -= m * ask_price_real
            except ZeroDivisionError as e:
                Time = datetime.datetime.utcnow()
                EventType = "ZeroDivisionError"
                Function = "get_arb_opp"
                Explanation = "ask_price_real is equal to 0"
                EventText = e
                ExceptionType = type(e)
                print("{}|{}|{}|{}|{}|{}|{}".format(Time, EventType, Function, File, Explanation, EventText,
                                                    ExceptionType))
                break

            if ask_exch in buy_orders:
                buy_orders[ask_exch][0] = max(buy_orders[ask_exch][0], ask_price_real)
                buy_orders[ask_exch][1] += m
            else:
                buy_orders[ask_exch] = [ask_price_real, m]

    else: #SELL
        m = 0
        while bx < bid_count and need_quote > 1e-4:

            bid_price = bids[bx][0]
            if bid_price < EPS:
                bx += 1
                continue

            bid_vol = bids[bx][1]
            if bid_vol < EPS:
                bx += 1
                continue

            bid_price_real = bids[bx][2]

            bid_exch = bids[bx][3]  # ASK: quote -> base

            bid_bal = current_balance[bid_exch][base_cur]
            if bid_bal < EPS:
                bx += 1
                continue

            m = min(bid_vol, bid_bal, need_quote / bid_price_real)  # current micro-trade volume
            if m < EPS:
                bx += 1
                continue
            need_quote -= bid_price * m
            bids[bx][1] -= m
            current_balance[bid_exch][base_cur] -= m

            if bid_exch in sell_orders:
                sell_orders[bid_exch][0] = min(sell_orders[bid_exch][0], bid_price_real)
                sell_orders[bid_exch][1] += m
            else:
                sell_orders[bid_exch] = [bid_price_real, m]

    our_orders = {}
    our_orders['buy'] = buy_orders
    our_orders['sell'] = sell_orders
    our_orders['required_base_amount'] = base_amount
    our_orders['required_quote_amount'] = quote_amount
    our_orders['profit'] = 0

    return our_orders

def rebalance(data, order_books, total_balances, balances):
    ok = False
    orders = {}
    cur_pair = ''
    try:
        for pair in data:
            cnt = 0
            rate = 0
            for exch in data[pair]:
                if len(data[pair][exch]['bids']) != 0 and len(data[pair][exch]['asks']) != 0:
                    rate += float(data[pair][exch]['bids'][0][2])
                    rate += float(data[pair][exch]['asks'][0][2])
                    cnt += 2
            if cnt == 0:
                continue
            rate /= cnt
            base, quote = pair.split('_')
            if total_balances[base] == 0.0 and total_balances[quote] == 0.0:
                continue
            minvol, maxvol = min(total_balances[base] * rate, total_balances[quote]), max(total_balances[base] * rate, total_balances[quote])
            if minvol / maxvol < 0.8:
                ok = True
                cur_pair = pair
                if total_balances[base] * rate < total_balances[quote]:
                    orders = make_orders(order_books, pair, balances, need_base=(total_balances[quote] - total_balances[base] * rate) / 2 / rate, need_quote=0.0)
                else:
                    orders = make_orders(order_books, pair, balances, need_base=0.0, need_quote=(total_balances[base] * rate - total_balances[quote]) / 2)

    except Exception as e:
        Time = datetime.datetime.utcnow()
        EventType = "Error"
        Function = "rebalance"
        Explanation = "Fail while getting rates for currencies"
        EventText = e
        ExceptionType = type(e)
        print("{}|{}|{}|{}|{}|{}|{}".format(Time, EventType, Function, File, Explanation, EventText,
                                            ExceptionType))

    return ok, cur_pair, orders


# -*- coding:utf-8 -*-
import json
import hashlib
import os
import re
from collect.custom.mysql import conn
f = open('quotes-qianku-data.json', 'r')

data = json.load(f)
handleData = {}
db = conn.conn().getConn()
# i = 0
# def addslashes(s):
#     d = {'"':'\\"', "'":"\\'", "\0":"\\\0", "\\":"\\\\"}
#     return ''.join(d.get(c, c) for c in s)
for sub in data:
    # print sub['username']
    # os._exit(0)
    # title = hashlib.md5(sub['subTitle'].encode('utf-8')).hexdigest()
    #
    # if(handleData.has_key(title) != True):
    #     handleData[title] = {}
    #
    # if (handleData[title].has_key('id') != True):
    #     handleData[title]['id'] = []
    # handleData[title]['title'] =  sub['subTitle'];
    # handleData[title]['id'].append(sub['subIds']);
    # handleData[title]['type'] = sub['subType'];
    # handleData[title]['keyword'] = ','.join(sub['subKeyword']);
    # handleData[title]['des'] = sub['subDesc'];
    # i = i+1
    # if(i == 10):
    #     break;

# for subData in handleData:
#     db = conn.conn().getConn()
#     client = db.cursor()
#     sql = "select last_id from tmp_scrapy limit 1"
#     client.execute(sql)
#     db.commit()
    id = re.match(r'(.*?)(\d+).html', sub['href']).group(2)
    link = sub['href']
    if(sub['box'] is not None):
         rst = re.match(r'(\d+)\*(\d+)px', sub['box'])
    if(rst is None):
        width = 0
        height = 0
    else:
        width = int(rst.group(1))
        height = int(rst.group(2))
    imgArr = sub['img'].split('!')
    img = imgArr[0][21:]
    keyword = ','.join(sub['keyword'])
    if(sub['format'] is None):
        format = ''
    else:
        format = sub['format']
    # print link
    # print width
    # print height
    # print img
    # print format
    # sql = "select * from source_pic_qianku where source_id = %d" % (int(id))
    # client = db.cursor()
    # client.execute(sql)
    # exist = client.fetchall()
    # if len(exist) != 0:
    #     sql = "update source_pic_qianku set format = '%s' where source_id = %d" % (format, int(id))
    #     client = db.cursor()
    #     client.execute(sql)
    #     db.commit()
    # else:
    sql = "insert ignore into tmp_handle_data (title, tag, cdn_path, width, height, link, cid, source, uid, weight_updated_time) values ('%s', '%s', '%s', '%d', '%d', '%s', '%d', '%s', 0, 0)" % (sub['title'], keyword, img, width, height, link, 26, 'qianku')
    client = db.cursor()
    client.execute(sql)
    db.commit()

    # sql = "update tmp_shetu set link = '%s' where pid = %d" % (link, int(id))
    # client = db.cursor()
    # client.execute(sql)
    # db.commit()
        # lastId = 123
        # if(lastId is not None):
        #     sql = "insert into source_pic_qianku (id, source_id, username, width, height, format) values (%d, %d, '%s', '%s', %d, %d, '%s')" % (int(lastId), int(id), str(addslashes(sub['username'].encode('utf-8'))), int(width), int(height), format)
        #     print sql
        #     iexit
        #     client = db.cursor()
        #     client.execute(sql)
        #     db.commit()
    # os._exit(0)
    # sql = "insert into tmp_shetu (title, type, keyword, des, pid, link, wid) values ('%s', '%s', '%s', '%s', %d, '', 0)" % (sub['subTitle'].encode('utf-8'), sub['subType'].encode('utf-8'), (','.join(sub['subKeyword'])).encode('utf-8'), sub['subDesc'].encode('utf-8'), int(sub['subIds']))
    # # print sql
    # # os._exit(0)
    # db = conn.conn().getConn()
    # client = db.cursor()
    # client.execute(sql)
    # db.commit()


# import library tfluna
import time
import sys
import tfli2c as tfl    # Import `tfli2c` module v0.0.1

# konfigurasi tfluna (cek i2c)
I2CPort = 1     # I2C(4), /dev/i2c-4, GPIO 8/9, pins 24/21
I2CAddr = 0x10   # Device address in Hex, Decimal 16
if( tfl.begin( I2CAddr, I2CPort)):
    print( "I2C mode: ready")
else:
    print( "I2C mode: not ready")
    sys.exit()

def readtfl():
    time.sleep(0.047)   # Add 47ms delay for 20Hz loop-rate
    if tfl.getData():
        # Display distance in centimeters,
        print( f"Dist:{tfl.dist:{4}}cm", end= " | ")
        # display signal-strength or quality,
        print( f"Flux:{tfl.flux:{6}d}",  end= " | ")
        # and display temperature in Centigrade.
        print( f"Temp:{tfl.temp:{3}}°C",  )
        if tfl.dist>51 or tfl.dist<46:
            print('jalan bergelombang')
    else:                  # If the command fails...
        tfl.printStatus()  # display the error status

# set tfluna condition
tfAttempt = 0

while tfAttempt < 3:
    try:
        while True:
            readtfl()
    except KeyboardInterrupt:
        print( 'Keyboard Interrupt')
        break
    except:
        eType = sys.exc_info()[0]  #  Return exception type
        print( eType)
        tfAttempt += 1
        print( "Attempts: " + str(tfAttempt))
        time.sleep(2.0)
sys.exit()


import os
import sys
import codecs
import collections
import pandas as pd
import numpy as np
import jenkspy
from sklearn.neighbors import NearestNeighbors

file_name = input('Please enter a name of csv file for input >> ')
if os.path.exists(file_name):
    with codecs.open(file_name, 'r', 'utf-8', 'ignore') as f:
        df_xy = pd.read_csv(f)
    base_name = os.path.splitext(os.path.basename(file_name))[0]
else:
    print('The processing was cancelled due to an invalid input. Please enter a correct file name')
    sys.exit()

grid_input = input('Input size of a side of grid (default=100) >> ')
try:
    grid_qt = int(grid_input)
except:
    grid_qt = 100
    print("The size of a side of grid is set to 100, due to an invalid input.")

d_input = input('Input distance decay parameter (default=3) >> ')
try:
    d_cost = int(d_input)
except:
    d_cost = 3
    print('Distance decay parameter is set to 3, due to an invalid input.')

log_doc = "Documents : " + str(len(df_xy))
print(log_doc)

min_X = df_xy["x"].min()
max_X = df_xy["x"].max()
min_Y = df_xy["y"].min()
max_Y = df_xy["y"].max()

x_coord = []
y_coord = []
x_side = ((max_X - min_X) / (grid_qt - 1))
a_side = 2/3 * x_side
y_side = (a_side * np.sqrt(3))
grid_count = 0
s_count = -1
for i in range(grid_qt+2):
    s = min_X + (x_side * (i-1))
    s_count += 1
    t = min_Y - y_side if s_count % 2 == 0 else min_Y - (y_side * 0.5)
    while t < max_Y + y_side:
        x_coord.append(s)
        y_coord.append(t)
        t += y_side
        grid_count += 1

df_grid_id = pd.DataFrame()
df_grid_id['grid_id'] = pd.RangeIndex(start=0, stop=grid_count, step=1)
print("The number of grid :" + str(grid_count+1))
df_grid_id["x"] = x_coord
df_grid_id["y"] = y_coord

count_density_all = []

for i in range(len(df_grid_id)):
    grid_x = df_grid_id["x"][i]
    grid_y = df_grid_id["y"][i]
    grid_dist = ((1 + (((df_xy["x"] - grid_x) ** 2 + (df_xy["y"] - grid_y) ** 2) ** 0.5) / a_side) ** d_cost)
    df_cnt_all = df_xy["size"]  / grid_dist
    count_density_all.append(df_cnt_all.sum())
    sys.stdout.write("\r%d/%d is done." % (i+1, len(df_grid_id)))

print("\ncreating break points of values...")
df_grid_id["density_all"] = list(map(lambda x: (x*1000).round(4), count_density_all))

num_breaks = 9
bin_breaks_all = jenkspy.jenks_breaks(df_grid_id["density_all"], nb_class=num_breaks)

df_grid_id["jenks_all"] = pd.cut(df_grid_id["density_all"], bins=bin_breaks_all, labels=False, include_lowest=False, duplicates='drop')

print("\ncreating grid-document relations...")
nearest_point = NearestNeighbors(n_neighbors=1, algorithm='auto', metric='euclidean').fit(df_grid_id[["x", "y"]].values)
distances, indices = nearest_point.kneighbors(df_xy[["x", "y"]])
df_nearest = pd.DataFrame(indices, columns=["grid_id"])
df_xyn = pd.concat([df_xy, df_nearest], axis=1)

grid_uid = df_xyn['grid_id'].unique()
grid_doc_uid = []
grid_doc_i = []
grid_key_i = []

for i in grid_uid:
    key_i_list = []
    df_key_uid = df_xyn.query("grid_id == @i")
    keys_i = np.array(df_key_uid["keywords"])
    s = keys_i.T

    for j in s:
        try:
            l = [x.strip() for x in j.split("|")]
            key_i_list.extend(l)
        except:
            pass

    num_topKey = 4
    ci = collections.Counter(key_i_list)
    if len(ci) > num_topKey - 1:
        ci_top = ci.most_common(num_topKey)[0:num_topKey]
        ci_top, ci_top_counts = zip(*ci.most_common(num_topKey)[0:num_topKey])
    elif len(ci) > 0:
        ci_top = ci.most_common(len(ci))
        ci_top, ci_top_counts = zip(*ci.most_common(len(ci)))
    else:
        ci_top = "N/A"

    grid_doc_uid.append(i)
    grid_doc_i.append(df_key_uid['size'].sum())
    grid_key_i.append(ci_top)

df_grid_key = pd.DataFrame()
df_grid_key["grid_id"] = grid_doc_uid
df_grid_key["size"] = grid_doc_i
df_grid_key["keywords"] = grid_key_i

df_grid = pd.merge(df_grid_id, df_grid_key, on='grid_id', how='outer', sort=True)

f_name = base_name +"_grid_" + str(len(df_grid_id)) + ".tsv"
df_grid.to_csv(f_name, sep='\t', index=False, encoding='utf-8')

print("Completed.")


from __future__ import print_function
from instantiate import gen_inst_task
from pddlstream.algorithms.downward import get_problem, task_from_domain_problem, get_cost_scale, \
    scale_cost, fd_from_fact, make_domain, make_predicate, evaluation_from_fd, fact_from_fd, \
    get_preimage_literals_with_idxes
from pddlstream.algorithms.instantiate_task import instantiate_task, sas_from_instantiated
from pddlstream.algorithms.scheduling.apply_fluents import convert_fluent_streams
from pddlstream.algorithms.scheduling.negative import convert_negative, recover_preimage_plan
from pddlstream.algorithms.scheduling.postprocess import postprocess_stream_plan
from pddlstream.algorithms.scheduling.recover_axioms import recover_axioms_plans
from pddlstream.algorithms.scheduling.recover_functions import compute_function_plan
from pddlstream.algorithms.scheduling.recover_streams import extract_stream_plan
from etamp.pddlstream.algorithms.scheduling.utils import get_instance_facts
from pddlstream.algorithms.search import solve_from_task

from pddlstream.language.constants import Not, get_prefix, EQ, pAction, pA_info, pAtom
from pddlstream.language.conversion import transform_plan_args, transform_action_args, pddl2obj
from pddlstream.language.conversion import objects_from_evaluations
from pddlstream.language.optimizer import UNSATISFIABLE

from pddlstream.utils import Verbose, flatten
from itertools import product
from copy import copy
from collections import namedtuple
from etamp.pddlstream.algorithms.instantiation import Instantiator
from pddlstream.algorithms.scheduling.recover_streams import get_achieving_streams, \
    expand_evals_by_results
from pddlstream.algorithms.constraints import PlanConstraints, WILD
from pddlstream.language.constants import FAILED_LEVEL, INFEASIBLE_EVAL, is_plan
from pddlstream.language.conversion import evaluation_from_fact, substitute_expression
from pddlstream.language.function import FunctionResult, Function
from pddlstream.language.stream import StreamResult
from pddlstream.language.statistics import check_effort, compute_stream_plan_effort
from pddlstream.language.object import Object, OptimisticObject
from pddlstream.utils import safe_zip, get_mapping, implies, INF
from pddlstream.algorithms.scheduling.skeleton_batch import SkeletonBatch

CONSTRAIN_STREAMS = True
CONSTRAIN_PLANS = True  # False  True
MAX_DEPTH = INF  # 1 | INF

PREDICATE_ORDER = "_applicable"
PARA_PROPO = '_p'
TYPE_PROPO = 'propo_action'

NT_Domain = namedtuple('NT_Domain',
                       ['name', 'requirements', 'types', 'type_to_constants',
                        'predicates', 'predicate_to_typeList', 'derived', 'action'])
NT_Problem = namedtuple('NT_Problem',
                        ['name', 'domain', 'type_to_objects', 'init', 'goal'])
NT_Action = namedtuple('NT_Action', ['name', 'parameters', 'precondition', 'effect'])
NT_Stream = namedtuple('NT_Stream', ['name', 'inputs', 'domain', 'outputs', 'certified'])

def add_stream_effort_to_action(node_from_atom, instantiated_task, effort_weight, **kwargs):
    """
    Count the estimated stream_cost on its corresponding actions.
    """
    if effort_weight is None:
        return

    for action in instantiated_task.fluent_actions:
        # TODO: prune stream actions here?
        # TODO: round each effort individually to penalize multiple streams
        facts = get_instance_facts(action, node_from_atom)
        # effort = COMBINE_OP([0] + [node_from_atom[fact].effort for fact in facts])
        stream_plan = []
        extract_stream_plan(node_from_atom, facts, stream_plan)

        estimated_effort = compute_stream_plan_effort(stream_plan, **kwargs)
        stream_cost = scale_cost(effort_weight * estimated_effort)
        action.cost += stream_cost

##################################################

def rename_instantiated_actions(instantiated):
    # TODO: rename SAS instead?
    actions = instantiated.fluent_actions[:]
    renamed_actions = []
    action_from_name = {}
    for i, action in enumerate(actions):
        renamed_actions.append(copy(action))
        renamed_name = 'a{}'.format(i)
        renamed_actions[-1].name = '({})'.format(renamed_name)
        action_from_name[renamed_name] = action  # Change reachable_action_params?
    instantiated.fluent_actions[:] = renamed_actions
    return action_from_name

# def pddl_from_instance(instance):
#     action = instance.action
#     args = [instance.var_mapping[para.name]
#             for para in action.parameters[:action.num_external_parameters]]
#     pa_info = pA_info(action.info.cost_fn, None)
#     return pAction(action.name, args, pa_info)

def create_pAtom(atom):
    return pAtom(atom.predicate, atom.args)

def propoAction2pddl(propo_action):
    """
    Transform a propositional action into a namedtuple Action.
    """
    action = propo_action.action
    full_args = [propo_action.var_mapping[para.name]
                 for para in action.parameters[:action.num_external_parameters]]

    pa_info = pA_info(action.info.cost_fn, [create_pAtom(e.effect) for e in propo_action.add_effects], propo_action)
    pa_info = pA_info(action.info.cost_fn, [create_pAtom(e.effect) for e in propo_action.add_effects], None)

    return pAction(action.name, full_args, pa_info)

##################################################

def get_plan_cost(action_plan, cost_from_action):
    """
    Sum up the cost of all actions in the plan.
    """
    if action_plan is None:
        return INF
    # return sum([0.] + [instance.cost for instance in action_plan])
    scaled_cost = sum([0.] + [cost_from_action[action]
                              for action in action_plan])
    return scaled_cost / get_cost_scale()

def instantiate_optimizer_axioms(instantiated, domain, results):
    # Needed for instantiating axioms before adding stream action effects
    # Otherwise, FastDownward will prune these unreachable axioms
    # TODO: compute this first and then apply the eager actions
    stream_init = {fd_from_fact(result.stream_fact)
                   for result in results if isinstance(result, StreamResult)}
    evaluations = list(map(evaluation_from_fd, stream_init | instantiated.atoms))
    temp_domain = make_domain(predicates=[make_predicate(UNSATISFIABLE, [])],
                              axioms=[ax for ax in domain.axioms if ax.name == UNSATISFIABLE])
    temp_problem = get_problem(evaluations, Not((UNSATISFIABLE,)), temp_domain)
    # TODO: UNSATISFIABLE might be in atoms making the goal always infeasible
    with Verbose():
        # TODO: the FastDownward instantiation will prune static preconditions
        new_instantiated = instantiate_task(task_from_domain_problem(temp_domain, temp_problem),
                                            check_infeasible=False, prune_static=False)
        assert new_instantiated is not None
    instantiated.axioms.extend(new_instantiated.axioms)
    instantiated.atoms.update(new_instantiated.atoms)

##################################################

def retrace_stream_plan(original_evaluation, evaluation, optms_evaluation, goal_expression, domain,
                        dict_fact_node, fluent_action_plan, list_fluent_axiom_plan,
                        negative_predicates, replan_step):
    original_task = task_from_domain_problem(domain, get_problem(original_evaluation, goal_expression, domain))
    real_task = task_from_domain_problem(domain, get_problem(evaluation, goal_expression, domain))
    optms_task = task_from_domain_problem(domain,
                                          get_problem(optms_evaluation, goal_expression, domain))  # for the solver

    negname_to_pred = {predicate.name: predicate
                       for predicate in negative_predicates}

    list_step_inits, preimage_plan = recover_preimage_plan(real_task, optms_task,
                                                           list_fluent_axiom_plan, fluent_action_plan,
                                                           negname_to_pred)

    function_plan = compute_function_plan(optms_evaluation, fluent_action_plan)

    literal_to_steps = get_preimage_literals_with_idxes(preimage_plan, [])  # of the full preimage.
    # Full preimage literals required by the plan.
    full_preimage = set(literal_to_steps)
    # Preimage literals to come from streams.
    stream_preimage = full_preimage - set(original_task.init)
    negative_preimage = set([literal for literal in stream_preimage
                             if literal.predicate in negname_to_pred])
    positive_preimage = stream_preimage - negative_preimage

    function_plan.update(
        convert_negative(negative_preimage, negname_to_pred, literal_to_steps, list_step_inits))

    # A mapping from a non-negative preimage fact (from Atom only) to positions where it arises.
    dict_fact_steps = {fact_from_fd(literal): literal_to_steps[literal]
                       for literal in positive_preimage
                       if not literal.negated}
    target_facts = {fact for fact in dict_fact_steps.keys()
                    if get_prefix(fact) != EQ}

    fact_to_minIdx = {fact: min(dict_fact_steps[fact])
                      for fact in target_facts}

    stream_to_minIdx = {}
    stream_plan = []  # empty container
    """stream_plan is recursively extracted to fulfill target facts according to dict_fact_node.
       Streams are basically used to fulfill the static facts of the (full) action_plan.
       ALL streams even for previously added facts are included."""
    extract_stream_plan(dict_fact_node, target_facts, stream_plan,
                        fact_to_minIdx, stream_to_minIdx)

    """evaluation:            steam_plan将仅包括与此次optimistic facts有关的streamResults;
       original_evaluation:   steam_plan将包括所有所依赖streamResults(包括之前生成的)"""
    post_stream_plan = postprocess_stream_plan(original_evaluation, domain, stream_plan, target_facts)

    applied_stream_plan = []
    step_to_actions = {}
    for result in (post_stream_plan + list(function_plan)):
        if (result.opt_index != 0) or (stream_to_minIdx.get(result, 0) < replan_step):
            """
            If the (optimistic) result is not unique, or its step index < replan_step,,
            then count it as an item of the stream plan.
            """
            applied_stream_plan.append(result)
        else:
            """
            If the result is unique, and its step index >= replan_step,
            then generate an action out of it. (Currently not used.)
            """
            step_to_actions.setdefault(stream_to_minIdx[result], []).append(result.get_action())
            assert len(step_to_actions) == 0

    # The stream plan with its streamResults grouped into static(s) and fluent(f)
    sf_stream_plan = convert_fluent_streams(applied_stream_plan, list_step_inits,
                                            fluent_action_plan, dict_fact_steps, dict_fact_node)
    # sf_stream_plan =applied_stream_plan

    """Generate an action plan, based on the fluent propositional action plan."""
    for index, fluent_action in enumerate(fluent_action_plan):
        paction = propoAction2pddl(fluent_action)
        paction_with_obj = transform_action_args(paction, pddl2obj)
        step_to_actions.setdefault(index, []).append(paction_with_obj)

    action_plan = list(flatten(step_to_actions[idx]
                               for idx in sorted(step_to_actions)))

    return sf_stream_plan, action_plan

##################################################

def solve_optimistic_sequential(instantiated_task, debug=False, max_cost=INF, **search_kwargs):
    action_from_name = rename_instantiated_actions(instantiated_task)
    with Verbose(debug):
        sas_task = sas_from_instantiated(instantiated_task)
        sas_task.metric = True

    # TODO: apply renaming to hierarchy as well
    # solve_from_task | serialized_solve_from_task | abstrips_solve_from_task | abstrips_solve_from_task_sequential
    renamed_plan, _ = solve_from_task(sas_task, debug=False,
                                      max_cost=max_cost, **search_kwargs)
    if renamed_plan is None:
        return None
    action_instances = [action_from_name[name]
                        for name, _, _ in renamed_plan]
    return action_instances

def is_refined(stream_plan):
    """
    This means that the results in stream_plan are refined,
    i.e., their input_objects are remapped to specifically adapt a existing plan.
    """
    # TODO: lazily expand the shared objects in some cases to prevent increase in size
    if not is_plan(stream_plan):
        return True
    # TODO: some of these opt_index equal None
    return all((result.opt_index is None) or (result.opt_index == 0)
               for result in stream_plan)

##################################################

def prune_high_effort_streams(streams, max_effort=INF, **effort_args):
    # TODO: convert streams to test streams with extremely high effort
    low_effort_streams = []
    for stream in streams:
        effort = stream.get_effort(**effort_args)
        if isinstance(stream, Function) or check_effort(effort, max_effort):
            low_effort_streams.append(stream)
    return low_effort_streams

def apply_streams(evaluations, streams, level_limit, **effort_args):
    pruned_streams = prune_high_effort_streams(streams, **effort_args)
    instantiator = Instantiator(pruned_streams)
    for evaluation, node in evaluations.items():
        if node.complexity <= level_limit:
            instantiator.add_atom(evaluation, node.complexity)
    optms_results = []

    while instantiator and (instantiator.min_level() <= level_limit):
        instance = instantiator.pop_stream()
        instance.opt_index = 0
        results = instance.next_optimistic()  # depending on if the instance is marked as refined, if it is, get an unique optimistic object

        for result in results:
            new_facts = False
            level = instantiator.compute_complexity(instance)
            result.level = level

            for fact in result.get_certified():
                new_facts |= instantiator.add_atom(evaluation_from_fact(fact), level)
            if isinstance(result, FunctionResult) or new_facts:
                # assert not isinstance(result, FunctionResult)
                optms_results.append(result)

        # TODO: instantiate and solve to avoid repeated work
    """Indicates nothing left in the queue.
       So higher level_limit will not enrich the optms results.
       Implies that the input evaluations are probably insufficient."""
    exhausted = not instantiator

    return optms_results, exhausted

##################################################

def is_plan(plan):
    """Return True
       if the plan is neither FAILED nor INFEASIBLE"""
    return plan not in [None, False]

def is_plan_certified(stream_plan):
    if stream_plan is None:
        return None

    # if all results in the stream plan is verified (optms_index=None or 0), then the plan is certified
    r = all((result.opt_index is None) or (result.opt_index == 0)
            for result in stream_plan)
    return r

def optimistic_stream_instantiation(instance, obj_mappings, evaluations, optms_evaluations, only_immediate=False):
    """
    Generating new instances by updating the input objects according to the sequence of stream_plan.
    For the new instance, reduce their optms_index and thereby make them more unique.
    They will be saved into their parent streams, and available for next Stream.get_instance()
    """
    new_instances = []
    for input_combo in product(*[obj_mappings.get(i, [i]) for i in instance.input_objects]):
        mapping = get_mapping(instance.input_objects, input_combo)
        domain_evaluations = set(map(evaluation_from_fact, substitute_expression(instance.get_domain(), mapping)))
        if domain_evaluations <= optms_evaluations:
            new_instance = instance.external.get_instance(input_combo)
            if (new_instance.opt_index != 0) and implies(only_immediate, domain_evaluations <= evaluations):
                # always -1
                new_instance.opt_index -= 1
            new_instances.append(new_instance)
    return new_instances

def optimistic_stream_evaluation(evaluations, stream_plan):
    """
    Generate new unique stream_results according to the new stream_instances from the given stream_plan.
    Those results are all related to the stream_plan.
    """
    evaluations = set(evaluations)  # For subset testing
    optms_evaluations = set(evaluations)
    related_results = []
    obj_mappings = {}  # the mapping from original outputs (optimistic) to stream-generated outputs
    active_streams = [s for s in stream_plan if not s.redundant]
    redundant_streams = [s for s in stream_plan if s.redundant]
    for result in active_streams:  # TODO: just refine the first step of the plan
        # unique instance
        new_instances = optimistic_stream_instantiation(result.instance, obj_mappings,
                                                        evaluations, optms_evaluations)
        for new_instance in new_instances:
            for new_result in new_instance.next_optimistic():
                optms_evaluations.update(map(evaluation_from_fact, new_result.get_certified()))
                new_result.level = result.level
                related_results.append(new_result)
                if isinstance(new_result, StreamResult):  # Could not add if same value
                    for old_obj, new_obj in safe_zip(result.output_objects, new_result.output_objects):
                        obj_mappings.setdefault(old_obj, []).append(new_obj)
    related_results.extend(redundant_streams)
    return related_results, obj_mappings

def compute_function_results(evaluations, related_results, externals, complexity_limit, **effort_args):
    # functions = list(filter(lambda s: type(s) is Function, externals))
    functions = [e for e in externals if type(e) is Function]
    extended_evaluations = expand_evals_by_results(evaluations, related_results)
    """Only functions are applied."""
    new_results, _ = apply_streams(extended_evaluations, functions, complexity_limit, **effort_args)
    return related_results + new_results

def compute_plan_constraints(action_plan, obj_mappings):
    """
    Generate an action plan with updated argument objects.
    And further generate a plan constraint.
    """
    new_plan = []
    groups = {old: news for old, news
              in obj_mappings.items() if len(news) != 1}
    for name, arg_tuple, _ in action_plan:
        new_arg_tuple = []
        for arg in arg_tuple:
            if isinstance(arg, Object):
                """For real object, arg remains."""
                new_arg_tuple.append(arg)
            elif isinstance(arg, OptimisticObject):
                assert obj_mappings.get(arg, [])
                if len(obj_mappings[arg]) == 1:
                    """For optm object with single mapping, arg updates."""
                    new_arg_tuple.append(obj_mappings[arg][0])
                else:
                    """For optm object with multiple mappings, arg remains/is marked as WILD."""
                    new_arg_tuple.append(WILD)
                    # new_arg_tuple.append(arg)
            else:
                raise ValueError(arg)
        new_plan.append((name, new_arg_tuple))
    # exact=False because we might need new actions
    return PlanConstraints(plans=[new_plan], groups=groups, exact=False, max_cost=INF)

def solve_and_refine(evaluations, externals, optms_results, optms_plan_solver, level_limit,
                     depth, constraints, **effort_args):
    """
    Plan refinement iteration
    """

    if MAX_DEPTH <= depth:
        return None, None, INF, depth

    stream_plan, action_plan, estimated_cost = optms_plan_solver(evaluations, optms_results,
                                                                 constraints)
    if estimated_cost == 0.514:
        return stream_plan, action_plan, estimated_cost, depth

    if not is_plan(action_plan):
        """Case1: None or False."""
        return stream_plan, action_plan, estimated_cost, depth

    if is_refined(stream_plan):
        """Case2: Already refined."""
        return stream_plan, action_plan, estimated_cost, depth

    related_results, obj_mappings = optimistic_stream_evaluation(evaluations, stream_plan)

    # next_results = compute_function_results(evaluations, related_results, externals, level_limit, **effort_args)

    next_constraints = compute_plan_constraints(action_plan, obj_mappings)

    """Re-run the plan solver with streams having refined stream instances."""
    return solve_and_refine(evaluations, externals, related_results, optms_plan_solver, level_limit,
                            depth + 1, next_constraints, **effort_args)

def plan_action_plan_stream(optms_plan_solver, evaluations, externals, level_limit, **effort_args):
    # externals = (streams + functions + optimizers)
    # Previously didn't have unique optimistic objects that could be constructed at arbitrary depths
    # evals = {e: n for e, n in evaluations.items() if n.complexity <= level_limit}
    evals = {e: n for e, n in evaluations.items()}
    assert len(evals) == len(evaluations)

    num_iterations = 0
    while True:
        num_iterations += 1

        optms_results, exhausted = apply_streams(evals, externals, level_limit, **effort_args)

        stream_plan, action_plan, cost, final_depth = solve_and_refine(evals, externals, optms_results.copy(),
                                                                       optms_plan_solver, level_limit,
                                                                       depth=0, constraints=None,
                                                                       **effort_args)
        print('  Attempt: {} | Results: {} | Depth: {} | Success: {}'.format(
            num_iterations, len(optms_results), final_depth, is_plan(action_plan)))

        if cost == 0.514:
            return stream_plan, action_plan, cost, exhausted

        if is_plan(action_plan):
            """
            Case2:
            """
            return stream_plan, action_plan, cost, exhausted
        """
        A: (final_depth==0 and not is_plan) concludes that at this level the solver cannot find any plausible plans.
           depth is the number of the refinement iteration.
        """
        if final_depth == 0:
            status = INFEASIBLE_EVAL if exhausted else FAILED_LEVEL
            """
            Case1:
            1) if exhausted, the search is failed due to the insufficiency of initial evaluations. -> INFEASIBLE
            2) if not, the search is failed due to the level limitation. -> FAILED
            """
            return status, status, cost, exhausted
        """
         B: Retry at current level
         """

class OptimisticPlanSolver(object):

    def __init__(self, goal_exp, domain, negative, search_kwargs, original_evaluation,
                 replan_actions=set(), max_effort=INF, effort_weight=None, dict_optms_cost={},
                 max_cost=INF):
        self.goal_exp = goal_exp
        self.domain = domain
        self.negative = negative  # negative predicates from the problem
        self.effort_weight = effort_weight
        self.max_effort = max_effort
        self.search_kwargs = search_kwargs
        self.replan_actions = replan_actions
        self.dict_optms_cost = dict_optms_cost
        self.original_evaluation = original_evaluation
        self.max_cost = max_cost

    def _planner_core(self, evaluations, optms_results, domain, goal_exp, max_cost):
        """
        Generate an action_plan with its stream_plan
        based on initial evaluations and all optimistic facts.
        """
        r_problem = get_problem(evaluations, goal_exp, domain)  # begin_metric
        original_init = r_problem.init
        original_goal = r_problem.goal

        dict_fact_node = get_achieving_streams(evaluations, optms_results,
                                               max_effort=self.max_effort)

        optms_evaluations = {evaluation_from_fact(f): n.result for f, n in dict_fact_node.items()}

        problem = get_problem(optms_evaluations, goal_exp, domain)  # begin_metric

        optms_init = problem.init

        if len(problem.init) > 300:
            sk_batch = SkeletonBatch('domain_p.pddl', original_init, original_goal, optms_results, optms_init)
            op_plan, results_for_step = sk_batch.get_next_operatorPlan()
            if op_plan:
                print('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
                return op_plan, results_for_step, 0.514

        pddl_task = task_from_domain_problem(domain, problem)

        # Mapping from string in pddl to Object/OptimiticObject
        pddl_to_obj = {obj.pddl: obj
                       for obj in objects_from_evaluations(optms_evaluations)}

        instantiated_task = gen_inst_task(pddl_task, pddl_to_obj)

        if instantiated_task is None:
            return None, None, INF

        # Estimated cost for actions.
        add_stream_effort_to_action(dict_fact_node, instantiated_task, self.effort_weight)
        action_to_EstmCost = {action: action.cost
                              for action in instantiated_task.fluent_actions}

        fluent_action_plan = solve_optimistic_sequential(instantiated_task,
                                                         max_cost=max_cost, **self.search_kwargs)

        # _plan = solve_optimistic_symk('found_plans/sas_plan.1')
        #
        # prop_action_plan = inst_task(_plan, pddl_task, pddl_to_obj)

        if fluent_action_plan is not None:
            print('')

        estimated_cost = get_plan_cost(fluent_action_plan, action_to_EstmCost)

        if fluent_action_plan is None:
            return None, None, estimated_cost

        list_fluent_axiom_plan = recover_axioms_plans(instantiated_task, fluent_action_plan)

        pddl_plan = transform_plan_args(map(propoAction2pddl, fluent_action_plan), pddl2obj)
        replan_step = min([step + 1 for step, action in enumerate(pddl_plan)
                           if action.name in self.replan_actions] or [len(pddl_plan)])

        stream_plan, action_plan = retrace_stream_plan(self.original_evaluation, evaluations, optms_evaluations,
                                                       goal_exp, domain, dict_fact_node, fluent_action_plan,
                                                       list_fluent_axiom_plan, self.negative, replan_step)

        return stream_plan, action_plan, estimated_cost

    def __call__(self, evaluations, optms_results, constraints):

        result = self._planner_core(evaluations, optms_results, self.domain, self.goal_exp, self.max_cost)

        return result


#!/usr/bin/python3

# Összehasonlítja a diak_jelszo fájl aktuális névsorát
# az ifa.db névsorával.

import sys
base_dir = '/home/szaszi/g/'
sys.path.append(base_dir)

import sqlite3
import utils
import hashlib

DB = '../db/ifa.db'
diak_jelszo = 'diak-jelszo.csv'

conn = sqlite3.connect(DB)
cur = conn.cursor()

diak_regi = {} # {'d12a-Pumpa Pál': 2012, ... }
q = "SELECT oszt || '-' || dnev, id FROM Diak_base"
for row in cur.execute(q):
    nev, diak_id = row
    diak_regi[nev] = diak_id

# Az aktuális diákok adatai
diak_aktualis = {} # {'d12a-Pumpa Pál': 'titok', ... }
for row in open(diak_jelszo):
    oid, jelszo, nev, oszt = row.strip().split(',')
    diak_aktualis[oszt + '-' + nev] = {'nev': nev, 'oszt': oszt, 'jelszo': jelszo}

A = set(diak_regi)
B = set(diak_aktualis)

print('PRAGMA foreign_keys=ON;')
# Ezek csak az SQL-ben vannak, törölhetők
for nev in A-B:
    print('DELETE FROM Diak_base WHERE id=%s; -- %s' % (diak_regi[nev], nev))

print()
max_id = max(diak_regi.values())
i = max_id
# Az új diákokat föl kell venni
for nev in B-A:
    i += 1
    d = diak_aktualis[nev]
    digest = hashlib.sha256(d['jelszo'].encode('utf-8')).hexdigest()
    print("INSERT INTO 'Diak_base' VALUES (%s, '%s', '%s', '%s');" % (i, digest, d['nev'], d['oszt']))

print('sqlite3', DB)


##########################################################
# TP  Programmer avec des Fonctions                      #
##########################################################

#------------------------
# Date
# Nom et prenom et classe
# -----------------------

#********************************************
#   Les fonctions d'entrée et de sortie
#********************************************
def menu():
   print (30*"-")
   print ("  0. Quitter")
   print ("  1. Compter les espaces dans un texte")
   print ("  2. Compter un caractère dans un texte")
   print ("  3. Supprimer les espaces dans un texte")
   print ("  4. Supprimer un caractère dans un texte")
   print ("  5. Modifier un caractère dans un texte")
   print ("  6. Convertir un texte en majuscules sans espace en minuscules")
   print ("  7. Convertir un texte en majuscules avec espace en minuscules")
   print ("  8. Rechercher le caractère le plus présent dans un texte")
   print ("  9. Rechercher une chaine de caractères dans un texte")
   print (30*"-")
   rep=int(input("         Votre reponse : "))
   print (30*"-")
   print()
   return rep

def attente(n):
# Entrée : n un entier
# Sortie : affichage
  print("   ------>   Réponse",n,"en attente   <------")

#********************************************
#   Les fonctions métiers
#********************************************

# Compter les espaces
#.....................

def CompteEspace(Txt):
# Donnée : txt une chaine de caractères
# Résultat : cpt entier prenant pour valeur le nombre d'espace de txt.
  cpt=0 # On initialise le compteur à 0
  for i in range(0,len(Txt)): # on parcourt la chaine de caracteres
    if Txt[i]==" ":
      cpt=cpt+1  # on augmente de 1 le compteur pour chaque espace rencontre
  return cpt

# Compter un caractère donné
#............................

def CompteCar(Txt,car):
# Donnée : txt une chaine de caractères, car un caractère
# Résultat : cpt entier prenant pour valeur le nombre de caractère car de txt.
  cpt=0 # On initialise le compteur à 0
  for k in range(0, len(Txt)):
    if Txt[k] == car:
      cpt = cpt + 1
  return cpt

# Supprimer les espaces
#.......................

def SupprEspace(Txt):
# Donnée : txt une chaine de caractères
# Résultat : cpt entier prenant pour valeur le nombre d'espace de txt.
  NvTxt="" # On initialise un nouveau texte à vide
  for k in range(0,len(Txt)): # on parcourt la chaine de caracteres
    if Txt[k]!=" ":
      NvTxt=NvTxt+Txt[k]  #Si le caractère n'est pas un espace, on écrit le caractère correspondant au texte
  return NvTxt

# Supprimer un caractère donné
#..............................

def SupprCar(Txt,car):
# Donnée : txt une chaine de caractères
# Résultat : cpt entier prenant pour valeur le nombrede caractère car de txt.
  NvTxt="" # On initialise un nouveau texte à vide
  for k in range(len(Txt)):
     if Txt[k] != car:
        NvTxt  = NvTxt + Txt[k]

  return NvTxt

# Remplacer un caractère donné par un autre caractère
#....................................................

def ModifCar(txt,c1,c2):
# Donnée : txt une chaine de caractères
#          c1 et c2 deux caractères
# Résultat : NvTxt, chaine de caractère correspondant à txt dont le caractère
#            c1 a été changé en c2
  NvTxt="" # On initialise un nouveau texte à vide
  for k in range(len(Txt)):
     if Txt[k] == c1:
        NvTxt = NvTxt + c2
     else:
         NvTxt  = NvTxt + Txt[k]

  return NvTxt

# Convertir un texte en majuscules sans espace en minuscules
#............................................................
def MajMin1(TxtMaj):  # arguments à définir
# Donnée : TxtMaj un texte en maj sans espaces
# Résultat : TxtMin: TxtMaj converti en minuscules
   TxtMin = "" # On initialise le texte à vide
   for k in range(len(TxtMaj)):
      TxtMin = TxtMin + chr(ord(TxtMaj)+32)
   return TxtMin

# Convertir un texte en majuscules avec espace en minuscules
#............................................................
def MajMin2(TxtMaj):  # arguments à définir
# Donnée : TxtMaj un texte en maj
# Résultat : TxtMin: TxtMaj converti en minuscules
   TxtMin = "" # On initialise le texte à vide
   for k in range(len(TxtMaj)):
      if TxtMaj[k] == " ":
         TxtMin = TxtMin + TxtMaj[k]
      else:
         TxtMin = TxtMin + chr(ord(TxtMaj)+32)
   return TxtMin

# Recherche le caractère le plus présent dans un texte
#......................................................
def MaxCar(ChaineCar, Car):  # arguments à définir
# Donnée : Car le caractere que l'on cherche
# Résultat : gCar le caractere le plus présent dans le texte
   Fréq = {}
   for Car in ChaineCar.lower():
      Fréq[Car] = ChaineCar.count(Car)
      for Key, Value in Fréq.items():
         if Value == max(Fréq.values()):
            gCar = {Key: Value}

   return gCar

# Recherche un mot dant un chaine de caractères
#...............................................
def ChercheMot(Chaine, Mot):  # arguments à définir
# Donnée : Chaine la chaine avec le mot a chercher
#          Mot le mot à chercher
# Résultat : pos la position
  return Chaine.find(Mot)

#********************************************
#   Le programme principal
#********************************************

rep=1
while rep!=0:
  rep=menu()
  if rep==1:
    Txt=input("texte dont on compte les espaces (non vide) : ")
    NbEspace=CompteEspace(Txt)
    print("Le texte saisi contient",NbEspace,"espace(s)")
  elif rep==2:
    Txt=input("texte dont on compte un caractère (non vide) : ")
    Car=input("Caractère à compter : ")
    NbCar=CompteCar(Txt,Car)
    print("Le texte saisi contient",NbCar,"caractère(s)",Car)
  elif rep==3:
    attente(rep)
    Txt = input("texte dont on veux supprimer les espaces: ")
    NvTxt = SupprEspace(Txt)
    print("bah sa donne", NvTxt)

  elif rep==4:
    attente(rep)
    Txt = input("texte dont on veux supprimer le car: ")
    Car = input("le car: ")
    NvTxt = SupprCar(Txt,Car)
    print(NvTxt)
  elif rep==5:
    attente(rep)
    txt = input("txt dont on veux modif un car par un autre")
    c1 = input("car à changer")
    c2 = input("car pour changer")
    NvTxt = ModifCar(txt,c1,c2)
    print(NvTxt)
  elif rep==6:
    attente(rep)
    TxtMaj = input("txt en maj sans espace à convertir en min")
    NvTxt = MajMin1(TxtMaj)
    print(NvTxt)
  elif rep==7:
    attente(rep)
    TxtMaj = input("txt en maj à convertir en min")
    NvTxt = MajMin2(TxtMaj)
    print(NvTxt)
  elif rep==8:
    attente(rep)
    ChaineCar = input("Chaine")
    Car = input("Car à trouver")
    NvTxt = MaxCar(ChaineCar, Car)
    print(NvTxt)
  elif rep==9:
    attente(rep)
    Chaine = input("Chaine")
    Mot = input("mot à trouver")
    NvTxt = ChercheMot(Chaine, Mot)
    print(NvTxt)
  else:
    if rep!=0:
      print("Réponse non valide")
  print()

Structures de données (matrices, arbres, files, piles, listes chaînées, graphes), parcours et tris/A few algorithms (personnal & school)/Quelques TPs de première/18.10.2019/Elev_TP_Fonc_Oct18_Théo_Tchilinguirian.py


#!/usr/bin/env python3
import sys
import os
import logging
import numpy as np
import pandas as pd
import dateutil

def tempF2C(x): return (x-32.0)*5.0/9.0
def tempC2F(x): return (x*9.0/5.0)+32.0

def load_temperature_hdf5(temps_fn, local_time_offset, basedir=None, start_year=None, truncate_to_full_day=False):
    ## Load temperature
    # temps_fn = "{}_AT_cleaned.h5".format(station_callsign)
    logging.info("Using saved temperatures file '{}'".format(temps_fn))
    if basedir is not None:
        temps_fn = os.path.join(basedir, temps_fn)
    tempdf = pd.read_hdf(temps_fn, 'table')

    tmp = local_time_offset.split(':')
    tmp = int(tmp[0])*3600+int(tmp[1])*60
    sitetz = dateutil.tz.tzoffset(local_time_offset, tmp)
    tempdf.index = tempdf.index.tz_convert(sitetz)

    if truncate_to_full_day:
        x = tempdf.index[-1]
        if x.hour != 23:
            x = x-pd.Timedelta(days=1)
            tmp = '{:04d}-{:02d}-{:02d}'.format(x.year, x.month, x.day)
            tempdf = tempdf.loc[:tmp]
    if start_year is not None:
        tempdf = tempdf.loc['{}-01-01'.format(start_year):]
    logging.info("Temperature data date range used: {} through {}".format(tempdf.index[0], tempdf.index[-1]))
    return tempdf

def load_temperature_csv(fn, local_time_offset=None):
    t = pd.read_csv(fn, index_col=0)
    if local_time_offset is not None:
        tmp = local_time_offset.split(':')
        tmp = int(tmp[0])*3600+int(tmp[1])*60
        sitetz = dateutil.tz.tzoffset(local_time_offset, tmp)
        #t.index = pd.to_datetime(t.index).tz_localize('UTC').tz_convert(sitetz) # @TCC this fails if csv contains datetimes with TZ
        t.index = pd.to_datetime(t.index)
        try:
            t.index = t.index.tz_localize('UTC')
        except TypeError:
            pass
        t.index = t.index.tz_convert(sitetz)
    return t

# Function which computes BM (single sine method) degree day generation from temperature data
def compute_BMDD_Fs(tmin, tmax, base_temp, dd_gen):
    # Used internally
    def _compute_daily_BM_DD(mint, maxt, avet, base_temp):
        """Use standard Baskerville-Ermin (single sine) degree-day method
        to compute the degree-day values for each a single day.
        """
        if avet is None:
            avet = (mint+maxt)/2.0 # simple midpoint (like in the refs)
        dd = np.nan # value which we're computing
        # Step 1: Adjust for observation time; not relevant
        # Step 2: GDD = 0 if max < base (curve all below base)
        if maxt < base_temp:
            dd = 0
        # Step 3: Calc mean temp for day; already done previously
        # Step 4: min > base; then whole curve counts
        elif mint >= base_temp:
            dd = avet - base_temp
        # Step 5: else use curve minus part below base
        else:
            W = (maxt-mint)/2.0
            tmp = (base_temp-avet) / W
            if tmp < -1:
                print('WARNING: (base_temp-avet)/W = {} : should be [-1:1]'.format(tmp))
                tmp = -1
            if tmp > 1:
                print('WARNING: (base_temp-avet)/W = {} : should be [-1:1]'.format(tmp))
                tmp = 1
            A = np.arcsin(tmp)
            dd = ((W*np.cos(A))-((base_temp-avet)*((np.pi/2.0)-A)))/np.pi
        return dd

    # compute the degree-days for each day in the temperature input (from tmin and tmax vectors)
    dd = pd.concat([tmin,tmax], axis=1)
    dd.columns = ['tmin', 'tmax']
    dd['DD'] = dd.apply(lambda x: _compute_daily_BM_DD(x[0], x[1], (x[0]+x[1])/2.0, base_temp), axis=1)
    # compute the degree-days for each day in the temperature input (from a daily groupby)
#     grp = t.groupby(pd.TimeGrouper('D'))
#     dd = grp.agg(lambda x: _compute_daily_BM_DD(np.min(x), np.max(x), None, base_temp))
#     dd.columns = ['DD']

    # Find the point where cumulative sums of degree days cross the threshold
    cDD = dd['DD'].cumsum(skipna=True)
    for cumdd_threshold,label in [[1*dd_gen,'F1'], [2*dd_gen,'F2'], [3*dd_gen,'F3']]:
        dtmp = np.zeros(len(dd['DD']))*np.nan
        tmp = np.searchsorted(cDD, cDD+(cumdd_threshold)-dd['DD'], side='left').astype(float)
        tmp[tmp>=len(tmp)] = np.nan
        #dd[label+'_idx'] = tmp
        # convert those indexes into end times
        e = pd.Series(index=dd.index, dtype='float64')#, dtype='datetime64[ns]')

        #e[~np.isnan(tmp)] = dd.index[tmp[~np.isnan(tmp)].astype(int)] # @TCC previous code
        e.loc[~np.isnan(tmp)] = dd.index[tmp[~np.isnan(tmp)].astype(int)]
        e.loc[np.isnan(tmp)] = np.nan
        dd[label+'_end'] = e
        # and duration...
        #dd[label] = (e-dd.index+pd.Timedelta(days=1)).apply(lambda x: np.nan if pd.isnull(x) else x.days) # @TCC previous code
        dd[label] = (pd.to_datetime(e)-dd.index+pd.Timedelta(days=1)).apply(lambda x: np.nan if pd.isnull(x) else x.days)
        #dd.loc[np.isnan(tmp), label] = np.nan

    print("DD dataframe min values\n", dd.min())
    return dd

def compute_year_over_year_norm(in_dataframe,
                                start, end,
                                norm_start=None, norm_end=None,
                                freq='daily',
                                interp_method='linear',
                                norm_method='mean'):
    """
    Parameters
    ----------
    start: convertable to Datetime
        start range of dates to output
    end: convertable to Datetime
        end range of dates to output
    norm_start : convertable to Datetime or None
        `None` will use in_dataframe.index[0]
    norm_end : convertable to Datetime or None
        if given (not None), output range does not include `norm_end` (it is half-open)
        `None` will use in_dataframe.index[-1]
    freq : {'daily', 'hourly'}
    interp_method : str or None
        `None` will skip resample and interpolation, so
        `in_dataframe` must already be daily or hourly (depending on `freq`)!
    norm_method : {'mean', 'median'}
    """
    if freq == 'hourly':
        hrs = 24
        hrs_freq = '1h'
    elif freq == 'daily':
        hrs = 1
        hrs_freq = '24h'
    else:
        raise ValueError("Invalid `freq` argument value: {}".format(freq))
    if norm_start is None:
        norm_start = in_dataframe.index[0]
    if norm_end is None:
        norm_end = in_dataframe.index[-1]
    else:
        norm_end = pd.to_datetime([norm_end])[0] - pd.Timedelta('1 second')
    print('Computing using range:', norm_start, 'to', norm_end)

    if interp_method is None: # skip resample+interpolation (assumes in_dataframe is daily!)
        t = in_dataframe.loc[norm_start:norm_end]
    else: # resample and interpolate to get hourly
        t = in_dataframe.resample(hrs_freq).interpolate(method=interp_method).loc[norm_start:norm_end]

    if norm_method == 'mean':
        norm = t.groupby([t.index.month, t.index.day, t.index.hour]).mean().sort_index()
    elif norm_method == 'median':
        norm = t.groupby([t.index.month, t.index.day, t.index.hour]).median().sort_index()
    else:
        assert False, "Error: Unknown norm_method '{}'".format(norm_method)

    # now replicate and trim to the desired output range

    start = pd.to_datetime(start)
    end = pd.to_datetime(end)

    # need a non-leapyear and leapyear version
    norm_ly = norm.copy()
    if norm.shape[0] == 366*hrs:
        norm = norm.drop((2,29,))
    else: # norm doesn't include any leapyear data
        assert norm.shape[0] == 365*hrs
        # make Feb 29 the mean of Feb 28 and Mar 1
        foo = (norm.loc[(2,28,)] + norm.loc[(3,1,)]) / 2.0
        foo.index = pd.MultiIndex.from_product( ([2],[29],list(range(hrs))) )
        norm_ly = pd.concat((norm_ly,foo)).sort_index()
        norm_ly.sort_index(inplace=True) # probably not needed

    # build up a 'long normal' (lnorm) dataframe year by year by appending the norm or norm_ly
    lnorm = None
    for yr in np.arange(start.year, end.year+1):
        #print(yr)
        idx = pd.date_range(start='{}-{:02d}-{:02d} {:02d}:00:00'.format(yr,*norm.index[0]),
                            end=  '{}-{:02d}-{:02d} {:02d}:00:00'.format(yr,*norm.index[-1]),
                            freq=hrs_freq)
        if idx.shape[0] == 366*hrs:
            foo = norm_ly.copy()
        else:
            assert norm.shape[0] == 365*hrs
            foo = norm.copy()
        foo.index = idx
        if lnorm is None:
            lnorm = foo
        else:
            lnorm = lnorm.append(foo)

    return lnorm.loc[start:end]


"""Resolwe Python process support."""
import logging
import sys

from .communicator import communicator
from .descriptor import Persistence, ProcessDescriptor, SchedulingClass, ValidationError
from .fields import (
    BooleanField,
    DataField,
    DateField,
    DateTimeField,
    DirField,
    FileField,
    FileHtmlField,
    FloatField,
    GroupField,
    IntegerField,
    JsonField,
    ListField,
    SecretField,
    StringField,
    TextField,
    UrlField,
)
from .models import Collection, Data, Entity
from .runtime import Process

try:
    from plumbum import local as Cmd

    # Log plumbum commands to standard output.

    formatter = logging.Formatter(
        "%(asctime)s - %(levelname)s - %(name)s[%(process)s]: %(message)s"
    )

    handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(formatter)

    plumbum_logger = logging.getLogger("plumbum.local")
    plumbum_logger.setLevel(logging.DEBUG)
    plumbum_logger.addHandler(handler)

except ImportError:
    # Generate a dummy class that defers the ImportError to a point where
    # the package actually tries to be used.
    class MissingPlumbumCmd:
        """Missing plumbum command dummy class."""

        def __getitem__(self, cmd):
            """Raise ImportError exception."""
            raise ImportError("Missing 'plumbum' Python package.")

    Cmd = MissingPlumbumCmd()

__all__ = (
    "BooleanField",
    "Cmd",
    "DataField",
    "DateField",
    "DateTimeField",
    "DirField",
    "FileField",
    "FileHtmlField",
    "FloatField",
    "GroupField",
    "IntegerField",
    "JsonField",
    "ListField",
    "Persistence",
    "Process",
    "ProcessDescriptor",
    "SchedulingClass",
    "SecretField",
    "StringField",
    "ValidationError",
    "TextField",
    "UrlField",
    "Data",
    "Entity",
    "Collection",
    "communicator",
)

# XXX: ImportedFormat is available from resolwe-runtime-utils v2.0.0. Docker
# images with older version of resolwe-runtime-utils should skip the import.
try:
    from resolwe_runtime_utils import ImportedFormat

    __all__ += (ImportedFormat,)
except ImportError:
    pass


import os
from loguru import logger
import torch
import torch.nn as nn

from decalib.utils.lossfunc import VGGFace2Loss
from .utils.config import cfg
from .utils import util

class ConvBlock(nn.Module):
    def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1, dilation=1, bias=True):
        super(ConvBlock, self).__init__()
        self.conv = nn.Conv2d(in_channel, out_channel, kernel_size, stride, padding, dilation, bias)
        self.bn = nn.BatchNorm2d(out_channel)
        self.relu = nn.ReLU(inplace=True)
    def forward(self, x):
        x = self.conv(x)
        x = self.bn(x)
        x = self.relu(x)
        return x

class Generator(nn.Module):
    '''
    Simple UNet Structures like:
        encoder: [16, 32, 32, 32]
        decoder: [32, 32, 32, 32, 32, 16, 16]
    '''
    def __init__(self, model_config):
        super(Generator, self).__init__()
        in_channel = model_config.in_channel
        out_channel = model_config.out_channel
        encoder_nf = model_config.encoder_nf
        decoder_nf = model_config.decoder_nf[:len(encoder_nf)]
        final_nf = model_config.decoder_nf[len(encoder_nf):]

        ## Encoder
        self.pooling = [nn.MaxPool2d(2) for _ in range(len(encoder_nf))]
        self.encoder = nn.ModuleList()
        prev_nf = in_channel
        for i in range(len(encoder_nf)):
            blk = ConvBlock(prev_nf, encoder_nf[i], bias=False)
            self.encoder.append(blk)
            prev_nf = encoder_nf[i]
        ## Decoder
        self.upsampling = [nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False) for _ in range(len(decoder_nf))]
        self.decoder = nn.ModuleList()
        for i in range(len(decoder_nf)):
            blk = ConvBlock(prev_nf, decoder_nf[i], bias=False)
            self.decoder.append(blk)
            prev_nf = decoder_nf[i] + encoder_nf[-i-1]
        ## Final
        self.final = nn.ModuleList()
        for i in range(len(final_nf)):
            blk = ConvBlock(prev_nf, final_nf[i], bias=False)
            self.final.append(blk)
            prev_nf = final_nf[i]
        self.final.append(nn.Conv2d(prev_nf, out_channel, 1, 1, 0, 1, 1, False))

    def forward(self, x):
        x_hist = [x]
        for i in range(len(self.encoder)):
            x = self.encoder[i](x)
            x_hist.append(x)
            x = self.pooling[i](x)
        for i in range(len(self.decoder)):
            x = self.decoder[i](x)
            x = self.upsampling[i](x)
            x = torch.cat([x, x_hist.pop()], dim=1)
        for i in range(len(self.final)):
            x = self.final[i](x)
        return x

class Discriminator(nn.Module):
    '''
    Input: VGG features of size
    '''
    def __init__(self, model_config):
        super(Discriminator, self).__init__()
        in_channel = model_config.in_channel
        self.fc = nn.Linear(in_channel, 1)

    def forward(self, x):
        return self.fc(x)

class Generator1D(nn.Module):
    '''
    Input: 2 1D tensor
    Output: 1 1D tensor
    '''
    def __init__(self, model_config):
        super(Generator1D, self).__init__()
        ldm_len, shp_len = model_config.in_channel
        hidden_channel = model_config.hidden_channel
        out_channel = shp_len
        self.ldm_fc = nn.Sequential(
            nn.Linear(ldm_len, hidden_channel[0]//2),
            nn.LeakyReLU(0.2, inplace=True),
        )
        self.shp_fc = nn.Sequential(
            nn.Linear(shp_len, hidden_channel[0]//2),
            nn.LeakyReLU(0.2, inplace=True),
        )
        self.fc = nn.ModuleList()
        for i in range(len(hidden_channel)):
            if i == 0:
                continue
            else:
                self.fc.append(nn.Sequential(
                    nn.Linear(hidden_channel[i-1], hidden_channel[i]),
                    nn.LeakyReLU(0.2, inplace=True),
                ))
        self.out_fc = nn.Linear(hidden_channel[-1], out_channel)

    def forward(self, ldm, shp):
        ldm = self.ldm_fc(ldm)
        shp = self.shp_fc(shp)
        x = torch.cat([ldm, shp], dim=1)
        for i in range(len(self.fc)):
            x = self.fc[i](x)
        x = self.out_fc(x)
        return x

class GAN1D(nn.Module):
    def __init__(self, model_config, deca = None):
        super(GAN1D, self).__init__()
        self.vgg = VGGFace2Loss(pretrained_model=model_config.fr_model_path).eval()
        # freeze vgg
        for param in self.vgg.parameters():
            param.requires_grad = False
        self.generator = Generator1D(model_config.generator)
        self.discriminator = Discriminator(model_config.discriminator)
        self.deca = deca
        self.ldm_idx = model_config.lmk_idx
        self.real_img_ldm = model_config.real_img_ldm
        self.cfg = model_config

    def load_chkp(self, path):
        model_chkp = path
        chkp = torch.load(model_chkp)
        cur_dict = self.model_dict()
        print(f'trained model found. load {model_chkp}')
        for k in chkp.keys():
            if k in cur_dict.keys():
                print('load module dict', k, 'from checkpoint')
                util.copy_state_dict(cur_dict[k], chkp[k])

    def model_dict(self):
        d =  dict(
            generator = self.generator.state_dict(),
            discriminator = self.discriminator.state_dict(),
        )
        d.update(self.deca.model_dict())
        return d

    def forward(self, batches):
        # self.deca.eval()
        # # freeze deca
        # for param in self.deca.parameters():
        #     param.requires_grad = False

        images = batches['images']
        batch_size = images.shape[0]
        # get camera, light, pose, expression, shape, texture
        codedict_in = self.deca.encode(images, use_detail=False)
        shape_in = codedict_in['shape']
        # real landmarks?
        if self.real_img_ldm:
            # Option 1: calculate landmarks in real settings
            if not self.cfg.normal_settings:
                opdict_in = self.deca.decode(codedict_in, rendering=True, vis_lmk=False, return_vis=False, use_detail=False)
                landmarks_in = opdict_in['verts'][:, self.ldm_idx, :]
                codedict_del = {}
                for k in codedict_in.keys():
                    if k not in ['shape', 'tex']:
                        codedict_del[k] = codedict_in[k]
                    else:
                        codedict_del[k] = codedict_in[k][list(range(batch_size//2, batch_size))+list(range(batch_size//2))]
                opdict_del = self.deca.decode(codedict_del, only_verts=True)
                landmarks_del = opdict_del['verts'][:, self.ldm_idx, :]
                delta_landmarks = landmarks_in - landmarks_del
            # Option 2: normalize settings
            else:
                codedict_norm = util.normalize_codedict(codedict_in.copy())
                opdict_norm = self.deca.decode(codedict_norm, rendering=True, vis_lmk=False, return_vis=False, use_detail=False)
                landmarks_in = opdict_norm['verts'][:, self.ldm_idx, :]
                landmarks_del = landmarks_in[list(range(batch_size//2, batch_size))+list(range(batch_size//2))]
                delta_landmarks = landmarks_in - landmarks_del
                opdict_in = opdict_norm
                codedict_in = codedict_norm
        # not supported yet
        else: delta_landmarks = batches['landmarks']

        img_in = opdict_in['rendered_images']
        # generate fake shape params
        shape_out = self.generator(delta_landmarks.view(batch_size,-1), \
            shape_in) + shape_in
        logdict = {
            'shape_out change max': (shape_out/shape_in-1).abs().max().item(),
            'shape_out change mean': (shape_out/shape_in-1).abs().mean().item(),
            'landmark_init_error': delta_landmarks.norm(dim=-1).mean().item(),
        }
        codedict_out = codedict_in.copy()
        codedict_out['shape'] = shape_out
        # use same texture as target images
        if self.cfg.use_target_tex:
            codedict_out['tex'] = codedict_in['tex'][list(range(batch_size//2, batch_size))+list(range(batch_size//2))]
        opdict_out = self.deca.decode(codedict_out, rendering=True, vis_lmk=False, return_vis=False, use_detail=False)
        img_out = opdict_out['rendered_images']
        landmarks_out = opdict_out['verts'][:, self.ldm_idx, :]
        logdict.update({
            'landmark_dist': ((landmarks_in-landmarks_out-delta_landmarks)).norm(dim=-1).mean().item()
        })
        # render fake images (everything but the tex and shape are as original images)
        # codedict_out = {}
        # for k in codedict_in.keys():
        #     if k not in ['shape', 'tex']:
        #         codedict_out[k] = codedict_in[k].view(2, batch_size//2, -1)[[1,0]].flatten(0,1)
        # codedict_out['tex'] = codedict_in['tex']

        # vgg features
        imgs = torch.cat([img_in, img_out], dim=0)
        # F = self.vgg.forward_features(imgs)
        # F_real, F_gen = F[:batch_size], F[batch_size:]
        F_real, F_gen = self.vgg.forward_features(img_in), self.vgg.forward_features(img_out)
        visdict = {
            'images': imgs[:batch_size],
            'gen_images': imgs[batch_size:],
            'verts': opdict_in['verts'],
            'gen_verts': opdict_out['verts'],
            'trans_verts': opdict_in['trans_verts'],
            'gen_trans_verts': opdict_out['trans_verts'],
            'lmk': opdict_in['trans_verts'][:, self.ldm_idx, :],
            'lmk_gen': opdict_out['trans_verts'][:, self.ldm_idx, :],
        }

        return F_real, F_gen, landmarks_in, landmarks_out, delta_landmarks, visdict, logdict


from django.urls import path
from django.conf.urls import url, handler404, handler500
from . import views

app_name = "board"

handler404 = 'board.views.error_404'

urlpatterns = [
    path('', views.login, name="login"),
    path('logout/', views.logout, name="logout"),
    path('index/', views.index, name="index"),
    path('register/', views.register, name="register"),
    path('password/', views.password, name="password"),
    path('charts/', views.charts, name="charts"),
    path('indexbarcharts/', views.indexbarcharts, name="indexbarcharts"),
    path('barcharts/', views.barcharts, name="barcharts"),
    path('areacharts/', views.areacharts, name="areacharts"),
    path('piecharts/', views.piecharts, name="piecharts"),
    path('tables/', views.tables, name="tables"),
    path('notice/', views.notice, name="notice"),
    path('notice/create/', views.create, name="create"),
    path('notice/<int:notice_pk>/', views.detail, name="detail"),
    path('notice/<int:notice_pk>/update', views.update, name="update"),
    path('notice/<int:notice_pk>/delete', views.delete, name="delete"),
    path('notice/<int:notice_pk>/comment_create/', views.comment_create, name="comment_create"),
    path('notice/<int:notice_pk>/comment_delete/<int:comment_pk>/', views.comment_delete, name="comment_delete"),
    path('search/', views.search, name="search"),
]


import numpy as np
import pandas as pd

'''
pivot
'''

df1 = pd.DataFrame({'cust_id': ['c1', 'c1', 'c1', 'c2', 'c2', 'c2', 'c3', 'c3', 'c3'],
                    'prod_cd': ['p1', 'p2', 'p3', 'p1', 'p2', 'p3', 'p1', 'p2', 'p3'],
                    'grade' : ['A', 'A', 'A', 'A', 'A', 'A', 'B', 'B', 'B'],
                    'pch_amt': [30, 10, 0, 40, 15, 30, 0, 0, 10]})

df_pivot = df1.pivot(index='cust_id', columns='prod_cd', values='pch_amt')
print(df_pivot)

'''
melt
'''

df2 = pd.DataFrame({'cust_ID' : ['C_001', 'C_001', 'C_002', 'C_002'],
                    'prd_CD' : ['P_001', 'P_002', 'P_001', 'P_002'],
                    'pch_cnt' : [1, 2, 3, 4], 'pch_amt' : [100, 200, 300, 400]})
print(df2.melt())

'''
cust_ID와 prd_CD 칼럼의 값
C_001, P_001, 의
'''
df_melt = pd.melt(df2, id_vars=['cust_ID', 'prd_CD'])
# print(df_melt)

# melt의 variable 이름 , value 이름을 직접 지정해 줄수 있다.
df_melt_setName = pd.melt(df2, id_vars=['cust_ID', 'prd_CD'], var_name='pch_CD', value_name='pch_value')
# print(df_melt_setName)


import sys
import types

PY2 = sys.version_info[0] == 2

if PY2:
    string_types = (basestring,)
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str
    long = long

else:
    string_types = (str,)
    integer_types = (int,)
    class_types = (type,)
    text_type = str
    binary_type = bytes
    long = int

def text_(s, encoding='latin-1', errors='strict'):
    """ If ``s`` is an instance of ``binary_type``, return
    ``s.decode(encoding, errors)``, otherwise return ``s``"""
    if isinstance(s, binary_type):
        return s.decode(encoding, errors)
    return s

if PY2:

    def is_nonstr_iter(v):
        return hasattr(v, '__iter__')

else:

    def is_nonstr_iter(v):
        if isinstance(v, str):
            return False
        return hasattr(v, '__iter__')

if PY2:  # pragma: no cover

    def exec_(code, globs=None, locs=None):
        """Execute code in a namespace."""
        if globs is None:
            frame = sys._getframe(1)
            globs = frame.f_globals
            if locs is None:
                locs = frame.f_locals
            del frame
        elif locs is None:
            locs = globs
        exec("""exec code in globs, locs""")

    exec_(
        """def reraise(tp, value, tb=None):
    raise tp, value, tb
"""
    )

else:  # pragma: no cover
    import builtins

    exec_ = getattr(builtins, "exec")

    def reraise(tp, value, tb=None):
        if value is None:
            value = tp
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

    del builtins