'''
Created on Apr 16, 2017

@author: MT
'''

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution(object):
    def pathSum(self, root, sumVal):
        if not root: return 0
        return self.helper(root, sumVal) +\
            self.pathSum(root.left, sumVal)+\
            self.pathSum(root.right, sumVal)

    def helper(self, root, sumVal):
        if not root: return 0
        if root.val == sumVal:
            res = 1
        else:
            res = 0
        res += self.helper(root.left, sumVal-root.val)
        res += self.helper(root.right, sumVal-root.val)
        return res

    def pathSum_second(self, root, sumVal):
        hashmap = {0:1}
        return self.dfs(root, 0, sumVal, hashmap)

    def dfs(self, root, sumVal, target, hashmap):
        if not root: return 0
        sumVal += root.val
        res = hashmap.get(sumVal-target, 0)
        hashmap[sumVal] = hashmap.get(sumVal, 0)+1
        res += self.dfs(root.left, sumVal, target, hashmap)
        res += self.dfs(root.right, sumVal, target, hashmap)
        hashmap[sumVal] = hashmap[sumVal]-1
        return res


'''
https://www.acmicpc.net/problem/7576
BFS

틀림

기존 bfs와 다른점
1. 탐색을 시작할 수 있는 노드가 한 개가 아님
2. 여러 노드에서 동시에 탐색을 허용
3. 큐에 시작 가능한 노드를 모두 담은 후 시작 Point!!
4. 노드 방문 기록을 남길 추가적인 배열이 필요 없음

왜 틀린지 모르겠음.
'''

import sys
from collections import deque

m, n = map(int, input().split())
box = []
for i in range(n):
    box.append(list(map(int, input().split())))
# quit_cond = min(map(min, box))
# if quit_cond >= 1: # 최소값이 1 이상 즉, 토마토가 모두 담긴 상태라면 0출력 종료
#     print(0)
#     exit()

queue = deque()
d_x = [-1,0,1,0]
d_y = [0,-1,0,1]

def bfs():
    while queue:
        x, y = queue.popleft()
        for i in range(4):
            new_x = x + d_x[i]
            new_y = y + d_y[i]
            if 0 <= new_x < n and 0 <= new_y < m:
                if box[new_x][new_y] == 0:
                    box[new_x][new_y] = box[x][y] + 1
                    queue.append([new_x, new_y])

# 시작 가능한 좌표를 큐에 저장
# 익은 토마토가 있는 모든 위치
for i in range(n):
    for j in range(m):
        if box[i][j] == 1:
            queue.append([i, j])
bfs()
if 0 in box:
    print(-1)
else:
    print(max(map(max, box))-1)


from toah_model import TOAHModel, IllegalMoveError

def move(model, origin, dest):
    """ Apply move from origin to destination in model.

    May raise an IllegalMoveError.

    @param TOAHModel model:
        model to modify
    @param int origin:
        stool number (index from 0) of cheese to move
    @param int dest:
        stool number you want to move cheese to
    @rtype: None
    """
    TOAHModel.move(model, origin, dest)

class ConsoleController:
    """ Controller for text console.
    """

    def __init__(self, number_of_cheeses, number_of_stools):
        """ Initialize a new ConsoleController self.

        @param ConsoleController self:
        @param int number_of_cheeses:
        @param int number_of_stools:
        @rtype: None
        """
        self.number_of_cheeses = number_of_cheeses
        self.number_of_stools = number_of_stools

    def play_loop(self):
        """ Play Console-based game.

        @param ConsoleController self:
        @rtype: None

        TODO:
        -Start by giving instructions about how to enter moves (which is up to
        you). Be sure to provide some way of exiting the game, and indicate
        that in the instructions.
        -Use python's built-in function input() to read a potential move from
        the user/player. You should print an error message if the input does
        not meet the specifications given in your instruction or if it denotes
        an invalid move (e.g. moving a cheese onto a smaller cheese).
        You can print error messages from this method and/or from
        ConsoleController.move; it's up to you.
        -After each valid move, use the method TOAHModel.__str__ that we've
        provided to print a representation of the current state of the game.
        """
        game = TOAHModel(self.number_of_stools)
        game.fill_first_stool(self.number_of_cheeses)
        print("~~~~~~~~~~WELCOME TO FAIYAZ AND RASU'S GAME~~~~~~~~~~\n")
        print("~~~~~~~~~~OBJECTIVES~~~~~~~~~~")
        print("To Move the entire stack of cheese from one stool to another "
              "stool\n")
        print("~~~~~~~~~~INSTRUCTIONS~~~~~~~~~~")
        print("1. Move cheese from one stool to another one at a time \n"
              "2. Can only move the top most cheese from a stool \n"
              "3. Can't place a larger cheese over a smaller cheese \n"
              "4. To exit the game, enter 'exit'")
        print('START \n \n ')
        print(game)
        start = 'Open'
        while start != 'exit':
            origin = int(input("Enter the stool from which you want to move the"
                               " cheese. Stools range from 0 to " + str
                               (self.number_of_stools-1)
                               + ': '))
            final = int(input("Now enter the stool you want to move the slice "
                              "of cheese to. Stools range from 0 to "
                              + str(self.number_of_stools - 1) + '\n'))
            try:
                move(game, origin, final)
            except IllegalMoveError as error:
                print("!!!!!!You caused an error!!!!!!!")
                print(error)
            print(game)
            start = input("type 'exit' to exit or press enter to continue")

if __name__ == '__main__':
    stool = int(input(
        "Enter the number of stools you would want to play the game with: "))
    if stool <= 0:
        while stool <= 0:
            print('Number of stools have to be greater than 0')
            stool = int(input('Enter again: '))
    model = TOAHModel(stool)
    cheese = int(input(
        "Now, enter the number of cheeses you would want to play the game "
        "with: "))
    if cheese <= 0:
        while cheese <= 0:
            print('Number of cheeses have to be greater than 0')
            cheese = int(input('Enter again: '))
    console = ConsoleController(cheese, stool)
    console.play_loop()
    # You should initiate game play here. Your game should be playable by
    # running this file.
    # Leave lines below as they are, so you will know what python_ta checks.
    # You will need consolecontroller_pyta.txt in the same folder.
    import python_ta
    python_ta.check_all(config="consolecontroller_pyta.txt")


#modules
import os
import csv

counties = ["Arapahoe", "Denver", "Jefferson"]

counties.append("El Paso")
print(counties)

counties_dict ={}
counties_dict["Arapahoe"] = 422829
counties_dict["Denver"] = 463353
counties_dict["Jefferson"] = 432438

counties_dict.items()

voting_data = []
voting_data.append({"county":"Arapahoe", "registered_voters": 422829})
voting_data.append({"county":"Denver", "registered_voters": 463353})
voting_data.append({"county":"Jefferson", "registered_voters": 432438})
print(voting_data)

#user votes
my_votes = int(input("How many cotes did you get in the election? "))
#total votes
total_votes = int(input("What was the total votes in the election? "))
#calculate and print percentage
percent_vote = (my_votes / total_votes) *100
print("I recieved " + str(percent_vote) + "% of the total votes")


import re
from pathlib import Path

import pytest
import spacy
from confection import Config
from spacy.util import make_tempdir

from spacy_llm.pipeline import LLMWrapper
from spacy_llm.registry import fewshot_reader, file_reader
from spacy_llm.ty import LLMTask
from spacy_llm.util import assemble_from_config

from ...tasks import make_summarization_task
from ..compat import has_openai_key

EXAMPLES_DIR = Path(__file__).parent / "examples"
TEMPLATES_DIR = Path(__file__).parent / "templates"

@pytest.fixture
def zeroshot_cfg_string():
    return """
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]

    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"
    max_n_words = 20

    [components.llm.model]
    @llm_models = "spacy.GPT-3-5.v2"
    """

@pytest.fixture
def fewshot_cfg_string():
    return f"""
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]

    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"
    max_n_words = 20

    [components.llm.task.examples]
    @misc = "spacy.FewShotReader.v1"
    path = {str((Path(__file__).parent / "examples" / "summarization.yml"))}

    [components.llm.model]
    @llm_models = "spacy.GPT-3-5.v2"
    """

@pytest.fixture
def ext_template_cfg_string():
    """Simple zero-shot config with an external template"""

    return f"""
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]
    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"
    max_n_words = 20

    [components.llm.task.template]
    @misc = "spacy.FileReader.v1"
    path = {str((Path(__file__).parent / "templates" / "summarization.jinja2"))}

    [components.llm.model]
    @llm_models = "spacy.GPT-3-5.v2"
    """

@pytest.fixture
def noop_config():
    return """
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]

    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"

    [components.llm.model]
    @llm_models = "test.NoOpModel.v1"
    """

@pytest.fixture
def example_text() -> str:
    """Returns string to be used as example in tests."""
    return (
        "The atmosphere of Earth is the layer of gases, known collectively as air, retained by Earth's gravity "
        "that surrounds the planet and forms its planetary atmosphere. The atmosphere of Earth creates pressure, "
        "absorbs most meteoroids and ultraviolet solar radiation, warms the surface through heat retention "
        "(greenhouse effect), allowing life and liquid water to exist on the Earth's surface, and reduces "
        "temperature extremes between day and night (the diurnal temperature variation)."
    )

@pytest.mark.external
@pytest.mark.skipif(has_openai_key is False, reason="OpenAI API key not available")
@pytest.mark.parametrize(
    "cfg_string",
    [
        "zeroshot_cfg_string",
        "fewshot_cfg_string",
        "ext_template_cfg_string",
    ],
)
def test_summarization_config(cfg_string, request):
    cfg_string = request.getfixturevalue(cfg_string)
    orig_config = Config().from_str(cfg_string)
    nlp = spacy.util.load_model_from_config(orig_config, auto_fill=True)
    assert nlp.pipe_names == ["llm"]

    # also test nlp config from a dict in add_pipe
    component_cfg = dict(orig_config["components"]["llm"])
    component_cfg.pop("factory")

    nlp2 = spacy.blank("en")
    nlp2.add_pipe("llm", config=component_cfg)
    assert nlp2.pipe_names == ["llm"]

    pipe = nlp.get_pipe("llm")
    assert isinstance(pipe, LLMWrapper)
    assert isinstance(pipe.task, LLMTask)

@pytest.mark.external
@pytest.mark.skipif(has_openai_key is False, reason="OpenAI API key not available")
@pytest.mark.parametrize(
    "cfg_string",
    [
        "zeroshot_cfg_string",
        "fewshot_cfg_string",
        "ext_template_cfg_string",
    ],
)
def test_summarization_predict(cfg_string, example_text, request):
    """Use OpenAI to get summarize text.
    Note that this test may fail randomly, as the LLM's output is unguaranteed to be consistent/predictable
    """
    orig_cfg_string = cfg_string
    cfg_string = request.getfixturevalue(cfg_string)
    orig_config = Config().from_str(cfg_string)
    nlp = spacy.util.load_model_from_config(orig_config, auto_fill=True)

    # One of the examples exceeds the set max_n_words, so we expect a warning to be emitted.
    if orig_cfg_string == "fewshot_cfg_string":
        with pytest.warns(
            UserWarning,
            match=re.escape(
                "The provided example 'Life is a quality th...' has a summary of length 28, but `max_n_words` == 20."
            ),
        ):
            doc = nlp(example_text)
    else:
        doc = nlp(example_text)

    # Check whether a non-empty summary was written and we are somewhat close to the desired upper length limit.
    assert 0 < len(doc._.summary)
    if "ext" not in orig_cfg_string:
        nlp.select_pipes(disable=["llm"])
        assert (
            len(nlp(doc._.summary))
            <= orig_config["components"]["llm"]["task"]["max_n_words"] * 1.5
        )

@pytest.mark.external
@pytest.mark.skipif(has_openai_key is False, reason="OpenAI API key not available")
@pytest.mark.parametrize(
    "cfg_string_and_field",
    [
        ("zeroshot_cfg_string", None),
        ("fewshot_cfg_string", None),
        ("ext_template_cfg_string", None),
        ("zeroshot_cfg_string", "summary_x"),
    ],
)
def test_summarization_io(cfg_string_and_field, example_text, request):
    cfg_string, field = cfg_string_and_field
    orig_cfg_string = cfg_string
    cfg_string = request.getfixturevalue(cfg_string)
    orig_config = Config().from_str(cfg_string)
    if field:
        orig_config["components"]["llm"]["task"]["field"] = field
    nlp = spacy.util.load_model_from_config(orig_config, auto_fill=True)
    assert nlp.pipe_names == ["llm"]
    # ensure you can save a pipeline to disk and run it after loading
    with make_tempdir() as tmpdir:
        nlp.to_disk(tmpdir)
        nlp2 = spacy.load(tmpdir)
    assert nlp2.pipe_names == ["llm"]

    if orig_cfg_string == "fewshot_cfg_string":
        with pytest.warns(
            UserWarning,
            match=re.escape(
                "The provided example 'Life is a quality th...' has a summary of length 28, but `max_n_words` == 20."
            ),
        ):
            doc = nlp2(example_text)
    else:
        doc = nlp2(example_text)

    field = "summary" if field is None else field
    nlp2.select_pipes(disable=["llm"])
    assert 0 < len(nlp2(getattr(doc._, field)))
    if "ext" not in orig_cfg_string:
        assert (
            len(nlp2(getattr(doc._, field)))
            <= orig_config["components"]["llm"]["task"]["max_n_words"] * 1.5
        )

def test_jinja_template_rendering_without_examples(example_text):
    """Test if jinja template renders as we expected

    We apply the .strip() method for each prompt so that we don't have to deal
    with annoying newlines and spaces at the edge of the text.
    """
    nlp = spacy.blank("en")
    doc = nlp.make_doc(example_text)

    llm_ner = make_summarization_task(examples=None, max_n_words=10)
    prompt = list(llm_ner.generate_prompts([doc]))[0]

    assert (
        prompt.strip()
        == f"""
You are an expert summarization system. Your task is to accept Text as input and summarize the Text in a concise way.
The summary must not, under any circumstances, contain more than 10 words.
Here is the Text that needs to be summarized:
'''
{example_text}
'''
Summary:""".strip()
    )

@pytest.mark.parametrize(
    "examples_path",
    [
        str(EXAMPLES_DIR / "summarization.json"),
        str(EXAMPLES_DIR / "summarization.yml"),
        str(EXAMPLES_DIR / "summarization.jsonl"),
    ],
)
def test_jinja_template_rendering_with_examples(examples_path, example_text):
    """Test if jinja2 template renders as expected

    We apply the .strip() method for each prompt so that we don't have to deal
    with annoying newlines and spaces at the edge of the text.
    """
    nlp = spacy.blank("en")
    doc = nlp.make_doc(example_text)

    prompt_examples = fewshot_reader(examples_path)
    llm_ner = make_summarization_task(examples=prompt_examples, max_n_words=20)

    with pytest.warns(
        UserWarning,
        match=re.escape(
            "The provided example 'Life is a quality th...' has a summary of length 28, but `max_n_words` == 20."
        ),
    ):
        prompt = list(llm_ner.generate_prompts([doc]))[0]

    assert (
        prompt.strip()
        == f"""
You are an expert summarization system. Your task is to accept Text as input and summarize the Text in a concise way.
The summary must not, under any circumstances, contain more than 20 words.
Below are some examples (only use these as a guide):

Text:
'''
The United Nations, referred to informally as the UN, is an intergovernmental organization whose stated purposes are to maintain international peace and security, develop friendly relations among nations, achieve international cooperation, and serve as a centre for harmonizing the actions of nations. It is the world's largest international organization. The UN is headquartered on international territory in New York City, and the organization has other offices in Geneva, Nairobi, Vienna, and The Hague, where the International Court of Justice is headquartered.

The UN was established after World War II with the aim of preventing future world wars, and succeeded the League of Nations, which was characterized as ineffective. On 25 April 1945, 50 nations met in San Francisco, California for a conference and started drafting the UN Charter, which was adopted on 25 June 1945. The charter took effect on 24 October 1945, when the UN began operations. The organization's objectives, as defined by its charter, include maintaining international peace and security, protecting human rights, delivering humanitarian aid, promoting sustainable development, and upholding international law. At its founding, the UN had 51 member states; as of 2023, it has 193 – almost all of the world's sovereign states.
'''
Summary:
'''
UN is an intergovernmental organization to foster international peace, security, and cooperation. Established after WW2 with 51 members, now 193.
'''

Text:
'''
Life is a quality that distinguishes matter that has biological processes, such as signaling and self-sustaining processes, from matter that does not, and is defined by the capacity for growth, reaction to stimuli, metabolism, energy transformation, and reproduction. Various forms of life exist, such as plants, animals, fungi, protists, archaea, and bacteria. Biology is the science that studies life.

The gene is the unit of heredity, whereas the cell is the structural and functional unit of life. There are two kinds of cells, prokaryotic and eukaryotic, both of which consist of cytoplasm enclosed within a membrane and contain many biomolecules such as proteins and nucleic acids. Cells reproduce through a process of cell division, in which the parent cell divides into two or more daughter cells and passes its genes onto a new generation, sometimes producing genetic variation.

Organisms, or the individual entities of life, are generally thought to be open systems that maintain homeostasis, are composed of cells, have a life cycle, undergo metabolism, can grow, adapt to their environment, respond to stimuli, reproduce and evolve over multiple generations. Other definitions sometimes include non-cellular life forms such as viruses and viroids, but they are usually excluded because they do not function on their own; rather, they exploit the biological processes of hosts.
'''
Summary:
'''
Life is a quality defined by biological processes, including reproduction, genetics, and metabolism. There are two types of cells and organisms that can grow, respond, reproduce, and evolve.
'''

Here is the Text that needs to be summarized:
'''
{example_text}
'''
Summary:
""".strip()
    )

def test_external_template_actually_loads(example_text):
    template_path = str(TEMPLATES_DIR / "summarization.jinja2")
    template = file_reader(template_path)
    nlp = spacy.blank("en")
    doc = nlp.make_doc(example_text)

    llm_ner = make_summarization_task(template=template)
    prompt = list(llm_ner.generate_prompts([doc]))[0]
    assert (
        prompt.strip()
        == """
This is a test summarization template.
Here is the text: The atmosphere of Earth is the layer of gases, known collectively as air, retained by Earth's gravity that surrounds the planet and forms its planetary atmosphere. The atmosphere of Earth creates pressure, absorbs most meteoroids and ultraviolet solar radiation, warms the surface through heat retention (greenhouse effect), allowing life and liquid water to exist on the Earth's surface, and reduces temperature extremes between day and night (the diurnal temperature variation).
""".strip()
    )

def test_ner_serde(noop_config):
    config = Config().from_str(noop_config)
    nlp1 = assemble_from_config(config)
    nlp2 = assemble_from_config(config)
    nlp2.from_bytes(nlp1.to_bytes())

def test_ner_to_disk(noop_config, tmp_path: Path):
    config = Config().from_str(noop_config)
    nlp1 = assemble_from_config(config)
    nlp2 = assemble_from_config(config)

    path = tmp_path / "model"
    nlp1.to_disk(path)

    cfgs = list(path.rglob("cfg"))
    assert len(cfgs) == 1

    nlp2.from_disk(path)


import base64
import datetime

from dataclasses import dataclass
from pydantic import BaseModel

@dataclass
class AcfgParams:
    creator: str
    decimals: int
    total: int
    round: int
    url: str = None
    url_b64: str = None
    name: str = None
    name_b64: str = None
    unit_name: str = None
    unit_name_b64: str = None
    default_frozen: bool = None
    metadata_hash: str = None
    manager: str = None
    freeze: str = None
    reserve: str = None
    clawback: str = None

class Txn:
    def __init__(self, txninfo):
        self.data = txninfo

    @property
    def round_time(self):
        return self.data["round-time"]

    @property
    def note(self):
        if "note" in self.data:
            return base64.b64decode(self.data["note"])
        return None

    @property
    def type(self):
        return self.data["tx-type"]

class ACfgTxn(Txn):
    def __init__(self, txninfo):
        Txn.__init__(self, txninfo)
        params = self.data["asset-config-transaction"]["params"]
        for key in list(params.keys()):
            if "-" in key:
                params[key.replace("-", "_")] = params.pop(key)
        params["round"] = self.data["confirmed-round"]
        self.params = AcfgParams(**params)
        self.txn_id = self.data["id"]

    @property
    def created_asset_id(self):
        self.data.get("created-asset-index", None)

    def __repr__(self):
        rep = f"ACfgTxn({self.txn_id})"
        return rep

class AssetBaseSchema(BaseModel):
    asset_id: int
    name: str
    description: str = None
    asset_info: dict
    asset_metadata: dict = None
    is_destroyed: bool
    media: dict = None
    collection_asset: list = []
    network: str
    blockchain_updated_at: datetime.datetime


""" Components
"""
import logging
import types
from zope.component import queryAdapter
from eea.relations.interfaces import IContentType
from eea.relations.component.interfaces import IContentTypeLookUp
from eea.relations.component.interfaces import IRelationsLookUp

logger = logging.getLogger('eea.relations.queryContentType')

def queryContentType(context):
    """ Lookup for context related content-type in portal_relations
    """
    connecter = queryAdapter(context, IContentTypeLookUp)
    if not connecter:
        if not isinstance(context, types.MethodType):
            logger.exception('No IContentTypeLookUp adapter found for '
                             '%s', context)
        return None
    return connecter()

def queryForwardRelations(context):
    """ Lookup for context possible forward relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return
    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return
    for relation in connecter.forward():
        yield relation

def queryBackwardRelations(context):
    """ Lookup for context possible backward relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return
    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return
    for relation in connecter.backward():
        yield relation

def getForwardRelationWith(context, ctype):
    """ Get forward relation with ctype

    Returns None if I can't find possible relation or
    possible relation object from portal_relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return None
    new_ctype = ctype
    if not IContentType.providedBy(ctype):
        new_ctype = queryContentType(ctype)
    if not new_ctype:
        return None

    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return None
    return connecter.forward_with(new_ctype)

def getBackwardRelationWith(context, ctype):
    """ Get backward relation with ctype

    Returns None if I can't find possible relation or
    possible relation object from portal_relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return None

    if not IContentType.providedBy(ctype):
        ctype = queryContentType(ctype)
    if not ctype:
        return None

    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return None
    return connecter.backward_with(ctype)


def reverse(x):
    res = 0
    x, isNeg = (-x, True) if x < 0 else (x, False)
    while x > 0:
        rem = x % 10
        x //= 10
        res = 10 * res + rem

    return -res if isNeg else res

from sys import exit

from test_framework import generic_test, test_utils

if __name__ == '__main__':
    exit(generic_test.generic_test_main('reverse_digits.tsv', reverse))


import numpy as np
import matplotlib.pyplot as plt
import numpy as np

def get_evasions_falues(path):
    vals = []
    with open(path, "r") as f:
        for line in f:
            vals.append(float(line.strip().split(",")[0]))
    return np.array(vals)

if __name__ == "__main__":
    # names = ["okreuk_sorel_malconv", "okreukvmi_sorel_malconv", "kreukmi_sorel_malconv", "nkreuk2_sorel_malconv", "kreuk_sorel_malconv"]
    # labels =["FGSM", "FGSM-VMI", "FGSM-MI", "FGSM(2)", "FGSM (og-t)"]
    # names = ["kreuk_sorel_malconv", "kreukmomentum_sorel_malconv", "kreukvmi_sorel_malconv"]
    # names = ["okreuk_sorel_malconv", "okreukvmi_sorel_malconv", "kreukmi_sorel_malconv"]
    # labels =["FGSM", "FGSM-MI", "FGSM-VMI"]
    #names = ["dosextend_malconv", "dosextend_sorel", "dosextend_gbt", "dosextend_malconv_gbtall_sorel_all", "dosextend_malconv_gbt", "dosextend_malconv_sorel", "dosextend_sorelall", "dosextend_gbt_all", "dosextend_malconv_gbtall_sorel", "dosextend_malconv_gbt_sorel_all"]
    #labels = ["Malconv", "SOREL", "GBT", "MalConv_GBTALL_SORELALL", "Malconv_GBT", "Malconv_SOREL", "SORELALL", "GBTALL", "Malconv_GBTALL_SOREL", "Mallconv_GBT_SORELALL"]
    names = ["dosextend_malconv_gbtall_sorel_all"]
    labels = ["i-Ensemble"]
    vals = [get_evasions_falues(f"./vt_outputs/{name}.txt") for name in names]
    # Plot histogram
    for i, val in enumerate(vals):
        plt.hist(val, bins=[1, 5, 10, 15, 20, 25, 30, 35], label=labels[i], alpha=0.5)
        # print("Average # of evasions for {}: {}".format(labels[i], val.mean()))
        print("Average % of evasions for {} (>= 15 evasions): {}".format(labels[i], 100 * (val >= 15).mean()))
        print("Average % of evasions for {} (>= 30 evasions): {}".format(labels[i], 100 * (val >= 30).mean()))
    # Save plot
    plt.xlabel("Number of classifiers evaded (out of 73)")
    plt.savefig(f"histogram.png")


from backend.db.base_class import Base
from sqlalchemy import Column, Integer, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.orm import backref

class CartProduct(Base):
    __tablename__ = 'cart_products'
    cart_product_id = Column(Integer, primary_key=True)
    product_id = Column(Integer, ForeignKey('products.id'), nullable=False)
    product = relationship(
        "Product", foreign_keys=[product_id],
        backref=backref(
            "cart_products", cascade="all,delete"
        )
    )
    user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
    quantity = Column(Integer, nullable=False)

    @property
    def is_available(self):
        try:
            return sum([availability.quantity for availability in self.product.available]) > 0
        except Exception as e:
            print(e)

class CartStock(Base):
    __tablename__ = 'cart_stocks'
    cart_stock_id = Column(Integer, primary_key=True)
    stock_id = Column(Integer, ForeignKey('stocks.id'), nullable=False)
    stock = relationship(
        "Stock", foreign_keys=[stock_id],
        backref=backref(
            "cart_stocks", cascade="all,delete"
        )
    )
    user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
    quantity = Column(Integer, nullable=False)

    @property
    def is_available(self):
        try:
            stock_products = self.stock.products
            stock_availability = 0
            for stock_product in stock_products:
                if len([availability for availability in stock_product.product.available if availability.quantity > 0]) > 0:
                    stock_availability += 1
            return stock_availability == len(stock_products)
        except Exception as e:
            print(e)


import click
from string import Template
import ast
from os.path import isfile
import inflect
import pkg_resources

from crafter.commands import create_model

p = inflect.engine()

def create_relationship(relationship_type, model_1, model_2, assoc = None):
    assoc_name = ''
    if (relationship_type == 'many_many'):
        assoc_name = assoc

    statements = {
        "one_one" : [
            f"{model_2} = db.relationship('{model_2.capitalize()}', backref='{model_1.lower()}', lazy=True, uselist=False)",
            f"{model_1}_id = db.Column(db.Integer, db.ForeignKey('{model_1}.id'), nullable=False)",
        ],
        "one_many": [
            f"{p.plural(model_2)} = db.relationship('{model_2.capitalize()}', backref='{model_1.lower()}', lazy=True)",
            f"{model_1}_id = db.Column(db.Integer, db.ForeignKey('{model_1}.id'), nullable=False)",
        ],
        "many_one": [
            f"{model_2}_id = db.Column(db.Integer, db.ForeignKey('{model_2}.id'), nullable=False)",
            f"{p.plural(model_1)} = db.relationship('{model_1.capitalize()}', backref='{model_2.lower()}', lazy=True)",
        ],
        "many_many": [
            f"{p.plural(assoc_name)} = db.relationship('{model_2.capitalize()}', secondary={assoc_name}, backref=db.backref('{assoc_name}', lazy=True))",
            None,
        ],

    }

    insert_statement(statements[relationship_type][0], model_1)
    insert_statement(statements[relationship_type][1], model_2)

    if (relationship_type == "many_many"):
        create_assoc_table(assoc_name, model_1, model_2)
        add_import(assoc_name, model_1)

    click.echo(f"{model_1} and {model_2} Relationship created successfully ✅")

def add_import(assoc_name, model):
    with open(f"app/models/{model}.py", "r") as f:
        root = ast.parse(f.read())
    import_node = ast.parse(f"import {assoc_name}")
    root.body.insert(1, import_node)
    ast.fix_missing_locations(root)

    unparsed = ast.unparse(root)

    with open(f"app/models/{model}.py", "w") as f:
        f.write(unparsed)

def create_assoc_table(assoc_name, model_1, model_2):

    d = {
        'assoc_name': assoc_name,
        'model_1': model_1,
        'model_2': model_2,

    }

    with open(pkg_resources.resource_filename("crafter", "templates/assoc_table.tpl"), "r") as f:
        src = Template(f.read())
    result_table = src.substitute(d)

    with open(f"app/models/{assoc_name}.py", "w") as f:
        f.write(result_table)

    click.echo(f"{assoc_name} Table created successfully ✅")

def insert_statement(statement, model):
    if statement is None:
        if not isfile(f"app/models/{model}.py"):
            create_model(model)
        return
    if not isfile(f"app/models/{model}.py"):
        create_model(model)

    with open(f"app/models/{model}.py", "r") as f:
        root = ast.parse(f.read())

    classdef = root.body[-1]
    assign_node = ast.parse(statement)
    classdef.body.insert(2, assign_node)
    ast.fix_missing_locations(root)

    unparsed = ast.unparse(root)

    with open(f"app/models/{model}.py", "w") as f:
        f.write(unparsed)


#!/usr/bin/env python3
import sys
import re
from urllib.request import urlopen
from bs4 import BeautifulSoup

def stockQuote(watchlist):
    # default url for a stock quote search
    url="https://markets.businessinsider.com/stocks/"
    data=[] # create a temp list to stuff each url content into
    # iterate over our watchlist extracting html data
    for stock in watchlist:
        temp=urlopen(url+stock+"-stock")
        bs = BeautifulSoup(temp.read(), 'html.parser')
        data.append(bs)

    counter=0
    # iterate over our stock html data to pull out the closing price
    for stock in data:
        nameList=stock.findAll('div', {'class':'col-md-3 col-xs-6 text-right bold black'})
        price=nameList[0].get_text()
        print(watchlist[counter] + " " + price.strip())
        counter+=1

stockQuote(["MSFT","GOOGL","T","BRKB","AAPL","FB","SBUX","MS","SCHW","CSCO","PG","DAL","JNJ","PFE"])


#  @author: Yifeng Peng PID: 730366058

import numpy as np
import matplotlib.pyplot as plt

def Dfunc(t,Y,params):
    [I1,I3,M,g,l] = params
    theta = Y[0]
    p_theta = Y[1]
    phi = Y[2]
    p_phi = Y[3]
    psi = Y[4]
    p_psi = Y[5]

    def kick(t):
        tperiod = 0.5
        if t/tperiod > 1 and t % tperiod < 0.01:
            kick = 1 # can be changed to other value to apply a short term kick in the vertical direction. Notice the amount is affected by the solver resolution dt.
        else:
            kick = 1
        return kick

    ret = np.zeros(6)

    ret[0] = p_theta/I1
    ret[1] = (p_phi-p_psi*np.cos(theta))**2*np.cos(theta)/(I1*np.sin(theta)**3) - p_psi*(p_phi-p_psi*np.cos(theta))/(I1*np.sin(theta)) + M*kick(t)*g*l*np.sin(theta)
    ret[2] = (p_phi-p_psi*np.cos(theta))/(I1*np.sin(theta)**2)
    ret[3] = 0
    ret[4] = p_psi/I3 - (p_phi-p_phi*np.cos(theta))*np.cos(theta)/(I1*np.sin(theta)**2)
    ret[5] = 0
    return ret

def RK4(f,Y0,dt,T,params):
    t = np.arange(0,T,dt)
    iteration = len(t)
    Y = np.zeros([len(Y0),iteration])
    Y[:,0] = Y0[:,0]
    for i in range(iteration-1):
        k1 = f(t[i],Y[:,i],params)
        k2 = f(t[i]+dt/2,Y[:,i]+dt/2*k1,params)
        k3 = f(t[i]+dt/2,Y[:,i]+dt/2*k2,params)
        k4 = f(t[i]+dt,Y[:,i]+dt*k3,params)
        Y[:,i+1] = Y[:,i]+dt/6*(k1+2*k2+2*k3+k4)
    return t,Y


# # Prototyping on mortgages

# Let's prototype a potentially useful ML app here.
# We'll abstract and test and expand later, but we can try to fail fast.

# ## Imports

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import roc_auc_score, precision_recall_fscore_support
from sklearn.preprocessing import label_binarize

# ## Read data

# We'll read the data and keep only those complaints with narratives.
# I don't like stateful manipulation of dataframes in the global scope,
# so we're going to wrap it in a function.

def read_complaints(filename):
    """
    Reads the consumer complaints data (in it's standard downloaded csv form)
    from the file `filename', relative to project root directory.
    """
    complaints = pd.read_csv(filename)
    complaints = complaints[complaints['Consumer complaint narrative'].notnull()]
    return complaints

complaints = read_complaints('data/raw/consumer_complaints.csv')

# Let's try to build an "issue classifier" from the free text.
# This could be used as an assistive system for the administrators that
# need to respond to the free text:
# complaints could be automatically routed to the right person
# (with the caveat that they need a method for correcting misclassifications!)

# As described in the field reference, the "Issue" identified for each
# complaint can take certain values depending on the product
# (which is specified by the user).

# Let's at least start simple, and focus on just one product.
# I'm choosing mortgages because I like the cut of it's jib.

# How many issues exist within the mortgage product category?

(
    complaints
    [complaints['Product'] == 'Mortgage'] # filter
    .groupby('Issue')
    ['Complaint ID'] # select column
    .count()
    .sort_values(ascending=False)
)

# Owing to the imbalanced number of entries for each issue, I suggest
# we take only the top 9 (arbitrarily) categories here,
# then introduce an "other" category.
# It's still an imbalanced dataset, but nonetheless, let's go ahead.
# Let's also encode the issue categories for ease of reference.

issues_abbreviations = {
    "Loan servicing, payments, escrow account": "loan_servicing",
    "Loan modification,collection,foreclosure": "loan_modification",
    "Trouble during payment process": "payment_process",
    "Struggling to pay mortgage": "struggling_to_pay",
    "Application, originator, mortgage broker": "application",
    "Settlement process and costs": "settlement",
    "Applying for a mortgage or refinancing an existing mortgage": "applying",
    "Closing on a mortgage": "closing",
    "Credit decision / Underwriting": "underwriting",
    "Incorrect information on your report": "other",
    "Applying for a mortgage": "applying",
    "Problem with a credit reporting company's investigation into an existing problem": "other",
    "Improper use of your report": "other",
    "Credit monitoring or identity theft protection services": "other",
    "Unable to get your credit report or credit score": "other",
    "Problem with fraud alerts or security freezes": "other"
}

# Our `issues_abbreviations` dict is not 1:1.
# Inverting won't work perfectly because we've mapped several categories to
# "other".
# We'll "fix" that by replacing the categories mapping to "other" with
# "Unknown" in the reverse mapping, since we can't be sure which issue they
# should be attributed to.
# Again, I'm going to encapsulate the stateful append to the dict with a
# function.

def create_abbreviations_issues():
    """
    Reverse the issues_abbreviations dict, encapsulating state
    """
    abbreviations_issues = {
        v:k for (k,v) in issues_abbreviations.items()
    }
    abbreviations_issues['other'] = 'Unknown'
    return abbreviations_issues

abbreviations_issues = create_abbreviations_issues()

# Now we can clean up the data.

def create_mortgages_frame():
    mortgages = (
        complaints
            [complaints['Product'] == 'Mortgage']
            [['Consumer complaint narrative', 'Issue']]
            .rename({
                'Consumer complaint narrative': 'complaint',
                'Issue': 'issue'
                },
                axis='columns')
            .reset_index(drop=True)
    )
    mortgages['issue'] = mortgages.issue.apply(
      lambda x: issues_abbreviations[x]
    )
    return mortgages

mortgages = create_mortgages_frame()

# Let's take a look.

mortgages.head()

def plot_mortgage_complaint_lengths():
    # This is the second time I've written this code, so maybe
    # it'd be a good thing to abstract!
    ax = mortgages.complaint.apply(len).plot(kind='hist', bins = 100)
    ax.set_xlabel('Character length of text')
    return ax

plot_mortgage_complaint_lengths()

# Still a wide range of lengths.

complaint_lengths = mortgages.complaint.apply(len)
complaint_lengths.describe()

# I'm suspicious of the extremely long messages.
# Let's look at some of them.

long_messages = (
    mortgages
    .where(mortgages.complaint.apply(lambda c: len(c) > 10000))
    .dropna()
)

long_messages.sample().complaint.values[0]

# Yup, they just seem like really long complaints ¯\\\_(ツ)_/¯

# How imbalanced is the dataset?

def plot_class_balance():
    ax = pd.value_counts(mortgages.issue).plot(kind='bar')
    ax.set_xlabel('Issue')
    ax.set_ylabel('Count')
    return ax

plot_class_balance()

# ## Classifying complaints

# So we have some handle on the data.
# Let's train a classifier to separate complaints into the issues identified.

# First, split into train and test sets, stratified across the classes.
# If we were going to do model selection too, we'd want to split into an
# additional set to give us some measure of expected real world performance
# (ie stop us overfitting the test set).
# We'll do that eventually, but the results of this experiment aren't going
# be used for anything except testing whether this problem is easily amenable
# to ML, so I'm not too concerned about holding out validation data right now.

train_X, test_X, train_y, test_y = train_test_split(
    mortgages.complaint,
    mortgages.issue,
    stratify=mortgages.issue,
    test_size=0.2
)

# We need a computable representation of text.
# For topic classification (which is what we're doing here),
# keywords usually work great, at least as a baseline.
# We'll use scikit's tf-idf.

vectorizer = TfidfVectorizer()
vectorized_train_X = vectorizer.fit_transform(train_X)
vectorized_test_X = vectorizer.transform(test_X)

# Yes, using vectorizer like this has broken my enforced rigour about
# state transforms in the global scope (vectorizer is now different to
# when it was created). I'll deal with it.

# We'll try a simple multinomial naive bayes classifier.
# Technically this should use integer counts (because that's what a multinomial
# distribution represents), but it works with td-idf in practice too.

clf = MultinomialNB()
clf.fit(vectorized_train_X, train_y)

# Predict on train and test using the fitted classifier.

train_y_pred = clf.predict(vectorized_train_X)
test_y_pred = clf.predict(vectorized_test_X)

# Calculate a measure of goodness.
# We'll use the area under the ROC curve (true positive vs false positive
# rate), with a weighted average over the multiple classes.
# To do this we must binarize the labels (because that's what the sklearn
# method wants).

labels = [x for x in set(issues_abbreviations.values())]

binary_true_train_y = label_binarize(train_y, classes=labels)
binary_pred_train_y = label_binarize(train_y_pred, classes=labels)
binary_true_test_y = label_binarize(test_y, classes=labels)
binary_pred_test_y = label_binarize(test_y_pred, classes=labels)

train_roc = roc_auc_score(
    binary_true_train_y,
    binary_pred_train_y,
    average='weighted'
)

test_roc = roc_auc_score(
    binary_true_test_y,
    binary_pred_test_y,
    average='weighted'
)

print(
  "The weighted average ROC AUC for the train set: {}"
  .format(train_roc)
)

print(
  "The weighted average ROC AUC for the test set: {}"
  .format(test_roc)
)

# OK. That's not a great score, but we used a simple
# method with all default params (for both td-idf and naive bayes).

# Let's also take a look at the precision, recall, f-score and support,
# again weighted by class imbalances.

train_prfs = precision_recall_fscore_support(
    binary_true_train_y,
    binary_pred_train_y,
    average='weighted'
)

test_prfs = precision_recall_fscore_support(
    binary_true_test_y,
    binary_pred_test_y,
    average='weighted'
)

# When calling the `precision...` function, we get an UndefinedMetricWarning
# telling us that there are labels with no predicted samples.
# That's likely because the classes in our training set are so imbalanced.
# There are multiple ways of dealing with that which we can investigate later.


import re
import snscrape.modules.twitter as sntwitter
import pandas as pd
from tqdm import tqdm

def scraperV1(query, maxTweets):
    scraper = sntwitter.TwitterSearchScraper(query)
    data = scraper.get_items()

    tweets = []

    counter = 0

    for tweet in tqdm(data, total=maxTweets):
        if counter > maxTweets:
            break
        if tweet.lang == 'en':
            text = tweet.renderedContent
            text = re.sub(',','\\,',text)
            text = re.sub('\n',' ',text)
            l = []
            l.append(tweet.id)
            l.append(tweet.date)
            l.append(text)
            l.append(tweet.likeCount)
            l.append(tweet.retweetCount)
            l.append(tweet.replyCount)
            l.append(tweet.hashtags)
            tweets.append(l)
            counter +=1

    df = pd.DataFrame(tweets, columns=['id', 'date', 'renderedContent', 'likeCount', 'retweetCount', 'replyCount', 'hashtags'])

    return df


"""add blogview without timestamp

Revision ID: b4893dd631cc
Revises: 8553a96593c0
Create Date: 2017-11-18 00:15:21.615630

"""

# revision identifiers, used by Alembic.
revision = 'b4893dd631cc'
down_revision = '8553a96593c0'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('ix_blog_view_timestamp', table_name='blog_view')
    op.drop_column('blog_view', 'timestamp')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('blog_view', sa.Column('timestamp', mysql.DATETIME(), nullable=True))
    op.create_index('ix_blog_view_timestamp', 'blog_view', ['timestamp'], unique=False)
    # ### end Alembic commands ###


import os
import numpy as np
import tensorflow as tf
import datetime
import pandas as pd
import logging as l
from . import model as m
import global_config_supervised_learning as gc
import supervised_learning.data_generator.game_manager  as gm

class MyCustomRobotEvalCallback(tf.keras.callbacks.Callback):
    def __init__(self,path, model):
        self.model = model
        self.path = path

        with open(path, 'w') as the_file:
            the_file.write('epoch,loss,val_loss,win_rate,draw_rate,loss_rate\n')

    def on_epoch_end(self, epoch, logs=None):
        l.info('start robot evaluation')
        win_rate, draw_rate, loss_rate = gm.robot_evaluate_by_model(self.model)

        with open(self.path, 'a') as the_file:
            the_file.write('{},{},{},{},{},{}\n'.format(epoch, logs['loss'], logs['val_loss'],
                    win_rate,draw_rate ,loss_rate) )

        l.info('end robot evaluation (win, draw, loss) : {} , {} , {}'.format(win_rate, draw_rate, loss_rate))

def get_numpy_data():
    path = os.path.join( gc.C_save_data_folder , 'data.npz' )
    l.info('loading numpy data file {}'.format(path))

    with open(path, 'rb') as f:
        dd = np.load(f)
        data_train = dd['data_train']
        data_dev = dd['data_dev']
        data_test = dd['data_test']
        win_stat = dd['win_stat']
        move_count_stat = dd['move_count_stat']
        winner_level_stat = dd['winner_level_stat']

    l.info('total size of training data {}'.format(data_train.shape))
    l.info('total size of dev data {}'.format(data_dev.shape))
    l.info('total size of test data {}'.format(data_test.shape))
    l.info('total size of win stat data {}'.format(win_stat.shape))
    l.info('total size of move data {}'.format(move_count_stat.shape))
    l.info('total size of player level data {}'.format(winner_level_stat.shape))

    return data_train , data_dev , data_test , win_stat , move_count_stat , winner_level_stat

def get_dataset(n_example=120000):
    l.info('loading dataset file')
    data_train , data_dev , data_test , win_stat , move_count_stat , winner_level_stat = get_numpy_data()

    l.info('shuffling...')

    np.random.shuffle(data_train)
    np.random.shuffle(data_dev)
    np.random.shuffle(data_test)

    all_data = []
    for name, d in [ ( 'training set' , data_train) , ( 'dev set' , data_dev ) , ( 'test set' , data_test )] :
        x = d[:,0:-1]
        y = d[:,-1]

        l.info('converting to tf dataset : {}'.format(name))
        dataset = tf.data.Dataset.from_tensor_slices((x[0:n_example], y[0:n_example]))
        dataset = dataset.batch(gc.HP_Batch)
        l.info('dataset shuffle')
        dataset.shuffle( gc.HP_DATA_SHUFFLE_SIZE )

        all_data.append(dataset)

    dataset_train = all_data[0]
    dataset_dev = all_data[1]
    dataset_test = all_data[2]

    return dataset_train , dataset_dev , dataset_test

def _get_model(save_model_path, create_new=False):
    if create_new:
        l.info('***** create new model *******')
        model = m.create_model()
    else:
        l.info('************************************')
        l.info('***** resume model *******')
        l.info('***** loading {} *******'.format( save_model_path ) )
        l.info('************************************')
        model = tf.keras.models.load_model( save_model_path )

    model.summary(print_fn=l.info)

    return model

def _get_callback(csv_logger, checkpoint_path, robot_eval_logger, model):
    robot_eval = MyCustomRobotEvalCallback(robot_eval_logger, model)
    csv_logger = tf.keras.callbacks.CSVLogger( csv_logger )
    cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=True,
                                                 verbose=1)

    cb = [ csv_logger , cp_callback ]
    if gc.MODE_ENABLE_MODEL_ROBOT_EVULATION :
        cb = cb + [ robot_eval]

    return cb

def _save_history(history_folder, history, run_time):

    x = run_time

    history_path = '{}/history_{}.csv'.format(history_folder, x.strftime( r'%Y%m%d_%H_%M_%S'))
    l.info('saving history to {}'.format(history_path))

    df = pd.DataFrame( history)
    df['date'] = x.strftime( r'%Y-%b-%d' )
    df['time'] = x.strftime( r'%H:%M:%S' )
    df.to_csv(history_path)

def train_model():
    working_folder = gc.C_save_model_current_folder
    base_folder = gc.C_save_model_base_folder

    run_time  = datetime.datetime.now()

    csv_logger = './{}/{}/training_{}.log'.format(base_folder, working_folder, run_time.strftime( r'%Y%m%d_%H_%M_%S'))
    robot_eval_logger = './{}/{}/training_robot_{}.log'.format(base_folder, working_folder, run_time.strftime( r'%Y%m%d_%H_%M_%S'))
    history_folder= './{}/{}/'.format(base_folder, working_folder)
    checkpoint_path = './{}/{}/checkpoint/checkpoint'.format(base_folder, working_folder)
    save_model_path = './{}/{}/savemodel/my_model'.format(base_folder, working_folder)

    to_create_new = (not gc.MODE_RESUME_TRAINING)
    model = _get_model(save_model_path, create_new=to_create_new)

    if gc.MODE_RESUME_TRAINING:
        model.load_weights(checkpoint_path)

    n_example = gc.HP_NUM_TRAINING_DATA
    epochs = gc.HP_EPOCH

    l.info('loading dataset . n_example {}'.format(n_example))
    dataset_train , dataset_dev , dataset_test  = get_dataset(n_example)

    l.info('ready to fit. n_example {}'.format(n_example))
    cb = _get_callback(csv_logger, checkpoint_path, robot_eval_logger, model)

    history = model.fit(dataset_train, epochs=epochs, validation_data=dataset_dev, callbacks=cb)

    l.info('saving model')
    tf.keras.models.save_model( model, save_model_path )
    model.save_weights(checkpoint_path)

    l.info('saving history')
    _save_history(history_folder, history.history, run_time)

    # Evaluate the model on the test data using `evaluate`
    l.info('Evaluate on test data')
    results = model.evaluate(dataset_test)
    l.info('result {}'.format(results))

    return model


import ipaddress

def check_ip(ip):
    try:
        ipaddress.ip_address(ip)
        return True
    except ValueError as err:
        return False

if __name__ == "__main__":
    result = check_ip('10.1.1.1')
    print('Function result:', result)


#Scrape stock data of NTC on NEPSE (2011 AD to present) from
#stocksansar.com using bs4 and selenium and save to a CSV file

import bs4, requests
import pandas as pd
from selenium import webdriver
import time

total_data = []
driver = webdriver.Chrome(r'C:\Users\HP\Downloads\Compressed\chromedriver_win32/chromedriver.exe')
company_id = 'ntc'
driver.get('https://www.sharesansar.com/company/'+company_id)
elem = driver.find_element_by_id('btn_cpricehistory')
elem.click()
time.sleep(3)
i=0
while True:
    i+=1
    soup = bs4.BeautifulSoup(driver.page_source, 'lxml')
    tables = soup.select('#myTableCPriceHistory')[0]
    df = pd.read_html(str(tables))[0]
    total_data.append(df)
    print(f'Added page {i}')
    nextPage = driver.find_element_by_id('myTableCPriceHistory_next')
    nextPage.click()
    if 'disabled' in nextPage.get_attribute('class'):
        break
    time.sleep(3)

total_df = pd.concat(total_data)
total_df.to_csv('lateststockdata.csv')
print('Successfully created csv file')


#!/usr/bin/python3
from fabric import SerialGroup
from fabric.exceptions import GroupException
from fabric.runners import Result
from termcolor import cprint
import sys

def format_proc(line):
    if len(line) < 4 or '-' * 5 in line or '=' * 5 in line or "/usr/" in line:
        return
    else:
        line_args = line.split()
        if "@QD" in line_args[4]:
            name, rtime = line_args[4].split('#')
            rdays = rtime.split("d")[0]
            if rdays == "first_epoch":
                color = "white"
            elif int(rdays) > 6:
                color = 'red'
            elif int(rdays) > 4:
                color = 'yellow'
            elif int(rdays) >= 1:
                color = 'blue'
            else: # = 0
                color = 'green'
            return [line_args[1], name[4:] + ":" + rtime, color, ['bold'], "QD"]
        else:
            return [line_args[1], line_args[4], None, ['dark'], "other"]

def retrieve_smis(server_list, out_filename=".smi_saves"):
    out_file = open(out_filename, "w", encoding="utf-8")
    server_group = SerialGroup(*server_list)
    try:
        return server_group.run('echo "new connection:" && hostname && \
                                 nvidia-smi', out_stream=out_file, hide=True)
    except GroupException as e:
        problem_list = []
        for key, val in e.result.items():
            if type(val) is not Result or val.failed:
                problem_list.append(key.original_host)
        return 'Err', problem_list

def format_smis(filename=".smi_saves"):
    smi_content = open(filename, "r", encoding="utf-8").read()
    all_processes_dict = {}
    smis = smi_content.split("new connection:")[1:]
    for smi in smis:
        smi = smi[1:]
        hostname = smi.partition('\n')[0]
        if "-------------------------------------------------" not in smi:
            all_processes_dict[hostname] = {}
            all_processes_dict[hostname]["ERR"] = ["Not able to retrieve processes"]
            continue
        processes = smi.split("="*77)[1].split("\n")[1:]
        gpu_set = False
        if "mlstudent" in hostname:
            gpu_num = hostname[9]
            hostname = "mlstudentpool"
            gpu_set = True
        if hostname not in all_processes_dict:
            all_processes_dict[hostname] = {}
        for process in processes:
            results = format_proc(process)
            if results is None:
                continue
            if not gpu_set:
                gpu_num = results.pop(0)
            else:
                results.pop(0)
            if gpu_num not in all_processes_dict[hostname]:
                all_processes_dict[hostname][gpu_num] = []
            all_processes_dict[hostname][gpu_num].append(results)
    return all_processes_dict

if __name__ == '__main__':
    server_list = ["mlstudentpool", "mlstudentpool2", "mlstudentpool3",
                   "mlstudentpool4",  "dl1", "stevens", "chichis",
                   "dgx-a", "dgx-b"]
    # server_list = ["dl1", "stevens"]
    # list = ["dl1", "chichis", "stevens", "dgx-a", "dgx-b"]
    retrieve_smis(server_list)
    if "-term" in sys.argv:
        formated_dict = format_smis()
        for hostname, gpus in formated_dict.items():
            print(hostname)
            for gpu, process_list in gpus.items():
                print(gpu, end=": ")
                for process in process_list:
                    process_name = process[0]
                    process_name += " " * (42 - len(process_name))
                    if len(process) == 4:
                        cprint(process_name, process[1], attrs=process[2], end="")
                    else:
                        print(process, end="")

                print()


# coding:utf-8
import random
from tkinter import *
import time

people = {1: '陈俊宇', 2: '陈铭隽', 3: '陈奕纶', 4: '邓历新', 5: '何启荣', 6: '黄炜钊', 7: '黄烨锋', 8: '黄钊华', 9: '李  可', 10: '李铭轩', 11: '李志翔', 12: '李宗轩', 13: '练浩诚', 14: '梁俊文', 15: '梁宇轩', 16: '林子尧', 18: '韦睿行', 19: '冼飘虹', 20: '许  航', 21: '叶孝轩', 22: '张  乐', 23: '郑  超', 24: '钟梓洋', 25: '朱信华', 26: '邹佳洵', 27: '陈楚原', 28: '陈隽欣', 29: '陈沛怡', 30: '陈易沛', 31: '陈钰甄', 32: '范海瑶', 33: '冯琬乔', 34: '何恩希', 35: '何文静', 36: '何宗阳', 37: '黄逸帆', 38: '黎晋旭', 39: '李烨彤', 40: '梁楚清', 41: '梁子怡', 42: '林雨涵', 43: '罗思薇', 44: '马麒越', 45: '邱  童', 46: '石粤礽', 47: '汪雨桐', 48: '谢砚琪', 49: '许艺馨', 50: '杨梓馨', 51: '詹  岚', 52: '张嘉宁', 53: '周熙临', 54: '朱怡晨', 55: '左芷溪'}

def get_random_seats():
    """获取随机座位数据
    input: None
    OUTPUT: List
    输出格式: [[同桌对1]， [同桌对2]， ...]
        E.g. [[2, 5], [7, 12], ...]
        其中，数字为学号
    """

    # 生成打乱后的男女学号
    boys = list(range(1, 27))
    boys.remove(17)
    girls = list(range(27, 56))
    print(str(girls))
    random.shuffle(boys)
    random.shuffle(girls)

    # 确定每对同桌性别
    result = []
    seat_sex = [0]*(len(boys)//2) + [1]*(len(girls)//2)
    if len(seat_sex) != len(boys+girls):
        seat_sex.append(0.5)  # 此处0.5为男女同桌
    random.shuffle(seat_sex)

    # 将男生女生学号随机填入座位
    for num in range(1, len(boys+girls)//2+1):
        sex = seat_sex.pop(0)
        if sex == 0:
            result.append([boys.pop(0), boys.pop(0)])
        elif sex == 1:
            result.append([girls.pop(0), girls.pop(0)])
        else:

            opposite = [boys.pop(0), girls.pop(0)]
            print('男女同桌', people[opposite[0]], people[opposite[1]])  # 男女同桌公开处刑
            print()
            random.shuffle(opposite)
            result.append(opposite)

    return result

def print_seats(seats):
    """格式化输出座位表
    """

    for y in range(7):
        print(y+1, ' ', end='')
        for x in range(4):
            if not (x == 3 and y == 6):
                pair = seats[y*4+x]

                print('|'+people[pair[0]]+' '+people[pair[1]], end='')
        print('|')

print_seats(get_random_seats())
input()


import tensorflow as tf
import numpy as np

class DataReader():
    def __init__(self, dataset, batch_size = None):
        npz = np.load('Audiobooks_data_{0}.npz'.format(dataset))
        self.inputs, self.targets = npz['inputs'].astype(np.float), npz['targets'].astype(np.int)

        if batch_size is None:
            self.batch_size = self.inputs.shape[0]
        else:
            self.batch_size = batch_size
        self.curr_batch = 0
        self.batch_count = self.inputs.shape[0] // self.batch_size

    def __next__(self):
        if self.curr_batch >= self.batch_count:
            self.curr_batch = 0
            raise StopIteration()

        batch_slice = slice(self.curr_batch * self.batch_size, (self.curr_batch + 1) * self.batch_size)
        inputs_batch = self.inputs[batch_slice]
        targets_batch = self.targets[batch_slice]
        self.curr_batch += 1

        classes_num = 2
        targets_one_hot = np.zeros((targets_batch.shape[0], classes_num))
        targets_one_hot[range(targets_batch.shape[0]), targets_batch] = 1

        return inputs_batch, targets_one_hot

    def __iter__(self):
        return self

input_size = 10
output_size = 2
hidden_layer_size = 50

tf.reset_default_graph()

inputs = tf.placeholder(tf.float32, [None, input_size])
targets = tf.placeholder(tf.int32, [None, output_size])

weights_1 = tf.get_variable("weights_1", [input_size, hidden_layer_size])
biases_1 = tf.get_variable("biases_1", [hidden_layer_size])
outputs_1 = tf.nn.relu(tf.matmul(inputs, weights_1) + biases_1)

weights_2 = tf.get_variable("weights_2", [hidden_layer_size, hidden_layer_size])
biases_2 = tf.get_variable("biases_2", [hidden_layer_size])
outputs_2 = tf.nn.sigmoid(tf.matmul(outputs_1, weights_2) + biases_2)

weights_3 = tf.get_variable("weights_3", [hidden_layer_size, hidden_layer_size])
biases_3 = tf.get_variable("biases_3", [hidden_layer_size])
outputs_3 = tf.nn.sigmoid(tf.matmul(outputs_2, weights_3) + biases_3)

weights_final = tf.get_variable("weights_final", [hidden_layer_size, output_size])
biases_final = tf.get_variable("biases_final", [output_size])
outputs = tf.matmul(outputs_3, weights_final) + biases_final

loss = tf.nn.softmax_cross_entropy_with_logits(logits=outputs, labels=targets)
mean_loss = tf.reduce_mean(loss)

out_equals_target = tf.equal(tf.argmax(outputs, 1), tf.argmax(targets, 1))
accuracy = tf.reduce_mean(tf.cast(out_equals_target, tf.float32))

optimize = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(mean_loss)

sess = tf.InteractiveSession()
initializer = tf.global_variables_initializer()
sess.run(initializer)

batch_size = 100
max_epoch = 500
prev_validation_loss = 99999999

train_data = DataReader('train', batch_size)
validation_data = DataReader('validation')

for epoch_counter in range(max_epoch):
    curr_epoch_loss = 0.

    for batch_train_inputs, batch_train_targets in train_data:
        _, batch_loss = sess.run([optimize, mean_loss],
                                 feed_dict={inputs: batch_train_inputs, targets: batch_train_targets})
        curr_epoch_loss += batch_loss

    curr_epoch_loss /= train_data.batch_count

    validation_loss = 0.
    validation_accuracy = 0.
    for batch_validation_inputs, batch_validation_targets in validation_data:
        validation_loss, validation_accuracy = sess.run([mean_loss, accuracy],
                                                        feed_dict={inputs: batch_validation_inputs,
                                                                   targets: batch_validation_targets})

    print('Epoch '+str(epoch_counter+1) +
          '. Training loss: '+'{0:.3f}'.format(curr_epoch_loss) +
          '. Validation loss: '+'{0:.3f}'.format(validation_loss) +
          '. Validation accuracy: '+'{0:.2f}'.format(validation_accuracy * 100.)+'%')

    if validation_loss > prev_validation_loss:
        break
    prev_validation_loss = validation_loss

print('End of training.')

test_data = DataReader('test')
for inputs_batch, targets_batch in test_data:
    test_accuracy = sess.run([accuracy], feed_dict={inputs: inputs_batch, targets: targets_batch})

test_accuracy_percent = test_accuracy[0] * 100.
print('Test accuracy: '+'{0:.2f}'.format(test_accuracy_percent)+'%')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 18 08:28:11 2023

@author: felipe.bortolletto
"""

import pandas as pd
import xarray as xr
import os

def cria_super_csv():

    diretorios = [
    "../catch/table2map",
    "../catch/meteo",
    "../catch/maps/soilhyd",
    "../catch/maps/landuse",
    "../catch/maps",
    "../catch/lai"
                        ]

    # diretorios = [
    # "./params_calibration/table2map",
    # "./params_calibration/meteo",
    # "./params_calibration/maps/soilhyd",
    # "./params_calibration/maps/landuse",
    # "./params_calibration/maps",
    # "./params_calibration/lai"
    #                     ]

    first_meteo = True
    first_lai   = True
    first_estatico = True
    for _entrada in diretorios:
        files = [f for f in os.listdir(f"{_entrada}") if f.endswith(".nc")]
        for arquivo in files:
            entrada_temp = _entrada.split("/")[-1]
            dataset= xr.open_dataset(f"{_entrada}/{arquivo}")
            df = dataset.to_dataframe()
            if entrada_temp in ["meteo"]: #meteorologicos
                if first_meteo:
                    df_meteo = pd.DataFrame(index = df.index)
                    first_meteo = False
                if "spatial_ref" in df.columns:
                    df.drop(columns = ["spatial_ref"],inplace= True)
                df_meteo = pd.merge(df_meteo,df,how = "inner",left_index=True,right_index=True)

            elif entrada_temp in ["lai"]: #indices de area foliar
                if "spatial_ref" in df.columns:
                    df.drop(columns = ["spatial_ref"],inplace= True)
                if "band" in df.index.names:
                    df = df.droplevel('band')

                if first_lai:
                    df_lai = pd.DataFrame(index = df.index)
                    first_lai = False
                df_lai = pd.merge(df_lai,df,how = "inner",left_index=True,right_index=True)
            else: #diversos mapas estaticos
                if "spatial_ref" in df.columns:
                    df.drop(columns = ["spatial_ref"],inplace= True)
                if "transverse_mercator" in df.columns:
                    df.drop(columns = ["transverse_mercator"],inplace= True)
                if "band_data" in df.columns:
                    df.rename(columns = {"band_data":arquivo},inplace = True)
                if first_estatico:
                    df_estatico = pd.DataFrame(index = df.index)
                    first_estatico = False
                df_estatico = pd.merge(df_estatico,df,how = "inner",left_index=True,right_index=True)

    return df_estatico ,df_meteo , df_lai

estaticos,meteo,lai = cria_super_csv()

df = pd.read_csv("/home/felipe.bortolletto/Downloads/dados_estaticos_com_erro_no_escoamento_de_base.csv",index_col = ["y","x"])

for i in estaticos.columns:

    temp1 = df[i]
    temp2 = estaticos[i]
    dif = temp1 - temp2

    print(f"A diferençã entre o note e o pc são: {i} " ,round(dif.sum(),4))

lisf_pm/fim de uma era/novo_calibrador/cria_super_csv.py


import pickle
from datetime import datetime, timedelta
from NotificationCenter import debug, info, warning, error, critical
import ast

# 監視している通貨リスト
def get_monitoring_currency_cache():
    try:
        f = open('save/monitoring_currency_cache.bin', 'rb')
        r = pickle.load(f)
        return r
    except:
        data = {}
        set_monitoring_currency_cache(data)
        return data

def set_monitoring_currency_cache(data):
    f = open('save/monitoring_currency_cache.bin', 'wb')
    pickle.dump(data, f)
    f.close()

# 今取引しているコインについての情報
# 取引してないときはNULL？
# userは1or2
def get_position_cache(user):
    user = str(user)
    if user != '1' and user != '2':
        return
    try:
        path = 'save/position_cache' + str(user) + '.bin'
        with open(path, 'rb') as web:
            r = pickle.load(web)
            return r
    except:
        data = {
            'user': 0,
            'status': False,
            'pair': None,
            'amount': 0,
            'buy_time': None,
            'sell_time': None,
            'buy_coin': 0,
            'sell_coin': 0,
            'profit': 0,
            'mode': 0,
        }
        set_position_cache(user, data)
        warning("[CacheManager]例外：ファイルがないので初期ファイルを作成します")
        return data

def set_position_cache(user, data):
    user = str(user)
    if user != '1' and user != '2':
        return
    path = 'save/position_cache' + str(user) + '.bin'
    with open(path, 'wb') as web:
        pickle.dump(data, web)

def get_binance_api():
    f = open('binance_api.txt', 'r', encoding='UTF-8')
    api_txt = f.read()
    api_key = ast.literal_eval(api_txt)
    return api_key

class CacheManagerClass:
    def __init__(self):
        pass

if __name__ == "__main__":
    dict = {
        'status': False,
        'dt_now': None,
        'price': 0,
        'usecoin': None,
        'amount': 0,
        'wasOverbuy': False,
        'wasOversold': False,
        'crossoverbuy': False,
        'crossoversold': False,
        'buy_coin': 0,
        'sell_coin': 0,
        'mode': 0,
    }
    data = {'XEMUSDT': datetime.now() + timedelta(hours=1),
            'BATUSDT': datetime.now() + timedelta(hours=1)
            }
    # set_position_cache(1, dict)
    # set_monitoring_currency_cache(data)
    print(type(get_binance_api()))


import functools

@functools.lru_cache(maxsize=None)
def binomial(n, k):
    if k == n or k ==0:                         #n == k 일때 헷갈림..
        return 1
    return binomial(n-1, k-1) + binomial(n-1, k)

l=[[0 for value in range(2)] for value in range(1000)]

for i in range(1000):
    l[i]=list(map(int, input().split()))
    if l[i][0]==0 and l[i][1]==0: break

print(l)

for i in range(1000):
    if l[i][0]==0 and l[i][1]==0: break
    print(binomial(l[i][0], l[i][1]))

이항계수/이항 계수3.py


"""
짝수가 주어지면 두 소수의 합으로 나타내라

우선 에라토스테네스의 체를 사용하여 10,000까지의 소수를 모두 판별할 후
값이 주어지면 해당 값의 절반까지 탐색하면서 값에 현재 소수의 값을 뺀 값이 소수라면 정답에 저장한다.
이를 절반까지 탐색하게되면 가장 소수의 차이가 작은 값이 저장될테고, 이를 출력하였다.

"""

import sys
import math

# 에라토스테네스의 체를 사용하여 소수 판별
MAX_VALUE = 10000
prime_num = [True] * (MAX_VALUE + 1)

for i in range(2, int(math.sqrt(MAX_VALUE)) + 1):
    if prime_num[i]:
        j = 2
        while i * j <= MAX_VALUE:
            prime_num[i * j] = False
            j += 1

# 테스트 케이스
T = int(sys.stdin.readline().strip())
for _ in range(T):
    answer = (0, 0)
    target = int(sys.stdin.readline().strip())

    for i in range(2, target // 2 + 1):
        # 현재 값이 소수이고, 골드바흐 파티션이 된다면
        if prime_num[i] and prime_num[target - i]:
            answer = (i, target - i)

    print(f"{answer[0]} {answer[1]}")


import logging

logging.basicConfig(filename='demo.log',level=logging.DEBUG)

# x=10
# y=20
# z=x+y
# logging.info(z)

try:
    n1=int(input("enter the number:"))
    n2=int(input("enter the number:"))
    result=n1/n2
    logging.info(result)
except:
    logging.error("an unknown error happened")

# logging.critical("critical error happened")

# logging.error("an unknown error happened")

# logging.warning("expected value is an integer")

# logging.info("normal message")

# logging.debug("for developers")


import sys, os
from  PyQt4 import QtGui, QtCore
from PyQt4.QtGui import QFileDialog, QAbstractItemView, QListView, QTreeView, QApplication, QDialog
#
# class getExistingDirectories(QFileDialog):
#     def __init__(self, *args):
#         super(getExistingDirectories, self).__init__(*args)
#         self.setOption(self.DontUseNativeDialog, True)
#         self.setFileMode(self.Directory)
#         # self.setOption(self.ShowDirsOnly, True)
#         self.findChildren(QListView)[0].setSelectionMode(QAbstractItemView.ExtendedSelection)
#         self.findChildren(QTreeView)[0].setSelectionMode(QAbstractItemView.ExtendedSelection)
#
# # qapp = QApplication(sys.argv)
# dlg = getExistingDirectories()
# if dlg.exec_() == QDialog.Accepted:
#     for fileq in dlg.selectedFiles():
#         print unicode(fileq)

class FileDialog(QtGui.QFileDialog):
    def __init__(self, *args):
        QtGui.QFileDialog.__init__(self, *args)
        self.setOption(self.DontUseNativeDialog, True)
        self.setFileMode(self.ExistingFiles)

        btns = self.findChildren(QtGui.QPushButton)
        self.openBtn = [x for x in btns if 'open' in str(x.text()).lower()][0]
        self.openBtn.clicked.disconnect()
        self.openBtn.clicked.connect(self.openClicked)
        self.tree = self.findChild(QtGui.QTreeView)

    def openClicked(self):
        inds = self.tree.selectionModel().selectedIndexes()
        files = []
        for i in inds:
            if i.column() == 0:
                files.append(os.path.join(str(self.directory().absolutePath()),str(i.data().toString())))
        self.selectedFiles = files
        # print files, self.selectedFiles
        self.hide()
    # @property
    def filesSelected(self):

        return self.selectedFiles

# app = QApplication(sys.argv)
# windows = FileDialog()
# if windows.exec_() == QDialog.Accepted:
#     windows.show()


from dataclasses import dataclass
import struct
from socket import inet_aton, IPPROTO_TCP
from array import array

"""
TCP Header Format

    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |          Source Port          |       Destination Port        |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                        Sequence Number                        |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                    Acknowledgment Number                      |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |  Data |           |U|A|P|R|S|F|                               |
   | Offset| Reserved  |R|C|S|S|Y|I|            Window             |
   |       |           |G|K|H|T|N|N|                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |           Checksum            |         Urgent Pointer        |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                    Options                    |    Padding    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                             data                              |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
"""

@dataclass
class TCPData:
    from_port: int
    to_port: int
    sequence: int
    acknowledgment: int
    flag_urg: int
    flag_ack: int
    flag_psh: int
    flag_rst: int
    flag_syn: int
    flag_fin: int

class TCPPackage:
    """ TCP package data """
    def __init__(self, from_host: str, from_port: int, to_host: str, to_port: int, flags: int = 0):
        self.from_host = from_host
        self.from_port = from_port
        self.to_host = to_host
        self.to_port = to_port
        self.flags = flags

    def build(self) -> bytes:
        """
        Build tcp package
        """
        package = struct.pack(
            '!HHIIBBHHH',
            self.from_port,  # Source Port
            self.to_port,  # Destination Port
            0,  # Sequence Number
            0,  # Acknoledgement Number
            5 << 4,  # Data offset
            self.flags,  # Flags
            8192,  # Window
            0,  # Checksum (initial value)
            0)  # Urgent pointer
        pseudo_hdr = struct.pack(
            '!4s4sHH',
            inet_aton(self.from_host),  # Source Address
            inet_aton(self.to_host),  # Destination Address
            IPPROTO_TCP,  # Protocol ID
            len(package)  # TCP Length
        )
        checksum = self.check_sum(pseudo_hdr + package)
        package = package[:16] + struct.pack('H', checksum) + package[18:]
        return package

    @staticmethod
    def check_sum(package: bytes) -> int:
        """
        Calculate checksum
        """
        if len(package) % 2 != 0:
            package += b'\0'
        res = sum(array("H", package))
        res = (res >> 16) + (res & 0xffff)
        res += res >> 16
        return (~res) & 0xffff

    @staticmethod
    def tcp_head_parse(data: bytes) -> TCPData:
        """
        Parse tcp header
        """
        (from_port, to_port, sequence, acknowledgment, offset_flags) = \
            struct.unpack('! H H L L H', data[:14])
        flag_urg = (offset_flags & 32) >> 5
        flag_ack = (offset_flags & 16) >> 4
        flag_psh = (offset_flags & 8) >> 3
        flag_rst = (offset_flags & 4) >> 2
        flag_syn = (offset_flags & 2) >> 1
        flag_fin = offset_flags & 1
        return TCPData(from_port, to_port, sequence, acknowledgment, flag_urg, flag_ack,
                       flag_psh, flag_rst, flag_syn, flag_fin)


from quantizers import uniform
import utils
import numpy as np
from tqdm import tqdm

def bucket(row_bucketer, col_bucketer, X):
    """
    We return a list of lists of buckets of data  The first list represents the
    row buckets. The second list represents the column buckets.  If you need to
    do the column wise method first just take the transpose upon entrance and
    exit of this method. The other two returned arrays specify the mapping from
    (1) the bucketed rows back to original indexes (in order) and (2) the
    bucketed columns back to the original indexes (in order).
    """

    row_buckets, row_reorder = row_bucketer.bucket(X)
    buckets, col_reorder = col_bucketer.bucket(row_buckets, X)
    return buckets, row_reorder, col_reorder

def quantize(buckets, quantizer):
    """
    Quantizes each bucket!
    """
    q_buckets = []
    num_bytes = 0
    print("Quantizing...")
    for row_bucket in tqdm(buckets):
        q_col_buckets = []
        for col_bucket in row_bucket:
            q_X, q_bytes = quantizer.quantize(col_bucket)
            num_bytes += q_bytes
            q_col_buckets.append(q_X)
        q_buckets.append(q_col_buckets)
    return q_buckets, num_bytes

def _reconstruct(X, buckets, row_reorder, col_reorder, reconstruction):
    """
    Reconstructs the buckets into inflated embeddings
    Allows special reconstruction specifications
    """
    print("Reconstructing...")
    codex = None
    if reconstruction == 'sorted':
        codex = row_reorder[1]
        row_reorder = row_reorder[0]

    # Allocate buffers for each row bucket. These are used when the columns are
    # reconstructed.
    reconstruction_buffers = []
    for i in range(len(buckets)):
        assert (len(buckets[i]) > 0)
        reconstruction_buffers.append(
            np.zeros([buckets[i][0].shape[0], X.shape[1]]))
    # Buffer for the final output.
    compressed_X = np.zeros(X.shape)

    # The indexes into 'compressed_X' for the rows.
    row_start = 0
    row_end = 0
    for i in tqdm(range(len(buckets))):
        reconstructed_bucket = None
        # all column buckets have the same number of rows.
        row_end = row_start + buckets[i][0].shape[0]

        # The buffer we will reorganize the columns back into.
        reconstructed_bucket = reconstruction_buffers[i]

        # The col indexes for the 'reconstructed_bucket'.
        col_start = 0
        col_end = 0

        for j in range(len(buckets[i])):
            # stitch into large 'reconstructed' matrix.
            col_end = col_start + buckets[i][j].shape[1]
            reconstructed_bucket[:, col_start:col_end] = buckets[i][j]
            col_start = col_end

        # Sort the buffer by the 'col_reorder' index.
        new_x = np.zeros(reconstructed_bucket.shape)
        for c_i in range(len(col_reorder[i])):
            new_x[:, col_reorder[i][c_i]] = reconstructed_bucket[:, c_i]

        # Place it into our final buffer after it is sorted.
        compressed_X[row_start:row_end, :] = new_x

        row_start = row_end

    # reorder the 'reconstructed' matrix. by row reorder
    new_x = np.zeros(compressed_X.shape)
    for i in range(compressed_X.shape[0]):
        new_x[row_reorder[i], :] = compressed_X[i, :]
    #compressed_X = compressed_X[row_reorder, :]
    compressed_X = np.copy(new_x, order="F")

    if reconstruction == 'sorted':
        #first reconstruct as per normal
        X_sorted = compressed_X
        #now apply the sorted insanity to reconstruct fully
        X_vect_sorted = X_sorted.reshape(-1, 1)
        X_vect_desort = np.zeros(X_vect_sorted.shape)
        for i in range(0, len(X_vect_desort)):
            X_vect_desort[codex[i]] = X_vect_sorted[i]
        X_desort = X_vect_desort.reshape(X_sorted.shape)
        return X_desort

    return compressed_X

def finish(buckets, num_bytes, X, V, row_reorder, col_reorder, filename,
           recons):
    compressed_X = _reconstruct(X, buckets, row_reorder, col_reorder, recons)

    # Print stats and send to file.
    utils.print_stats(X, compressed_X, num_bytes)
    utils.to_file(filename, V, compressed_X)


import contextlib
import logging
import os
import sys

logging.basicConfig(level=logging.ERROR)

self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                       os.pardir,
                                       os.pardir))
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)

from oslo_utils import uuidutils

import taskflow.engines
from taskflow.patterns import linear_flow as lf
from taskflow.persistence import models
from taskflow import task

import example_utils as eu  # noqa

# INTRO: In this example linear_flow is used to group three tasks, one which
# will suspend the future work the engine may do. This suspend engine is then
# discarded and the workflow is reloaded from the persisted data and then the
# workflow is resumed from where it was suspended. This allows you to see how
# to start an engine, have a task stop the engine from doing future work (if
# a multi-threaded engine is being used, then the currently active work is not
# preempted) and then resume the work later.
#
# Usage:
#
#   With a filesystem directory as backend
#
#     python taskflow/examples/resume_from_backend.py
#
#   With ZooKeeper as backend
#
#     python taskflow/examples/resume_from_backend.py \
#       zookeeper://127.0.0.1:2181/taskflow/resume_from_backend/

# UTILITY FUNCTIONS #########################################

def print_task_states(flowdetail, msg):
    eu.print_wrapped(msg)
    print("Flow '%s' state: %s" % (flowdetail.name, flowdetail.state))
    # Sort by these so that our test validation doesn't get confused by the
    # order in which the items in the flow detail can be in.
    items = sorted((td.name, td.version, td.state, td.results)
                   for td in flowdetail)
    for item in items:
        print(" %s==%s: %s, result=%s" % item)

def find_flow_detail(backend, lb_id, fd_id):
    conn = backend.get_connection()
    lb = conn.get_logbook(lb_id)
    return lb.find(fd_id)

# CREATE FLOW ###############################################

class InterruptTask(task.Task):
    def execute(self):
        # DO NOT TRY THIS AT HOME
        engine.suspend()

class TestTask(task.Task):
    def execute(self):
        print('executing %s' % self)
        return 'ok'

def flow_factory():
    return lf.Flow('resume from backend example').add(
        TestTask(name='first'),
        InterruptTask(name='boom'),
        TestTask(name='second'))

# INITIALIZE PERSISTENCE ####################################

with eu.get_backend() as backend:

    # Create a place where the persistence information will be stored.
    book = models.LogBook("example")
    flow_detail = models.FlowDetail("resume from backend example",
                                    uuid=uuidutils.generate_uuid())
    book.add(flow_detail)
    with contextlib.closing(backend.get_connection()) as conn:
        conn.save_logbook(book)

    # CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################

    flow = flow_factory()
    engine = taskflow.engines.load(flow, flow_detail=flow_detail,
                                   book=book, backend=backend)

    print_task_states(flow_detail, "At the beginning, there is no state")
    eu.print_wrapped("Running")
    engine.run()
    print_task_states(flow_detail, "After running")

    # RE-CREATE, RESUME, RUN ####################################

    eu.print_wrapped("Resuming and running again")

    # NOTE(harlowja): reload the flow detail from backend, this will allow us
    # to resume the flow from its suspended state, but first we need to search
    # for the right flow details in the correct logbook where things are
    # stored.
    #
    # We could avoid re-loading the engine and just do engine.run() again, but
    # this example shows how another process may unsuspend a given flow and
    # start it again for situations where this is useful to-do (say the process
    # running the above flow crashes).
    flow2 = flow_factory()
    flow_detail_2 = find_flow_detail(backend, book.uuid, flow_detail.uuid)
    engine2 = taskflow.engines.load(flow2,
                                    flow_detail=flow_detail_2,
                                    backend=backend, book=book)
    engine2.run()
    print_task_states(flow_detail_2, "At the end")


## Imports
# Data related libraries
import numpy as np
import pandas as pd

# System
from os import listdir, remove, mkdir, makedirs
import os.path
from os.path import isfile, join, splitext
import sys, getopt
from pathlib import Path
from datetime import date
import tarfile
from genericpath import exists

# Persistance
import pickle

# Plotting
import matplotlib.pyplot as plt

# Validation
import ot
import ot.plot
from bisect import bisect_left

# Modeling
from sklearn.preprocessing import minmax_scale, StandardScaler, MinMaxScaler
from sklearn import preprocessing, linear_model
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

def reduceFilePath(filepath):
    return os.path.split(filepath)[1]

def readGraphOnlyData(folder):
    """
    Reads in the graph features from unlabelled data points (without stress-strain curve)
    """

    # Get all the filenames in the directory
    path = join("features", folder)
    files = [f for f in listdir(path) if isfile(join(path, f)) and "_graph" in f]

    li = []

    # Iterate over files
    for f in files:
        file = join(path, f)
        data = pd.read_csv(file, delimiter=',', encoding='utf-8', index_col = 0)
        li.append(data)

    # Combine into dataframe and return
    data_graphonly = pd.concat(li, axis=0, ignore_index=False)
    data_graphonly.index = data_graphonly.index.str.replace("_Microstructure.graphml", "")

    data_graphonly.index = data_graphonly.index.map(reduceFilePath)

    data_graphonly = data_graphonly.sort_index()

    return data_graphonly

def readStretchFeatures(folder):
    """
    Reads in calculated stretch features placed in 'folder'
    """

    # Get all the filenames in the directory
    path = join("features", folder)
    files = [f for f in listdir(path) if isfile(join(path, f)) and "_stretch" in f]

    # Creat empty list
    li = []

    # Iterate over files
    for f in files:
        file = join(path, f)
        data = pd.read_csv(file, delimiter=',', encoding='utf-8', index_col = 0)
        li.append(data)

    # Combine into dataframe and return
    data_stretch = pd.concat(li, axis=0, ignore_index=False)
    data_stretch.index = data_stretch.index.str.replace("_Microstructure.graphml", "")
    data_stretch.index = data_stretch.index.map(reduceFilePath)
    data_stretch = data_stretch.sort_index()

    return data_stretch

if __name__ == "__main__":

    # Get full command-line arguments
	full_cmd_arguments = sys.argv

	# Load trained models
	file_alpha = sys.argv[1]
	file_beta = sys.argv[2]

	## Alpha
	print(f"LOG: Loading file: {file_alpha}")
	with open(file_alpha, 'rb') as f:
		final_linreg_alpha2 = pickle.load(f)
	#print(final_linreg_alpha2.coef_)

	## Beta
	print(f"LOG: Loading file: {file_beta}")
	with open(file_beta, 'rb') as f:
		final_linreg_beta2 = pickle.load(f)
	#print(final_linreg_beta2.coef_)

	print(f"LOG: Loaded models successfully")

	## Read in Graph_only data
	folder_graphonly_source = sys.argv[3]
	data_graphonly = readGraphOnlyData(folder_graphonly_source)

	## Read in stretch data
	data_stretch = readStretchFeatures(folder_graphonly_source)

	data_joined = data_graphonly.join(data_stretch)
	#print(data_joined)
	print(f"LOG: Loaded data successfully")

	pred_alpha = final_linreg_alpha2.predict(data_joined[data_joined.columns[4:]])
	pred_beta = final_linreg_beta2.predict(data_joined[data_joined.columns[4:]])

	pred = pd.DataFrame([data_joined.index, pred_alpha, pred_beta]).T
	pred.columns = ["file", "alpha", "beta"]
	pred = pred.set_index('file')
	print(f"LOG: Calculated predictions")

	#if not exists(join('../results/predictions/', folder_graphonly_source)):
	#	mkdir(join('../results/predictions/', folder_graphonly_source))

	if not exists(f'../results/predictions/{folder_graphonly_source}'):
		makedirs(f'../results/predictions/{folder_graphonly_source}')

	for i in pred.index:
		pred.loc[i].to_csv(f"../results/predictions/{folder_graphonly_source}/{i}.csv")

	print(f"LOG: Exported predictions")


def ale_or_pilsen(text):
    if "ale" in text.lower():
        return "ale"
    elif "lager" in text.lower():
        return "lager"
    else:
        return "who_knows"

def ebc_to_group(ebc_color: float) -> str:
    lovibond = ebc_color / 1.97
    if lovibond <= 7.5:
        return "yellow"
    elif lovibond > 7.5 and lovibond <= 14:
        return "amber"
    elif lovibond > 14 and lovibond <= 25:
        return "brown"
    elif lovibond > 25:
        return "black"

def group_ph(ph_value):
    if ph_value <= 3.8:
        return "(3.198, 3.8]"
    elif ph_value > 3.8 and ph_value <= 4.4:
        return "(3.8, 4.4]"
    elif ph_value > 4.4 and ph_value <= 5:
        return "(4.4, 5.0]"
    elif ph_value > 5:
        return "(5.0, 5.6]"


# ============= Imports ==================
from re import X
import pygame
import numpy as np
from pynput import keyboard  # using module keyboard
# from math import *

direction = 0

def on_press(key):
    global direction
    try:
        print('alphanumeric key {0} pressed'.format(
            key.char))
        if key.char == "a":
            direction = 0
        elif key.char == "d":
            direction = 1
        elif key.char == "w":
            direction = 2
        elif key.char == "s":
            direction = 3
    except AttributeError:
        print('special key {0} pressed'.format(
            key))

def on_release(key):
    print('{0} released'.format(
        key))
    if key == keyboard.Key.esc:
        # Stop listener
        return False

WHITE = (255, 255, 255)
RED = (255, 0, 0)
BLACK = (0, 0, 0)

scale = 100
WIDTH, HEIGHT = 1000, 800
screen_center = [WIDTH/2, HEIGHT/2]  # x, y
sc_center_vec = np.array([[WIDTH/2], [HEIGHT/2]])
pygame.display.set_caption("3D projection Lab#2")
screen = pygame.display.set_mode((WIDTH, HEIGHT))

points = []
p1 = np.array([-1, -1, 1])
p2 = np.array([1, -1, 1])

# ============= Matrices ==================

# matrix = np.array([
#     [1, 0, 0],
#     [0, 1, 0],
#     [0, 0, 1]])

projection_matrix = np.array([
    [1, 0, 0, 0],
    [0, 1, 0, 0]
])

def get_matrix_Rx(angle):
    rx = np.array([
        [1,             0,              0, 0],
        [0, np.cos(angle), -np.sin(angle), 0],
        [0, np.sin(angle),  np.cos(angle), 0],
        [0,             0,              0, 1]])

    return rx

def get_matrix_Ry(angle):
    ry = np.array([
        [np.cos(angle) , 0,  np.sin(angle), 0],
        [0             , 1,              0, 0],
        [-np.sin(angle), 0,  np.cos(angle), 0],
        [0             , 0,              0, 1]])

    return ry

def get_matrix_Rz(angle):
    rz = np.array([
        [np.cos(angle), -np.sin(angle), 0, 0],
        [np.sin(angle),  np.cos(angle), 0, 0],
        [0,             0,              1, 0],
        [0,             0,              0, 1]])

    return rz

def get_matrix_Tr(tx, ty, tz):
    translation_matrix = np.array([
        [1, 0, 0, tx],
        [0, 1, 0, ty],
        [0, 0, 1, tz],
        [0, 0, 0, 1]])
    return translation_matrix

# ============= Functions ==================
def drawline(point1, point2):
    # tuple1 = (point1.vec[0, 0], point1.vec[1, 0])
    # tuple2 = (point2.vec[0, 0], point2.vec[1, 0])
    pygame.draw.line(screen, BLACK, (point1.x,
                        point1.y), (point2.x, point2.y), width = 4)

# ============= Classes ==================
class Point():
    def __init__(self, x, y, z):
        self.vec = np.array([[x], [y], [z], [1]])
        self.x = self.vec[0, 0]
        self.y = self.vec[1, 0]
        self.z = self.vec[2, 0]

    # def __add__(self, other_point):

    def vecprod(self, matrix):
        self.vec = matrix @ self.vec
        self.x = self.vec[0, 0]
        self.y = self.vec[1, 0]
        self.z = self.vec[2, 0]

        return True

# print(Rz(np.pi/2)@a.vec)
class Wheel:
    def __init__(self, radius, height=0, x=0, y=0, z=0, color='black'):

        self.radius = radius
        self.height = height

        self.diag_coef = radius/np.sqrt(2)
        self.color = color

        self.center = Point(x, y, z)
        self.top = Point(x, y, height)

        self.cirle_points = []
        self.cirle_points.append( Point(self.center.x, self.center.y + self.radius, self.center.z ))
        self.cirle_points.append( Point(self.center.x + self.diag_coef, self.center.y + self.diag_coef,\
            self.center.z))
        self.cirle_points.append( Point(self.center.x + self.radius, self.center.y, self.center.z ))
        self.cirle_points.append( Point(self.center.x + self.diag_coef, self.center.y - self.diag_coef,\
            self.center.z))
        self.cirle_points.append( Point(self.center.x, self.center.y - self.radius, self.center.z ))
        self.cirle_points.append( Point(self.center.x - self.diag_coef, self.center.y - self.diag_coef,\
            self.center.z))
        self.cirle_points.append( Point(self.center.x - self.radius, self.center.y, self.center.z ))
        self.cirle_points.append( Point(self.center.x - self.diag_coef, self.center.y + self.diag_coef,\
            self.center.z))

        self.points = list(self.cirle_points)
        self.points.append(self.top)
        self.points.append(self.center)

    def translate_2_origin(self,):
        cx, cy, cz = self.center.x, self.center.y, self.center.z
        for i in range(len(self.points)):
            self.points[i].vecprod(get_matrix_Tr(-cx, -cy, -cz))
        return True

    def translate(self, tx, ty, tz):
        for i in range(len(self.points)):
            self.points[i].vecprod(get_matrix_Tr(tx, ty, tz))
        return True

# ====================== Rotation ======================
    def rotate_x(self, angle):
        cx, cy, cz = self.center.x, self.center.y, self.center.z
        self.translate_2_origin()
        for i in range(len(self.points)):
            self.points[i].vecprod(get_matrix_Rx(angle))
        self.translate(cx, cy, cz)
        return True

    def rotate_y(self, angle):
        cx, cy, cz = self.center.x, self.center.y, self.center.z
        self.translate_2_origin()
        for i in range(len(self.points)):
            self.points[i].vecprod(get_matrix_Ry(angle))
        self.translate(cx, cy, cz)
        return True

    def rotate_z(self, angle):
        cx, cy, cz = self.center.x, self.center.y, self.center.z
        self.translate_2_origin()
        for i in range(len(self.points)):
            self.points[i].vecprod(get_matrix_Rz(angle))
        self.translate(cx, cy, cz)
        return True

# ====================== Priject and draw ======================
    def project_to_screen(self):
        for i in range(len(self.points)):
            # print(self.points[i].vec, 'mult to\n', projection_matrix, '  n={}\n\n'.format(i))
            projection2d = projection_matrix @ self.points[i].vec + sc_center_vec
        return projection2d

    def draw(self):
        # self.project_to_screen()
        for i in range(len(self.cirle_points)):
            drawline(self.points[i], (self.points[(i+1) % len(self.cirle_points)]))
            drawline(self.points[i], (self.top))

if __name__ == '__main__':
    '''Main method'''

    # # Keyboard:
    # # Collect events until released
    # listener = keyboard.Listener(
    #     on_press=on_press,
    #     on_release=on_release)
    # listener.start()

    FPS = 30

    translation = (0.00,0)
    rotation = 0.001

    # ============= Window and Draw ==================
    pygame.init()
    bg = pygame.Surface(screen.get_size())
    bg.fill((255, 255, 255))
    bg.convert()
    clock = pygame.time.Clock()
    milliseconds = clock.tick(FPS)
    screen.fill(WHITE)
    screen.blit(bg, (0,0))

    # ============= Create object ==================
    w = Wheel(100, height=200, x=0, y=0)

    while True:
        # clock.tick(5)
        screen.blit(bg, (0,0))
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                exit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    pygame.quit()
                    exit()

        # update stuff
        # print('transform1\n', w.center.vec == w.top.vec)
        # w.translate(5, 5, 0)
        w.rotate_x(0.001)
        w.rotate_y(0.001)
        w.rotate_z(0.001)

        # print('transform2\n', w.center.vec == w.top.vec)
        # w.translate_2_origin()

        # print('draw\n',w.center.vec == w.top.vec, '\n\n')
        w.draw()

        pygame.display.update()