import concurrent.futures
from utils import handle_rate_limit_errors, create_chat_completion
from typing import List, Dict

def answer_question(user_question: str) -> List[str]:
    """
    Answer the user's question.
    """
    messages = []
    messages = chain_of_thought(user_question, messages)
    messages = reflexion(messages)
    messages = resolver(messages)
    return messages

def chain_of_thought(question: str, messages: List[Dict], temperatures: List[float] = None) -> List[str]:
    if temperatures is None:
        temperatures = [0.0, 0.5, 1.0]

    user_prompt = f"Question: {question}. Answer: Let's work this out in a step by step way to be sure we have the right answer."
    messages.append({"role": "user", "content": user_prompt})

    # define the function to be run in parallel
    def generate_response(temperature):
        response = handle_rate_limit_errors(create_chat_completion, 300, messages, temperature)
        answer = response.choices[0].message.to_dict()
        # Add Option index to the answer
        answer["content"] = f"Reponse Option {temperature}:\n{answer['content']}"
        return answer

    answers = []
    with concurrent.futures.ThreadPoolExecutor(len(temperatures)) as executor:
        futures = {executor.submit(generate_response, temp) for temp in temperatures}
        for future in concurrent.futures.as_completed(futures):
            answers.append(future.result())

    # Add answers to messages
    messages.extend(answers)
    return messages

def reflexion(messages: List[Dict]) -> List[str]:
    """
    Reflexion: An autonomous agent with dynamic memory and self-reflection.

    Prompt: You are a researcher tasked with investigaging the esponse options provided. List the flaws and faulty logic of each answer option.
            Let's work this out in a step by step way to be sure we have all the errors.

    Args:
        question (str): The question to classify.

    Returns:
        str: The answer.
    """
    prompt = "You are a researcher tasked with investigaging the response options (Response Option X:) provided. List the flaws and faulty logic of each response option. Let's work this out in a step by step way to be sure we have all the errors."
    messages.append({"role": "user", "content": prompt})
    response = handle_rate_limit_errors(create_chat_completion, 300, messages)
    messages.append(response.choices[0].message.to_dict())
    return messages

def resolver(messages: List[Dict]) -> List[str]:
    """
    DERA: Enhancing Large Language Model Completions with Dialog-Enabled Resolving Agents.

    Prompt: You are a resolver tasked with
            1) finding which of the answer options the researcher thought was best
            2) improving that answer
            3) printing the improved answer in full.
            Let's work this out in a step by step way to be sure we have the right answer.
            Print your final answer on the last line.

    Args:
        messages (List[str]): The past communication with the model.

    Returns:
        messages (List[str]): The updates communication with the model.
    """
    prompt = (
        "You are a resolver tasked with\n"
        "1) finding which of the answer options the researcher thought was best\n"
        "2) improving that answer\n"
        "3) printing the improved answer in full.\n"
        "Let's work this out in a step by step way to be sure we have the right answer.\n"
        "Print your final answer on the last line."
    )
    messages.append({"role": "user", "content": prompt})
    response = handle_rate_limit_errors(create_chat_completion, 300, messages)
    messages.append(response.choices[0].message.to_dict())
    return messages

def main():
    user_question = input("Question: ")
    messages = answer_question(user_question)
    print(messages)

if __name__ == "__main__":
    main()


import papermill

to_do = {"HNRNPA1" : "_rep1",
         "HNRNPK" : "",
         "RBFOX2" : "_rep2"}

for rbp,rep in to_do.items():
    print(rbp)
    papermill.execute_notebook('tune_CNNs.ipynb',
                        'tune_CNN_notebooks/%s_CNN.ipynb' % rbp,
                        parameters = {"RBP" : rbp,  "rep" : rep})


#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
#  See https://llvm.org/LICENSE.txt for license information.
#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

from __future__ import annotations

from .support import get_user_loc

import circt.support as support
from circt.dialects import comb, hw, msft, seq

import mlir.ir as ir

from typing import Union
import re

class Value:

  @staticmethod
  def get(value, type=None):
    from .pycde_types import PyCDEType
    value = support.get_value(value)
    if type is None:
      type = value.type
    type = PyCDEType(type)

    if isinstance(type.strip, hw.ArrayType):
      return ListValue(value, type)
    if isinstance(type.strip, hw.StructType):
      return StructValue(value, type)
    if isinstance(type.strip, ir.IntegerType):
      return BitVectorValue(value, type)
    return RegularValue(value, type)

  _reg_name = re.compile(r"^(.*)__reg(\d+)$")

  def reg(self, clk, rst=None, name=None):
    if name is None:
      name = self.name
    if name is not None:
      m = Value._reg_name.match(name)
      if m:
        basename = m.group(1)
        reg_num = m.group(2)
        name = f"{basename}__reg{int(reg_num)+1}"
      else:
        name = name + "__reg1"
    with get_user_loc():
      return Value.get(seq.reg(self.value, clock=clk, reset=rst, name=name))

  @property
  def name(self):
    owner = self.value.owner
    if hasattr(owner, "attributes") and "name" in owner.attributes:
      return ir.StringAttr(owner.attributes["name"]).value
    if isinstance(owner, ir.Block) and isinstance(owner.owner,
                                                  msft.MSFTModuleOp):
      mod = owner.owner
      return ir.StringAttr(
          ir.ArrayAttr(mod.attributes["argNames"])[self.value.arg_number]).value
    if hasattr(self, "_name"):
      return self._name

  @name.setter
  def name(self, new: str):
    owner = self.value.owner
    if hasattr(owner, "attributes"):
      owner.attributes["name"] = ir.StringAttr.get(new)
    else:
      self._name = new

class RegularValue(Value):

  def __init__(self, value, type):
    self.value = value
    self.type = type

class BitVectorValue(Value):

  def __init__(self, value, type):
    self.value = value
    self.type = type

  def __getitem__(self, idxOrSlice: Union[int, slice]):
    if isinstance(idxOrSlice, int):
      s = slice(idxOrSlice, idxOrSlice + 1)
    elif isinstance(idxOrSlice, slice):
      s = idxOrSlice
    else:
      raise TypeError("Expected int or slice")
    idxs = s.indices(len(self))
    if idxs[2] != 1:
      raise ValueError("Integer / bitvector slices do not support steps")

    from .pycde_types import types
    ret_type = types.int(idxs[1] - idxs[0])
    with get_user_loc():
      extracted = comb.ExtractOp.create(idxs[0], ret_type, self.value)
      ret = Value.get(extracted.result)
      if self.name is not None:
        ret.name = f"{self.name}_{idxs[0]}upto{idxs[1]}"
      return ret

  def __len__(self):
    return self.type.width

class ListValue(Value):

  def __init__(self, value, type):
    self.value = value
    self.type = type

  def __getitem__(self, sub):
    if isinstance(sub, int):
      idx = int(sub)
      if idx >= self.type.size:
        raise ValueError("Subscript out-of-bounds")
    else:
      idx = support.get_value(sub)
      if idx is None or not isinstance(support.type_to_pytype(idx.type),
                                       ir.IntegerType):
        raise TypeError("Subscript on array must be either int or MLIR int"
                        f" Value, not {type(sub)}.")
    with get_user_loc():
      v = Value.get(hw.ArrayGetOp.create(self.value, idx))
      if self.name and isinstance(idx, int):
        v.name = self.name + f"__{idx}"
      return v

  def __len__(self):
    return self.type.strip.size

class StructValue(Value):

  def __init__(self, value, type):
    self.value = value
    self.type = type

  def __getitem__(self, sub):
    fields = self.type.strip.get_fields()
    if sub not in [name for name, _ in fields]:
      raise ValueError(f"Struct field '{sub}' not found in {self.type}")
    with get_user_loc():
      return Value.get(hw.StructExtractOp.create(self.value, sub))

  def __getattr__(self, attr):
    ty = self.type.strip
    fields = ty.get_fields()
    if attr in [name for name, _ in fields]:
      with get_user_loc():
        v = Value.get(hw.StructExtractOp.create(self.value, attr))
        if self.name:
          v.name = f"{self.name}__{attr}"
        return v
    raise AttributeError(f"'Value' object has no attribute '{attr}'")


"""Given a list of tuples featuring names and grades on a test,
write a function normalize_grades to normalize the values of the grades to a linear scale between 0 and 1."""
import numpy as np

def normalize_grades(tuples):
    low_value = min(x[1] for x in tuples)
    high_value = max(x[1] for x in tuples)
    return [(x[0], (x[1]-low_value)/(high_value-low_value)) for x in tuples]

def z_normalize_grades(tuples):
    values = [x[1] for x in tuples]
    mean = sum(values) / len(values)
    differences = [(value - mean)**2 for value in values]
    sum_of_differences = sum(differences)
    standard_deviation = (sum_of_differences / (len(values) - 1)) ** 0.5
    # standard_deviation = np.std([values], ddof=1)
    return [(x[0], (x[1]-mean)/standard_deviation) for x in tuples]

tuples = [
    ('Jason', 94),
    ('Tessa', 80),
    ('Carla', 38),
    ('Matt', 43),
    ('Jessica', 100)
]
print(normalize_grades(tuples))
print(z_normalize_grades(tuples))


import datetime
import pandas as pd
import shutil
import os
from dateutil.relativedelta import relativedelta
# from common.logger import write_to_log
from ge_sm.facebook_code.data_creation.query_pages import construct_insight_data, get_fb_demo_data  # derive_dates
from ge_sm.facebook_code.data_creation.query_posts import construct_post_data
from ge_sm.facebook_code.data_creation.query_ads import construct_ad_data

from ge_sm.youtube_code.data_creation.query_reports import run_video_report, run_channel_report, run_demo_report

from ge_sm.google_analytics_code.data_creation.query_reports import construct_user_report, make_signedup_df

from ge_sm.common.create_database_tables import upload_with_pandas, delete_all_data
from ge_sm.control_details import reportlist

# put in path name for secrets et al as running from batch file
upload_dir = 'ge_sm/sm-upload'
path_dir = os.path.abspath('')
pathname = os.path.join(path_dir, upload_dir)

# from common.create_database_tables import update_monthly_table

# # set logger
# logging.basicConfig(filename='facebook_api_log',
#                     filemode='a',
#                     format='%(asctime)s - %(levelname)s - %(module)-20s: %(message)s',
#                     level=logging.DEBUG)
# log = logging.getLogger('facebook_api_log')

# @write_to_log
def main(s, page_start, e):

    user_stdt = '2019-06-12'
    post_start = '2019-06-01'

    # note adding in page / demo start dates as it only seems to like <=90 days (!)
    # demo_start = '2019-08-01'
    #
    # # # # Get various dataframes and add to dictionary
    ## FBADs
    reportlist['fb_ads']['df'] = construct_ad_data(post_start, e)

    # Other FB
    reportlist['fb_demos']['df'] = get_fb_demo_data(page_start, e, 'days_28')
    reportlist['fb_pages_day']['df'] = construct_insight_data(page_start, e, 'day')
    reportlist['fb_pages_week']['df'] = construct_insight_data(page_start, e, 'week')
    reportlist['fb_pages_days_28']['df'] = construct_insight_data(page_start, e, 'days_28')
    reportlist['fb_posts']['df'] = construct_post_data(post_start, e)

    ## YT
    reportlist['yt_channels']['df'] = run_channel_report(post_start, e)
    reportlist['yt_demos']['df'] = run_demo_report(post_start, e)
    reportlist['yt_video']['df'] = run_video_report(post_start, e)
    #
    # # # # Import kadence Id files
    # cancelled it no longer needed
    # if os.path.exists(pathname + '/Kadence_Ids.csv'):
    #     df_kadence = pd.read_csv(pathname + '/Kadence_Ids.csv')
    #     df_kadence.columns = ['KadenceId', 'Centre']
    #     reportlist['ga_signed_up']['df'] = make_signedup_df(reportlist['ga_user_pages_CK']['df'],
    #                                                     reportlist['ga_user_pages_CKW']['df'],
    #                                                     reportlist['ga_user_events_CK']['df'],
    #                                                     df_kadence)
    # user_pages_dfs = construct_user_report(user_stdt, e, 'pages')

    # ## GA
    # reportlist['ga_user_pages_C']['df'] = user_pages_dfs['ga_user_pages_C']
    # reportlist['ga_user_pages_CK']['df'] = user_pages_dfs['ga_user_pages_CK']
    # reportlist['ga_user_pages_CW']['df'] = user_pages_dfs['ga_user_pages_CW']
    # reportlist['ga_user_pages_CKW']['df'] = user_pages_dfs['ga_user_pages_CKW']
    # user_events_dfs = construct_user_report(user_stdt, e, 'events')
    # reportlist['ga_user_events_C']['df'] = user_events_dfs['ga_user_events_C']
    # reportlist['ga_user_events_CK']['df'] = user_events_dfs['ga_user_events_CK']
    # reportlist['ga_user_events_CW']['df'] = user_events_dfs['ga_user_events_CW']
    # reportlist['ga_user_events_CKW']['df'] = user_events_dfs['ga_user_events_CKW']
    #

    #
    # # # # # # # Loop over each element in report list
    for rpt, data in reportlist.items():
        # if rpt != 'fb_ads' and rpt != 'fb_posts' and rpt[:2] == 'fb':
        if rpt[:2] == 'yt' or rpt[:2] == 'fb':
        # try:
            print(rpt)
            # First make table if need (normally keep commented out)
            #     create_table_from_csv(data['tablename'], data['tablestructure'])
            # Copy old .csv files into same location to keep them for future checking
            csvlen = len(data['outputcsvloc']) - 4
            if os.path.exists(pathname + data['outputcsvloc']):
                shutil.copy2(pathname + data['outputcsvloc'], pathname +data['outputcsvloc'][:csvlen] + '_Old.csv')
            # # Delete all data from given table
            # delete_all_data(data['tablename'])
            # temp thing to sort out FB page issue on 3 months
            df = data['df']
            if data['FBPAGE'] == 'Yes' and os.path.exists(pathname + data['outputcsvloc']):
                # get historic data for this FB object
                olddata_df = pd.read_csv(pathname + data['outputcsvloc'])
                # knock out current month then merge with downloaded data
                olddata_df['Date'] = pd.to_datetime(olddata_df['Date'].astype(str).str.slice(0, 10), format = '%Y-%m-%d')
                olddata_df = olddata_df[olddata_df['Date'] < page_start]
                df = pd.concat([olddata_df, df], ignore_index = True, sort = False)
            print(df.to_string())

            os.makedirs(os.path.dirname(pathname + data['outputcsvloc']), exist_ok=True)
            # Export dataframe to csv
            if not os.path.exists(pathname + data['outputcsvloc']):
                open(pathname + data['outputcsvloc'], 'a').close()

            df.to_csv(pathname + data['outputcsvloc'], index = False)
            # Upload to data platform
            # upload_with_pandas(df, data['tablename'])
            # except:
            #     pass

if __name__ == '__main__':
    today = datetime.datetime.now()
    tm = datetime.datetime.strftime(today + datetime.timedelta(days=2), "%Y-%m-%d")
    td = datetime.datetime.strftime(today, "%Y-%m-%d")
    yd = datetime.datetime.strftime(today + datetime.timedelta(days=-1), "%Y-%m-%d")
    print(yd, td, tm)

    # keep eye on month for now
    if not os.environ.get('DYNAMIC_DATES'):
        main(
            os.environ.get('START_DATE', '2019-06-01'),
            os.environ.get('END_DATE', '2019-08-01'),
            tm
        )
    else:
        months = os.environ.get('REPORT_MONTHS', 2)
        main(
            today.replace(day=1).date().strftime('%Y-%m-%d'),
            (today - relativedelta(months=months)).replace(day=1).strftime('%Y-%m-%d'),
            tm
        )


class Move:
    def __init__(self, name, element, damage, accuracy, level_needed=0, unlock=0):
        self.name = name
        self.element = element
        self.damage = int(damage)
        self.accuracy = int(accuracy)
        self.level_needed = int(level_needed)
        self.unlock = int(unlock)

    def has_move(self, level, element):
        """Checks if the level and element allow the dragon to use this move."""
        if level >= self.level_needed:
            if element == "all":
                return True
            if element == self.element:
                return True
        return False

    def equals(self, move):
        """Checks if the name of this object and the name of the other object are the same."""
        return move.name == self.name

    def string_equals(self, name):
        """Checks if the name of this object and the string that is passed are the same."""
        return self.name == name

    def get_file_lines(self):
        """Gets a list of each line of this move."""
        lines = [
            f"{self.name}\n",
            f"{self.element}\n",
            f"{self.damage}\n",
            f"{self.accuracy}\n",
            f"{self.level_needed}\n",
            f"{self.unlock}"
        ]
        return {"name": f"{self.name}.txt", "lines": lines}


from django.shortcuts import render
import pickle
import json
import pandas as pd

# load the model from disk
model = pickle.load(open('model.pkl', 'rb'))
map_add = json.load(open('map.json', 'r'))

def predict(Address, Size):
    X = pd.DataFrame([[Address, Size]], columns=['Address', 'Size(Acres)'])
    X['Address'] = X['Address'].map(map_add)
    prediction = model.predict(X)
    return int(prediction)

def home(request):
    params = {'add': list(map_add.keys())}
    return render(request, 'HouseRentPrediction/index.html', params)

def index(request):
    address = str(request.POST.get('address', 'others'))
    size = int(request.POST.get('size', '500'))
    pred = predict(address, size)
    params = {'add': list(map_add.keys()),
              'pred': f"Monthly Rent for {size} sqft in {address} is Rs {pred}"}
    return render(request, 'HouseRentPrediction/index.html', params)


from lxml import etree
import webob

from nova.api.openstack.compute.contrib import keypairs
from nova.api.openstack import wsgi
from nova import db
from nova import exception
from nova.openstack.common import jsonutils
from nova import quota
from nova import test
from nova.tests.api.openstack import fakes

QUOTAS = quota.QUOTAS

def fake_keypair(name):
    return {'public_key': 'FAKE_KEY',
            'fingerprint': 'FAKE_FINGERPRINT',
            'name': name}

def db_key_pair_get_all_by_user(self, user_id):
    return [fake_keypair('FAKE')]

def db_key_pair_create(self, keypair):
    pass

def db_key_pair_destroy(context, user_id, name):
    if not (user_id and name):
        raise Exception()

def db_key_pair_get(context, user_id, name):
    pass

class KeypairsTest(test.TestCase):

    def setUp(self):
        super(KeypairsTest, self).setUp()
        self.Controller = keypairs.Controller()
        fakes.stub_out_networking(self.stubs)
        fakes.stub_out_rate_limiting(self.stubs)

        self.stubs.Set(db, "key_pair_get_all_by_user",
                       db_key_pair_get_all_by_user)
        self.stubs.Set(db, "key_pair_create",
                       db_key_pair_create)
        self.stubs.Set(db, "key_pair_destroy",
                       db_key_pair_destroy)
        self.flags(
            osapi_compute_extension=[
                'nova.api.openstack.compute.contrib.select_extensions'],
            osapi_compute_ext_list=['Keypairs'])
        self.app = fakes.wsgi_app(init_only=('os-keypairs',))

    def test_keypair_list(self):
        req = webob.Request.blank('/v2/fake/os-keypairs')
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 200)
        res_dict = jsonutils.loads(res.body)
        response = {'keypairs': [{'keypair': fake_keypair('FAKE')}]}
        self.assertEqual(res_dict, response)

    def test_keypair_create(self):
        body = {'keypair': {'name': 'create_test'}}
        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 200)
        res_dict = jsonutils.loads(res.body)
        self.assertTrue(len(res_dict['keypair']['fingerprint']) > 0)
        self.assertTrue(len(res_dict['keypair']['private_key']) > 0)

    def test_keypair_create_with_empty_name(self):
        body = {'keypair': {'name': ''}}
        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 400)

    def test_keypair_create_with_invalid_name(self):
        body = {
            'keypair': {
                'name': 'a' * 256
            }
        }
        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 400)

    def test_keypair_create_with_non_alphanumeric_name(self):
        body = {
            'keypair': {
                'name': 'test/keypair'
            }
        }
        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        res_dict = jsonutils.loads(res.body)
        self.assertEqual(res.status_int, 400)

    def test_keypair_import(self):
        body = {
            'keypair': {
                'name': 'create_test',
                'public_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBYIznA'
                              'x9D7118Q1VKGpXy2HDiKyUTM8XcUuhQpo0srqb9rboUp4'
                              'a9NmCwpWpeElDLuva707GOUnfaBAvHBwsRXyxHJjRaI6Y'
                              'Qj2oLJwqvaSaWUbyT1vtryRqy6J3TecN0WINY71f4uymi'
                              'MZP0wby4bKBcYnac8KiCIlvkEl0ETjkOGUq8OyWRmn7lj'
                              'j5SESEUdBP0JnuTFKddWTU/wD6wydeJaUhBTqOlHn0kX1'
                              'GyqoNTE1UEhcM5ZRWgfUZfTjVyDF2kGj3vJLCJtJ8LoGc'
                              'j7YaN4uPg1rBle+izwE/tLonRrds+cev8p6krSSrxWOwB'
                              'bHkXa6OciiJDvkRzJXzf',
            },
        }

        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 200)
        # FIXME(ja): sholud we check that public_key was sent to create?
        res_dict = jsonutils.loads(res.body)
        self.assertTrue(len(res_dict['keypair']['fingerprint']) > 0)
        self.assertFalse('private_key' in res_dict['keypair'])

    def test_keypair_import_quota_limit(self):

        def fake_quotas_count(self, context, resource, *args, **kwargs):
            return 100

        self.stubs.Set(QUOTAS, "count", fake_quotas_count)

        body = {
            'keypair': {
                'name': 'create_test',
                'public_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBYIznA'
                              'x9D7118Q1VKGpXy2HDiKyUTM8XcUuhQpo0srqb9rboUp4'
                              'a9NmCwpWpeElDLuva707GOUnfaBAvHBwsRXyxHJjRaI6Y'
                              'Qj2oLJwqvaSaWUbyT1vtryRqy6J3TecN0WINY71f4uymi'
                              'MZP0wby4bKBcYnac8KiCIlvkEl0ETjkOGUq8OyWRmn7lj'
                              'j5SESEUdBP0JnuTFKddWTU/wD6wydeJaUhBTqOlHn0kX1'
                              'GyqoNTE1UEhcM5ZRWgfUZfTjVyDF2kGj3vJLCJtJ8LoGc'
                              'j7YaN4uPg1rBle+izwE/tLonRrds+cev8p6krSSrxWOwB'
                              'bHkXa6OciiJDvkRzJXzf',
            },
        }

        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 413)

    def test_keypair_create_quota_limit(self):

        def fake_quotas_count(self, context, resource, *args, **kwargs):
            return 100

        self.stubs.Set(QUOTAS, "count", fake_quotas_count)

        body = {
            'keypair': {
                'name': 'create_test',
            },
        }

        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 413)

    def test_keypair_create_duplicate(self):
        self.stubs.Set(db, "key_pair_get", db_key_pair_get)
        body = {'keypair': {'name': 'create_duplicate'}}
        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 409)

    def test_keypair_import_bad_key(self):
        body = {
            'keypair': {
                'name': 'create_test',
                'public_key': 'ssh-what negative',
            },
        }

        req = webob.Request.blank('/v2/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 400)

    def test_keypair_delete(self):
        req = webob.Request.blank('/v2/fake/os-keypairs/FAKE')
        req.method = 'DELETE'
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 202)

    def test_keypair_get_keypair_not_found(self):
        req = webob.Request.blank('/v2/fake/os-keypairs/DOESNOTEXIST')
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 404)

    def test_keypair_delete_not_found(self):

        def db_key_pair_get_not_found(context, user_id, name):
            raise exception.KeypairNotFound(user_id=user_id, name=name)

        self.stubs.Set(db, "key_pair_get",
                       db_key_pair_get_not_found)
        req = webob.Request.blank('/v2/fake/os-keypairs/WHAT')
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 404)

    def test_keypair_show(self):

        def _db_key_pair_get(context, user_id, name):
            return {'name': 'foo', 'public_key': 'XXX', 'fingerprint': 'YYY'}

        self.stubs.Set(db, "key_pair_get", _db_key_pair_get)

        req = webob.Request.blank('/v2/fake/os-keypairs/FAKE')
        req.method = 'GET'
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        res_dict = jsonutils.loads(res.body)
        self.assertEqual(res.status_int, 200)
        self.assertEqual('foo', res_dict['keypair']['name'])
        self.assertEqual('XXX', res_dict['keypair']['public_key'])
        self.assertEqual('YYY', res_dict['keypair']['fingerprint'])

    def test_keypair_show_not_found(self):

        def _db_key_pair_get(context, user_id, name):
            raise exception.KeypairNotFound(user_id=user_id, name=name)

        self.stubs.Set(db, "key_pair_get", _db_key_pair_get)

        req = webob.Request.blank('/v2/fake/os-keypairs/FAKE')
        req.method = 'GET'
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        self.assertEqual(res.status_int, 404)

    def test_show_server(self):
        self.stubs.Set(db, 'instance_get',
                        fakes.fake_instance_get())
        req = webob.Request.blank('/v2/fake/servers/1')
        req.headers['Content-Type'] = 'application/json'
        response = req.get_response(fakes.wsgi_app(init_only=('servers',)))
        self.assertEquals(response.status_int, 200)
        res_dict = jsonutils.loads(response.body)
        self.assertTrue('key_name' in res_dict['server'])
        self.assertEquals(res_dict['server']['key_name'], '')

    def test_detail_servers(self):
        self.stubs.Set(db, 'instance_get_all_by_filters',
                        fakes.fake_instance_get_all_by_filters())
        req = fakes.HTTPRequest.blank('/v2/fake/servers/detail')
        res = req.get_response(fakes.wsgi_app(init_only=('servers',)))
        server_dicts = jsonutils.loads(res.body)['servers']
        self.assertEquals(len(server_dicts), 5)

        for server_dict in server_dicts:
            self.assertTrue('key_name' in server_dict)
            self.assertEquals(server_dict['key_name'], '')

    def test_keypair_create_with_invalid_keypairBody(self):
        body = {'alpha': {'name': 'create_test'}}
        req = webob.Request.blank('/v1.1/fake/os-keypairs')
        req.method = 'POST'
        req.body = jsonutils.dumps(body)
        req.headers['Content-Type'] = 'application/json'
        res = req.get_response(self.app)
        res_dict = jsonutils.loads(res.body)
        self.assertEqual(res.status_int, 400)
        self.assertEqual(res_dict['badRequest']['message'],
                         "Invalid request body")

class KeypairsXMLSerializerTest(test.TestCase):
    def setUp(self):
        super(KeypairsXMLSerializerTest, self).setUp()
        self.deserializer = wsgi.XMLDeserializer()

    def test_default_serializer(self):
        exemplar = dict(keypair=dict(
                public_key='fake_public_key',
                private_key='fake_private_key',
                fingerprint='fake_fingerprint',
                user_id='fake_user_id',
                name='fake_key_name'))
        serializer = keypairs.KeypairTemplate()
        text = serializer.serialize(exemplar)

        tree = etree.fromstring(text)

        self.assertEqual('keypair', tree.tag)
        for child in tree:
            self.assertTrue(child.tag in exemplar['keypair'])
            self.assertEqual(child.text, exemplar['keypair'][child.tag])

    def test_index_serializer(self):
        exemplar = dict(keypairs=[
                dict(keypair=dict(
                        name='key1_name',
                        public_key='key1_key',
                        fingerprint='key1_fingerprint')),
                dict(keypair=dict(
                        name='key2_name',
                        public_key='key2_key',
                        fingerprint='key2_fingerprint'))])
        serializer = keypairs.KeypairsTemplate()
        text = serializer.serialize(exemplar)

        tree = etree.fromstring(text)

        self.assertEqual('keypairs', tree.tag)
        self.assertEqual(len(exemplar['keypairs']), len(tree))
        for idx, keypair in enumerate(tree):
            self.assertEqual('keypair', keypair.tag)
            kp_data = exemplar['keypairs'][idx]['keypair']
            for child in keypair:
                self.assertTrue(child.tag in kp_data)
                self.assertEqual(child.text, kp_data[child.tag])

    def test_deserializer(self):
        exemplar = dict(keypair=dict(
                name='key_name',
                public_key='public_key'))
        intext = ("<?xml version='1.0' encoding='UTF-8'?>\n"
                  '<keypair><name>key_name</name>'
                  '<public_key>public_key</public_key></keypair>')

        result = self.deserializer.deserialize(intext)['body']
        self.assertEqual(result, exemplar)


# -*- coding: utf-8 -*-
"""

"""

from __future__ import absolute_import

from os.path import dirname, join
from setuptools import setup, find_namespace_packages

__copyright__ = "Copyright (c) 2015-2019 Ing. Petr Jindra. All Rights Reserved."

SRC_DIR = "src"

requirements_path = join(dirname(__file__), "requirements.txt")

def get_requirements():
    with open(requirements_path, "r") as fd:
        return [r for r in fd]

def main():
    setup(
        version="1.0.0",
        name="saw",
        package_dir={"": SRC_DIR},
        packages=find_namespace_packages(SRC_DIR),
        install_requires=get_requirements(),
        data_files=[("", ["requirements.txt"])]
    )

if __name__ == "__main__":
    main()


from flask import Blueprint, request

from .._params import extract_integer, extract_integers, extract_strings
from .._query import execute_query, QueryBuilder
from .._validate import require_all

bp = Blueprint("fluview_clinical", __name__)

@bp.route("/", methods=("GET", "POST"))
def handle():
    require_all(request, "epiweeks", "regions")

    epiweeks = extract_integers("epiweeks")
    regions = extract_strings("regions")
    issues = extract_integers("issues")
    lag = extract_integer("lag")

    # basic query info
    q = QueryBuilder("fluview_clinical", "fvc")

    fields_string = ["release_date", "region"]
    fields_int = ["issue", "epiweek", "lag", "total_specimens", "total_a", "total_b"]
    fields_float = ["percent_positive", "percent_a", "percent_b"]
    q.set_fields(fields_string, fields_int, fields_float)
    q.set_sort_order("epiweek", "region", "issue")

    q.where_integers("epiweek", epiweeks)
    q.where_strings("region", regions)

    if issues is not None:
        q.where_integers("issue", issues)
    elif lag is not None:
        q.where(lag=lag)
    else:
        # final query using most recent issues
        q.with_max_issue("epiweek", "region")

    # send query
    return execute_query(str(q), q.params, fields_string, fields_int, fields_float)


from tracetools.signatures.xpdf_poppler import XpdfPopplerFrame, LexType, \
    LexObj, XpdfPopplerMomentSignature, XpdfPopplerValueSignature, \
    XpdfPopplerPTTracker as Tracker, \
    LexerObjFrame as XpdfPopplerLexerObjFrame, \
    MakeStreamFrame as XpdfPopplerMakeStreamFrame, \
    NewParserMoment as XpdfPopplerNewParserMoment, \
    XpdfPopplerPTMoment, \
    GfxStreamSig as XpdfPopplerGfxStreamSig, \
    XRefConstructSigBase

from tracetools.signatures.pdf import PDFPT as PT
from tracetools.signatures.pdf import PDFEnum
from tracetools.signatures.signatures import ReturnSignature, \
    NewFrameMoment, SigID
from tracetools.signatures.versions import Version
from tracetools.log_entries import is_kind, MemEntry, CallEntry, PCEntry
from aenum import IntEnum, auto
from tracetools.signatures.context import DataContext, ParseReason
import dataclasses

class PopplerFrame(XpdfPopplerFrame):
    lib_name = "libpoppler.so.94"

class LexerObjFrame(XpdfPopplerLexerObjFrame, PopplerFrame):

    @classmethod
    def setup(cls):
        cls.callpcs = [cls.addrs_of("lexergetobj%d" % i, 1)
                       for i in range(1, 4)]
        cls.lib_start = Version.lib_starts(cls.lib_name, 1)

class MakeStreamFrame(XpdfPopplerMakeStreamFrame, PopplerFrame):
    fn_names = ["_ZN6Parser10makeStreamEO6ObjectPh14CryptAlgorithmiiiib"]

    def on_pop(self, new_top):
        super(MakeStreamFrame, self).on_pop(new_top)
        if self.shift_count < 2:
            self.OOPS("Shift should have been called at least 2 times",
                      f"but was only called {self.shift_count} time(s)")
        elif self.shift_count == 2:
            print("Note: endstream() missing from stream or incorrect "
                  "stream length, this is a fixup")
        # todo: if `make_stream_set_length` address/flag reached
        # this means that poppler substituted calculated length into
        # stream's dictionary
        new_top.register_pt_node(self.pt, self)

class XrefStmEntryType(IntEnum):
    FREE = 0
    UNCOMPRESSED = auto()
    COMPRESSED = auto()
    NONE = auto()

class ReadXRefStreamFrame(PopplerFrame):
    sig_id_name = "READ_XREF_STREAM"
    log_type = CallEntry
    attr_name = "target_addr"
    fn_names = ["_ZN4XRef21readXRefStreamSectionEP6StreamPKiii"]
    entry_type_idx = [XrefStmEntryType.FREE,
                      XrefStmEntryType.UNCOMPRESSED,
                      XrefStmEntryType.COMPRESSED]

    @classmethod
    def setup(cls):
        cls.gen_addrs = cls.addrs_of("xref_stm_section_gen")

    def __init__(self, flagged_sig):
        sigs = set([self.manager.sig_from_id(s) for s in
                    [SigID.XREF_STREAM_ENTRY_NUM,
                     SigID.XREF_STREAM_ENTRY_GEN,
                     SigID.XREF_STREAM_ENTRY_OFFSET]])
        self.num = None
        self.gen = None
        self.type = None
        self.offset = None
        pt = PT(PDFEnum.XREF_STREAM_TABLE)
        pt.add_context(ParseReason.create(self))
        super(ReadXRefStreamFrame, self).__init__(
            flagged_sig, pt_tracking_sigs=sigs, pt=pt,
        )

    def register_num(self, val, pc):
        self.num = val

    def register_gen(self, val, pc):
        self.gen = val
        self.type = int(self.entry_type_idx[self.gen_addrs.index(pc)])
        children = [PT(PDFEnum.INT, value=getattr(self, n))
                    for n in ["num", "gen", "offset", "type"]]
        self.add_pt_child(
            PT(PDFEnum.XREF_TABLE_ENTRY, children=children)
        )

    def register_offset(self, val, pc):
        self.offset = val

    def on_pop(self, new_top):
        self.do_register_pt(self.pt)
        super(ReadXRefStreamFrame, self).on_pop(new_top)

    def debug_string(self):
        return super(ReadXRefStreamFrame, self).debug_string() + \
            f"\nnum: {self.num}, gen: {self.gen}, offset: " + \
            f"{self.offset}, type: {self.type}"

class PopplerMomentSignature(XpdfPopplerMomentSignature):
    lib_name = "libpoppler.so.94"

class PopplerValueSignature(XpdfPopplerValueSignature, PopplerMomentSignature):
    pass

class PopplerPTTracker(Tracker):
    tracker_name = "poppler"
    _name = "tracetools.signatures.poppler"
    sig_bases = [PopplerMomentSignature]
    frame_bases = [PopplerFrame]
    _old = Tracker.frame_bases
    Tracker.frame_bases = frame_bases
    ArrayFrame = Tracker.do_register_subcls("ArrayFrame")
    DictFrame = Tracker.do_register_subcls("DictFrame")
    IntFrame = Tracker.do_register_subcls("IntFrame")
    GetObjFrame = Tracker.do_register_subcls("GetObjFrame")
    NewParserFrame = Tracker.do_register_subcls("NewParserFrame")
    GfxStreamFrame = Tracker.do_register_subcls("GfxStreamFrame",
                                                attr={"fn_names":
                                                      ["_ZN3Gfx2goEb"]})
    ShiftObjFrame = Tracker.do_register_subcls("ShiftObjFrame",
                                               attr={"fn_names":
                                                     ["_ZN6Parser5shiftEi",
                                                      "_ZN6Parser5shiftEPKci"]})
    FetchFrame = Tracker.do_register_subcls("FetchFrame",
                                            attr={"fn_names":
                                                  ["_ZN4XRef5fetchEiii"]})
    ConstructXRefFrame = Tracker.do_register_subcls("ConstructXRefFrame",
                                                    attr={"fn_names":
                                                          ["_ZN4XRef13constructXRefEPbb"]})
    Tracker.frame_bases = _old
    register_subcls = [
        ("ParserGetObj", sig_bases,
         {
            "push_frame_class": GetObjFrame,
            "fn_names": ["_ZN6Parser6getObjEbPh14CryptAlgorithmiiiib"]
         }),
        ("GetParserObjId", sig_bases, {"parent_frame_class": GetObjFrame}),
        ("ShiftObjId", sig_bases, {}),
        ("LexerGetObjCalled", sig_bases, {"fn_names": ["_ZN5Lexer6getObjEi",
                                                       "_ZN5Lexer6getObjEPKci"]}),
        ("GetObjSimpleMoment", sig_bases, {}),
        ("GetObjIntMoment", sig_bases, {"push_frame_class": IntFrame}),

        ("DictValueMoment", sig_bases, {}),
        ("DoubleObj", sig_bases, {}),
        ("BoolObj", sig_bases, {}),
        ("LexStrVal", sig_bases, {}),
        ("IntObj", sig_bases, {}),
        ("ObjInfoSig", sig_bases, {}),
        ("NewDictMoment", sig_bases, {"push_frame_class": DictFrame,
                                      "parent_frame_class": DictFrame}),
        ("NewArrayMoment", sig_bases, {"push_frame_class": ArrayFrame,
                                       "parent_frame_class": ArrayFrame}),
        ("DictFrameReturn", sig_bases, {"enable_sig_frame_class": GetObjFrame,
                                        "parent_frame_class": DictFrame}),
        ("CopyStringMoment", [], {"caller_lib_name": "libpoppler.so.94",
                                  "additional_libs": ["libpoppler.so.94"],
                                  "supported_group_ids": ["poppler"]}),
        ("ObjFetchXRefGen", sig_bases, {"parent_frame_class": FetchFrame}),
        ("ObjFetchXRefNum", sig_bases, {"parent_frame_class": FetchFrame}),
        ("ErrorSig", sig_bases, {"parent_frame_class": GfxStreamFrame,
                                 "flag_addr_fn_name":
                                 "_Z5error13ErrorCategoryxPKcz"}),
        ("XRefConstructNum", sig_bases, {"parent_frame_class":
                                         ConstructXRefFrame}),
        ("XRefConstructGen", sig_bases, {"parent_frame_class":
                                         ConstructXRefFrame}),
        ("XRefConstructOffset", sig_bases, {"parent_frame_class":
                                            ConstructXRefFrame}),
        ("XRefConstructId", sig_bases, {"parent_frame_class":
                                        ConstructXRefFrame}),

    ]

class PopplerNewFrameMoment(PopplerMomentSignature, NewFrameMoment):
    pass

class PopplerPTMoment(XpdfPopplerPTMoment, PopplerMomentSignature):
    parent_frame_class = PopplerPTTracker.GetObjFrame

class RecursionErrorMoment(PopplerPTMoment):
    sig_id_name = "RECURSION_ERROR"
    log_type = CallEntry
    attr_name = "pc"
    remove_when_flagged = True

    def get_lex_obj(self):
        return LexObj(LexType.ERROR)

    @classmethod
    def setup(cls):
        # rec_virtpc = 0x2bb20e  # Parser.cc:88, Object::Object()
        cls.rec_value = cls.addrs_of("recursion_error", 1)
        cls.check_values = [cls.rec_value]

class ArrayFrameReturn(ReturnSignature, PopplerPTMoment):
    sig_id_name = "ARRAY_FRAME_RETURN"
    log_type = CallEntry
    attr_name = "pc"
    pt_container_type = PDFEnum.ARRAY_END
    expected_lex_objvalue = b"]"
    expected_lex_objtype = LexType.CMD
    parent_frame_class = PopplerPTTracker.ArrayFrame

    @classmethod
    def setup(cls):
        # virtpc = 0x2bb346  # Parser.cc:104, basic block when shift() called
        # eofvirtpc = 0x2bb31f  # Parser.cc:101, getPos()
        # recursionerror = 0x2bb61f  # Parser.cc:196 Object(objError)
        cls.check_value = cls.addrs_of("array_end", 1)
        cls.eof_value = cls.addrs_of("array_end_eof", 1)
        cls.rec_error_pc = cls.addrs_of("array_end_recursion_error", 1)
        cls.check_values = [cls.eof_value, cls.rec_error_pc]

    def reset(self):
        self.eof = False
        self.recursion_error = False

    def package_pt_obj(self, obj: LexObj) -> PT:
        if self.recursion_error:
            return PT(PDFEnum.ERROR)
        elif self.eof:
            return PT(PDFEnum.EOF)
        else:
            return super(ArrayFrameReturn, self).package_pt_obj(obj)

    def get_lex_obj(self):
        return self.manager.shift_objs.obj1()

    def check_lex_obj(self, obj: LexObj) -> bool:
        if self.eof:
            return self._do_check("obj type", obj, LexType.EOF, "type")
        elif not self.recursion_error:
            return self._do_check("obj type", obj, LexType.CMD, "type") and \
                self._do_check("obj value", obj, b"]", "value")
        else:
            # nothing to enforce if there was a recursion error
            return True

    def do_log_entry(self, log_entry):
        if (is_kind(log_entry, PCEntry) and
           log_entry.pc == self.check_value) or \
           (is_kind(log_entry, CallEntry) and
            log_entry.pc in self.check_values):
            if log_entry.pc == self.eof_value:
                self.eof = True
            elif log_entry.pc == self.rec_error_pc:
                self.recursion_error = True
            self.do_flag(log_entry)

class DictKeyMoment(PopplerPTMoment):
    sig_id_name = "DICT_KEY"
    remove_when_flagged = True
    pt_container_type = PDFEnum.DICT_KEY
    parent_frame_class = PopplerPTTracker.DictFrame

    def __init__(self, key_obj=None):
        # when DICT_KEY is registered by DictFrame or by DICT_VALUE,
        # the DICT_KEY object is obj1, but it it is registered by
        # a DICT_KEY moment, which happens when there is an error
        # parsing the previous key, the key object is obj2() b/c
        # shift() is called after the dict key error moment happens
        # and before the next DICT_KEY moment
        super(DictKeyMoment, self).__init__()
        self.first_lex_obj = key_obj if key_obj else self.first_lex_obj

    @classmethod
    def setup(cls):
        # self.dict_start_obj = dict_start_obj
        # virtpc = 0x2bb3f9  # Parser.cc:123
        # not_name_err = 0x2bb4c8  # Parser.cc:112 Parser::getPos()
        # other_err = 0x2bb61f  # Parser.cc:196 Object(objError)
        cls.dict_key = cls.addrs_of("dict_key", 1)
        cls.not_name_err_pc = cls.addrs_of("dict_key_not_name", 1)
        cls.other_err_pc = cls.addrs_of("dict_key_other_err", 1)
        cls.check_values = [cls.dict_key, cls.not_name_err_pc,
                            cls.other_err_pc]

    def reset(self):
        self.error = False
        self.not_name = False

    def do_log_entry(self, log_entry):
        if is_kind(log_entry, PCEntry) and \
           self.dict_key == log_entry.pc:
            self.do_flag(log_entry)
        elif is_kind(log_entry, CallEntry):
            if self.not_name_err_pc == log_entry.pc:
                self.not_name = True
                self.do_flag(log_entry)
            elif self.other_err_pc == log_entry.pc:
                self.error = True
                self.do_flag(log_entry)

    def package_pt_obj(self, obj):
        self.pt_container_type = PDFEnum.DICT_KEY_ERROR \
            if self.error or self.not_name else PDFEnum.DICT_KEY
        return super(DictKeyMoment, self).package_pt_obj(obj)

    def flag_enable_sigs(self):
        if self.not_name:
            next_sig = self.manager.sig_from_id(SigID.DICT_KEY,
                                                self.manager.shift_objs.obj2())
        elif not self.error:
            next_sig = self.manager.sig_from_id(SigID.DICT_VALUE)
        if next_sig:
            self.parent_sig_enable_frame.add_ghostsite_sig(next_sig)

class Int64Obj(PopplerValueSignature):
    sig_id_name = "INT64_VAL"
    flag_addr_name = "obj_int64_val"

class NewParserXrefID(PopplerValueSignature):
    sig_id_name = "NEW_PARSER_XREF_ID"
    parent_frame_class = PopplerPTTracker.NewParserFrame
    flag_addr_name = "lexer_xref_id"
    remove_when_flagged = False
    log_type = MemEntry
    attr_name = "pc"
    struct_format = "Q"

    def flag(self):
        self.parent_frame.register_xref_id(
            self.unpack_val(self.flagged_entry.value)
        )

# not really using this for anything at the moment,
# but i'm curious as to whether poppler creates
# multiple XRef instances/XRef copies
class NewXrefID(PopplerMomentSignature):
    sig_id_name = "NEW_XREF"
    flag_addr_name = "new_xref_id"
    remove_when_flagged = False
    log_type = MemEntry
    attr_name = "pc"

    @dataclasses.dataclass(repr=False)
    class XRefInfo(DataContext):
        xref_id: str
        stack_str: str

    def reset(self):
        self.all_objs = []

    def flag(self):
        # addr is address of XRef->capacity field
        # print("New XRef at %x" % self.flagged_entry.addr - 24)
        self.all_objs.append(
            self.XRefInfo("%x" % (self.flagged_entry.addr - 24),
                          str(self.manager.ml.stack))
        )

class NewParserMoment(XpdfPopplerNewParserMoment):
    lib_name = "libpoppler.so.94"
    push_frame_class = PopplerPTTracker.NewParserFrame
    fn_names = ["_ZN6ParserC1EP4XRefP6Streamb",
                "_ZN6ParserC1EP4XRefP6Objectb",
                "_ZN6ParserC2EP4XRefP6Streamb",
                "_ZN6ParserC2EP4XRefP6Objectb"]

    class Where(IntEnum):
        UNKNOWN = 0
        PARSE_ENTRY = auto()
        OBJ_STREAM_0 = auto()
        OBJ_STREAM_1 = auto()
        READ_XREF = auto()
        XREF_FETCH = auto()
        CONSTRUCT_XREF = auto()
        LINEARIZATION = auto()
        GFX_STREAM = auto()
        HINTS = auto()

    @classmethod
    def setup(cls):
        cls.parse_entry_start = cls.addrs_of("xref_parse_entry_start", 1)
        cls.parse_entry_end = cls.addrs_of("xref_parse_entry_end", 1)
        cls.obj_stream = cls.addrs_of("xref_obj_new_parser")
        cls.read_xref_start = cls.addrs_of("xref_read_start", 1)
        cls.read_xref_end = cls.addrs_of("xref_read_end", 1)
        cls.xref_fetch_start = cls.addrs_of("xref_fetch_start", 1)
        cls.xref_fetch_end = cls.addrs_of("xref_fetch_end", 1)
        cls.xref_fetch_obj = cls.addrs_of("xref_fetch", 1)
        cls.construct_xref_start = cls.addrs_of("construct_xref_start", 1)
        cls.construct_xref_end = cls.addrs_of("construct_xref_end", 1)
        cls.linearization_start = cls.addrs_of("linearization_start", 1)
        cls.linearization_end = cls.addrs_of("linearization_end", 1)
        cls.hints_new = cls.addrs_of("hint_new_parser", 1)
        cls.gfx_new = cls.addrs_of("gfx_display_new_parser", 1)

    def flag(self):
        # lookup address of caller
        callsite = self.manager.ml.stack.top()
        pc = callsite.pc

        def _in_range(start, end):
            return start <= pc and pc < end
        if _in_range(self.parse_entry_start, self.parse_entry_end):
            self.where = self.Where.PARSE_ENTRY
        elif _in_range(self.read_xref_start, self.read_xref_end):
            self.where = self.Where.READ_XREF
        elif _in_range(self.construct_xref_start, self.construct_xref_end):
            self.where = self.Where.CONSTRUCT_XREF
        elif pc == self.obj_stream[0]:
            self.where = self.Where.OBJ_STREAM_0
        elif pc == self.obj_stream[1]:
            self.where = self.Where.OBJ_STREAM_1
        elif _in_range(self.linearization_start, self.linearization_end):
            self.where = self.Where.LINEARIZATION
        elif _in_range(self.xref_fetch_start, self.xref_fetch_end):
            self.where = self.Where.XREF_FETCH
        elif pc == self.gfx_new:
            self.where = self.Where.GFX_STREAM
        elif pc == self.hints_new:
            self.where == self.Where.HINTS
        else:
            self.where = self.Where.UNKNOWN
        # now determine container type
        if self.where in [self.Where.LINEARIZATION, self.Where.HINTS,
                          self.Where.XREF_FETCH]:
            self.container = PDFEnum.INDIRECT_OBJ
        elif self.where in [self.Where.OBJ_STREAM_0, self.Where.OBJ_STREAM_1]:
            self.container = PDFEnum.STREAM_CONTENTS
        elif self.where == self.Where.PARSE_ENTRY:
            self.container = PDFEnum.XREF_OBJ
        elif self.where == self.Where.CONSTRUCT_XREF:
            self.container = PDFEnum.CONTAINER
        elif self.where == self.Where.GFX_STREAM:
            self.container = PDFEnum.GFX_IMAGE_STREAM
        elif self.where == self.Where.READ_XREF:
            self.container = PDFEnum.XREF_TABLE
        else:
            self.container = PDFEnum.UNKNOWN

class NewParserIdMoment(PopplerMomentSignature):
    sig_id_name = "NEW_PARSER_ID"
    log_type = MemEntry
    attr_name = "pc"
    remove_when_flagged = True
    parent_frame_class = PopplerPTTracker.NewParserFrame

    @classmethod
    def setup(cls):
        # cls.new_parser_stream_id = cls.addrs_of("new_parser_stream_id", 1)
        # cls.new_parser_obj_id = cls.addrs_of("new_parser_obj_id", 1)
        cls.new_parser_stream_id_2 = cls.addrs_of("new_parser_stream_id_2", 1)
        cls.new_parser_obj_id_2 = cls.addrs_of("new_parser_obj_id_2", 1)
        # cls.check_pcs = [cls.new_parser_stream_id, cls.new_parser_obj_id]
        cls.check_values = [cls.new_parser_stream_id_2,
                            cls.new_parser_obj_id_2]

    def reset(self):
        self.obj_id = None
        self.flagged = False

    def flag(self):
        # we generally use inlineImg as the parser ID (but here we use
        # allowStreams because it is accessed before lexer.getObj) ,
        # so add the difference between the offsets of the two
        # instance fields
        if self.flagged_entry.typ == self.flagged_entry.READ:
            self.obj_id = self.flagged_entry.value + 0xe0
        else:
            # some versions of poppler are hooked differently
            self.obj_id = self.flagged_entry.addr
        self.parent_frame.register_parser_id(self.obj_id)

class GetObjError(PopplerMomentSignature):
    sig_id_name = "GET_OBJ_ERROR"
    remove_when_flagged = True

    @classmethod
    def setup(cls):
        cls.check_values = cls.addrs_of("array_end_recursion_error")

    def flag(self):
        self.parent_frame.set_pt_as(PDFEnum.ERROR)

class IntFrameReturn(ReturnSignature, PopplerMomentSignature):
    sig_id_name = "INT_FRAME_RETURN"
    remove_when_flagged = True
    log_type = CallEntry
    attr_name = "pc"
    flag_addr_name = "int_frame_ret"

    @classmethod
    def setup(cls):
        cls.check_values = cls.addrs_of("int_frame_ret")

    def reset(self):
        self.is_reference = False
        self.is_error = False

    def flag(self):
        idx = self.check_values.index(self.flagged_entry.pc)
        self.is_reference = idx == 0
        self.is_error = idx == 2

class XRefStmSig(XRefConstructSigBase, PopplerMomentSignature):
    parent_frame_class = ReadXRefStreamFrame

    def flag(self):
        getattr(self.parent_frame, f"register_{self.entry_field}")(
            self.unpack_val(self.flagged_entry.value), self.flagged_entry.pc
        )

class XrefStreamOffset(XRefStmSig):
    sig_id_name = "XREF_STREAM_ENTRY_OFFSET"
    flag_addr_name = "xref_stm_section_offset"
    entry_field = "offset"
    struct_format = "Q"

class XrefStreamGen(XRefStmSig):
    sig_id_name = "XREF_STREAM_ENTRY_GEN"
    flag_addr_name = "xref_stm_section_gen"
    entry_field = "gen"
    struct_format = "Q"

class XrefStreamNum(XRefStmSig):
    sig_id_name = "XREF_STREAM_ENTRY_NUM"
    flag_addr_name = "xref_stm_section_idx2"
    entry_field = "num"
    struct_format = "i"

class GfxStreamSig(XpdfPopplerGfxStreamSig, PopplerMomentSignature):
    parent_frame_class = PopplerPTTracker.GfxStreamFrame
    reg_names = XpdfPopplerGfxStreamSig.reg_names
    reg_names["gfx_update_display"] = "register_update_display"
    reg_names["gfx_command_aborted"] = "register_command_aborted",

# embedded CMap/CIDFont parsing -- see also CharCodeToUnicode::parseCMap1, CharCodeToUnicode.cc


"""
    https://leetcode.com/problems/n-ary-tree-level-order-traversal/
"""
from collections import deque
from typing import Optional

from src.data_structures.leetcode.tree import Node

class Solution:
    # pylint: disable=too-few-public-methods
    """
    Solution
    """

    def level_order(self, root: Optional[Node] = None) -> list[list[int]]:
        # pylint: disable=no-self-use
        """
        Given an n-ary tree, return the level order traversal of its nodes' values.
        """
        if root is None:
            return []

        # BFS with a queue of (Node, level)
        queue: deque[Node] = deque([root])

        results: list[list[int]] = []

        while queue:
            level_results = []
            queue_next: deque[Node] = deque()
            while queue:
                node = queue.popleft()
                level_results.append(node.val)

                for next_node in node.children or []:
                    queue_next.append(next_node)

            results.append(level_results)
            queue = queue_next

        return results


import json
from PyQt5 import QtGui, QtWidgets, QtCore
from PyQt5.QtCore import *
from PyQt5.QtGui import *
import os

from src.SettingsManager import SettingsManager

class TSHLocaleHelperSignals(QObject):
    localeChanged = pyqtSignal()

class TSHLocaleHelper(QObject):
    exportLocale = "en-US"
    programLocale = "en-US"
    roundLocale = "en-US"
    translator = None
    languages = []
    remapping = {}

    def LoadLocale():

        settingsProgramLocale = SettingsManager.Get("program_language", None)
        settingsExportLocale = SettingsManager.Get("export_language", None)
        settingsRoundLocale = SettingsManager.Get("round_language", None)

        if settingsProgramLocale and settingsProgramLocale != "default":
            current_locale = [settingsProgramLocale]
        else:
            current_locale = QtCore.QLocale().uiLanguages()

        print("OS locale", current_locale)

        oldTranslator = TSHLocaleHelper.translator

        if oldTranslator:
            QGuiApplication.instance().removeTranslator(oldTranslator)

        TSHLocaleHelper.translator = QTranslator()
        for locale in current_locale:
            for f in os.listdir("./src/i18n/"):
                if f.endswith(".qm"):
                    lang = f.split("_", 1)[1].split(".")[0]
                    if lang == locale:
                        TSHLocaleHelper.translator.load(
                            QLocale(lang), "./src/i18n/"+f)
                        TSHLocaleHelper.programLocale = locale
                        break
                    elif lang == locale.split("-")[0]:
                        TSHLocaleHelper.translator.load(
                            QLocale(lang), "./src/i18n/"+f)
                        TSHLocaleHelper.programLocale = locale
                        break

        QGuiApplication.instance().installTranslator(TSHLocaleHelper.translator)

        if settingsExportLocale and settingsExportLocale != "default":
            TSHLocaleHelper.exportLocale = settingsExportLocale
        else:
            TSHLocaleHelper.exportLocale = current_locale[0]

        if settingsRoundLocale and settingsRoundLocale != "default":
            TSHLocaleHelper.roundLocale = settingsRoundLocale
        else:
            TSHLocaleHelper.roundLocale = current_locale[0]

    def LoadLanguages():
        try:
            languages_json = json.load(open("./src/i18n/mapping.json", 'rt', encoding='utf-8'))
            TSHLocaleHelper.languages = languages_json.get("languages")
            TSHLocaleHelper.remapping = languages_json.get("remapping")
        except Exception as e:
            raise Exception(f"Error loading languages") from e

    def GetRemaps(language: str):
        for remap, langs in TSHLocaleHelper.remapping.items():
            if language.replace('-', '_') in langs:
                print("Loaded remap: ", remap)
                return remap
        return None

TSHLocaleHelper.LoadLanguages()


import itertools
import numpy as np

class Weight:
    def __init__(self,
            cost, # コスト項
            p, # 制約項の重み
            ineq, # B(t) <= B_max
        ):
        self._cost = cost
        self._p = p
        self._ineq = ineq

    @property
    def cost(self):
        return self._cost

    @property
    def p(self):
        return self._p

    @property
    def ineq(self):
        return self._ineq

class WeightStore:
    def __init__(self):
        self._store = {}

    def register(self,
            machine,
            cost=1.0, # コスト項
            p=1.0, # 制約項の重み
            ineq=1.0, # B(t) <= B_max
        ):

        def make_range(input):
            if isinstance(input, list) or isinstance(input, tuple):
                return np.arange(input[0], input[1], input[2])
            return [input]

        cost_range = make_range(cost)
        p_range = make_range(p)
        ineq_range = make_range(ineq)

        weights = []
        for cost, p, ineq in itertools.product(cost_range, p_range, ineq_range):
            weights.append(Weight(cost, p, ineq))

        self._store[machine] = weights

    def weights(self, machine):
        return self._store[machine]


import pandas as pd
import cv2
import numpy as np
from omegaconf import OmegaConf
from tqdm import tqdm

from vision.misc.help_func import get_repo_dir
from vision.tools.video_wrapper import video_wrapper
from vision.tools.sensors_alignment import SensorAligner
from vision.tools.sensors_alignment import align_sensors_cuda

def downsample_by_log(log_fp, zed_fp, jai_fp, jai_rgb_fp, output_movie_fp, alignment_output_fp, cfg):
    sa = SensorAligner(cfg.sensor_aligner)
    log_df = pd.read_csv(log_fp)
    jai_frame_ids = list(log_df['JAI_frame_number'])
    zed_frame_ids = list(log_df['ZED_frame_number'])

    cam_zed = video_wrapper(zed_fp, rotate=2, channels=1)
    cam_jai = video_wrapper(jai_fp, rotate=1)
    cam_jai_rgb = video_wrapper(jai_rgb_fp, rotate=1)

    alignment_data = []
    f_id = 0

    (x1, y1, x2, y2) = 0, 0, 0, 0
    output_video = cv2.VideoWriter(output_movie_fp, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
                                   10, (1000, 700), isColor=True)
    jai_first_id, zed_first_id = 0, 0
    p_zed_id = -1e5
    last_jai = 0
    last_zed = 0
    for jai_id, zed_id in tqdm(zip(jai_frame_ids, zed_frame_ids)):
        if p_zed_id > zed_id:
            jai_first_id = jai_id
            p_zed_id = zed_id
            continue
        f_id = jai_id - jai_first_id
        while f_id > last_jai  + 1:
            _, jai_frame = cam_jai.get_frame()
            _, jai_rgb_frame = cam_jai_rgb.get_frame()
            last_jai += 1

        _, jai_frame = cam_jai.get_frame()
        _, jai_rgb_frame = cam_jai_rgb.get_frame()

        while zed_id > last_zed + 1:
            _, zed_frame = cam_zed.get_frame()
            last_zed += 1
        _, zed_frame = cam_zed.get_frame()

        last_jai = f_id
        last_zed = zed_id

        (px1, py1, px2, py2) = (x1, y1, x2, y2)
        (x1, y1, x2, y2), tx, ty = align_sensors_cuda(zed_frame,
                                                      jai_rgb_frame,
                                                      sa.sx,
                                                      sa.sy,
                                                      sa.origins[0],
                                                      [sa.roix, sa.roiy],
                                                      sa.ransac)
        alignment_data.append({'z_shift': sa.zed_shift, 'tx': tx, 'f_id': f_id})

        # winname = f"zed {zed_id} | jai {jai_id}"
        # if np.isnan(x1):
        #     winname = "!!!!!!!!!!!!!!!!!!!!!!"
        #     (x1, y1, x2, y2) = px1, py1, px2, py2
        #
        # canvas = np.zeros((zed_frame.shape[0] + jai_frame.shape[0] + 20, zed_frame.shape[1] + jai_frame.shape[1] + 30, 3), dtype=np.uint8)
        #
        # x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
        # print(x1, y1, x2, y2)
        # zed_frame = cv2.rectangle(zed_frame, (x1, y1), (x2, y2), (0, 0, 255), 5)
        #
        # canvas[10:10+zed_frame.shape[0], 10:10+zed_frame.shape[1], :] = zed_frame
        # canvas[10:10+jai_frame.shape[0], 20+zed_frame.shape[1]:20+zed_frame.shape[1]+jai_frame.shape[1], :] = jai_frame
        #
        # cv2.destroyAllWindows()
        # cv2.namedWindow(winname, cv2.WND_PROP_FULLSCREEN)
        # cv2.setWindowProperty(winname, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_NORMAL)
        # cv2.resizeWindow(winname, 1000, 1000)
        # cv2.imshow(winname, canvas)
        # cv2.waitKey()

        cropped = zed_frame[int(y1):int(y2), int(x1):int(x2), :3]
        cropped = cv2.resize(cropped, (480, 640))

        jai_frame = cv2.resize(jai_frame, (480, 640))

        canvas = np.zeros((700, 1000, 3), dtype=np.uint8)
        canvas[10:10 + cropped.shape[0], 10:10+cropped.shape[1], :] = cropped
        canvas[10:10 + jai_frame.shape[0], 510:510 + jai_frame.shape[1], :] = jai_frame

        f_id += 1
        output_video.write(canvas)

        if f_id > 300:
            break

    output_video.release()
    df = pd.DataFrame(alignment_data, columns=['z_shift', 'tx', 'matches', 'f_id'])
    df.to_csv(alignment_output_fp)

if __name__ == "__main__":
    repo_dir = get_repo_dir()
    runtime_config = "/vision/pipelines/config/pipeline_config.yaml"
    args = OmegaConf.load(repo_dir + runtime_config)

    zed_fp = "/home/mic-730ai/Desktop/MOTCHA/VALENCIA/160523/row_1/ZED_1.mkv"
    jai_fp = "/home/mic-730ai/Desktop/MOTCHA/VALENCIA/160523/row_1/FSI_EQUALIZE_HIST_1.mkv"
    jai_rgb_fp = "/home/mic-730ai/Desktop/MOTCHA/VALENCIA/160523/row_1/RGB_1.mkv"
    log_fp = "/home/mic-730ai/Desktop/MOTCHA/VALENCIA/160523/row_1//jaized_timestamps_1.log"
    alignment_output_fp = "/home/mic-730ai/Desktop/MOTCHA/VALENCIA/160523/row_1/sa/alignment.csv"
    output_movie_fp = "/home/mic-730ai/Desktop/MOTCHA/VALENCIA/160523/row_1/sa/synced_clip.mkv"

    downsample_by_log(log_fp, zed_fp, jai_fp, jai_rgb_fp, output_movie_fp, alignment_output_fp, args)


import pickle

import numpy
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt

_output_dir = 'figures/'
_output_ext = '.png'
#_output_ext = '.pdf'

class Hist1():
    def __init__(self, plot_name, plot_title, overlay_list, bin_count, bin_range, **kwargs):
        arg_vals = {
            'normalize': True, 'legend_args':{}, 'xlabel':'', 'ylabel':'',
            'xlim':None, 'ylim':None, 'ylog':False, 'labelmaker':None,
            'cumulative': 0, 'zooms': []
        }
        self.plot_name = plot_name
        self.plot_title = plot_title
        self.data = { label:([],[]) for label in overlay_list }
        self.bins = bin_count
        self.range = bin_range

        arg_vals.update(kwargs)
        for kw,arg in arg_vals.items(): setattr(self, kw, arg)

    def fill(self, value, key=None, weight=1):
        if key == None: key = list(self.data)[0]
        self.data[key][0].append(value)
        self.data[key][1].append(weight)

    def generate_plot(self, cache, refresh=None):
        print('Plotting '+self.plot_name)
        if refresh == None and cache != None: self.data = cache
        elif refresh: cache[self.plot_name] = self.data
        else: self.data = cache[self.plot_name]

        plot_values = {'x':[],'weights':[],'label':[]}
        for label, (values, weights) in self.data.items():
            counts, bins = numpy.histogram(values, weights=weights, bins=self.bins, range=self.range)
            if counts.sum() == 0:
                print('WARNING: '+self.plot_name + ' has no data for label ' + str(label))
                if len(values) == 0:
                    print('Label list is empty')
                else:
                    print( 'Label list contains {} values between {} and {}'.format(len(values), min(values), max(values)) )
                return
            binned_vals = counts / counts.sum() if self.normalize else counts
            if self.cumulative != 0:
                if isinstance(self.cumulative,dict):
                    sumdir = self.cumulative[label]
                else:
                    sumdir = self.cumulative
                binned_vals = binned_vals[::sumdir].cumsum()[::sumdir]

            plot_values['x'].append( bins[:-1] )
            plot_values['weights'].append(binned_vals)
            label_text = label if self.labelmaker == None else self.labelmaker(label)
            plot_values['label'].append(label_text)
        plot_values['range'] = self.range
        plot_values['bins'] = self.bins

        fig,ax = plt.subplots()
        counts, bins, hist = plt.hist( **plot_values, linewidth=2, histtype='step')

        if len(plot_values['label']) > 1: ax.legend(**self.legend_args)
        plt.grid()
        if self.ylog: plt.yscale('log')
        if self.xlim != None: plt.xlim(*self.xlim)
        if self.ylim != None: plt.ylim(*self.ylim)
        plt.xlabel(self.xlabel)
        plt.ylabel(self.ylabel)
        plt.title(self.plot_title)
        dpi=500
        fig.savefig(_output_dir+self.plot_name+_output_ext, dpi=dpi)
        for index, (xlim, ylim) in enumerate(self.zooms):
            plt.xlim(xlim)
            plt.ylim(ylim)
            plt.savefig(_output_dir+self.plot_name+f'_zoom{index}'+_output_ext, dpi=dpi)
        plt.close()

class Hist2():
    def __init__(self, plot_name, plot_title, bin_count, bin_range, **kwargs):
        arg_vals = { 'normalize': True, 'xlabel':'', 'ylabel':'', 'zlim':None }
        self.plot_name = plot_name
        self.plot_title = plot_title
        self.data = ([],[])
        self.bins = bin_count
        self.range = bin_range

        arg_vals.update(kwargs)
        for kw,arg in arg_vals.items(): setattr(self, kw, arg)

    def fill(self, value0, value1):
        self.data[0].append(value0)
        self.data[1].append(value1)

    def generate_plot(self, cache, refresh=None):
        print('Plotting '+self.plot_name)
        if refresh == None and cache != None: self.data = cache
        elif refresh: cache[self.plot_name] = self.data
        else: self.data = cache[self.plot_name]

        hist_counts, xedges, yedges = numpy.histogram2d(*self.data, bins=self.bins, range=self.range)
        flat_counts = hist_counts.flatten()
        if flat_counts.sum() == 0:
            print('WARNING: '+self.plot_name + ' has no data')
            print(*self.data)
            return
        if self.normalize: flat_counts /= hist_counts.sum()
        bin_edges = numpy.array([ (x,y) for x in xedges[:-1] for y in yedges[:-1] ]).transpose()

        fig,ax = plt.subplots()
        counts, xbins, ybins, hist = plt.hist2d( *bin_edges, weights=flat_counts, bins=self.bins, range=self.range)

        cbar = plt.colorbar()
        if self.zlim != None: plt.clim(self.zlim)

        #plt.grid()
        #plt.yscale('log')
        #plt.ylim(10e-6, 1)
        plt.xlabel(self.xlabel)
        plt.ylabel(self.ylabel)
        plt.title(self.plot_title)
        fig.savefig(_output_dir+self.plot_name+_output_ext)
        plt.close()

class Roc():
    def __init__(self, plot_name, plot_title, overlay_list, **kwargs):
        arg_vals = {
            'legend_args':{}, 'labelmaker':None, 'normalize': True, 'scale_to_y': False,
            'zooms': []
        }
        self.plot_name = plot_name
        self.plot_title = plot_title
        self.data = { label:( ([],[]), ([],[]) ) for label in overlay_list }
        self.marker_requests = { label:[] for label in overlay_list }

        arg_vals.update(kwargs)
        for kw,arg in arg_vals.items(): setattr(self, kw, arg)

    def add_marker(self, label, marker_value, **kwargs):
        if 'annotation' not in kwargs: kwargs['annotation'] = ''
        self.marker_requests[label].append( (marker_value, kwargs) )

    def fill(self, value, bgd, key=None, weight=1):
        if key == None: key = list(self.data)[0]
        self.data[key][bgd][0].append(value)
        self.data[key][bgd][1].append(weight)

    def generate_plot(self, cache, refresh=None):
        print('Plotting '+self.plot_name)
        if refresh == None and cache != None: self.data = cache
        elif refresh: cache[self.plot_name] = self.data
        else: self.data = cache[self.plot_name]

        roc_ax = plt.subplots()
        marker_list = []

        num_bins = 10000
        for label, (signal,background) in self.data.items():
            maximum = max( max(signal[0]), max(background[0]) )
            minimum = min( min(signal[0]), min(background[0]) )
            efficiency,bin_list = numpy.histogram( signal[0], weights=signal[1], bins=num_bins, range=(minimum, maximum) )
            rejection = numpy.histogram( background[0], weights=background[1], bins=num_bins, range=(minimum, maximum) )[0]
            efficiency = ( efficiency/efficiency.sum() )[::-1].cumsum()[::-1]
            rejection = ( rejection/rejection.sum() ).cumsum()
            plt.plot(efficiency, rejection, label=label, linewidth=1)

            bin_width = bin_list[1] - bin_list[0]
            for marker_val, kwargs in self.marker_requests[label]:
                bin_index = int( (marker_val-minimum) / bin_width )
                location = (efficiency[bin_index], rejection[bin_index])
                print(f'Marker: {label}={marker_val} @ {location[0]},{location[1]}')
                marker_list.append( (location, kwargs) )

        plt.legend(prop={'size':6})
        plt.xlabel('Signal Efficiency')
        plt.ylabel('Background Rejection')
        plt.title(self.plot_title)
        plt.grid(True)
        for location, kwargs in marker_list:
            annotation_text = kwargs.pop('annotation')
            plt.annotate(annotation_text, xy=location, fontsize='x-small')
            plt.plot(*location, **kwargs)

        dpi=500
        plt.savefig(_output_dir+self.plot_name+_output_ext, dpi=dpi)
        for index, (xlim, ylim) in enumerate(self.zooms):
            plt.xlim(xlim)
            plt.ylim(ylim)
            plt.savefig(_output_dir+self.plot_name+f'_zoom{index}'+_output_ext, dpi=dpi)
        plt.close()

class plot_wrapper():
    def __init__(self, blacklist):
        self.plot_dict = {}
        self.blacklist = blacklist

    def add_hist1(self, plot_name, *args, **kwargs):
        self.plot_dict[plot_name] = Hist1(plot_name, *args, **kwargs)

    def add_hist2(self, plot_name, *args, **kwargs):
        self.plot_dict[plot_name] = Hist2(plot_name, *args, **kwargs)

    def add_roc(self, plot_name, *args, **kwargs):
        self.plot_dict[plot_name] = Roc(plot_name, *args, **kwargs)

    def plot_all(self, refresh, cache):
        for key,plot in self.plot_dict.items():
            blacklisted = False
            for substring in self.blacklist:
                if substring in key:
                    blacklisted = True
                    break
            if not blacklisted: plot.generate_plot(cache, refresh=refresh)

    def __getitem__(self, plot_name):
        return self.plot_dict[plot_name]


import ply.lex as lex
from ply.lex import TOKEN

class CoolLexer:
	"""
	CoolLexer Class.

	Deals with lexical analysis of Cool programs

	To use the lexer, create an object from the class and feed the source code to the object by calling
	'input(str)', where 'str' is the program's code as a string.

	New tokens are fetched from the lexer using the 'token()' function (1 by 1).

	To access the last token that was fetched without moving on to a new token, use 'last_token' property
	"""

	def __init__(self):
		# Properties used by ply.lex
		self.tokens = self.token_names + tuple(self.reserved_keywords.values())  # ply tokens collection
		self.reserved = self.reserved_keywords.keys()  # ply reserved keywords map

		# Build Ply.Lex
		self.lexer = lex.lex(module=self)

		# Save program's code to check for error's columns
		self.program = ""

	def test(self,input):
		"""
		Prints all tokens from the input source code to stdout.
		"""
		self.input(input)
		a = self.lexer.token()
		while a:
			print(a)
			a = self.lexer.token()

	def input(self, input):
		"""
		Feeds input source code to the lexical analyzer.
		"""
		self.lexer.input(input)

	def token(self):
		"""
		Returns the next token in the program's code. This token can be accessed
		again with the last_token property.
		"""
		self.last_token = self.lexer.token()
		return self.last_token

   #################################### ITERATOR PROTOCOL ############################################

	def __iter__(self):
		return self

	def __next__(self):
		t = self.token()
		if t is None:
			raise StopIteration
		return t

	def next(self):
		return self.__next__()

	# ####################################	TOKENS DEFINITIONS	######################################

	#	 Collection of COOL Syntax Tokens.
	token_names = (
		# Identifiers
		"ID", "TYPE",

		# Primitive Types
		"INTEGER", "STRING", "BOOLEAN",

		# Literals
		"LPAREN", "RPAREN", "LBRACE", "RBRACE", "COLON", "COMMA", "DOT", "SEMICOLON", "AT",

		# Operators
		"PLUS", "MINUS", "MULTIPLY", "DIVIDE", "EQ", "LT", "LTEQ", "ASSIGN", "INT_COMP",

		# Special Operators
		"ARROW"
	)

	#	 Map of COOL reserved keywords.
	reserved_keywords = {
		"case": "CASE",
		"class": "CLASS",
		"else": "ELSE",
		"esac": "ESAC",
		"fi": "FI",
		"if": "IF",
		"in": "IN",
		"inherits": "INHERITS",
		"isvoid": "ISVOID",
		"let": "LET",
		"loop": "LOOP",
		"new": "NEW",
		"of": "OF",
		"pool": "POOL",
		"self": "SELF",
		"then": "THEN",
		"while": "WHILE",
		"not": "NOT"
	}

	# #################################### LEXICAL RULES	######################################

	# Ignore rule for single line comments
	t_ignore_SINGLE_LINE_COMMENT = r"\-\-[^\n]*"

	# Simple tokens
	t_LPAREN = r'\('        # (
	t_RPAREN = r'\)'        # )
	t_LBRACE = r'\{'        # {
	t_RBRACE = r'\}'        # }
	t_COLON = r'\:'         # :
	t_COMMA = r'\,'         # ,
	t_DOT = r'\.'           # .
	t_SEMICOLON = r'\;'     # ;
	t_AT = r'\@'            # @
	t_MULTIPLY = r'\*'      # *
	t_DIVIDE = r'\/'        # /
	t_PLUS = r'\+'          # +
	t_MINUS = r'\-'         # -
	t_INT_COMP = r'~'       # ~
	t_LT = r'\<'            # <
	t_EQ = r'\='            # =
	t_LTEQ = r'\<\='        # <=
	t_ASSIGN = r'\<\-'      # <-
	t_ARROW = r'\=\>'       # =>

	@TOKEN(r"(true|false)")
	def t_BOOLEAN(self, token):
		token.value = True if token.value == "true" else False
		return token

	@TOKEN(r"\d+")
	def t_INTEGER(self, token):
		token.value = int(token.value)
		return token

	@TOKEN(r"[A-Z][a-zA-Z_0-9]*")
	def t_TYPE(self, token):
		token.type = self.reserved_keywords.get(token.value, 'TYPE')
		return token

	@TOKEN(r"[a-z_][a-zA-Z_0-9]*")
	def t_ID(self, token):
		token.type = self.reserved_keywords.get(token.value, 'ID')
		return token

	@TOKEN(r"\n+")
	def t_newline(self, token):
		token.lexer.lineno += len(token.value)

	# Ignore Whitespace Character Rule
	t_ignore = ' \t\r\f'

	########################## STATEFUL LEXICAL RULES ###############################

	# LEXER STATES
	@property
	def states(self):
		return (
			("STRING", "exclusive"),
			("COMMENT", "exclusive")
		)

	######################### THE STRING STATE #####################

	@TOKEN(r"\"")
	def t_start_string(self, token):
		token.lexer.push_state("STRING")
		token.lexer.string_backslashed = False
		token.lexer.stringbuf = ""

	@TOKEN(r"\n")
	def t_STRING_newline(self, token):
		token.lexer.lineno += 1
		if not token.lexer.string_backslashed:
			print("String newline not escaped")
			token.lexer.skip(1)
		else:
			token.lexer.string_backslashed = False

	@TOKEN(r"\"")
	def t_STRING_end(self, token):
		if not token.lexer.string_backslashed:
			token.lexer.pop_state()
			token.value = token.lexer.stringbuf
			token.type = "STRING"
			return token
		else:
			token.lexer.stringbuf += '"'
			token.lexer.string_backslashed = False

	@TOKEN(r"[^\n]")
	def t_STRING_anything(self, token):
		if token.lexer.string_backslashed:
			if token.value == 'b':
					token.lexer.stringbuf += '\b'
			elif token.value == 't':
					token.lexer.stringbuf += '\t'
			elif token.value == 'n':
					token.lexer.stringbuf += '\n'
			elif token.value == 'f':
					token.lexer.stringbuf += '\f'
			elif token.value == '\\':
					token.lexer.stringbuf += '\\'
			else:
					token.lexer.stringbuf += token.value
			token.lexer.string_backslashed = False
		else:
			if token.value != '\\':
					token.lexer.stringbuf += token.value
			else:
					token.lexer.string_backslashed = True

	# STRING error handler
	def t_STRING_error(self, token):
		print("Illegal character! Line: {0}, character: {1}".format(token.lineno, token.value[0]))
		token.lexer.skip(1)

	# STRING ignored characters
	t_STRING_ignore = ''

	####################### THE COMMENT STATE ######################
	@TOKEN(r"\(\*")
	def t_start_comment(self, token):
		token.lexer.push_state("COMMENT")
		token.lexer.comment_count = 0

	@TOKEN(r"\(\*")
	def t_COMMENT_startanother(self, t):
		t.lexer.comment_count += 1

	@TOKEN(r"\*\)")
	def t_COMMENT_end(self, token):
		if token.lexer.comment_count == 0:
			token.lexer.pop_state()
		else:
			token.lexer.comment_count -= 1

	# COMMENT error rule
	def t_COMMENT_error(self, token):
		token.lexer.skip(1)

	# COMMENT ignored characters
	t_COMMENT_ignore = ''

	##################### ERROR REPORTING RULE ######################

	def t_error(self, token):
		line_start = self.program.rfind('\n', 0, token.lexpos) + 1
		column = (token.lexpos - line_start) + 1
		print("Illegal character! Line: {0}, Column: {1}".format(token.lineno, column))
		token.lexer.skip(1)

# #----------- TESTS

# s = CoolLexer()
# fpath = "D:\Scripts\cool-compiler-hieu-luis-alejandro\examples\\graph.cl"
# with open(fpath, encoding="utf-8") as file:
# 	cool_program_code = file.read()
# 	s.test(cool_program_code)


import os
import time
import glob
import argparse
import os.path as osp

import numpy as np
from tqdm import tqdm

from typing import Optional, List, NamedTuple

import torch
import torch.nn.functional as F

from pytorch_lightning.callbacks import ModelCheckpoint
from pytorch_lightning import Trainer, seed_everything

from ogb.lsc import MAG240MEvaluator

from utils.data_loader import MAG240M
from model.gnn import GNN
ROOT = './dataset'

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--hidden_channels', type=int, default=1024)
    parser.add_argument('--batch_size', type=int, default=1024)
    parser.add_argument('--dropout', type=float, default=0.5)
    parser.add_argument('--epochs', type=int, default=100)
    parser.add_argument('--model', type=str, default='graphsage',
                        choices=['gat', 'graphsage'])
    parser.add_argument('--sizes', type=str, default='25-15')
    parser.add_argument('--device', type=str, default='0')
    parser.add_argument('--evaluate', action='store_true')
    args = parser.parse_args()
    args.sizes = [int(i) for i in args.sizes.split('-')]
    print(args)

    seed_everything(42)
    datamodule = MAG240M(ROOT, args.batch_size, args.sizes)

    if not args.evaluate:
        model = GNN(args.model, datamodule.num_features,
                    datamodule.num_classes, args.hidden_channels,
                    num_layers=len(args.sizes), dropout=args.dropout)
        print(f'#Params {sum([p.numel() for p in model.parameters()])}')
        checkpoint_callback = ModelCheckpoint(monitor='val_acc', mode = 'max', save_top_k=1)
        trainer = Trainer(gpus=args.device, max_epochs=args.epochs,
                          callbacks=[checkpoint_callback],
                          default_root_dir=f'logs/{args.model}')
        trainer.fit(model, datamodule=datamodule)

    if args.evaluate:
        dirs = glob.glob(f'logs/{args.model}/lightning_logs/*')
        version = max([int(x.split(os.sep)[-1].split('_')[-1]) for x in dirs])
        logdir = f'logs/{args.model}/lightning_logs/version_{version}'
        print(f'Evaluating saved model in {logdir}...')
        ckpt = glob.glob(f'{logdir}/checkpoints/*')[0]

        trainer = Trainer(gpus=args.device, resume_from_checkpoint=ckpt)
        model = GNN.load_from_checkpoint(checkpoint_path=ckpt,
                                         hparams_file=f'{logdir}/hparams.yaml')

        datamodule.batch_size = 16
        datamodule.sizes = [160] * len(args.sizes)  # (Almost) no sampling...

        trainer.test(model=model, datamodule=datamodule)

        evaluator = MAG240MEvaluator()
        loader = datamodule.hidden_test_dataloader()

        model.eval()
        device = f'cuda:{args.device}' if torch.cuda.is_available() else 'cpu'
        model.to(device)
        y_preds = []
        for batch in tqdm(loader):
            batch = batch.to(device)
            with torch.no_grad():
                out = model(batch.x, batch.adjs_t).argmax(dim=-1).cpu()
                y_preds.append(out)
        res = {'y_pred': torch.cat(y_preds, dim=0)}
        evaluator.save_test_submission(res, f'results/{args.model}', mode = 'test-dev')


# -*- coding: utf8 -*-

"""
How to use

1. debug printを使用したいファイルで以下のようにimportする。

.. code-block:: python

    from debug import ERROR, WARN, INFO, DEBUG, TRACE

2. DEBUG_LEVELに、ログ出力のレベルを指定する。

| 指定したレベル以上のログが出力される。
| 例： ``DEBUG_LEVEL_INFO`` を指定すると、ERROR,WARN,INFOが出力される。

3. 使用したいlevelのdebug printを通常のprint文のように記述する。

| 複数引数にも対応している。

4. debug levelの目安

| ERROR : プレー続行不可能になるようなエラー
| WARN  : プレー続行不可能とはならないような警告
| INFO  : 他スレッドとの結合動作において表示したい情報
| DEBUG : 自スレッドのみの単体デバッグにおいて表示したい情報
| TRACE : 処理の開始終了や分岐の出入りなど、ルートを確認するための情報
"""

import datetime

DEBUG_LEVEL_OFF = -1
DEBUG_LEVEL_ERROR = 0
DEBUG_LEVEL_WARN = 1
DEBUG_LEVEL_INFO = 2
DEBUG_LEVEL_DEBUG = 3
DEBUG_LEVEL_TRACE = 4
DEBUG_LEVEL_ALL = 5

DEBUG_DISABLE = 0
DEBUG_ENABLE = 1

# ログ出力のレベルを指定
DEBUG_LEVEL = DEBUG_LEVEL_INFO

def dummyFunc(level, *args):
    pass

def debugPrint(level, *args):
    now = datetime.datetime.now()
    timestamp = ''.join(['[', now.strftime('%H:%M:%S.%f')[:-3], ']'])
    print(timestamp + level + ' ' + ' '.join(map(str, args)))

debugFuncList = [dummyFunc, dummyFunc, dummyFunc, dummyFunc, dummyFunc]

for i in range(DEBUG_LEVEL_ALL):
    if DEBUG_LEVEL >= i:
        debugFuncList[i] = debugPrint

def ERROR(*args):
    debugFuncList[DEBUG_LEVEL_ERROR]('[ERROR]', *args)

def WARN(*args):
    debugFuncList[DEBUG_LEVEL_WARN]('[WARN ]', *args)

def INFO(*args):
    debugFuncList[DEBUG_LEVEL_INFO]('[INFO ]', *args)

def DEBUG(*args):
    debugFuncList[DEBUG_LEVEL_DEBUG]('[DEBUG]', *args)

def TRACE(*args):
    debugFuncList[DEBUG_LEVEL_TRACE]('[TRACE]', *args)

if __name__ == '__main__':
    ERROR('error test')
    WARN('warn test')
    INFO('info test')
    DEBUG('debug test')
    TRACE('trace test')


class Autocomplete:
    def __init__(self):
        self.trie = {}

    def get_words(self, start, temp, words):
        if start['isWord']:
            words.append((temp, start['isWord']))

        for key, val in start.items():
            if key != 'isWord':
                self.get_words(val, temp + key, words)

    def get(self, prefix):
        current = self.trie
        for char in prefix:
            if char not in current:
                return []
            current = current[char]

        words = []
        self.get_words(current, prefix, words)
        sorted_words = sorted(words, key=lambda x: x[1], reverse=True)
        return sorted_words

    def put(self, word):
        current = self.trie
        for char in word:
            if char not in current:
                current[char] = {'isWord': 0}
            current = current[char]
        current['isWord'] += 1

autocomplete = Autocomplete()
autocomplete.put('boat')
autocomplete.put('boat')
autocomplete.put('boat')

autocomplete.put('goat')
autocomplete.put('book')
autocomplete.put('booking')
print(autocomplete.get('bo'))


game_board = list(range(1, 10))
wins_comb = (
    (1, 2, 3), (4, 5, 6), (7, 8, 9),
    (1, 4, 7), (2, 5, 8), (3, 6, 9),
    (1, 5, 9), (3, 5, 7)
)

def draw_board():
    print('-------------')
    for i in range(3):
        print('|', game_board[0 + i * 3], '|', game_board[1 + i * 3], '|', game_board[2 + i * 3], '|')
    print('-------------')

def player_step(token):
    while True:
        value = input('Куда поставить: ' + token + '? ')
        if not (value in '123456789'):
            print('Ошибка. Повторите ввод')
            continue
        value = int(value)
        if not (0 < value < 10):
            print('Неверное значени. Повторите!')
            continue
        if str(game_board[value - 1]) in 'XO':
            print('Клетка занята.')
            continue
        game_board[value - 1] = token
        break

def check_win():
    for i in wins_comb:
        if game_board[i[0] - 1] == game_board[i[1] - 1] == game_board[i[2] - 1]:
            return game_board[i[1] - 1]
    else:
        return False

def main():
    counter = 0
    while True:
        draw_board()
        if counter % 2 == 0:
            player_step('X')
        else:
            player_step('O')
        if counter > 3:
            winner = check_win()
            if winner:
                draw_board()
                print(winner, 'выиграл!')
                break
        counter += 1
        if counter > 8:
            draw_board()
            print('Ничья!')
            break

main()