from collections import namedtuple
from datetime import datetime
from typing import Optional
from typing import Union

from .connection import join_url
from .connection import root
from .exceptions import _raise_exception
from .parse import BeautifulSoup
from .parse import check_page_raise
from .parse import html_to_bbcode
from .parse import parse_comments
from .parse import parse_journal_page
from .parse import parse_journal_section
from .parse import Tag
from .user import UserPartial

class JournalStats(namedtuple("JournalStats", ["comments"])):
    """
    This object contains the journal's statistics:
    * comments
    """

class JournalBase:
    def __init__(self):
        self.id: int = 0
        self.title: str = ""
        self.date: datetime = datetime.fromtimestamp(0)
        self.author: UserPartial = UserPartial()
        self.stats: JournalStats = JournalStats(0)
        self.content: str = ""
        self.mentions: list[str] = []

    def __hash__(self) -> int:
        return hash(self.id)

    def __eq__(self, other) -> bool:
        if isinstance(other, JournalBase):
            return other.id == self.id
        elif isinstance(other, int):
            return other == self.id
        return False

    def __gt__(self, other) -> bool:
        if isinstance(other, JournalBase):
            return self.id > other.id
        elif isinstance(other, int):
            return self.id > other
        return False

    def __ge__(self, other) -> bool:
        if isinstance(other, JournalBase):
            return self.id >= other.id
        elif isinstance(other, int):
            return self.id >= other
        return False

    def __lt__(self, other) -> bool:
        if isinstance(other, JournalBase):
            return self.id < other.id
        elif isinstance(other, int):
            return self.id < other
        return False

    def __le__(self, other) -> bool:
        if isinstance(other, JournalBase):
            return self.id <= other.id
        elif isinstance(other, int):
            return self.id <= other
        return False

    def __iter__(self):
        yield "id", self.id
        yield "title", self.title
        yield "date", self.date
        yield "author", dict(self.author)
        yield "stats", self.stats._asdict()
        yield "content", self.content
        yield "mentions", self.mentions

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return f"{self.id} {self.author} {self.title}"

    @property
    def content_bbcode(self) -> str:
        """
        The journal content formatted to BBCode

        :return: BBCode content
        """
        return html_to_bbcode(self.content)

    @property
    def url(self) -> str:
        """
        Compose the full URL to the journal.

        :return: The URL to the journal.
        """
        return join_url(root, "journal", self.id)

class JournalPartial(JournalBase):
    """
    Contains partial journal information gathered from journals pages.
    """

    def __init__(self, journal_tag: Optional[Tag] = None):
        """
        :param journal_tag: The tag from which to parse the journal.
        """
        assert journal_tag is None or isinstance(journal_tag, Tag), \
            _raise_exception(TypeError(f"journal_item must be {None} or {Tag.__name__}"))
        self.journal_tag: Optional[Tag] = journal_tag

        super(JournalPartial, self).__init__()

        self.parse()

    def parse(self, journal_tag: Optional[Union[Tag, BeautifulSoup]] = None):
        """
        Parse a journal tag, overrides any information already present in the object.

        :param journal_tag: The tag from which to parse the journal.
        """
        assert journal_tag is None or isinstance(journal_tag, BeautifulSoup), \
            _raise_exception(TypeError(f"journal_item must be {None} or {BeautifulSoup.__name__}"))

        self.journal_tag = journal_tag or self.journal_tag
        if self.journal_tag is None:
            return

        parsed: dict = parse_journal_section(self.journal_tag)

        # noinspection DuplicatedCode
        self.id = parsed["id"]
        self.title = parsed["title"]
        self.author.name = parsed.get("user_name", "")
        self.author.status = parsed.get("user_status", "")
        self.author.title = parsed.get("user_title", "")
        self.author.join_date = parsed.get("user_join_date", "")
        self.author.avatar_url = parsed.get("avatar_url", "")
        self.stats = JournalStats(parsed["comments"])
        self.date = parsed["date"]
        self.content = parsed["content"]
        self.mentions = parsed["mentions"]

class Journal(JournalBase):
    """
    Contains complete journal information gathered from journal pages, including comments.
    """

    def __init__(self, journal_page: Optional[BeautifulSoup] = None):
        """
        :param journal_page: The page from which to parse the journal.
        """
        assert journal_page is None or isinstance(journal_page, BeautifulSoup), \
            _raise_exception(TypeError(f"journal_item must be {None} or {BeautifulSoup.__name__}"))
        self.journal_page: Optional[BeautifulSoup] = journal_page

        super(Journal, self).__init__()

        self.header: str = ""
        self.footer: str = ""
        from .comment import Comment
        self.comments: list[Comment] = []

        self.parse()

    def __iter__(self):
        for k, v in super(Journal, self).__iter__():
            yield k, v
        yield "header", self.header
        yield "footer", self.footer
        from .comment import _sort_comments_dict
        yield "comments", _sort_comments_dict(self.comments)

    @property
    def header_bbcode(self) -> str:
        """
        The journal header formatted to BBCode

        :return: BBCode header
        """
        return html_to_bbcode(self.header)

    @property
    def footer_bbcode(self) -> str:
        """
        The journal footer formatted to BBCode

        :return: BBCode footer
        """
        return html_to_bbcode(self.footer)

    def parse(self, journal_page: Optional[Union[Tag, BeautifulSoup]] = None):
        """
        Parse a journal page, overrides any information already present in the object.

        :param journal_page: The page from which to parse the journal.
        """
        assert journal_page is None or isinstance(journal_page, BeautifulSoup), \
            _raise_exception(TypeError(f"journal_item must be {None} or {BeautifulSoup.__name__}"))

        self.journal_page = journal_page or self.journal_page
        if self.journal_page is None:
            return

        check_page_raise(self.journal_page)

        parsed: dict = parse_journal_page(self.journal_page)

        # noinspection DuplicatedCode
        self.id = parsed["id"]
        self.title = parsed["title"]
        self.author.name = parsed["user_info"]["name"]
        self.author.status = parsed["user_info"]["status"]
        self.author.title = parsed["user_info"]["title"]
        self.author.join_date = parsed["user_info"]["join_date"]
        self.author.avatar_url = parsed["user_info"]["avatar_url"]
        self.stats = JournalStats(parsed["comments"])
        self.date = parsed["date"]
        self.content = parsed["content"]
        self.header = parsed["header"]
        self.footer = parsed["footer"]
        self.mentions = parsed["mentions"]
        from .comment import sort_comments, Comment
        self.comments = sort_comments([Comment(t, self) for t in parse_comments(self.journal_page)])


# Задайте двумерный массив из целых чисел. Количество строк и столбцов задается с клавиатуры.
# Отсортировать элементы по возрастанию слева направо и сверху вниз.

import random
try:
    print('Задайте размеры двумерного массива')
    x = int(input('Введите количество строк: '))
    y = int(input('Введите вколичество столбцов: '))
    maxNumber = int(input('Введите максимальное число массива: '))
    minNumber = int(input('Введите минимальное число массива: '))
    if (maxNumber > minNumber):
        doubleList = []
        for i in range(x):
            doubleList.append([])
            for j in range(y):
                doubleList[i].append(random.randint(minNumber, maxNumber))
        print(doubleList)
        finalList = sum(doubleList, [])
        finalList.sort()
        print(finalList)
    else:
        print('Ошибка, максимальное число не может быть больше минимального')
except:
    print('Введены некорректные данные')


from microbit import *
from utime import sleep_ms

I2C_ADDRESS = 0x08

REG_ADD_REVISION = 0;
REG_ADD_SERVO_1 = 1;
REG_ADD_SERVO_2 = 2;
REG_ADD_SERVO_3 = 3;
REG_ADD_SERVO_4 = 4;
REG_ADD_M1A = 5;
REG_ADD_M1B = 6;
REG_ADD_M2A = 7;
REG_ADD_M2B = 8;
REG_ADD_LB_UTH = 9;
REG_ADD_LB_LTH = 10;
REG_ADD_OV_TH = 11;
REG_ADD_VIN = 12;
REG_ADD_PWR_STATE = 13;
REG_ADD_LB_STATE = 14;
REG_ADD_OV_STATE = 15;

Servo_S1 = REG_ADD_SERVO_1
Servo_S2 = REG_ADD_SERVO_2
Servo_S3 = REG_ADD_SERVO_3
Servo_S4 = REG_ADD_SERVO_4
Servo_All = 1000

Motor_M1 = 0
Motor_M2 = 1
Motor_All = 1000

Direction_Forward = 0
Direction_Backward = 1

RGB_LED_PIN = pin8

flag = 0

def limit(value,min,max):
    if value < min:
        value = min
    elif value > max:
        value = max
    return value

def i2cRead(register):
    buf = bytearray(1)
    buf[0] = register
    value = 0
    i2c.write(I2C_ADDRESS,buf,True)
    value = i2c.read(I2C_ADDRESS,1)
    #value = int.from_bytes(value, "little")
    return value[0]

def i2cWrite(register,data):
    buffer = bytearray(2)
    buffer[0] = register
    buffer[1] = data
    i2c.write(I2C_ADDRESS,buffer)

def power_monitor():
    global flag
    global oldPowerState

    if flag == 0:
        oldPowerState = is_power_on()
        flag = 1

    if is_power_on():
        if oldPowerState == False:
            brake_motor(Motor_M1)
            brake_motor(Motor_M2)
            disable_servo(Servo_S1)
            disable_servo(Servo_S2)
            disable_servo(Servo_S3)
            disable_servo(Servo_S4)
            reset()
        oldPowerState = True
    else:
        oldPowerState = False
    sleep_ms(200)

def brake_motor(motorChannel):
    if motorChannel == Motor_M1:
        i2cWrite(REG_ADD_M1A, 0)
        i2cWrite(REG_ADD_M1B, 0)

    elif motorChannel == Motor_M2:
        i2cWrite(REG_ADD_M2A, 0)
        i2cWrite(REG_ADD_M2B, 0)

    elif motorChannel == Motor_All:
        i2cWrite(REG_ADD_M1A, 0)
        i2cWrite(REG_ADD_M1B, 0)
        i2cWrite(REG_ADD_M2A, 0)
        i2cWrite(REG_ADD_M2B, 0)

def run_motor(motorChannel,direction,speed):
    speed = limit(speed,0,255)
    if motorChannel == Motor_M1:
        if direction == Direction_Forward:
            i2cWrite(REG_ADD_M1A,speed)
            i2cWrite(REG_ADD_M1B,0)
        else:
            i2cWrite(REG_ADD_M1A,0)
            i2cWrite(REG_ADD_M1B,speed)
    elif motorChannel == Motor_M2:
        if direction == Direction_Forward:
            i2cWrite(REG_ADD_M2A,speed)
            i2cWrite(REG_ADD_M2B,0)
        else:
            i2cWrite(REG_ADD_M2A,0)
            i2cWrite(REG_ADD_M2B,speed)
    elif motorChannel == Motor_All:
        if direction == Direction_Forward:
            i2cWrite(REG_ADD_M1A,speed)
            i2cWrite(REG_ADD_M1B,0)
            i2cWrite(REG_ADD_M2A,speed)
            i2cWrite(REG_ADD_M2B,0)
        else:
            i2cWrite(REG_ADD_M1A,0)
            i2cWrite(REG_ADD_M1B,speed)
            i2cWrite(REG_ADD_M2A,0)
            i2cWrite(REG_ADD_M2B,speed)

def disable_servo(servo):
    if servo == Servo_All:
        i2cWrite(Servo_S1,0)
        i2cWrite(Servo_S2,0)
        i2cWrite(Servo_S3,0)
        i2cWrite(Servo_S4,0)
    else:
        i2cWrite(servo,0)

def sets_servo_position(servo,position):
    position = limit(position,0,180)

    pulseWidth = int(position * 20 / 18 + 50)
    if servo == Servo_All:
        i2cWrite(Servo_S1,pulseWidth)
        i2cWrite(Servo_S2,pulseWidth)
        i2cWrite(Servo_S3,pulseWidth)
        i2cWrite(Servo_S4,pulseWidth)
    else:
        i2cWrite(servo,pulseWidth)

def is_power_on():
    if i2cRead(REG_ADD_PWR_STATE) != 0:
        return True
    else:
        return False

def is_low_batt():
    if i2cRead(REG_ADD_LB_STATE) != 0:
        return True
    else:
        return False

def is_overvoltage():
    if i2cRead(REG_ADD_OV_STATE) != 0:
        return True
    else:
        return False

def read_Vin():
    return i2cRead(REG_ADD_VIN) / 10


from dict_base import 훈민정음
# 사전(dict) 처리 예: 추가
# ㄱ
훈민정음['창제'] = '1443년' # 내용 추가
항목수 = len(훈민정음)
print(f"ㄱ) 사전(dict) [{항목수}]:", 훈민정음)
# ㄴ)
훈민정음.update({'언어':'한글'}) # 사전 방식 내용 추가
항목수 = len(훈민정음)
print(f"ㄴ) 사전(dict) [{항목수}]:", 훈민정음)
# ㄷ)
훈민정음.update(사용자='8천만') # 키=값 방식 내용 추가
항목수 = len(훈민정음)
print(f"ㄷ) 사전(dict) [{항목수}]:", 훈민정음)

'''
처리 결과:
=========================
ㄱ) 사전(dict) [4]: {'제작': '세종대왕', '반포': '1446년', '소장': '간송미술관', '창제': '1443년'}
ㄴ) 사전(dict) [5]: {'제작': '세종대왕', '반포': '1446년', '소장': '간송미술관', '창제': '1443년', '언어': '한글'}
ㄷ) 사전(dict) [6]: {'제작': '세종대왕', '반포': '1446년', '소장': '간송미술관', '창제': '1443년', '언어': '한글', '사용자': '8천만'}

'''


# 객체지향 프로그래밍으로 링크드 리스트 구현하기
class Node:
  def __init__(self, data, next=None):
    self.data = data
    self.next = next

class NodeMgmt:
  def __init__(self, data):
    self.head = Node(data)

  def add(self, data):
    if self.head == '':
      self.head = Node(data)
    else:
      node = self.head
      while node.next:
        node = node.next
      node.next = Node(data)

  def desc(self):
    node = self.head
    while node:
      print(node.data)
      node = node.next

  def delete(self, data):
    if self.head == '':
      print("해당 값을 가진 노드가 없습니다.")
      return

    if self.head.data == data: # head 삭제
      temp = self.head
      self.head = self.head.next
      del temp
    else: # 마지막 or 중간 노드 삭제
      node = self.head
      while node.next:
        if node.next.data == data:
          temp = node.next
          node.next = node.next.next
          del temp
        else:
          node = node.next

  def search_node(self, data):
    node = self.head
    while node:
      if node.data == data:
        return node
      else:
        node = node.next

linkedlist1 = NodeMgmt(0)
linkedlist1.desc()

for data in range(1, 10):
  linkedlist1.add(data)
linkedlist1.desc()

# 링크드 리스트의 복잡한 기능2 - 노드 삭제
# (1) 위의 class NodeMgmt에 def delete 추가

# (2) 테스트 위해 1개 노드 생성
linkedlist1 = NodeMgmt(0)
linkedlist1.desc()

# (3) head가 살아있음을 확인
print(linkedlist1.head)

# (4) head를 지워봄
linkedlist1.delete(0)

# (5) 정상적으로 삭제되었음을 확인
print(linkedlist1.head) # None

# (6) 여러 노드 추가
linkedlist1 = NodeMgmt(0)
linkedlist1.desc()

for data in range(1, 10):
  linkedlist1.add(data)
linkedlist1.desc()

# (7) 노드 중 한 개 삭제
linkedlist1.delete(4)
linkedlist1.desc()

linkedlist1.delete(9)
linkedlist1.desc()

# 연습1 - 위 코드에서 노드 데이터가 2인 노드 삭제해보기
node_mgmt = NodeMgmt(0)
for data in range(1, 10):
  node_mgmt.add(data)
node_mgmt.desc()

node_mgmt.delete(2)
node_mgmt.desc()

# 연습2 - 위 코드에서 노드 데이터가 특정 숫자인 노드를 찾는 함수 만들고, 테스트 해보기
# (테스트: 임의로 1~9까지 데이터를 링크드 리스트에 넣어보고, 데이터 값이 4인 노드의 데이터 값 출력해보기)
# (1) 위의 class NodeMgmt에 def search_node 추가

# (2) 테스트
node_mgmt = NodeMgmt(0)
for data in range(1, 10):
  node_mgmt.add(data)

node = node_mgmt.search_node(4)
print(node.data)


import math
from collections import defaultdict

def fact(n):

    res = 1

    for i in range(2, n+1):
        res = res * i

    return res

def ncr(n, r):

    return (fact(n) / (fact(r)
                * fact(n - r)))

n = int(input())
a = []
for i in range(n):
	x = input()
	a.append(x)

d = defaultdict(lambda : 0)

for i in range(n):
	z = a[i][0]
	d[z]+=1

count_1, count_2 = 0, 0
j = 0

for keys, values in d.items():
	if values<2:
		continue
	s = values//2
	rest = values-s
	if j%2==0:
		if s>rest:
			if s>1:
				count_1+=ncr(s,2)
			if rest>1:
				count_2+=ncr(rest,2)
		else:
			if rest>1:
				count_1+=ncr(rest,2)
			if s>1:
				count_2+=ncr(s,2)
	else:
		if s>rest:
			if s>1:
				count_2+=ncr(s,2)
			if rest>1:
				count_1+=ncr(rest,2)
		else:
			if rest>1:
				count_2+=ncr(rest,2)
			if s>1:
				count_1+=ncr(s,2)
print(int(count_1+count_2))


from scrapy.utils.serialize import ScrapyJSONEncoder
import json
import csv
import os

def return_spider_output(output):
    """
    Encode the output into json
    """
    _encoder = ScrapyJSONEncoder(ensure_ascii=False)
    return _encoder.encode(output)

def save_finance_json(output):
    """
    Save a list of spyder's outputs into jsonlines
    """
    path = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'files/finance.jsonl'))
    with open(path, 'w', encoding='utf-8') as outfile:
        for item in output:
            json.dump(dict(item), outfile, ensure_ascii=False)
            outfile.write('\n')
    return output

def save_b3_csv(output):
    """
    Save the B3 spyder's outputs into csv
    """
    path = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'files/b3.csv'))
    with open(path, 'w', newline='') as file:
        writer = csv.writer(file)
        for item in output:
            writer.writerow((item['name'], item['symbol']))

    return output

def load_b3():
    """
    Read a b3 csv file and return a set with it's items.
    """
    file_path = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'files/b3.csv'))
    if not os.path.exists(file_path):
        raise "B3 file not found"
    b3_set = {"B3", "Bovespa"}
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            b3_set |= set(line.rstrip('\n|\r').split(','))
    return b3_set

def get_finance(number):
    data = []
    json_path = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'files/finance.jsonl'))
    if not os.path.exists(file_path):
        raise "Finance file not found"
    with open(json_path) as f:
        for index, line in enumerate(f):
            if index == number:
                break
            data.append(json.loads(line))

    return data


from .easy import sheet, column, cell_range
from .utils import extract_data, get_cell_type, get_cell_numeric_format

def from_array(array):
    """ Helper function for creating a sheet out of a NumPy Array

    Args:
        array (NumPy Array):

    Returns:
        ``Sheet`` widget

    Example:
        >>> import numpy as np
        >>> from ipysheet import from_array
        >>>
        >>> arr = np.random.randn(6, 26)
        >>>
        >>> sheet = from_array(arr)
        >>> display(sheet)
    """
    if len(array.shape) > 2:
        raise RuntimeError('The NumPy Array should be of 1-D or 2-D')

    rows = array.shape[0]
    columns = 1 if len(array.shape) == 1 else array.shape[1]

    kwargs = {
        'numeric_format': get_cell_numeric_format(array.dtype),
        'type': get_cell_type(array.dtype)
    }

    out_sheet = sheet(rows=rows, columns=columns)
    if columns == 1:
        column(0, array, **kwargs)
    else:
        cell_range(array, **kwargs)
    return out_sheet

def to_array(sheet):
    """ Helper function for creating a NumPy Array out of a sheet

    Args:
        sheet (Sheet widget):

    Returns:
        A NumPy Array

    Example:
        >>> import ipysheet
        >>>
        >>> sheet = ipysheet.sheet(rows=3, columns=4)
        >>> ipysheet.cell(0, 0, 'Hello')
        >>> ipysheet.cell(2, 0, 'World')
        >>>
        >>> arr = to_array(sheet)
        >>> display(arr)
    """
    import numpy as np

    data = extract_data(sheet)

    return np.array(
        [
            [cell['value'] for cell in row]
            for row in data
        ]
    )


from googletrans import Translator
import time
from city import city
import csv
from dhaka import dhaka

level_0_bn = ""
level_1_bn = ""
level_2_bn = ""

def translate_to_bn(text):
    translator = Translator(service_urls=[
      'translate.google.co.kr',
    ])

    translated = translator.translate(text, src='en', dest='bn').text

    # print(translated)
    return translated

def dumpcsv():
    with open('Dhakabn.csv', 'a') as csvfile:
    # creating a csv writer object
        csvwriter = csv.writer(csvfile)

    # writing the fields

        rows= ["level_0","level_0_bn","level_1","level_1_bn","level_2","level_2_bn","level_3","level_3_bn","level_4","level_4_bn","level_5","level_5_bn"]
    # writing the data rows
        csvwriter.writerow(rows)
def dump(data):
    with open('Dhakabn.csv','a') as csvfile:
         csvwriter = csv.writer(csvfile)

         rows = [data['level_0'],data['level_0_bn'],data['level_1'],data['level_1_bn'],data['level_2'],data['level_2_bn'],data['level_3'],data['level_3_bn'],data['level_4'],data['level_4_bn'],data['level_5'],data['level_5_bn']]
         print(rows)
         csvwriter.writerow(rows)
def execute():
    dumpcsv()
    level_0_bn = translate_to_bn(dhaka[0]['level_0'])
    level_1_bn = "ঢাকা"
    level_2_bn = "ঢাকা"
    for i in range(0,10):
        level_3_bn = translate_to_bn(dhaka[i]['level_3'])
        level_4_bn = translate_to_bn(dhaka[i]['level_4'])
        level_5_bn = translate_to_bn(dhaka[i]['level_5'])
        dhaka[i]['level_0_bn'] = level_0_bn
        dhaka[i]['level_1_bn'] = level_1_bn
        dhaka[i]['level_2_bn'] = level_2_bn
        dhaka[i]['level_3_bn'] = level_3_bn
        dhaka[i]['level_4_bn'] = level_4_bn
        dhaka[i]['level_5_bn'] = level_5_bn
        print(dhaka[i])
        data = dhaka[i]
        dump(data)

execute()

#print(translate_to_bn(dhaka[0]['level_1']))


import pathlib
from collections import defaultdict

import matplotlib.pyplot as plt
import numpy as np
import pybedtools
from pybedtools import BedTool
from tqdm import tqdm

import utils
from utils.miscellaneous import is_valid_chromosome

########################################################################################################################
blacklisted = BedTool(utils.paths.BLACKLIST).filter(is_valid_chromosome).sort()
repmasker = BedTool(utils.paths.REPEATMASKER).filter(is_valid_chromosome).sort().subtract(blacklisted)

# calculate coverage for each repeat family
coverage = defaultdict(int)
coverage['blacklisted'] = blacklisted.total_coverage()
for repeat in tqdm(repmasker):
    name, family, cls = utils.repmasker.classify(repeat.name)
    cls = cls.replace("?", "")
    if name in ("L1Md_A", "L1Md_T"):
        coverage[name] += repeat.length
    elif family == "L1":
        coverage['other LINE-1s'] += repeat.length
    elif cls == "LINE":
        coverage['other LINE'] += repeat.length
    else:
        coverage[cls] += repeat.length

# calculate repeat-free fraction of the genome
mm10 = {k: v for k, (_, v) in pybedtools.chromsizes('mm10').items() if is_valid_chromosome(k)}
mm10_totalsize = sum(mm10.values())

coverage['repeat-free'] = mm10_totalsize - sum(coverage.values())
assert sum(coverage.values()) == mm10_totalsize

# normalize coverage and get percents
coverage = {k: v / mm10_totalsize * 100 for k, v in coverage.items()}
coverage['other repeats'] = coverage.pop('Other')

# merge rare repeats
threshold = 2
for k in list(coverage.keys()):
    if coverage[k] < threshold and k not in ("L1Md_A", "L1Md_T", "blacklisted"):
        coverage['other repeats'] += coverage.pop(k)

# rename for simplicity
coverage['simple_repeats'] = coverage.pop('Simple_repeat')

########################################################################################################################
# make the plot
order = ['repeat-free', 'blacklisted', 'other repeats', 'SINE', 'LTR',
         'simple_repeats', 'other LINE-1s', 'L1Md_A', 'L1Md_T']
fontsize = 'large'

repeats = [k.replace("_", " ") for k in order]
shares = [coverage[k] for k in order]

colorscheme = {
    "blacklisted": "#7f7f7f",
    "repeat-free": "#2CA674",
    "other LINE-1s": "#F49E2D",
    "LTR": "#D079B1",
    "SINE": "#882255",
    "simple repeats": "#4B0092",
    "other repeats": "#636EFA",

    "L1Md A": "#E75C21",
    "L1Md T": "#F6E358"
}

fig = plt.figure(figsize=(16, 8))
ax = fig.gca()

colors = [colorscheme[k] for k in repeats]
wedges, _ = ax.pie(shares, startangle=90, wedgeprops=dict(width=0.5), colors=colors)

kw = dict(arrowprops=dict(arrowstyle="-", relpos=(0, 0.5)), va="center")

for i, p in enumerate(wedges):
    p.set_linewidth(0.3)
    p.set_edgecolor('white')
    if repeats[i] in ("L1Md A", "L1Md T"):
        continue

    ang = (p.theta2 - p.theta1) / 2. + p.theta1
    y = np.sin(np.deg2rad(ang))
    x = np.cos(np.deg2rad(ang))

    xytext = (x * 1.2, y * 1.2)
    xy = (x * 0.92, y * 0.92)

    horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))]
    ax.annotate(repeats[i], xy=xy, xytext=xytext,
                horizontalalignment=horizontalalignment, **kw, fontsize=fontsize)

kw['va'] = 'bottom'

# L1Md A
i = repeats.index("L1Md A")

ang = (wedges[i].theta2 - wedges[i].theta1) / 2. + wedges[i].theta1
y = np.sin(np.deg2rad(ang))
x = np.cos(np.deg2rad(ang))

xytext = (x * 1.08, y * 1.08)
xy = (x * 0.92, y * 0.92)
ax.annotate(repeats[i], xy=xy, xytext=xytext,
            horizontalalignment='left', **kw, fontsize=fontsize)

# L1Md T
i = repeats.index("L1Md T")

ang = (wedges[i].theta2 - wedges[i].theta1) / 2. + wedges[i].theta1
y = np.sin(np.deg2rad(ang))
x = np.cos(np.deg2rad(ang))

xytext = (x * 1.25, y * 1.25)
xy = (x * 0.92, y * 0.92)

ax.annotate(repeats[i], xy=xy, xytext=xytext, horizontalalignment='left', **kw, fontsize=fontsize)

ax.set_title("Mouse genome coverage by repeats", fontsize=fontsize).set_position([.5, 1.05])

saveto = pathlib.Path(__file__).name.replace(".py", ".svg")
fig.savefig(utils.paths.RESULTS.joinpath(saveto), bbox_inches="tight", pad_inches=0)


import requests
import json
import time

i = 0
offset = 1
limit = 1000
headers = {'Token': 'iPUFLyvvyzixkBcMRarLLeoLASMlWkEE'
           }
payload = {'limit': limit,
           'offset': offset}
data_set_name = 'locations'
r = requests.get(f'https://www.ncdc.noaa.gov/cdo-web/api/v2/{data_set_name}', headers=headers, params=payload)
r_dict = r.json()
total_results = r_dict['metadata']['resultset']['count']
r_str = json.dumps(r_dict, indent=4)
while offset <= total_results:
    time.sleep(.75)
    with open(f'{data_set_name}_{i}.json', 'w') as outfile:
        outfile.write(r_str)
    i += 1
    offset = offset + limit
    payload = {'limit': limit,
               'offset': offset}
    r = requests.get(f'https://www.ncdc.noaa.gov/cdo-web/api/v2/{data_set_name}', headers=headers, params=payload)
    r_dict = r.json()
    r_str = json.dumps(r_dict, indent=4)


from setuptools import setup, find_packages

# install_requires = open_requirements('requirements.txt')

# d = {}
# exec(open("spikefinder/version.py").read(), None, d)
# version = d['version']

long_description = open("README.md").read()

setup(name='SpikeFinder',
      version='0.1',
      description='Python toolkit for analysis, visualization and extraction of neural features from continuous neural recordings.',
      long_description=long_description,
      long_description_content_type="text/markdown",
      url="https://github.com/pabloslash/SpikeFinder",
      packages=find_packages(),
      author='Pablo M. Tostado',
      author_email='tostadomarcos@gmail.com',
      license='MIT',
#       install_requires = install_requires,
#       packages=['SpikeFinder'],
      install_requires=['numpy',
                        'matplotlib',
                        'pandas>=0.23',
                        'librosa'
                       ],

      classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ),
      zip_safe=False)


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="python_magic_file",
    version="0.0.7",
    author="Valtteri Remes",
    description="Small Python module to aid developers in getting file extensions from files securely.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
    ],
    python_requires='>=3.6',
    py_modules=["python_magic_file"],
    install_requires=[
        "python-magic == 0.4.27; platform_system != 'Windows'",
        "python-magic-bin == 0.4.14; platform_system == 'Windows'"
    ],
    url="https://github.com/vremes/python-magic-file",
)


'''
Author: Sungjun Han
Description:
here we can build either

1. regular vocabulary
2. byte pair encoding vocabulary

'''
import logging
import os
import numpy as np
import argparse
import pickle
import time
import json

from nli.preprocess import token_to_idx, idx_to_token, frequency_count
from nli.utils import open_tsv_file

parser = argparse.ArgumentParser()
logger = logging.getLogger(__name__)

#directory for data/train/val
parser.add_argument('--data_path', default='data/alphanli/tsv/train.tsv', type=str, help='data to build the vocabulary on')
parser.add_argument('--out_dir', default='data', type=str, help='output directory for vocab json')
parser.add_argument('--vocab_type', default='reg', type=str, help='build regular or bpe')
parser.add_argument('--lower', default=True, type=bool)
parser.add_argument('--min_occurence', default=1, type=str, help='minimum occurence of a token to include in a vocabulary')
parser.add_argument('--vocabulary_size', default = None, type=int)

parser.add_argument('--start_symbol', default='<sos>', type=str, help='start of sentence token')
parser.add_argument('--end_symbol', default='<eos>', type=str, help='end of sentence token')
parser.add_argument('--split_symbol', default='</s>', type=str, help='split token')
parser.add_argument('--null_symbol', default='<unk>', type=str, help='null token')
parser.add_argument('--pad_symbol', default='<pad>', type=str, help='pad token')

def main(args): #so that the function can be called not from command line only

	if args.vocab_type == 'reg': # this option is enough for BoW model
		data = open_tsv_file(args.data_path)
		freq_count = frequency_count(data, args.lower)

		#filter by vocabulary size
		logger.info('Saving the vocabulary to {}'.format(args.out_dir))
		logger.info('Unfiltered vocabulary length: {}'.format(len(freq_count)))

		#min occurence
		if args.min_occurence is not None:
			for token, count in freq_count.items():
				if count < args.min_occurence:
					del freq_count[key]

		if args.vocabulary_size is not None:
			freq_count = sorted(((k,v) in k,v in freq_count.items()), reverse=True, key = lambda x: x[1])[:args.vocabulary_size]
			freq_count = {k:v for k,v in freq_count}

		token2idx = token_to_idx(freq_count,
								pad_symbol = args.pad_symbol,
								start_symbol = args.start_symbol,
								end_symbol = args.end_symbol,
								unk_symbol = args.null_symbol,
								split_symbol = args.split_symbol)

		idx2token = idx_to_token(token2idx)

		logger.info('filtered vocabulary length + special symbols: {}'.format(len(token2idx)))

	elif args.vocab_type == 'bpe':
		'''
		build vocabulary here using byte pair encoding
		'''
		NotImplementedError

	#finally save
	vocab = {
	'token2idx': token2idx,
	'idx2token': idx2token,
	'pad_token': args.pad_symbol,
	'unk_token': args.null_symbol,
	'end_token': args.end_symbol,
	'start_token':args.start_symbol,
	'split_token':args.split_symbol,
	'lower': args.lower
	}

	with open(os.path.join(args.out_dir, 'vocab.json'), 'w') as f:
		json.dump(vocab, f, indent=4)

	return vocab

if __name__ == '__main__':
	args = parser.parse_args()
	#If you set the log level to INFO, it will include INFO, WARNING, ERROR, and CRITICAL messages
	logging.basicConfig(
		level=logging.INFO,
		format='%(name)s: %(asctime)s: %(message)s'
		)
	main(args)


"""
  File with the utility functions
  Y Fan, L Lin, L Ying, L Zepeda-Nunez, A multiscale neural network based on hierarchical matrices,
  arXiv preprint arXiv:1807.01883

"""
import numpy as np
from keras import backend as K
import argparse

# functions
def padding(x, size):
    return K.concatenate([x[:,x.shape[1]-size//2:x.shape[1],:], x, x[:,0:(size-size//2-1),:]], axis=1)

def rel_err_loss(y_true, y_pred):
    return K.sum(K.square(y_pred - y_true), axis=-1)/K.sum(K.square(y_true), axis=-1)

# Output functions
def outputFunc(log):
	def output(obj):
	    print(obj)
	    log.write(str(obj)+'\n')
	def outputnewline():
	    log.write('\n')
	    log.flush()
	def outputvec(vec, string):
	    log.write(string+'\n')
	    for i in range(0, vec.shape[0]):
	        log.write("%.6e\n" % vec[i])
	return (output,outputnewline,outputvec)

def initParser(descStr,
			   trainResStr='trainresultH.txt'):
	parser = argparse.ArgumentParser(description=descStr)
	parser.add_argument('--epoch', type=int, default=4000, metavar='N',
	                    help='input number of epochs for training (default: %(default)s)')
	parser.add_argument('--input-prefix', type=str, default='nlse2v2', metavar='N',
	                    help='prefix of input data filename (default: %(default)s)')
	parser.add_argument('--alpha', type=int, default=6, metavar='N',
	                    help='number of channels for training (default: %(default)s)')
	parser.add_argument('--L', type=int, default=7, metavar='N',
	                    help='number of grids (L+1, N=2^L*m) (default: %(default)s)')
	parser.add_argument('--n-cnn', type=int, default=5, metavar='N',
	                    help='number layer of CNNs (default: %(default)s)')
	parser.add_argument('--lr', type=float, default=0.001, metavar='LR',
	                    help='learning rate (default: %(default)s)')
	parser.add_argument('--batch-size', type=int, default=0, metavar='N',
	                    help='batch size (default: #train samples/100)')
	parser.add_argument('--verbose', type=int, default=2, metavar='N',
	                    help='verbose (default: %(default)s)')
	parser.add_argument('--output-suffix', type=str, default='None', metavar='N',
	                    help='suffix output filename(default: )')
	parser.add_argument('--percent', type=float, default=0.5, metavar='precent',
	                    help='percentage of number of total data(default: %(default)s)')
	parser.add_argument('--sum-file', type=str, default=trainResStr, metavar='N',
	                    help='file to summarize the error of the training (default: %(default)s)')
	return parser

def splitData(InputArray, OutputArray,n_train,n_test,output):
	"""function for splitting the data"""
	Nsamples = InputArray.shape[0]

	X_train = InputArray[0:n_train, :]
	Y_train = OutputArray[0:n_train, :]
	X_test  = InputArray[(Nsamples-n_test):Nsamples, :]
	Y_test  = OutputArray[(Nsamples-n_test):Nsamples, :]

	n_input = X_train.shape[1]
	n_output= Y_train.shape[1]

	output("[n_input, n_output] = [%d, %d]" % (n_input, n_output))
	output("[n_train, n_test] = [%d, %d]" % (n_train, n_test))

	# reshaping hte input data
	X_train = np.reshape(X_train, [X_train.shape[0], X_train.shape[1], 1])
	X_test  = np.reshape(X_test,  [X_test.shape[0],  X_test.shape[1],  1])

	return (X_train,Y_train,X_test,Y_test)


# Import libraries for pre-processing
import numpy as np
np.random.seed(1000)
import matplotlib.pyplot as plt
import os
import cv2
from PIL import Image
import keras

# Set the path & size of images
image_directory = 'D:/ML_Projects/Melanoma Classification/skin-lesions/train/'
SIZE = 128
dataset_train = []  #Many ways to handle data, you can use pandas. Here, we are using a list format.
label_train = []  #Place holders to define add labels. We will add 0 to all parasitized images and 1 to uninfected.

""" We shall load the images from melanoma, nevus and keratosis path
    into a single variable called dataset. The label of each shall also be loaded into label variabe.
    0 - Melanoma
    1 - Nevus
    2 - Keratosis """

Mel_images = os.listdir(image_directory + 'melanoma/')

for i, image_name in enumerate(Mel_images):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory + 'melanoma/' + image_name)
        image = Image.fromarray(image, '1')
        image = image.resize((SIZE, SIZE))
        dataset_train.append(np.array(image))
        label_train.append(0)

Nev_images = os.listdir(image_directory + 'nevus/')

for i, image_name in enumerate(Nev_images):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory + 'nevus/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_train.append(np.array(image))
        label_train.append(1)

Ker_images = os.listdir(image_directory + 'seborrheic_keratosis/')

for i, image_name in enumerate(Ker_images):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory + 'seborrheic_keratosis/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_train.append(np.array(image))
        label_train.append(2)

""" Now we shall import the validation data in a similar manner """
image_directory_valid = 'D:/ML_Projects/Melanoma Classification/skin-lesions/valid/'
SIZE = 128
dataset_valid = []  #Many ways to handle data, you can use pandas. Here, we are using a list format.
label_valid = []

Mel_images_valid = os.listdir(image_directory_valid + 'melanoma/')

for i, image_name in enumerate(Mel_images_valid):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory_valid + 'melanoma/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_valid.append(np.array(image))
        label_valid.append(0)

Nev_images_valid = os.listdir(image_directory_valid + 'nevus/')

for i, image_name in enumerate(Nev_images_valid):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory_valid + 'nevus/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_valid.append(np.array(image))
        label_valid.append(1)

Ker_images_valid = os.listdir(image_directory_valid + 'seborrheic_keratosis/')

for i, image_name in enumerate(Ker_images_valid):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory_valid + 'seborrheic_keratosis/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_valid.append(np.array(image))
        label_valid.append(2)

""" TRAIN =
    dataset_train - X_train
    label_train - y_train

    VALIDATION =
    dataset_valid - X_test
    label_valid - y_test """

# Lets import test dataset also
image_directory_test = 'D:/ML_Projects/Melanoma Classification/skin-lesions/test/'
SIZE = 128
dataset_test = []  #Many ways to handle data, you can use pandas. Here, we are using a list format.
label_test = []

Mel_images_test = os.listdir(image_directory_test + 'melanoma/')

for i, image_name in enumerate(Mel_images_test):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory_test + 'melanoma/' + image_name)
        image = Image.fromarray(image, 'BW')
        image = image.resize((SIZE, SIZE))
        dataset_test.append(np.array(image))
        label_test.append(0)

Nev_images_test = os.listdir(image_directory_test + 'nevus/')

for i, image_name in enumerate(Nev_images_test):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory_test + 'nevus/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_test.append(np.array(image))
        label_test.append(1)

Ker_images_test = os.listdir(image_directory_test + 'seborrheic_keratosis/')

for i, image_name in enumerate(Ker_images_test):    #Remember enumerate method adds a counter and returns the enumerate object

    if (image_name.split('.')[1] == 'jpg'):
        image = cv2.imread(image_directory_test + 'seborrheic_keratosis/' + image_name)
        image = Image.fromarray(image, 'RGB')
        image = image.resize((SIZE, SIZE))
        dataset_test.append(np.array(image))
        label_test.append(2)

# Shuffle the dataset
from sklearn.utils import shuffle
dataset_train, label_train = shuffle(dataset_train, label_train)
dataset_valid, label_valid = shuffle(dataset_valid, label_valid)
dataset_test, label_test = shuffle(dataset_test, label_test)

# Noe lets add the train and valid dataset
dataset_train = dataset_train + dataset_valid
label_train = label_train + label_valid

# Import libraries for CNN
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import to_categorical
from keras.preprocessing import image
import pandas as pd
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical

#Apply CNN
# ### Build the model

#############################################################
###2 conv and pool layers. with some normalization and drops in between.

INPUT_SHAPE = (SIZE, SIZE, 1)
inp = keras.layers.Input(shape=INPUT_SHAPE)

conv1 = keras.layers.Conv2D(32, kernel_size=(3, 3),
                               activation='relu', padding='same')(inp)
pool1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv1)
norm1 = keras.layers.BatchNormalization(axis = -1)(pool1)
drop1 = keras.layers.Dropout(rate=0.1)(norm1)
conv2 = keras.layers.Conv2D(32, kernel_size=(3, 3),
                               activation='relu', padding='same')(drop1)
pool2 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv2)
norm2 = keras.layers.BatchNormalization(axis = -1)(pool2)
drop2 = keras.layers.Dropout(rate=0.1)(norm2)

flat = keras.layers.Flatten()(drop2)  #Flatten the matrix to get it ready for dense.

hidden1 = keras.layers.Dense(256, activation='relu')(flat)
norm3 = keras.layers.BatchNormalization(axis = -1)(hidden1)
drop3 = keras.layers.Dropout(rate=0.1)(norm3)
hidden2 = keras.layers.Dense(128, activation='relu')(drop3)
norm4 = keras.layers.BatchNormalization(axis = -1)(hidden2)
drop4 = keras.layers.Dropout(rate=0.1)(norm4)

out = keras.layers.Dense(4, activation='sigmoid')(drop4)   #units=1 gives error

model = keras.Model(inputs=inp, outputs=out)
model.compile(optimizer='adam',
                loss='categorical_crossentropy',   #Check between binary_crossentropy and categorical_crossentropy
                metrics=['accuracy'])
print(model.summary())

# One hot encoding our categorical variables
from keras.utils import to_categorical
label_train = to_categorical(label_train)
label_valid = to_categorical(label_valid)

history = model.fit(np.array(dataset_train), label_train, epochs=15, validation_split=0.1)

### Accuracy calculation
#
# I'll now calculate the accuracy on the test data.
label_test = to_categorical(label_test)
print("Test_Accuracy: {:.2f}%".format(model.evaluate(np.array(dataset_test), label_test)[1]*100))

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
t = f.suptitle('CNN Performance', fontsize=12)
f.subplots_adjust(top=0.85, wspace=0.3)

max_epoch = len(history.history['accuracy'])+1
epoch_list = list(range(1,max_epoch))
ax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')
ax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')
ax1.set_xticks(np.arange(1, max_epoch, 5))
ax1.set_ylabel('Accuracy Value')
ax1.set_xlabel('Epoch')
ax1.set_title('Accuracy')
l1 = ax1.legend(loc="best")

ax2.plot(epoch_list, history.history['loss'], label='Train Loss')
ax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')
ax2.set_xticks(np.arange(1, max_epoch, 5))
ax2.set_ylabel('Loss Value')
ax2.set_xlabel('Epoch')
ax2.set_title('Loss')
l2 = ax2.legend(loc="best")


import argparse
import os
import time

import cv2
import numpy as np
import tensorrt as trt

import common

TRT_LOGGER = trt.Logger(trt.Logger.INFO)
trt.init_libnvinfer_plugins(TRT_LOGGER, "")

WINDOW_NAME = "TensorRT Fast-SCNN example."

mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]

def get_engine(engine_file_path):
    print("Reading engine from file {}".format(engine_file_path))
    with open(engine_file_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
        return runtime.deserialize_cuda_engine(f.read())

def normalize(im):
    im = np.asarray(im, dtype="float32")
    im = (im / 255.0 - mean) / std
    im = im.transpose(2, 0, 1)
    im = np.expand_dims(im, axis=0)
    return im.astype("float32")

def draw_caption(image, box, caption):
    b = np.array(box).astype(int)
    cv2.putText(
        image, caption, (b[0], b[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2
    )
    cv2.putText(
        image, caption, (b[0], b[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 1
    )

def create_label_colormap():
    """Creates a label colormap used in PASCAL VOC segmentation benchmark.
    Returns:
        A Colormap for visualizing segmentation results.
    """
    colormap = np.zeros((256, 3), dtype=np.uint8)
    ind = np.arange(256, dtype=np.uint8)

    for shift in reversed(range(8)):
        for channel in range(3):
            colormap[:, channel] |= ((ind >> channel) & 1) << shift
        ind >>= 3
    return colormap

def label_to_color_image(colormap, label):
    if label.ndim != 2:
        raise ValueError("Expect 2-D input label")

    if np.max(label) >= len(colormap):
        raise ValueError("label value too large.")

    return colormap[label]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", help="File path of trt model.", required=True)
    parser.add_argument(
        "--input_shape",
        type=str,
        default="576,768",
        help="Specify an input shape for inference.",
    )
    parser.add_argument(
        "--input", help="File path of input image file.", required=True, type=str
    )
    parser.add_argument(
        "--output", help="File path of output image.", default=None, type=str
    )
    parser.add_argument(
        "--count", help="Nun of inference.", type=int, default=101
    )
    parser.add_argument(
        "--with_argmax", help="Model has argmax.", action="store_true"
    )
    args = parser.parse_args()

    # Initialize colormap
    colormap = create_label_colormap()

    # Load model.
    model_name = os.path.splitext(os.path.basename(args.model))[0]
    input_shape = tuple(map(int, args.input_shape.split(",")))
    engine = get_engine(args.model)
    context = engine.create_execution_context()

    frame = cv2.imread(args.input)
    h, w, _ = frame.shape
    im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    resized_im = cv2.resize(im, (input_shape[1], input_shape[0]))
    normalized_im = normalize(resized_im)

    elapsed_list = []

    for count in range(args.count):
        # inference.
        start = time.perf_counter()
        inputs, outputs, bindings, stream = common.allocate_buffers(engine)
        inputs[0].host = np.ascontiguousarray(normalized_im)
        outputs = common.do_inference_v2(
            context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream
        )

        seg_map = np.array(outputs[0])

        if args.with_argmax:
            seg_map = seg_map.reshape([input_shape[0], input_shape[1]])
        else:
            seg_map = seg_map.reshape([-1, input_shape[0], input_shape[1]])
            seg_map = np.argmax(seg_map, axis=0)
        inference_time = (time.perf_counter() - start) * 1000

        if count % 20 == 0:
            print("count: {0:04} Inference: {1:.2f} ms".format(count, inference_time))

        if (count != 0):
            elapsed_list.append(inference_time)

    if elapsed_list:
        print("Mean inference: {0:.2f} ms".format(np.mean(elapsed_list)))

    if args.output:
        # Post process.
        seg_map += 1
        seg_image = label_to_color_image(colormap, seg_map)
        seg_image = cv2.resize(seg_image, (w, h))
        im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) // 2 + seg_image // 2
        im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
        draw_caption(im, (10, 30), model_name)

        # Output image file.
        cv2.imwrite(args.output, im)

if __name__ == "__main__":
    main()


# Bellman-Ford File Transfer Host Configurations
#
# Written by Sean Liu

# infinity
INFINITY = float('inf')

# maximum segment size
MSS = 4096	# bytes

# user commands
C_LINK_DOWN = 'LINKDOWN'
C_LINK_UP = 'LINKUP'
C_CHGCOST = 'CHANGECOST'
C_SHOWRT = 'SHOWRT'
C_CLOSE = 'CLOSE'
C_TRANSFER = 'TRANSFER'

# packet headers
H_FILE = 'f'
H_LINK_DOWN = 'ld'
H_LINK_UP = 'lu'
H_CHGCOST = 'c'
H_ROUTE_UPDATE = 'ru'


from django.shortcuts import render, redirect
from crud.models import players, game, player_game_scoreboard
from django.http import JsonResponse, HttpResponse
from django.views.decorators.csrf import csrf_exempt
import json
from django.contrib.auth.decorators import login_required

@login_required()
def teamSelection(request):
    playerCheck = game.objects.filter(is_active=True).exists()
    gameStatus = ''
    if playerCheck:
        gameStatus = "True"
        print("yes theres a game")
    else:
        gameStatus = "False"
        print("no sir")
    return render(request, 'teamSelection.html', {'gameStatus': gameStatus})
@login_required()
def alternateSelection(request):
    playerList = players.objects.values_list('name', flat=True)
    data = list(playerList)
    return render(request, 'alternateSelection.html', {'completeList': data})

def getPlayers(request):
    playerList = players.objects.values_list('name')
    data = list(playerList)

    return JsonResponse(data, safe=False)

@login_required()
def addPlayer(request):

    if request.method == 'POST':
        if players.objects.filter(name=request.POST['fname']).exists():
            return render(request, 'addPlayer.html')
        else:
            newPlayer = players(name=request.POST['fname'])
            newPlayer.save()

    return render(request,'addPlayer.html')

@login_required()
def gamePage(request):
    activeGame = game.objects.filter(is_active=True)
    activePlayers = player_game_scoreboard.objects.filter(is_active=True)
    gameId = activeGame.values('id')
    t1playerList = []
    t1tagList = []
    t2playerList = []
    teamNames = []
    t1Points = []
    t2Points = []
    t1playerPoints = []
    t2playerPoints = []
    if request.method == 'POST':
        gameDict = request.POST.items()
        for label, player in gameDict:
            if label.startswith('tname'):
                teamNames.append(player)
                continue

            if player == 'empty':
                continue
            elif label.startswith('t1p'):
                t1playerList.append(player)
                # t1tagList.append(label)
            elif label.startswith('t2p'):
                t2playerList.append(player)
        while len(t2playerList) < 5:
            t2playerList.append('None')
        while len(t1playerList) < 5:
            t1playerList.append('None')

        if activeGame:

            activeGame.delete()
            activePlayers.delete()
            #delete rows in player scoreboard here
            enterGame = game(
                team1=teamNames[0],
                team2=teamNames[1],
            )
            enterGame.save()

            newGameId = activeGame.values('id')

            enterTech = player_game_scoreboard(
                player='Tech',
                game=newGameId[0]['id'],
                team=teamNames[0],
            )
            enterTech.save()
            for player in t1playerList:
                if player == 'None':
                    continue
                else:
                    enterPlayer = player_game_scoreboard(
                        player=player,
                        team=teamNames[0],
                        game=gameId[0]['id'],

                    )
                    enterPlayer.save()

            enterSecondTech = player_game_scoreboard(
                player='Tech',
                game=newGameId[0]['id'],
                team=teamNames[1],

            )
            enterSecondTech.save()

            for player in t2playerList:
                if player == 'None':
                    continue
                else:
                    enterPlayer = player_game_scoreboard(
                        player=player,
                        team=teamNames[1],
                        game=gameId[0]['id'],

                    )
                    enterPlayer.save()

            #create rows for next set of players AND tech
        else:
            enterGame = game(
            team1=teamNames[0],
            team2=teamNames[1],
            )
            enterGame.save()
            newGameId = activeGame.values('id')
            enterTech = player_game_scoreboard(
                player='Tech',
                game=newGameId[0]['id'],
                team=teamNames[0],
            )
            enterTech.save()
            for player in t1playerList:
                if player == 'None':
                    continue
                else:
                    enterPlayer = player_game_scoreboard(
                        player=player,
                        team=teamNames[0],
                        game=gameId[0]['id'],

                    )
                    enterPlayer.save()
            enterSecondTech =player_game_scoreboard(
                player='Tech',
                game=newGameId[0]['id'],
                team=teamNames[1],

            )
            enterSecondTech.save()

            for player in t2playerList:
                if player == 'None':
                    continue
                else:
                    enterPlayer = player_game_scoreboard(
                        player=player,
                        team=teamNames[1],
                        game=gameId[0]['id'],

                    )
                    enterPlayer.save()

        # print(t1playerList)
        #
        # print(t2playerList)
        # print(teamNames)
        return render(request,'gamePage.html', {'t1players': t1playerList, 't2players': t2playerList, 'tnames': teamNames})

    else:  # if Get request
        if activeGame:
            activePlayersNoTech = activePlayers.exclude(player='Tech').order_by('id')

            playerLength = len(activePlayersNoTech)  #-2 maybe for the two techs if name is tech, pass exclude TECH
            t1Name = ''
            t2Name = ''
            for playerData in activePlayersNoTech:
                if (playerLength / 2) == len(t1playerList):
                    t2playerList.append(playerData.player)
                    t2Name = playerData.team
                    t2playerPoints.append(playerData.points)
                else:
                    t1playerList.append(playerData.player)
                    t1Name = playerData.team
                    t1playerPoints.append(playerData.points)
                # print(playerData)
                # print(playerData.player)
                # print(playerData.team)
            while len(t1playerList) < 5:
                t1playerList.append('None')
                t1playerPoints.append(0)
            while len(t2playerList) < 5:
                t2playerList.append('None')
                t2playerPoints.append(0)
            teamNames.append(t1Name)
            teamNames.append(t2Name)
            print(teamNames)
            print(t1playerList)
            print(t2playerList)
            print(t1playerPoints)
            print(t2playerPoints)
            return render(request, 'gamePage.html',
                          {'t1players': t1playerList, 't2players': t2playerList, 'tnames': teamNames,
                           't1playerPoints': t1playerPoints, 't2playerPoints': t2playerPoints})
        else:
            return redirect('/')

@login_required()
@csrf_exempt
def dataRetrieval(request):
    activeGame = game.objects.filter(is_active=True)
    gameId = activeGame.values('id')

    clonePlayers = player_game_scoreboard.objects.filter(is_active=True)
    #create query for game table to recieve game id
    #use gameid to update rows of player games that are active and change from is active to false
    if request.method == 'POST':
        body = json.loads(request.body)
        enterTech = player_game_scoreboard.objects.filter(player='Tech', is_active=True)
        enterTechFirst = enterTech.first()
        enterTechSecond = enterTech.last()

        for keys,values in body.items():
            if keys == 'team1tech':

                t1Name = enterTechFirst.team
                enterTechFirst.points = values
                enterTechFirst.is_active = False
                enterTechFirst.save()
                # enterTech = player_game_scoreboard(
                #
                #     player='Tech',
                #     points=values,
                #     is_active=False,
                #     game=gameId[0]['id'],
                # )
                # enterTech.save()
            elif keys == 'team2tech':
                t2Name = enterTechSecond.team
                enterTechSecond.points = values
                enterTechSecond.is_active = False
                enterTechSecond.save()

                # enterTech = player_game_scoreboard(
                #     player='Tech',
                #     points=values,
                #     is_active=False,
                #     game=gameId[0]['id'],
                # )
                # enterTech.save()
            elif keys == 'gameNumber':
                continue
            else:
                if keys == 'None':
                    continue
                elif body['gameNumber'] == 1:

                    newPlayer = player_game_scoreboard.objects.get(is_active=True, player=keys)
                    player_game_scoreboard.objects.filter(player=keys, is_active=True).update(
                        points=values,
                        is_active=False,
                    )
                    newPlayer.pk = None
                    newPlayer.is_active = True
                    newPlayer.points = 0
                    newPlayer.game = int((gameId[0]['id']) + 1)
                    newPlayer.save()

                else:
                    player_game_scoreboard.objects.filter(player=keys, is_active=True).update(
                        points=values,
                        is_active=False,
                    )

        # Updated current active game
        newGame = game.objects.get(is_active=True)

        # game.objects.filter(is_active=True).update(is_active=False)

        if body['gameNumber'] == 1:
            newGame.is_active = False
            newGame.save()  # saving game to be done
            # Cloning game
            newGame.pk = None
            newGame.is_active = True
            newGame.save()

            newTech = player_game_scoreboard(
                points=0,
                game=int((gameId[0]['id'])),
                team=t1Name,
                player='Tech',
                is_active=True
            )
            newTech.save()

            newTech1 = player_game_scoreboard(
                points=0,
                game=int((gameId[0]['id'])),
                team=t2Name,
                player='Tech',
                is_active=True
            )
            newTech1.save()

        else:
            newGame.is_active = False
            newGame.save()  # saving game to be done

        return JsonResponse({'test': "yee"}, safe=False)

    else:

        return HttpResponse('no sir')

def stats(request):
    completeList = []

    playerList = players.objects.all().values_list('name', flat=True)
    gameList = player_game_scoreboard.objects.filter(is_active=False)
    # perPlayer = gameList.filter(player='Christos')

    for player in playerList:
        subList = []
        sum = 0
        pointsPerPlayer = gameList.filter(player=player).values_list('points', flat=True)

        for i in pointsPerPlayer:

            sum = sum + i

        if sum <= 0:
            continue
        else:

            subList.append(player)
            subList.append(sum)
            subList.append(round(sum/len(pointsPerPlayer),2))
            subList.append(len(pointsPerPlayer))

            completeList.append(subList)

    return render(request, 'stats.html', {'completeList': completeList})

@csrf_exempt
def postUpdatePoints(request):
    if request.is_ajax:
        requestedData = request.body.decode("utf-8")
        new = json.loads(requestedData)
        # print(type(new))
        print(new['player'])
        print(new['points'])
        print(new['team'])
        playerName = new['player']
        playerPoints = new['points']
        playerTeam = new['team']
        player_game_scoreboard.objects.filter(player=playerName, team= playerTeam, is_active=True).update(
            points=playerPoints,
        )

    return HttpResponse('no sir')

@csrf_exempt
def ajaxUpdatePoints(request):
    if request.is_ajax:
        t1Points = []
        t2Points = []
        techScores = []
        activePlayers = player_game_scoreboard.objects.filter(is_active=True).order_by('id')
        activePlayersNoTech = activePlayers.exclude(player='Tech')
        activePlayersTech = activePlayers.filter(player='Tech')
        playerLength = len(activePlayersNoTech)
        for player in activePlayersNoTech:
            print(player.player)
            print(player.points)
            if (playerLength / 2) == len(t1Points):
                t2Points.append(player.points)

            else:
                t1Points.append(player.points)

        while len(t1Points) < 5:

            t1Points.append(0)
        while len(t2Points) < 5:

            t2Points.append(0)

        for techPlayer in activePlayersTech:
            techScores.append(techPlayer.points)

        return JsonResponse({"t1pointsArray": t1Points, "t2pointsArray": t2Points, "techPoints": techScores})  # Pass scores of tech while passing players scores

#view here for ajax call--- this page will render game page. very similar logic to gaempage. send same data, etc. Team name is experimental Team...


# https://www.hackerrank.com/challenges/collections-counter/problem
from collections import Counter

if __name__ == '__main__':
    numberofshoes = int(input())
    shoevarietysize = Counter(map(int,input().split()))
    numberofcustomer = int(input())
    totalincome = 0

    for _ in range(numberofcustomer):
        shoesizeaskedbycustomer , priceofshoe = map(int,input().split())
        if shoevarietysize[shoesizeaskedbycustomer]:
            totalincome += priceofshoe
            shoevarietysize[shoesizeaskedbycustomer] -=1
print(totalincome)


# -*- coding: utf-8 -*-
import os
import xlrd
import logging
from sources.repository.noderepository import NodeRepo
from sources.domain.nodesubscriber import NodeSubscriber
from sources.domain.errors import MigrateError
from sources.config.config import *

log_def_subs_repo = logging.getLogger("migrate.defsubsrepo")

class MockRepo(NodeRepo):
    """ Xранилище номеров Макета АТС для типа SIP """
    def __init__(self, config):
        super().__init__(config)
        try:
            if os.path.isfile(config.source_file_db):
                self.book = xlrd.open_workbook(config.source_file_db)
            else:
                self.book = None
        except TypeError as e:
            raise MigrateError(e)

    @property
    def check_book(self):
        """ Проверка на непустые входные данные """
        return bool(self.book)

    @property
    def sheet(self):
        """ Выбор 0 закладки книги """
        return self.book.sheet_by_index(0)

    @property
    def header(self):
        """ Читаем заголовки в 0 строке """
        return self.sheet.row_values(0)

    def zip_number(self, number):
        _n = dict(zip(self.header, number))
        return _n

    def get_numbers(self):
        _index = range(1, self.sheet.nrows)
        _numbers_gen = [self.sheet.row_values(i) for i in _index]
        return list(map(self.zip_number, _numbers_gen))

    def execute(self):

        if self.check_book:

            dn = dict()

            for row in self.get_numbers():
                dn['dn'] = str(int(row['Номер']))
                dn['type_dn'] = SIP
                dn['password'] = row['Пароль']
                dn['list_dn_options'] = [row['Категория'].strip()]
                dn['list_dn_options'].extend([suppl.strip() for suppl in row['Услуги'].split(',')])

                self.add(NodeSubscriber.from_dict(dn))

        return self


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 22 11:05:03 2022

@author: belayv
"""

import mdtraj as md
import paramiko
from scp import SCPClient, SCPException
import os
import sys

def send_to_fah(lilac_out_dir,fah_in_dir):

    def remove_solvent(lilac_out_dir):

        for i in os.listdir(lilac_out_dir):

            if i.endswith('.pdb'):

                try:
                    os.chdir(lilac_out_dir)
                    print('removing solvent from '+i)
                    md.load(i).remove_solvent().save_pdb(lilac_out_dir+'/nosolvent_equilibrated.pdb')
                    print('removed solvent from '+ i)
                except:
                    print('something went awry :(')

            elif i.endswith('.cif'):

                try:
                    os.chdir(lilac_out_dir)
                    print('removing solvent from '+i)
                    md.load(i).save_pdb(lilac_out_dir+i)
                    md.load(i).remove_solvent().save_pdb(lilac_out_dir+'/nosolvent_equilibrated.pdb')
                    print('removed solvent from '+i)
                except:
                    print('something went wrong')
            else:

                pass

    host = 'pllwskifah1'
    username = 'server'
    port = 22
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(host,port,username=username)
    x=1

    if x == 1:

        try:
            for i in os.listdir(lilac_out_dir):

                if i.endswith('.xml.bz2'):

                    try:
                        os.chdir(lilac_out_dir)
                        scp=SCPClient(ssh.get_transport())
                        scp.put(i,remote_path=fah_in_dir+"/.")
                        scp.close()

                    except:
                        print('something went wrong while trying to transport zipped files')

        except:
            print('something went wrong!')

        try:
        	for i in os.listdir(lilac_out_dir):

        		if i.endswith('.pdb'):

        				remove_solvent(lilac_out_dir)

        				os.chdir(lilac_out_dir)

        				scp=SCPClient(ssh.get_transport())
        				scp.put(i,remote_path=fah_in_dir+"/.")
        				scp.close()

        except:

        	print('Something went wrong with transporting topology files.')

    ssh.close()

if __name__ == '__main__':
	send_to_fah(sys.argv[1],sys.argv[2])


#!/home/zeyu/anaconda3/bin/python3.6

"""
    Preprocessing

    Author:
        Zeyu Li <zyli@cs.ucla.edu> or <zeyuli@ucla.edu>

    Description:
        Take in the Very raw data and produce a ready-to-use version
"""

import sys, os
import re
import logging
import numpy as np
from lxml import etree
from bs4 import BeautifulSoup

import nltk
nltk.data.path.append("/workspace/nltk_data")
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

import string, random
from collections import Counter

try:
    import ujson as json
except:
    import json

part_user = set()
    # Users participated in Asking and Answering

# count how many questions an users asked
# count how many questions an answerer responded
count_Q, count_A = {}, {}

qa_map = {}
test_candidates = set()

def clean_html(x):
    return BeautifulSoup(x, 'lxml').get_text()

def clean_str(string):
    """Clean up the string

    Cleaning strings of content or title
    Original taken from [https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py]

    Args:
        string - the string to clean

    Return:
        _ - the cleaned string
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def clean_str2(s):
    """Clean up the string

    * New version, removing all punctuations

    Cleaning strings of content or title
    Original taken from [https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py]

    Args:
        string - the string to clean

    Return:
        _ - the cleaned string
    """
    ss = s
    translator = str.maketrans("", "", string.punctuation)
    ss = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", ss)
    ss = re.sub(r"\'s", "s", ss)
    ss = re.sub(r"\'ve", "ve", ss)
    ss = re.sub(r"n\'t", "nt", ss)
    ss = re.sub(r"\'re", "re", ss)
    ss = re.sub(r"\'d", "d", ss)
    ss = re.sub(r"\'ll", "ll", ss)
    ss = re.sub(r"\s{2,}", " ", ss)
    ss = ss.translate(translator)
    return ss.strip().lower()

def remove_stopwords(string, stopword_set):
    """Removing Stopwords

    Args:
        string - the input string to remove stopwords
        stopword_set - the set of stopwords

    Return:
        _ - the string that has all the stopwords removed
    """
    word_tokens = word_tokenize(string)
    filtered_string = [word for word in word_tokens
                       if word not in stopword_set]
    return " ".join(filtered_string)

def split_post(raw_dir, data_dir):
    """ Split the post

    Split post to question and answer,
    keep all information, output to file

    Args:
        raw_dir - raw data directory
        data_dir - parsed data directory
    """
    if os.path.exists(data_dir + "Posts_Q.json") \
        and os.path.exists(data_dir + "Posts_A.json"):
        print("\t\tPosts_Q.json, Posts_A.json already exists."
              "Skipping the split_post.")
        return

    with open(data_dir + "Posts_Q.json", "w") as fout_q, \
            open(data_dir + "Posts_A.json", "w") as fout_a:
        parser = etree.iterparse(raw_dir + 'Posts.xml',
                                 events=('end',), tag='row')
        for event, elem in parser:
            attr = dict(elem.attrib)
            attr['Body'] = clean_html(attr['Body'])

            # Output to separate files
            if attr['PostTypeId'] == '1':
                fout_q.write(json.dumps(attr) + "\n")
            elif attr['PostTypeId'] == '2':
                fout_a.write(json.dumps(attr) + "\n")
    return

def process_QA(data_dir):
    """Process QA

    Extract attributes used in this project
    Get rid of the text information,
    only record the question-user - answer-user relation

    Args:
        data_dir - the dir where primitive data is stored
    """
    POST_Q = "Posts_Q.json"
    POST_A = "Posts_A.json"
    OUTPUT = "Record_All.json"
    RAW_STATS = "question.stats.raw"

    # Get logger to log exceptions
    logger = logging.getLogger(__name__)

    no_acc_question = 0

    raw_question_stats = []

    if not os.path.exists(data_dir + POST_Q):
        raise IOError("file {} does NOT exist".format(data_dir + POST_Q))

    if not os.path.exists(data_dir + POST_A):
        raise IOError("file {} does NOT exist".format(data_dir + POST_A))

    # Process question information
    with open(data_dir + POST_Q, 'r') as fin_q:
        for line in fin_q:
            data = json.loads(line)
            try:
                qid, rid = data.get('Id', None), data.get('OwnerUserId', None)
                # If such
                if qid and rid:
                    acc_id = data.get('AcceptedAnswerId', None)
                    answer_count = int(data.get('AnswerCount', -1))
                    if acc_id:
                        qa_map[qid] = {
                            'QuestionId': qid,
                            'QuestionOwnerId': rid,
                            'AcceptedAnswerId': acc_id,
                            'AcceptedAnswererId': None,
                            'AnswererIdList': [],
                            'AnswererAnswerTuples': []
                        }
                        count_Q[rid] = count_Q.get(rid, 0) + 1
                    else:
                        no_acc_question += 1

                    if answer_count >= 0:
                        raw_question_stats.append(answer_count)
            except:
                logger.error("Error at process_QA 1: " + str(data))
                continue
    print("\t\t{} questions do not have accepted answer!"
          .format(no_acc_question))

    # Count raw question statistics
    raw_question_stats_cntr = Counter(raw_question_stats)
    with open(data_dir + RAW_STATS, "w") as fout:
        for x in sorted(list(raw_question_stats_cntr.keys())):
            print("{}\t{}".format(x, raw_question_stats_cntr[x]), file=fout)
        print("Total\t{}".format(sum(raw_question_stats)), file=fout)

    # Process answer information
    with open(data_dir + POST_A, 'r') as fin_a:
        for line in fin_a:
            data = json.loads(line)
            try:
                answer_id = data.get('Id', None)
                aid = data.get('OwnerUserId', None)
                qid = data.get('ParentId', None)
                entry = qa_map.get(qid, None)
                if answer_id and aid and qid and entry:
                    entry['AnswererAnswerTuples'].append((aid, answer_id))
                    entry['AnswererIdList'].append(aid)
                    count_A[aid] = count_A.get(aid, 0) + 1

                    # Check if we happen to hit the accepted answer
                    if answer_id == entry['AcceptedAnswerId']:
                        entry['AcceptedAnswererId'] = aid
                else:
                    logger.error(
                        "Answer {} belongs to unknown Question {} at Process QA"
                        .format(answer_id, qid))
            except IndexError as e:
                logger.error(e)
                logger.info("Error at process_QA 2: " + str(data))
                continue

    # Fill in the blanks of `AcceptedAnswererId`
    # for qid in qa_map.keys():
    #    acc_id = qa_map[qid]['AcceptedAnswerId']
    #    for aid, answer_id in qa_map[qid]['AnswererAnswerTuples']:
    #        if answer_id == acc_id:
    #            qa_map[qid]['AcceptedAnswererId'] = aid
    #            break

    print("\t\tWriting the Record for ALL to disk.")
    with open(data_dir + OUTPUT, 'w') as fout:
        for q in qa_map.keys():
            fout.write(json.dumps(qa_map[q]) + "\n")

def question_stats(data_dir):
    """Find the question statistics for `Introduction`

    Args:
        data_dir -
    Return
    """
    OUTPUT = "question.stats"
    count = []
    for qid in qa_map.keys():
        ans_count = len(qa_map[qid]['AnswererIdList'])
        count.append(ans_count)
        if ans_count == 0:
            print("0 answer id list", qid)
    question_stats_cntr = Counter(count)

    with open(data_dir + OUTPUT, "w") as fout:
        for x in sorted(list(question_stats_cntr.keys())):
            print("{}\t{}".format(x, question_stats_cntr[x]), file=fout)
        print("Total\t{}".format(sum(count), file=fout), file=fout)
    return

def build_test_set(data_dir, parsed_dir, threshold, test_sample_size,
                   test_proportion):
    """
    Building test datase,
    test_proportiont
    Args:
        parse_dir - the directory to save parsed set.
        threshold - the selection threshold

    Return:
    """
    TEST = "test.txt"
    OUTPUT_TRAIN = "Record_Train.json"

    accept_no_answerer = 0

    ordered_count_A = sorted(
        count_A.items(), key=lambda x:x[1], reverse=True)
    ordered_aid = [x[0] for x in ordered_count_A]
    ordered_aid = ordered_aid[: int(len(ordered_aid) * 0.1)]

    question_count = len(qa_map)

    for qid in qa_map.keys():
        accaid = qa_map[qid]['AcceptedAnswererId']
        rid = qa_map[qid]['QuestionOwnerId']
        if not accaid:
            accept_no_answerer += 1
            continue
        if count_Q[rid] >= threshold and count_A[accaid] >= threshold:
            test_candidates.add(qid)

    print("\t\tSample table size {}. Using {} instances for test."
          .format(len(test_candidates), int(question_count * test_proportion)))

    test = np.random.choice(list(test_candidates),
                            size=int(question_count * test_proportion),
                            replace=False)

    print("\t\tAccepted answer without Answerer {}".format(accept_no_answerer))

    print("\t\tWriting the sampled test set to disk")
    with open(parsed_dir + TEST, "w") as fout:
        for qid in test:
            rid = qa_map[qid]['QuestionOwnerId']
            accaid = qa_map[qid]['AcceptedAnswererId']
            aid_list = qa_map[qid]['AnswererIdList']
            if len(aid_list) <= test_sample_size:
                neg_sample_size = test_sample_size - len(aid_list)
                neg_samples = random.sample(ordered_aid, neg_sample_size)
                samples = neg_samples + aid_list
            else:
                samples = random.sample(aid_list, test_sample_size)
                if accaid not in samples:
                    samples.pop()
                    samples.append(accaid)
            samples = " ".join(samples)
            print("{} {} {} {}".format(rid, qid, accaid, samples),
                  file=fout)

    # if qid is a test instance or qid doesn't have an answer
    qid_list = list(qa_map.keys())
    for qid in qid_list:
        if qid in test\
            or not len(qa_map[qid]['AnswererIdList'])\
            or not qa_map[qid]['AcceptedAnswererId']:
            del qa_map[qid]

    # Write QA pair to file
    print("\t\tWriting the Record for training to disk")
    with open(data_dir + OUTPUT_TRAIN, 'w') as fout:
        for q in qa_map.keys():
            fout.write(json.dumps(qa_map[q]) + "\n")
    return

def extract_question_user(data_dir, parsed_dir):
    """Extract Question User pairs and output to file.
    Extract "Q" and "R". Format:
        <Qid> <Rid>
    E.g.
        101 40
        145 351

    Args:
        data_dir - data directory
        parsed_dir - parsed file directory
    """
    # INPUT = "Record_Train.json"
    INPUT = "Record_All.json"
    OUTPUT = "Q_R.txt"

    if not os.path.exists(data_dir + INPUT):
        IOError("Can NOT find {}".format(data_dir + INPUT))

    with open(data_dir + INPUT, "r") as fin:
        with open(parsed_dir + OUTPUT, "w") as fout:
            for line in fin:
                data = json.loads(line)
                qid = data['QuestionId']
                rid = data['QuestionOwnerId']
                part_user.add(int(rid))  # Adding participated questioners
                print("{} {}".format(str(qid), str(rid)), file=fout)

def extract_question_answer_user(data_dir, parsed_dir):
    """Extract Question, Answer User pairs and output to file.

    (1) Extract "Q" - "A"
        The list of AnswerOwnerList contains <aid>-<owner_id> pairs
        Format:
            <Qid> <Aid>
        E.g.
            100 1011
            21 490

    (2) Extract "Q" - Accepted answerer
        Format:
            <Qid> <Acc_Aid>
    Args:
        data_dir - data directory
        parsed_dir - parsed file directory
    """
    INPUT = "Record_Train.json"
    OUTPUT_A = "Q_A.txt"
    OUTPUT_ACC = "Q_ACC.txt"

    if not os.path.exists(data_dir + INPUT):
        IOError("Can NOT find {}".format(data_dir + INPUT))

    with open(data_dir + INPUT, "r") as fin, \
            open(parsed_dir + OUTPUT_A, "w") as fout_a, \
            open(parsed_dir + OUTPUT_ACC, "w") as fout_acc:
        for line in fin:
            data = json.loads(line)
            qid = data['QuestionId']
            aid_list = data['AnswererIdList']
            accaid = data['AcceptedAnswererId']
            for aid in aid_list:
                part_user.add(int(aid))
                print("{} {}".format(str(qid), str(aid)), file=fout_a)
            print("{} {}".format(str(qid), str(accaid)), file=fout_acc)

def extract_question_content(data_dir, parsed_dir):
    """Extract questions, content pairs from question file

    Question content pair format:
        <qid> <content>
    We extract both with and without stop-word version
        which is signified by "_nsw"

    Args:
        data_dir - data directory
        parsed_dir - parsed file directory
    """
    INPUT = "Posts_Q.json"
    OUTPUT_T = "Q_title.txt"  # Question title
    OUTPUT_T_NSW = "Q_title_nsw.txt"  # Question title, no stop word
    OUTPUT_C = "Q_content.txt"  # Question content
    OUTPUT_C_NSW = "Q_content_nsw.txt"  # Question content, no stop word

    logger = logging.getLogger(__name__)

    if not os.path.exists(data_dir + INPUT):
        IOError("Can NOT locate {}".format(data_dir + INPUT))

    sw_set = set(stopwords.words('english'))  # Create the stop word set

    # We will try both with or without stopwords to
    # check out the performance.
    with open(data_dir + INPUT, "r") as fin, \
            open(parsed_dir + OUTPUT_T, "w") as fout_t, \
            open(parsed_dir + OUTPUT_T_NSW, "w") as fout_t_nsw, \
            open(parsed_dir + OUTPUT_C, "w") as fout_c, \
            open(parsed_dir + OUTPUT_C_NSW, "w") as fout_c_nsw:
        for line in fin:
            data = json.loads(line)
            try:
                qid = data.get('Id')
                if qid not in qa_map:
                    continue
                title = data.get('Title')
                content = data.get('Body')

                content, title = clean_str2(content), clean_str2(title)
                content_nsw = remove_stopwords(content, sw_set)
                title_nsw = remove_stopwords(title, sw_set)

                print("{} {}".format(qid, content_nsw),
                      file=fout_c_nsw)  # Without stopword
                print("{} {}".format(qid, content),
                      file=fout_c)  # With stopword
                print("{} {}".format(qid, title_nsw),
                      file=fout_t_nsw)  # Without stopword
                print("{} {}".format(qid, title),
                      file=fout_t)  # With stopword
            except:
                logger.info("Error at Extracting question content and title: "
                            + str(data))
                continue

def extract_answer_score(data_dir, parsed_dir):
    """Extract the answers vote, a.k.a. Scores.

    This information might be useful when
        the accepted answer is not selected.

    Args:
        data_dir - Input data dir
        parsed_dir - Output data dir
    """
    INPUT = "Posts_A.json"
    OUTPUT = "A_score.txt"

    logger = logging.getLogger(__name__)

    if not os.path.exists(data_dir + INPUT):
        IOError("Cannot find file{}".format(data_dir + INPUT))

    with open(data_dir + INPUT, "r") as fin, \
        open(parsed_dir + OUTPUT, "w") as fout:
        for line in fin:
            data = json.loads(line)
            try:
                aid = data.get('Id')
                score = data.get('Score')
                print("{} {}".format(aid, score), file=fout)
            except:
                logging.info("Error at Extracting answer score: "
                             + str(data))
                continue

def extract_question_best_answerer(data_dir, parsed_dir):
    """Extract the question-best-answerer relation

    Args:
        data_dir  - as usual
        parsed_dir  -  as usual
    """
    INPUT_A = "Posts_A.json"
    INPUT_MAP = "Record_Train.json"
    OUTPUT = "Q_ACC_A.txt"

    if not os.path.exists(data_dir + INPUT_A):
        IOError("Cannot find file {}".format(data_dir + INPUT_A))
    if not os.path.exists(data_dir + INPUT_MAP):
        IOError("Cannot find file {}".format(data_dir + INPUT_MAP))

    accanswerid_uaid = {}  # Accepted answer id to Answering user id
    answerid_score = {}  # Answer id to answer scores
    with open(data_dir + INPUT_A, "r") as fin_a, \
        open(data_dir + INPUT_MAP, "r") as fin_map, \
        open(parsed_dir + OUTPUT, "w") as fout:

        # build acc-a dict
        for line in fin_a:
            data = json.loads(line)
            try:
                answerid = data.get("Id")
                if answerid == "5":
                    print(100)
                score = data.get("Score")
                uaid = data.get("OwnerUserId")
                answerid_score[answerid] = score
                accanswerid_uaid[answerid] = uaid  # uaid is rid
            except:
                logging.info(
                    "Error at Extracting question, best answer user: "
                    + str(data))

        print(len(accanswerid_uaid))
        for line in fin_map:
            data = json.loads(line)
            try:
                qid = data.get('QuestionId')
                if "AcceptedAnswerID" in data:  # If acc answer exists
                    acc_answerid = data.get('AcceptedAnswerId')
                else:
                # If acc answer doesn't exist, choose highest score answer
                    ans = data.get('AnswerIdList')
                    ans = list(zip(*ans))[0]
                    scores = [answerid_score[answerid] for answerid in ans]
                    max_ind = scores.index(max(scores))
                    acc_answerid = ans[max_ind]
                uaccid = accanswerid_uaid[acc_aid]
                print("{} {}".format(qid, uaccid), file=fout)
            except:
                print(1)
                logging.info(
                    "Error at Extracting question, best answer user: "
                     + str(data))

def extract_question_best_answerer_2(data_dir, parsed_dir):
    """Extract the question-best-answerer relation

    Args:
        data_dir  - as usual
        parsed_dir  -  as usual
    """
    INPUT_A = "Posts_A.json"
    # INPUT_MAP = "Record_Train.json"
    # Uncomment this when running NeRank
    INPUT_MAP = "Record_All.json"
    OUTPUT = "Q_ACC_A.txt"

    if not os.path.exists(data_dir + INPUT_A):
        IOError("Cannot find file {}".format(data_dir + INPUT_A))
    if not os.path.exists(data_dir + INPUT_MAP):
        IOError("Cannot find file {}".format(data_dir + INPUT_MAP))

    accanswerid_uaid = {}  # Accepted answer id to Answering user id
    answerid_score = {}  # Answer id to answer scores
    with open(data_dir + INPUT_MAP, "r") as fin_map, \
        open(parsed_dir + OUTPUT, "w") as fout:

        for line in fin_map:
            data = json.loads(line)
            try:
                qid = data.get('QuestionId')
                acc_aid = data.get("AcceptedAnswererId")
                if qid and acc_aid:
                    print("{} {}".format(qid, acc_aid), file=fout)
            except:
                print(1)
                logging.info(
                    "Error at Extracting question, best answer user: "
                     + str(data))

def write_part_users(parsed_dir):
    OUTPUT = "QA_ID.txt"
    with open(parsed_dir + OUTPUT, "w") as fout:
        IdList = list(part_user)
        IdList.sort()
        for index, user_id in enumerate(IdList):
            print("{} {}".format(index + 1, user_id), file=fout)

def preprocess_(dataset, threshold, prop_test, sample_size):
    DATASET = dataset
    RAW_DIR = os.getcwd() + "/raw/{}/".format(DATASET)
    DATA_DIR= os.getcwd() + "/data/{}/".format(DATASET)
    PARSED_DIR = os.getcwd() + "/data/parsed/{}/".format(DATASET)

    print("Preprocessing {} ...".format(dataset))

    if not os.path.exists(RAW_DIR):
        print("{} dir or path doesn't exist.\n"
              "Please download the raw data set into the /raw."
              .format(RAW_DIR), file=sys.stderr)
        sys.exit()

    if not os.path.exists(DATA_DIR):
        print("{} data dir not found.\n"
              " Creating a folder for that."
              .format(DATA_DIR))
        os.makedirs(DATA_DIR)

    if not os.path.exists(PARSED_DIR):
        print("{} dir or path NOT found.\n"
              "Creating a folder for that."
              .format(PARSED_DIR))
        os.makedirs(PARSED_DIR)

    if os.path.exists(DATA_DIR + "log.log"):
        os.remove(DATA_DIR + "log.log")

    # Setting up loggers
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)

    log_fh = logging.FileHandler(DATA_DIR + "log.log")
    log_fh.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    log_fh.setFormatter(formatter)
    logger.addHandler(log_fh)

    # Split contest to question and answer
    print("\tSpliting post")
    # split_post(raw_dir=RAW_DIR, data_dir=DATA_DIR)

    # Extract question-user, answer-user, and question-answer information
    # Generate Question and Answer/User map
    print("\tProcessing QA")
    # process_QA(data_dir=DATA_DIR)

    print("\tGenerating question statistics...")
    # question_stats(data_dir=DATA_DIR)

    print("\tExtracting question content ...")
    # extract_question_content(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    print("\tBuilding test sets")
    # build_test_set(data_dir=DATA_DIR, parsed_dir=PARSED_DIR,
    #                threshold=threshold, test_sample_size=sample_size,
    #                test_proportion=prop_test)

    print("\tExtracting Q, R, A relations ...")
    extract_question_user(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    # extract_question_answer_user(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    # extract_answer_score(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)
    # extract_question_best_answerer_2(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    # write_part_users(parsed_dir=PARSED_DIR)

    print("Done!")

if __name__ == "__main__":
    if len(sys.argv) < 3 + 1:
        print("\t Usage: {} [name of dataset] [threshold] [prop of test] [test sample size]"
              .format(sys.argv[0]), file=sys.stderr)
        sys.exit(0)
    threshold = int(sys.argv[2])
    test_proportion = float(sys.argv[3])
    sample_size = int(sys.argv[4])
    preprocess_(sys.argv[1], threshold, test_proportion, sample_size)


# 227. Basic Calculator II

from collections import deque

class Solution:
    def calculate(self, s: str) -> int:
        # is there +1-3
        # handle operand:
        # 1, 10, -> cur = 0, cur = cur*10 + int(val), cur=0
        # since the operator is to calculate with the nex digit, er need a var to hold the operator
        # since calculate the next number, we need to hold prev value
        # since we will refresh the curand prev, we need res to store last action
        res, cur, prev = 0, 0, 0
        operator = "+"
        # stack = deque(1)
        for val in s + "+":
            if val.isdigit():
                cur = cur * 10 + int(val)
            elif val != " ":
                if operator == "+" or operator == "-":
                    res += prev
                    prev = cur if operator == "+" else -cur
                elif operator == "*" or operator == "/":
                    prev = (
                        cur * prev if operator == "*" else int(prev / cur)
                    )  # no prev//cur dur to negative num
                cur, operator = 0, val

        res += prev
        return res