# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.shortcuts import render
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, WebDriverException
from operator import itemgetter
from decimal import Decimal
from threading import Thread
import requests, time

def scrapRedmart(item, redmart):
    r = requests.get('https://api.redmart.com/v1.6.0/catalog/search?q='+item+'&pageSize=18&sort=1024')
    if 'products' in r.json():
        for i in r.json()['products']:
            item = {}
            item['name'] = i['title'] + ' (' + i['measure']['wt_or_vol'] + ')'
            item['price'] = i['pricing']['price']
            item['url'] = 'https://redmart.com/product/' + i['details']['uri']
            item['website'] = 'Redmart'
            redmart.append(item)

def scrapNtuc(item, ntuc, options):
    driver = webdriver.Chrome(settings.CHROME_PATH, chrome_options=options)
    try:
        driver.get("https://www.fairprice.com.sg/searchterm/"+item)
        # load_more = 1
        # for i in range(0, load_more):
        #     driver.find_element_by_link_text('LOAD MORE').click()
        #     time.sleep(1)
        elements = driver.find_elements_by_css_selector("div.pdt_list_wrap")
        for element in elements:
            try:
                item = {}
                item['name'] = element.find_element_by_tag_name("img").get_attribute("title") + ' (' + element.find_element_by_css_selector("span.pdt_Tweight").text + ')'
                item['price'] = Decimal(element.find_element_by_css_selector("span.pdt_C_price").text[1:])
                try:
                    item['url'] = element.find_element_by_css_selector("a.pdt_title").get_attribute("href")
                except NoSuchElementException:
                    item['url'] = None
                item['website'] = 'Ntuc'
                ntuc.append(item)
            except NoSuchElementException:
                continue
            except StaleElementReferenceException:
                continue
    except WebDriverException:
        pass
    driver.quit()

def scrapHonestbee(item, honestbee):
    headers = {'Accept': 'application/vnd.honestbee+json;version=2'}
    r = requests.get('https://www.honestbee.sg/api/api/stores/9345?q='+item+'&sort=relevance&page=1', headers=headers)
    if 'products' in r.json():
        for i in r.json()['products']:
            item = {}
            item['name'] = (i['productBrand'] + " " if i['productBrand'] else "") + i['title'] + ' (' + i['size'] + ')'
            item['price'] = Decimal(i['price'])
            item['url'] = 'https://www.honestbee.sg/en/groceries/stores/fresh-by-honestbee/products/' + str(i['id'])
            item['website'] = 'Honestbee'
            honestbee.append(item)

def scrapAmazonPrime(item, amazonprime, options):
    driver = webdriver.Chrome(settings.CHROME_PATH, chrome_options=options)
    try:
        driver.get("https://www.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords="+item)
        # load_more = 1
        # for i in range(0, load_more):
        elements = driver.find_elements_by_css_selector("div.s-item-container")
        for element in elements:
            try:
                item = {}
                try:
                    item['price'] = Decimal(element.find_element_by_css_selector("span.sx-price-whole").text.replace(",", "") + '.' + element.find_element_by_css_selector("sup.sx-price-fractional").text)
                except NoSuchElementException:
                    continue
                try:
                    item['name'] = element.find_element_by_css_selector("h2.a-size-base.s-inline.s-access-title.a-text-normal").get_attribute("data-attribute")
                except NoSuchElementException:
                    item['name'] = element.find_element_by_css_selector("h2.a-size-medium.s-inline.s-access-title.a-text-normal").get_attribute("data-attribute")
                try:
                    item['url'] = element.find_elements_by_css_selector("a.a-link-normal.a-text-normal")[0].get_attribute("href")
                except IndexError:
                    item['url'] = None
                item['website'] = 'Amazon Prime'
                amazonprime.append(item)
                # if i < load_more - 1:
                #     driver.find_element_by_link_text('Next Page').click()
                #     time.sleep(2)
            except StaleElementReferenceException:
                continue
    except WebDriverException:
        pass
    driver.quit()

def index(request):
    current_name, items = '', []
    if request.method == 'POST':
        options = webdriver.ChromeOptions()
        options.add_argument('--headless')

        current_name, redmart, ntuc, honestbee, amazonprime = request.POST['item'], [], [], [], []

        # create threads
        redmart_thread = Thread(target=scrapRedmart, args=(request.POST['item'], redmart,))
        ntuc_thread = Thread(target=scrapNtuc, args=(request.POST['item'], ntuc, options,))
        honestbee_thread = Thread(target=scrapHonestbee, args=(request.POST['item'], honestbee,))
        amazonprime_thread = Thread(target=scrapAmazonPrime, args=(request.POST['item'], amazonprime, options,))

        # run all threads
        redmart_thread.start()
        ntuc_thread.start()
        honestbee_thread.start()
        amazonprime_thread.start()

        # wait until all threads finished
        redmart_thread.join()
        ntuc_thread.join()
        honestbee_thread.join()
        amazonprime_thread.join()

        items = redmart + ntuc + honestbee + amazonprime
        items = sorted(items, key=itemgetter('price'))
    return render(request, 'scrap/index.html', {'current_name':current_name, 'items':items})


from turtle import *
from floodfillv2 import FloodFill
from processmaze import genBitmap
import os, math

class Maze:
    def __init__(self,screenx,screeny,mazename):
        hideturtle()
        tracer(0,0)
        speed(3)

        self.window = Screen()
        self.window.screensize(screenx,screeny)
        self.window.setup(width=1.0,height=1.0,startx=None,starty=None)

        self.maze = genBitmap(os.path.dirname(os.path.abspath(__file__))+ r"\mazes\{}.bmp".format(mazename))
        self.botx = None
        self.boty = None
        self.botdir = self.findStartDir()
        onscreenclick(self.updateMaze)
        self.currentGoal = [0,0]
        self.solver = FloodFill()
        self.moving = False

    def findStartDir(self):
        for i in self.maze:
            for j in i:
                if j >= 3:
                    return ["e","s","w","n"][j-3]

    def drawBox(self,x1,y1,x2,y2):
        penup()
        goto(x1,y1)
        pendown()
        goto(x1,y2)
        goto(x2,y2)
        goto(x2,y1)
        goto(x1,y1)

    def drawFilledBox(self,x1,y1,x2,y2,c):
        color(c)
        begin_fill()
        self.drawBox(x1,y1,x2,y2)
        end_fill()
        color("black")
        self.drawBox(x1,y1,x2,y2)
        self.drawBox(x1,y1,x2,y2)

    #x123/y123 are pixel offsets from bottom left corner
    def drawBot(self,x1,y1,x2,y2,x3,y3,c):
        color(c)
        penup()
        goto(self.botx+x1,self.boty+y1)
        pendown()
        begin_fill()
        goto(self.botx+x2,self.boty+y2)
        goto(self.botx+x3,self.boty+y3)
        goto(self.botx+x1,self.boty+y1)
        end_fill()
        color("black")

    def forward(self):
        if self.botdir == "n":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.boty += 50
            self.drawBot(12,5,38,5,25,45,"blue")
        elif self.botdir == "e":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.botx += 50
            self.drawBot(5,12,5,38,45,25,"blue")
        elif self.botdir == "s":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.boty -= 50
            self.drawBot(12,45,25,5,38,45,"blue")
        elif self.botdir == "w":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.botx -= 50
            self.drawBot(45,12,5,25,45,38,"blue")

    def right(self):
        if self.botdir == "n":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(5,12,5,38,45,25,"blue")
            self.botdir = "e"
        elif self.botdir == "e":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(12,45,25,5,38,45,"blue")
            self.botdir = "s"
        elif self.botdir == "s":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(45,12,5,25,45,38,"blue")
            self.botdir = "w"
        elif self.botdir == "w":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(12,5,38,5,25,45,"blue")
            self.botdir = "n"

    def left(self):
        if self.botdir == "n":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(45,12,5,25,45,38,"blue")
            self.botdir = "w"
        elif self.botdir == "e":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(12,5,38,5,25,45,"blue")
            self.botdir = "s"
        elif self.botdir == "s":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(5,12,5,38,45,25,"blue")
            self.botdir = "e"
        elif self.botdir == "w":
            self.drawFilledBox(self.botx,self.boty,self.botx+50,self.boty+50,"white")
            self.drawBot(12,45,25,5,38,45,"blue")
            self.botdir = "n"

    def drawMaze(self):
        width = len(self.maze[0])
        height = len(self.maze)
        startX = -int((width*50)/2)
        startY = int((height*50)/2)

        for i in range(height):
            for j in range(width):
                xOffset = startX+(50*j)
                if self.maze[i][j] == 0:
                    self.drawFilledBox(xOffset,startY,xOffset+50,startY-50,"white")
                elif self.maze[i][j] == 1:
                    self.drawFilledBox(xOffset,startY,xOffset+50,startY-50,"black")
                elif self.maze[i][j] == 2:
                    self.drawFilledBox(xOffset,startY,xOffset+50,startY-50,"white")
                    self.currentGoal = [j,i]
                else:
                    self.botx = xOffset
                    self.boty = startY-50
                    self.left()
                self.drawBox(xOffset,startY,xOffset+50,startY-50)
                self.drawBox(xOffset,startY,xOffset+50,startY-50)
            startY -= 50

    #click event handler
    def updateMaze(self,x,y):
        inside = self.isInsideMaze(x,y)
        if inside and not self.moving:
            self.moving = True
            coords = self.findTileCoord(x,y)
            if self.maze[coords[1]][coords[0]] != 1 and [coords[0],coords[1]] != self.currentGoal:
                tracer(0,0)
                width = len(self.maze[0])
                height = len(self.maze)
                startX = -int((width*50)/2)

                startY = int((height*50)/2) - 50*self.currentGoal[1]
                xOffset = startX+50*self.currentGoal[0]
                self.drawFilledBox(xOffset,startY,xOffset+50,startY-50,"white")

                startY = int((height*50)/2) - 50*coords[1]
                xOffset = startX+50*coords[0]
                self.drawFilledBox(xOffset,startY,xOffset+50,startY-50,"gray")

                self.currentGoal = coords
                self.go()
            self.moving = False

    #detects if turtle coordinates are inside the tile grid
    def isInsideMaze(self,x,y):
        size = (((len(self.maze))%2))*25
        if x <= -len(self.maze[0]*25):
            return False
        if x >= len(self.maze[0]*25):
            return False
        if y >= len(self.maze*25):
            return False
        if y <= -len(self.maze*25):
            return False
        return True

    #converts board coordinates to tile grid coordinates
    def findTileCoord(self,x,y):
        size = (((len(self.maze))%2))*25        #deals with odd/even board size offset
        x = 50*math.floor((x+size)/50)          #floors turtle coordinates to multiple of 50
        y = 50*math.floor((y+size)/50)          #floors turtle coordinates to multiples of 50
        xCoord = int(x/50+len(self.maze)/2)     #converts turtle coordinates to tile grid coordinates
        yCoord = int(-(y/50-len(self.maze)/2))  #converts turtle coordinates to tile grid coordinates
        return [xCoord, yCoord]

    def go(self):
        botcoord = self.findTileCoord(self.botx,self.boty)
        self.solver.update(self.maze,[botcoord[0],botcoord[1]],self.currentGoal)
        for direction in self.solver.path():
            if direction != self.botdir:
                self.rotate(direction)
            self.forward()
            penup()
            goto(20000,20000)
            color("white")
            tracer(1,20)
            forward(50)
            tracer(0,0)
            pendown()
            color("black")

    def rotate(self,direction):
        possible = ["ne","es","sw","wn","nw","ws","se","en","ns","sn","we","ew"]
        if possible.index(self.botdir+direction) < 4:
            self.right()
        elif possible.index(self.botdir+direction) < 8:
            self.left()
        else:
            self.left()
            self.left()
        self.botdir = direction


import sys

def base_2(n):
    base = 2
    result = []
    aux = n

    if n > base-1:
        while aux != 0:
            if aux % base == 0:
                result.append(0)
            else:
                result.append(1)
            aux = int(aux / base)
        result.reverse()
    else:
        result.append(aux)

    return result

def base_16(n):
    base = 16
    map = ['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F']
    aux = n
    result = []
    if n > base-1:
        while aux != 0:
            if aux % base == 0:
                result.append(map[0])
            else:
                result.append(map[int(aux%base)])
            aux = int(aux / base)
    else:
        result.append(map[aux])

    result.reverse()
    return result

# main
# $ python3 ej7.py 26
# numero base 2: [1, 1, 0, 1, 0]
# numero base 16: ['1', 'A']

n = None
try:
    n = int(sys.argv[1])
except:
    print('ingresar parametros correctos!')

if n is not None:
    print('numero base 2: ' + str(base_2(n)))
    print('numero base 16: ' + str(base_16(n)))


from flask import Flask
import custom

app = Flask(__name__)

@app.route("/model", methods=["GET"])
def model():
	if request.method == "GET":
		url = request.args.get('url')
		image = url_to_image(url)
		custom.detect_and_color_splash(image)

def url_to_image(url):

	resp = urllib.request.urlopen(url)
	image = np.asarray(bytearray(resp.read()), dtype="uint8")
	image = cv2.imdecode(image, cv2.IMREAD_COLOR)

	return image

if __name__=="__main__":
	app.run(debug=True)


#Sher Khan
#insertion sort - just like sorting deck of cards with hands 1) pile sorted and second pile not sorted
#always compare to previous one elements in the deck.

def InsertionSort(array):
    length = len(array)
    #compare it with previous index, can not start at 0 because nothing previous to it.
    for index in range(1,length):
         j = index -1  #compare to the previous index
         value = array[index]  # value at index

         while(j>= 0 and array[j] > value):  #shifting elements till this condition holds
             array[j+1] = array[j]
             j = j -1

         array[j+1] = value    #correct position

array = [3,1,2,0,5]
InsertionSort(array)
print(array)


import asyncio
import socketio
import requests
from chatbot import messageBot

sio = socketio.AsyncClient()

@sio.event
async def connect():
    await sio.emit("currentRoom", 'personal')
    await sio.emit("chat message", ('hello','personal'))
    print('connection established')

@sio.event
async def my_message(data):
    print('message received with ', data)
    await sio.emit('my response', {'response': 'my response'})

@sio.on('chat message')
async def handleChatMessage(msg, UrlRoomId):
    if (msg['username'] != 'bot' and UrlRoomId == 'personal'):
        res = messageBot(msg['message'])
        await sio.emit("chat message", (res,'personal'))
        print(f'I received a message!{msg, UrlRoomId}')

@sio.event
async def disconnect():
    print('disconnected from server')

async def main():
    await sio.connect('http://localhost:3000')
    await sio.wait()

if __name__ == '__main__':
    # sending post request and saving response as response object
    r = requests.post(url='http://localhost:3000/user', json= {'username': 'bot'})
    # extracting response text
    pastebin_url = r.text
    print("The pastebin URL is:%s" % pastebin_url)
    asyncio.run(main())


import numpy as np
from advanced_buffer import get_reward_key
from state_encoder import unifying_transform_encode, state_2_buffer

actions = [
    {'idx':0, 'left':-1, 'move':'R1', 'sign':1},
    {'idx':0, 'left':1, 'move':'R1', 'sign':-1},
    {'idx':1, 'left':-1, 'move':'R1', 'sign':-1},
    {'idx':1, 'left':-1, 'move':'R1', 'sign':1},
    {'idx':1, 'left':1, 'move':'R1', 'sign':-1},
    {'idx':1, 'left':1, 'move':'R1', 'sign':1},
    {'idx':2, 'left':-1, 'move':'R1', 'sign':-1},
    {'idx':2, 'left':1, 'move':'R1', 'sign':1},
    {'left':-1, 'move':'R2', 'over_before_under':-1, 'over_idx':1, 'under_idx':0},
    {'left':-1, 'move':'R2', 'over_before_under':-1, 'over_idx':1, 'under_idx':1},
    {'left':-1, 'move':'R2', 'over_before_under':-1, 'over_idx':2, 'under_idx':0},
    {'left':-1, 'move':'R2', 'over_before_under':-1, 'over_idx':2, 'under_idx':1},
    {'left':-1, 'move':'R2', 'over_before_under':-1, 'over_idx':2, 'under_idx':2},
    {'left':-1, 'move':'R2', 'over_before_under':1, 'over_idx':0, 'under_idx':0},
    {'left':-1, 'move':'R2', 'over_before_under':1, 'over_idx':0, 'under_idx':1},
    {'left':-1, 'move':'R2', 'over_before_under':1, 'over_idx':0, 'under_idx':2},
    {'left':-1, 'move':'R2', 'over_before_under':1, 'over_idx':1, 'under_idx':1},
    {'left':-1, 'move':'R2', 'over_before_under':1, 'over_idx':1, 'under_idx':2},
    {'left':1, 'move':'R2', 'over_before_under':-1, 'over_idx':1, 'under_idx':0},
    {'left':1, 'move':'R2', 'over_before_under':-1, 'over_idx':1, 'under_idx':1},
    {'left':1, 'move':'R2', 'over_before_under':-1, 'over_idx':2, 'under_idx':0},
    {'left':1, 'move':'R2', 'over_before_under':-1, 'over_idx':2, 'under_idx':1},
    {'left':1, 'move':'R2', 'over_before_under':-1, 'over_idx':2, 'under_idx':2},
    {'left':1, 'move':'R2', 'over_before_under':1, 'over_idx':0, 'under_idx':0},
    {'left':1, 'move':'R2', 'over_before_under':1, 'over_idx':0, 'under_idx':1},
    {'left':1, 'move':'R2', 'over_before_under':1, 'over_idx':0, 'under_idx':2},
    {'left':1, 'move':'R2', 'over_before_under':1, 'over_idx':1, 'under_idx':1},
    {'left':1, 'move':'R2', 'over_before_under':1, 'over_idx':1, 'under_idx':2},
    {'move':'cross', 'over_idx':0, 'sign':-1, 'under_idx':1},
    {'move':'cross', 'over_idx':0, 'sign':-1, 'under_idx':2},
    {'move':'cross', 'over_idx':0, 'sign':1, 'under_idx':1},
    {'move':'cross', 'over_idx':0, 'sign':1, 'under_idx':2},
    {'move':'cross', 'over_idx':2, 'sign':-1, 'under_idx':0},
    {'move':'cross', 'over_idx':2, 'sign':-1, 'under_idx':1},
    {'move':'cross', 'over_idx':2, 'sign':1, 'under_idx':0},
    {'move':'cross', 'over_idx':2, 'sign':1, 'under_idx':1}
]

def hash_dict(abstract_action):
    tokens = [k+'-'+str(v) for k,v in abstract_action.items()]
    return '_'.join(sorted(tokens))

merged_buffers = {}
for Tac in actions:
    buffer_name = hash_dict(Tac) + '_buffer.npz'
    data = np.load(buffer_name)
    obs = data['obs']
    actions = data['actions']
    obs_u, actions_u = [], []
    over_seg_range_u, under_seg_range_u = [], []
    for ob, ac in zip(obs, actions):
        ob_u, ac_u, Tac_u, transform = unifying_transform_encode(ob, ac, Tac)
        ob_u, over_seg_u, under_seg_u = state_2_buffer(ob_u, Tac_u)
        obs_u.append(ob_u)
        actions_u.append(ac_u)
        over_seg_range_u.append(over_seg_u)
        under_seg_range_u.append(under_seg_u)
    reward_key = get_reward_key(Tac_u, obs_u[0])
    if reward_key in merged_buffers:
        merged_buffers[reward_key]['obs'].extend(obs_u)
        merged_buffers[reward_key]['actions'].extend(actions_u)
        merged_buffers[reward_key]['over_seg_range'].extend(over_seg_range_u)
        merged_buffers[reward_key]['under_seg_range'].extend(under_seg_range_u)
    else:
        merged_buffers[reward_key] = {
                                      'obs':obs_u, 'actions':actions_u,
                                      'over_seg_range':over_seg_range_u,
                                      'under_seg_range':under_seg_range_u
                                     }

for key, buffer in merged_buffers.items():
    save_name = key + '_init_buffer.npz'
    np.savez(save_name, **buffer)


import binascii
import csv
import json
import math
import os
import re
import pickle
from collections import defaultdict

from tqdm import tqdm
import numpy as np

import torch
from torch.utils.data import Dataset, random_split
from torch_geometric.data import Batch, Data, DataLoader, InMemoryDataset
from torch_geometric.utils import (
    add_remaining_self_loops,
    dense_to_sparse,
    remove_self_loops,
)

import sys

sys.path.append("..")
from models import BinaryModel
from .dataset import FuncDataset, preprocess_integer

from utils import ConfigParse, get_logger

device = "cuda" if torch.cuda.is_available() else "cpu"

model_file = "model.pt"

config_file = json.load(open("config.json", "r"))
config = ConfigParse(config_file)
model = BinaryModel(args=config).to(device)
model.load_state_dict(torch.load(model_file))

block_dataset = FuncDataset(
    args=config, max_len=100, asm_max_len=300, integer_max_len=20
)

def process_deep_block(code):
    byte, asm = "", ""
    for c in code:
        byte += c[0]
        asm += c[1]
    # process block asm, byte, integer
    data = {}
    data["asm"] = np.array([ord(i) for i in asm], dtype=np.uint8) + 1
    data["code"] = np.frombuffer(binascii.unhexlify(byte), dtype=np.uint8) + 1
    data["integer"] = preprocess_integer(asm)

    data["asm"] = block_dataset.process_asm(data["asm"]).to(device)
    data["code"] = block_dataset.process_code(data["code"]).to(device)
    data["integer"] = block_dataset.process_integer(data["integer"]).to(device)

    # model predict
    return data

def get_deep_acfg(block, code):
    block_code = defaultdict(list)
    bstart_index = [b[2] for b in block]

    start = 0
    try:
        for c in code:
            if c[0] < bstart_index[start]:
                block_code[start].append([c[1], c[2]])
            else:
                start += 1
                block_code[start].append([c[1], c[2]])
    except:
        pass

    acfg = []
    for idx, c in block_code.items():
        acfg.append(process_block(c))

    code_data, asm_data, integer_data = [], [], []
    for data in acfg:
        code_data.append(data["code"])
        asm_data.append(data["asm"])
        integer_data.append(data["integer"])

    code_data = torch.vstack(code_data)
    asm_data = torch.vstack(asm_data)
    integer_data = torch.vstack(integer_data)

    new_data = {
        "code": code_data,
        "asm": asm_data,
        "integer": integer_data,
    }

    with torch.no_grad():
        model.eval()
        x = model(new_data).detach().cpu().numpy()

    return x

data_regex = re.compile(
    "MOV|LDR|LDRB|LDRH|STR|STRB|STRH|MVN|PUSH|POP|XCHG|IN|OUT|XLAT|LEA|LDS|LES|LAHF|SAHF|PUSHF|POPF|\
move|movf|movt|movn|movz|mfhi|mflo|dla|la|dli|li|lw|sw|lh|lhu|sh|lb|lbu|sb|ll|sc|lui",
    re.IGNORECASE,
)

arithmetic_regex = re.compile(
    "ADD|ADC|SUB|SBC|CMP|CMN|MUL|MLA|UMULL|UMLAL|SMULL|SMLAL|\
INC|AAA|DAA|SUB|SBB|DEC|NEG|CMP|AAS|DAS|IMUL|AAM|DIV|IDIV|AAD|CBW|CWD|CWDE|CDQ|\
addi|abs|neg|div|mul|rem|mad|slt|slti|beg|bne|bgt|bge|blt|ble",
    re.IGNORECASE,
)

logic_regex = re.compile(
    "AND|ORR|EOR|BIC\
|OR|XOR|NOT|TEST|SHL|SAL|SHR|SAR|ROL|ROR|RCL|RCR\
|nor|andi|ori|sll|srl",
    re.IGNORECASE,
)

call_regex = re.compile("call", re.IGNORECASE)

jump_regex = re.compile(
    "B|BL|BX\
|JMP|RET|RETF|JA|JNBE|JAE|JNB|JB|JNAE|JBE|JNA|JG|JNLE|JGE|JNL|JL|JNGE|JLE|JNG|JE|JZ|JNE|JNZ|JC|JNC|JNO|JNP/JPO|JNS|JO|JP/JPE|JS|\
j|jr|jal",
    re.IGNORECASE,
)

def process_block(code):
    asm, opcode = "", ""
    for c in code:
        asm += c[1]
        opcode += c[1].split("\t")[0]
    data = (
        len(data_regex.findall(opcode)),
        len(arithmetic_regex.findall(opcode)),
        len(logic_regex.findall(opcode)),
        len(call_regex.findall(opcode)),
        len(jump_regex.findall(opcode)),
        len(code),
    )

    return data

def get_acfg(block, code):
    block_code = defaultdict(list)
    bstart_index = [b[2] for b in block]

    start = 0
    try:
        for c in code:
            if c[0] < bstart_index[start]:
                block_code[start].append([c[1], c[2]])
            else:
                start += 1
                block_code[start].append([c[1], c[2]])
    except:
        pass

    acfg = []
    for idx, c in block_code.items():
        acfg.append(process_block(c))

    return acfg

def process_graph(cfg, block, code):
    edges = cfg
    edges = np.array(edges, dtype=np.int64)

    num_of_nodes = edges.max() + 1
    node_feature_dim = 64

    features = get_acfg(block, code)
    if features.shape[0] < num_of_nodes:
        features = np.vstack(
            [features, np.ones((num_of_nodes - features.shape[0], node_feature_dim))]
        )
    x = torch.FloatTensor(features)

    edge_index = torch.from_numpy(edges.T).type(torch.long)

    data = Data(x=x, edge_index=edge_index)
    return data

class BinaryFuncDataset(InMemoryDataset):
    def __init__(
        self,
        func_file,
        train_group_file,
        root=None,
        transform=None,
        pre_transform=None,
        pre_filter=None,
    ):
        self.func_file = func_file
        self.train_group_file = train_group_file
        self.number_features = 64
        self.func2graph = dict()
        super().__init__(root, transform, pre_transform, pre_filter)
        self.func2graph, self.number_features = torch.load(self.processed_paths[0])

    @property
    def processed_file_names(self):
        return ["data.pt"]

    def get_group(self):
        file = self.train_group_file
        fid2group = {}
        self.all_pairs = []
        with open(file, "r") as f:
            reader = csv.reader(f)
            for i, row in enumerate(reader):
                group_id = row[0]
                group_fids = row[1:]
                for fid in group_fids:
                    fid2group[int(fid)] = int(group_id)
                if i >= 1000:
                    break

        return fid2group

    def process(self):
        fid2group = self.get_group()
        print("all fids: ", len(fid2group))
        with open(self.func_file, "r") as f:
            for i, line in tqdm(enumerate(f)):
                pline = json.loads(line)
                fid, cfg = pline["fid"], pline["cfg"]
                if len(cfg) < 1 or fid not in fid2group:
                    continue
                codes, blocks = pline["code"], pline["block"]
                group_id = fid2group[fid]
                data = process_graph(cfg=cfg, block=blocks, code=codes)

                if group_id in self.func2graph:
                    self.func2graph[group_id].append(data)
                else:
                    self.func2graph[group_id] = [data]

        torch.save((self.func2graph, self.number_features), self.processed_paths[0])

class GraphEmbeddingDataset(object):
    def __init__(self, args):
        self.args = args
        self.training_funcs = dict()
        self.validation_funcs = dict()
        self.testing_funcs = dict()
        self.number_features = None
        self.id2name = None
        self.func2graph = None
        self.process_dataset()

    def process_dataset(self):
        print("\nPreparing datasets.\n")
        self.dataset = BinaryFuncDataset(
            func_file="data/public/train.func.json",
            train_group_file="data/public/train.group.csv",
            root="data/",
        )
        self.number_features = self.dataset.number_features
        self.func2graph = self.dataset.func2graph
        self.id2name = dict()

        cnt = 0
        for k, v in self.func2graph.items():
            self.id2name[cnt] = k
            cnt += 1

        self.train_num = int(len(self.func2graph) * 0.8)
        self.val_num = int(len(self.func2graph) * 0.1)
        self.test_num = int(len(self.func2graph)) - (self.train_num + self.val_num)

        random_idx = np.random.permutation(len(self.func2graph))
        self.train_idx = random_idx[0 : self.train_num]
        self.val_idx = random_idx[self.train_num : self.train_num + self.val_num]
        self.test_idx = random_idx[self.train_num + self.val_num :]

        self.training_funcs = self.split_dataset(self.training_funcs, self.train_idx)
        self.validation_funcs = self.split_dataset(self.validation_funcs, self.val_idx)
        self.testing_funcs = self.split_dataset(self.testing_funcs, self.test_idx)

    def split_dataset(self, funcdict, idx):
        for i in idx:
            funcname = self.id2name[i]
            funcdict[funcname] = self.func2graph[funcname]
        return funcdict

    def collate(self, data_list):
        code_data, asm_data, graph_data, wide_data = [], [], [], []
        for data in data_list:
            for i in range(len(data)):
                graph_data.append(data[i])

        return Batch.from_data_list(graph_data)

    def create_batches(self, funcs, collate, shuffle_batch=True):
        data = FuncDataset(funcs)
        loader = torch.utils.data.DataLoader(
            data,
            batch_size=self.args.batch_size,
            shuffle=shuffle_batch,
            collate_fn=collate,
            num_workers=8,
            pin_memory=True,
        )

        return loader

class FuncDataset(Dataset):
    def __init__(self, funcdict):
        super(FuncDataset, self).__init__()
        self.funcdict = funcdict
        self.id2key = dict()  # idx to groupid
        cnt = 0
        for k, v in self.funcdict.items():
            self.id2key[cnt] = k
            cnt += 1

    def __len__(self):
        return len(self.funcdict)

    def get_graph_pair(self, graphset):
        count = 0
        while True:
            pos_idx = np.random.choice(range(len(graphset)), size=2, replace=True)
            origin_graph = graphset[pos_idx[0]]
            pos_graph = graphset[pos_idx[1]]
            # if cfg diff > 0.3*min, repeat sample
            if np.absolute(len(origin_graph) - len(pos_graph)) < int(
                0.3 * min(len(origin_graph), len(pos_graph))
            ):
                break
            # force quit
            count += 1
            if count > 10:
                break
        return origin_graph, pos_graph

    def __getitem__(self, idx):
        # idx to groupid, groupid to funcset
        graphset = self.funcdict[self.id2key[idx]]
        origin_graph, pos_graph = self.get_graph_pair(graphset)
        return origin_graph, pos_graph


#!/usr/bin/env python
'''
Classes to parse charset names
'''

#####################
# IMPORT OPERATIONS #
#####################

import GlobalVariables as GlobVars
import MyExceptions as ME

import sys

###############
# AUTHOR INFO #
###############

__author__ = 'Michael Gruenstaeudl <m.gruenstaeudl@fu-berlin.de>'
__copyright__ = 'Copyright (C) 2016-2017 Michael Gruenstaeudl'
__info__ = 'nex2embl'
__version__ = '2017.02.02.1100'

#############
# DEBUGGING #
#############

import pdb
#pdb.set_trace()

###########
# CLASSES #
###########

class GetEntrezInfo:
    ''' This class contains functions to obtain gene information from gene
    symbols. '''

    def __init__(self, email_addr):
        self.email_addr = email_addr

    @staticmethod
    def _id_lookup(gene_sym, retmax=10):
        ''' An internal static function to convert a gene symbol to an Entrez ID
            via ESearch.
        Args:
            gene_sym (str): a gene symbol; example: 'psbI'
        Returns:
            entrez_id_list (list): a list of Entrez IDs; example: ['26835430',
                            '26833718', '26833393', ...]
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> gene_sym = 'psbI'
#                >>> _id_lookup(gene_sym)
#                Out: ['26835430', '26833718', '26833393', ...]

        from Bio import Entrez
        if not gene_sym:
            raise ME.MyException('No gene symbol detected.')
        if '_' in gene_sym:
            raise ME.MyException('Gene symbol `%s` contains an '\
                'underscore, which is not allowed.' % (gene_sym))
        query_term = gene_sym + ' [sym]'
        try:
            esearch_records = Entrez.esearch(db='gene', term=query_term,
                retmax = retmax, retmod='xml')
        except:
            raise ME.MyException('An error occurred while retrieving '\
                'data from %s.' % ('ESearch'))
        parsed_records = Entrez.read(esearch_records)
        entrez_id_list = parsed_records['IdList']
        return entrez_id_list

    @staticmethod
    def _gene_product_lookup(entrez_id_list):
        ''' An internal static function to convert a list of Entrez IDs to a
        list of Entrez gene records via EPost and ESummary.
        Args:
            entrez_id_list (list): a list of Entrez IDs; example: ['26835430',
                                   '26833718', '26833393', ...]
        Returns:
            entrez_rec_list (list): a list of Entrez gene records
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> entrez_id_list = ['26835430', '26833718', '26833393']
#                >>> _record_lookup(entrez_id_list)
#                Out: ???

        from Bio import Entrez
        epost_query = Entrez.epost('gene', id=','.join(entrez_id_list))
        try:
            epost_results = Entrez.read(epost_query)
        except:
            raise ME.MyException('An error occurred while retrieving data from '\
                '%s.' % ('EPost'))
        webenv = epost_results['WebEnv']
        query_key = epost_results['QueryKey']
        try:
            esummary_records = Entrez.esummary(db='gene', webenv=webenv,
                query_key=query_key)
        except:
            raise ME.MyException('An error occurred while retrieving data from '\
                '%s.' % ('ESummary'))
        entrez_rec_list = Entrez.read(esummary_records)
        return entrez_rec_list

    @staticmethod
    def _parse_gene_products(entrez_rec_list):
        ''' An internal static function to parse out relevant information from .
        Args:
            entrez_rec_list (list): a list of Entrez gene records
        Returns:
            gene_info_list (list): a list of dictionaries
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> entrez_rec_list = []
#                >>> _parse_records(entrez_rec_list)

        from collections import Counter
        try:
            documentSummarySet = entrez_rec_list['DocumentSummarySet']
            docs = documentSummarySet['DocumentSummary']
        except:
            raise ME.MyException('An error occurred while parsing the '\
            'data from %s.' % ('ESummary'))
        list_gene_product = [doc['Description'] for doc in docs]
        #list_gene_symbol = [doc['NomenclatureSymbol'] for doc in docs]
        #list_gene_name = [doc['Name'] for doc in docs]

        # Avoiding that spurious first hit biases gene_product
        gene_product = Counter(list_gene_product).most_common()[0][0]
        return gene_product

    @staticmethod
    def _taxname_lookup(taxon_name, retmax=1):
        ''' An internal static function to look up a taxon name at NCBI
            Taxonomy via ESearch.
        Args:
            taxon_name (str): a taxon name; example: 'Pyrus tamamaschjanae'
            retmax (int):     the number of maximally retained hits
        Returns:
            entrez_hitcount (int): an integer
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> taxon_name = 'Pyrus tamamaschjanae'
#                >>> _taxname_lookup(taxon_name)
#                Out: 0

        from Bio import Entrez
        if not taxon_name:
            raise ME.MyException('No taxon name detected.')
        if '_' in taxon_name:
            raise ME.MyException('Taxon name `%s` contains an underscore, '
            'which is not allowed.' % (taxon_name))
        query_term = taxon_name
        try:
            esearch_records = Entrez.esearch(db='taxonomy', term=query_term,
                retmax=retmax, retmod='xml')
        except:
            raise ME.MyException('An error occurred while retrieving data from '\
                '%s.' % ('ESearch'))
        parsed_records = Entrez.read(esearch_records)
        entrez_hitcount = parsed_records['Count']
        return entrez_hitcount

    def obtain_gene_product(self, gene_sym):
        ''' This function performs something.
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> gene_sym = 'psbI'
#                >>> GetGeneInfo()._entrezid_lookup(gene_sym)
#                Out: ['26835430', '26833718', '26833393', ...]

        from Bio import Entrez
        Entrez.email = self.email_addr
        try:
            entrez_id_list = GetEntrezInfo._id_lookup(gene_sym)
        except ME.MyException as e:
            raise e
        try:
            entrez_rec_list = GetEntrezInfo._gene_product_lookup(entrez_id_list)
        except ME.MyException as e:
            raise e
        try:
            gene_product = GetEntrezInfo._parse_gene_products(entrez_rec_list)
        except ME.MyException as e:
            raise e
        return gene_product

    def does_taxon_exist(self, taxon_name):
        ''' This function calls _taxname_lookup and thus evaluates if a taxon exists.
        Args:
            taxon_name (str): a taxon name; example: 'Pyrus tamamaschjanae'
            retmax (int):     the number of maximally retained hits
        Returns:
            entrez_id_list (list): a list of Entrez IDs; example: ['26835430',
                            '26833718', '26833393', ...]
        Raises:
            none
        '''
        from Bio import Entrez
        Entrez.email = self.email_addr
        try:
            entrez_hitcount = GetEntrezInfo._taxname_lookup(taxon_name)
        except ME.MyException as e:
            raise e
        if entrez_hitcount == '0':
            return False
        if entrez_hitcount == '1':
            return True

class ConfirmAdjustTaxonName:
    ''' This class contains functions to confirm or adjust a sequence's
    taxon name.
    '''

    def __init__(self):
        pass

    def go(self, seq_record, email_addr):
        ''' This function evaluates a taxon name against NCBI taxonomy;
            if not listed, it adjusts the taxon name and appends it
            as ecotype info.
            Args:
                seq_record (obj):   a seqRecord object
                email_addr (dict):  your email address; example:
                                    "m.gruenstaeudl@fu-berlin.de"
            Returns:
                seq_record (obj):   a seqRecord object
            Raises:
                currently nothing
        '''
        try:
            genus_name, specific_epithet = seq_record.name.split(' ', 1)
        except ME.MyException as e:
            sys.exit('%s annonex2embl ERROR: Could not locate a '\
                'whitespace between genus name and specific epithet '\
                'in taxon name of sequence `%s`.' % ('\n', seq_record.id))
        if not GetEntrezInfo(email_addr).does_taxon_exist(seq_record.name):
            print('%s annonex2embl WARNING: Taxon name of sequence `%s` '\
                'not found in NCBI Taxonomy: `%s`. Please consider sending '\
                'a taxon request to ENA.'
                % ('\n', seq_record.id, seq_record.name))
            if not GetEntrezInfo(email_addr).does_taxon_exist(genus_name):
                sys.exit('%s annonex2embl ERROR: Neither genus name, '\
                    'nor species name of sequence `%s` were found in '\
                    'NCBI Taxonomy.' % ('\n', seq_record.id))
            else:
                species_name_original = seq_record.name
                species_name_new = genus_name + ' sp. ' + specific_epithet
                seq_record.name = species_name_new
                seq_record.features[0].qualifiers['organism'] = species_name_new
                seq_record.description = seq_record.description.\
                    replace(species_name_original, species_name_new)
                print('%s annonex2embl WARNING: Taxon name of sequence '\
                      '`%s` converted to the informal name: `%s`'
                      % ('\n', seq_record.id, species_name_new))
        return seq_record

class ParseCharsetName:
    ''' This class contains functions to parse charset names.
    Args:
        charset_name (str): a string that represents a charset name; example:
                            "psbI_CDS"
        email_addr (dict):  your email address; example:
                            "m.gruenstaeudl@fu-berlin.de"
    Raises:
        currently nothing
    '''

    def __init__(self, charset_name, email_addr):
        self.charset_name = charset_name
        self.email_addr = email_addr

    @staticmethod
    def _extract_charset_type(charset_name):
        ''' An internal static function to extract the charset type from a
        string. '''
        fk_present = [fk for fk in GlobVars.nex2ena_valid_INSDC_featurekeys
            if fk in charset_name]
        if not fk_present:
            raise ME.MyException('%s annonex2embl ERROR: No feature '\
            'key encountered in the name of charset `%s`.' % ('\n',
            charset_name))
        if len(fk_present) > 1:
            raise ME.MyException('%s annonex2embl ERROR: More than '\
            'one feature key encountered in the name of charset '\
            '`%s`.' % ('\n', charset_name))
        charset_type = fk_present[0]
        return charset_type

    @staticmethod
    def _extract_charset_sym(charset_name, charset_type):
        ''' An internal static function to extract the charset symbol from a
        string. '''
        try:
            charset_sym = charset_name.strip(charset_type)
        except:
            raise ME.MyException('%s annonex2embl ERROR: No charset '\
            'symbol encountered in the name of charset `%s`.' % (
            '\n', charset_name))
        charset_sym = charset_sym.strip('_')
        charset_sym = charset_sym.rstrip('_') # Remove trailing underscores
        return charset_sym

    def parse(self):
        ''' This function parses the charset_name.
        Returns:
            tupl.   The return consists of three strings in the order
                    "charset_sym, charset_type, charset_product"
        '''
        try:
            charset_type = ParseCharsetName._extract_charset_type(\
                self.charset_name)
        except ME.MyException as e:
            raise e
        try:
            charset_sym = ParseCharsetName._extract_charset_sym(\
                self.charset_name, charset_type)
        except ME.MyException as e:
            raise e
        entrez_handle = GetEntrezInfo(self.email_addr)
        if charset_type == 'CDS' or charset_type == 'gene':
            try:
                charset_product = entrez_handle.obtain_gene_product(\
                    charset_sym)
            except ME.MyException as e:
                raise e
        else:
            charset_product = None
        return (charset_sym, charset_type, charset_product)

#############
# FUNCTIONS #
#############

########
# MAIN #
########


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 13 01:52:40 2021

@author: dileepn

Plot of a hyperboloid: isobar in an ideal vortex
"""

# Preliminaries
import numpy as np
import matplotlib.pyplot as plt

# Radius of vessel
R = 1.0
r1 = np.linspace(0.1,R,100)
r2 = np.linspace(-R,-0.1,100)

# Free surface (air-water interface height)
H0 = 2.

# v(theta component) = C/r
C = 5.

# Acceleration due to gravity
g = 9.805

# Isobar curve
z1 = H0 + (C**2/(2*g))*(1/R**2 - 1/r1**2)
z2 = H0 + (C**2/(2*g))*(1/R**2 - 1/r2**2)

# Plot
plt.figure(figsize=(12,12))
plt.plot(r1,z1,'b',linewidth=2)
plt.plot(r2,z2,'b',linewidth=2)
plt.grid(axis="both")
plt.show()


from typing import TypeVar

INITIAL_SIZE = 0
MIN_INITIAL_CAPACITY = 1
MIN_REAL_CAPACITY = 16
GROWTH_FACTOR = 2
SHRINK_FACTOR = 4

INDEX_OUT_OF_RANGE_ERROR = "Index out of range."
NOT_MATCH_VALUE_FOUND_ERROR = "No match value found."

T = TypeVar("T", int, str, None)

class Array:
    def __init__(self, capacity: int = MIN_INITIAL_CAPACITY) -> None:
        """
        Creates a new array according to the provided capacity.

        About: An array is a contiguous area of memory consisting of equal-size
        elements indexed by contiguous integers.

        - Complexity (AVERAGE and WORST):
            * Access: O(1)
            * Search: O(n)
            * Insertion: O(n)
            * Deletion: O(n)
        """
        self._size: int = INITIAL_SIZE
        self._capacity: int = self._set_capacity(capacity)
        self.data: list[T] = []

    def __iter__(self) -> T:
        return iter(self.data)

    def __getitem__(self, index: int) -> T:
        """
        Returns item at given index.

        - Complexity: O(1)

        Args:
            index: Index where the item is found.

        Raises:
            - IndexError: If index out of range.
            - ValueError: If the index is not an Integer.

        Return:
            T: The found item
        """
        self._validate_index_type(index)
        if index == -1:
            return self._get_last_item()

        for i, value in enumerate(self.data):
            if i == index:
                return value
        raise IndexError(INDEX_OUT_OF_RANGE_ERROR)

    def append(self, value: T) -> None:
        """
        Add an item to the end of the list.

        - Complexity: O(1)

        Args:
            value: The value that will be add.
        """
        self._upsize_capacity()
        if self.is_empty():
            self.data = [value]
        else:
            self.data += [value]
        self._size += 1

    def insert(self, index: int, value: T) -> None:
        """
        Insert an item at a given position.

            * [index = 0]: Insert the item at the beginning - Complexity: O(n);\n
            * [index > current size or index < 0]: Insert the item at the end
                (is equivalent to append(value)) - Complexity: O(1);\n
            * [index < current size or any other numbers]: Insert the item among
                other elements - Complexity: O(n).
        --------

        Args:
            index: The index where you want to insert the value.
            value: The value that will be insert.

        Raises:
            ValueError: If the index is not an Integer
        """
        self._validate_index_type(index)

        self._upsize_capacity()
        if index == INITIAL_SIZE:  # Insert the element at the beginning
            shifted_array = [value]
            for item in self.data:
                shifted_array += [item]
            self.data = shifted_array
            self._size += 1
        elif (
            index > self._size or index < INITIAL_SIZE
        ):  # Insert the element at the end
            self.append(value)
        else:  # Insert the element among other elements
            left_content = self.data[:index]
            right_content = self.data[index:]

            left_content += [value]
            for item in right_content:
                left_content += [item]

            self.data = left_content
            self._size += 1

    def pop(self, index: int = None) -> T:
        """
        Remove a value at the end, and return it.

        - Complexity:
            * O(1) if the index is at the end.
            * O(n) from specific index.

        Args:
            index: Index where the item is found.

        Raises:
            - IndexError:
                * If there is nothing in the array;
                * And if index is greater than size.
            - ValueError: If the index is not an integer.

        Return:
            T: The removed value
        """
        if index is not None:
            self._validate_index_type(index)

        if self.is_empty() or index is not None and index > self._size:
            raise IndexError(INDEX_OUT_OF_RANGE_ERROR)

        target_value: T = None
        if index is None or index < INITIAL_SIZE or index == self._size - 1:
            self._downsize_capacity()
            target_value = self.data[-1]
            del self.data[-1]
            self._size -= 1
        else:
            # Searching...
            for value in self.data:
                if value == self.data[index]:
                    target_value = value
                    self._downsize_capacity()
                    self._reorganize(index)

                    del self.data[-1]
                    self._size -= 1
                    break
            else:
                raise IndexError(INDEX_OUT_OF_RANGE_ERROR)

        return target_value

    def remove(self, target_value: T) -> None:
        """
        Remove the first item from the array that is equal to the target value.

        Complexity: O(n)

        Args:
            target_value: value you want to remove.

        Raises:
            ValueError:
                * if no matching value is found.
        """
        index_found = self.find(target_value)
        self._downsize_capacity()
        self._reorganize(index_found)
        del self.data[-1]
        self._size -= 1

    def remove_all(self, target_values: list[T]) -> None:
        """
        Remove values from the array that are equal to the list of target values.

        Complexity: O(n)

        Args:
            target_values: list of values you want to remove.

        Raises:
            ValueError: If no matching value is found.
        """
        self._downsize_capacity()
        for target in target_values:
            index_found = self.find(target)
            self._reorganize(index_found)
            del self.data[-1]
            self._size -= 1

    def find(self, value: T) -> int:
        """
        Search for the given value and returns its index.

        - Complexity: O(n)

        Args:
            value: value you want to find.

        Raises:
            ValueError: If the value is not found.
        """
        for index, item in enumerate(self.data):
            if item == value:
                break
        else:
            raise ValueError(NOT_MATCH_VALUE_FOUND_ERROR)
        return index

    def is_empty(self) -> bool:
        """Check if the array is empty."""
        return self._size == INITIAL_SIZE

    def size(self) -> int:
        return self._size

    def capacity(self) -> int:
        return self._capacity

    def _reorganize(self, target_index: int) -> None:
        for i in range(target_index, self._size - 1):
            self.data[i] = self.data[i + 1]

    def _get_last_item(self):
        return self.data[::-1][0]

    @staticmethod
    def _set_capacity(capacity: int) -> int:
        """
        Arrays will be created with a minimium of 16 bits (2 bytes) and
        a growth factor of 2.

        Examples:
            * capacity = 8 ->  Real capacity: 16 bits (2 bytes);
            * capacity = 9 ->  Real capacity: 32 bits (4 bytes);
            * capacity = 17 -> Real capacity: 64 bits (8 bytes);
            * ....
        """
        if capacity < MIN_INITIAL_CAPACITY:
            raise ValueError(
                f"The minimium initial capacity must be greater than or equal to {MIN_INITIAL_CAPACITY}"
            )

        real_capacity = MIN_REAL_CAPACITY
        while capacity > real_capacity // GROWTH_FACTOR:
            real_capacity *= GROWTH_FACTOR

        return real_capacity

    def _upsize_capacity(self) -> None:
        if self._size == self._capacity:
            old_capacity = self._capacity
            self._capacity = self._set_capacity(old_capacity)

    def _downsize_capacity(self) -> None:
        if self._size < self._capacity // SHRINK_FACTOR:
            old_capacity = self._capacity
            new_capacity = self._capacity // GROWTH_FACTOR

            if new_capacity < MIN_REAL_CAPACITY:
                new_capacity = MIN_REAL_CAPACITY

            if new_capacity != old_capacity:
                self._capacity = new_capacity

    @staticmethod
    def _validate_index_type(index: int) -> None:
        if not isinstance(index, int):
            raise ValueError("Index must be a integer!")


import pandas as pd
import openpyxl

class ErrorAnalysis:

    def __init__(self, path, expected_sheets, expected_columns):

        self.excel_path: str = path
        self.expected_sheets: list = expected_sheets
        self.expected_columns: dict = expected_columns
        self.customers: pd.DataFrame = pd.read_excel(self.excel_path, sheet_name='Customers')
        self.locations: pd.DataFrame = pd.read_excel(self.excel_path, sheet_name='Locations')
        self.suppliers: pd.DataFrame = pd.read_excel(self.excel_path, sheet_name='Suppliers')
        self.customers_names: list = self.customers['Name'].values.tolist()
        self.locations_names: list = self.locations['Name'].values.tolist()
        self.suppliers_names: list = self.suppliers['Name'].values.tolist()

    def sheet_analysis(self, alerts, errors) -> (list, list):

        wb = openpyxl.load_workbook(self.excel_path)
        sheet_names = wb.sheetnames

        alerts += [f'Table {sheet} is not a expected table'
                   for sheet in sheet_names if sheet not in self.expected_sheets]

        errors += [f'Table {sheet} is not in archive tables'
                   for sheet in self.expected_sheets if sheet not in sheet_names]

        return alerts, errors

    def columns_analysis(self, alerts, errors) -> (list, list):

        customer_columns = list(self.customers)
        location_columns = list(self.locations)
        suppliers_columns = list(self.suppliers)

        errors += [f'There is no {column} column in Customers table'
                   for column in self.expected_columns['Customers'] if column not in customer_columns]
        errors += [f'There is no {column} column in suppliers table'
                   for column in self.expected_columns['Suppliers'] if column not in suppliers_columns]
        errors += [f'There is no {column} column in locations table'
                   for column in self.expected_columns['Locations'] if column not in location_columns]

        alerts += [f'Column {column} is not a expected column in customers table'
                   for column in customer_columns if column not in self.expected_columns['Customers']]
        alerts += [f'Column {column} is not a expected column in suppliers table'
                   for column in suppliers_columns if column not in self.expected_columns['Suppliers']]
        alerts += [f'Column {column} is not a expected column in locations table'
                   for column in location_columns if column not in self.expected_columns['Locations']]

        return alerts, errors

    def check_duplicates(self, alerts) -> (list, list):

        duplicates = self.suppliers['Name'].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.suppliers['Name'][i]} is duplicated in Suppliers"
                      for i in range(len(self.suppliers_names)) if duplicates[i]]
            self.suppliers.drop_duplicates('Name', inplace=True)

        duplicates = self.locations[['Name', 'Status']].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.locations['Name'][i]} is duplicated with the same status in Locations"
                      for i in range(len(self.locations_names)) if duplicates[i]]
            self.locations.drop_duplicates(['Name', 'Status'], inplace=True)

        duplicates = self.customers['Name'].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.customers['Name'][i]} is duplicated in Customers"
                      for i in range(len(self.customers_names)) if duplicates[i]]
            self.customers.drop_duplicates('Name', inplace=True)

        duplicates = self.locations['Name'].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.locations['Name'][i]} is duplicated in Locations"
                      for i in range(len(self.locations_names)) if duplicates[i]]
            self.locations.drop_duplicates('Name', inplace=True)

        return alerts

    def check_names_inconsistency(self, alerts, errors) -> (list, list):

        self.customers.set_index('Name', inplace=True, drop=False)
        self.locations.set_index('Name', inplace=True, drop=False)
        self.suppliers.set_index('Name', inplace=True, drop=False)

        errors += [f'Name {name} in Customers does not exist in Locations table'
                   for name in self.customers_names if name not in self.locations_names]
        errors += [f'Name {name} in Suppliers does not exist in Locations table'
                   for name in self.suppliers_names if name not in self.locations_names]

        if errors:
            return alerts, errors

        errors += [f'Status divergence in the name {name} between tables Customers and Locations'
                   for name in self.customers_names if self.customers['Status'][name] != self.locations['Status'][name]]

        errors += [f'Status divergence in the name {name} between tables Suppliers and Locations'
                   for name in self.suppliers_names if self.suppliers['Status'][name] != self.locations['Status'][name]]

        return alerts, errors

    def check_supply_and_demand_inconsistency(self, alerts, errors) -> (list, list):

        errors += [f' Negative value for Maximum Supply from the supplier {name}'
                  for name in self.suppliers_names if self.suppliers['Maximum Supply'][name] < 0]
        errors += [f' Negative value for Fixed Demand from the customer {name}'
                  for name in self.customers_names if self.customers['Fixed Demand'][name] < 0]

        alerts += [f' Null value for Maximum Supply from the supplier {name}'
                  for name in self.suppliers_names if self.suppliers['Maximum Supply'][name] == 0]
        alerts += [f' Null value for Fixed Demand from the customer {name}'
                  for name in self.customers_names if self.customers['Fixed Demand'][name] == 0]

        return alerts, errors

    def check_status_inconsistency(self, alerts, errors) -> (list, list):

        if (self.suppliers['Status'] == 'Exclude').all():
            alerts += ['All values for status in the table suppliers are Exclude']

        if (self.customers['Status'] == 'Exclude').all():
            alerts += ['All values for status in the table customers are Exclude']

        if (self.locations['Status'] == 'Exclude').all():
            alerts += ['All values for status in the table locations are Exclude']

        errors += [f'Status {status} in Locations is not supported, status must be Exclude or Include'
                   for status in self.locations['Status'].values.tolist() if status not in ['Exclude', 'Include']]

        errors += [f'Status {status} in Suppliers is not supported, status must be Exclude or Include'
                   for status in self.suppliers['Status'].values.tolist() if status not in ['Exclude', 'Include']]

        errors += [f'Status {status} in Customers is not supported, status must be Exclude or Include'
                   for status in self.customers['Status'].values.tolist() if status not in ['Exclude', 'Include']]

        return alerts, errors

    def check_coordenates_inconsistency(self, alerts, errors) -> (list, list):

        alerts += [f'Location {name} latitude is not in the correct format' for name in self.locations_names
                   if abs(self.locations['Latitude'][name]) > 90]

        alerts += [f'Location {name} longitude is not in the correct format' for name in self.locations_names
                   if abs(self.locations['Longitude'][name]) > 180]

        return alerts, errors

    def errors_search(self) -> (list, list):

        alerts = []
        errors = []

        alerts, errors = self.sheet_analysis(alerts, errors)

        if errors:
            return alerts, errors

        alerts, errors = self.columns_analysis(alerts, errors)

        if errors:
            return alerts, errors

        alerts = self.check_duplicates(alerts)

        alerts, errors = self.check_names_inconsistency(alerts, errors)

        if errors:
            return alerts, errors

        alerts, errors = self.check_supply_and_demand_inconsistency(alerts, errors)

        alerts, errors = self.check_coordenates_inconsistency(alerts, errors)

        alerts, errors = self.check_supply_and_demand_inconsistency(alerts, errors)

        return alerts, errors


#!/usr/bin/env python

import sys
import json
import time
import serial

try:
    import mosquitto
except ImportError:
    import paho.mqtt.client as mosquitto

class MQTTClient(mosquitto.Mosquitto):
    def publish_utf8(self, topic, payload, *args, **kwargs):
        if isinstance(topic, unicode):
            topic = topic.encode('utf8', 'ignore')
        if isinstance(payload, unicode):
            payload = payload.encode('utf8', 'ignore')

        self.publish(topic, payload, *args, **kwargs)

class TMHZ19(object):
    REQ_READ_CONCENTRATION = "\xFF\x01\x86\x00\x00\x00\x00\x00\x79"

    class TCommunicationError(Exception):
        pass

    def __init__(self, port, timeout = 1):
        self.port = port
        self.timeout = timeout
        self.ser = serial.Serial(port, baudrate = 9600, timeout = self.timeout)

    def send_request(self, request):
        try:
            self.ser.write(request)
        except serial.SerialTimeoutException:
            raise self.TCommunicationError("timeout while sending request")

    def read_concentration(self):
        self.send_request(self.REQ_READ_CONCENTRATION)
        payload = self.read_response()
        return payload[1] * 256 + payload[2]

    def read_response(self):
        resp_len = 9
        resp = self.ser.read(resp_len)
        if len(resp) != resp_len:
            raise self.TCommunicationError("expected %d bytes, got %d" % (resp_len, len(resp)))

        payload = [ord(x) for x in resp[1:-1]]
        crc16 =((sum(payload) % 256) ^ 0xFF) + 1
        if crc16 != ord(resp[-1]):
            raise self.TCommunicationError("checksum error")

        return payload

class TMQTTMHZ19Sensor(object):
    def init_config(self, fname):
        self.config = json.load(open(fname))

        self.config.setdefault('mqtt_id', 'mh-z19')
        self.config.setdefault('name', 'MHZ19 sensor')
        self.config.setdefault('debug', False)
        self.config.setdefault('port', '/dev/ttyAPP2')
        self.config.setdefault('poll_interval', 10)
        if self.config['poll_interval'] < 6:
            raise RuntimeError("poll interval cannot be smaller than 6 seconds")

    def init_sensor(self):
        self.mhz19 = TMHZ19(self.config['port'])

    def init_mqtt(self):
        self.mqtt_client = MQTTClient()
        self.mqtt_client.connect('localhost')
        self.mqtt_client.loop_start()

        self.mqtt_client.publish_utf8("/devices/%s/meta/name" % self.config['mqtt_id'], self.config['name'], qos=1, retain=True)

        self.mqtt_client.publish_utf8("/devices/%s/controls/%s/meta/type" %
                                      (self.config['mqtt_id'], "co2"), "concentration", qos=1, retain=True)

    def set_channel_error(self, control_id, error):
        if self.error_cache.get(control_id) != error:
            self.mqtt_client.publish("/devices/%s/controls/%s/meta/error" %
                                     (self.config['mqtt_id'], control_id), error, qos=1, retain=1)
        self.error_cache[control_id] = error

    def publish_channel(self, control_id, value, decimal_places=1):
        self.set_channel_error(control_id, "")
        format = "%%.%df" % decimal_places
        value_str = format % value
        self.mqtt_client.publish_utf8("/devices/%s/controls/%s" %
                                      (self.config['mqtt_id'], control_id), value_str, qos=1, retain=True)

    def __init__(self, fname):
        self.fname = fname
        self.error_cache = {}

    def start(self):
        self.init_config(self.fname)
        self.init_sensor()
        self.init_mqtt()

        while True:
            try:
                co2_ppm = self.mhz19.read_concentration()
            except TMHZ19.TCommunicationError:
                self.set_channel_error('co2', "r")
            else:
                self.publish_channel('co2', co2_ppm)

            time.sleep(self.config['poll_interval'])

        return 0

def main():
    if len(sys.argv) != 2:
        print >>sys.stderr, "USAGE: %s <config file>"
        return 1

    sensor = TMQTTMHZ19Sensor(sys.argv[1])
    sensor.start()

    return 0

if __name__ == '__main__':
    main()


import itertools
import logging
import os
import numpy as np

import pytest
import cocotb_test.simulator

import cocotb
from cocotb.clock import Clock
from cocotb.triggers import RisingEdge, FallingEdge
from cocotb.regression import TestFactory

from cocotbext.axi import AxiStreamBus, AxiStreamFrame, AxiStreamSource, AxiStreamSink
from cocotbext.axi import AxiLiteBus, AxiLiteMaster, AxiLiteRam

class TB:
    def __init__(self, dut):
        self.dut = dut
        #self.signal = []

        self.log = logging.getLogger("cocotb.tb")
        self.log.setLevel(logging.DEBUG)

        cocotb.cocotb.start_soon(Clock(dut.CLK100MHz, 10,units="ns").start())

        self.axil_master = AxiLiteMaster(AxiLiteBus.from_prefix(dut, "s_axi"), dut.CLK100MHz, dut.ARESETN,reset_active_level=False)

        self.simulation_time = int(40)
        self.serial_clk_div = 33

    async def reset(self):
        self.dut.ARESETN.setimmediatevalue(0)
        self.dut.i_data_rx.setimmediatevalue(0)

        self.dut.ARESETN.value = 0
        await RisingEdge(self.dut.CLK100MHz)
        await RisingEdge(self.dut.CLK100MHz)
        self.dut.ARESETN.value = 1

    async def delay(self, clocks):
        for i in range(clocks):
            await RisingEdge(self.dut.CLK100MHz)

    async def write_clk(self):
        for j in range(self.simulation_time):
            for i in range(36):
                await self.delay(self.serial_clk_div)
                if i < 4:
                    self.dut.i_data_rx[16].value = 1
                else:
                    self.dut.i_data_rx[16].value = 0

    async def write_enable(self):
        for j in range(self.simulation_time):
            for i in range(36):
                await self.delay(self.serial_clk_div)
                if i < 34:
                    if (i % 2) == 0:
                        self.dut.i_data_rx[17].value = 0
                    else:
                        self.dut.i_data_rx[17].value = 1
                else:
                    self.dut.i_data_rx[17].value = 0

    async def write(self, data, size, pos):
        for j in range(self.simulation_time):
            # self.dut.i_data_rx[pos].value  = 0
            await self.delay(self.serial_clk_div)
            await self.delay(self.serial_clk_div)
            # print("====================================================")
            for i in range(size+1):
                value = data

                self.dut.i_data_rx[pos].value        = data & 1 << i != 0
                # print(self.dut.i_data_rx[pos].value )
                await self.delay(self.serial_clk_div)
                await self.delay(self.serial_clk_div)

    async def write_to_axi_lite(self,addr, value):

        try:
            send_data = value.to_bytes(4, 'little') #bytearray(value)
            self.log.info("Writed: %s", send_data)
            await self.axil_master.write(addr*4, send_data)

            # self.log.info("Writed: %s", data.data)
        except:
            print("An exception occurred")

    async def read_from_axi_lite(self,addr):

        self.log.info("Reading data")
        try:

            await RisingEdge(self.dut.CLK100MHz)
            data = await self.axil_master.read(addr, 4)

            self.log.info("Writed: %s", data.data)
        except:
            print("An exception occurred")

    async def read_loop(self):
        for j in range(self.simulation_time):
            # self.dut.i_data_rx[pos].value  = 0
            await self.delay(200)
            # await self.read_from_axi_lite(0) # var1
            await self.write_to_axi_lite(8, j) # var2

async def run_test(dut):

    tb = TB(dut)

    await tb.reset()

    # for i in range(4095):
    #     await tb.write(i, 16)
    write_thread_0 = cocotb.start_soon(tb.write_clk())
    write_thread_1 = cocotb.start_soon(tb.write_enable())

    write_thread_2 = cocotb.start_soon(tb.write(1001,16,0))
    write_thread_3 = cocotb.start_soon(tb.write(1002,16,1))
    write_thread_4 = cocotb.start_soon(tb.write(1003,16,2))
    write_thread_5 = cocotb.start_soon(tb.write(1004,16,3))
    write_thread_6 = cocotb.start_soon(tb.write(1005,16,4))
    write_thread_7 = cocotb.start_soon(tb.write(1006,16,5))
    write_thread_8 = cocotb.start_soon(tb.write(1007,16,6))
    write_thread_9 = cocotb.start_soon(tb.write(1008,16,7))
    write_thread_10 = cocotb.start_soon(tb.write(1009,16,8))
    write_thread_11 = cocotb.start_soon(tb.write(1010,16,9))
    write_thread_12 = cocotb.start_soon(tb.write(1011,16,10))
    write_thread_13 = cocotb.start_soon(tb.write(1012,16,11))
    write_thread_14 = cocotb.start_soon(tb.write(1013,16,12))
    write_thread_15 = cocotb.start_soon(tb.write(1014,16,13))
    write_thread_16 = cocotb.start_soon(tb.write(1015,16,14))
    write_thread_17 = cocotb.start_soon(tb.write(1016,16,15))

    read_thread_0 = cocotb.start_soon(tb.read_loop())

    assert 1 == 1
    await RisingEdge(dut.CLK100MHz)
    await RisingEdge(dut.CLK100MHz)

    await write_thread_0
    await write_thread_1
    await write_thread_2
    await write_thread_3
    await write_thread_4
    await write_thread_5
    await write_thread_6
    await write_thread_7
    await write_thread_8
    await write_thread_9
    await write_thread_10
    await write_thread_11
    await write_thread_12
    await write_thread_13
    await write_thread_14
    await write_thread_15
    await write_thread_16
    await write_thread_17
    await read_thread_0

if cocotb.SIM_NAME:
    factory = TestFactory(run_test)
    factory.generate_tests()

# cocotb-test

tests_dir = os.path.abspath(os.path.dirname(__file__))
hdl_dir = os.path.abspath(os.path.join(tests_dir, '..',  'hdl'))

@pytest.mark.parametrize("data_width", [12])
def test_opal_com_wrapper(request, data_width):
    dut = "opal_com_wrapper"
    module = os.path.splitext(os.path.basename(__file__))[0]
    toplevel = dut

    verilog_sources = [
        os.path.join(hdl_dir, f"{dut}.v"),
    ]

    parameters = {}

    parameters['ADC_DATA_WIDTH'] = data_width

    extra_env = {f'PARAM_{k}': str(v) for k, v in parameters.items()}

    sim_build = os.path.join(tests_dir, "sim_build",
        request.node.name.replace('[', '-').replace(']', ''))

    cocotb_test.simulator.run(
        python_search=[tests_dir],
        verilog_sources=verilog_sources,
        toplevel=toplevel,
        module=module,
        parameters=parameters,
        sim_build=sim_build,
        extra_env=extra_env,
    )


from tkinter import *
from tkinter import ttk
from time    import strftime
from PIL     import Image , ImageTk
from io      import BytesIO
import requests

##########################
# Global Variables
##########################
API_key_file   = 'weatherapi.key'
weatherAPI_key = None # Variable to store the API Key
imageTk_icon   = None # Variable for ImageTk icon

##########################
# Common functions
##########################

# Clear root GUI window
def clear_root( root:object ):
    for frame in root.winfo_children():
        frame.destroy()

##########################
# Help frame
##########################

def close_help_form( root:object ):
    clear_root( root )    # Clear the root window
    forecast_form( root ) # Show the forecast form

# Help form will be called by a help button to show query examples
def help_form( root:object ):
    clear_root( root )
    global imageTk_icon
    instructions = '''

For detailed documentation visit:

            www.weatherapi.com

For quick reference, see these query examples:
Latitude and Longitude (Decimal degree) e.g: 48.8567,2.3508
city name e.g.: Paris
US zip e.g.: 10001
UK postcode e.g: SW1
Canada postal code e.g: G2J
metar:<metar code> e.g: metar:EGLL
iata:<3 digit airport code> e.g: iata:DXB
auto:ip IP lookup e.g: auto:ip
IP address (IPv4 and IPv6 supported) e.g: 100.0.0.1

'''
    frm = ttk.Frame( root , padding=5 )
    frm.grid()
    # Row 1 - Logo
    image_icon   = Image.open( 'logo.png' )
    imageTk_icon = ImageTk.PhotoImage( image_icon )
    ttk.Label( frm , image=imageTk_icon ).grid( row=0 , column=0 )
    # Row 2 - Instructions
    ttk.Label( frm , text=instructions ).grid( row=1 , column=0 )
    # Row 3 - Close Button
    ttk.Button( frm, text="Close",
        command = lambda:
            close_help_form( root )
    ).grid( row=2 , column=0 )

########################################
# Weather forecast frame (main frame)
########################################

# Retrive data from API and update form widgets
def fetch_forecast( query:str , form:object ):
    global weatherAPI_key
    global imageTk_icon
    # Send API request like:
    # http://api.weatherapi.com/v1/current.json?key=< API KEY >&q=London&aqi=no
    # For full documentation visit https://www.weatherapi.com site
    url = "http://api.weatherapi.com/v1/forecast.json"
    params = {
        'key': weatherAPI_key,
        'q': query,
        'days':7,
        'aqi': 'no',   # To get air quality data or not
        'alerts':'no'
    }
    res = requests.get( url , params=params )
    # Update widgets
    if res.status_code == 200: # HTTP status code 200 - OK
        # Parse the response to a variable
        data = res.json()
        condition_text = 'The weather condition at {0} is: {1}'.format(
            data['location'][ 'name' ],
            data['current'][ 'condition' ][ 'text' ]
        )
        form.nametowidget( 'condition_text' ).config( text=condition_text )

        res = requests.get( 'http:' + str( data['current'][ 'condition' ][ 'icon' ] ) )
        if res.status_code == 200:
            image_icon   = Image.open( BytesIO( res.content ) )
            imageTk_icon = ImageTk.PhotoImage( image_icon )
            form.nametowidget( 'condition_icon' ).config( image=imageTk_icon )
        else:
            print( res.reason )

        form.nametowidget( 'temp_c' ).config( text='{} °C'.format( str( data['current'][ 'temp_c' ] ) ) )
        form.nametowidget( 'temp_f' ).config( text='{} °F'.format( str( data['current'][ 'temp_f' ] ) ) )
    else:
        form.nametowidget( 'condition_text' ).config( text='Ooops, something went wrong. The API response is:\n{}'.format( res.reason ) )

# Main forecast form
def forecast_form( root:object ):
    global imageTk_icon
    # Initilise Frame for all widgets
    frm = ttk.Frame( root , padding=5)
    frm.grid()

    # Row 1 - title
    ttk.Label( frm , text="Local time" ).grid( row=0 , column=0 )
    # Local Time label
    def my_time( label:object ): # Help function to update the time label every second
        label.config( text= strftime( '%H:%M:%S %p | %x' ) )
        label.after( 1000 , my_time , label ) # recursive call every 1000ms

    time_label = ttk.Label( frm )
    time_label.grid( row=0 , column=1 )
    my_time( time_label ) # start lable update loop

    # Row 2 - current time and date
    query_entry = ttk.Entry( frm )
    query_entry.grid( row=1 , column=0 )
    query_entry.focus()
    ttk.Button( frm , text="Get Weather",
        command = lambda:
            fetch_forecast( query_entry.get() , frm )
    ).grid( row=1 , column=1 )
    ttk.Button( frm , text="Help",
        command = lambda:
            help_form( root )
    ).grid( row=1 , column=2 )

    # Row 3 - Current conditions - icon
    image_icon   = Image.open( 'logo.png' )
    imageTk_icon = ImageTk.PhotoImage( image_icon )
    ttk.Label( frm , name = 'condition_icon' , image=imageTk_icon ).grid( row=2 , column=0 , columnspan=3 )

    # Row 4 - Current conditions - text
    ttk.Label( frm,  name = 'condition_text' ).grid( row=3 , column=0 , columnspan=3 )

    # Row 5 - Current conditions - temperatures
    ttk.Label( frm , name = 'temp_c' ).grid( row=4 , column=0 )
    ttk.Label( frm,  name = 'temp_f' ).grid( row=4 , column=1 , columnspan=2 )

##########################
# API Key frame
##########################

# Save API key into a file
def save_API_key( key:str , root:object ):
    global API_key_file
    global weatherAPI_key
    weatherAPI_key = key
    try:
        f = open( API_key_file , "w" )
        f.write( key )
        f.close()
    except Exception: pass # For now do nothing if fail to write
    clear_root( root )
    forecast_form( root )

# API Key form to be displaied if there is no API Key file
def api_key_form( root:object ):
    global imageTk_icon
    instructions = '''

This application requires an API Key from

           www.weartherapi.com

Please visit the site and read the documentation.

Once you have an API Key, use this form
to save it and use this application.

'''
    frm = ttk.Frame( root , padding=5 )
    frm.grid()
    # Row 1 - Logo
    image_icon   = Image.open( 'logo.png' )
    imageTk_icon = ImageTk.PhotoImage( image_icon )
    ttk.Label( frm , image=imageTk_icon ).grid( row=0 , column=0 , columnspan=2 )
    # Row 2 - Instructions
    ttk.Label( frm , text=instructions ).grid( row=1 , column=0 , columnspan=2 )
    # Row 3 - Enter Key
    query_entry = ttk.Entry( frm )
    query_entry.grid( row=2 , column=0 )
    query_entry.focus()
    ttk.Button( frm, text="Save API Key",
        command = lambda:
            save_API_key( query_entry.get() , root )
    ).grid( row=2 , column=1 )

##################
# MAIN APP
##################

root = Tk(  ) # Creates a window

try:
    # Get WeatherAPI Key from file
    f = open( API_key_file , "r")
    weatherAPI_key = f.read()
    f.close()
    forecast_form( root ) # Display the forecast form
except FileNotFoundError as e:
    # User entery of API Key
    api_key_form( root )

root.mainloop() # Shows the window


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 31 20:46:08 2018

@author: Gaurav
"""

#WebCrawler for Dortmund Data Bank

import requests
from bs4 import BeautifulSoup
import scipy as sci
import win32com.client
import os
import sys

#This Function gets the constant value of Temperature or Pressure from above
#the table
def GetConstantValue(useful_header):
    useful_value=useful_header.find_previous("table")
    col=useful_value.find_all("td")
    constvalue=col[0].get_text()
    constvalue=float(constvalue)
    return constvalue

#This function captures the useful headers in the webpage that point to
#Data Tables
def GetUsefulHeaders(soup):
    headers=soup.find_all("h4")
    useful_headers=[]
    for header in headers:
        text=header.get_text()
        if(text=="Data Table"):
            useful_headers.append(header)
    return useful_headers

#This function captures useful tables from the useful headers and stores the
#tables as numpy arrays which are themselves stored in a list

def GetUsefulTables(useful_headers):
    number_of_tables=len(useful_headers)
    list_of_arrays=[]
    constvalues=sci.zeros(number_of_tables)
    stats=sci.zeros(number_of_tables)
    for i in range(number_of_tables):
        useful_header=useful_headers[i]
        useful_table=useful_header.find_next("table")
        rows=useful_table.find_all("tr")
        number_of_cols=len(rows[-1].find_all("td"))
        if(number_of_cols<3):
            continue
        constvalues[i]=GetConstantValue(useful_header)
        stats[i]=GetTableStat(useful_header)
        arr=sci.zeros((len(rows),3))

        row_counter=0
        for row in rows:
            cols=row.find_all("td")
            for cell in range(len(cols)):
                arr[row_counter,cell]=cols[cell].get_text()
            row_counter+=1

        arr=sci.delete(arr,(0),axis=0)
        if(arr[0,1]==0.):
            arr=sci.delete(arr,(0),axis=0)
        if(arr[-1,1]==1.):
            arr=sci.delete(arr,(-1),axis=0)
        list_of_arrays.append(arr)

    final_constvalues=constvalues[constvalues!=0]
    final_stats=stats[constvalues!=0]
    return [list_of_arrays,final_constvalues,final_stats]

#This function is used to print the VLEData to an Excel file (which is created)
#using the win32com library
def PrintToExcel(list_of_tables,compound1,compound2,switch):
    app=win32com.client.Dispatch("Excel.Application")
    app.Visible=False
    wb=app.Workbooks.Add()
    cwdir=os.getcwd()
    filename=f"{compound1}-{compound2}_VLEData.xlsx"
    path=cwdir+"\\"+filename
    wb.SaveAs(path)
    sh=wb.Sheets("Sheet1")

    offset=0
    for table in list_of_tables:
        one_table=table
        [rows,cols]=one_table.shape
        one_table=one_table.tolist()
        for i in range(rows):
            for j in range(cols):
                sh.Cells(i+1+offset,j+1).Value=one_table[i][j]
                if(switch==0):
                    sh.Cells(i+1+offset,1).Interior.ColorIndex=19
                if(switch==1):
                    sh.Cells(i+1+offset,1).Interior.ColorIndex=20
        offset+=rows+1

    wb.Save()
    wb.Close()
    app.Quit()

#This function returns 0 if the table is PXY, else it returns 1 for TXY
def GetTableStat(useful_header):
    useful_value=useful_header.find_previous("table")
    col=useful_value.find_all("td")
    check=col[1].get_text()
    if(check=="K"):
        return 0
    else:
        return 1

#The main function that accepts compound names, calls all the relevant functions
#and returns/prints the final tables
def Crawl(compound1,compound2):
    comp1=f"{compound1}"
    comp2=f"{compound2}"
    page=requests.get(f"http://www.ddbst.com/en/EED/VLE/VLE%20{comp1}%3B{comp2}.php")
    if(page.status_code==200):
        switch=0
    else:
        switch=1
        page=requests.get(f"http://www.ddbst.com/en/EED/VLE/VLE%20{comp2}%3B{comp1}.php")
    soup=BeautifulSoup(page.text,"html.parser")

    useful_headers=GetUsefulHeaders(soup)
    [tables,constantvalues,stats]=GetUsefulTables(useful_headers)

    PrintToExcel(tables,comp1,comp2,switch)
    return [tables,constantvalues,stats,switch]

if __name__=="__main__":
    if(len(sys.argv)==3):
        compound1=f"{sys.argv[1]}"
        compound2=f"{sys.argv[2]}"
    if(len(sys.argv)==4):
        compound1=f"{sys.argv[1]} {sys.argv[2]}"
        compound2=f"{sys.argv[3]}"
    if(len(sys.argv)==5):
        compound1=f"{sys.argv[1]} {sys.argv[2]}"
        compound2=f"{sys.argv[3]} {sys.argv[4]}"

    Crawl(f"{compound1}",f"{compound2}")