# Works with Python 3.6
# Discord 1.2.2
import asyncio
import json
import sqlite3 as sl
import os
import sys
import traceback
import datetime
import discord
from discord.ext import commands
from discord.ext.commands import has_permissions, CheckFailure
import prettytable
import requests

import csv
from modules.core.utils import CommandChecker, RateBucket, sendLong, CoreConfig, Tools
from modules.core.Log import log

class CommandRconDatabase(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.path = os.path.dirname(os.path.realpath(__file__))

        self.con = sl.connect(self.path+'/users.db')
        self.c = self.con.cursor()

        self.upgrade_database()
        #self.alter_database() only needed to upgrade from old db
        #self.player_db = CoreConfig.cfg.new(self.path+"/player_db.json")

        self.cfg = CoreConfig.modules["modules/rcon_database"]["general"]

        asyncio.ensure_future(self.on_ready())
        self.players = None

    async def on_ready(self):
        await self.bot.wait_until_ready()
        if("CommandRcon" not in self.bot.cogs):
            log.error("[module] 'CommandRcon' required, but not found in '{}'. Module unloaded".format(type(self).__name__))
            del self
            return
        try:
            self.CommandArma = self.bot.cogs["CommandArma"]
            self.CommandArma.readLog.EH.add_Event("Player connected", self.playerConnected)
        except KeyError:
            self.CommandArma = None
        self.CommandRcon = self.bot.cogs["CommandRcon"]

        asyncio.ensure_future(self.fetch_player_data_loop())
        #self.check_all_users()

    def upgrade_database(self):
        try:
            json_f = self.path+"/player_db.json"
            if(not os.path.isfile(json_f)):
                return

            #get the count of tables with the name
            self.c.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='users' ''')
            if self.c.fetchone()[0]==0 :

                #load old table
                with open(json_f) as json_file:
                    data_db = json.load(json_file)

                data = []
                for key, item in data_db.items():
                    for _item in item:
                        id = None
                        name = None
                        beid = None
                        ip  = None
                        date = None
                        if "ID" in _item:
                            id = int(_item["ID"])
                        if "name" in _item:
                            name = _item["name"]
                        if "beid" in _item:
                            beid = _item["beid"]
                        if "ip" in _item:
                            ip = _item["ip"]
                        if "last-seen" in _item:
                            date = _item["last-seen"]
                        data.append((id, name, beid, ip, date))

                self.c.execute("""
                    CREATE TABLE users (
                        id INTEGER NOT NULL,
                        name  TEXT,
                        beid TEXT,
                        ip TEXT,
                        stamp DATETIME,
                        profileid INTEGER
                    );
                """)

                #date yyyy-MM-dd HH:mm:ss
                sql = 'INSERT INTO users (id, name, beid, ip, stamp) values(?, ?, ?, ?, ?)'
                self.c.executemany(sql, data)
                self.con.commit()
                json_f_new = json_f.replace(".json", "_old.json")
                os.rename(json_f, json_f_new)
                log.info("*** Database has been upgraded! ***")
        except Exception as e:
            log.print_exc()
            log.error(e)

    def alter_database(self):
        try:
            ## Alter table (upgrade by adding the profileid column)
            self.c.execute("SELECT COUNT(*) AS CNTREC FROM pragma_table_info('users') WHERE name='profileid'")
            if self.c.fetchone()[0]==0:

                sql = """   ALTER TABLE users
                            ADD COLUMN profileid INTEGER;"""
                self.c.execute(sql)
                self.con.commit()
                log.info("Altered DB Table: 'Added COLUMN profileid'")
        except Exception as e:
            log.print_exc()
            log.error(e)
            log.info("If you are not upgrading an old database, you can ignore the error above.")

    async def fetch_player_data_loop(self):
        while True:
            try:
                if(self.CommandRcon.arma_rcon.disconnected==True):
                    await asyncio.sleep(60)
                    continue
                try:
                    self.players = await self.CommandRcon.arma_rcon.getPlayersArray()
                except Exception as e:
                    await asyncio.sleep(60)
                    continue
                #self.player_db.save = False

                c_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

                if(self.cfg["setTopicPlayerList_channel"]>0):
                    await self.setTopicPlayerList(self.players)

                for player in self.players:
                    name = player[4]
                    if(name.endswith(" (Lobby)")): #Strip lobby from name
                        name = name[:-8]
                    d_row = {
                        "id": player[0],
                        "name": name,
                        "beid": player[3],
                        "ip": player[1].split(":")[0], #removes port from ip
                        "note": None,
                        "stamp": c_time
                    }
                    self.update_insert(d_row)

                #self.player_db.save = True
                #self.player_db.json_save()

            except Exception as e:
                log.print_exc()
                log.error(e)
            await asyncio.sleep(60)

    async def setTopicPlayerList(self, players):
        #log.info("[DEBUG] {}".format(players))#
        channel = self.bot.get_channel(self.cfg["setTopicPlayerList_channel"])
        if(not channel):
            return
        playerlist = "Players online: "+str(len(players))+" - "
        for player in players:
            playerlist += player[4]+", "
        await channel.edit(topic=playerlist[:-2])

    # fetches players profile ID from the log file,
    # and adds it to the database if it can be matched to a user.
    async def playerConnected(self, event, data):
        try:
            player_name = data["event_match"].group(2)
            player_profileID = int(data["event_match"].group(3))
            for i in range(2):
                for player in self.players:
                    name = player[4]
                    if(name.endswith(" (Lobby)")): #Strip lobby from name
                        name = name[:-8]

                    if name == player_name:
                        beid = player[3]
                        ip = player[1].split(":")[0]
                        sql = """   UPDATE users
                                    SET profileid = ?
                                    WHERE name = ?
                                          AND beid = ?
                                          AND ip = ?
                                          AND stamp > date('now','-1 day')"""

                        log.debug(sql)
                        self.c.execute(sql, (player_profileID, name, beid, ip))
                        self.con.commit()
                        break
                    await asyncio.sleep(60)
        except Exception as e:
            log.print_exc()
            log.error(e)

###################################################################################################
#####                                   General functions                                      ####
###################################################################################################

    async def new_data_entry(self, row):
        linked = self.find_by_linked(row["beid"])

        if(len(linked["beids"])>1):
            channel = self.bot.get_channel(self.cfg["post_channel"])
            if(channel):
                await channel.send(":warning: Player '{name}' with BEID '{beid}' might be using >=2 accounts from the same ip".format(**row))

    def update_insert(self, row):
        #update only if user if last seen > 1 day or diffrent ip/beid
        sql = "SELECT * FROM users WHERE name = ? AND beid = ? AND ip = ? AND stamp > date('now','-1 day')"
        self.c.execute(sql, (row["name"], row["beid"], row["ip"]))
        r = self.c.fetchone()
        if r is None:
            #TODO Handel stamp = Null
            sql = 'INSERT INTO users (id, name, beid, ip, stamp) values(?, ?, ?, ?, ?)'
            self.c.execute(sql, (row["id"], row["name"], row["beid"], row["ip"], row["stamp"]))
            self.con.commit()

    def find_by_linked(self, beid, beids = None, ips = None, names = None):
        try:
            if(beids == None):
                beids = set()
            if(ips == None):
                ips = set()
            if(names == None):
                names = set()
            self.c.execute("SELECT count(beid) FROM users WHERE beid = ?", (beid, ))
            if self.c.fetchone()[0]==0:
                return {"beids": beids, "ips": ips, "names": names}

            entries = self.c.execute("SELECT * FROM users WHERE beid = ?", (beid, ))
            entries = entries.fetchall()
            for data in entries:
                beids.add(data[2])
                ips.add(data[3])
                names.add(data[1])

                ip_list = self.c.execute("SELECT * FROM users WHERE ip = ?", (data[3], ))
                ip_list = ip_list.fetchall()
                for row in ip_list:
                    ips.add(row[3])
                    if(row[1] not in names):
                        names.add(row[1])
                    if(row[2] not in beids):
                        beids.add(row[2])
                        r = self.find_by_linked(row[2], beids, ips, names)
                        beids = beids.union(r["beids"])
                        ips = ips.union(r["ips"])
                        names = names.union(r["names"])
            return {"beids": beids, "ips": ips, "names": names}
        except Exception as e:
            log.print_exc()
            log.error(e)

        #log.info(init)

    #Checks all users in the database for possible multi account usage.
    #The generated list will be printed into console.
    def check_all_users(self, min = 2):
        for key in self.player_db.keys():
            data = self.find_by_linked(key)
            if(len(data["beids"]) >= min):
                log.info(data)
###################################################################################################
#####                                       Commands                                           ####
###################################################################################################

    @CommandChecker.command(name='find',
            brief="find user data by field",
            pass_context=True)
    async def find_data(self, ctx, field, data):
        valid = ["name", "beid", "ip", "stamp"]
        if field not in valid:
            raise Exception("Invalid field '{}', must be one of these values: {}".format(field, valid))

        result = self.c.execute("SELECT * FROM users WHERE {} LIKE ? GROUP BY beid ORDER BY stamp DESC".format(field), (data, ))

        if(result):
            msg = ""
            for row in result:
                msg += "{} ``{}`` ({})\n".format(row[1], row[2], row[4])
            await sendLong(ctx, msg)
        else:
            await sendLong(ctx, "Sorry, I could not find anything")

    @CommandChecker.command(name='find_linked',
            brief="Does a cross search over IPs and BEID",
            pass_context=True)
    async def find_linked(self, ctx, beid):

        result = self.find_by_linked(beid)
        msg = ""
        if(len(result["beids"]) > 0):
            msg+= "BEID: ```\n"
            for beid in result["beids"]:
                msg+=beid+"\n"
            msg+= "```\n"

            msg+= "Names: ```"
            for name in result["names"]:
                msg+=str(name)+"\n"
            msg+= "```"
            await sendLong(ctx, msg)
        else:
            await sendLong(ctx, "Sorry, I could not find anything")

    @CommandChecker.command(name='players+',
        brief="Lists players on the server and checks if they are regulars",
        pass_context=True)
    async def players(self, ctx):
        players = await self.CommandRcon.arma_rcon.getPlayersArray()
        msgtable = prettytable.PrettyTable()
        msgtable.field_names = ["ID", "Name", "IP", "GUID", "First Seen", "Visits"]
        msgtable.align["ID"] = "r"
        msgtable.align["Name"] = "l"
        msgtable.align["IP"] = "l"
        msgtable.align["GUID"] = "l"
        msgtable.align["First Seen"] = "l"
        msgtable.align["Visits"] = "r"

        limit = 100
        i = 1
        new = False
        msg  = ""
        for player in players:
            if(i <= limit):
                first_seen = self.c.execute("SELECT * FROM users WHERE beid = ? AND stamp IS NOT NULL ORDER BY stamp ASC LIMIT 1", (player[3], ))
                first_seen = list(first_seen)
                if(len(first_seen) > 0):
                    first_seen = first_seen[0][4].split(" ")[0]
                else:
                    first_seen = ""
                visits = self.c.execute("SELECT COUNT(*) FROM users WHERE beid = ?", (player[3], ))
                visits = list(visits)
                if(len(visits) > 0):
                    visits = visits[0][0]
                else:
                    visits = 0
                msgtable.add_row([player[0], discord.utils.escape_markdown(player[4], as_needed=True), player[1],player[3], first_seen, visits])
                if(len(str(msgtable)) < 1800):
                    i += 1
                    new = False
                else:
                    msg += "```"
                    msg += str(msgtable)
                    msg += "```"
                    await ctx.send(msg)
                    msgtable.clear_rows()
                    msg = ""
                    new = True
        if(new==False):
            msg += "```"
            msg += str(msgtable)
            msg += "```"
            await ctx.send(msg)

    @CommandChecker.command(name='regulars',
        brief="Lists the most seen players",
        pass_context=True)
    async def regulars(self, ctx):
        sql = """   SELECT name,COUNT(beid) AS cnt FROM users
                    GROUP BY beid
                    ORDER BY cnt DESC;
        """
        players = self.c.execute(sql)
        players = (list(players)[:10])

        msgtable = prettytable.PrettyTable()
        msgtable.field_names = ["Name", "Visits"]
        msgtable.align["Name"] = "l"
        msgtable.align["Visits"] = "r"

        limit = 100
        i = 1
        new = False
        msg  = ""
        for player in players:
            if(i <= limit):
                msgtable.add_row([discord.utils.escape_markdown(player[0], as_needed=True), player[1]])
                if(len(str(msgtable)) < 1800):
                    i += 1
                    new = False
                else:
                    msg += "```"
                    msg += str(msgtable)
                    msg += "```"
                    await ctx.send(msg)
                    msgtable.clear_rows()
                    msg = ""
                    new = True
        if(new==False):
            msg += "```"
            msg += str(msgtable)
            msg += "```"
            await ctx.send(msg)

    @CommandChecker.command(name='query',
        brief="Runs and commits an SQL command",
        help="""table: 'users'
        columns:
        id INTEGER NOT NULL,
        name  TEXT,
        beid TEXT,
        ip TEXT,
        stamp DATETIME

        returns at most 15 rows
        """,
        pass_context=True)
    async def query(self, ctx, *query):
        try:
            query = " ".join(query)
            log.info(query)
            result = self.c.execute(query)
            result = list(result)
            self.con.commit()

            msg = ""
            for row in result[:15]:
                msg += "{}\n".format(row)
            if(msg == ""):
                msg = "Query returned nothing"
        except Exception as e:
            msg = str(e)
        await ctx.send(msg)

    @CommandChecker.command(name='convertID',
        brief="Convert a players ID. Can be Steam ID, BattlEye ID or Bohemia ID",
        aliases=["convertid"],
        pass_context=True)
    async def convertid(self, ctx, id):
        url = '	https://api.devt0ols.net/converter/'

        params = dict(id=id)
        resp = requests.get(url=url, params=params, verify=False)
        data = resp.json() # Check the JSON Response Content documentation below
        msg=""
        for key, val in data["result"].items():
            if(key == "steam_url"):
                msg += "{} {}\n".format(key, val)
            else:
                msg += "{} ``{}``\n".format(key, val)
        await ctx.send(msg)

async def setup(bot):
    await bot.add_cog(CommandRconDatabase(bot))


import os
import datetime
import pickle
import requests
import bs4 as bs
import pandas as pd
import pandas_datareader as web
from quantipy.basics.data_struct import StocksData, OptionsData, FuturesData

class BaseImport(object):
    mapping = {}
    datatypes = {}

    def __init__(self):
        self.dataframe = pd.DataFrame()

    def _enhance_data(self):
        pass

    def _data_cleansing(self):
        pass

    def _remap(self, mapping):
        self.dataframe.rename(columns=mapping, inplace=True)

    def import_data(self, **kwargs):
        if "datasource" not in kwargs:
            if self.dataframe.empty:
                return
        else:
            self.dataframe = kwargs["datasource"]

        if "mapping" in kwargs:
            self._remap(kwargs["mapping"])
        else:
            self._remap(self.mapping)

        self._data_cleansing()
        self._enhance_data()

        return self.dataframe

class BaseCSVImport(BaseImport):
    def __init__(self):
        super().__init__()
        self.datatypes = {}
        self.mapping = {}

    def import_data(self, filename, delimiter=",", **kwargs):
        if "datatypes" in kwargs:
            self.datatypes = kwargs["datatypes"]

        if "mapping" in kwargs:
            self.mapping = kwargs["mapping"]

        self.dataframe = pd.read_csv(filename, delimiter=delimiter, dtype=self.datatypes)
        super().import_data(mapping=self.mapping)
        return self.dataframe

class OptionVueImport(BaseCSVImport):
    """" Import data from a CSV file that was exported in OptionVue

         Todo: Futures import does not work correctly. Also the OptionVue export does not supply an expiration date
               for futures contracts as of 17.FEB.2017.
    """

    def __init__(self):
        super().__init__()
        self.datatypes = {
            "Time": "str",
            "Date": "str",
            "Exp.Date": "str"
        }
        self.mapping = {"Symbol": "symbol",
                           "Item Type": "type",
                           "Last": "last",
                           "Change": "change",
                           "High": "high",
                           "Low": "low",
                           "Volume": "volume",
                           "Theta": "theta",
                           "Time": "time",
                           "Asked": "ask",
                           "Bid": "bid",
                           "Date": "date",
                           "Delta": "delta",
                           "Description": "description",
                           "Exp.Date": "exp_date",
                           "Gamma": "gamma",
                           "Mid IV": "iv",
                           "Market Price": "market_price",
                           "Open": "open",
                           "Open Interest": "open_interest",
                           "Vega": "vega",
                           "Call/Put": "p_c",
                           "Previous Close": "prev_close",
                           "Rho": "rho",
                           "Strike Price": "strike",
                           "SV": "atm_sv"}

    def _data_cleansing(self):
        #drop columns that are not needed
        self.dataframe.drop(["Original Price", "Prob.Finish.ITM", "Projected Volty", "Time Premium", "Ask IV",
                             "At Price", "Existing Posn.", "Bid IV", "Pcnt To Double", "Percent O/U", "Th.Price",
                             "Trade"], axis=1, inplace=True)

        #fill NaN values for columns and set datatypes
        self.dataframe["volume"].fillna(0, inplace=True)
        self.dataframe["volume"].astype(int)
        self.dataframe["open_interest"].fillna(0, inplace=True)
        self.dataframe["open_interest"].astype(int)
        self.dataframe["exp_date"].fillna("", inplace=True)

        # Filter out all rows that don't have a market price, these couldn't be traded
        self.dataframe = self.dataframe[self.dataframe["market_price"] > 0.0]

        # If date columns are filled, prefix them with the century and convert to datetime object
        self.dataframe.loc[self.dataframe["date"] != "", "date"] = "20" + self.dataframe["date"]
        self.dataframe.loc[self.dataframe["exp_date"] != "", "exp_date"] = "20" + self.dataframe["exp_date"]
        self.dataframe["date"] = pd.to_datetime(self.dataframe["date"])
        self.dataframe["exp_date"] = pd.to_datetime(self.dataframe["exp_date"])

    def _enhance_dataframe(self):
        # Extract Symbol
        self.dataframe["symbol"] = self.dataframe["symbol"].str.split(' ').str[0]
        # Add close data to current data
        self.dataframe["close"] = self.dataframe["market_price"]
        # Add percent change
        self.dataframe["change_pct"] = (self.dataframe["close"]
                                        - self.dataframe["prev_close"]) / self.dataframe["prev_close"]
        self.dataframe["dte"] = (self.dataframe["exp_date"] - self.dataframe["date"]).dt.days

    def import_data(self, filename, delimiter=",", **kwargs):
        """ Import CSV files exported from OptionVue. These files contain stocks, futures and options data
            in the same file. The export structure is dynamic, depending on the list selected in OptionVue
        """
        super().import_data(filename, delimiter, **kwargs)

        #self.dataframe.to_csv("./Data/OptionVue/conv.csv", index=False, float_format='%.12g')

        stock_data = self.dataframe[self.dataframe["type"] == "S"]
        option_data = self.dataframe[self.dataframe["type"] == "G"].append(self.dataframe[self.dataframe["type"] == "O"])
        future_data = self.dataframe[self.dataframe["type"] == "F"]

        return stock_data, future_data, option_data

class YahooStockImport(BaseImport):
    _symbol_pickle_filename = "./Data/Yahoo/sp500tickers.pkl"
    _stock_dir = "./Data/Yahoo/Stocks"

    def __init__(self):
        super().__init__()
        # This is the mapping from Yahoo's web-data to the fieldnames used internally in QuantiPy
        self.mapping = {
            "Date": "date",
            "Open": "open",
            "High": "high",
            "Low": "low",
            "Volume": "volume",
            "Close": "close",
            "Adj Close": "adj_close",
            "Description": "description",
            "Symbol": "symbol"
        }

    def _enhance_data(self):
        self.dataframe["factor"] = self.dataframe["close"] / self.dataframe["adj_close"]
        self.dataframe["close"] = self.dataframe["adj_close"]
        self.dataframe["open"] = self.dataframe["open"] / self.dataframe["factor"]
        self.dataframe["high"] = self.dataframe["high"] / self.dataframe["factor"]
        self.dataframe["low"] = self.dataframe["low"] / self.dataframe["factor"]

        # Last data = close
        self.dataframe["last"] = self.dataframe["close"]
        # Previous close from previously processed data
        self.dataframe["prev_close"] = self.dataframe["close"].shift(1)
        # Open Interest, yahoo doesn't have this info so set to 0
        self.dataframe["open_interest"] = 0
        # Market price is set to close, because no bid/ask available
        self.dataframe["market_price"] = self.dataframe["close"]
        # Bid/ask is set to close
        self.dataframe["bid"] = self.dataframe["close"]
        self.dataframe["ask"] = self.dataframe["close"]
        # Time is always EOD
        self.dataframe["time"] = datetime.time(22, 0)
        # Create change and change percent from previous
        self.dataframe["change"] = self.dataframe["close"] - self.dataframe["prev_close"]
        self.dataframe["change_pct"] = self.dataframe["change"] / self.dataframe["prev_close"]

    def _web_load_symbols(self):
        resp = requests.get('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
        soup = bs.BeautifulSoup(resp.text, "lxml")
        table = soup.find('table', {'class': 'wikitable sortable'})
        tickers = []
        for row in table.findAll('tr')[1:]:
            ticker, description = row.findAll('td')[0].text, row.findAll('td')[1].text
            tickers.append((ticker, description))

        with open(self._symbol_pickle_filename, "wb") as f:
            pickle.dump(tickers, f)
        return tickers

    def _load_symbols(self):
        if os.path.exists(self._symbol_pickle_filename):
            with open(self._symbol_pickle_filename, "rb") as f:
                tickers = pickle.load(f)
        else:
            tickers = self._web_load_symbols()
        return tickers

    def import_data(self, stock_symbol=[], from_date="2001-01-01", to_date="2099-01-01",
                    reload_sp500_symbols=False, reload_data=False, **kwargs):
        if not os.path.exists(self._stock_dir):
            os.makedirs(self._stock_dir)

        symbols = []
        if stock_symbol:
            symbols = stock_symbol
        else:
            if reload_sp500_symbols:
                symbols = self._web_load_symbols()
            else:
                symbols = self._load_symbols()

        if not symbols:
            return

        start_date = datetime.datetime.strptime(from_date, "%Y-%m-%d")
        end_date = datetime.datetime.strptime(to_date, "%Y-%m-%d")

        del self.dataframe
        self.dataframe = pd.DataFrame()

        for symbol in symbols:
            if type(symbol) == str:
                symbol_to_load, description = symbol, symbol
            else:
                symbol_to_load, description = symbol[0], symbol[1]

            df = pd.DataFrame()
            if reload_data or not os.path.exists(self._stock_dir + "/{}.csv".format(symbol_to_load)):
                try:
                    df = web.DataReader(symbol_to_load, 'yahoo', start_date, end_date)
                    df["Description"] = description
                    df["Symbol"] = symbol_to_load
                    df.to_csv(self._stock_dir + "/{}.csv".format(symbol_to_load))
                    df.reset_index(inplace=True)
                except:
                    continue
            else:
                with open(self._stock_dir + "/{}.csv".format(symbol_to_load)) as datafile:
                    df = pd.read_csv(self._stock_dir + "/{}.csv".format(symbol_to_load), parse_dates=True)
                    # Filter all dates that are not within the selected period. Filtering a vector should be faster than
                    # Looping through csv
                    df = df[(df["Date"] >= from_date) & (df["Date"] <= to_date)]

            self.dataframe = self.dataframe.append(df)

        super().import_data()
        return self.dataframe


from django import forms
from django.http import HttpResponse
from django.shortcuts import render,redirect
from db.models import details
from db.forms import detailsfrom

def index(request):
    obs=details.objects.all()
    context={'obs':obs}

    return render(request,'index.html',context)

def addmember(request):
    form=detailsfrom
    if request.method=='POST':
        form=detailsfrom(request.POST)
        if form.is_valid():
            form.save()
            return redirect('index')

    context={'form':form}
    return render(request,'add_member.html',context)

def update(request,pk):
    form=detailsfrom
    member=details.objects.get(id=pk)
    if request.method == 'POST':
        form=detailsfrom(request.POST , instance=member)
        if form.is_valid():
            form.save()
            return redirect('index')
    context={'form':form}
    return render(request,'add_member.html',context)

def delete(request,pk):
    member=details.objects.get(id=pk)
    member.delete()
    return redirect('index')


from urllib.request import urlopen,Request
from bs4 import BeautifulSoup

def page_soup(url):
    req = Request(url, headers = {'User-Agent': 'Mozilla/5.0'})
    furl = urlopen(req)
    html = furl.read()
    furl.close()
    soup = BeautifulSoup(html,'html.parser')
    return soup

url =  "https://www.tutorialspoint.com/tutorialslibrary.htm"
soup = page_soup(url)

containers = soup.findAll('div',{'class': 'featured-box'})

for container in containers[1:2]:
    headings = container.findAll('h4')
    ul = container.findAll('ul')
    no_of_headings = len(headings)

    for i in range(7,9):
        print(i)

        topics = ul[i].findAll('li')
        print(len(topics))

        for topic in topics:
            topic_name = topic.text
            f = open(topic_name.replace('/',' ')+'_'+headings[i].text.split()[0]+'.txt','w',encoding = 'utf-8')
            url = 'https://www.tutorialspoint.com/' + topic.a['href']
            soup = page_soup(url)
            con = soup.findAll('aside',{'class':'sidebar'})
            if (len(con)==0):
                continue
            uls = con[0].findAll('ul')
            for u in uls:
                lis = u.findAll('li')
                if (lis[0].text == 'Selected Reading'):
                    continue
                for li in lis[1:]:
                    f.write(li.text+'\n')
            f.close()


#!/usr/bin/env python3

import os
import tensorflow as tf
import numpy as np
from Bio import SeqIO
from Utility import run
import re
from Utility import append_time_ram_stamp_to_file
from itertools import product
import time

def convert_data_one_hot_with_Ns_spread_str_to_numbers(seqs) -> list[list[float]]:
    return list(map(lambda l: [[float(x) for x in i.split("_")] for i in l], seqs))

def read_data_one_hot_with_Ns_spread_str(config, add_one_terminal_symbol = False) -> list[list[str]]:
    '''
    bc i can only pad batches with scalar, i have multi hot encoded strings for every base
    and pad with str that encodes terminal emission
    then i can convert these a string corresponding to a base to its multi hot encoded version
    '''
    start = time.perf_counter()
    append_time_ram_stamp_to_file(f"read_data_one_hot_with_Ns_spread_str() start ", config.bench_path, start)
    seqs = []
    base_to_id_dict = dict([(base, id) for id, base in enumerate("ACGTI")])
    def base_to_id(b):
        b = b.upper()
        try:
            return base_to_id_dict[b]
        except:
            return 5

    with open(config.fasta_path,"r") as handle:
        if config.only_primates or config.only_diverse:
            with open(config.primates_path, "r") as primates_handle:
                primates = set()
                # this file is just a list of ids of primates
                for line in primates_handle:
                    primates.add(line.strip())

            # a list containing the ids of the species that should be used as input
            species_to_use = []
            species_in_combinded_fasta = set()
            number_of_primats_in_current_fasta = 0

            for record in SeqIO.parse(handle,"fasta"):
                species_name = re.search("(\w+?_\w+?)\.", record.id).group(1)
                species_in_combinded_fasta.add(species_name)
                if species_name in primates:
                    number_of_primats_in_current_fasta += 1
                    if config.only_primates:
                        species_to_use.append(species_name)
            # print("number_of_primats_in_current_fasta", number_of_primats_in_current_fasta)

        if config.only_diverse and config.only_diverse:
            # list all files in a dir whos path is stored in config.only_diverse
            all_files = []
            for file in os.listdir(config.diverse_trees_path):
                if file.endswith(".txt"):
                    all_files.append(os.path.join(config.diverse_trees_path, file))
            all_files = sorted(all_files)
            print(all_files)
            for file in all_files:
                species_in_file = set()
                with open(file, "r") as file_handle:
                    for line in file_handle:
                        species_in_file.add(line.strip())
                # find the size of the overlap of the set species_in_combinded_fasta and species_in_file
                overlap = species_in_combinded_fasta.intersection(species_in_file)
                print("species_in_combinded_fasta", len(species_in_combinded_fasta))
                print("species_in_file", len(species_in_file))
                if len(overlap) == number_of_primats_in_current_fasta:
                    overlap.add("Homo_sapiens")
                    species_to_use = list(overlap)
                    break

    with open(config.fasta_path,"r") as handle:
        for record in SeqIO.parse(handle,"fasta"):
            if config.only_primates or config.only_diverse:
                # print("using species:", species_to_use)
                species_name = re.search("(\w+?_\w+?)\.", record.id).group(1)
                # print("species_to_use", species_to_use)
                # print("species_name", species_name)
                if species_name not in species_to_use:
                    continue

            if config.only_human:
                species_name = re.search("(\w+?_\w+?)\.", record.id).group(1)
                # print("species_to_use", species_to_use)
                # print("species_name", species_name)
                if not re.search("Homo_sap",species_name):
                    continue

            seq = record.seq
            seq_of_one_hot = []
            last_bases = [4] * config.order # 4 is padded left flank by place holder symbol
            for i, base in enumerate(seq):
                t = (last_bases + [base_to_id(base)])
                matching_ids = [] # t might contain N, so more that one id match to this tuple
                allowd_bases = [[0,1,2,3] if b == 5 else [b] for b in t]
                # TODO here, stop codons are allowed. B has just zeros at these positions.
                # this results in a very slight shift of there parameters
                allowed_ho_emissions = list(product(*allowd_bases))
                entry_of_one_hot = np.zeros(config.model.number_of_emissions)
                for allowed_ho_emission in allowed_ho_emissions:
                    entry_of_one_hot[config.model.emission_tuple_to_id(allowed_ho_emission)] = 1
                entry_of_one_hot /= sum(entry_of_one_hot)
                entry_of_one_hot = "_".join([str(x) for x in entry_of_one_hot])
                seq_of_one_hot.append(entry_of_one_hot)
                last_bases = last_bases[1:] + [base_to_id(base)] if config.order > 0 else []
                # print(list(entry_of_one_hot))
                # for i in range(len(entry_of_one_hot)):
                #     if entry_of_one_hot[i] != 0:
                #         print(config.model.emission_id_to_str(i), end = ", ")
            if add_one_terminal_symbol:
                entry_of_one_hot = np.zeros(config.model.number_of_emissions, dtype = np.float32)
                entry_of_one_hot[config.model.emission_tuple_to_id("X")] = 1
                entry_of_one_hot = "_".join([str(x) for x in entry_of_one_hot])
                seq_of_one_hot.append(entry_of_one_hot)
            seqs.append(seq_of_one_hot)
    append_time_ram_stamp_to_file(f"read_data_one_hot_with_Ns_spread_str() end ", config.bench_path, start)
    assert len(seqs) != 0, "no seqs read"
    print("actual number of species used as input:", len(seqs))

    if config.only_human:
        seqs.append(seqs[0])
        print("added human sequence to input")
        print("len(seqs)", len(seqs))

    return seqs

def remove_long_seqs(seqs):
    len_of_seqs = [len(seq) for seq in seqs]
    # median
    med = np.median(len_of_seqs)
    # create a list starting in 2 ending in 10 with step size 0.1
    l = np.arange(2,10,0.1)
    for f in l:
        number_of_too_long_seqs = 0
        for lenght in len_of_seqs:
            if lenght > f * med:
                number_of_too_long_seqs += 1
        if number_of_too_long_seqs < 0.09 * len(seqs):
            break

    # remove all seqs that are longer than f * med
    seqs = [seq for seq in seqs if len(seq) <= f * med]

    # assert that there are still more than 90% of the seqs left
    assert len(seqs) > 0.89 * len(len_of_seqs), "to many seqs removed"
    return seqs

def read_data_with_order(config, alphabet = ["A","C","G","T"], add_one_terminal_symbol = False, verbose = False):
    seqs = []
    def log(s):
        if verbose:
            print(s)

    base_to_id = dict([(base, id) for id, base in enumerate(alphabet)])
    with open(config.fasta_path,"r") as handle:
        log(f"opened: {config.fasta_path}")
        for record in SeqIO.parse(handle,"fasta"):
            seq = record.seq
            log(f"seq = {seq}")
            seq_of_tuple_ids = [] # 21, 124
            last_bases = [4] * config.order # 4 is padded left flank
            for base in seq:
                t = (last_bases + [base_to_id[base]])
                seq_of_tuple_ids.append(config.model.emission_tuple_to_id(t))
                last_bases = last_bases[1:] + [base_to_id[base]] if config.order > 0 else []
            if add_one_terminal_symbol:
                seq_of_tuple_ids.append(config.model.emission_tuple_to_id("X"))
            seqs.append(seq_of_tuple_ids)
    log(f"read {len(seqs)} sequnces")
    return seqs

def get_batch_input_from_tf_printed_file(path):
    input = []
    with open(path, "r") as file:
        seq = []
        for line in file:
            line = line.strip()
            if len(line) == 0:
                input.append(seq)
                seq = []
            else:
                line = re.sub("[\[\]]","", line)
                line = line.split(" ")
                line = [float(x) for x in line]
                seq.append(line)
        if len(seq) != 0:
            input.append(seq)
    return tf.constant(input, dtype = tf.float32)


#from GUI import tk_entry_alb_suffix

def findtextures():
    import os, fnmatch
    import Settings
    #Set directory
    os.chdir(Settings.folder_path)

    #CREATE LIST WITH FOLDER FILES
    file_list = []
    for i in os.listdir():
        file_list.append(i)

    def findfile(pattern, path):
        result = []
        for name in os.listdir():
            if fnmatch.fnmatch(name, pattern):
                result.append(name)
        return result
    #find('jpg', folder_path)
    Settings.tex_Albedo.append(findfile('*albedo*', Settings.folder_path))
    Settings.tex_Normal.append(findfile('*normal*', Settings.folder_path))
    Settings.tex_Roughness.append(findfile('*roughness*', Settings.folder_path))
    Settings.tex_Metalness.append(findfile('*metalness*', Settings.folder_path))
    Settings.tex_Height.append(findfile('*disp*', Settings.folder_path))
    Settings.tex_AO.append(findfile('*_ao*', Settings.folder_path))

if __name__ == "__main__":
    findtextures()

def ResetTextureValues():
    import Settings
    Settings.tex_Albedo = []
    Settings.tex_Normal = []
    Settings.tex_Roughness = []
    Settings.tex_Metalness = []
    Settings.tex_Height = []
    Settings.tex_AO = []

if __name__ == "__main__":
    ResetTextureValues()

def debug():
    import Settings
    print(Settings.tex_Albedo)
    print(Settings.tex_Normal)
    print(Settings.tex_Roughness)
    print(Settings.tex_Metalness)
    print(Settings.tex_AO)
    print(Settings.tex_Height)
if __name__ == "__main__":
    debug()

def convertimg():
    import os, fnmatch
    from shutil import copy2
    from PIL import Image as pillowimage
    import Settings

    import Settings
    print(Settings.tex_Albedo)
    print(Settings.tex_Normal)
    print(Settings.tex_Roughness)
    print(Settings.tex_Metalness)
    print(Settings.tex_AO)
    print(Settings.tex_Height)

    #print(dir(os))
    #print(os.getcwd())
    #Set directory
    os.chdir(Settings.folder_path)
    conv_fold_name = 'Converted'

    #CREATE FOLDER CONVERTED
    file_list = []
    for i in os.listdir():
        file_list.append(i)
    if conv_fold_name not in file_list:
        os.mkdir('Converted')

    def imgprocess(srcname,folder,width,height,format, name, suffix):
        imgn = pillowimage.open(srcname)
        img_adj = imgn.resize((width, height))
        img_adj.save(folder+name+suffix+format)

    print('FINAL DEBUG BEFOR CONVERSION')
    debug()

    if Settings.tex_Albedo:
        if not Settings.tex_Albedo == 'None':
            print('Converting: '+Settings.tex_albedo)
            imgprocess(Settings.tex_Albedo,Settings.destination_folder, int(Settings.width_px), int(Settings.height_px),Settings.c_format, Settings.c_TextureName,Settings.c_AlbedoSuffix)

    if Settings.tex_Normal:
        if not Settings.tex_Normal == 'None':
            print('Converting: '+Settings.tex_normal)
            imgprocess(Settings.tex_Normal,Settings.destination_folder, int(Settings.width_px), int(Settings.height_px),Settings.c_format, Settings.c_TextureName,Settings.c_NormalSuffix)

    if Settings.tex_Roughness:
        if not Settings.tex_Roughness == 'None':
            print('Converting: '+Settings.tex_roughness)
            imgprocess(Settings.tex_Roughness,Settings.destination_folder, int(Settings.width_px), int(Settings.height_px),Settings.c_format, Settings.c_TextureName,Settings.c_RoughnessSuffix)

    if Settings.tex_Metalness:
        if not Settings.tex_Metalness == 'None':
            print('Converting: '+Settings.tex_metalness)
            imgprocess(Settings.tex_Metalness,Settings.destination_folder, int(Settings.width_px), int(Settings.height_px),Settings.c_format, Settings.c_TextureName,Settings.c_MetalnessSuffix)

    if Settings.tex_Height:
        if not Settings.tex_Height == 'None':
            print('Converting: '+Settings.tex_height)
            imgprocess(Settings.tex_Height,Settings.destination_folder, int(Settings.width_px), int(Settings.height_px),Settings.c_format, Settings.c_TextureName,Settings.c_HeightSuffix)
    if Settings.tex_AO:
        if not Settings.tex_AO == 'None':
            print('Converting: '+Settings.tex_AO)
            imgprocess(Settings.tex_AO,Settings.destination_folder, int(Settings.width_px), int(Settings.height_px),Settings.c_format, Settings.c_TextureName,Settings.c_AOSuffix)

    print('DONE')

if __name__ == "__main__":
    convertimg()

#IF RESOLUTION LEFT AT NONE FUNCTION KEEPS THE ORIGINAL RESOLUTION

#EXAMPLE:
#dir = r'C:\Users\lreznicek\Desktop\Asphalt_Fine_tk3mebkew_8K_surface_ms\packer'
#aimgName = 'TESTIMAGE.tga'
#aimgR = 'tk3mebkew_8K_AO.jpg'
#aimgG = 'tk3mebkew_8K_Roughness.jpg'
#aimgB = None
#packRGBA(dir,aimgName,SimgR=None, SimgG=aimgG,SimgB=aimgB,imgResW=None,imgResH=None, SimgA=aimgR)

def packRGBA(dir,imgName, SimgR=None, SimgG=None, SimgB=None, SimgA=None, imgResW=None, imgResH=None):
    import os
    from PIL import Image as P_Image
    from PIL import ImageOps as P_ImageOps

    #SET DIRECTORY
    os.chdir(dir)

    #DETERMINE SOURCE RESOLUTION
    for i in [SimgR,SimgG,SimgB,SimgA]:
        if i == None:
            continue
        else:
            imr = P_Image.open(i)
            SwidthR, SheightR = imr.size
            break

    #SET SOURCE WIDTH AND HEIGHT IF NOT SPECIFIED IN FUNCTION
    if imgResW is None:
        imgResW = SwidthR

    if imgResH is None:
        imgResH = SheightR

    tRes = (imgResW, imgResH)

    #CREATE EMPTY IMAGES IF NONE ARE SPECIFIED IN FUNCTION, CONVERT INPUT TEXTURES TO GRAYSCALE, SET RESOLUTION
    if SimgR is None:
        imgRg = P_Image.new('L', tRes, color='#808080')
    else:
        imgR = P_Image.open(SimgR)
        imgR = imgR.resize(tRes)
        imgRg = P_ImageOps.grayscale(imgR)

    if SimgG is None:
        imgGg = P_Image.new('L',tRes,color='#808080')
    else:
        imgG = P_Image.open(SimgG)
        imgG = imgG.resize(tRes)
        imgGg = P_ImageOps.grayscale(imgG)

    if SimgB is None:
        imgBg = P_Image.new('L',tRes,color='#808080')
    else:
        imgB = P_Image.open(SimgB)
        imgB = imgB.resize(tRes)
        imgBg = P_ImageOps.grayscale(imgB)

    if SimgA is None:
        pass
    else:
        imgA = P_Image.open(SimgA)
        imgA = imgA.resize(tRes)
        imgAg = P_ImageOps.grayscale(imgA)

    #MERGE GRAYSCALE IMAGES
    if SimgA is None:
        imageRGB = P_Image.merge('RGB', (imgRg,imgGg,imgBg))
    else:
        imageRGB = P_Image.merge('RGBA', (imgRg,imgGg,imgBg,imgAg))

    #SAVE PACKED IMAGE
    imageRGB.save(imgName)

    #OUTPUT
    print('PACKED')

if __name__ == "__main__":
    pass


import json
from dataclasses import dataclass
from typing import Any, List

from dataclasses_json import DataClassJsonMixin

from llama_index.output_parsers.base import OutputParserException, StructuredOutput
from llama_index.output_parsers.utils import _marshal_llm_to_json
from llama_index.types import BaseOutputParser

def _escape_curly_braces(input_string: str) -> str:
    # Replace '{' with '{{' and '}' with '}}' to escape curly braces
    return input_string.replace("{", "{{").replace("}", "}}")

FORMAT_STR = """The output should be ONLY JSON formatted as a JSON instance.

Here is an example:
[
    {
        choice: 1,
        reason: "<insert reason for choice>"
    },
    ...
]
"""

@dataclass
class Answer(DataClassJsonMixin):
    choice: int
    reason: str

class SelectionOutputParser(BaseOutputParser):
    REQUIRED_KEYS = frozenset(Answer.__annotations__)

    def _filter_dict(self, json_dict: dict) -> dict:
        """Filter recursively until a dictionary matches all REQUIRED_KEYS."""
        output_dict = json_dict
        for key, val in json_dict.items():
            if key in self.REQUIRED_KEYS:
                continue
            elif isinstance(val, dict):
                output_dict = self._filter_dict(val)
            elif isinstance(val, list):
                for item in val:
                    if isinstance(item, dict):
                        output_dict = self._filter_dict(item)

        return output_dict

    def _format_output(self, output: List[dict]) -> List[dict]:
        output_json = []
        for json_dict in output:
            valid = True
            for key in self.REQUIRED_KEYS:
                if key not in json_dict:
                    valid = False
                    break

            if not valid:
                json_dict = self._filter_dict(json_dict)

            output_json.append(json_dict)

        return output_json

    def parse(self, output: str) -> Any:
        json_string = _marshal_llm_to_json(output)
        try:
            json_obj = json.loads(json_string)
        except json.JSONDecodeError as e_json:
            try:
                import yaml

                # NOTE: parsing again with pyyaml
                #       pyyaml is less strict, and allows for trailing commas
                #       right now we rely on this since guidance program generates
                #       trailing commas
                json_obj = yaml.safe_load(json_string)
            except yaml.YAMLError as e_yaml:
                raise OutputParserException(
                    f"Got invalid JSON object. Error: {e_json} {e_yaml}. "
                    f"Got JSON string: {json_string}"
                )
            except NameError as exc:
                raise ImportError("Please pip install PyYAML.") from exc

        if isinstance(json_obj, dict):
            json_obj = [json_obj]

        if not json_obj:
            raise ValueError(f"Failed to convert output to JSON: {output!r}")

        json_output = self._format_output(json_obj)
        answers = [Answer.from_dict(json_dict) for json_dict in json_output]
        return StructuredOutput(raw_output=output, parsed_output=answers)

    def format(self, prompt_template: str) -> str:
        return prompt_template + "\n\n" + _escape_curly_braces(FORMAT_STR)


from msrest.serialization import Model

class RunbookCreateOrUpdateParameters(Model):
    """The parameters supplied to the create or update runbook operation.

    All required parameters must be populated in order to send to Azure.

    :param log_verbose: Gets or sets verbose log option.
    :type log_verbose: bool
    :param log_progress: Gets or sets progress log option.
    :type log_progress: bool
    :param runbook_type: Required. Gets or sets the type of the runbook.
     Possible values include: 'Script', 'Graph', 'PowerShellWorkflow',
     'PowerShell', 'GraphPowerShellWorkflow', 'GraphPowerShell'
    :type runbook_type: str or ~azure.mgmt.automation.models.RunbookTypeEnum
    :param draft: Gets or sets the draft runbook properties.
    :type draft: ~azure.mgmt.automation.models.RunbookDraft
    :param publish_content_link: Gets or sets the published runbook content
     link.
    :type publish_content_link: ~azure.mgmt.automation.models.ContentLink
    :param description: Gets or sets the description of the runbook.
    :type description: str
    :param log_activity_trace: Gets or sets the activity-level tracing options
     of the runbook.
    :type log_activity_trace: int
    :param name: Gets or sets the name of the resource.
    :type name: str
    :param location: Gets or sets the location of the resource.
    :type location: str
    :param tags: Gets or sets the tags attached to the resource.
    :type tags: dict[str, str]
    """

    _validation = {
        'runbook_type': {'required': True},
    }

    _attribute_map = {
        'log_verbose': {'key': 'properties.logVerbose', 'type': 'bool'},
        'log_progress': {'key': 'properties.logProgress', 'type': 'bool'},
        'runbook_type': {'key': 'properties.runbookType', 'type': 'str'},
        'draft': {'key': 'properties.draft', 'type': 'RunbookDraft'},
        'publish_content_link': {'key': 'properties.publishContentLink', 'type': 'ContentLink'},
        'description': {'key': 'properties.description', 'type': 'str'},
        'log_activity_trace': {'key': 'properties.logActivityTrace', 'type': 'int'},
        'name': {'key': 'name', 'type': 'str'},
        'location': {'key': 'location', 'type': 'str'},
        'tags': {'key': 'tags', 'type': '{str}'},
    }

    def __init__(self, **kwargs):
        super(RunbookCreateOrUpdateParameters, self).__init__(**kwargs)
        self.log_verbose = kwargs.get('log_verbose', None)
        self.log_progress = kwargs.get('log_progress', None)
        self.runbook_type = kwargs.get('runbook_type', None)
        self.draft = kwargs.get('draft', None)
        self.publish_content_link = kwargs.get('publish_content_link', None)
        self.description = kwargs.get('description', None)
        self.log_activity_trace = kwargs.get('log_activity_trace', None)
        self.name = kwargs.get('name', None)
        self.location = kwargs.get('location', None)
        self.tags = kwargs.get('tags', None)


"""
concentration of frozen particles (unactivated, activated or both)
"""
import numpy as np

from PySDM.products.impl.concentration_product import ConcentrationProduct

class FrozenParticleConcentration(ConcentrationProduct):
    def __init__(
        self,
        *,
        count_unactivated: bool,
        count_activated: bool,
        unit="m^-3",
        name=None,
        specific=False,
        stp=False
    ):
        super().__init__(specific=specific, stp=stp, unit=unit, name=name)
        self.__filter_range = [-np.inf, 0]
        if not count_activated:
            self.__filter_range[0] = -1
        if not count_unactivated:
            self.__filter_range[1] = -1

    def register(self, builder):
        super().register(builder)
        builder.request_attribute("wet to critical volume ratio")

    def _impl(self, **kwargs):
        self._download_moment_to_buffer(
            attr="volume",
            rank=0,
            filter_attr="wet to critical volume ratio",
            filter_range=self.__filter_range,
        )
        return super()._impl(**kwargs)

class FrozenParticleSpecificConcentration(FrozenParticleConcentration):
    def __init__(self, *, count_unactivated, count_activated, unit="kg^-1", name=None):
        super().__init__(
            unit=unit,
            name=name,
            count_activated=count_activated,
            count_unactivated=count_unactivated,
            specific=True,
        )


from flask import Flask, jsonify, request
from PIL import Image
from itertools import groupby
import numpy as np
import tensorflow as tf
from tensorflow import keras
import urllib.request

app = Flask(__name__)
@app.route("/")
def idx():
    return "abcfd"

#Run on http://127.0.0.1:5000

# @app.route("/calculate")
# def apitest():
#     return "API working"

# if __name__ == "__main__":
#   app.run(host="0.0.0.0", debug=False, port=5005)

elements_pred=[]
def calculate(img_loc):
    global elements_pred
    'loading image'
    urllib.request.urlretrieve(img_loc,"input.png")
    image = Image.open("input.png").convert("L")
    image.show()

    'resizing to 28 height pixels'
    w = image.size[0]
    h = image.size[1]
    r = w / h # aspect ratio
    new_w = int(r * 28)
    new_h = 28
    new_image = image.resize((new_w, new_h))

    'converting to a numpy array'
    new_image_arr = np.array(new_image)

    'inverting the image to make background = 0'
    new_inv_image_arr = 255 - new_image_arr

    'rescaling the image'
    final_image_arr = new_inv_image_arr / 255.0

    'splitting image array into individual digit arrays using non zero columns'
    m = final_image_arr.any(0)
    out = [final_image_arr[:,[*g]] for k, g in groupby(np.arange(len(m)), lambda x: m[x] != 0) if k]

    '''
    iterating through the digit arrays to resize them to match input
    criteria of the model = [mini_batch_size, height, width, channels]
    '''
    num_of_elements = len(out)
    elements_list = []

    for x in range(0, num_of_elements):

        img = out[x]

        #adding 0 value columns as fillers
        width = img.shape[1]
        filler = (final_image_arr.shape[0] - width) / 2

        if filler.is_integer() == False:    #odd number of filler columns
            filler_l = int(filler)
            filler_r = int(filler) + 1
        else:                               #even number of filler columns
            filler_l = int(filler)
            filler_r = int(filler)

        arr_l = np.zeros((final_image_arr.shape[0], filler_l)) #left fillers
        arr_r = np.zeros((final_image_arr.shape[0], filler_r)) #right fillers

        #concatinating the left and right fillers
        help_ = np.concatenate((arr_l, img), axis= 1)
        element_arr = np.concatenate((help_, arr_r), axis= 1)

        element_arr.resize(28, 28, 1) #resize array 2d to 3d

        #storing all elements in a list
        elements_list.append(element_arr)

    elements_array = np.array(elements_list)

    'reshaping to fit model input criteria'
    elements_array = elements_array.reshape(-1, 28, 28, 1)

    'predicting using the model'
    #model = keras.models.load_model("C:\\Users\\Priyam_Shukla\\Downloads\\Final Year Project\\model1")
    model = keras.models.load_model("model1")
    elements_pred =  model.predict(elements_array)
    elements_pred = np.argmax(elements_pred, axis = 1)
    print(elements_pred)

def math_expression_generator(arr):

    op = {
              10,   # = "/"
              11,   # = "+"
              12,   # = "-"
              13    # = "*"
                  }

    m_exp = []
    temp = []

    'creating a list separating all elements'
    for item in arr:
        if item not in op:
            temp.append(item)
        else:
            m_exp.append(temp)
            m_exp.append(item)
            temp = []
    if temp:
        m_exp.append(temp)

    'converting the elements to numbers and operators'
    i = 0
    num = 0
    for item in m_exp:
        if type(item) == list:
            if not item:
                m_exp[i] = ""
                i = i + 1
            else:
                num_len = len(item)
                for digit in item:
                    num_len = num_len - 1
                    num = num + ((10 ** num_len) * digit)
                m_exp[i] = str(num)
                num = 0
                i = i + 1
        else:
            m_exp[i] = str(item)
            m_exp[i] = m_exp[i].replace("10","/")
            m_exp[i] = m_exp[i].replace("11","+")
            m_exp[i] = m_exp[i].replace("12","-")
            m_exp[i] = m_exp[i].replace("13","*")

            i = i + 1

    'joining the list of strings to create the mathematical expression'
    separator = ' '
    m_exp_str = separator.join(m_exp)

    return (m_exp_str)

@app.route("/func", methods = ['POST'])

def func():
    'creating the mathematical expression'
    with app.app_context():
        #test code
        image_loc = request.form.get('image_url')
        calculate(image_loc)
        m_exp_str = math_expression_generator(elements_pred)

    'calculating the mathematical expression using eval()'
    while True:
        try:
            answer = eval(m_exp_str)    #evaluating the answer
            answer = round(answer, 2)
            equation  = m_exp_str + " = " + str(answer)
            print(equation)   #printing the equation
            result={"answer":equation}
            return jsonify(result)
            #break

        except SyntaxError:
            print("Invalid predicted expression!!")
            print("Following is the predicted expression:")
            print(m_exp_str)
            break

#print(func())

# if __name__=="__main__":
#     app.run(debug=True)


"""
Code for colorization network adapted from
Colorization as a Proxy Task for Visual Understanding, Larsson, Maire, Shakhnarovich, CVPR 2017
https://github.com/gustavla/self-supervision
"""

import os
import tensorflow as tf
import deepdish as dd

from imm.models.selfsup import info
from imm.models.selfsup import vgg16

def build_vgg16(input, reuse=False, pretrained_file=None):
  with tf.variable_scope('vgg16', reuse=reuse):
    data = dd.io.load(pretrained_file, '/data')
    inf = info.create(scale_summary=True)
    testing = True

    input_raw = input
    # convert to grayscale
    input = tf.reduce_mean(input, 3, keep_dims=True)
    # normalize
    input = input / 255.0
    # centre
    input = input - 114.451 / 255.0
    net = vgg16.build_network(input, info=inf, parameters=data,
                             final_layer=False,
                             phase_test=testing,
                             pre_adjust_batch_norm=True,
                             use_dropout=True)

    # replace the input with the original input in RGB
    net['input'] = input_raw
    return net

if __name__ == '__main__':
  pretrained_file = '/users/tomj/minmaxinfo/data/models/vgg16.caffemodel.h5'
  input = tf.placeholder(tf.float32, [None, 128, 128, 1])
  net = build_vgg16(input, pretrained_file=pretrained_file)