# coding: utf-8

"""
This is only valid for Brazilian Real and B3 (Brazil stock exchange).
Todo localization
"""

from decimal import *
import csv, re, copy, json
from calendar import monthrange
from collections import defaultdict
from datetime import datetime, timedelta, date

# Options month codes
CALL = ['A','B','C','D','E','F','G','H','I','J','K','L']
PUT = ['M','N','O','P','Q','R','S','T','U','V','W','X']

MKT = {
        'VIS':{'threshold_exempt':20000},
        'OPV':{'threshold_exempt':0}
        }

class InsufficientStocks(Exception):
    """ Attempt to sell more stocks than in wallet """
    pass

class StockNotFound(Exception):
    """ Attempt to sell more stocks than in wallet """
    pass

class ObjectifyData():
    """
    Opens CSV file and objetify to a sequence of months and a list of stocks
    """
    def __init__(self, mkt_type, file, path="files/"):
        self.mkt_type = mkt_type
        self.file_path = path
        self.file = file
        self.mm = Months(mkt_type)
        self.stocks = {}
        self.running_options = RunningOptions()
        self.stocks_wallet = defaultdict()
        self.illegal_operation = defaultdict(list)

    def two_digits_month(self, month):
        """
        Four digits year plus two digits months for use in dict keys,
        to allows order keys
        """
        if month < 10:
            month = "0%d" % (month)
        return month

    def read_line(self, line):
        # line = [10/07/2019,1-Bovespa,C,VIS,TAEE11 UNT N2,,100,"28,69",2869,D]
        line_dict = {}
        try:
            if self.mkt_type != line[3]:
                return

            # Column 10 filed, to ignore.
            # import pdb; pdb.set_trace()
            if len(line) == 11 and line[10]:
                return

            """
            !!! Compatibilizar as datas. Provavelmente tudo com 2 digitos no ano.
            """
            dt = datetime.strptime(line[0], '%d/%m/%Y').date()
            month = self.two_digits_month(dt.month)
            year_month_id = "%d%s" % (int(dt.year), month)
            year_month_id = int(year_month_id)
            stock = line[4].split()[0]
            qt = int(line[6].replace('.', ''))

            # Comma separator for decimal
            if ',' in line[7]:
                unit_price = Decimal(line[7].replace('.', '').replace(',', '.').replace('"', ''))
            # Dot separator for decimal
            else:
                unit_price = Decimal(line[7].replace('"', ''))
            unit_price = round(unit_price, 2)
#            print(line)
            value = Decimal(line[8].replace('.', '').replace(',', '.').replace('"', ''))
            value = round(value, 2)
            buy_sell = line[2]

            line_dict = {'year_month_id':year_month_id,
                            'dt':dt,
                            'stock':stock,
                            'value':value,
                            'qt':qt,
                            'unit_price': unit_price,
                            'buy_sell': buy_sell,
                            }
            # print(line_dict)
            # import pdb; pdb.set_trace()
            return line_dict
        except Exception as e:
            print(e)
            raise

    def is_stock(self, asset):
        """ Whether is stock or option """
        """ There is a specific field for this on the csv """
        pattern = '^[A-Z]{4}\d+$'
        regexp = re.compile(pattern, re.IGNORECASE)
        if regexp.search(asset):
            return True
        return False

    def file2object(self, csv_only=False):
        """
        Process file line by line using the file's returned iterator
        to permit working with large files.
        Build months object and stocks objects.
        """
        try:
            file_path = '%s%s' % (self.file_path, self.file)
            with open(file_path) as file_handler:
                # import pdb; pdb.set_trace()
                csv_reader = csv.reader(file_handler, quotechar='"', delimiter=',')

                # To remove excel header if exists
                # next(csv_reader, None)
                csv_reader = reversed(list(csv_reader))

                if csv_only:
                    return csv_reader

                while True:
                    try:
                        line = next(csv_reader)
                        line = self.read_line(line)
                        if not line:
                            continue

                        # if not 'ITSA4' in line['stock']:
                        #      continue

                        # Whenever the months change (for each new month),
                        # verify option expiration (OTM)
                        if not line['year_month_id'] in self.mm:
                            if self.running_options:
                                updated_options = self.running_options.chk_running_options(line)
                                self.months_update_option(updated_options)

                        # Objectify stock and months
                        stock_updated_instance = self.objectify_stock(line)
                        if not stock_updated_instance:
                            continue
                        self.objectify_months(stock_updated_instance)

                        # Create or remove a running option.
                        if not self.is_stock(line['stock']): # If is option
                            if stock_updated_instance['qt_total'] == 0:
                                del self.running_options[line['stock']]
                            else:
                                self.running_options[line['stock']] = stock_updated_instance

                    except Exception as e:
                        raise

        except (IOError, OSError):
            print("Error opening / processing file")
        except StopIteration:
            pass

        # After the last operation check if there are OTM options
        updated_options = self.running_options.chk_running_options(None)
        self.months_update_option(updated_options)

        return self.mm

    def months_update_option(self, updated_options):
        if updated_options:
            for uo in updated_options:
                self.objectify_months(uo)

    def objectify_stock(self, line):
        """
        Receives a csv line in dict format and create a stock object.
        Each stock object instance is a stock.
        For each line, create/update the object which returns the object __dict__
        """
        # Create or get instance of a stock account
        # It is populating self.stocks, a still not used dict.
        stock = self.stocks.setdefault(line['stock'], StockCheckingAccount(line['stock']))

        # Set operation attributes
        stock.start_operation(**line)

        # Calculate and set instance values
        if line['buy_sell'] == 'C':
            stock.buy(**line)
        if line['buy_sell'] == 'V':
            try:
                stock.sell(**line)
            except InsufficientStocks:
                line['exception'] = InsufficientStocks('%s: Insufficient stocks to sale (%s)'  % (line['stock'], line['dt']))
                self.illegal_operation[line['stock']].append(line)
                return None

        # deep copy operation values
        cp_stock = copy.deepcopy(stock.__dict__)

        #if cp_stock['qt_total']:
        self.stocks_wallet[cp_stock['stock']] = cp_stock
        return cp_stock

    def objectify_months(self, stock_updated_instance):
        #line_dict = {'year_month_id':year_month_id, 'dt':dt, 'stock':stock, 'value':value, 'buy_sell': buy_sell }
        month = stock_updated_instance['year_month_id']
        month_dict = self.mm[month]
        # Set operation attributes
        self.mm.month_populate(month, stock_updated_instance)

class RunningOptions():
    def __init__(self):
        self._running_options = {}

    def __iter__(self):
        return iter(self._running_options)

    def __getitem__(self, option):
        return self._running_options[option]

    def __setitem__(self, option, updated_instance):
        self._running_options[option] = updated_instance

    def __delitem__(self, option):
        self._running_options.pop(option, None)

    def get_expiring_month(self):
        """ Get the month validity on option code.
        It is the fourth letter of the option code, following
        PUT and CALL list codes """
        option_type = ''
        option_code_letter = self.running_option['stock'][4]
        try:
            expiring_month = CALL.index(option_code_letter) + 1
            option_type =  'call'
#            expiring_date = self.expiring_date(expiring_month)
        except ValueError:
            expiring_month = PUT.index(option_code_letter) + 1
            option_type =  'put'
#            expiring_date = self.calc_expiring_date(expiring_month)
        except:
            raise
        return {'option_type':option_type, 'expiring_month':expiring_month}

    def calc_expiring_date(self, expiring_month):
        """ Calculates the expiration date of the option, assuming that
        it is not possible to carry for more than a year. """
        buy_month = self.running_option['dt'].month
        current_year = datetime.today().year
        expiring_year = self.running_option['dt'].year

        """ If the month of the option is smaller than the actual month,
        it refers to the month of the next year. """
        if expiring_month < buy_month:
            expiring_year += 1
        expiring_date = date(expiring_year,
                                expiring_month,
                                monthrange(expiring_year, expiring_month)[1])
        return expiring_date

    def chk_option_expiring(self, reference_date):
        """ Discover option expiring date based on option code and buy date.
        If expired, set loss to the option object. """

        def expired_otm(expiring_date):
            """ Verify if option has lost validity """
            if reference_date > expiring_date:
                return True
            return False

        def set_option_loss():
            """ Create an operation of selling option when it expires """
            sold_option = copy.deepcopy(self.running_option)
            sold_option['loss'] = self.running_option['qt_total'] * self.running_option['avg_price']
            sold_option['value'] = 0
            sold_option['qt_total'] = 0
            sold_option['buy_sell'] = 'V'
            return sold_option

        decoded_option = self.get_expiring_month()
        expiring_date = self.calc_expiring_date(decoded_option['expiring_month'])

        if expired_otm(expiring_date):
            sold_option = set_option_loss()
            return sold_option

    def cleanup_running_option(self, clean_running_option):
        for cro in clean_running_option:
            del self._running_options[cro['name']]

    def chk_running_options(self, line):
        try:
            # Check options status against any date (reference date).
            # Using a negotiation date (year_month_id).
            reference_date = line['dt']
        except:
            # After the end of the loop of the csv file, check against today.
            reference_date = datetime.now().date()

        clean_running_option = []
        sold_options = []
        for key in self._running_options:
            self.running_option = self._running_options[key]
            sold_option = self.chk_option_expiring(reference_date)
            if sold_option:
                sold_options.append(sold_option)
                clean_running_option.append(sold_option)

        self.cleanup_running_option(clean_running_option)
        return sold_options

class StockCheckingAccount():
    def __init__(self, name):
        self.name = name
        self.qt_total = 0
        self._avg_price = 0
        self._qt_total_prev = 0 # for balance
        self._avg_price_prev = 0 # for balance

    @property
    def avg_price(self):
        return self._avg_price

    @avg_price.setter
    def avg_price(self, new_avg_price):
        self._avg_price = new_avg_price

    @property
    def avg_price_prev(self):
        return self._avg_price_prev

    def new_avg_price(self, qt, unit_price):
        current_position = self.qt_total * self.avg_price
        new_dock = qt * unit_price
        new_avg_price = (current_position + new_dock) / (self.qt_total + qt)
        self._avg_price_prev = self.avg_price
        self.avg_price = new_avg_price

        # !!! This is a workaround that was working before p3 migration
        self.__dict__['avg_price'] = self.avg_price
        self.__dict__['avg_price_prev'] = self.avg_price_prev

    def profit_loss(self, qt, unit_price):
        if unit_price <= self.avg_price:
            self.loss += (qt * self.avg_price) - (qt * unit_price)
        elif unit_price >= self.avg_price:
            self.profit += (qt * unit_price) - (qt * self.avg_price)

    def start_operation(self, **kwargs):
        # Pass all csv line values to stockCheckingAccount instance
        self.__dict__.update(kwargs)
        # Init profit and loss for this operation
        # Will calculate this based on the avg_price * sell price.
        self.profit = 0
        self.loss = 0

    def buy(self, **line):
        self.new_avg_price(line['qt'], line['unit_price'])
        self._qt_total_prev = self.qt_total
        self.qt_total += line['qt']

    def sell(self, **line):
        self.profit_loss(line['qt'], line['unit_price'])
        if line['qt'] > self.qt_total:
            raise InsufficientStocks()
        self.qt_total -= line['qt']
        if self.qt_total == 0:
            self._qt_total_prev = 0
            self._avg_price_prev = 0
            self.avg_price = 0

    def __repr__(self):
        return 'name:{}, dt:{}, qt: {}, avg_price: {}, lucro:{}, prejuizo:{}'\
                    .format(self.name, self.dt, self.qt_total, self.avg_price, self.profit, self.loss)

class Months():
    def __init__(self, mkt_type):
        self._months = {}
        self.mkt_type = mkt_type

    def __getitem__(self, month):
        # stocks = {'operations':[], 'totalization':{}}
        return self._months.setdefault(month, {'dt':datetime,
                                                'month_buy':0, 'month_sell':0,
                                                'month_gain':0, 'month_loss':0,
                                                'cumulate_gain':0, 'cumulate_loss':0,
                                                'operations':defaultdict(list),
                                                'tax':{}})

    def __iter__(self):
        return iter(self._months)

    def get_month(self, month):
        return self._months[month]

    def keys(self):
        return sorted(self._months.keys())

    def month_populate(self, month, stock_updated_instance):
        line = stock_updated_instance
        # if 'BMGB' in line['stock']:
        #     import pdb; pdb.set_trace()
        month_dict = self._months[month]
        month_dict['dt'] = line['dt']

        if (line['buy_sell'] == 'C'):
            month_dict['month_buy'] += line['value']
        elif (line['buy_sell'] == 'V'):
            month_dict['month_sell'] += line['value']
        if stock_updated_instance is not None:
            month_dict['operations'][line['stock']].append(stock_updated_instance)

    def subtract_one_month(self, this_month):
        """ receives current month, returns the key of previous active month """
        self._months
        try:
            this_month_index = self.keys().index(this_month)
            if this_month_index:
                return self.keys()[this_month_index - 1]
            return None
        except:
            raise

    def cumulate_loss(self, this_month):
        previous_month = self.subtract_one_month(this_month)
        self._months[this_month]['cumulate_loss'] = self._months[this_month]['month_loss']
        if previous_month:
            self._months[this_month]['cumulate_loss'] += self._months[previous_month]['cumulate_loss']

    def threshold_exempt(self, this_month):
        """
        When the total amount sold on one month is less than R$20,000.00
        no tax are applied over gain
        """
        return True if self._months[this_month]['month_sell'] > MKT[self.mkt_type]['threshold_exempt'] else False

    def tax_calc(self, final_balance):
        #return {'final_balance':final_balance, 'tax_amount': final_balance * 0.15}
        #import pdb; pdb.set_trace()
        return {'final_balance':final_balance, 'tax_amount': final_balance * Decimal(0.15)}

    def month_add_detail(self):
        """
        Add gain, loss for all months.
        Calculate final balance and due tax.
        """
        # import pdb; pdb.set_trace()
        for month in self.keys():
            # print("\n\n"
            # print(month

            balance_current_month = 0
            # if self._months[month]['month_sell'] > 20000:
            # print("%s Total vendas : %s" % (month, self._months[month]['month_sell'])

            for stock, values in self._months[month]['operations'].items():
                for operation in values:
                    if operation['buy_sell'] == 'V':
                        if operation['profit']:
                            balance_current_month += operation['profit']
                        elif operation['loss']:
                            balance_current_month -= operation['loss']

                # print(operation
            if balance_current_month > 0:
                self._months[month]['month_gain'] =  balance_current_month
            elif balance_current_month < 0:
                self._months[month]['month_loss'] =  balance_current_month

            self.cumulate_loss(month)

            if self.threshold_exempt(month):
                if balance_current_month > 0:
                    final_balance = balance_current_month + self._months[month]['cumulate_loss']
                    if final_balance > 0:
                        self._months[month]['cumulate_loss'] = 0
                        self._months[month]['tax'] = self.tax_calc(Decimal(final_balance))
                    elif final_balance <= 0:
                        self._months[month]['cumulate_loss'] = final_balance

            # print(balance_current_month)
            # print(self._months[month]['month_gain'])
            # print(self._months[month]['month_loss'])
            # print(self._months[month]['cumulate_loss'])
            # print(self._months[month]['tax'])

class Report():
    def __init__(self, stock_price_file, stocks_wallet, months):
        self.current_prices = dict
        self.curr_prices_dt = ''
        self.illegal_operation = defaultdict(list)
        self.months = months
        self.stock_price_file = stock_price_file
        self.stocks_wallet = stocks_wallet

    def get_current_quotations(self):
        """
        Whatever market data source you have.
        This is getting form a serialized JSON from Yahoo Finance, from the
        stock_price.py script
        """
        # import pdb; pdb.set_trace()
        with open(self.stock_price_file, 'r') as f:
            stock_price = json.load(f)
            self.current_prices = stock_price[list(stock_price.keys())[0]]
            self.curr_prices_dt = list(stock_price.keys())[0]
            # import pdb; pdb.set_trace()

    def build_statement(self, values):
        statement = {'qt_total_start':0,
                     'avg_price_start':0,
                     'values':[],
                     'qt_total_end':0,
                     'avg_price_end':0 }

        if len(values) == 1:
            operation = values[0]

            statement = {'qt_total_start':operation['_qt_total_prev'],
                         'avg_price_start':operation['_avg_price_prev'],
                         'ops':[operation],
                         'qt_total_end':operation['qt_total'],
                         'avg_price_end':operation['avg_price'] }
        elif len(values) > 1:
            operation_start = values[0]
            operation_end = values[-1]
            statement = {'qt_total_start':operation_start['_qt_total_prev'],
                         'avg_price_start':operation_start['_avg_price_prev'],
                         'ops':values,
                         'qt_total_end':operation_end['qt_total'],
                         'avg_price_end':operation_end['avg_price'] }
        return statement

    def months_build_data(self):
        months = {}
        months_operations = {}
        months_keys = self.months.keys()
        months_keys.reverse()
        for month in months_keys:
            month_data = self.months.get_month(month)
            months[month] = month_data
            months_operations[month] = dict(month_data['operations'])
        return (months, months_operations)

    def report(self):
        """ to refactor - was used for command line tests
        use the months_build_data method as web app """
        months_keys = self.months.keys()
        for month in months_keys:
            month_data = self.months.get_month(month)
            print("\n\n===================================== \n\n\n")
            print('Mês: {0} / Vendas no mes: {1}'.format(month, month_data['month_sell']))
            print('Lucro: {0} / PositionViewPrejuízo: {1} / Prejuizo acumulado: {2}'.format(month_data['month_gain'], month_data['month_loss'], month_data['cumulate_loss']))

            if month_data['tax']:
                print('Balanço mês: {1} / Imposto devido: {0}'.format(month_data['tax']['tax_amount'], month_data['tax']['final_balance']))

            for stock, values in month_data['operations'].items():
                # if stock != 'KLBN11':
                #     continue
                print
                print("Stock: %s" % (stock))
                for ops in values:
                    if ops['buy_sell'] == 'V':
                        if ops['profit']:
                            print("%s: %s " % ('Balance', ops['profit']))
                        if ops['loss']:
                            print("%s: -%s " % ('Balance', ops['loss']))
                statement = self.build_statement(values)
                print(statement)

    def current_position(self):
        """ to refactor - was used for command line tests
        use the months_build_data method as web app """

        """
        On the last day of the month you can sell your stocks where you are
        losing, to reduce from your futures gains.
        """
        position = []
        summary = defaultdict(Decimal)
        def get_stock_position(stock, values):
            try:
                current_price = self.current_prices[stock]
                buy_position = values['qt_total'] * values['avg_price']

                if current_price['price']=='None':
                    raise StockNotFound

                curr_position = values['qt_total'] * round(Decimal(current_price['price']), 2)
                balance = curr_position - buy_position
                balance_pct = round((balance * 100 / buy_position), 2)
                # import pdb; pdb.set_trace()
                return (stock, values, Decimal(buy_position), Decimal(curr_position), Decimal(balance), balance_pct)
            except KeyError as e:
                # If KeyError 'avg_price', means that no buy input was provided.
                import pdb; pdb.set_trace()
                raise StockNotFound

        print("\n\n\nCurrent prices: %s" % (self.curr_prices_dt))
        for stock, values in self.stocks_wallet.items():
            if values['qt_total']:
                try:
                    (stock, values, buy_position, curr_position, balance, balance_pct) = get_stock_position(stock, values)
                    print("%s: Qt:%d - Buy avg: R$%.2f - Cur Price: R$%s - Buy Total: R$%.2f - Cur Total: R$%.2f - Balance: R$%.2f ( %.2f%s )" % (stock, values['qt_total'], values['avg_price'], self.current_prices[stock]['price'], buy_position, curr_position, balance, balance_pct, '%'))
                    position.append({
                        'stock': stock,
                        'qt': values['qt_total'],
                        'buy_avg': values['avg_price'],
                        'curr_price': self.current_prices[stock]['price'],
                        'buy_total':buy_position,
                        'cur_total':curr_position,
                        'balance': balance,
                        'balance_pct': balance_pct
                    })
                    summary['balance'] += balance
                    # import pdb; pdb.set_trace()
                    summary['buy_total'] += buy_position
                    summary['cur_total'] += curr_position

                    # import pdb; pdb.set_trace()

                except StockNotFound:
                    values['exception'] = StockNotFound('%s: StockNotFound (%s)'  % (values['stock'], values['dt']))
                    self.illegal_operation[values['stock']].append(values)
        # import pdb; pdb.set_trace()
        if not summary['buy_total'] == 0:
            summary['balance_pct'] =  (100 * (summary['cur_total'] - summary['buy_total'])) / summary['buy_total']
        return (position, summary)

    def illegal_operations(self, illegal_operations):
        print("\n\n\n")
        print("Illigal operations")
        print("==================")
        for list_io in illegal_operations:
            for stock, values in list_io.items():
                for err in values:
                    print(err['exception'])

if __name__ == "__main__":
    import argparse
    illegal_operations = []
    def get_args():
        """
        To get stock prices from your CSV file, call this script wiht arguments.
        python stock_price.py --path --file
        """
        parser = argparse.ArgumentParser()
        parser.add_argument('--mkt_type', default='VIS')
        parser.add_argument('--path', default='test_sample')
        parser.add_argument('--file', default='sample.csv')
        args = parser.parse_args()
        return args

    def handle_data(args):
        b3_tax_obj = ObjectifyData(mkt_type='VIS', path=args.path, file=args.file)
        return b3_tax_obj

    def gather_illegal_operation(illegal_operation):
        illegal_operations.append(illegal_operation)

    def months_reconcile(b3_tax_obj):
        months = b3_tax_obj.file2object()
        months.month_add_detail()
        return months

    def generate_reports(stocks_wallet, months):
        print("\n\nMercado à vista")
        print("===============")
        # report = Report()
        report = Report('/var/tmp/stock_price.json', stocks_wallet, months)

        report.report()

        report.get_current_quotations()
        report.current_position()
        gather_illegal_operation(report.illegal_operation)
        report.illegal_operations(illegal_operations)

    args = get_args()
    b3_tax_obj = handle_data(args)
    gather_illegal_operation(b3_tax_obj.illegal_operation)
    months = months_reconcile(b3_tax_obj)
    generate_reports(b3_tax_obj.stocks_wallet, months)

    exit(0)

    print("\n\nOpcoes PUT")
    print("==========")
    b3_tax_obj = ObjectifyData('OPV', path=args.path, file=args.file)
    months = b3_tax_obj.file2object()
    months.month_add_detail()
    report.report(months)


'''
Caitlin J. Corbin
Intro to Python
Finds Numbers
6/22/2020

'''
#uses an iterator and selection structure to calculate findNum
count = 0
for i in range (100, 201):
    if (i % 6 == 0 and i % 5 != 0) or (i % 5 == 0 and i % 6 != 0):
        print(str(i), ""),
        count = count + 1
    if count % 10 == 0:
        print("")


import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import pandas as pd
from utils import data_path

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets = external_stylesheets)

df = pd.read_csv(data_path)

fig = px.histogram(df, x="Rating", range_x=[0.8, 5.2])

app.layout = html.Div(children=[
    html.H1('Dashboard aplicaciones Android'),
    html.Div(children='''
    Primera app de aprendizaje con Dash
    '''),
    dcc.Graph(
        id='example-graph',
        figure=fig
    )
])

if __name__ == '__main__':
    app.run_server(host="0.0.0.0", port=1010, debug=True)


from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
import sys
#from clean_string import CleanString
from get_data_from_db import GetDataFromDB
#from jpype import getDefaultJVMPath, shutdownJVM, startJVM
import Levenshtein as lev
from similarity.ngram import NGram
import os

class Main(GetDataFromDB):

	def levenshtein(self, machined_product_cleaned_name, machined_master_product_cleaned_name):
		lev_sim = lev.ratio(machined_product_cleaned_name,machined_master_product_cleaned_name)
		return lev_sim

	def cosinus(self, machined_product_cleaned_name, machined_master_product_cleaned_name):
		values = []
		values.append(machined_product_cleaned_name)
		values.append(machined_master_product_cleaned_name)

		vectorizer =  CountVectorizer().fit_transform(values)
		vectors = vectorizer.toarray()
		cos_sim = cosine_similarity(vectors)
		return cos_sim[0][1]

	def jaccard(self, first, second):
		fcount = 0
		scount = 0
		for f in first:
			if f in second:
				scount += 1
		fcount = len(first)+len(second)
		jac_sim = float(scount) / float(fcount)

		return jac_sim

	def q_gram(self, machined_product_cleaned_name, machined_master_product_cleaned_name, q_gram_number=3):
		q_gram = NGram(q_gram_number)
		q_gram_sim = 1-q_gram.distance(machined_product_cleaned_name, machined_master_product_cleaned_name)

		return q_gram_sim

	def compare_cleaned_names(self, machined_products_df, machined_master_products_df):
		result_df = pd.DataFrame()
		product_counter = 0
		true_and_upper_limit_matching_counter = 0
		false_and_upper_limit_matching_counter = 0
		true_and_under_limit_matching_counter = 0
		false_and_under_limit_matching_counter = 0
		for ind1 in machined_products_df.index:
			product_counter += 1
			print(f"compared_product_count: {product_counter}")
			biggest_sim = -1
			result_dict = {}
			for ind2 in machined_master_products_df.index:
				#sim = self.cosinus(machined_products_df['cleaned_name'][ind1], machined_master_products_df['cleaned_name'][ind2])
				#sim = self.levenshtein(machined_products_df['cleaned_name'][ind1], machined_master_products_df['cleaned_name'][ind2])
				#sim = self.jaccard(machined_products_df['cleaned_name'][ind1], machined_master_products_df['cleaned_name'][ind2])
				sim = self.q_gram(machined_products_df['cleaned_name'][ind1], machined_master_products_df['cleaned_name'][ind2],6)
				if sim > biggest_sim:
					#if machined_products_df['master_product_id'][ind1] == machined_master_products_df['master_product_id'][ind2]:
					biggest_sim = sim
					result_dict['product_id'] = machined_products_df['product_id'][ind1]
					result_dict['current_master_product_id'] = machined_products_df['master_product_id'][ind1]
					result_dict['new_master_product_id'] = machined_master_products_df['master_product_id'][ind2]
					result_dict['sim'] = sim
					# ürünün sitedeki ismi
					result_dict['product_actual_name'] = machined_products_df['name'][ind1]
					# ürünü eşleşmesi gereken master product ismi
					result_dict['must_match_master_product_actual_name'] = machined_products_df['must_match_master_product_actual_name'][ind1]
					# ürünün eşleştiği master product ismi
					result_dict['matched_master_product_actual_name'] = machined_master_products_df['name'][ind2]
					# ürünün sitedeki isminin temizlenmiş hali
					result_dict['product_cleaned_name'] = machined_products_df['cleaned_name'][ind1]
					# ürünün eşleştiği master ürünün temizlenmiş hali
					result_dict['matched_master_product_cleaned_name'] = machined_master_products_df['cleaned_name'][ind2]

			if biggest_sim > 0.50: # benzerlik limitin üstünde ise
				if result_dict['new_master_product_id'] == result_dict['current_master_product_id']:# eşleşme doğru ve ve benzerlik limitin üstünde ise
					true_and_upper_limit_matching_counter += 1
					print(f"true_and_upper_limit_matching_counter: {true_and_upper_limit_matching_counter}")
					# insert result df
					result_dict['status'] = 1
					result_df = result_df.append(result_dict, ignore_index=True)
				else:# eşleşme yanlış ve benzerlik limitin üstünde ise
					false_and_upper_limit_matching_counter += 1
					print(f"false_and_upper_limit_matching_counter: {false_and_upper_limit_matching_counter}")
					# insert result df
					result_dict['status'] = 2
					result_df = result_df.append(result_dict, ignore_index=True)
			else: # benzerlik limitin altında ise
				if result_dict['new_master_product_id'] == result_dict['current_master_product_id']:# eşleşme doğru ve ve benzerlik limitin altında ise
					true_and_under_limit_matching_counter += 1
					print(f"true_and_under_limit_matching_counter: {true_and_under_limit_matching_counter}")
					# insert result df
					result_dict['status'] = 3
					result_df = result_df.append(result_dict, ignore_index=True)
				else:# eşleşme yanlış ve benzerlik limitin altında ise
					false_and_under_limit_matching_counter += 1
					print(f"false_and_under_limit_matching_counter: {false_and_under_limit_matching_counter}")
					# insert result df
					result_dict['status'] = 4
					result_df = result_df.append(result_dict, ignore_index=True)

		if result_df.empty:
			return "No Result Found"
		convert_dict = {'current_master_product_id': int,'new_master_product_id': int,'product_id': int,
						'product_cleaned_name': str,'matched_master_product_cleaned_name': str,'sim': float,'status':int,
						'product_actual_name':str ,'matched_master_product_actual_name':str,'must_match_master_product_actual_name':str}
		data_typed_result_df = result_df.astype(convert_dict)
		result_df = data_typed_result_df[['product_id','current_master_product_id','new_master_product_id','sim',
										'product_actual_name','must_match_master_product_actual_name','matched_master_product_actual_name',
										'product_cleaned_name','matched_master_product_cleaned_name','status']]
		return result_df

	def write_to_excel(self,df, file_name):
		writer = pd.ExcelWriter(file_name)
		df.to_excel(writer)
		writer.save()

	def change_color_group(self,x):
		color_df = x.copy()
		color_df.loc[color_df['status'] == 1, :] = 'background-color: white'
		color_df.loc[color_df['status'] == 2, :] = 'background-color: red'
		color_df.loc[color_df['status'] == 3, :] = 'background-color: blue'
		color_df.loc[color_df['status'] == 4, :] = 'background-color: yellow'
		return color_df

	def run(self):
		# start jvm
		# ZEMBEREK_PATH = r'%s/zemberek-full.jar' % os.getcwd()
		# startJVM(getDefaultJVMPath(), '-ea', '-Djava.class.path=%s' % (ZEMBEREK_PATH))

		# cleaned products from db
		# products_df = self.get_products_from_db()
		# clean_string = CleanString(products_df)
		# machined_products_df = clean_string.run()

		machined_products_df = pd.read_excel('cleaned_data/cleaned_products.xlsx', index_col=0)

		#self.write_to_excel(machined_products_df, "machined_products_output.xlsx")

		# cleaned master_products from excel
		machined_master_products_df = pd.read_excel('cleaned_data/cleaned_master_products.xlsx', index_col=0)

		# cleaned master_products from db
		# master_products_df = self.get_limit_master_products_from_db()
		# clean_string = CleanString(master_products_df)
		# machined_master_products_df = clean_string.run()

		result_df = self.compare_cleaned_names(machined_products_df, machined_master_products_df)
		print(result_df)
		colored_df = result_df.style.apply(self.change_color_group, axis=None)

		self.write_to_excel(colored_df, "result_data/qgram6_result_output.xlsx")

		# close jvm
		# shutdownJVM()

main = Main()
main.run()


# Escreva um programa que lê do usuário uma lista de nomes e que lê um nome 'n' de
# pesquisa. O programa deverá escrever na tela “Nome encontrado” se ‘n’ for encontrado na
# lista. Senão, deverá imprimir “Nome não encontrado”.

nameList = []
nameSearch = False
nameSearchInput = ''
choose = ''

while choose != 'N':
    nameInput = input("Type a name: ").capitalize()

    if nameInput.count(' ') >= 1:
        print("Only single names. Type again.")
        while nameInput.count(' ') >= 1: # Each time the user type a compound name,
            # the loop will be actived
            nameInput = input("Type a name: ")
    else:
        nameList.append(nameInput)

    choose = input("Do you want to add another name? (Y/N)\n").upper()

    while choose != 'Y' and choose != 'N':
        print("Invalid option.")
        choose = input("Do you want to add another name? (Y/N)\n").upper()

while True:
    nameSearchInput = input("Type a name to find in a name list: ").capitalize()

    if nameSearchInput.isalpha():
        print(f"Typed name: {nameSearchInput}\n")
        if nameSearchInput in nameList:
            print("Name found.")
        break
    else:
        print("Type only names. Type again.")


from __future__ import division

import argparse
import codecs
from collections import defaultdict
import json
import csv
import os
import re
import sys
import asyncio
import time
from konlpy.tag import Kkma, Twitter
from bs4 import BeautifulSoup

try:
    from urlparse import urljoin
    from urllib import urlretrieve
except ImportError:
    from urllib.parse import urljoin, quote
    from urllib.request import urlretrieve

import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException

import django
from django.conf import settings

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
django.setup()

from app.serializers import InstaSerializer
from django.db.utils import IntegrityError
import datetime

# SELENIUM CSS SELECTOR
CSS_LOAD_MORE = "a._1cr2e._epyes"
CSS_RIGHT_ARROW = "a[class='_de018 coreSpriteRightPaginationArrow']"
FIREFOX_FIRST_POST_PATH = "//div[contains(@class, '_8mlbc _vbtk2 _t5r8b')]"
TIME_TO_CAPTION_PATH = "../../../div/ul/li/span"

# FOLLOWERS/FOLLOWING RELATED
CSS_EXPLORE_MAIN = "main._8fi2q._2v79o"
CSS_EXPLORE = "a[href='/explore/']"
CSS_EXPLORE_MAIN_LIST = "article._gupiw"
CSS_LOGIN = "a[href='/accounts/login/']"
CSS_FOLLOWERS = "a[href='/{}/followers/']"
CSS_FOLLOWING = "a[href='/{}/following/']"
FOLLOWER_PATH = "//div[contains(text(), 'Followers')]"
FOLLOWING_PATH = "//div[contains(text(), 'Following')]"

# JAVASCRIPT COMMANDS
SCROLL_UP = "window.scrollTo(0, 0);"
SCROLL_DOWN = "window.scrollTo(0, document.body.scrollHeight);"

POST_REMOVE = "if('undefined' !== typeof arguments && arguments[0] > 0){ \
 var ele = document.getElementsByClassName('_70iju'); \
 var prnt = ele[ele.length-1].parentNode; \
 for(var i=0; i<arguments[0]; i++){ \
 var temp_ele = document.getElementsByClassName('_70iju'); \
 temp_ele[0].parentNode.removeChild(temp_ele[0].parentNode.firstChild); \
 } \
 }"
# POST_REMOVE = "if('undefined' !== typeof arguments && arguments[0] > 0){ \
#  var ele = document.getElementsByClassName('_70iju'); \
#  var prnt = ele[ele.length-1].parentNode; \
#  for(var i=0; i<arguments[0]; i++){ \
#  var temp_ele = document.getElementsByClassName('_70iju'); \
#  console.log(arguments[0], temp_ele);\
#  temp_ele[0].parentNode.removeChild(temp_ele[0].parentNode.firstChild); \
#  } \
#  }"

now = datetime.datetime.now()

class InstagramCrawler(object):
    """
        Crawler class
    """

    def __init__(self, setting_path='settings.json'):
        # Setting
        with open(setting_path) as data_file:
            self.setting = json.load(data_file)

        # if headless:
        self._driver = webdriver.PhantomJS(self.setting['PHANTOMJS_PATH'])
        self._driver.set_window_size(1120, 550)
        # else:
        # self._driver = webdriver.Firefox()

        self._driver.implicitly_wait(10)
        self.data = defaultdict(list)

    # DB connection
    # connection = pymongo.MongoClient(self.setting['DB_HOST'], self.setting['DB_PORT'])
    # db_name = self.setting['DB_NAME']
    # self.db = connection[db_name]
    # collectionName = "in-explore-{}-Collection".format(now.strftime("%Y-%m-%d"))
    # self.collection = self.db[collectionName]

    def crawl(self, query, number):
        self.query = query
        self.accountIdx = 0
        self.totalNum = number
        self.browse_target_page()
        self.resultArr = []

        try:
            self.scrape_tags_aco(number)
        except TimeoutException:
            # print("Quitting driver...")
            self.quit()
            self.preprocess()
            return self.resultArr2

        # print("Quitting driver...")
        self.quit()
        self.preprocess()
        return self.resultArr2

    def preprocess(self):
        """
            ACO용 전처리 작업
        """
        kkma = Kkma()
        t = Twitter()
        newArr = []
        sentArr = []
        nounsArr = []
        tokens_ko = []
        index = 0

        self.resultArr = sorted(self.resultArr, key=lambda t: t[0], reverse=False)

        for data in self.resultArr:
            sentPosArr = kkma.pos(data[1])
            inArr = []
            for outA in sentPosArr:
                # for inA in outA:
                inArr.append("/".join(outA))

            morph_arr = t.morphs(data[1])
            morphWords = [word for word in morph_arr if not word in tokens_ko]
            for word in morphWords:
                if not word in nounsArr:
                    nounsArr.append(word)

            tokens_ko.extend(morphWords)

            newArr.append({"sentence": "", "words": morph_arr, "score": 0})

            index = index + 1
            sentArr.append(";".join(inArr))

        index = 0
        for eaSent in sentArr:
            sentiScore = 0
            for corp in settings.KOSAC:
                if eaSent.find(corp['ngram']) > -1:
                    if corp['max.value'] == 'NEG':
                        sentiScore = sentiScore - float(corp['max.prop'])
                    elif corp['max.value'] == 'POS':
                        sentiScore = sentiScore + float(corp['max.prop'])

            newArr[index]["sentence"] = eaSent
            newArr[index]["score"] = sentiScore

            index = index + 1

        self.resultArr2 = newArr

    def quit(self):
        """
            Exit Method
        """
        self._driver.quit()

    def browse_target_page(self):
        # Browse Hashtags
        query = quote(self.query.encode("utf-8"))
        relative_url = query.strip('#')

        target_url = urljoin(self.setting['INSTA_DOMAIN'], relative_url)

        self._driver.get(target_url)
        self._driver.save_screenshot("/Users/Andrew-MB/DEV/05.GIT/GSCIT-sns-sentiment/WEB/screenshot_test.png")

    def scrape_tags_aco(self, number):
        """
            scrape_tags method : scraping Instagram image URL & tags
        """

        last_post_num_pre = 0

        user_h1_tag = WebDriverWait(self._driver, 3).until(
            EC.presence_of_element_located((By.CSS_SELECTOR,
                                            "h1._rf3jb"))
        )

        user_id = user_h1_tag.get_attribute("title")

        while last_post_num_pre < number:

            WebDriverWait(self._driver, 3).until(
                EC.presence_of_element_located((By.XPATH,
                                                "//div[contains(@class, '_mck9w') and contains(@class,'_gvoze') and contains(@class,'_f2mse')]"))
            )

            user_post_list_new = self._driver.find_elements_by_xpath(
                "//div[contains(@class, '_mck9w') and contains(@class,'_gvoze') and contains(@class,'_f2mse')]")

            if len(user_post_list_new) < 1:
                self.quit()

            user_cur_post = user_post_list_new[last_post_num_pre].find_elements_by_xpath(".//a")[0]
            post_url = user_cur_post.get_attribute("href")

            post_url_arr = post_url.split('/')
            post_id = post_url_arr[len(post_url_arr) - 2]

            self._driver.get(post_url)

            single_post = WebDriverWait(self._driver, 3).until(
                EC.presence_of_element_located((By.XPATH, "//article[contains(@class, '_7hhq6')]"))
            )

            article_src = BeautifulSoup(single_post.get_attribute("innerHTML"), "html.parser")

            data_box = article_src.find('div', class_='_ebcx9')
            media_box = article_src.find('div', class_='_sxolz')

            write_date = data_box.find('time', class_='_p29ma').get('datetime')
            write_date_ymd = write_date.split('T')[0]

            ul = data_box.find('ul', class_='_b0tqa')
            li = ul.find_all('li')[0]

            cleanr = re.compile('<.*?>')
            text = re.sub(cleanr, '', str(li.span))

            media_src = media_box.find_all(['video', 'img'])[0].get('src')

            reg_date = datetime.datetime.now()

            instaSerial = InstaSerializer(data={"post_id": post_id
                , "user_id": user_id
                , "img": media_src
                , "text": text
                , "write_date": write_date
                , "reg_date": reg_date})

            if instaSerial.is_valid():
                row = instaSerial.validated_data

                # text preprocess
                row['text'] = re.sub(r'@\w+', '', row['text'])
                row['text'] = re.sub(
                    'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '',
                    row['text'])
                row['text'] = re.sub(r'[\[]|[\]]', '', row['text'])
                row['text'] = re.sub(r'[\r]|[\n]', ' ', row['text'])
                row['text'] = re.sub(r'[.]|[ㆍ]', '', row['text'])
                row['text'] = re.sub(r'#', ' ', row['text'])

                row['write_date'] = datetime.datetime(int(row['write_date'][0:4]), int(row['write_date'][5:7]),
                                                      int(row['write_date'][8:10]),
                                                      int(row['write_date'][11:13]), int(row['write_date'][14:16]),
                                                      int(row['write_date'][17:19]))

                # csvwriter.writerow([post_id, user_id, media_src, text, write_date, reg_date])
                try:
                    instaSerial.save()
                    self.resultArr.append((row['write_date'], row['text']))
                except IntegrityError:
                    self.resultArr.append((row['write_date'], row['text']))
                    pass

            last_post_num_pre = last_post_num_pre + 1
            # print("user's post count : {} ---------------------------------".format(last_post_num_pre))
            self._driver.back()

    def nextAuth(self):
        self.accountIdx = 0 if len(self.auth_dict["INSTAGRAM"]) - 1 == self.accountIdx else self.accountIdx + 1

    # def logoutAndLogin(self):
    #     self._driver.get(urljoin(self.setting['INSTA_DOMAIN'], "accounts/logout"))
    #
    #     self._driver.get(urljoin(self.setting['INSTA_DOMAIN'], "accounts/login/"))
    #
    #     EnvPrint.log_info(
    #         "Since Instagram provides 5000 post views per Hour, relogin with annother username and password loaded from {}".format(
    #             authentication))
    #
    #     # Input username
    #     try:
    #         username_input = WebDriverWait(self._driver, 5).until(
    #             EC.presence_of_element_located((By.NAME, 'username'))
    #         )
    #         username_input.send_keys(self.auth_dict["INSTAGRAM"][self.accountIdx]['username'])
    #
    #     except Exception:
    #         self._driver.save_screenshot('img/{}'.format('screenshot_relogin_01.png'))
    #
    #     # Input password
    #     try:
    #         password_input = WebDriverWait(self._driver, 5).until(
    #             EC.presence_of_element_located((By.NAME, 'password'))
    #         )
    #         password_input.send_keys(self.auth_dict["INSTAGRAM"][self.accountIdx]['password'])
    #         # Submit
    #         password_input.submit()
    #
    #     except Exception:
    #         self._driver.save_screenshot('img/{}'.format('screenshot_relogin_02.png'))
    #
    #     WebDriverWait(self._driver, 60).until(
    #         EC.presence_of_element_located((By.CSS_SELECTOR, CSS_EXPLORE))
    #     )

def main():
    #   Arguments  #
    parser = argparse.ArgumentParser(description='Pengtai Instagram Crawler')
    parser.add_argument('-q', '--query', type=str,
                        help="target to crawl, add '#' for hashtags")
    parser.add_argument('-n', '--number', type=int, default=0,
                        help='Number of posts to download: integer')
    parser.add_argument('-a', '--authentication', type=str, default='auth.json',
                        help='path to authentication json file')
    parser.add_argument('-s', '--setting', type=str, default='settings.json',
                        help='path to setting json file')

    args = parser.parse_args()
    #  End Argparse #

    crawler = InstagramCrawler(setting_path=args.setting)
    print(crawler.crawl(query=args.query,
                  number=args.number))

if __name__ == "__main__":
    main()


from enlace import *
import numpy as np
import math

def ligaCom(serialName):
    com1 = enlace(serialName)
    start_time = time.time()
    com1.enable()
    if com1.enable() == True:
        print("Comunicação Aberta")

    return com1, start_time

def desligaCom(com1, start_time):
    print("-------------------------")
    print("Comunicação encerrada")
    print("-------------------------")
    com1.disable()

    print("--- {:.4f} seconds ---".format(time.time() - start_time))

def int_1_byte(data):
    entireData = bytearray()
    for i in data:
        intByte = (i).to_bytes(1, byteorder ='big')
        entireData.append(intByte[0])
    return entireData

def makeHead(arquivo, tipo):
    """Se tipo = 0 é handshake se tipo != 0 é parte do arquivo"""
    tamanhoBytes = len(arquivo)
    print(f"O arquivo tem {tamanhoBytes} bytes" )
    qtdPayloads = math.ceil(tamanhoBytes/114)
    print(f"Quantidade de Pacotes: {qtdPayloads}")
    tamUltimoPacote = tamanhoBytes - 114*(qtdPayloads-1)
    print(f"Tamanho do último pacote: {tamUltimoPacote}")
    pacoteAtual = 0

    heads = [0, tamanhoBytes, qtdPayloads, tamUltimoPacote,pacoteAtual,0, 0, 255, 0, 0]

    return heads

def makePacote(arquivo,com1):
    headInt = makeHead(arquivo,1)
    eopInt = [0,255,0,0]
    eopByte = int_1_byte(eopInt)
    time.sleep(2)
    for i in range(0, headInt[2]):
        payload = arquivo[:114]
        del arquivo[:114]
        print("Payload feito")
        headInt[1] = len(payload)

        print("-------------------------")
        print("número do pacote: {}".format(headInt[4]))
        time.sleep(0.5)
        if headInt[2] == headInt[4]+1:
            headInt[5] = headInt[3]
        else:
            headInt[5] = 114
        headByte = int_1_byte(headInt)
        print(headInt)
        headInt[4] += 1
        print(headByte)
        pacote = headByte + payload + eopByte
        print(pacote)
        com1.sendData(pacote)

        print("-----------------")
        print("Pacote enviado: ", len(pacote))
        print("-----------------")

def recebePacotes(arquivo, com1):
    i = 0
    while True:
        head, lenHead = com1.getData(10)
        print("......")
        print("head recebido: ", head[4])
        print(head[5])
        if head[4] != i:
            print("Erro enviar de novo pacote: ", i)
        payload, lenPayload = com1.getData(head[5])
        print(payload)
        arquivo += payload
        eop, lenEOP = com1.getData(4)
        i += 1
        if head[2] == head[4]+1:
            break

    return arquivo


import csv
import math

class Neighbor:

    def __init__(self, realgm_id, name):
        self.name = name
        self.bref_id = None
        self.gleague_cluster = None
        self.nba_cluster = None
        self.height = None
        self.weight = None
        self.position = "N/A"
        self.gl_threepm = None
        self.gl_g = None
        self.gl_ast = None
        self.gl_fga = None
        self.gl_pts = None
        self.gl_reb = None
        self.nba_g = None
        self.nba_threepm = None
        self.nba_ast = None
        self.nba_fga = None
        self.nba_pts = None
        self.nba_reb = None
        self.pic_id = self.get_data(realgm_id)

    def get_data(self, realgm_id):

        with open('data/training_data.csv', 'r') as training:
            players = csv.reader(training, delimiter=',')
            next(players, None)
            for player in players:
                if player[2] == realgm_id:
                    self.height = str(math.floor(int(player[9]) / 12)) + "-" + str(int(player[9]) % 12)
                    self.weight = player[10]
                    break

        with open('data/2010-17_dataset_clustered.csv', 'r') as database:
            rows = csv.reader(database, delimiter=",")
            next(rows, None)
            next(rows, None)
            for player in rows:
                if realgm_id == player[152]:
                    self.bref_id = player[153]
                    self.gleague_cluster = player[82]
                    self.nba_cluster = player[80]
                    self.gl_g = player[40]
                    self.gl_threepm = player[12]
                    self.gl_ast = player[15]
                    self.gl_fga = player[28]
                    self.gl_pts = player[56]
                    self.gl_reb = player[59]
                    self.nba_g = player[119]
                    self.nba_threepm = player[91]
                    self.nba_ast = player[94]
                    self.nba_fga = player[107]
                    self.nba_pts = player[131]
                    self.nba_reb = player[139]
                    pic_id = self.bref_id.split("/")[3].split(".")[0]
                    return pic_id

        return None


#!/usr/bin/python

import datetime as dt

def dayofweek(date):
    # slicing ftw? https://i.imgur.com/0cWeO6h.gif
    date = date[:date.index(' ')].zfill(4) + date[date.index(' '):]
    dateobj = dt.datetime.strptime(date, '%Y %m %d')
    return dateobj.strftime('%A')

def test_dayofweek():
    inputdata = """2017 10 30
2016 2 29
2015 2 28
29 4 12
570 11 30
1066 9 25
1776 7 4
1933 1 30
1953 3 6
2100 1 9
2202 12 15
7032 3 26"""

    answers = "Monday Monday Saturday Thursday Friday Tuesday\
     Thursday Monday Friday Saturday Wednesday Monday"

    lookup = dict(zip(inputdata.splitlines(), answers.split()))

    for k, v in lookup.items():
        try:
            assert(dayofweek(k) == v)
        except AssertionError:  # elaborate in case of a fail.
            print(f"Failed Test: dayofweek({k}) == {v}: {dayofweek(k)}")

test_dayofweek()


import pygame
from pygame import *
from blocks import *
from monsters import *
from pygame.rect import Rect

import helperspygame  # Преобразует tmx карты в формат  спрайтов pygame
import tmxreader  # Может загружать tmx файлы
from player import *

# Объявляем переменные
WIN_WIDTH = 800  # Ширина создаваемого окна
WIN_HEIGHT = 600  # Высота
DISPLAY = (WIN_WIDTH, WIN_HEIGHT)  # Группируем ширину и высоту в одну переменную
CENTER_OF_SCREEN = WIN_WIDTH / 2, WIN_HEIGHT / 2

FILE_DIR = os.path.dirname(__file__)

class Camera(object):
    def __init__(self, camera_func, width, height):
        self.camera_func = camera_func
        self.state = Rect(0, 0, width, height)

    def apply(self, target):
        return target.rect.move(self.state.topleft)

    def update(self, target):
        self.state = self.camera_func(self.state, target.rect)

    def reverse(self, pos):  # получение внутренних координат из глобальных
        return pos[0] - self.state.left, pos[1] - self.state.top

def camera_configure(camera, target_rect):
    l, t, _, _ = target_rect
    _, _, w, h = camera
    l, t = -l + WIN_WIDTH / 2, -t + WIN_HEIGHT / 2

    l = max(-(camera.width - WIN_WIDTH), min(0, l))
    t = min(0, max(-(camera.height - WIN_HEIGHT), t))

    return Rect(l, t, w, h)

def loadLevel(name):
    world_map = tmxreader.TileMapParser().parse_decode('%s/%s.tmx' % (FILE_DIR, name))  # загружаем карту
    resources = helperspygame.ResourceLoaderPygame()  # инициируем преобразователь карты
    resources.load(world_map)  # и преобразуем карту в понятный pygame формат

    sprite_layers = helperspygame.get_layers_from_map(resources)  # получаем все слои карты

    # берем слои по порядку
    # 0 - слой фона,
    # 1 - слой блоков,
    # 2 - слой смертельных блоков
    # 3 - слой объектов монстров,
    # 4 - слой объектов телепортов
    platforms_layer = sprite_layers[1]
    dieBlocks_layer = sprite_layers[2]

    for row in range(0, platforms_layer.num_tiles_x):  # перебираем все координаты тайлов
        for col in range(0, platforms_layer.num_tiles_y):
            if platforms_layer.content2D[col][row]:
                pf = Platform(row * PLATFORM_WIDTH, col * PLATFORM_WIDTH)  # как и прежде создаем объкты класса Platform
                platforms.append(pf)
            if dieBlocks_layer.content2D[col][row]:
                bd = BlockDie(row * PLATFORM_WIDTH, col * PLATFORM_WIDTH)
                platforms.append(bd)

    teleports_layer = sprite_layers[4]
    for teleport in teleports_layer.objects:
        goX = int(teleport.properties["goX"]) * PLATFORM_WIDTH
        goY = int(teleport.properties["goY"]) * PLATFORM_HEIGHT
        x = teleport.x
        y = teleport.y - PLATFORM_HEIGHT
        tp = BlockTeleport(x, y, goX, goY)
        entities.add(tp)
        platforms.append(tp)
        animatedEntities.add(tp)

    playerX = 65
    playerY = 65

    monsters_layer = sprite_layers[3]
    for monster in monsters_layer.objects:
        x = monster.x
        y = monster.y
        if monster.name == "Player":
            playerX = x
            playerY = y - PLATFORM_HEIGHT
        elif monster.name == "Princess":
            pr = Princess(x, y - PLATFORM_HEIGHT)
            platforms.append(pr)
            entities.add(pr)
            animatedEntities.add(pr)
        else:
            up = int(monster.properties["up"])
            maxUp = int(monster.properties["maxUp"])
            left = int(monster.properties["left"])
            maxLeft = int(monster.properties["maxLeft"])
            mn = Monster(x, y - PLATFORM_HEIGHT, left, up, maxLeft, maxUp)
            entities.add(mn)
            platforms.append(mn)
            monsters.add(mn)

    total_level_width = platforms_layer.num_tiles_x * PLATFORM_WIDTH  # Высчитываем фактическую ширину уровня
    total_level_height = platforms_layer.num_tiles_y * PLATFORM_HEIGHT  # высоту

    return playerX, playerY, total_level_height, total_level_width, sprite_layers

def main():
    pygame.init()
    screen = pygame.display.set_mode(DISPLAY)
    pygame.display.set_caption("Super Mario Boy")
    bg = Surface((WIN_WIDTH, WIN_HEIGHT))
    renderer = helperspygame.RendererPygame()  # визуализатор

    playerX, playerY, total_level_height, total_level_width, sprite_layers = loadLevel("levels/map_1")
    bg.fill(Color("#000000"))  # Заливаем поверхность сплошным цветом

    left = right = up = running = False
    hero = Player(playerX, playerY)  # создаем героя по (x,y) координатам
    entities.add(hero)

    timer = pygame.time.Clock()
    camera = Camera(camera_configure, total_level_width, total_level_height)

    while not hero.winner:
        timer.tick(60)
        for e in pygame.event.get():
            if e.type == QUIT:
                return
            elif e.type == KEYDOWN:
                if e.key == K_UP:
                    up = True
                if e.key == K_LEFT:
                    left = True
                if e.key == K_RIGHT:
                    right = True
                if e.key == K_LSHIFT:
                    running = True
            elif e.type == KEYUP:
                if e.key == K_UP:
                    up = False
                if e.key == K_LEFT:
                    left = False
                if e.key == K_RIGHT:
                    right = False
                if e.key == K_LSHIFT:
                    running = False

        for sprite_layer in sprite_layers:  # перебираем все слои
            if not sprite_layer.is_object_group:  # и если это не слой объектов
                renderer.render_layer(screen, sprite_layer)  # отображаем его

        for e in entities:
            screen.blit(e.image, camera.apply(e))

        animatedEntities.update()  # показываеaм анимацию
        monsters.update(platforms)  # передвигаем всех монстров
        camera.update(hero)  # центризируем камеру относительно персонаж
        center_offset = camera.reverse(CENTER_OF_SCREEN)  # получаем координаты внутри длинного уровня
        renderer.set_camera_position_and_size(
            center_offset[0], center_offset[1], WIN_WIDTH, WIN_HEIGHT, "center")
        hero.update(left, right, up, running, platforms)  # передвижение
        pygame.display.update()  # обновление и вывод всех изменений на экран
        screen.blit(bg, (0, 0))  # Каждую итерацию необходимо всё перерисовывать

    for sprite_layer in sprite_layers:
        if not sprite_layer.is_object_group:
            renderer.render_layer(screen, sprite_layer)

    for e in entities:
        screen.blit(e.image, camera.apply(e))

    font = pygame.font.Font(None, 38)
    text = font.render(("Thank you MarioBoy! but our princess is in another level!"), 1,
                       (255, 255, 255))  # выводим надпись
    screen.blit(text, (10, 100))

    pygame.display.update()

level = []
entities = pygame.sprite.Group()
animatedEntities = pygame.sprite.Group()
monsters = pygame.sprite.Group()
platforms = []

if __name__ == "__main__":
    main()


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="fstq-worker-python",
    version="0.0.1",
    author="Cyril Diagne",
    author_email="diagne.cyril@gmail.com",
    description="Python worker lib for FSTQ",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/cyrildiagne/fstq",
    packages=setuptools.find_packages(),
    entry_points='''
        [console_scripts]
        fstq=fstq.cli.cli:main
    ''',
    install_requires=[
        'click==7.1.2', 'firebase-admin==4.4.0', 'docker==4.3.1',
        'google-cloud-container==2.1.0', 'kubernetes==11.0.0'
    ],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.7',
)


import random

class Sarsa:

    def __init__(self, actions, epsilon, alpha, gamma):
        self.q = {}             # Stores (state, action)-> q mappings
        self.epsilon = epsilon  # Exploration constant
        self.alpha = alpha      # Learning rate
        self.gamma = gamma      # Discount factor
        self.actions = actions  # Available actions

    def get_q(self, state, action):
        return self.q.get((state, action), 0.0)

    def learn_q(self, state, action, reward, value):
        old_quality = self.q.get((state, action), None)
        if old_quality is None:
            self.q[(state, action)] = reward
        else:
            self.q[(state, action)] = old_quality + self.alpha * (value - old_quality)

    def choose_action(self, state):
        if random.random() < self.epsilon:
            action = random.choice(self.actions)
        else:
            qs = [self.get_q(state, action) for action in self.actions]
            max_q = max(qs)
            if qs.count(max_q) > 1:
                max_q_indices = [i for i in range(len(self.actions)) if qs[i] == max_q]
                index = random.choice(max_q_indices)
            else:
                index = qs.index(max_q)
            action = self.actions[index]
        return action

    def learn(self, state, action, reward, next_state, next_action):
        next_q = self.get_q(next_state, next_action)
        self.learn_q(state, action, reward, reward + self.gamma * next_q)


import socket

class Server:

    def __init__(self, port, handle_message):
        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.handle_message = handle_message
        self.server_address = ('localhost', port)
        self.server_socket.bind(self.server_address)

    """ starts to listen for clients """
    def listen(self):
        while True:
            try:
                message, client_address = self.server_socket.recvfrom(4096)
                answer = self.handle_message(message)
                self.send(answer, client_address)
            except:
                continue

    """ sends a message to the given address """
    def send(self, message, address):
        self.server_socket.sendto(bytes(message, encoding="utf-8"), address)


import pytest
import re

pytest.importorskip(modname="torch", reason="torch is not installed")
pytest.importorskip(modname="transformers", reason="transformers is not installed")

from medkit.core import Attribute, ProvTracer  # noqa: E402
from medkit.core.text import Segment, Span  # noqa: E402
from medkit.text.ner.hf_entity_matcher import HFEntityMatcher  # noqa: E402

_SPAN_OFFSET = 10

# mock of TokenClassificationPipeline class used by HFEntityMatcher
class _MockedPipeline:
    def __init__(self, *args, **kwargs):
        self._regexps_and_labels = [
            (re.compile(r"\basthma\b"), "problem"),
            (re.compile(r"\bdiabetes\b"), "problem"),
            (re.compile(r"\bventoline\b"), "treatment"),
        ]

    def __call__(self, texts):
        all_match_dicts = []
        for text in texts:
            match_dicts = []
            for regexp, label in self._regexps_and_labels:
                for match in regexp.finditer(text):
                    match_dicts.append(
                        {
                            "start": match.start(),
                            "end": match.end(),
                            "entity_group": label,
                            "score": 1.0,
                        }
                    )
            all_match_dicts.append(match_dicts)

        return all_match_dicts

@pytest.fixture(scope="module", autouse=True)
def _mocked_pipeline(module_mocker):
    module_mocker.patch(
        "medkit.text.ner.hf_entity_matcher.hf_utils.check_model_for_task_HF",
        lambda m, t, hf_auth_token=None: True,
    )
    module_mocker.patch("transformers.pipeline", _MockedPipeline)

def _get_sentence_segment(text):
    return Segment(
        label="sentence",
        spans=[Span(_SPAN_OFFSET, _SPAN_OFFSET + len(text))],
        text=text,
    )

def test_single_match():
    """Basic behavior, single match in one input segment"""

    sentence = _get_sentence_segment("The patient has asthma.")
    matcher = HFEntityMatcher(model="mock-model")
    entities = matcher.run([sentence])
    assert len(entities) == 1

    # entity
    entity = entities[0]
    assert entity.label == "problem"
    assert entity.text == "asthma"
    assert entity.spans == [Span(26, 32)]

    # score attribute
    attrs = entity.attrs.get(label="score")
    assert len(attrs) == 1
    attr = attrs[0]
    assert attr.label == "score"
    assert 0.0 <= attr.value <= 1.0

def test_multiple_matches():
    """Basic behavior, multiple matches in multiple input segments"""

    sentence_1 = _get_sentence_segment("The patient has asthma and is using ventoline.")
    sentence_2 = _get_sentence_segment("The patient has diabetes.")
    sentences = [sentence_1, sentence_2]
    matcher = HFEntityMatcher(model="mock-model")
    entities = matcher.run(sentences)
    assert len(entities) == 3

    # 1st entity
    entity_1 = entities[0]
    assert entity_1.label == "problem"
    assert entity_1.text == "asthma"
    assert entity_1.spans == [Span(26, 32)]

    # 2nd entity
    entity_2 = entities[1]
    assert entity_2.label == "treatment"
    assert entity_2.text == "ventoline"
    assert entity_2.spans == [Span(46, 55)]

    # 3rd entity
    entity_3 = entities[2]
    assert entity_3.label == "problem"
    assert entity_3.text == "diabetes"
    assert entity_3.spans == [Span(26, 34)]

def test_attrs_to_copy():
    """Copying of selected attributes from input segment to created entity"""

    sentence = _get_sentence_segment("The patient has asthma.")
    # copied attribute
    neg_attr = Attribute(label="negation", value=False)
    sentence.attrs.add(neg_attr)
    # uncopied attribute
    sentence.attrs.add(Attribute(label="hypothesis", value=False))

    matcher = HFEntityMatcher(model="mock-model", attrs_to_copy=["negation"])
    entity = matcher.run([sentence])[0]

    assert len(entity.attrs.get(label="score")) == 1
    # only negation attribute was copied
    neg_attrs = entity.attrs.get(label="negation")
    assert len(neg_attrs) == 1
    assert len(entity.attrs.get(label="hypothesis")) == 0

    # copied attribute has same value but new id
    copied_neg_attr = neg_attrs[0]
    assert copied_neg_attr.value == neg_attr.value
    assert copied_neg_attr.uid != neg_attr.uid

def test_prov():
    """Generated provenance nodes"""

    # use file containing voice signal
    sentence_1 = _get_sentence_segment("The patient has asthma and is using ventoline.")
    sentence_2 = _get_sentence_segment("The patient has diabetes.")
    sentences = [sentence_1, sentence_2]

    matcher = HFEntityMatcher(model="mock-model")
    prov_tracer = ProvTracer()
    matcher.set_prov_tracer(prov_tracer)
    entities = matcher.run(sentences)
    assert len(entities) == 3

    # data item uid and operation uid are correct
    entity_1 = entities[0]
    prov_1 = prov_tracer.get_prov(entity_1.uid)
    assert prov_1.data_item == entity_1
    assert prov_1.op_desc == matcher.description

    # 1st and 2nd entities have 1st sentence as source
    assert prov_1.source_data_items == [sentence_1]
    entity_2 = entities[1]
    prov_2 = prov_tracer.get_prov(entity_2.uid)
    assert prov_2.source_data_items == [sentence_1]

    # 3d entity has 2nd sentence as source
    entity_3 = entities[2]
    prov_3 = prov_tracer.get_prov(entity_3.uid)
    assert prov_3.source_data_items == [sentence_2]


# 226. Invert Binary Tree
# https://leetcode.com/problems/invert-binary-tree/
from typing import List
from typing import Optional

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.data = val
        self.left = left
        self.right = right

    def insert(self, data):
        if data < self.data:
            if self.left is None:
                self.left = TreeNode(data)
            else:
                self.left.insert(data)
        elif data > self.data:
            if self.right is None:
                self.right = TreeNode(data)
            else:
                self.right.insert(data)

    def __str__(self):
        left = self.left.data if self.left else 'None'
        right = self.right.data if self.right else 'None'
        return f'{self.data}, {left}, {right}'

def build_tree(nums: List[int]):
    root = TreeNode(nums[0])
    for i, n in enumerate(nums[1:], start=1):
        root.insert(n)
    return root

def print_tree(root: Optional[TreeNode]):
    q = [root]
    while len(q) > 0:
        node = q.pop(0)
        print(node.data, end=',')
        # print(node)
        if node.left:
            q.append(node.left)
        if node.right:
            q.append(node.right)
    print()

class Solution:
    # BFS, iterative
    def invertTree(self, root: Optional[TreeNode]) -> Optional[TreeNode]:
        q = [root]
        while len(q) > 0:
            node = q.pop(0)
            if not node:
                continue
            node.left, node.right = node.right, node.left
            q += [node.left, node.right]
        return root

class Solution2:
    # BFS, recursive
    def invertTree(self, root: Optional[TreeNode]) -> Optional[TreeNode]:
        if root:
            root.right, root.left = self.invertTree(root.left), self.invertTree(root.right)
        return root

print_tree(Solution().invertTree(build_tree([4, 2, 7, 1, 3, 6, 9])))
# [4,7,2,9,6,3,1]
print_tree(Solution().invertTree(build_tree([2, 1, 3])))


import sys
import argparse
import json2csv.api
import logging

def main(args=None):
    """The main routine."""
    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser(
        description='JSON TO CSV Converter',
        epilog="Example: json2csv input_file.json output_file.csv")
    parser.add_argument("input", help="Path of the json file that you want to convert")
    parser.add_argument("output", help="Path of the output csv file")
    parser.add_argument("-l", "--log", help="Log file's path")
    args = parser.parse_args()

    if args.log:
        logger = logging.getLogger('json2csv')
        logger.setLevel(logging.DEBUG)
        fh = logging.FileHandler(args.log)
        fh.setLevel(logging.DEBUG)
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)
        formatter = logging.Formatter('%(asctime)s:%(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        ch.setFormatter(formatter)
        logger.addHandler(fh)
        logger.addHandler(ch)

    json2csv.api.json_to_csv(args.input, args.output)

if __name__ == "__main__":
    main()


import pytest

from apps.utils.kmp import knuth_morris_prath,kmp_first_occurence

def test_kmp_should_should_match():

    pattern = "aaa"
    text = "aabaabaaab"

    assert len(knuth_morris_prath(pattern, text)) > 0, "Pattern matching should resulted in more than one occurence"

def test_kmp_should_not_match():
    pattern = "aaa"
    text = "aabaabaab"

    assert len(knuth_morris_prath(pattern, text)) == 0, "Pattern matching should resulted in more than one occurence"


import numpy as np

def forward(model, input, target):
	for layer in model[:-1]:
		input = layer.forward(input)
	output = model[-1].forward(input, target)
	return output

def backward(model):
	gradient = model[-1].backward()
	for layer in reversed(model[:-1]):
		gradient = layer.backward(gradient)
	return gradient

def load_params(model, W):
	for layer in model:
		w = layer.get_params()
		if w is None:
			continue
		w_shape = w.shape
		w, W = np.split(W, [np.prod(w_shape)])
		layer.set_params(w.reshape(w_shape))

def extract_params(model):
	params = []
	for layer in model:
		w = layer.get_params()
		if w is None:
			continue
		params.append(w)
	W = np.concatenate(params)
	return np.array(W)

def extract_grads(model):
	grads = []
	for layer in model:
		g = layer.get_grads()
		if g is None:
			continue
		grads.append(g)
	dW = np.concatenate(grads)
	return np.array(dW)

def forget(model):
	for layer in model:
		layer.forget()

def print_info(model):
	for layer in model:
		layer.print_info()


import streamlit as st
from PIL import Image
import os
import cv2
from mtcnn import MTCNN
import numpy as np
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet50 import preprocess_input
import pickle
from sklearn.metrics.pairwise import cosine_similarity
from tensorflow.keras.layers import GlobalMaxPooling2D
import tensorflow.python.keras

model = ResNet50(weights='imagenet',include_top=False,input_shape=(224,224,3))
model.trainable=False

model = tensorflow.keras.Sequential([
    model,
    GlobalMaxPooling2D()
])

detector = MTCNN()
feature_list = np.array(pickle.load(open('embedding1.pkl','rb')))
filenames = pickle.load(open('filename.pkl','rb'))

def save_uploaded_file(uploaded_image):
    try:
        with open(os.path.join('uploads',uploaded_image.name),'wb') as f:
            f.write(uploaded_image.getbuffer())
        return True
    except:
        return False

def extract_feature(img_path,model,detector):
    img = cv2.imread(img_path)
    results = detector.detect_faces(img)
    x, y, width, height = results[0]['box']

    # for croping the image
    face = img[y:y + width, x:x + width]
    # cv2.imshow('output',face)
    # cv2.waitKey(0)
    # detection done

    # extraction from image start
    image = Image.fromarray(face)
    image = image.resize((224, 224))

    face_array = np.asarray(image)
    face_array = face_array.astype('float32')
    expended_image = np.expand_dims(face_array, axis=0)

    preprocessed_image = preprocess_input(expended_image)
    result = model.predict(preprocessed_image).flatten()
    return result

def recommend(feature_list,features):
    similarity = []

    for i in range(len(feature_list)):
        similarity.append(cosine_similarity(features.reshape(1, -1), feature_list[i].reshape(1, -1))[0][0])
    index_pos = sorted(list(enumerate(similarity)), reverse=True, key=lambda x: x[1])[0][0]
    return index_pos

st.title('which bollywood celebrity are you?')

uploaded_img = st.file_uploader('choose an image')

if uploaded_img is not None:
#     save in directory
    if save_uploaded_file(uploaded_img):
#         load
#           feature extract
#          recomed
        display_img = Image.open(uploaded_img)
        # st.image(display_img)

        features = extract_feature(os.path.join('uploads',uploaded_img.name),model,detector)
        # st.text(features)
        # st.text(features.shape)
        index_pos = recommend(feature_list,features)
        predicted_actor = " ".join(filenames[index_pos].split('\\')[1].split('_'))
        # st.text(index_pos)
        col1,col2 = st.columns(2)
        with col1:
            st.header('Your upload image')
            st.image(display_img,width = 300)
        with col2:

            st.header('seems like '+predicted_actor)
            st.image(filenames[index_pos],width = 300)

# display


import thorpy.flipmount.flipmount as fm

from tango import AttrWriteType, DispLevel, DevState
from tango.server import Device, attribute, command, device_property
from enum import IntEnum

class MirrorState(IntEnum):
    closed = 0
    open = 1
    moving = 2
    unknown = 3

class ThorlabsMFF100(Device):
    mffstate = attribute(
        label="Mirror State",
        dtype=MirrorState,
        display_level=DispLevel.OPERATOR,
        access=AttrWriteType.READ,
        doc="State can be 0=closed, 1=open, 2=moving, 3=unknown",
    )

    serial_num = attribute(
        label="Serialnumber",
        dtype=str,
        display_level=DispLevel.OPERATOR,
        access=AttrWriteType.READ,
        doc="Serial number of device",
    )

    inverted = attribute(
        label="Invert states",
        dtype=bool,
        display_level=DispLevel.EXPERT,
        access=AttrWriteType.READ_WRITE,
        doc="Invert open and close states of mirror in TANGO only!",
        memorized=True,
        hw_memorized=True,
    )

    serial_number = device_property(dtype=str)

    def init_device(self):
        Device.init_device(self)
        self.info_stream(
            "Thorlabs Flip Mirror Mount with serial {:s}".format(self.serial_number)
        )
        self.mount = fm.flipMount(self.serial_number)
        self.__mff_state = 3
        self.__inverted = False

    def always_executed_hook(self):
        if self.mount.is_moving():
            self.set_status("Thorlabs Flip Mirror is MOVING")
            self.__mff_state = 2
            self.set_state(DevState.MOVING)
        elif self.mount.is_open():
            if self.__inverted:
                self.set_status("Thorlabs Flip Mirror is CLOSED")
                self.__mff_state = 0
                self.set_state(DevState.CLOSE)
            else:
                self.set_status("Thorlabs Flip Mirror is OPEN")
                self.__mff_state = 1
                self.set_state(DevState.OPEN)
        elif self.mount.is_close():
            if self.__inverted:
                self.set_status("Thorlabs Flip Mirror is OPEN")
                self.__mff_state = 1
                self.set_state(DevState.OPEN)
            else:
                self.set_status("Thorlabs Flip Mirror is CLOSED")
                self.__mff_state = 0
                self.set_state(DevState.CLOSE)
        else:
            self.set_status("Thorlabs Flip Mirror is UNKOWN")
            self.__mff_state = 3
            self.set_state(DevState.UNKNOWN)

    def read_mffstate(self):
        return self.__mff_state
        if self.__mff_state == 0:
            return MirrorState.closed
        elif self.__mff_state == 1:
            return MirrorState.open
        if self.__mff_state == 2:
            return MirrorState.moving
        else:
            return MirrorState.unknown

    def read_serial_num(self):
        return self.serial_number

    def read_inverted(self):
        return self.__inverted

    def write_inverted(self, value):
        self.__inverted = bool(value)

    def read_info(self):
        return "Information", dict(
            manufacturer="Thorlabs", model="MFF101", version_number=self.__serial_num
        )

    @command
    def Flip(self):
        self.mount.flip()

    @command
    def Close(self):
        if self.__inverted:
            self.mount.open()
        else:
            self.mount.close()

    @command
    def Open(self):
        if self.__inverted:
            self.mount.close()
        else:
            self.mount.open()

    @command
    def Identify(self):
        self.mount.identify()

if __name__ == "__main__":
    ThorlabsMFF100.run_server()


from typing import List, Dict, Callable
import numpy as np
import torch

from simulai.templates import NetworkTemplate

class WeightsEstimator:
    def __init__(self) -> None:
        pass

    def _clip_grad(self, grad: torch.Tensor = None, size: int = None) -> torch.Tensor:
        """Filtering gradients.
        Args:
            grad (torch.Tensor): Tensor containing the gradients of all the parameters.
            size (int): Number of parameters.

        Returns:
            torch.Tensor: The clipped gradients.

        """
        if not isinstance(grad, torch.Tensor):
            return torch.zeros(size).detach()
        else:
            return grad.detach()

    def _grad(
        self, loss: torch.tensor = None, operator: NetworkTemplate = None
    ) -> torch.Tensor:
        """It evaluates the gradients of loss function w.r.t. all the model parameters.

        Args:
            loss (torch.tensor): The current state of the loss function.
            operator (NetworkTemplate): The model being trained.

        Returns:
            torch.Tensor: The gradients evaluated for all the parameters.

        """
        if loss.requires_grad:
            loss.backward(retain_graph=True)

            grads = [
                self._clip_grad(grad=v.grad, size=v.shape)
                for v in operator.parameters()
            ]

            for v in operator.parameters():
                v.grad = None

            gradients = torch.hstack([v.flatten() for v in grads])
        else:
            gradients = torch.zeros(operator.n_parameters)

        return gradients

##########################################################
# Adjusters designed for equation-based loss terms (PINNs)
##########################################################

class GeometricMean:
    def __init__(self):
        """Simple and naive approach for balancing the loss terms in which
           they are rescaled to have the same order of magnitude of the geometric
           mean between all the terms.
        """
        pass

    def __call__(
        self,
        residual=List[torch.Tensor],
        loss_evaluator=Callable,
        loss_history=Dict[str, float],
        **kwargs,
    ) -> None:
        """

        Args:
            residual (List[torch.Tensor]): List containing all the equation-based
                loss terms.
            loss_evaluator (Callable): A Python class or function for evaluating
                the loss function.

        """
        n_res = len(residual)
        residual_norms = [loss_evaluator(res).detach() for res in residual]
        exps = [torch.log(res) for res in residual_norms]
        mean_exps = torch.mean(torch.Tensor(exps))
        shifts = [mean_exps - exp for exp in exps]

        weights = [torch.exp(shift).detach() for shift in shifts]

        return weights

class ShiftToMax:
    def __init__(self):
        """Simple and naive approach for balancing the loss terms in which
           they are rescaled to have the same order of magnitude of the maximum value
           of all the terms.
        """

        pass

    def __call__(
        self,
        residual=List[torch.Tensor],
        loss_evaluator=Callable,
        loss_history=Dict[str, float],
        **kwargs,
    ) -> None:
        """

        Args:
            residual (List[torch.Tensor]): List containing all the equation-based
                loss terms.
            loss_evaluator (Callable): A Python class or function for evaluating
                the loss function.
        """

        n_res = len(residual)
        residual_norms = [loss_evaluator(res).detach() for res in residual]
        exps = [torch.log(res) for res in residual_norms]
        max_exps = torch.max(torch.Tensor(exps))
        shifts = [max_exps - exp for exp in exps]

        weights = [torch.exp(shift.to(int).detach()) / n_res for shift in shifts]

        return weights

########################################################################
# Adjusters designed for balancing overall residual (PINN) contributions
# and data-driven and  initial/boundary conditions
########################################################################

class AnnealingWeights(WeightsEstimator):
    def __init__(
        self,
        alpha: float = None,
        init_weight: float = 1.0,
        bound_weight: float = 1.0,
        extra_data_weight: float = 1.0,
    ) -> None:
        """Learning rate Annealing technique used
            for scaling equation-based loss terms (see https://arxiv.org/abs/2001.04536)

        Args:
            alpha (float): 1 - update step.
            init_weight (float): Initial value for the initial condition weight.
            bound_weight (float): Initial value for the boundary condition weight.
            extra_data_weight (float): Initial value for the weight related to
                data-drive loss terms.

        """
        super().__init__()

        self.alpha = alpha

        self.init_weight = init_weight
        self.bound_weight = bound_weight
        self.extra_data_weight = extra_data_weight

    def _coeff_update(self, loss_ref: torch.tensor = None, loss: torch.tensor = None):
        loss_grad_mean = torch.mean(torch.abs(loss))

        if torch.abs(loss_grad_mean) >= 1e-15:
            coeff_hat = torch.max(torch.abs(loss_ref)) / loss_grad_mean
        else:
            coeff_hat = 0

        return coeff_hat

    def __call__(
        self, residual: torch.tensor = None, operator: NetworkTemplate = None, **kwargs
    ) -> List[torch.tensor]:

        """

        Args:
            residual (torch.tensor): Tensor containing the equation residual.
            operator (NetworkTemplate): Model being trained.

        Returns:
            List[torch.tensor]: A list containing the updated loss weights.

        """

        pde = residual[0]
        init = residual[1]
        bound = residual[2]
        extra_data = residual[3]

        pde_grads = self._grad(loss=pde, operator=operator)
        init_grads = self._grad(loss=init, operator=operator)
        bound_grads = self._grad(loss=bound, operator=operator)
        extra_data_grads = self._grad(loss=extra_data, operator=operator)

        init_weight_update = self._coeff_update(loss_ref=pde_grads, loss=init_grads)
        bound_weight_update = self._coeff_update(loss_ref=pde_grads, loss=bound_grads)
        extra_data_weight_update = self._coeff_update(
            loss_ref=pde_grads, loss=extra_data_grads
        )

        self.init_weight = (self.alpha) * self.init_weight + (
            1 - self.alpha
        ) * init_weight_update
        self.bound_weight = (self.alpha) * self.bound_weight + (
            1 - self.alpha
        ) * bound_weight_update
        self.extra_data_weight = (self.alpha) * self.extra_data_weight + (
            1 - self.alpha
        ) * extra_data_weight_update

        return [1.0, self.init_weight, self.bound_weight, self.extra_data_weight]

class InverseDirichletWeights(WeightsEstimator):
    def __init__(
        self, alpha: float = None, initial_weights: List[float] = None
    ) -> None:

        """Inverse Dirichlet technique used
            for scaling equation-based loss terms (see https://iopscience.iop.org/article/10.1088/2632-2153/ac3712/pdf)

        Args:
            alpha (float): 1 - update step.
            initial_weights (List[float]): List containing the initial states of all the loss
                function terms.
        """

        super().__init__()

        self.alpha = alpha
        self.default_number = 100

        if not initial_weights:
            self.weights = [1.0] * self.default_number
        else:
            self.weights = initial_weights

    def _coeff_update(self, nominator: torch.tensor = None, loss: torch.tensor = None):
        loss_grad_std = torch.std(loss)

        if torch.abs(loss_grad_std) >= 1e-15:
            coeff_hat = nominator / loss_grad_std
        else:
            coeff_hat = 0

        return coeff_hat

    def __call__(
        self,
        residual: List[torch.Tensor] = None,
        loss_evaluator: Callable = None,
        loss_history: Dict[str, float] = None,
        operator: Callable = None,
        **kwargs,
    ) -> List[torch.Tensor]:
        """

        Args:
            residual (List[torch.Tensor]): List of equation-based loss terms.
            loss_evaluator (Callable): Python function or class which evaluates the
                loss function.
            operator (Callable): Model being trained.
        Returns:
            List[torch.Tensor]: List of loss updated loss terms.
        """

        residual_grads = list()

        for res in residual:
            res_loss = res
            residual_grads.append(self._grad(loss=res_loss, operator=operator))

        losses_std = [torch.std(l) for l in residual_grads]

        nominator = torch.max(torch.Tensor(losses_std))

        for j in range(len(residual)):
            weight_update = self._coeff_update(
                nominator=nominator, loss=residual_grads[j]
            )

            self.weights[j] = (self.alpha) * self.weights[j] + (
                1 - self.alpha
            ) * weight_update

        return self.weights[: len(residual)]


import utils

class CoreMemory(object):
    def __init__(self, persona, human):
        self.persona = persona
        self.human = human

    def __repr__(self) -> str:
        return f"\n### CORE MEMORY ###" + f"\n=== Persona ===\n{self.persona}" + f"\n\n=== Human ===\n{self.human}"

    def edit_persona(self, new_persona):
        self.persona = new_persona

    def edit_human(self, new_human):
        self.human = new_human

    def append(self, field, content, sep="\n"):
        if field == "persona":
            new_content = self.persona + sep + content
            self.edit_persona(new_content)
        elif field == "human":
            new_content = self.human + sep + content
            self.edit_human(new_content)
        else:
            raise KeyError

    def replace(self, field, old_content, new_content):
        if field == "persona":
            if old_content in self.persona:
                new_persona = self.persona.replace(old_content, new_content)
                self.edit_persona(new_persona)
            else:
                raise ValueError("Content not found in persona (make sure to use exact string)")
        elif field == "human":
            if old_content in self.human:
                new_human = self.human.replace(old_content, new_content)
                self.edit_human(new_human)
            else:
                raise ValueError("Content not found in human (make sure to use exact string)")
        else:
            raise KeyError

class ArchivalMemory(object):
    def __init__(self):
        self.archive = []

    def __len__(self):
        return len(self.archive)

    def __repr__(self) -> str:
        if len(self.archive) == 0:
            memory_str = "<empty>"
        else:
            memory_str = "\n".join([d["content"] for d in self.archive])
        return f"\n### ARCHIVAL MEMORY ###" + f"\n{memory_str}"

    def insert(self, memory_string):
        self.archive.append(
            {
                "timestamp": utils.get_local_time(),
                "content": memory_string,
            }
        )

    def search(self, query_string, count=None, start=None):
        # run an (inefficient) case-insensitive match search
        matches = [s for s in self.archive if query_string.lower() in s["content"].lower()]
        # start/count support paging through results
        if start is not None and count is not None:
            return matches[start: start + count], len(matches)
        elif start is None and count is not None:
            return matches[:count], len(matches)
        elif start is not None and count is None:
            return matches[start:], len(matches)
        else:
            return matches, len(matches)

class RecallMemory(object):
    def __init__(self):
        self.message_logs = []

    def __len__(self):
        return len(self.message_logs)

    def __repr__(self) -> str:
        system_count = user_count = assistant_count = function_count = other_count = 0
        for msg in self.message_logs:
            role = msg["message"]["role"]
            if role == "system":
                system_count += 1
            elif role == "user":
                user_count += 1
            elif role == "assistant":
                assistant_count += 1
            elif role == "function":
                function_count += 1
            else:
                other_count += 1
        memory_str = (
                f"Statistics:"
                + f"\n{len(self.message_logs)} total messages"
                + f"\n{system_count} system"
                + f"\n{user_count} user"
                + f"\n{assistant_count} assistant"
                + f"\n{function_count} function"
                + f"\n{other_count} other"
        )
        return f"\n### RECALL MEMORY ###" + f"\n{memory_str}"

    def text_search(self, query_string, count=None, start=None):
        # run an (inefficient) case-insensitive match search
        message_pool = [d for d in self.message_logs if d["message"]["role"] not in ["system", "function"]]
        matches = [
            d for d in message_pool if
            d["message"]["content"] is not None and query_string.lower() in d["message"]["content"].lower()
        ]
        # start/count support paging through results
        if start is not None and count is not None:
            return matches[start: start + count], len(matches)
        elif start is None and count is not None:
            return matches[:count], len(matches)
        elif start is not None and count is None:
            return matches[start:], len(matches)
        else:
            return matches, len(matches)


import cv2
name = "blue_overhang"
cap = cv2.VideoCapture(f'../test_examples/videos/{name}.mp4')
fps = int(cap.get(cv2.CAP_PROP_FPS))
print('fps :'+str(fps))
# Change the FPS whatever you want
FPS=30
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
writer = cv2.VideoWriter('outpy.avi',cv2.VideoWriter_fourcc('M','J','P','G'), FPS, (frame_width,frame_height) )

if (cap.isOpened()== False):
  print("Error opening video stream or file")

while(cap.isOpened()):
  ret, frame = cap.read()
  if ret == True:
    cv2.imshow('Frame',frame)
    writer.write( frame)
    if cv2.waitKey(25) & 0xFF == ord('q'):
      break
  else:
    break
writer.release()
cap.release()
cv2.destroyAllWindows()


#!/usr/bin/env python3

import numpy as np

class Model:

    def __init__(self):
        self.g = 9.807
        self.v0 = 5
        self.theta = 80*np.pi/180
        self.U0 = [0, self.v0*np.cos(self.theta), 0, self.v0*np.sin(self.theta)]

    def f(u, t):
        x, vx, y, vy = u
        return [vx, 0, vy, -1*self.g]


"""
Given newline JSON with order data for each customer
The app read the data into Spark and calculate the net merchandise value of the ordered products).
After transforming the data it writes it into a mongodb file server.
"""

from pyspark.sql import SparkSession
from pyspark.sql import DataFrame
from pymongo import MongoClient
from loguru import logger

from pathlib import Path
from typing import List

from config import config

# Temporary hardcoded filepath. Would be good to fix in future
FILE_PATH = str(Path.cwd() / Path('etl/resources/data.json'))

def main():
    # start Spark application and get Spark session
    spark = SparkSession \
        .builder \
        .appName("net merchandise per customer") \
        .getOrCreate()

    df = load_json_data(FILE_PATH, spark)
    t_df = transform_data(df, spark)
    rows = dataframe_to_rows(t_df)

    # store the rows in mongodb database
    store_rows_in_database(rows)

def transform_data(df: DataFrame, spark) -> DataFrame:
    logger.debug("Starting data transformation...")

    df.createOrReplaceTempView('table')
    transformed = spark.sql(
        """
        with nett AS (
        select customerId
        , o.orderId as orderId,
            CASE
                WHEN b.productType = 'cold food' THEN b.grossMerchandiseValueEur - (b.grossMerchandiseValueEur * 0.15)
                WHEN b.productType = 'hot food' THEN b.grossMerchandiseValueEur - (b.grossMerchandiseValueEur * 0.07)
                WHEN b.productType = 'beverage' THEN b.grossMerchandiseValueEur - (b.grossMerchandiseValueEur * 0.09)
            END as nettMerchandiseValueEur
        from table
        lateral view explode(orders) as o
        lateral view explode(o.basket) as b
        )

        SELECT customerId
        , COUNT(orderId) as orders
        , ROUND(SUM(nettMerchandiseValueEur), 2) as totalNetMerchandiseValueEur
        FROM nett
        GROUP BY 1
        """)
    return transformed

def load_json_data(path: str, spark) -> DataFrame:
    logger.debug(f"Loading data at file path: {path}")
    return spark.read.json(path)

def store_rows_in_database(rows: List[dict]) -> None:
    collection = MongoClient()[config.DATABASE][config.COLLECTION]

    logger.info("Inserting rows into db...")

    collection.insert_many(rows)

def dataframe_to_rows(df: DataFrame) -> List[dict]:
    """
    creates a list of dicts where each dict is a row
    :param df: spark Dataframe
    :return: each row from the dataframe in a dict
    """
    m = map(lambda row: row.asDict(), df.collect())
    rows = [i for i in m]
    return rows

if __name__ == "__main__":
    main()