# -*- coding: utf-8 -*-

from odoo import models, fields, api, exceptions, _
from datetime import datetime, date

class SaleOrder(models.Model):
    _inherit = 'sale.order'

    #Pestana de documentos
    use_letter = fields.Binary(string="Carta de buen uso", attachment=True, required= True, copy=False)
    bank_transfer_receipt = fields.Binary(string="Comprobante de Transferencia", attachment=True, copy=False)
    contract = fields.Binary(string="Contrato", attachment=True, copy=False)
    foreign_currency_payroll = fields.Binary(string="Planilla de Recaudación Moneda Extranjera", attachment=True, copy=False)
    fund_availability = fields.Binary(string="Disponibilidad de Fondos", attachment=True, copy=False)
    release_authorization = fields.Binary(string="Autorización de la Exoneración", attachment=True, copy=False)
    sales_operational_worksheet = fields.Binary(string="Planilla de Operativo de Venta", attachment=True, copy=False)
    warehouse_domain = fields.Many2many('stock.warehouse')

    warehouse_id = fields.Many2one('stock.warehouse', string='Almacén',
        default=False, domain="[('id', 'in', warehouse_domain)]")

    currency_selection = fields.Selection(
    [('0', 'Bolívares'),
    ('1', 'Divisas'),
    ('2', 'No Aplica')
    ], string="Moneda", copy=False)

    bolivar_payment = fields.Selection([
        ('0', 'Transferencia Bolívares'),
        ('1', 'Débito en Cuenta'),
    ], string="Forma de Pago (Bolívares)", copy=False)

    dollar_payment = fields.Selection([
        ('0', 'Divisas en Taquilla'),
        ('1', 'Transferencia Divisas'),
    ], string="Forma de Pago (Divisas)", copy=False)

    not_apply = fields.Selection([
        ('0', 'Exonerado'),
    ], string="Forma de Pago (No Aplica)", copy=False)

    #General pedido de venta
    finance_approved_date = fields.Datetime(string="Fecha", readonly=True, copy=False)
    approved_regional_management_date = fields.Datetime(string="Fecha", readonly=True, copy=False)
    approved_regional_management_user = fields.Many2one(comodel_name ='res.users', string='Aprobado por', copy=False)
    finance_approved_user = fields.Many2one(comodel_name ='res.users', string='Aprobado por', copy=False)

    order_ids = fields.One2many('sale.order', 'opportunity_id', related="opportunity_id.order_ids")

    state = fields.Selection([
        ('draft', 'Quotation'),
        ('sent', 'Quotation Sent'),
        ('reg_manag', 'Espera de aprobación Gerencia Regional'),
        ('fin_approve', 'Espera de aprobación de Finanzas'),
        ('sale', 'Sales Order'),
        ('done', 'Locked'),
        ('cancel', 'Cancelled'),
        ], string='Status', readonly=False, copy=False, index=True, tracking=3, default='draft')
    appointment_date = fields.Date(string="Fecha de cita")
    sales_executive = fields.Many2one(comodel_name='res.users',string="Ejecutivo vendedor")

    #Campos traidos del modulo contactos
    affiliated = fields.Char(string="Numero de afiliación")
    denomination = fields.Char(string="Denominación comercial")
    cluster_id = fields.Many2one(comodel_name ='segmentation.cluster', string='Cluster', ondelete="restrict")
    name_owner =fields.Char(string ="Nombre y Apellido del Propietario")
    region_id = fields.Many2one('crm.region',string='Región', ondelete="restrict")
    phone_one = fields.Char(string="Teléfono")
    phone_two = fields.Char(string="Teléfono 2")
    phone_three = fields.Char(string="Teléfono 3")
    regional_manager = fields.Many2one(comodel_name='res.users', string="Gerente Regional")
    bank_ids = fields.One2many(related='partner_id.bank_ids', string='Banks')
    parent_id = fields.Many2one('res.partner', string='Empresa relacionada')
    street = fields.Char()
    street2 = fields.Char()
    zip = fields.Char(change_default=True)
    city = fields.Char()
    state_id = fields.Many2one("res.country.state", string='State', ondelete='restrict', domain="[('country_id', '=?', country_id)]")
    country_id = fields.Many2one('res.country', string='Country', ondelete='restrict')

    #Campos traidos del modulo CRM
    type_negotiation_id = fields.Many2one(comodel_name='crm.negotiation', string="Tipo de negociación", ondelete="restrict")
    product_type = fields.Selection([
        ('1', 'POS'),
        ('2', 'Accesorios'),
        ('3', 'POS y Accesorios')
    ], string="Tipo de Producto")
    type_point_sale_id = fields.Many2one(comodel_name='crm.point.sale', string="Tipo de comunicación", ondelete="restrict")
    company_pos_id = fields.Many2one(comodel_name='crm.company.pos', string="Operadora Telefónica", ondelete="restrict")
    origin_id = fields.Many2one(comodel_name='crm.origin', string="Origen", ondelete="restrict")
    kind_attention = fields.Selection([('1', 'Tradicional'), ('2', 'Evento'), ('3', 'VIP')], string="Tipo de atención", ondelete="restrict")
    event_name_id = fields.Many2one(comodel_name='event.name', string="Nombre del evento", ondelete="restrict")
    type_code = fields.Integer(related="type_point_sale_id.type_code")
    change_opportunity = fields.Boolean(compute='_compute_change_opportunity')
    inv = fields.Boolean(compute='_compute_inv')
    mail_partner_id = fields.Many2one('res.partner')

    def _compute_change_opportunity(self):
        if self.opportunity_id and self.state == 'sent' and len(self.opportunity_id.order_ids.filtered(lambda s: s.state == 'sent').ids) <= 1:
            self.change_opportunity = False
        else:
            self.change_opportunity = True

    #validaciones para el campo fecha de cita
    @api.constrains('appointment_date')
    def _onchange_appointment_date(self):
        if self.appointment_date:
            if self.appointment_date < date.today():
                raise exceptions.ValidationError('La fecha de la cita no puede ser menor a la fecha actual')
            if self.opportunity_id.appointment_date != self.appointment_date:
                self.opportunity_id.appointment_date = self.appointment_date

    def write(self, values):
        state = self.state
        result = super(SaleOrder, self).write(values)
        if self.state not in ['cancel', 'sale', 'done'] and not self._context.get('no_validate_stock', False):
            self.restrict_sale_without_stock(state)
        if self.is_rental_order and any( line.product_template_id.product_type == '0' and not line.product_template_id.rent_ok for line in self.order_line):
            raise exceptions.ValidationError("No esta permitido vender productos de tipo \"POS\" a travez de un presupuesto de alquiler")
        return result

    #Carga de datos del cliente y validacion de cliente activo
    @api.onchange('partner_id')
    def _onchange_partner_id(self):
        if ((self.partner_id.contact_type == '0' or self.partner_id.contact_type == '1') and self.partner_id.status_customer == '1') or not self.partner_id:
            self.denomination = self.partner_id.denomination
            self.cluster_id = self.partner_id.cluster_id
            self.name_owner = self.partner_id.name_owner
            self.region_id = self.partner_id.region_id
            self.phone_one = self.partner_id.phone_one
            self.phone_two = self.partner_id.phone_two
            self.phone_three = self.partner_id.phone_three
            self.regional_manager = self.partner_id.regional_manager
            self.user_id = self.partner_id.user_id
            self.sales_executive = self.partner_id.user_id
            self.affiliated = self.partner_id.affiliated
            self.street = self.partner_id.street
            self.street2 = self.partner_id.street2
            self.zip = self.partner_id.zip
            self.city = self.partner_id.city
            self.state_id = self.partner_id.state_id
            self.country_id = self.partner_id.country_id
            self.parent_id  = self.partner_id.parent_id
        else:
            raise exceptions.ValidationError('El contacto no esta habilitado para la creacion de un presupuesto')

    #carga de datos del CRM
    @api.onchange('opportunity_id')
    def _onchange_opportunity_id(self):
        if self.opportunity_id:
            self.type_negotiation_id = self.opportunity_id.type_negotiation_id
            self.product_type = self.opportunity_id.product_type
            self.type_point_sale_id = self.opportunity_id.type_point_sale_id
            self.company_pos_id = self.opportunity_id.company_pos_id
            self.origin_id = self.opportunity_id.origin_id
            self.kind_attention = self.opportunity_id.kind_attention
            self.event_name_id = self.opportunity_id.event_name_id
            self.appointment_date = self.opportunity_id.appointment_date
            self.partner_invoice_id = self.opportunity_id.company_address
            self.origin = self.opportunity_id.lead_sequence
            self.set_domain()

    @api.constrains('opportunity_id')
    def set_budget_send(self):
        if not self.opportunity_id.budget_send and self.state == 'sent':
            self.opportunity_id.budget_send = True

    @api.onchange('origin')
    def _onchange_origin(self):
        if self.origin:
            opport = self.env['crm.lead'].search([('lead_sequence', '=', self.origin),('partner_id', '=', self.partner_id.id),('stage_code', '=', 3)], limit=1)
            if opport:
                if self.opportunity_id:
                    if self.opportunity_id.id != opport.id:
                        if opport.budget_send:
                            raise exceptions.ValidationError(_("You cannot assign sales quotes to opportunities with submitted quotes."))
                        else:
                            self.opportunity_id = opport
                elif opport.budget_send:
                    raise exceptions.ValidationError(_("You cannot assign sales quotes to opportunities with submitted quotes."))
                else:
                    self.opportunity_id = opport
            else:
                raise exceptions.ValidationError(_("No opportunity was found that matches the source document provided."))

    @api.onchange('kind_attention', 'event_name_id')
    def set_domain(self):
        if self.kind_attention == '2':
            self.warehouse_domain = self.env['stock.warehouse'].search([('kind_attention', '=', '2'), ('event', '=', self.event_name_id.id)])
        elif self.kind_attention in ['1', '3']:
            self.warehouse_domain = self.env['stock.warehouse'].search([('kind_attention', '=', '1')])

        if len(self.warehouse_domain) == 1:
            self.warehouse_id = self.warehouse_domain[0].id.origin
        else:
            self.warehouse_id = False

    # direccion de factura debe ser igual en presupuestos y oportunidades
    @api.constrains('partner_invoice_id')
    def constrains_partner_invoice_id(self):
        if self.partner_invoice_id != self.opportunity_id.company_address:
            self.opportunity_id.company_address = self.partner_invoice_id

    #Buscar el contacto por el numero de afiliado
    @api.onchange('affiliated')
    def _onchange_affiliated(self):
        if self.affiliated:
            partner = self.env['res.partner'].search([('affiliated', '=', self.affiliated)], limit=1)
            if partner:
                self.partner_id = partner.id
            else:
                raise exceptions.ValidationError('La afiliación ingresada no pertenece a ningun contacto')

    def action_quotation_send(self):
        self.restrict_sale_without_stock()
        return super(SaleOrder, self).action_quotation_send()

    #Extencion de funcion confirmar pedido de venta
    def action_confirm(self):
        if self.partner_id.status_customer == '1':
            if self.kind_attention != '2':
                self.opportunity_id.stage = self.env['crm.stage'].search([('stage_code', '=', 6)])
                self.opportunity_id.stage_code = 6
                self.opportunity_id.scheduled_date = True
            if not self.opportunity_id.budget_confirmed:
                self.opportunity_id.budget_confirmed = True
                self.opportunity_id.budget_confirmed_date = datetime.now()
            self.appointment_date = self.opportunity_id.appointment_date
            context = self._context
            self.finance_approved_user = context.get('uid')
            self.finance_approved_date = datetime.now()
            for order in self.opportunity_id.order_ids.filtered(lambda s: s.state in ['draft', 'sent', 'reg_manag', 'fin_approve'] and s.id != self.id):
                order.action_cancel()

            #send assigned to email
            lang = self.user_id.partner_id.lang if self.user_id.partner_id else None
            self.mail_partner_id = self.user_id.partner_id
            self.with_context(lang=lang)._message_auto_subscribe_notify([self.user_id.partner_id.id], 'sale_customization.message_sale_assigned')

            return super().action_confirm()
        else:
            raise exceptions.ValidationError('Cliente INACTIVO. No se puede confirmar el presupuesto')

    #Limpiar los campos al cambiar el tipo de moneda
    @api.onchange('currency_selection')
    def _onchange_currency_selection(self):
        self.bolivar_payment = False
        self.dollar_payment = False
        self.not_apply = False

    #Boton de aprobacion en estado de gerencia regional
    def approved_regional_management_button(self):
        context = self._context
        self.state = 'fin_approve'
        self.approved_regional_management_user = context.get('uid')
        self.approved_regional_management_date = datetime.now()

    asigned_partner = fields.Many2one('res.partner', copy=False)

    #cambio de chek presupuesto enviado en el CRM
    @api.constrains('state')
    def constrains_sent_invoice(self):
        if self.state == 'sent':
            if not self.opportunity_id.budget_send:
                self.opportunity_id.budget_send = True
                self.opportunity_id.budget_send_date = datetime.now()

        if self.state == 'fin_approve':
            if not self.asigned_partner:
                #get all users in the group
                partners = self.env['res.users'].search([]).filtered(lambda s: s.has_group("sale_customization.group_sale_finance_approval_notification")).partner_id
                if partners:
                    self.asigned_partner = partners.sorted(key = lambda s: s.asigned_orders)[0]
                    self.sudo().asigned_partner.asigned_orders +=1

                    if self.asigned_partner.asigned_orders == 2147480000:
                        for partner in partners:
                            partner.asigned_orders = 0

            if self.asigned_partner:
                if self.asigned_partner.id not in self.message_partner_ids.ids:
                    vals = {
                        'res_id': self.id,
                        'res_model': 'sale.order',
                        'partner_id': self.asigned_partner.id
                        }
                    self.env['mail.followers'].create(vals)
                lang = self.asigned_partner.lang if self.asigned_partner else None
                self.mail_partner_id = self.asigned_partner
                self.with_context(lang=lang)._message_auto_subscribe_notify([self.asigned_partner.id], 'sale_customization.message_sale_assigned')

    #Funcion de boton Solicitud de aprobación
    def approved_regional_management_change(self):
        cont = 0
        if self.partner_id.cedula == False or self.partner_id.rif == False:
            raise exceptions.ValidationError('El contacto no posee sus documentos completos')
        if self.opportunity_id:
            for record in self.opportunity_id.order_ids:
                if record.state == 'reg_manag' or record.state == 'fin_approve' or record.state == 'sale':
                    cont += 1
        if cont > 0:
            raise exceptions.ValidationError('Ya existe un presupuesto en espera de aprobación para esta oportunidad')
        else:
            self.opportunity_id.type_negotiation_id = self.type_negotiation_id
            self.opportunity_id.product_type = self.product_type
            self.opportunity_id.type_point_sale_id = self.type_point_sale_id
            self.opportunity_id.company_pos_id = self.company_pos_id
            self.opportunity_id.origin_id = self.origin_id
            self.opportunity_id.kind_attention = self.kind_attention
            self.opportunity_id.event_name_id = self.event_name_id
            self.state = 'reg_manag'

        # add to followers
        if self.regional_manager.partner_id.id not in self.message_partner_ids.ids:
            vals = {
                'res_id': self.id,
                'res_model': 'sale.order',
                'partner_id': self.regional_manager.partner_id.id
                }
            self.env['mail.followers'].create(vals)

        # send approval notification
        self.send_approval_message(self.regional_manager)

        # self.env['mail.message'].create({
        #     'subject': _("Need approval"),
        #     'body': "
El pedido de venta necesita aprobación
",
        #     'model': 'sale.order',
        #     'res_id': self.id,
        #     'record_name': self.display_name,
        #     'message_type': 'notification',
        #     'partner_ids': [2, 3, 4],
        # })

    def send_approval_message(self, user):
        view = self.env['ir.ui.view'].browse(self.env['ir.model.data'].xmlid_to_res_id('sale_customization.message_sale_approval'))

        model_description = self.env['ir.model']._get(self._name).display_name
        values = {
            'object': self,
            'model_description': model_description,
        }
        assignation_msg = view.render(values, engine='ir.qweb', minimal_qcontext=True)
        assignation_msg = self.env['mail.thread']._replace_local_links(assignation_msg)
        self.message_notify(
            subject=_('Approval of %s') % self.display_name,
            body=assignation_msg,
            partner_ids=[user.partner_id.id],
            record_name=self.display_name,
            email_layout_xmlid='mail.mail_notification_light',
            model_description=model_description,
        )

    condition_create_invoice = fields.Boolean(default=True)
    @api.constrains('currency_selection', 'bolivar_payment', 'dollar_payment', 'not_apply', 'use_letter', 'bank_transfer_receipt', 'contract', 'foreign_currency_payroll', 'fund_availability', 'release_authorization', 'sales_operational_worksheet', 'appointment_date')
    def _onchange_condition_create_invoice(self):
        self.condition_create_invoice = True
        if self.kind_attention == '2':
            if self.currency_selection == '0' and self.bolivar_payment == '0' and self.bank_transfer_receipt and self.contract and self.use_letter and self.appointment_date:
                self.condition_create_invoice = False
            if self.currency_selection == '0' and self.bolivar_payment == '1' and self.fund_availability and self.contract  and self.use_letter and self.appointment_date:
                self.condition_create_invoice = False
            if self.currency_selection == '1' and self.dollar_payment == '0' and self.foreign_currency_payroll and self.contract and self.use_letter and self.appointment_date:
                self.condition_create_invoice = False
            if self.currency_selection == '1' and self.dollar_payment == '1' and self.bank_transfer_receipt and self.contract and self.use_letter and self.appointment_date:
                self.condition_create_invoice = False
            if self.currency_selection == '2' and self.not_apply == '0' and self.release_authorization and self.appointment_date:
                self.condition_create_invoice = False
        else:
            if self.currency_selection == '0' and self.bolivar_payment == '0' and self.bank_transfer_receipt and self.contract and self.use_letter:
                self.condition_create_invoice = False
            if self.currency_selection == '0' and self.bolivar_payment == '1' and self.fund_availability and self.contract  and self.use_letter:
                self.condition_create_invoice = False
            if self.currency_selection == '1' and self.dollar_payment == '0' and self.foreign_currency_payroll and self.contract and self.use_letter:
                self.condition_create_invoice = False
            if self.currency_selection == '1' and self.dollar_payment == '1' and self.bank_transfer_receipt and self.contract and self.use_letter:
                self.condition_create_invoice = False
            if self.currency_selection == '2' and self.not_apply == '0' and self.release_authorization:
                self.condition_create_invoice = False

    def _compute_condition_create_invoice_scheduled_date(self):
            if self.opportunity_id and self.kind_attention == '2' :
                if self.opportunity_id.scheduled_date or self.appointment_date:
                    self.create_invoice_scheduled_date = True
                else:
                    self.create_invoice_scheduled_date = False
            else:
                self.create_invoice_scheduled_date = True
    create_invoice_scheduled_date = fields.Boolean(compute=_compute_condition_create_invoice_scheduled_date)

    def no_approve(self):
        self.with_context(no_validate_stock = True).state = 'sent'

    def action_cancel(self):
        if self.state == 'sent' and self.opportunity_id and self.opportunity_id.stage_id.stage_code in [3, 4, 5] and not any(order.state == 'sale' for order in self.opportunity_id.order_ids):
            raise exceptions.ValidationError('No puede ser cancelado un presupuesto de venta si la oportunidad no posee un pedido de venta, si la oportunidad se encuentra en "En negociación", "En evaluación del cliente" o "En espera de cita"')
        else:
            super(SaleOrder, self).action_cancel()

    def action_draft(self):
        orders = self.with_context(no_validate_stock = True).filtered(lambda s: s.state in ['cancel', 'sent'])
        return orders.write({
            'state': 'draft',
            'signature': False,
            'signed_by': False,
            'signed_on': False,
        })

    @api.depends('regional_manager')
    def _compute_inv(self):
        for rec in self:
            if rec.state == 'reg_manag' and rec.regional_manager:
                rec.inv = (rec.regional_manager.id != rec.env.user.id and not (rec.env.user.has_group('base.user_admin') or rec.env.user.has_group('sale_customization.group_sale_secundary_approver')))
            else:
                rec.inv = True

    def restrict_sale_without_stock(self, state=''):
        no_stock_product = {}
        actual_state = state if state != '' else self.state
        for line in self.order_line.filtered(lambda l: l.product_id.type == 'product'):
            if not line.is_rental and line.prod_qty_for_location < line.product_uom_qty:
                if no_stock_product.get(line.product_id, False):
                    no_stock_product[line.product_id][0] += int(line.product_uom_qty)
                else:
                    no_stock_product[line.product_id] = [int(line.product_uom_qty), int(line.prod_qty_for_location)]

            if line.is_rental and actual_state=='draft' and line.prod_qty_for_location_rental < line.product_uom_qty:
                if no_stock_product.get(line.product_id, False):
                    no_stock_product[line.product_id][0] += int(line.product_uom_qty)
                else:
                    no_stock_product[line.product_id] = [int(line.product_uom_qty), int(line.prod_qty_for_location_rental)]
        if no_stock_product:
            message = "No existen cantidades disponibles para los siguientes productos:\n"
            for product, counters in no_stock_product.items():
                message += "- {}. Necesario: {} Disponible: {} \n".format(product.name, counters[0], counters[1])
            raise exceptions.ValidationError(message)

class SaleOrderLine(models.Model):
    _inherit = "sale.order.line"

    prod_qty_for_location = fields.Integer(compute='_compute_prod_qty_for_location')
    prod_qty_for_location_rental = fields.Integer(compute='_compute_prod_qty_for_location')

    @api.depends('product_id', 'product_uom_qty', 'qty_delivered', 'state', 'product_uom')
    def _compute_prod_qty_for_location(self):
        for rec in self:
            rec.prod_qty_for_location = 0
            rec.prod_qty_for_location_rental = 0
            if rec.order_id.warehouse_id and self.product_id:
                if rec.order_id.warehouse_id.sale_stock_location:
                    rec.prod_qty_for_location = rec.env['stock.quant']._get_available_quantity(rec.product_id, rec.order_id.warehouse_id.sale_stock_location)
                if rec.order_id.warehouse_id.rental_stock_location:
                    rec.prod_qty_for_location_rental = rec.env['stock.quant']._get_available_quantity(rec.product_id, rec.order_id.warehouse_id.rental_stock_location)

    @api.depends('product_id', 'product_uom_qty', 'qty_delivered', 'state', 'product_uom')
    def _compute_qty_to_deliver(self):
        """Compute the visibility of the inventory widget."""
        for line in self:
            line.qty_to_deliver = line.product_uom_qty - line.qty_delivered
            if line.product_type == 'product' and line.product_uom and line.qty_to_deliver > 0:
                line.display_qty_widget = True
            else:
                line.display_qty_widget = False

    product_uom_qty = fields.Float(digits='None Decimal')
    qty_delivered = fields.Float(digits='None Decimal')
    qty_invoiced = fields.Float(digits='None Decimal')


# -*- coding: utf-8 -*-
"""
Created on Mon Jan 18 10:31:19 2021

@author: Alexandre

Binary/Adaptive Threshold:

    https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html

"""
from abc import ABC, abstractmethod

from matplotlib.colors import LogNorm, Normalize

import numpy as np
import pandas as pd
import cv2
from skimage import filters
from scipy.ndimage import gaussian_filter

class Filter(ABC):
    def __init__(self, threshold):
        self.threshold = threshold

    @abstractmethod
    def apply(self):
        pass

class Laplacian(Filter):
    def __init__(self, depth = cv2.CV_32F, kernel_size = 5, threshold = 80):
        super().__init__(threshold)

        self.depth = depth
        self.kernel_size = kernel_size

    def apply(self, image):
        image = cv2.GaussianBlur(image, (3, 3), 0)
        image = cv2.Laplacian(image, self.depth,  ksize=self.kernel_size)
        image = cv2.convertScaleAbs(image)
        image = np.flip(image, 0)
        ret,image = cv2.threshold(image,self.threshold, 255,cv2.THRESH_BINARY)
        return image

class Sobel(Filter):
    def __init__(self, threshold = 20):
        super().__init__(threshold)

    def apply(self, image):
        image = filters.sobel(image)
        image = cv2.normalize(src = image, dst=None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1)
        ret,image = cv2.threshold(image, self.threshold, 255,cv2.THRESH_BINARY)
        return image

class Gaussian(Filter):
    def __init__(self, sigma = 1, norm = False, threshold = 80):
        super().__init__(threshold)
        self.sigma = sigma
        self.norm = norm

    def apply(self, image):
        image = image.astype(float)
        image = gaussian_filter(image, sigma = self.sigma)

        df = pd.DataFrame(image)
        gx = df.diff(axis = 1).drop(columns = df.columns[0])
        gy = df.diff().drop(df.index[0])

        image = (gx.drop(df.index[0]).values**2 + gy.drop(columns = df.columns[0]).values**2)**(1/2)

        image = cv2.normalize(src = image, dst=None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1)

        if self.norm == True:
            #TODO: apply norm to magnitude
            norm = LogNorm(image.mean().mean() + 0.5 * image.std().std(), image.max().max())
            image *= norm
        return image

class Adaptative(Filter):
    def __init__(self, sigma = 1, norm = False, threshold = 20):
        super().__init__(threshold)
        self.sigma = sigma
        self.norm = norm

    def apply(self, image):
        image = image.astype(float)
        image = gaussian_filter(image, sigma = self.sigma)

        df = pd.DataFrame(image)
        gx = df.diff(axis = 1).drop(columns = df.columns[0])
        gy = df.diff().drop(df.index[0])

        image = (gx.drop(df.index[0]).values**2 + gy.drop(columns = df.columns[0]).values**2)**(1/2)

        vmin = image.mean().mean() + 0.5 * image.std().std()
        vmax = image.max().max()
        np.log(image, image)
        result = image.copy()
        result -= np.log(vmin)
        result /= (np.log(vmax) - np.log(vmin))
        image = result * 255

        #TODO: add binary threshold there
        ret,image = cv2.threshold(image, self.threshold, 255,cv2.THRESH_BINARY)

        image = cv2.normalize(src = image, dst=None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1)
        return image

class Canny(Filter):
    def __init__(self, threshold = 80):
        super().__init__(threshold)

    def apply(self, image):
        image = cv2.Canny(image, 50, 200, None, 3)
        return image


from Task import *
from TaskHelperTest import *
import subprocess

class TaskHelperImpl(TaskHelper):
    """Implementation of TaskHelper with interaction with the real schtasks instead of
    an abstraction for testing. This should always be used in production."""
    loaded_tasks = []
    deleted_tasks = []

    def __init__(self):
        print("Running Task Helper, interaction with actual schtasks")
        self.loaded_tasks = []
        self.schtasks_loaded_tasks = []
        self.get_tasks()

    def log(self, *content):
        """Logs content with information specifying it comes from the task helper"""
        print("(TASK HELPER [prod]): ", str(*content).replace('\\n','\n'))

    def run_command_with_output(self, command,shell=True):
        self.log("Running command: "+str(command))
        return subprocess.check_output(command,shell=shell)

    def save(self):
        """Saves loaded tasks"""
        self.log("TODO: SAVE LOADED TASKS")
        for task in self.deleted_tasks:
            try:
                sample_command = f"schtasks /f /delete /tn \"\\task master\\{task.hash_inclusive_name}\""
                task_delete_output = str(self.run_command_with_output(sample_command, shell=True))
                self.log(task_delete_output)
            except subprocess.CalledProcessError as e:
                print(e)
                continue
        for task in self.loaded_tasks:
            print(f'Task: name {task.name} h_inc {task.hash_inclusive_name} in_db {task.name_in_database} trigger {task.trigger} target {task.target}')
            if not task.name_in_database == task.hash_inclusive_name:
                try:
                    sample_command = f"schtasks /f /delete /tn \"\\task master\\{task.name_in_database}\""
                    task_delete_output = str(self.run_command_with_output(sample_command, shell=True))
                    self.log(task_delete_output)
                except subprocess.CalledProcessError as e:
                    print(e)
                    continue
        for task in self.loaded_tasks:
            sample_command = f"schtasks /f /create /tn taskname /tr taskrun /sc taskschedule"
            if task.trigger is Trigger.NONE:
                """Literal garbage. There's no way to make a disabled task directly from the command line
                So it's necessary to make it a scheduled one with an impossible time range.
                Schtasks doesn't need a wrapper, it needs to be abandoned. The code I'm interacting with
                Was written before the sample range I'm using. Honestly annoyed. 02/09/2022"""
                sample_command += " /ST 00:00 /SD 01/01/1900 /ED 02/01/1900"
                sample_command = sample_command.replace("taskschedule", "MONTHLY")
            else:
                sample_command = sample_command.replace("taskschedule", task.trigger.name)
            sample_command = sample_command.replace("taskname",f"\"\\task master\\{task.hash_inclusive_name}\"")\
                .replace("taskrun",task.target if task.target else '/')
            task_create_output = str(self.run_command_with_output(sample_command, shell=True)).replace(":\\", ";\\").replace("\\\\", "\\")
            self.log(task_create_output)
        self.deleted_tasks=[]

    def new_task(self, task):
        """Appends a new task to the loaded tasks. Does not save."""
        self.loaded_tasks.append(task)
        self.log("TODO: ADD TASK")

    def delete_task(self, task):
        """Removes a task from the loaded tasks. Does not save."""
        self.deleted_tasks.append(task)
        self.loaded_tasks.remove(task)
        self.log("TODO: DELETE TASK")

    def delete_task_by_index(self, index):
        """Removes a task from the loaded tasks by array index. Does not save."""
        del self.loaded_tasks[index]

    def get_tasks(self):
        """Returns loaded tasks.
        Eldritch arbitrary output format parsing code. Requires a wizard at the helm."""
        self.log("TODO: RETURN SAVED TASKS")
        task_query_output = ""
        try:
            task_query_output = str(self.run_command_with_output("schtasks /query /tn \"\\task master\"\\ /fo list /v", shell=True)).replace(":\\",";\\").replace(":/",";/").replace("\\\\","\\")
        except subprocess.CalledProcessError as exc:
            print("Schtasks query failed. Assuming no tasks - query fails when no tasks are found."+str(exc.returncode)+str(exc.output))
        task_query_output = [dict([j.strip() for j in i.split(":")[:2]] if len(i.split(":")[:2])==2 else ["key", "value"] for i in task.split("\\r\\n")) for task in task_query_output.split("HostNam")]
        self.log([i for i in task_query_output])
        temporary_tasks = []
        for task_dict in task_query_output:
            if task_dict.get("TaskName"):
                temporary_task = Task(name=task_dict.get("TaskName").replace("\\task master\\", "").replace("\\\\", ""), new = False)
                temporary_task.target=task_dict.get("Task To Run").replace(";\\",":\\").replace(";/",":/")
                temporary_task.trigger=Trigger(task_dict.get("Schedule Type"))
                temporary_tasks.append(temporary_task)
        self.loaded_tasks=self.schtasks_loaded_tasks=temporary_tasks
        return self.loaded_tasks

    def get_task_by_index(self, index):
        """Returns loaded task by index, returning an empty task if it doesn't exist"""
        try:
            return self.loaded_tasks[index]
        except IndexError:
            return EmptyTask()

    def set_task_by_index(self, index, new_task):
        """Sets task by index, producing an error message if the task at given index does not exist."""
        self.log("TODO: SET TASK BY INDEX")
        try:
            self.loaded_tasks[index] = new_task
        except IndexError:
            self.log("User has attempted to edit nonetask")
            from tkinter import messagebox
            messagebox.showerror("Error", "Please create a task first.")


def dodaj_do_rownania(rownanie, wartosci, x):
    wartosc = int(wartosci.pop(0))
    wynik = rownanie * x + wartosc
    if len(wartosci) == 0:
        print("Wynik wielomianu to:")
        print(wynik)
        # return wynik
    else:
        dodaj_do_rownania(wynik, wartosci, x)

def main():
    wartosci = []
    print("Podaj niewiadomą x.")
    x = int(input("> "))
    print("Podaj stopień wielomianu.")
    stopien = int(input("> "))
    stopien = stopien + 1

    for liczbaPetli in range(0, stopien):
        print("Podaj wartość kolejnego składnika wielomianu (od najwyższego stopnia).")
        y = int(input("> "))
        wartosci.append(y)

    dodaj_do_rownania(0, wartosci, x)

main()


import torch
from torch.nn import Module

class ContrastLoss(Module):
    def __init__(self, temperature=0.1):
        super(ContrastLoss, self).__init__()
        self.temperature = temperature

    def forward(self, feat):
        anchors, positives, negatives = feat['anchor'], feat['positive'], feat['negative']
        anchor = anchors[:, 0]
        loss_1 = self.loss(anchor, positives, negatives)
        positive = positives[:, 0]
        loss_2 = self.loss(positive, anchors, negatives)
        return loss_1 + loss_2

    def loss(self, anchor, positives, negatives):
        pos = torch.exp(torch.bmm(positives, anchor[..., None]) / self.temperature).squeeze(-1)
        neg = torch.exp(torch.bmm(negatives, anchor[..., None]) / self.temperature).squeeze(-1)
        dominator = pos.sum(dim=1) + neg.sum(dim=1)
        loss = - torch.log(pos).mean(dim=1) + torch.log(dominator)
        return loss.mean()


from collections import OrderedDict
import numpy as np
from math import ceil
import cv2
from io import BytesIO
from pathlib import Path
from PIL import Image

from utils import *
from huffman import *

def decode_header(reader: BytesIO):
    read = reader.read
    assert read(2) == MARKER.SOI  # Start of Image
    marker = read(2)
    if marker == MARKER.APP0:  # Application Marker 0
        APP0 = dict(
            length=read(2),
            identifier=read(5),
            version=read(2),  # b'\x01\x01'
            unit=read(1),
            x_density=read(2),
            y_density=read(2),
        )
        APP0['thumbnail-data'] = read(bytes2int(APP0['length']) - 14)
        marker = read(2)

    APPns = []
    while MARKER.APPn[0] <= marker <= MARKER.APPn[1]:  # Application Marker n
        APPn = dict(marker=marker, length=read(2))
        APPns.append(APPn)
        APPn['content'] = read(bytes2int(APPn['length']) - 2)
        marker = read(2)

    qts = {}
    while marker == MARKER.DQT:  # Define Quantization Table
        length, byte = bytes2int(read(2)), read(1)[0]
        precision, id_ = byte >> 4, byte & 0b1111
        assert precision == 0, 'only support uint8'
        table_size = (precision + 1) * 64
        assert length == table_size + 3
        dtype = np.uint8 if precision == 0 else np.uint16
        qts[id_] = np.frombuffer(
            read(table_size), dtype=dtype, count=64).astype(np.int32)
        marker = read(2)

    assert marker == MARKER.SOF0  # Start of Frame
    length = bytes2int(read(2))
    precision = read(1)[0]
    assert precision == 8, 'only support 8'
    height, width = bytes2int(read(2)), bytes2int(read(2))
    cop_num = read(1)[0]  # 1: grayscale; 3: YCrCb;
    assert cop_num in (1, 3), 'only support grayscale and RGB images'
    # components_name: ['Y', 'Cr', 'Cb']
    cop_infos = {}
    for i in range(cop_num):
        cop_id, ratio, qt_id = read(3)  # cop_id: 1, 2, 3; qt_id: 0, 1
        horizontal_ratio, vertical_ratio = ratio >> 4, ratio & 15
        cop_infos[cop_id] = ComponentInfo(
            id_=cop_id,
            horizontal=horizontal_ratio,
            vertical=vertical_ratio,
            qt_id=qt_id,
            dc_ht_id=None,
            ac_ht_id=None,
        )

    marker = read(2)
    hts = {}  # (type, id): table
    # type 0: DC, 1: AC
    # Huffman table id: 0, 1
    while marker == MARKER.DHT:  # Define Huffman Table
        length, byte = bytes2int(read(2)), read(1)[0]
        type_, id_ = byte >> 4, byte & 0b1111
        count = np.frombuffer(read(16), dtype=np.uint8, count=16)
        total = count.sum()
        weigh = np.frombuffer(read(total), dtype=np.uint8, count=total)
        assert length == 19 + total
        hts[type_, id_] = canonical_huffman_table(count, weigh)
        marker = read(2)

    DRI = None
    if marker == MARKER.DRI:  # Define Restart Interval
        DRI = dict(marker=marker, length=bytes2int(read(2)))
        DRI['content'] = read(length - 2)
        marker = read(2)
    assert DRI is None, 'not support DRI yet'

    assert marker == MARKER.SOS
    length = bytes2int(read(2))
    assert read(1)[0] == cop_num
    for i in range(cop_num):
        cop_id, ht_id = read(2)
        # ht_id: (dc_ht_id, ac_ht_id)
        cop_infos[cop_id].dc_ht_id = ht_id >> 4
        cop_infos[cop_id].ac_ht_id = ht_id & 0b1111
    # cop_id: 1 Y, 2 Cb, 3 Cr
    cop_infos = [cop_infos[i + 1] for i in range(cop_num)]
    assert read(3) == b'\x00\x3f\x00'
    return qts, hts, cop_infos, height, width

def decode_mcu(stream, mcu_num, block_num, hts):
    mcu = np.zeros([mcu_num, block_num, 64], dtype=np.int32)
    for cur in mcu:
        for dct, (dc_ht, ac_ht) in zip(cur, hts):
            # decode DC coefficients
            dc_size = read_huffman_code(dc_ht, stream)
            dc_complement = stream.read_int(dc_size)
            dct[0] = decode_2s_complement(dc_complement, dc_size)

            ac_index = 1
            rss_ = []
            while ac_index < 64:
                # obtain (run, size) pair
                rs = read_huffman_code(ac_ht, stream)
                run_length, ac_size = rs >> 4, rs & 15
                if (run_length, ac_size) == (0, 0):
                    dct[ac_index:] = 0
                    break
                dct[ac_index:ac_index + run_length] = 0
                ac_index += run_length
                ac_complement = stream.read_int(ac_size)
                dct[ac_index] = decode_2s_complement(ac_complement, ac_size)
                rss_.append((run_length, ac_size,
                             decode_2s_complement(ac_complement, ac_size)))
                ac_index += 1
    # Due to byte alignment, some bits (< 8) may be not used.
    return mcu

def iDPCM(mcu, cop_infos):
    index, mcu_num = 0, mcu.shape[0]
    for cop in cop_infos:
        num = cop.horizontal * cop.vertical
        dc = mcu[:, index:index + num, 0].flat
        for i in range(1, num * mcu_num):
            dc[i] += dc[i - 1]
        index += num
    return mcu

def de_quantization(mcu, cop_infos, qts):
    index = 0
    for cop in cop_infos:
        num = cop.horizontal * cop.vertical
        mcu[:, index:index + num] *= qts[cop.qt_id][None, None]
        index += num
    return mcu

def zigzag_decode(zz):
    dct = np.zeros((zz.shape[0], zz.shape[1], BH, BW), dtype=np.float32)
    trace = zigzag_points(BH, BW)
    for i, p in enumerate(trace):
        dct[:, :, p[0], p[1]] = zz[:, :, i]
    return dct

def iDCT(dct):
    block = np.zeros_like(dct)
    # the input of DCT/iDCT must be float number
    for i in range(dct.shape[0]):
        for j in range(dct.shape[1]):
            block[i, j] = cv2.idct(dct[i, j])
    return block

def decode_jpeg(reader):
    qts, hts, cop_infos, height, width = decode_header(reader)
    remains = reader.read()
    data, marker = remains[:-2].replace(b'\xff\x00', b'\xff'), remains[-2:]
    bit_stream = BitStreamReader(data)
    assert marker == MARKER.EOI

    # for blocks in MCU, choose the corresponding huffman tables to decode
    mcu_hts = []
    for cop in cop_infos:
        for _ in range(cop.horizontal * cop.vertical):
            mcu_hts.append((hts[0, cop.dc_ht_id], hts[1, cop.ac_ht_id]))

    # the height and width of MCU
    mh, mw = 8 * cop_infos[0].vertical, 8 * cop_infos[0].horizontal
    mcu_num = ceil(height / mh) * ceil(width / mw)
    block_num = len(mcu_hts)  # the number of blocks in each MCU
    mcu = decode_mcu(bit_stream, mcu_num, block_num, mcu_hts)

    # reverse the differential encoding (DPCM) of DC coefficients
    mcu = iDPCM(mcu, cop_infos)
    # inverse quantization
    mcu = de_quantization(mcu, cop_infos, qts)

    # zigzag to block, the returned result is float32
    mcu = zigzag_decode(mcu)

    # iDCT
    block = iDCT(mcu)

    # arrange blocks to form images
    ims, index = [], 0
    for cop in cop_infos:
        hr, vr = cop.horizontal, cop.vertical
        num = hr * vr
        # the number of MCUs in each column/row
        nh, nw = ceil(height / mh), ceil(width / mw)
        if num == 1:  # Cb, Cr, or Y when 4:4:4
            im = restore_image(block[:, index], nh, nw)
        else:  # Y when 4:2:2 or 4:2:0
            mat = block[:, index:index + num].reshape(
                -1, vr, hr, BH, BW).swapaxes(2, 3).reshape(-1, vr*BH, hr*BW)
            im = restore_image(mat, nh, nw)
        index += num
        ims.append(im)
    # reverse DC level shift for luminance
    ims[0] += 128
    if len(ims) == 3:
        # interpolating for chroma
        if ims[0].shape != ims[1].shape:
            h_, w_ = ims[0].shape
            for i in range(1, 3):
                ims[i] = cv2.resize(ims[i], (w_, h_))
        im = np.stack(ims, axis=-1)
        im = YCbCr2RGB(im)
    else:
        im = ims[0]
    im = im[:height, :width].round()
    im[im < 0], im[im > 255] = 0, 255
    return im.astype(np.uint8)

def read_jpeg(filename):
    reader = BytesIO(Path(filename).read_bytes())
    return decode_jpeg(reader)

def main():
    im = read_jpeg('image/color.jpg')
    Image.fromarray(im).save('image/color-decode.bmp')

if __name__ == '__main__':
    main()


import requests
import json
import pandas as pd
import streamlit as st
from os import getcwd, listdir
from json_normalize import json_transformation
from db_connection import cursor
from db_load import load_json_data
from duckdb_con import cursor

# Create a title
st.title("Pandascore API Data Retrieval")

# Encrypted Text Input
api_token = st.text_input("Paste in your Pandascore API Token: ", type='password')

# # Customization Text Inputs
# game_options = ['Counter Strike: Global Offensive', 'Call of Duty: Modern Warfare', 'Dota 2', 'FIFA', 'League of Legends', 'Overwatch', 'Public Underground Battle Ground (PUBG)', 'Rainbow Six Siege', 'Rocket League', 'Valorant', 'King of Glory', 'League of Legends: Wild Rift', 'Starcraft 2', 'Starcraft: Brood War']
# game_short_forms = ['csgo', 'codmw', 'dota2', 'fifa', 'lol', 'ow', 'pubg', 'r6siege', 'rl', 'valorant', 'kog', 'lol-wild-rift', 'starcraft-2', 'starcraft-brood-war']
game_df = pd.read_sql('games', con=cursor)

game_options = game_df['name'].tolist()
game_short_forms = game_df['short_name'].tolist()
game_api_map = {game: game_short_forms[i] for i, game in enumerate(game_options)}

game_name = st.selectbox("Choose a game: ", options=game_options)

endpoint_df = pd.read_sql('endpoints', con=cursor)
endpoint_options = endpoint_df['endpoint'].tolist()
# endpoint_options = ['leagues', 'matches', 'players', 'series', 'teams', 'tournaments']
endpoint_name = st.selectbox("Choose an endpoint: ", options=endpoint_options)

st.markdown("---")
st.markdown("<h3 style='text-align:center'> Query Parameters </h3>", unsafe_allow_html=True)

# Query Parameters
page_number = st.number_input("Choose a page number: ", min_value=1, max_value=10, value=1)
per_page = st.number_input("Choose number of entries per page: ", min_value=10, max_value=100, step=10)

# Create the Logic
data_path = getcwd() + "/data"

# Get JSON File Name
json_file_name = f"{game_name}_{endpoint_name}_{page_number}_{per_page}.json"

# Customized URL
url = f"https://api.pandascore.co/{game_api_map[game_name]}/{endpoint_name}?sort=&page={page_number}&per_page={per_page}&token={api_token}"

# Function to execute API call
def api_call(url):

    # Retrieve a response
    response = requests.get(url)

    # JSON Data Variable
    json_data = response.json()

    # Create context manager
    with open(f"{data_path}/{json_file_name}", "w") as my_file:
        json.dump(json_data, my_file)

    # Save to database
    load_json_data(game_name.lower(), endpoint_name, json_data, cursor)

    return url

# Create the button
if st.button("Retrieve Data"):
    if json_file_name in listdir(data_path):
        st.info(f"Your data is already saved in {json_file_name}")
    else:
        api_call(url)
        st.success(f"URL {url} data received successfully.")

# Create button for viewing the transformation
if st.button("View Transformation"):
    # Create context manager
    with open(f"{data_path}/{json_file_name}", "r") as my_file:
        result = json.load(my_file)

    result_df = json_transformation(result)

    st.dataframe(result_df)


N = int(input())
data = [list(input()) for _ in range(N)]

result = []

def solve(x, y, N):
    color = data[x][y]
    if N == 1:  # 크기가 1일 경우
        result.append(color)
        return

    for i in range(x, x+N):
        for j in range(y, y+N):
            if color != data[i][j]:
                result.append('(')
                solve(x, y, N//2)
                solve(x, y+N//2, N//2)
                solve(x+N//2, y, N//2)
                solve(x+N//2, y+N//2, N//2)
                result.append(')')
                return
    else:
        result.append(color)

solve(0, 0, N)
print(''.join(result))


# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution(object):
    def sortedListToBST(self, head):
        """
        :type head: ListNode
        :rtype: TreeNode
        """
        if not head:
            return head
        return self.build_tree(head, tail=None)

    def find_middle(self, head, tail):
        """
            Find the middle node in the linked list given head and tail node
        """
        slow = head
        fast = head

        while fast is not tail and fast.next is not tail:
            # slow move one step ahead
            slow = slow.next
            fast = fast.next.next
        if slow:
            return slow
        else:
            return None

    def build_tree(self, head, tail=None):
        # base case
        if head == tail:
            return None

        mid = self.find_middle(head, tail)
        # create a node
        root = TreeNode(mid.val)
        # left side from the mid val
        root.left = self.build_tree(head, mid)
        # right side from the mid value
        root.right = self.build_tree(mid.next, tail)

        return root


from rest_auth.registration.serializers import RegisterSerializer
from rest_framework import serializers, exceptions
from rest_framework.exceptions import ValidationError
from django.contrib.auth import get_user_model, authenticate
from rest_auth.serializers import TokenSerializer, LoginSerializer
from rest_auth.models import TokenModel
from .models import DealerDetail, ClientDetail, CustomUser

class CustomRegisterSerializer(RegisterSerializer):
    username = serializers.CharField(required=True)
    email = serializers.EmailField(required=True)
    password1 = serializers.CharField(write_only=True)
    number = serializers.CharField(required=True)

    user_type = serializers.ChoiceField(
        choices=(
            ('Client', 'Client'),
            ('Dealer', 'Dealer'),

        ),
        style={'base_template': 'radio.html'},
        required=True, write_only=True)

    def get_cleaned_data(self):
        super(CustomRegisterSerializer, self).get_cleaned_data()
        print(self.validated_data.get('number', ''))
        return {
            'password1': self.validated_data.get('password1', ''),
            'email': self.validated_data.get('email', ''),
            'username': self.validated_data.get('username', ''),
            'user_type': self.validated_data.get('user_type', ''),
            'number': self.validated_data.get('number', ''),
        }

class DealerProfileSerializer(serializers.ModelSerializer):
    dealer_name=serializers.CharField(read_only=True)

    class Meta:
        model = DealerDetail
        fields = ('id', 'dealer_name','extra_info', 'thumbnail',
                  'image', 'latitude', 'longitude', 'on_hour','off_hour','has_bike'
                  ,'add_1','add_2','area','city')
        read_only_fields = ('thumbnail', )
        #depth=1

class ClientProfileSerializer(serializers.ModelSerializer):

    class Meta:
        model = ClientDetail
        fields = '__all__'
        read_only_fields = ('thumbnail', )
        # depth=1

class CheckSerializer(serializers.ModelSerializer):
    class Meta:
        model = CustomUser
        fields = ("user_type",)

class TokenSerializer(serializers.ModelSerializer):
    """
    Serializer for Token model.
    """
    user = CheckSerializer(
        many=False, read_only=True)  # this is add by myself.

    class Meta:
        model = TokenModel
        fields = ('key', 'user')


from dpattack.libs.nlpaug import Augmenter
from dpattack.libs.nlpaug import CharAugmenter
from dpattack.libs.nlpaug import Method

class Pipeline(Augmenter, list):
    def __init__(self, action, name='Pipeline', aug_min=1, aug_p=1, flow=None, verbose=0):
        Augmenter.__init__(self, name=name, method=Method.FLOW,
                           action=action, aug_min=aug_min, verbose=verbose)
        self.aug_p = aug_p
        if flow is None:
            list.__init__(self, [])
        elif isinstance(flow, (Augmenter, CharAugmenter)):
            list.__init__(self, [flow])
        elif isinstance(flow, list):
            for subflow in flow:
                if not isinstance(subflow, Augmenter):
                    raise ValueError('At least one of the flow does not belongs to Augmenter')
            list.__init__(self, flow)
        else:
            raise Exception(
                'Expected None, Augmenter or list of Augmenter while {} is passed'.format(
                    type(flow)))

    def draw(self):
        raise NotImplementedError

    def augment(self, data, n=1):
        """
        :param data: Data for augmentation
        :param int n: Number of augmented output
        :return: Augmented data

        >>> augmented_data = flow.augment(data)
        """
        results = [data]
        for _ in range(n*5):
            augmented_data = data[:]
            for aug in self:
                if not self.draw():
                    continue
                augmented_data = aug.augment(augmented_data, n=1)

            # Data format output of each augmenter should be same
            for aug in self:
                if aug.__class__.__bases__[0] == Pipeline:
                    results.append(augmented_data)
                    continue
                if not aug.is_duplicate(results, augmented_data):
                    results.append(augmented_data)
                break

            if len(results) >= n+1:
                break

        # only have input data
        if len(results) == 1:
            if n == 1:
                return results[0]
            else:
                return [results[0]]

        # return 1 record as n == 1
        if n == 1 and len(results) >= 2:
            return results[1]

        # return all records
        return results[1:]


import os
from bs4 import BeautifulSoup
import re

'''Location of crawled files'''
CRAWLED_FILES_LOCATION="BFSFiles"
CORPUS_FOLDER_NAME="corpus"

'''Function for generating the corpus'''
def corpus_generator(case_folding,punctuation):
    path=os.path.join(os.getcwd(),CRAWLED_FILES_LOCATION)
    for f in os.listdir(path):

        filePath=os.path.join(path,f)
        generate_article(filePath,case_folding,punctuation)

''' Function for extracting content form each crawled files, processing them ,and finally calls another functions to create the corpus'''
def generate_article(filePath,case_folding,punctuation):
        fh=open(filePath,"r")
        title=fh.readline()
        title=get_title_from_url(title)
        title=title[:-2]
#        print(title)
        content=fh.read()
        '''Eliminates refences section of the wikipedia page'''

        if(''in content):
            pos=content.index('')
            content=content[:pos]

        ''' Eliminates the See Also section of the wikipedia page, if present'''

        if('' in content):
            pos=content.index('')
            content=content[:pos]

        ''' Eliminates the Notes section of the wikipedia page, if present'''

        if('' in content):
            pos=content.index('')
            content=content[:pos]

        soup=BeautifulSoup(content,"html5lib")
        fh.close()
        content=soup.find("div",attrs={"id":"bodyContent"})

        ''' Eliminates the Content links  section of the wikipedia page, if present'''

        if(content.find("div",attrs={"class":"toc","id":"toc"})):
            content.find("div",attrs={"class":"toc","id":"toc"}).decompose()

        ''' Eliminates the Mathematical formulas of the wikipedia page, if present'''

        if(content.find("annotation",attrs={"encoding":"application/x-tex"})):
            for elem in content.findAll("annotation",attrs={"encoding":"application/x-tex"}):
                elem.decompose()

        ''' Eliminates the "table" tags of the wikipedia page, if present'''

        if(content.find("table")):
            for elem in content.findAll("table"):
                elem.decompose()

        ''' Eliminates the images from  of the wikipedia page, if present'''

        if(content.find("img")):
            for elem in content.findAll("img"):
                elem.decompose()

        content=content.get_text().encode("utf-8").split()

        content_string=""
        for word in content:
            ''' checks for non ASCHII characters or spaces or "-" or "--"   '''
            if(not all(ord(char) < 128 for char in word) or word=="-" or word=="--" or word==" "):
                continue

            '''punctuation are eliminated if required'''

            if(punctuation):
                if(word.isdigit()):
                    p=re.compile('[^\d.,]')
                    word=p.sub("",word)
                else:
                    p=re.compile('[^\w-]')
                    word=p.sub("",word)

            ''' case-folding is done if required'''

            if(case_folding):
                word=word.lower()

            content_string+=word+" "
        ''' Functions call to create the corpus'''
        create_file(title,content_string)

''' creates a file for each page crawled where the main textual content is stored and is stored inside a
    folder called "corpus",which is present in the current directory'''

def create_file(title,content_string):
    file_name=title+".txt"
    file_location=open(os.getcwd()+"\\"+CORPUS_FOLDER_NAME+"\\"+file_name,"a")

    file_location.write(title+"\n")
    for s in content_string.split():
        file_location.write(s+"\n")
    file_location.close()

''' Function to extract the title form url of each page'''

def get_title_from_url(url):
    title=""
    for i in range(len(url)-1,-1,-1):
        if(url[i]=="/"):
            break
        title+=url[i]
    title=title[::-1]
    return title

def main():
    corpus_generator(True,True)

if __name__=="__main__":main()


"""frag_model."""
import numpy as np
import copy
import torch
import pytorch_lightning as pl
import torch.nn as nn
import torch_scatter as ts
import dgl.nn as dgl_nn

import ms_pred.common as common
import ms_pred.dag_pred.dag_data as dag_data
import ms_pred.nn_utils as nn_utils
import ms_pred.magma.fragmentation as fragmentation
import ms_pred.magma.run_magma as run_magma

class IntenGNN(pl.LightningModule):
    def __init__(
        self,
        hidden_size: int,
        gnn_layers: int = 2,
        mlp_layers: int = 0,
        set_layers: int = 2,
        learning_rate: float = 7e-4,
        lr_decay_rate: float = 1.0,
        weight_decay: float = 0,
        dropout: float = 0,
        mpnn_type: str = "PNA",
        pool_op: str = "avg",
        node_feats: int = common.ELEMENT_DIM + common.MAX_H,
        pe_embed_k: int = 0,
        max_broken: int = run_magma.FRAGMENT_ENGINE_PARAMS["max_broken_bonds"],
        frag_set_layers: int = 0,
        loss_fn: str = "cosine",
        root_encode: str = "gnn",
        inject_early: bool = False,
        warmup: int = 1000,
        embed_adduct=False,
        binned_targs: bool = True,
        encode_forms: bool = False,
        add_hs: bool = False,
        **kwargs,
    ):
        super().__init__()
        self.save_hyperparameters()
        self.hidden_size = hidden_size
        self.pe_embed_k = pe_embed_k
        self.root_encode = root_encode
        self.pool_op = pool_op
        self.inject_early = inject_early
        self.embed_adduct = embed_adduct
        self.binned_targs = binned_targs
        self.encode_forms = encode_forms
        self.add_hs = add_hs

        self.tree_processor = dag_data.TreeProcessor(
            root_encode=root_encode, pe_embed_k=pe_embed_k, add_hs=add_hs
        )

        self.formula_in_dim = 0
        if self.encode_forms:
            self.embedder = nn_utils.get_embedder("abs-sines")
            self.formula_dim = common.NORM_VEC.shape[0]

            # Calculate formula dim
            self.formula_in_dim = self.formula_dim * self.embedder.num_dim

            # Account for diffs
            self.formula_in_dim *= 2

        self.gnn_layers = gnn_layers
        self.set_layers = set_layers
        self.frag_set_layers = frag_set_layers
        self.mpnn_type = mpnn_type
        self.mlp_layers = mlp_layers

        self.learning_rate = learning_rate
        self.lr_decay_rate = lr_decay_rate
        self.warmup = warmup

        self.weight_decay = weight_decay
        self.dropout = dropout

        self.max_broken = max_broken + 1
        self.broken_onehot = torch.nn.Parameter(torch.eye(self.max_broken))
        self.broken_onehot.requires_grad = False
        self.broken_clamp = max_broken

        edge_feats = fragmentation.MAX_BONDS

        orig_node_feats = node_feats
        if self.inject_early:
            node_feats = node_feats + self.hidden_size

        adduct_shift = 0
        if self.embed_adduct:
            adduct_types = len(common.ion2onehot_pos)
            onehot = torch.eye(adduct_types)
            self.adduct_embedder = nn.Parameter(onehot.float())
            self.adduct_embedder.requires_grad = False
            adduct_shift = adduct_types

        # Define network
        self.gnn = nn_utils.MoleculeGNN(
            hidden_size=self.hidden_size,
            num_step_message_passing=self.gnn_layers,
            set_transform_layers=self.set_layers,
            mpnn_type=self.mpnn_type,
            gnn_node_feats=node_feats + adduct_shift,
            gnn_edge_feats=edge_feats,
            dropout=self.dropout,
        )

        if self.root_encode == "gnn":
            self.root_module = self.gnn

            # if inject early, need separate root and child GNN's
            if self.inject_early:
                self.root_module = nn_utils.MoleculeGNN(
                    hidden_size=self.hidden_size,
                    num_step_message_passing=self.gnn_layers,
                    set_transform_layers=self.set_layers,
                    mpnn_type=self.mpnn_type,
                    gnn_node_feats=node_feats + adduct_shift,
                    gnn_edge_feats=edge_feats,
                    dropout=self.dropout,
                )
        elif self.root_encode == "fp":
            self.root_module = nn_utils.MLPBlocks(
                input_size=2048,
                hidden_size=self.hidden_size,
                output_size=None,
                dropout=self.dropout,
                num_layers=1,
                use_residuals=True,
            )
        else:
            raise ValueError()

        # MLP layer to take representations from the pooling layer
        # And predict a single scalar value at each of them
        # I.e., Go from size B x 2h -> B x 1
        self.intermediate_out = nn_utils.MLPBlocks(
            input_size=self.hidden_size * 3 + self.max_broken + self.formula_in_dim,
            hidden_size=self.hidden_size,
            output_size=self.hidden_size,
            dropout=self.dropout,
            num_layers=self.mlp_layers,
            use_residuals=True,
        )

        trans_layer = nn_utils.TransformerEncoderLayer(
            self.hidden_size,
            nhead=8,
            batch_first=True,
            norm_first=False,
            dim_feedforward=self.hidden_size * 4,
        )
        self.trans_layers = nn_utils.get_clones(trans_layer, self.frag_set_layers)

        self.loss_fn_name = loss_fn
        if loss_fn == "mse":
            raise NotImplementedError()
        elif loss_fn == "cosine":
            self.loss_fn = self.cos_loss
            self.cos_fn = nn.CosineSimilarity()
            self.output_activations = [nn.Sigmoid()]
        else:
            raise NotImplementedError()

        self.num_outputs = len(self.output_activations)
        self.output_size = run_magma.FRAGMENT_ENGINE_PARAMS["max_broken_bonds"] * 2 + 1
        self.output_map = nn.Linear(
            self.hidden_size, self.num_outputs * self.output_size
        )

        # Define map from output layer to attn
        self.isomer_attn_out = copy.deepcopy(self.output_map)

        # Define buckets
        buckets = torch.DoubleTensor(np.linspace(0, 1500, 15000))
        self.inten_buckets = nn.Parameter(buckets)
        self.inten_buckets.requires_grad = False

        if self.pool_op == "avg":
            self.pool = dgl_nn.AvgPooling()
        elif self.pool_op == "attn":
            self.pool = dgl_nn.GlobalAttentionPooling(nn.Linear(hidden_size, 1))
        else:
            raise NotImplementedError()

        self.sigmoid = nn.Sigmoid()

    def cos_loss(self, pred, targ):
        """cos_loss.

        Args:
            pred:
            targ:
            optionlens:
        """
        if not self.binned_targs:
            raise ValueError()
        loss = 1 - self.cos_fn(pred, targ)
        loss = loss.mean()
        return {"loss": loss}

    def predict(
        self,
        graphs,
        root_reprs,
        ind_maps,
        num_frags,
        max_breaks,
        adducts,
        max_add_hs=None,
        max_remove_hs=None,
        masses=None,
        root_forms=None,
        frag_forms=None,
        binned_out=False,
    ) -> dict:
        """predict.

        Args:
            graphs:
            root_reprs:
            ind_maps:
            num_frags:
            max_breaks:
            adducts
            max_add_hs:
            max_remove_hs:
            masses
            root_forms
            frag_forms
            binned_out

        Returns:
            dict:
        """
        # B x nodes x num outputs x inten items
        out = self.forward(
            graphs,
            root_reprs,
            ind_maps,
            num_frags,
            adducts=adducts,
            broken=max_breaks,
            max_add_hs=max_add_hs,
            max_remove_hs=max_remove_hs,
            masses=masses,
            root_forms=root_forms,
            frag_forms=frag_forms,
        )

        if self.loss_fn_name not in ["mse", "cosine"]:
            raise NotImplementedError()

        output = out["output"][
            :,
            :,
            0,
        ]
        output_binned = out["output_binned"][:, 0, :]
        out_preds_binned = [i.cpu().detach().numpy() for i in output_binned]
        out_preds = [
            pred[:num_frag, :].cpu().detach().numpy()
            for pred, num_frag in zip(output, num_frags)
        ]

        if binned_out:
            out_dict = {
                "spec": out_preds_binned,
            }
        else:
            out_dict = {
                "spec": out_preds,
            }
        return out_dict

    def forward(
        self,
        graphs,
        root_repr,
        ind_maps,
        num_frags,
        broken,
        adducts,
        max_add_hs=None,
        max_remove_hs=None,
        masses=None,
        root_forms=None,
        frag_forms=None,
    ):
        """predict spec from graphs

        Args:
            graphs: DGL Graphs of all mols in batch
            root_reprs
            ind_maps: Mapping to pair graphs to root graphs (allows parallel)
            num_frags: Number of fragments
            broken: num bonds broken
            adducts
            max_add_hs
            max_remove_hs
            masses
            root_forms
            frag_forms
        """
        device = num_frags.device

        # if root fingerprints:
        embed_adducts = self.adduct_embedder[adducts.long()]
        if self.root_encode == "fp":
            root_embeddings = self.root_module(root_repr)
            raise NotImplementedError()
        elif self.root_encode == "gnn":
            with root_repr.local_scope():
                if self.embed_adduct:
                    embed_adducts_expand = embed_adducts.repeat_interleave(
                        root_repr.batch_num_nodes(), 0
                    )
                    ndata = root_repr.ndata["h"]
                    ndata = torch.cat([ndata, embed_adducts_expand], -1)
                    root_repr.ndata["h"] = ndata
                root_embeddings = self.root_module(root_repr)
                root_embeddings = self.pool(root_repr, root_embeddings)
        else:
            pass

        # Line up the features to be parallel between fragment avgs and root
        # graphs
        ext_root = root_embeddings[ind_maps]
        # Extend the root further to cover each individual atom
        ext_root_atoms = torch.repeat_interleave(
            ext_root, graphs.batch_num_nodes(), dim=0
        )
        concat_list = [graphs.ndata["h"]]

        if self.inject_early:
            concat_list.append(ext_root_atoms)

        if self.embed_adduct:
            adducts_mapped = embed_adducts[ind_maps]
            adducts_exp = torch.repeat_interleave(
                adducts_mapped, graphs.batch_num_nodes(), dim=0
            )
            concat_list.append(adducts_exp)

        with graphs.local_scope():
            graphs.ndata["h"] = torch.cat(concat_list, -1).float()

            frag_embeddings = self.gnn(graphs)

            # Average embed the full root molecules and fragments
            avg_frags = self.pool(graphs, frag_embeddings)

        # expand broken and map it to each fragment
        broken_arange = torch.arange(broken.shape[-1]).to(device)
        broken_mask = broken_arange[None, :] < num_frags[:, None]

        broken = torch.clamp(broken[broken_mask], max=self.broken_clamp)
        broken_onehots = self.broken_onehot[broken.long()]

        ### Build hidden with forms
        mlp_cat_list = [ext_root, ext_root - avg_frags, avg_frags, broken_onehots]

        hidden = torch.cat(mlp_cat_list, dim=1)

        # Pack so we can use interpeak attn
        padded_hidden = nn_utils.pad_packed_tensor(hidden, num_frags, 0)

        if self.encode_forms:
            diffs = root_forms[:, None, :] - frag_forms
            form_encodings = self.embedder(frag_forms)
            diff_encodings = self.embedder(diffs)
            new_hidden = torch.cat(
                [padded_hidden, form_encodings, diff_encodings], dim=-1
            )
            padded_hidden = new_hidden

        padded_hidden = self.intermediate_out(padded_hidden)
        batch_size, max_frags, hidden_dim = padded_hidden.shape

        # Build up a mask
        arange_frags = torch.arange(padded_hidden.shape[1]).to(device)
        attn_mask = ~(arange_frags[None, :] < num_frags[:, None])

        hidden = padded_hidden
        for trans_layer in self.trans_layers:
            hidden, _ = trans_layer(hidden, src_key_padding_mask=attn_mask)

        # hidden: B x L x h
        # attn_mask: B x L

        # Build mask
        max_inten_shift = (self.output_size - 1) / 2
        max_break_ar = torch.arange(self.output_size, device=device)[None, None, :].to(
            device
        )
        max_breaks_ub = max_add_hs + max_inten_shift
        max_breaks_lb = -max_remove_hs + max_inten_shift

        ub_mask = max_break_ar <= max_breaks_ub[:, :, None]
        lb_mask = max_break_ar >= max_breaks_lb[:, :, None]

        # B x Length x Mass shifts
        valid_pos = torch.logical_and(ub_mask, lb_mask)

        # B x Length x outputs x Mass shift
        valid_pos = valid_pos[:, :, None, :].expand(
            batch_size, max_frags, self.num_outputs, self.output_size
        )
        # B x L x Output
        output = self.output_map(hidden)
        attn_weights = self.isomer_attn_out(hidden)

        # B x L x Out x Mass shifts
        output = output.reshape(batch_size, max_frags, self.num_outputs, -1)
        attn_weights = attn_weights.reshape(batch_size, max_frags, self.num_outputs, -1)

        # Mask attn weights
        # attn_weights.masked_fill_(~valid_pos, -float("inf"))
        attn_weights.masked_fill_(~valid_pos, -99999)  # -float("inf"))

        # B x Out x L x Mass shifts
        output = output.transpose(1, 2)
        attn_weights = attn_weights.transpose(1, 2)
        valid_pos_binned = valid_pos.transpose(1, 2)

        # Calc inverse indices => B x Out x L x shift
        inverse_indices = torch.bucketize(masses, self.inten_buckets, right=False)
        inverse_indices = inverse_indices[:, None, :, :].expand(attn_weights.shape)

        # B x Out x (L * Mass shifts)
        attn_weights = attn_weights.reshape(batch_size, self.num_outputs, -1)
        output = output.reshape(batch_size, self.num_outputs, -1)
        inverse_indices = inverse_indices.reshape(batch_size, self.num_outputs, -1)
        valid_pos_binned = valid_pos.reshape(batch_size, self.num_outputs, -1)

        # B x Outs x ( L * mass shifts )
        pool_weights = ts.scatter_softmax(attn_weights, index=inverse_indices, dim=-1)
        weighted_out = pool_weights * output

        # B x Outs x (UNIQUE(L * mass shifts))
        output_binned = ts.scatter_add(
            weighted_out,
            index=inverse_indices,
            dim=-1,
            dim_size=self.inten_buckets.shape[-1],
        )

        output = output.reshape(batch_size, max_frags, self.num_outputs, -1)
        pool_weights_reshaped = pool_weights.reshape(
            batch_size, max_frags, self.num_outputs, -1
        )
        inverse_indices_reshaped = inverse_indices.reshape(
            batch_size, max_frags, self.num_outputs, -1
        )

        # B x Outs x binned
        valid_pos_binned = ts.scatter_max(
            (valid_pos_binned).long(),
            index=inverse_indices,
            dim_size=self.inten_buckets.shape[-1],
            dim=-1,
        )[0].bool()

        # Activate each dim with its respective output activation
        # Helpful for hurdle or probabilistic models
        new_outputs_binned = []
        for output_ind, act in enumerate(self.output_activations):
            new_outputs_binned.append(
                act(output_binned[:, output_ind : output_ind + 1, :])
            )
        output_binned = torch.cat(new_outputs_binned, -2)
        output_binned.masked_fill_(~valid_pos_binned, 0)

        # Index into output binned using inverse_indices_reshaped
        # Revert the binned output back to frags for attribution
        # B x Out x L x Mass shifts
        inverse_indices_reshaped_temp = inverse_indices_reshaped.transpose(
            1, 2
        ).reshape(batch_size, self.num_outputs, -1)
        output_unbinned = torch.take_along_dim(
            output_binned, inverse_indices_reshaped_temp, dim=-1
        )
        output_unbinned = output_unbinned.reshape(
            batch_size, self.num_outputs, max_frags, -1
        ).transpose(1, 2)
        output_unbinned_alpha = output_unbinned * pool_weights_reshaped

        return {"output_binned": output_binned, "output": output_unbinned_alpha}

    def _common_step(self, batch, name="train"):
        pred_obj = self.forward(
            batch["frag_graphs"],
            batch["root_reprs"],
            batch["inds"],
            batch["num_frags"],
            broken=batch["broken_bonds"],
            adducts=batch["adducts"],
            max_remove_hs=batch["max_remove_hs"],
            max_add_hs=batch["max_add_hs"],
            masses=batch["masses"],
            root_forms=batch["root_form_vecs"],
            frag_forms=batch["frag_form_vecs"],
        )
        pred_inten = pred_obj["output_binned"]
        pred_inten = pred_inten[:, 0, :]
        batch_size = len(batch["names"])

        loss = self.loss_fn(pred_inten, batch["inten_targs"])
        self.log(
            f"{name}_loss", loss["loss"].item(), batch_size=batch_size, on_epoch=True
        )

        for k, v in loss.items():
            if k != "loss":
                self.log(f"{name}_aux_{k}", v.item(), batch_size=batch_size)
        return loss

    def training_step(self, batch, batch_idx):
        """training_step."""
        return self._common_step(batch, name="train")

    def validation_step(self, batch, batch_idx):
        """validation_step."""
        return self._common_step(batch, name="val")

    def test_step(self, batch, batch_idx):
        """test_step."""
        return self._common_step(batch, name="test")

    def configure_optimizers(self):
        """configure_optimizers."""
        optimizer = torch.optim.Adam(
            self.parameters(), lr=self.learning_rate, weight_decay=self.weight_decay
        )
        scheduler = nn_utils.build_lr_scheduler(
            optimizer=optimizer, lr_decay_rate=self.lr_decay_rate, warmup=self.warmup
        )
        ret = {
            "optimizer": optimizer,
            "lr_scheduler": {
                "scheduler": scheduler,
                "frequency": 1,
                "interval": "step",
            },
        }
        return ret


import numpy as np
from PIL import Image

def read_pixel_data(png_path):
    im = Image.open(png_path)
    pixels = np.array(im.getdata())
    width, height = im.size
    pixels = pixels.reshape(height, width, 4)
    return pixels


import functools
from os import stat
import pytest

import numpy as np
import torch

from disaster_tweets.model import model

@pytest.fixture()
def data_iterable():
    def inner_gen():
        while True:
            yield labels, sentences, offsets

    labels = torch.tensor(np.ones(32))
    sentences = torch.tensor(np.random.randint(low=0, high=128, size=32 * 3))
    offsets = torch.tensor(range(0, 32 * 3, 3))
    return inner_gen()

class TestLinearModel:
    @staticmethod
    def test_given_data_output_shape_is_correct(data_iterable):
        vocab = list(range(128))
        model_instance = model.LinearModel(
            embedding_size=32,
            vocab=vocab,
        )
        with torch.no_grad():
            _, sentence, offset = next(data_iterable)
            outputs = model_instance(sentence, offset)
            len(outputs) == 32

class TestCNNModel:
    @staticmethod
    def get_activation(name, activation):
        def hook(model, input, output):
            activation[name] = output.detach()

        return hook

    def test_given_input_output_shape_is_correct(self):
        batch_size = 8
        length = 16
        embedding_size = 32

        vocab = list(range(128))

        input = torch.randint(0, 128, size=(length, batch_size))

        model_instance = model.CNNModel(embedding_size, vocab)

        activations = {}

        model_instance.conv_1d_16.register_forward_hook(
            self.get_activation("conv_1d_16", activation=activations)
        )
        model_instance.conv_1d_8.register_forward_hook(
            self.get_activation("conv_1d_8", activation=activations)
        )
        model_instance.conv_1d_4.register_forward_hook(
            self.get_activation("conv_1d_4", activation=activations)
        )

        model_instance.max_pool_1.register_forward_hook(
            self.get_activation("max_pool_1", activation=activations)
        )
        model_instance.max_pool_2.register_forward_hook(
            self.get_activation("max_pool_2", activation=activations)
        )
        model_instance.max_pool_3.register_forward_hook(
            self.get_activation("max_pool_3", activation=activations)
        )
        output = model_instance(input)

        assert activations["conv_1d_16"].shape == (batch_size, 18, 1)
        assert activations["conv_1d_8"].shape == (batch_size, 9, 9)
        assert activations["conv_1d_4"].shape == (batch_size, 5, 13)

        assert activations["max_pool_1"].shape == (batch_size, 5, 1)
        assert activations["max_pool_2"].shape == (batch_size, 9, 1)
        assert activations["max_pool_3"].shape == (batch_size, 18, 1)

        assert output.shape == (8, 1)

class TestLSTMModel:
    @staticmethod
    def test_given_data_output_shape_is_correct():
        batch_size = 8
        length = 16
        embedding_size = 32

        vocab = list(range(128))
        model_instance = model.LSTMModel(
            embedding_size=embedding_size,
            vocab=vocab,
        )
        input = torch.randint(0, 128, size=(length, batch_size))
        with torch.no_grad():
            outputs = model_instance(input)
            assert outputs.shape == (8, 1)


'''
启动websocket，发送和接收数据
redmorningcn 2020.10.19
'''
from   websocket_server import WebsocketServer
import threading
import json
from   datetime import datetime
import threading

def startServer():
    #t = threading.Timer(60, Server.run)
    t = threading.Thread(target = Server.run)
    t.start()

def server_send(text):
    #print("Server.send_message",Server.serverflg,text)
    if Server.serverflg == 2:
        Server.send_message(text)

def server_recv():
    if Server.messagelen > 0:
        #print(Server.messagelen)
        Server.messagelen = 0
        return Server.messagetxt
    return 0

class Server:
    serverflg  = 1
    messagetxt = ''
    messagelen = 0

    @classmethod
    def run(cls):
        PORT = 1234
        server = WebsocketServer(PORT, host='0.0.0.0')
        cls.myserver = server

        server.set_fn_new_client(cls.new_client)
        server.set_fn_client_left(cls.client_left)
        server.set_fn_message_received(cls.message_received)
        print("Web socket server has started at %s." % datetime.now())
        #showTimer()
        server.run_forever()

    @classmethod
    def getEvent(cls, data):
        try:
            jsonDic = json.loads(data)
            dic = dict(jsonDic)
            event = dic["event"]
        except Exception as error:
            print("Get event error:%s" % error)
            return None, {}
        return event, dic

    @classmethod
    def new_client(cls, client, server):
        print("A new client connected and was given id %d at %s." % (client['id'], datetime.now()))
        server.send_message_to_all("Hey all, a new client has joined us at %s." % datetime.now())
        Server.serverflg = 2    #连接正常
    @classmethod
    def send_message(cls,text):
        #print("A new client connected and was given id %d at %s." % (client['id'], datetime.now()))
        cls.myserver.send_message_to_all(text)

    @classmethod
    def client_left(cls, client, server):
        Server.serverflg   = 1  #连接断开
        print("Client(%d) disconnected at %s." % (client['id'], datetime.now()))

    @classmethod
    def message_received(cls, client, server, message):
        event, dic = cls.getEvent(message)
        #print(event, dic)
        #if event == 1:
        #    print(dic["tree"])

        Server.messagetxt = message
        Server.messagelen = len(message)
        #print(Server.messagelen,Server.messagetxt)
        #print("Client(%d) said: %s at %s" % (client['id'], message, datetime.now()))

import   time
if __name__=='__main__':
    startServer()      #启动服务

    while True:
        time.sleep(1)

        txt =  server_recv()
        if txt:
            print(txt)


from parameter_learner import *
import tensorflow as tf
import numpy as np

slim = tf.contrib.slim

def get_random_dynamic_param(dynamic_parameters_shape_map, batch_size):
  dynamic_parameters_map = dict()
  for name, shape in dynamic_parameters_shape_map.iteritems():
    dynamic_parameters_map[name] = tf.random_normal([batch_size] + shape)
  return dynamic_parameters_map

def weight_hashing_parameter_learner_net():
  num_classes = 1
  k_shot = 5

  meta_batch_size = 1
  height = 1
  width = 1
  depth_in = 5

  params = dict(is_training = True,
    after_hashing_scale_hyperparameters={},
    parameter_prediction_convline=None,
    fc_hyperparameters ={},
    decompression_factor=num_classes*k_shot)

  inputs = tf.random_normal([
      meta_batch_size, num_classes*k_shot,
      height, width, depth_in])
  dynamic_params_shape_map = dict(w=[3,3,5,50], b=[50])
  param_learner = WeightHashingParameterLearner(**params)
  net = param_learner.learn(inputs, dynamic_params_shape_map)
  return net

from IPython import embed
if __name__ == '__main__':
  net = weight_hashing_parameter_learner_net()
  init_op = tf.global_variables_initializer()
  with tf.Session() as sess:
    sess.run(init_op)
    out = sess.run(net)
    embed()


import pandas
import os
from urllib.request import urlopen, Request
import time
from tld import get_tld, get_fld

if not os.path.exists("domain_html"):
	os.mkdir("domain_html")

df = pandas.read_csv("domain.csv")

#print(df)

for link in df['domain_name']:
	# print("original" + link)
	# print("fld version:", get_fld("https://" + link))

	f = open('domain_html/' + link, "wb")
	try:
		print("Downloading (1st):", link)
		headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}
		fld_link = get_fld("https://" + link)
		request_link = "https://" + fld_link
		req = Request(url = request_link, headers = headers)
		response = urlopen(req)
		html = response.read()
		f.write(html)
		f.close()
	except:
		print("Downloading (2nd):", link)
		headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}
		fld_link = get_fld("https://" + link)
		request_link = "https://" + fld_link
		req = Request(url = request_link, headers = headers)
		response = urlopen(req)
		html = response.read()
		f.write(html)
		f.close()
	time.sleep(10)


#! /bin/python
#-*- coding:utf-8 -*-

import sys
import numpy

#reads libsvm format and returns pairs (LABEL,FEATVECT)
def read_libsvm(stream):
    for line in stream:
        line = line.strip()
        if line != '':
            entries = line.split(' ')
            label = int(entries[0])
            featvec = [int(feat[:-2]) for feat in entries [1:]]
            yield label,featvec
 #           for feat in entries[1:]:
 #               index,_,value = feat.partition(':')
 #               featvec.append(int(index))

class Perceptron(object):
    def __init__(self):
        self.weightvector = None
        self.averages = None
        self.numlabels = 0
        self.numfeatures = 0

    def save (self,filename):
        with open(filename,mode='w') as fout:
            print >> fout,self.numlabels
            print >> fout,self.numfeatures
            for l in range(self.numlabels):
                print >> fout, ' '.join(map(str,self.weightvector[l]))

    def load (self,filename):
        with open (filename, mode = 'r') as fin:
            self.numlables = int(fin.readline())
            self.numfeatures = int (fin.readline())
            for line in fin:
                line = line.strip()
                if line != '':
                    self.weightvector.append(map(float,line.split(' ')))

    def train (self,data,epochs):
        #determine numlabels and numfeatures
        for label,featvec in data:
            if label >= self.numlabels:
                self.numlabels = label+1
            for feat in featvec:
                if feat >= self.numfeatures:
                    self.numfeatures = feat+1

        #initialize the weight vector
        #self.weightvector = [0.0]*self.numlables*numfeatures     line version
        #matrix version
        #for _ in range(self.numlables):
         #   self.weightvector.append([0.0]*self.numlables*numfeatures)
        self.weightvector = numpy.zeros((self.numlabels,self.numfeatures))
        self.averages = numpy.zeros((self.numlabels,self.numfeatures))

        #train weight vector
        q = 0
        for e in range (epochs):
            for goldlabel,featvec in data:
                q +=1
                predlabel = self.predict(featvec)
                if goldlabel != predlabel:
                    self.update(goldlabel, predlabel,featvec,q)
            print >> sys.stderr,'epoch %d done' % e

        #average the weights
        self.weightvector -= self.averages/q
    #return predicted label

    def predict (self,featvec):
        #highest_score = None
        #best_label = None
        #for l in range(self.numlabels):
        #    score = 0.0
         #   for f in featvec:
        #       score += self.weightvector[l][f]
         #   if score > highest_score:
         #      highest_score = score
        #       best_label = l
        #return best_label

        return self.weightvector[:,featvec].sum(1).argmax()

    def update (self,goldlabel,predlabel,featvec,q):
        #for f in featvec:
        #    self.weightvector[goldlabel][f] +=0.5
        #    self.weightvector[goldlabel][f] +=0.5

        self.weightvector[goldlabel,featvec] +=0.5
        self.weightvector[predlabel,featvec] -=0.5
        self.averages[goldlabel,featvec] += q*0.5
        self.averages[predlabel,featvec] -= q*0.5
if __name__ == '__main__':
    import argparse

    argpar = argparse.ArgumentParser(description='train and apply a Perceptron')
    mode = argpar.add_mutually_exclusive_group(required=True)
    mode.add_argument('-t','--train',dest='train',action='store_true',help='run in training mode')
    mode.add_argument('-p','--predict',dest='predict',action='store_true',help='run in test mode')
    argpar.add_argument('-i','--input',dest='inputfile',required=True,help='the file containing training/test data')
    argpar.add_argument('-m','--model',dest='modelfile',required=True,help='the file containing the learned weight vector')
    argpar.add_argument('-o','--output',dest='outputfile',default = 'output.libsvm',help='output the file to write the predictions to')
    argpar.add_argument('-e','--epochs',dest='numepochs',default = 10,help='numbers of epochs to run the perceptron')

    args = argpar.parse_args()

    perceptron = Perceptron()
    if args.train:
        with open(args.inputfile) as fin:
            data = [datapoint for datapoint in read_libsvm(fin)]
            perceptron.train(data,int(args.numepochs))
            perceptron.save(args.modelfile)
    elif args.predict:
        perceptron.load(args.model(file))
        with open(args.inputfile) as fin:
            with open(args.outputfile,mode='w') as fout:
                data = [datapoint for datapoint in read_libsvm(fin)]
                predictions = [perceptron.predict(featvec) for (_,featvec)in data]
                print >> fout, '\n'.join(map(str,predictions))


from model import *
import json
import re
import os

def load(f):
    data = json.loads(re.compile(r'^var\s+\w+\s*=(.*);\s*$', re.S).sub(r'\1', f.read()))
    model = Model()
    model.vertices = [Vertex(*v) for v in data['vertices']]
    if 'normals' in data:
        for v, n in zip(model.vertices, data['normals']):
            v.normal = Vector3(*n)
    if 'coords' in data:
        for v, c in zip(model.vertices, data['coords']):
            v.coord = Vector2(*c)
    model.triangles = [Triangle(*t) for t in data['triangles']]
    return model

def dump(model, f):
    data = {
        'vertices': [[v.pos.x, v.pos.y, v.pos.z] for v in model.vertices],
        'coords': [[v.coord.x, v.coord.y] for v in model.vertices],
        'normals': [[v.normal.x, v.normal.y, v.normal.z] for v in model.vertices],
        'triangles': [[t.a, t.b, t.c] for t in model.triangles],
    }

    # only write out coords and normals if they are non-zero
    if not any(any(c) for c in data['coords']):
        del data['coords']
    if not any(any(n) for n in data['normals']):
        del data['normals']

    # we could just use json.dump(data, f) but that bloats the file format with a lot of numbers like 1.00000000000000001
    f.write('var %s = {\n' % re.sub(r'\..+$', '', os.path.basename(f.name)))
    for name in ['vertices', 'coords', 'normals', 'triangles']:
        if name not in data: continue
        elements = ','.join('[%s]' % ','.join(re.sub(r'\.?0+$', '', '%f' % y) for y in x) for x in data[name])
        f.write('    "%s": [%s]%s\n' % (name, elements, '' if name == 'triangles' else ','))
    f.write('};\n')


# 100/100
# Could be refactored a bit more - e.g. with if/else clauses

parking = {}

counter = int(input())

for _ in range(counter):
    tokens = input().split()
    command = tokens[0]
    username = tokens[1]

    if command == 'register':
        plate = tokens[2]

        if username not in parking.keys():
            parking[username] = plate
            print(f'{username} registered {plate} successfully')
        else:
            print(f'ERROR: already registered with plate number {plate}')
    else:
        if username not in parking.keys():
            print(f'ERROR: user {username} not found')
        else:
            parking.pop(username)
            print(f'{username} unregistered successfully')

[print(f'{name} => {plate}') for (name, plate) in parking.items()]


#!/usr/bin/env python3

from Bio import SeqIO
from sys import stderr
import csv
import re
from crc64iso.crc64iso import crc64

info_file = str(snakemake.input['info'])
a2m_file = str(snakemake.input['a2m'])
pos_file = str(snakemake.input['pos'])
usearch_file = str(snakemake.input['usearch'])
blasso_file  = str(snakemake.input['blasso'])
out_file = str(snakemake.output)

infos = dict()
data_type = 'JGI'
header = []
id_cols = { 'OM-RGC_ID', 'genome_id', 'gene_oid', 'ID' }
with open(info_file) as fh:
    tsv = csv.reader(fh, delimiter = '\t')
    header = next(tsv)
    header = [v + str(header[:i].count(v) + 1) if header.count(v) > 1 else v for i, v in enumerate(header)]
    id_cols = set(header) & id_cols
    assert len(id_cols) > 0, "Id column not found"
    id_col = id_cols.pop()
    has_sequence = 'sequence' in header
    for line in tsv:
        row = dict(zip(header, line))
        record_id = row.pop(id_col).replace(' ', '_')
        if has_sequence:
            del row['sequence']
        infos[record_id] = row
    header.remove(id_col)
    if has_sequence:
        header.remove('sequence')

positions = []
groups = []
with open(pos_file) as fh:
    for line in fh:
        pos, group = line.rstrip().split()
        pos0 = int(pos) - 1
        positions.append((pos0, group))
        if len(group) > 1 and group not in groups:
            groups.append(group)

blassos = dict()
with open(blasso_file) as fh:
    for line in fh:
        record_id, wl_mean, wl_sd = line.split()
        blassos[record_id] = { 'wl_mean': wl_mean, 'wl_sd': wl_sd }

matches = dict()
with open(usearch_file) as fh:
    for line in fh:
        query, target, ident = line.split('\t')
        if query not in matches:
            target_id, *rest = target.split()
            search = re.search('{(.+?)}', target)
            assert search, "Unexpected reference header format: %s" % target
            family = search.group(1)
            matches[query] = { 'target': target, 'family': family, 'ident': float(ident) }

aminoacids = [ 'A','C','D','E','F','G','H','I','K','L','M','N','P','Q','R','S','T','V','W','Y' ]
with open(out_file, 'w') as out_fh:
    tsv = csv.writer(out_fh, delimiter = "\t")
    tsv.writerow([ 'record_id', 'checksum', 'target', 'family', 'ident', 'wl_mean', 'wl_sd' ] + groups + header)
    with open(a2m_file) as a2m_fh:
        a2m = SeqIO.parse(a2m_fh, 'fasta')
        ref = next(a2m)
        ref_pos = 0
        aln_pos = []
        for pos in range(len(ref.seq)):
            if ref.seq[pos] not in [ '-', '.' ]:
                for pos0, group in positions:
                    if ref_pos == pos0:
                        aln_pos.append((pos, group))
                ref_pos += 1
        for rec in a2m:
            if rec.id in matches:
                seq = re.sub('[^A-Z]', '', str(rec.seq))
                cks = crc64(seq)
                match  = matches[rec.id]
                blasso = blassos[rec.id]
                info_id, *rest = rec.id.split('/')
                info   = infos[info_id]
                res = { group: [] for group in groups }
                for pos, group in aln_pos:
                    if group in groups:
                        res[group].append(rec.seq[pos])
                    elif rec.seq[pos] != group:
                        break
                else:
                    row = [ rec.id, cks, match['target'], match['family'], match['ident'], blasso['wl_mean'], blasso['wl_sd'] ]
                    row += [ ''.join(res[x]) for x in groups ]
                    row += info.values()
                    tsv.writerow(row)


from dataStructures import *

def insert_linked_list(first_ll, second_ll, index):
    temp_ll=LinkedList()
    list1pointer=first_ll.get_head()
    list2pointer=second_ll.get_head()
    counter=0
    while list1pointer!=None :
          if counter==index:
             while list2pointer!=None:
                temp_ll.append(list2pointer.get_data())
                list2pointer=list2pointer.get_next()
          else:
           temp_ll.append(list1pointer.get_data())
           list1pointer=list1pointer.get_next()

          counter+=1

    return temp_ll

firstlist = LinkedList()
secondlist = LinkedList()
################
firstlist.add(4)
firstlist.add(3)
firstlist.add(2)
firstlist.add(7)
##################
secondlist.add(2)
secondlist.add(1)
##################

print("First")
firstlist.printList()
print("Second")
secondlist.printList()
print("_________")
firstlist = insert_linked_list(firstlist, secondlist,2)
firstlist.printList() # should print 7 2 1 2 3 4