import logging
import struct

import numpy as np
from pkg_resources import resource_filename

from .memory import Memory
from .network import Mac, IpAddress
from .gbe import Gbe

LOGGER = logging.getLogger(__name__)

STRUCT_CTYPES = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'}
STRUCT_CTYPES_TO_B = {'B': 1, 'H': 2, 'L': 4, 'Q': 8}

TENGBE_UNIFIED_MMAP_TXT = resource_filename('casperfpga', 'tengbe_mmap.txt')
TENGBE_MMAP_LEGACY_TXT  = resource_filename('casperfpga', 'tengbe_mmap_legacy.txt')

# Offsets for fields in the memory map, in bytes
OFFSET_CORE_TYPE   = 0x0
OFFSET_BUFFER_SIZE = 0x4
OFFSET_WORD_LEN    = 0x8
OFFSET_MAC_ADDR    = 0xc
OFFSET_IP_ADDR     = 0x14
OFFSET_GW_ADDR     = 0x18
OFFSET_NETMASK     = 0x1c
OFFSET_MC_IP       = 0x20
OFFSET_MC_MASK     = 0x24
OFFSET_BUF_VLD     = 0x28
OFFSET_FLAGS       = 0x2c
OFFSET_PORT        = 0x30
OFFSET_STATUS      = 0x34
OFFSET_CONTROL     = 0x40
OFFSET_ARP_SIZE    = 0x44
OFFSET_TX_PKT_RATE = 0x48
OFFSET_TX_PKT_CNT  = 0x4c
OFFSET_TX_VLD_RATE = 0x50
OFFSET_TX_VLD_CNT  = 0x54
OFFSET_TX_OF_CNT   = 0x58
OFFSET_TX_AF_CNT   = 0x5c
OFFSET_RX_PKT_RATE = 0x60
OFFSET_RX_PKT_CNT  = 0x64
OFFSET_RX_VLD_RATE = 0x68
OFFSET_RX_VLD_CNT  = 0x6c
OFFSET_RX_OF_CNT   = 0x70
OFFSET_RX_AF_CNT   = 0x74
OFFSET_COUNT_RST   = 0x78

OFFSET_ARP_CACHE   = 0x1000
OFFSET_TX_BUFFER   = 0x4000
OFFSET_RX_BUFFER   = 0x8000

# Sizes for fields in the memory map, in bytes
SIZE_CORE_TYPE   = 0x4
SIZE_BUFFER_SIZE = 0x4
SIZE_WORD_LEN    = 0x4
SIZE_MAC_ADDR    = 0x8
SIZE_IP_ADDR     = 0x4
SIZE_GW_ADDR     = 0x4
SIZE_NETMASK     = 0x4
SIZE_MC_IP       = 0x4
SIZE_MC_MASK     = 0x4
SIZE_BUF_AVAIL   = 0x4
SIZE_FLAGS       = 0x4
SIZE_PORT        = 0x4
SIZE_STATUS      = 0x8
SIZE_CONTROL     = 0x8
SIZE_ARP_SIZE    = 0x4
SIZE_TX_PKT_RATE = 0x4
SIZE_TX_PKT_CNT  = 0x4
SIZE_TX_VLD_RATE = 0x4
SIZE_TX_VLD_CNT  = 0x4
SIZE_TX_OF_CNT   = 0x4
SIZE_TX_AF_CNT   = 0x4
SIZE_RX_PKT_RATE = 0x4
SIZE_RX_PKT_CNT  = 0x4
SIZE_RX_VLD_RATE = 0x4
SIZE_RX_VLD_CNT  = 0x4
SIZE_RX_OF_CNT   = 0x4
SIZE_RX_AF_CNT   = 0x4
SIZE_COUNT_RST   = 0x4

SIZE_ARP_CACHE   = 0x3000
SIZE_TX_BUFFER   = 0x4000
SIZE_RX_BUFFER   = 0x4000

def read_memory_map_definition(filename):
    """ Read memory map definition from text file.

    Returns a python dictionary:
        {REGISTER_NAME1: {'offset': offset, 'size': size, 'rwflag': rwflag},
         REGISTER_NAME2: {'offset': offset, 'size': size, 'rwflag': rwflag}
         ...}

    Notes:
        Used by TenGbe.configure_core() to write to mmap.
    """
    mmap_arr = np.genfromtxt(filename, dtype='str', skip_header=1)
    mmap_keys    = list(mmap_arr[:, 0])
    mmap_offsets = [int(x, 0) for x in mmap_arr[:, 1]]
    mmap_size    = [int(x, 0) for x in mmap_arr[:, 2]]
    mmap_rw      = list(mmap_arr[:, 3])
    mmap  = {}
    for ii, k in enumerate(mmap_keys):
        mmap[k] = {'offset': mmap_offsets[ii], 'size': mmap_size[ii], 'rwflag': mmap_rw[ii]}
    return mmap

class TenGbe(Memory, Gbe):
    """
    To do with the CASPER ten GBE yellow block implemented on FPGAs,
    and interfaced-to via KATCP memory reads/writes.
    """
    def __init__(self, parent, name, address, length_bytes, device_info=None):
        """

        :param parent: Parent object who owns this TenGbe instance
        :param name: Unique name of the instance
        :param address:
        :param length_bytes:
        :param device_info: Information about this device
        """
        Memory.__init__(self, name, 32, address, length_bytes)
        Gbe.__init__(self, parent, name, address, length_bytes, device_info)
        self.memmap_compliant = self._check_memmap_compliance()

    @property
    def mac(self):
        return self.get_gbe_core_details()['mac']

    @property
    def ip_address(self):
        return self.get_gbe_core_details()['ip']

    @property
    def port(self):
        return self.get_gbe_core_details()['fabric_port']

    def _check_memmap_compliance(self):
        """
        Look at the first word of the core's memory map and try to
        figure out if it compliant with the harmonized ethernet map.
        This isn't flawless, but unless the user sets a very weird
        MAC address for their core (which is what the old core's map
        stored in register 0, it should be OK).
        """
        x = self.parent.read(self.name, 4)
        cpu_tx_en, cpu_rx_en, rev, core_type = struct.unpack('4B', x)
        if (cpu_tx_en > 1) or (cpu_rx_en > 1) or ((core_type != 4) and (core_type != 2)):
            return False
        else:
            return True

    def post_create_update(self, raw_device_info):
        """
        Update the device with information not available at creation.

        :param raw_device_info: info about this block that may be useful
        """
        super(TenGbe, self).post_create_update(raw_device_info)
        self.snaps = {'tx': None, 'rx': None}
        for snapshot in self.parent.snapshots:
            if snapshot.name.find(self.name + '_') == 0:
                name = snapshot.name.replace(self.name + '_', '')
                if name == 'txs_ss':
                    self.snaps['tx'] = snapshot.name
                elif name == 'rxs_ss':
                    self.snaps['rx'] = snapshot.name
                else:
                    errmsg = '%s: incorrect snap %s under tengbe ' \
                             'block' % (self.fullname, snapshot.name)
                    LOGGER.error(errmsg)
                    raise RuntimeError(errmsg)

    def read_txsnap(self):
        """
        Read the TX snapshot embedded in this TenGBE yellow block
        """
        return self.snaps['tx'].read(timeout=10)['data']

    def read_rxsnap(self):
        """
        Read the RX snapshot embedded in this TenGBE yellow block
        """
        return self.snaps['rx'].read(timeout=10)['data']

    def _memmap_write(self, register, value):
        """ Write to memory map
        :param register: register to write to. Register must be in memmap.
        :param value: Value to write.
        """
        offset   = self.memmap[register]['offset']
        bytesize = self.memmap[register]['size']
        rw_addr  = offset - offset % 4               # Round in case of non 32-bit writes
        ctype    = STRUCT_CTYPES.get(bytesize, 'L')  # Treat as 32-bit if longer than 8B

        if self.memmap[register]['rwflag'] == 'r':
            raise RuntimeError("Warning: %s is read-only!" % register)

        if bytesize in (1, 2):
            n_elem = int(4 / bytesize)
            pcode  = '>%i%s' % (n_elem, ctype)
            current_value  = self.parent.read(self.name, size=4, offset=rw_addr)
            new_arr = list(struct.unpack(pcode, current_value))[::-1]
            new_arr[offset % n_elem] = value
            new_arr = new_arr[::-1]
            packed = struct.pack(pcode, *new_arr)

        elif bytesize in (4, 8):
            if isinstance(value, str):
                packed = value
            else:
                packed = struct.pack('>%s' % ctype, value)
        else:
            raise RuntimeError("Can only write 1,2,4,8 Byte registers with this function.")
        self.parent.blindwrite(self.name, packed, offset=rw_addr)

    def _memmap_read(self, register):
        """ Read from memory map

        :param register: register to read from. Must be in memmap.
        """
        offset   = self.memmap[register]['offset']
        bytesize = self.memmap[register]['size']
        ctype    = STRUCT_CTYPES.get(bytesize)  # Treat as 32-bit if longer than 8B

        if bytesize in (4, 8):
            value = self.parent.read(self.name, size=bytesize, offset=offset)
            value = struct.unpack('>%s' % ctype, value)[0]
        elif bytesize in (1, 2):
            if bytesize == 2 and offset % 4 not in (0, 2):
                raise RuntimeError("Attempted to read 16-bits from 32-bit word with %iB offset. "
                                   "Not supported." % (offset%4))
            read_addr = offset - offset % 4
            value = self.parent.read(self.name, size=4, offset=read_addr)
            valuearr = struct.unpack('>%i%s' % (int(4 / bytesize), ctype), value)
            value = valuearr[::-1][int((offset % 4) / bytesize)]
        else:
            raise RuntimeError("Cannot read %s of size %i: only 1,2,4,8 B supported" % (register, bytesize))
        return value

    def _memmap_read_array(self, register, ctype='L'):
        """ Read array chunk from mem-map """
        offset   = self.memmap[register]['offset']
        bytesize = self.memmap[register]['size']

        if isinstance(ctype, str):
            wsize = STRUCT_CTYPES_TO_B[ctype]
        elif isinstance(ctype, int):
            wsize = ctype
            ctype = STRUCT_CTYPES[ctype]
        else:
            raise RuntimeError('Unknown ctype: %s' % ctype)

        value = self.parent.read(self.name, size=bytesize, offset=offset)
        value = struct.unpack('>%i%s' % (int(bytesize / wsize), ctype), value)
        return value

    def _memmap_write_array(self, register, value, ctype='L'):
        offset   = self.memmap[register]['offset']
        bytesize = self.memmap[register]['size']

        if isinstance(ctype, str):
            wsize = STRUCT_CTYPES_TO_B[ctype]
        elif isinstance(ctype, int):
            wsize = ctype
            ctype = STRUCT_CTYPES[ctype]
        else:
            raise RuntimeError('Unknown ctype: %s' % ctype)

        n_elem = int(bytesize / wsize)  # Differs to n_elem in small bytesize code!
        if len(value) != n_elem:
            raise RuntimeError("Register is %i 32-bit words long, but array is "
                               "of length %i. Make sure these match." % (len(value), n_elem))
        if isinstance(value, str):
            packed = value
        else:
            packed = struct.pack('>%i%s' % (n_elem, ctype), *value)
        self.parent.blindwrite(self.name, packed, offset=offset)

    def configure_core(self, mac, ip, port, gateway=None, subnet_mask=None):
        """
        Configure the interface by writing to its internal configuration registers.

        :param mac: String or Integer input, MAC address (e.g. '02:00:00:00:00:01')
        :param ipaddress: String or Integer input, IP address (eg '10.0.0.1')
        :param port: String or Integer input
        :param gateway: String or Integer input, an IP address
        :param subnet_mask: string or integer, subnet mask (e.g. '255.255.255.0')
        """

        mac_to_write         = Mac(mac).mac_int
        ip_address_to_write  = IpAddress(ip).ip_int
        port_to_write        = port if isinstance(port, int) else int(port)
        gateway_to_write = IpAddress(gateway).ip_int if gateway is not None else 1
        subnet_mask_to_write = IpAddress(subnet_mask).ip_int if subnet_mask is not None else 0xffffff00

        self._memmap_write('MAC_ADDR', mac_to_write)
        self._memmap_write('IP_ADDR',  ip_address_to_write)
        self._memmap_write('NETMASK',  subnet_mask_to_write)
        self._memmap_write('GW_ADDR',  gateway_to_write)
        self._memmap_write('PORT',     port_to_write)

        self.fabric_enable()
        self.fabric_soft_reset_toggle()

    def dhcp_start(self):
        """
        Configure this interface, then start a DHCP client on ALL interfaces.
        """
        #if self.mac is None:
            # TODO get MAC from EEPROM serial number and assign here
            #self.mac = '0'
        reply, _ = self.parent.transport.katcprequest(
            name='tap-start', request_timeout=5,
            require_ok=True,
            request_args=(self.name, self.name, '0.0.0.0',
                          str(self.port), str(self.mac), ))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('%s: failure starting tap driver.' % self.name)

        reply, _ = self.parent.transport.katcprequest(
            name='tap-arp-config', request_timeout=1,
            require_ok=True,
            request_args=(self.name, 'mode', '0'))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('%s: failure disabling ARP.' % self.name)

        reply, _ = self.parent.transport.katcprequest(
            name='tap-dhcp', request_timeout=30,
            require_ok=True,
            request_args=(self.name, ))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('%s: failure starting DHCP client.' % self.name)

        reply, _ = self.parent.transport.katcprequest(
            name='tap-arp-config', request_timeout=1,
            require_ok=True,
            request_args=(self.name, 'mode', '-1'))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('%s: failure re-enabling ARP.' % self.name)
        # it looks like the command completed without error, so
        # update the basic core details
        self.get_gbe_core_details()

    def tap_start(self, restart=False):
        """
        Program a 10GbE device and start the TAP driver.

        :param restart: stop before starting
        """
        if len(self.name) > 8:
            raise NameError('%s: tap device identifier must be shorter than 9 '
                            'characters..' % self.fullname)
        if restart:
            self.tap_stop()
        if self.tap_running():
            LOGGER.info('%s: tap already running.' % self.fullname)
            return
        LOGGER.info('%s: starting tap driver.' % self.fullname)
        reply, _ = self.parent.transport.katcprequest(
            name='tap-start', request_timeout=-1, require_ok=True,
            request_args=(self.name, self.name, str(self.ip_address),
                          str(self.port), str(self.mac), ))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('%s: failure starting tap driver.' %
                               self.fullname)

    def tap_stop(self):
        """
        Stop a TAP driver.
        """
        if not self.tap_running():
            return
        LOGGER.info('%s: stopping tap driver.' % self.fullname)
        reply, _ = self.parent.transport.katcprequest(
            name='tap-stop', request_timeout=-1,
            require_ok=True, request_args=(self.name, ))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('%s: failure stopping tap '
                               'device.' % self.fullname)

    def tap_info(self):
        """
        Get info on the tap instance running on this interface.
        """
        uninforms = []

        def handle_inform(msg):
            uninforms.append(msg)

        self.parent.unhandled_inform_handler = handle_inform
        _, informs = self.parent.transport.katcprequest(
            name='tap-info', request_timeout=-1,
            require_ok=False, request_args=(self.name, ))
        self.parent.unhandled_inform_handler = None
        # process the tap-info
        if len(informs) == 1:
            return {'name': informs[0].arguments[0],
                    'ip': informs[0].arguments[1]}
        elif len(informs) == 0:
            return {'name': '', 'ip': ''}
        else:
            raise RuntimeError('%s: invalid return from tap-info?' %
                               self.fullname)
        # TODO - this request should return okay if the tap isn't
        # running - it shouldn't fail
        # if reply.arguments[0] != 'ok':
        #     log_runtime_error(LOGGER, 'Failure getting tap info for '
        #                               'device %s." % str(self))

    def tap_running(self):
        """
        Determine if an instance if tap is already running on for this
        ten GBE interface.
        """
        tapinfo = self.tap_info()
        if tapinfo['name'] == '':
            return False
        return True

    def tap_arp_reload(self):
        """
        Have the tap driver reload its ARP table right now.
        """
        reply, _ = self.parent.transport.katcprequest(
            name="tap-arp-reload", request_timeout=-1,
            require_ok=True, request_args=(self.name, ))
        if reply.arguments[0] != 'ok':
            raise RuntimeError('Failure requesting ARP reload for tap '
                               'device %s.' % str(self))

    def multicast_receive(self, ip_str, group_size):
        """
        Send a request to KATCP to have this tap instance send a multicast
        group join request.

        :param ip_str: A dotted decimal string representation of the base
            mcast IP address.
        :param group_size: An integer for how many mcast addresses from
            base to respond to.
        """
        # mask = 255*(2 ** 24) + 255*(2 ** 16) + 255*(2 ** 8) + (255-group_size)
        # self.parent.write_int(self.name, str2ip(ip_str), offset=12)
        # self.parent.write_int(self.name, mask, offset=13)

        # mcast_group_string = ip_str + '+' + str(group_size)
        mcast_group_string = ip_str
        reply, _ = self.parent.transport.katcprequest(
            'tap-multicast-add', -1, True, request_args=(self.name, 'recv',
                                                         mcast_group_string, ))
        if reply.arguments[0] == 'ok':
            if mcast_group_string not in self.multicast_subscriptions:
                self.multicast_subscriptions.append(mcast_group_string)
            return
        else:
            raise RuntimeError('%s: failed adding multicast receive %s to '
                               'tap device.' % (self.fullname,
                                                mcast_group_string))

    def multicast_remove(self, ip_str):
        """
        Send a request to be removed from a multicast group.

        :param ip_str: A dotted decimal string representation of the base
            mcast IP address.
        """
        try:
            reply, _ = self.parent.transport.katcprequest(
                'tap-multicast-remove', -1, True,
                request_args=(self.name, IpAddress.str2ip(ip_str), ))
        except:
            raise RuntimeError('%s: tap-multicast-remove does not seem to '
                               'be supported on %s' % (self.fullname,
                                                       self.parent.host))
        if reply.arguments[0] == 'ok':
            if ip_str not in self.multicast_subscriptions:
                LOGGER.warning(
                    '%s: That is odd, %s removed from mcast subscriptions, but '
                    'it was not in its list of sbscribed addresses.' % (
                        self.fullname, ip_str))
                self.multicast_subscriptions.remove(ip_str)
            return
        else:
            raise RuntimeError('%s: failed removing multicast address %s '
                               'from tap device' % (self.fullname,
                                                    IpAddress.str2ip(ip_str)))

    def _fabric_enable_disable(self, target_val):
        """

        :param target_val:
        """
        if self.memmap_compliant:
            word_bytes = list(
                struct.unpack('>4B', self.parent.read(self.name, 4, OFFSET_FLAGS)))
            if word_bytes[3] == target_val:
                return
            word_bytes[3] = target_val
            word_packed = struct.pack('>4B', *word_bytes)
            self.parent.write(self.name, word_packed, OFFSET_FLAGS)
        else:
            # 0x20 or (0x20 / 4)? What was the /4 for?
            word_bytes = list(
                struct.unpack('>4B', self.parent.read(self.name, 4, 0x20)))
            if word_bytes[1] == target_val:
                return
            word_bytes[1] = target_val
            word_packed = struct.pack('>4B', *word_bytes)
            self.parent.write(self.name, word_packed, 0x20)

    def fabric_enable(self):
        """
        Enable the core fabric
        """
        self._fabric_enable_disable(1)

    def fabric_disable(self):
        """
        Enable the core fabric
        """
        self._fabric_enable_disable(0)

    def fabric_soft_reset_toggle(self):
        """
        Toggle the fabric soft reset
        """
        if self.memmap_compliant:
            word_bytes = struct.unpack('>4B', self.parent.read(self.name, 4, OFFSET_FLAGS))
            word_bytes = list(word_bytes)

            def write_val(val):
                word_bytes[1] = val
                word_packed = struct.pack('>4B', *word_bytes)
                if val == 0:
                    self.parent.write(self.name, word_packed, OFFSET_FLAGS)
                else:
                    self.parent.blindwrite(self.name, word_packed, OFFSET_FLAGS)
            if word_bytes[1] == 1:
                write_val(0)
            write_val(1)
            write_val(0)
        else:
            word_bytes = struct.unpack('>4B', self.parent.read(self.name, 4, 0x20))
            word_bytes = list(word_bytes)

            def write_val(val):
                word_bytes[0] = val
                word_packed = struct.pack('>4B', *word_bytes)
                if val == 0:
                    self.parent.write(self.name, word_packed, 0x20)
                else:
                    self.parent.blindwrite(self.name, word_packed, 0x20)
            if word_bytes[0] == 1:
                write_val(0)
            write_val(1)
            write_val(0)

    def get_gbe_core_details(self, read_arp=False, read_cpu=False):
        """
        Get 10GbE core details.
        assemble struct for header stuff...

        .. code-block:: python

            \"\"\"
            0x00 - 0x07: MAC address
            0x08 - 0x0b: Not used
            0x0c - 0x0f: Gateway addr
            0x10 - 0x13: IP addr
            0x14 - 0x17: Not assigned
            0x18 - 0x1b: Buffer sizes
            0x1c - 0x1f: Not assigned
            0x20    :    Soft reset (bit 0)
            0x21    :    Fabric enable (bit 0)
            0x22 - 0x23: Fabric port
            0x24 - 0x27: XAUI status (bit 2,3,4,5 = lane sync, bit6 = chan_bond)
            0x28 - 0x2b: PHY config
            0x28    :    RX_eq_mix
            0x29    :    RX_eq_pol
            0x2a    :    TX_preemph
            0x2b    :    TX_diff_ctrl
            0x30 - 0x33: Multicast IP RX base address
            0x34 - 0x37: Multicast IP mask
            0x38 - 0x3b: Subnet mask
            0x1000  :    CPU TX buffer
            0x2000  :    CPU RX buffer
            0x3000  :    ARP tables start
            word_width = 8
            \"\"\"
            self.add_field(Bitfield.Field('mac0', 0,            word_width,                 0, 0 * word_width))
            self.add_field(Bitfield.Field('mac1', 0,            word_width,                 0, 1 * word_width))
            self.add_field(Bitfield.Field('mac2', 0,            word_width,                 0, 2 * word_width))
            self.add_field(Bitfield.Field('mac3', 0,            word_width,                 0, 3 * word_width))
            self.add_field(Bitfield.Field('mac4', 0,            word_width,                 0, 4 * word_width))
            self.add_field(Bitfield.Field('mac5', 0,            word_width,                 0, 5 * word_width))
            self.add_field(Bitfield.Field('mac6', 0,            word_width,                 0, 6 * word_width))
            self.add_field(Bitfield.Field('mac7', 0,            word_width,                 0, 7 * word_width))
            self.add_field(Bitfield.Field('unused_1', 0,        (0x0c - 0x08) * word_width, 0, 8 * word_width))
            self.add_field(Bitfield.Field('gateway_ip0', 0,     word_width,                 0, 0x0c * word_width))
            self.add_field(Bitfield.Field('gateway_ip1', 0,     word_width,                 0, 0x0d * word_width))
            self.add_field(Bitfield.Field('gateway_ip2', 0,     word_width,                 0, 0x0e * word_width))
            self.add_field(Bitfield.Field('gateway_ip3', 0,     word_width,                 0, 0x0f * word_width))
            self.add_field(Bitfield.Field('ip0', 0,             word_width,                 0, 0x10 * word_width))
            self.add_field(Bitfield.Field('ip1', 0,             word_width,                 0, 0x11 * word_width))
            self.add_field(Bitfield.Field('ip2', 0,             word_width,                 0, 0x12 * word_width))
            self.add_field(Bitfield.Field('ip3', 0,             word_width,                 0, 0x13 * word_width))
            self.add_field(Bitfield.Field('unused_2', 0,        (0x18 - 0x14) * word_width, 0, 0x14 * word_width))
            self.add_field(Bitfield.Field('buf_sizes', 0,       (0x1c - 0x18) * word_width, 0, 0x18 * word_width))
            self.add_field(Bitfield.Field('unused_3', 0,        (0x20 - 0x1c) * word_width, 0, 0x1c * word_width))
            self.add_field(Bitfield.Field('soft_reset', 2,      1,                          0, 0x20 * word_width))
            self.add_field(Bitfield.Field('fabric_enable', 2,   1,                          0, 0x21 * word_width))
            self.add_field(Bitfield.Field('port', 0,            (0x24 - 0x22) * word_width, 0, 0x22 * word_width))
            self.add_field(Bitfield.Field('xaui_status', 0,     (0x28 - 0x24) * word_width, 0, 0x24 * word_width))
            self.add_field(Bitfield.Field('rx_eq_mix', 0,       word_width,                 0, 0x28 * word_width))
            self.add_field(Bitfield.Field('rq_eq_pol', 0,       word_width,                 0, 0x29 * word_width))
            self.add_field(Bitfield.Field('tx_preempth', 0,     word_width,                 0, 0x2a * word_width))
            self.add_field(Bitfield.Field('tx_diff_ctrl', 0,    word_width,                 0, 0x2b * word_width))
            #self.add_field(Bitfield.Field('buffer_tx', 0,       0x1000 * word_width,        0, 0x1000 * word_width))
            #self.add_field(Bitfield.Field('buffer_rx', 0,       0x1000 * word_width,        0, 0x2000 * word_width))
            #self.add_field(Bitfield.Field('arp_table', 0,       0x1000 * word_width,        0, 0x3000 * word_width))
        """
        if self.memmap_compliant:
            data = self.parent.read(self.name, 16384)
            data = list(struct.unpack('>16384B', data))
            returnval = {
                'ip_prefix': '%i.%i.%i.' % (data[0x14], data[0x15], data[0x16]),
                'ip': IpAddress('%i.%i.%i.%i' % (data[0x14], data[0x15],
                                                 data[0x16], data[0x17])),
                'subnet_mask': IpAddress('%i.%i.%i.%i' % (
                                  data[0x1c], data[0x1d], data[0x1e], data[0x1f])),
                'mac': Mac('%i:%i:%i:%i:%i:%i' % (data[0x0e], data[0x0f],
                                                  data[0x10], data[0x11],
                                                  data[0x12], data[0x13])),
                'gateway_ip': IpAddress('%i.%i.%i.%i' % (data[0x18], data[0x19],
                                                         data[0x1a], data[0x1b])),
                'fabric_port': ((data[0x32] << 8) + (data[0x33])),
                'fabric_en': bool(data[0x2f] & 1),
                'multicast': {'base_ip': IpAddress('%i.%i.%i.%i' % (
                    data[0x20], data[0x21], data[0x22], data[0x23])),
                              'ip_mask': IpAddress('%i.%i.%i.%i' % (
                                  data[0x24], data[0x25], data[0x26], data[0x27])),
                              'rx_ips': []}
            }
        else:
            data = self.parent.read(self.name, 16384)
            data = list(struct.unpack('>16384B', data))
            returnval = {
                'ip_prefix': '%i.%i.%i.' % (data[0x10], data[0x11], data[0x12]),
                'ip': IpAddress('%i.%i.%i.%i' % (data[0x10], data[0x11],
                                                 data[0x12], data[0x13])),
                'subnet_mask': IpAddress('%i.%i.%i.%i' % (
                                  data[0x38], data[0x39], data[0x3a], data[0x3b])),
                'mac': Mac('%i:%i:%i:%i:%i:%i' % (data[0x02], data[0x03],
                                                  data[0x04], data[0x05],
                                                  data[0x06], data[0x07])),
                'gateway_ip': IpAddress('%i.%i.%i.%i' % (data[0x0c], data[0x0d],
                                                         data[0x0e], data[0x0f])),
                'fabric_port': ((data[0x22] << 8) + (data[0x23])),
                'fabric_en': bool(data[0x21] & 1),
                'xaui_lane_sync': [bool(data[0x27] & 4), bool(data[0x27] & 8),
                                   bool(data[0x27] & 16), bool(data[0x27] & 32)],
                'xaui_status': [data[0x24], data[0x25], data[0x26], data[0x27]],
                'xaui_chan_bond': bool(data[0x27] & 64),
                'xaui_phy': {'rx_eq_mix': data[0x28], 'rx_eq_pol': data[0x29],
                             'tx_preemph': data[0x2a], 'tx_swing': data[0x2b]},
                'multicast': {'base_ip': IpAddress('%i.%i.%i.%i' % (
                    data[0x30], data[0x31], data[0x32], data[0x33])),
                              'ip_mask': IpAddress('%i.%i.%i.%i' % (
                                  data[0x34], data[0x35], data[0x36], data[0x37])),
                              'rx_ips': []}
            }
            possible_addresses = [int(returnval['multicast']['base_ip'])]
            mask_int = int(returnval['multicast']['ip_mask'])
            for ctr in range(32):
                mask_bit = (mask_int >> ctr) & 1
                if not mask_bit:
                    new_ips = []
                    for ip in possible_addresses:
                        new_ips.append(ip & (~(1 << ctr)))
                        new_ips.append(new_ips[-1] | (1 << ctr))
                    possible_addresses.extend(new_ips)
            tmp = list(set(possible_addresses))
            for ip in tmp:
                returnval['multicast']['rx_ips'].append(IpAddress(ip))
        if read_arp:
            returnval['arp'] = self.get_arp_details(data)
        if read_cpu:
            returnval.update(self.get_cpu_details(data))
        self.core_details = returnval
        return returnval

    def get_arp_details(self, N=256):
        """ Get ARP details from this interface. """
        arp_table = self._memmap_read_array('ARP_CACHE', ctype='Q')
        return list(map(Mac, arp_table[:N]))

    def get_cpu_details(self, port_dump=None):
        """
        Read details of the CPU buffers.

        :param port_dump:
        """
        #TODO Not memmap compliant
        if port_dump is None:
            port_dump = self.parent.read(self.name, 16384)
            port_dump = list(struct.unpack('>16384B', port_dump))
        returnval = {'cpu_tx': {}}
        for ctr in range(4096 / 8):
            tmp = []
            for ctr2 in range(8):
                tmp.append(port_dump[4096 + (8 * ctr) + ctr2])
            returnval['cpu_tx'][ctr*8] = tmp
        returnval['cpu_rx_buf_unack_data'] = port_dump[6 * 4 + 3]
        returnval['cpu_rx'] = {}
        for ctr in range(port_dump[6 * 4 + 3] + 8):
            tmp = []
            for ctr2 in range(8):
                tmp.append(port_dump[8192 + (8 * ctr) + ctr2])
            returnval['cpu_rx'][ctr * 8] = tmp
        return returnval

    def set_single_arp_entry(self, ip, mac):
        """
        Set a single MAC address entry in the ARP table.

        :param ip (string) : IP whose MAC we should set. E.g. '10.0.1.123'
        :param mac (int)   : MAC address to associate with this IP. Eg. 0x020304050607
        """
        arp_addr = self.memmap['ARP_CACHE']['offset']
        arp_addr += 8*int(ip.split('.')[-1])
        mac_pack = struct.pack('>Q', mac)
        self.parent.write(self.name, mac_pack, offset=arp_addr)

    def set_arp_table(self, macs):
        """Set the ARP table with a list of MAC addresses. The list, `macs`,
        is passed such that the zeroth element is the MAC address of the
        device with IP XXX.XXX.XXX.0, and element N is the MAC address of the
        device with IP XXX.XXX.XXX.N"""
        if self.memmap_compliant:
            arp_addr = OFFSET_ARP_CACHE
        else:
            arp_addr = 0x3000
        macs = list(macs)
        macs_pack = struct.pack('>%dQ' % (len(macs)), *macs)
        self.parent.write(self.name, macs_pack, offset=arp_addr)

# end


from loader import courses_db
from .lecture import Lecture

class Course:
    def __init__(self, data: tuple, full: bool = False):
        self.table_name = data[1]
        self.name = data[2]
        self.description = data[3]
        self.poster = data[4]
        self.tg_id = data[5]
        self.tg_url = data[6]
        self.lectures_seminars = data[7]
        self.start_date = data[8]
        self.price = data[9]
        self.finished = data[10]
        self.lecture = [Lecture(lecture, self.table_name)
                        for lecture in courses_db.lecture(self.table_name, full)]

    @property
    def quantity(self) -> int:
        return sum(map(int, self.lectures_seminars.split(':')))

    @property
    def progress(self) -> str:
        all_lectures = courses_db.lecture(self.table_name, True)
        total = len(all_lectures)
        done = len([1 for lecture in all_lectures if lecture[-1]])
        return f'Завершен: {round(done / total * 100, 1)}% ({done}/{total})'

    @property
    def is_done(self):
        all_lectures = courses_db.lecture(self.table_name, True)
        return all([(lambda x: x[-1] == 1)(item) for item in all_lectures])

    @property
    def button(self):
        return self.name + (' (Лекции)' if self.finished else ' (Онлайн)')

    def info(self, full: bool = False):
        return f'{self.name}\n\n{self.description}\n\nДата начала курса: {self.start_date}' \
               f'\n\nЦена курса: {self.price} руб'


from flask import Flask, request,url_for,render_template,abort
from lxml import etree
app = Flask(__name__)

@app.route('/',methods=["GET","POST"])
def inicio():
	doc=etree.parse("sevilla.xml")
	municipios=doc.findall("municipio")
	return render_template("inicio.html",municipios=municipios)

@app.route('/<code>')
def temperatura(code):
	try:
		doc=etree.parse("http://www.aemet.es/xml/municipios/localidad_"+code+".xml")
	except:
		abort(404)
	name=doc.find("nombre").text
	max=doc.find("prediccion/dia/temperatura").find("maxima").text
	min=doc.find("prediccion/dia/temperatura").find("minima").text
	return render_template("temperaturas.html",name=name,max=max,min=min)

if __name__ == '__main__':
	app.run('0.0.0.0',5000, debug=True)


#analysis_authority.py
import birdnest
import sentimentSample
#import chunkGen
import matplotlib.pyplot as plt
import matplotlib.dates
import seaborn as sns
from datetime import datetime
import re
import json
import pandas as pd
import numpy as np
from wordcloud import WordCloud
import chunkGen
import os
import more_itertools as itools
import textflow
from multiprocessing import Pool
#THIS IS A TERRIBLE FUNCTION IT NEEDS TO BE CHANGED
def generate_tweet_timeline_x(timestring):
    timestring = timestring.replace("[", '')
    timestring = timestring.replace("]", '')
    timestring = timestring.replace(':', " ")
    timestring = timestring.replace('-', " ")
    timestring = timestring.split(" ")
    for x in range(0, len(timestring)):
        timestring[x] = int(timestring[x])
    time = timestring
    time = datetime(time[0], time[1], time[2], time[3], time[4])
    return(time)

def coresent(clist):
    returnlist = []
    for iterat in range(0, len(clist)):
        q = sentimentSample.retsent(clist[iterat])
        returnlist.append(q)
        print(q)
        print(str((iterat/len(clist)*100))+"%")
    return(returnlist)

def core_rip_sentiment(cores):
    tweet_row = birdnest.t_dump_by_row('text')
    author = birdnest.t_dump_by_row('authorUSN')
    time = birdnest.t_dump_by_row('time')
    #inefficiency right here
    for x in range(0, len(tweet_row)):
        tweet_row[x] = tweet_row[x][0]
    tweetsforcores = [list(c) for c in itools.divide(cores, tweet_row)]
    with Pool(cores) as p:
        returnedlist = p.map(coresent, tweetsforcores)
    retval = []
    for x in range(0, len(returnedlist)):
        retval.extend(returnedlist[x])
    return(retval)

def stash_sentiment(di):
    for keyq in di:
        for x in range(0, len(di[keyq]['x'])):
            di[keyq]['x'][x] = str(di[keyq]['x'][x])
    with open('memory/analysis/sentimentstash.json', 'w') as outfile:
        json.dump(di, outfile)

def open_sentiment():
    with open('memory/analysis/sentimentstash.json') as json_file:
        data = json.load(json_file)
    for keyq in data:
        for x in range(0, len(data[keyq]['x'])):
            data[keyq]['x'][x] = generate_tweet_timeline_x(data[keyq]['x'][x])
    return(data)

def vis_sentiment(di):
    fig, ax = plt.subplots()
    colors  = sns.color_palette(None, len(di))
    it = 0
    for key in di:
        color = colors[it]
    plt.legend()
    plt.show()

def sentiment_by_author_ripped():
    graph_dict = {}
    #['2022-03-14 23:21:51']
    tweet_row = birdnest.t_dump_by_row('text')
    author = birdnest.t_dump_by_row('authorUSN')
    time = birdnest.t_dump_by_row('time')
    rippedlist = core_rip_sentiment(15)
    for x in range(0, len(tweet_row)):
        if(not (author[x][0] in graph_dict)):
            print("Creating dict for author: " + author[x][0])
            graph_dict[author[x][0]] = {"x": [], "y": []}
        timex = generate_tweet_timeline_x(time[x][0])
        graph_dict[author[x][0]]["x"].append(timex)

        sent = rippedlist[x]
        if(sent == 'Positive'):
            graph_dict[author[x][0]]["y"].append(1)
        elif(sent == 'Negative'):
            graph_dict[author[x][0]]["y"].append(-1)
        else:
            graph_dict[author[x][0]]["y"].append(0)
    stash_sentiment(graph_dict)

def sentiment_by_author_single_core():
    graph_dict = {}
    #['2022-03-14 23:21:51']
    tweet_row = birdnest.t_dump_by_row('text')
    author = birdnest.t_dump_by_row('authorUSN')
    time = birdnest.t_dump_by_row('time')
    for x in range(0, len(tweet_row)):
        if( not (author[x][0] in graph_dict)):
            print("Creating dict for author: " + author[x][0])
            graph_dict[author[x][0]] = {"x": [], "y": []}
        sent = sentimentSample.retsent(tweet_row[x][0])
        #append x
        timex = generate_tweet_timeline_x(time[x][0])
        graph_dict[author[x][0]]["x"].append(timex)
        #append sentiment
        if(sent == 'Positive'):
            graph_dict[author[x][0]]["y"].append(1)
        elif(sent == 'Negative'):
            graph_dict[author[x][0]]["y"].append(1)
        else:
            graph_dict[author[x][0]]["y"].append(0)
    for key in graph_dict:
        plt.plot(graph_dict[key]["x"], graph_dict[key]["y"], label=key)
    plt.legend()
    plt.show()

def chunk_by_tweet():
    spGen = chunkGen.chunkGenerator()
    #wait until generator is alive
    while(next(spGen)!=True):
        time.sleep(0.1)
    tweet_row = birdnest.t_dump_by_row('text')
    author = birdnest.t_dump_by_row('authorUSN')
    for x in range(0, len(tweet_row)):
        next(spGen)
        print(spGen.send(tweet_row[x][0])[2])
#chunk_by_tweet()
#sentiment_by_author_ripped()
graph_dict = open_sentiment()

class language_loop:
    def __init__(self):
        print("< ------- Text Flow Initializing ------ >")
        self.tfobj = textflow.stream()
        self.flow = self.tfobj.routine()
        self.seg_s = chunkGen.docprocgen()
        while(next(self.flow)!=True):
            time.sleep(0.1)
        while(next(self.seg_s)!=True):
            time.sleep(0.1)
        print("< ------- Text Flow Online ------ >")
    def read_complete_tweets(self):
        tweet_row = birdnest.t_dump_by_row('text')
        author = birdnest.t_dump_by_row('authorUSN')
        time = birdnest.t_dump_by_row('time')
        graph_dict = open_sentiment()
        totaltweetsentlist = []
        for key in graph_dict:
            for x in range(0, len(graph_dict[key]['y'])):
                totaltweetsentlist.append(int(graph_dict[key]['y'][x]))
        for x in range(0, len(tweet_row)):
            self.flow.send([author[x][0], tweet_row[x][0], totaltweetsentlist[x]])
    def read_complete_youtube(self):
            path = "memory/youtube"
            q = os.listdir(path)
            q = sorted(q)
            for file in q:
                speaker = file.split("_")[0]
                vid_ext = file.split("_")[1]
                filetr = open(path+"/"+file, 'r')
                Lines = filetr.readlines()
                for line in Lines:
                    line = line.strip()
                    line = line.split(",")
                    timespoken = line[len(line)-1]
                    timeend = line[len(line)-2].split(".")[0]
                    timestart = line[len(line)-3].split(".")[0][1:len(line[len(line)-3].split(".")[0])]
                    words = line[0].split(":")[1]
                    if(words[0]==" "):
                        words = words[1:len(words)]
                    self.flow.send([speaker, words, 0, "https://www.youtube.com/watch?v="+vid_ext[0:len(vid_ext)-4], timespoken,"https://www.youtube.com/watch?v="+vid_ext[0:len(vid_ext)-4]+"?t="+str(timestart)+"s"])
    def spin(self):
        self.tfobj.spin_trace()
    def compare(self):
        self.tfobj.compare_speakers()
def print_author_tweets(authorUSN):
    tweet_row = birdnest.t_dump_by_row('text')
    retweet_row = birdnest.t_dump_by_row('retweetsNum')
    like_tweet_row = birdnest.t_dump_by_row('likesNum')
    author = birdnest.t_dump_by_row('authorUSN')
    time = birdnest.t_dump_by_row('time')
    for x in range(0, len(tweet_row)):
        if(author[x][0] == authorUSN):
            print("============")
            print(tweet_row[x])
            print(retweet_row[x])
            print(like_tweet_row[x])
            print(time[x])
            print("============")

q = language_loop()
#q.read_complete_youtube()
#q.spin()
q.compare()
#q.read_complete_youtube()
#q.spin()
#q.spin()
#print_author_tweets('@KristinaKaramo')


# -*- coding: utf-8 -*-
import logging
import socket
import struct
import time
import os
import datetime
import numpy as np
from celery import Celery

from analysis import create_app
from analysis.models import Flow
from analysis_machine import main, load_model, build_model
from celery import platforms
from config import ENV

platforms.C_FORCE_ROOT = True

app = create_app(ENV, name=__name__)
app.config['CELERY_BROKER_URL'] = 'redis://10.252.99.41:6379/0'

# app.config['CELERY_RESULT_BACKEND'] = 'redis://10.252.99.41:6379/0'

def make_celery(_app):
    _celery = Celery(_app.name,
                     broker=_app.config['CELERY_BROKER_URL']
                     # backend=_app.config['CELERY_RESULT_BACKEND']
                     )

    # TaskBase = _celery.Task
    #
    # class ContextTask(TaskBase):
    #     abstract = True
    #
    #     def __call__(self, *args, **kwargs):
    #         with app.app_context():
    #             return TaskBase.__call__(self, *args, **kwargs)
    #
    # _celery.Task = ContextTask
    _celery.config_from_object(app.config)

    return _celery

celery = make_celery(app)

def find_first_timestamp(data, timestamp, idx=0):
    length = len(data)
    if length == 1:
        return idx
    elif length == 0:
        return idx + 1
    else:
        i = int(length / 2)
        if data[i]['probe_ts'] == timestamp:
            return find_first_timestamp(data[:i], timestamp, idx)
        else:
            idx = idx + i + 1
            return find_first_timestamp(data[i + 1:], timestamp, idx=idx)

def create_queue(data):
    # queue_obj = SingletonQueue()
    timestamp_min = data[0].get('probe_ts')
    timestamp_max = data[-1].get('probe_ts')
    # 默认为请求来的数据是一个由时间戳升序列表
    # 判断最大的时间戳与最小时间戳是否相等
    # 相等则表示该flow内为同一秒
    # 不等则找出不等的点进行分割
    # 认为一次请求中的连接时间戳最多相差1秒
    # diff = timestamp_min - queue_obj.timestamp_current
    if timestamp_min == timestamp_max:
        queue = [[], data]
        # if diff == 0:
        #     queue_obj.push_current_con(data)
        #
        # # 代表上一秒的链接都发完了
        # elif diff == 1:
        #     queue_obj.push_next_con(data)
        #     queue = queue_obj.get_queue()
        #
        # # 重洗队列
        # elif diff >= 2:
        #     queue_obj.clean_all()
        #     queue_obj.push_last_con(data)
        #     queue_obj.set_timestamp_last(timestamp_min)
        #     queue_obj.set_timestamp_current(timestamp_min+1)
        #
        # elif diff < 0:
        #     queue_obj.push_last_con(data)

    else:
        timestamp = timestamp_min + 1
        idx = find_first_timestamp(data, timestamp)
        data_min = data[:idx]
        data_max = data[idx:]
        queue = [data_min, data_max]
        # if diff == 0:
        #     queue_obj.push_current_con(data_min)
        #     queue_obj.push_next_con(data_max)
        #
        # # 代表上一秒的链接都发完了
        # elif diff == 1:
        #     queue_obj.push_next_con(data_min)
        #     queue = queue_obj.get_queue()
        #
        # # 重洗队列
        # elif diff >= 2:
        #     queue_obj.clean_all()
        #     queue_obj.push_last_con(data_min)
        #     queue_obj.push_current_con(data_max)
        #     queue_obj.set_timestamp_last(timestamp_min)
        #     queue_obj.set_timestamp_current(timestamp)
        # elif diff < 0:
        #     queue_obj.push_last_con(data_min)
        #     queue_obj.push_current_con(data_max)

    # queue = queue_obj.get_queue()
    return queue

class SingletonModel(object):
    __instance = None
    model = ''

    def __init__(self):
        pass

    def __new__(cls, *args, **kwd):
        if SingletonModel.__instance is None:
            SingletonModel.__instance = object.__new__(cls, *args, **kwd)
            basedir = os.path.abspath(os.path.dirname(__file__))
            model_name = 'model_ip_port.h5'
            cls.model = load_model(os.path.join(basedir, 'analysis', 'utils', model_name))
            cls.model.predict(np.zeros((1, 25)))
        return SingletonModel.__instance

class SingletonDGAModel(object):
    __instance = None
    model = ''
    max_features = 38
    maxlen = 53

    def __init__(self):
        pass

    def __new__(cls, *args, **kwd):
        if SingletonDGAModel.__instance is None:
            SingletonDGAModel.__instance = object.__new__(cls, *args, **kwd)
            basedir = os.path.abspath(os.path.dirname(__file__))
            model_name = 'lstm_model.h5'
            cls.model = build_model(cls.max_features, cls.maxlen,
                                    os.path.join(basedir, 'analysis', 'utils', model_name))
            cls.model.predict([[16 for i in range(53)]])

        return SingletonDGAModel.__instance

@celery.task
def my_celery(data):
    m = SingletonModel()
    dga = SingletonDGAModel()
    model = m.model
    dga_model = dga.model
    # logging.info('receive data numbers1111111111111: {0}'.format(len(data)))
    queue = data
    if queue:
        logging.info('******{0}, {1}, **********'.format(len(queue[0]), len(queue[1])))
        start = datetime.datetime.now()
        res = main(queue, model, dga_model)
        # model.predict(np.zeros((1, 25)))
        end = datetime.datetime.now()
        a = (end - start).seconds
        logging.info('运行时间:{0}'.format(a))
        for error_con in res:
            dip = error_con['content'][0]
            dport = error_con['content'][1]
            sip = error_con['content'][2]
            sport = error_con['content'][3]
            timestamp = error_con['content'][4]
            error_type = error_con['error_type']
            kwargs = {
                'dip': socket.inet_ntoa(struct.pack('I', socket.htonl(int(dip)))),
                'dport': str(int(dport)),
                'sip': socket.inet_ntoa(struct.pack('I', socket.htonl(int(sip)))),
                'sport': str(int(sport)),
                'error_type': error_type[0],
                'error_per': str(error_type[1]),
                'timestamp': str(int(timestamp)),
                'createdtime': str(int(time.time())),
                'domain': error_con['domain']
            }
            Flow.objects.create(**kwargs)


from msrest.serialization import Model

class Property(Model):
    """Property.

    :param reported:
    :type reported: ~service.models.Reported
    :param desired:
    :type desired: ~service.models.Desired
    """

    _attribute_map = {
        'reported': {'key': 'reported', 'type': 'Reported'},
        'desired': {'key': 'desired', 'type': 'Desired'},
    }

    def __init__(self, reported=None, desired=None):
        super(Property, self).__init__()
        self.reported = reported
        self.desired = desired


import os
import re
import time

from sklearn.metrics import f1_score

from src.classifiers.classification_pipeline import ClassificationPipeline
from src.data_handlers.labelled_data_reader import LabelledDataReader
from src.data_handlers.unlabelled_data_reader import UnlabelledDataReader
from src.loggers.log_utils import setup_logger

logger = setup_logger(__name__)

class SymbolicClassifier:

    def __init__(self, data_filepath: str = None, validate: bool = False) -> None:
        logger.info("Instantiating Predictor...")
        self.validate: bool = validate
        self.data_filepath: str = data_filepath
        self.data: dict[str, int] = self._load_data()
        self.classification_pipeline: ClassificationPipeline = ClassificationPipeline(self._load_train_data())
        self.predictions: dict[str, int] = {}
        self.scorer: Scorer = None
        self.f1_score: float = None

    def predict(self, continuous: bool = True, discreet: bool = False) -> int:
        logger.info("Predicting queries...")
        for query in self.data:
            category: int = self.predict_query(query)
            self.predictions[query] = category
        if self.validate:
            f1_score: float = self.score()
            logger.info(f"F1 score: {f1_score}")
        self._save_predictions()

    def predict_query(self, query: str) -> int:
        category: int = self.classification_pipeline.classify(query)
        return category

    def score(self) -> float:
        self.scorer: Scorer = Scorer(self.predictions, self.data)
        f1_score: float = self.scorer.calculate_f1_score()
        self.f1_score: float = f1_score
        return f1_score

    def _load_data(self) -> dict[str, int]:
        if self.validate or self.data_filepath is None:
            logger.warning("No data filepath provided. Loading validation data...")
            self.data_filepath: str = os.environ["PATH_LABELLED_TEST_DATA"]
            data_reader: LabelledDataReader = LabelledDataReader(self.data_filepath)
        else:
            logger.info("Loading unlabelled test data...")
            data_reader: UnlabelledDataReader = UnlabelledDataReader(self.data_filepath)
        labelled_data: dict[str, int] = data_reader.read_data()
        return labelled_data

    def _load_train_data(self) -> dict[str, int]:
        data_filepath: str = os.environ["PATH_LABELLED_TRAIN_DATA"]
        data_reader: LabelledDataReader = LabelledDataReader(data_filepath)
        labelled_train_data: dict[str, int] = data_reader.read_data()
        return labelled_train_data

    def _save_predictions(self):
        filename: str = self._filename_to_save_string(self.data_filepath)
        filepath: str = os.path.join(os.environ["PATH_PREDICTIONS"], filename)
        logger.info(f"Saving predictions to {filepath}")
        try:
            with open(filepath, "w") as csv_file:
                for query in self.predictions:
                    category: int = self.predictions[query]
                    csv_file.write(f"{query},{category}\n")
            logger.info(f"Predictions saved successfully to {filepath}")
        except Exception as e:
            logger.error(f"Failed to save predictions. Exception: {e}")

    def _filename_to_save_string(self, filepath):
        base_name = os.path.basename(filepath)
        name_without_extension, _ = os.path.splitext(base_name)
        save_string = re.sub(r'\W+', '_', name_without_extension)
        timestamp = int(time.time())
        save_string = f"predictions-{timestamp}-{save_string}.csv"
        return save_string

class Scorer:

    def __init__(self, predictions: dict[str, int], validation_data: dict[str, int]) -> None:
        self.predictions: dict[str, int] = predictions
        self.validation_data: dict[str, int] = validation_data
        self.f1_score: float = None

    def calculate_f1_score(self) -> float:
        true_labels: list[int] = [self.validation_data[query] for query in self.predictions]
        predicted_labels: list[int] = [self.predictions[query] for query in self.predictions]
        self.f1_score: float = f1_score(true_labels, predicted_labels, average='macro')
        return self.f1_score


from statistics import stdev, mean
from typing import List, Dict, Set
from collections import Counter
from example import Example
import numpy as np
import random

def calculate_stats(arr_of_confusion: List[Dict]):
    accuracies = []
    precisions = []
    recalls = []
    for matrix in arr_of_confusion:
        accuracies.append((matrix["tp"] + matrix["tn"]) / (sum(matrix.values())))
        precisions.append(matrix["tp"] / (matrix["tp"] + matrix["fp"]))
        recalls.append(matrix["tp"] / (matrix["tp"] + matrix["fn"]))

    print_formatted_stats("Accuracy", accuracies)
    print_formatted_stats("Precision", precisions)
    print_formatted_stats("Recall", recalls)

def print_formatted_stats(stat_str, to_stat):
    if len(to_stat) > 1:
        print(f"{stat_str}: {round(mean(to_stat), 3)}, {round(stdev(to_stat), 3)}")
    else:
        print(f"{stat_str}: {round(to_stat[0], 3)}")

def calculate_aroc(arr_of_confidence):
    sort_conf = sorted(arr_of_confidence, key=lambda conf: conf[1], reverse=True)
    total_positive = len([x for x in sort_conf if x[0]])
    total_negative = len(sort_conf) - total_positive
    roc_points = []
    fp, tp = 0, 0
    for label, confidence in sort_conf:
        if label:
            tp += 1
        else:
            fp += 1
        fpr = fp / total_negative if total_negative > 0 else 0
        tpr = tp / total_positive if total_positive > 0 else 0
        roc_points.append([fpr, tpr])

    print(f"Area under ROC: {calculate_integral(roc_points):0.3f}")

def calculate_integral(points):
    area = 0
    pt_idx = 1
    while pt_idx < len(points):
        dx = points[pt_idx][0] - points[pt_idx - 1][0]
        area += ((points[pt_idx][1] + points[pt_idx - 1][1]) / 2) * dx
        pt_idx += 1

    return area

def n_fold_cross_validation(examples, num_folds=5):
    """
    returns a list of length num_folds containing
    tuples of the form (train_set, test_set)
    """
    random.shuffle(examples, random.seed(12345))
    batches = [examples[i::num_folds] for i in range(num_folds)]
    n_fold_sets = [
        (
            [example for flat in (batches[:i] + batches[i + 1 :]) for example in flat],
            batch,
        )
        for i, batch in enumerate(batches)
    ]

    return np.array(n_fold_sets)

def most_common_labels(examples: List[Example], top_n: int = 1) -> List:
    """
    return a list of the top n class labels from a list of examples, where
    each example is an array of feature values and a class label
    """
    top_labels = Counter([example.label for example in examples]).most_common(top_n)
    return [label[0] for label in top_labels]

def calculate_label_occurrences(examples):
    """
    Finds the occurrences of positive examples for a given attribute or value
    attr_idx: this can be specified if a specific attribute should be counted
    """
    positive_examples = sum([example.label for example in examples])
    return [positive_examples, len(examples) - positive_examples]

def calculate_nominal_occurrences(examples: List[Example], attr_idx: int):
    """
    Finds the occurrences of each value of a nominal attribute
    """
    value_occs = {}

    for example in examples:
        value = example[attr_idx]

        if value not in value_occs:
            value_occs[value] = [0, 0]

        value_occs[example[attr_idx]][int(example.label != 1)] += example.weight

    return [value_occ for key, value_occ in value_occs.items()]


import os
import csv
class notes_worker():
    def __init__(self):
        self.file = "notes.csv"
        if not os.path.exists(self.file):
            with open(self.file, "w") as notes:
                self.writer = csv.DictWriter(notes, fieldnames=["ID", "Note"])
                self.writer.writeheader()
                self.num_note = 1
        else:
            with open(self.file, "r") as notes:
                reader = csv.DictReader(notes)
                for i in reader:
                    self.num_note = int(i["ID"]) + 1

    def readAll(self):
        with open(self.file) as notes:
            file_reader = csv.DictReader(notes)
            for i in file_reader:
                print(f"{i['ID']}: {i['Note']}")

    def readNote(self, num_note: int):
        with open(self.file) as notes:
            file_reader = csv.DictReader(notes)
            for i in file_reader:
                if int(i["ID"]) == num_note:
                    print(i)
                    return
        print("This note not found")

    def updateNote(self, num_note:int, new_data:str):
        with open(self.file, "r") as notes:
            reader = csv.DictReader(notes)
            tmp_data = []
            found = False
            for i in reader:
                tmp_data.append(i)
                if int(i["ID"]) == num_note:
                    found = True
            if not found:
                print("This note not found!")
                return
        with open(self.file, "w") as notes:
            writer = csv.DictWriter(notes, fieldnames=["ID", "Note"])
            writer.writeheader()

            for i in tmp_data:
                if int(i["ID"]) == num_note:
                    writer.writerow({"ID": num_note, "Note": new_data})
                    continue
                writer.writerow({"ID": i["ID"], "Note": i["Note"]})

    def addNote(self, note: str):
        with open(self.file, "a") as notes:
            self.writer = csv.DictWriter(notes, fieldnames=["ID", "Note"])
            self.writer.writerow({"ID": self.num_note,
                                  "Note":note})
            self.num_note += 1

    def deleteNote(self, num_note: int):
        with open(self.file, "r") as notes:
            reader = csv.DictReader(notes)
            tmp_data = []
            found = False
            for i in reader:
                if int(i["ID"]) == num_note:
                    found = True
                    continue
                tmp_data.append(i)
            if not found:
                print("This note not found!")
                return
        with open(self.file, "w") as notes:
            writer = csv.DictWriter(notes, fieldnames=["ID", "Note"])
            writer.writeheader()

            counter = 1
            for i in tmp_data:
                writer.writerow({"ID": counter, "Note": i["Note"]})
                counter += 1
            self.num_note = counter + 1


#!/usr/bin/env python3
import sys
import random

def knuthshuffle(songs):
   """ Shuffle the elements in an list of songs """
   i = 0
   while i<len(songs)-1:
      swap(songs, i, random.randint(i, len(songs)-1))
      i += 1
   return songs

def swap(songs, a, b):
   temp = songs[a]
   songs[a] = songs[b]
   songs[b] = temp


import logging
import typing

from celery.utils.log import get_task_logger
from django.conf import settings
from django.utils import timezone

from apps.labels.client import LabelsAPIClient
from apps.labels.utils import LABEL_OUTDATED_TIMEOUT_MINUTES, LabelKeyData, LabelsData, get_associating_label_model
from apps.user_management.models import Organization
from common.custom_celery_tasks import shared_dedicated_queue_retry_task

logger = get_task_logger(__name__)
logger.setLevel(logging.DEBUG)

class ValueData(typing.TypedDict):
    value_name: str
    key_name: str

def unify_labels_data(labels_data: LabelsData | LabelKeyData) -> typing.Dict[str, ValueData]:
    values_data: typing.Dict[str, ValueData]
    if isinstance(labels_data, list):  # LabelsData
        values_data = {
            label["value"]["id"]: {"value_name": label["value"]["name"], "key_name": label["key"]["name"]}
            for label in labels_data
        }
    else:  # LabelKeyData
        values_data = {
            label["id"]: {"value_name": label["name"], "key_name": labels_data["key"]["name"]}
            for label in labels_data["values"]
        }
    return values_data

@shared_dedicated_queue_retry_task(
    autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
)
def update_labels_cache(labels_data: LabelsData | LabelKeyData):
    from apps.labels.models import LabelKeyCache, LabelValueCache

    values_data: typing.Dict[str, ValueData] = unify_labels_data(labels_data)
    values = LabelValueCache.objects.filter(id__in=values_data).select_related("key")
    now = timezone.now()

    if not values:
        return

    keys_to_update = set()

    for value in values:
        if value.name != values_data[value.id]["value_name"]:
            value.name = values_data[value.id]["value_name"]
        value.last_synced = now

        if value.key.name != values_data[value.id]["key_name"]:
            value.key.name = values_data[value.id]["key_name"]
        value.key.last_synced = now
        keys_to_update.add(value.key)

    LabelKeyCache.objects.bulk_update(keys_to_update, fields=["name", "last_synced"])
    LabelValueCache.objects.bulk_update(values, fields=["name", "last_synced"])

@shared_dedicated_queue_retry_task(
    autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else 10
)
def update_instances_labels_cache(organization_id: int, instance_ids: typing.List[int], instance_model_name: str):
    from apps.labels.models import LabelValueCache

    now = timezone.now()
    organization = Organization.objects.get(id=organization_id)

    model = get_associating_label_model(instance_model_name)
    field_name = model.get_associating_label_field_name()
    associated_instances = {f"{field_name}_id__in": instance_ids}
    values_ids = model.objects.filter(**associated_instances).values_list("value_id", flat=True)
    outdated_last_synced = now - timezone.timedelta(minutes=LABEL_OUTDATED_TIMEOUT_MINUTES)
    values = LabelValueCache.objects.filter(id__in=values_ids, last_synced__lte=outdated_last_synced)

    if not values:
        return

    keys_ids = set(value.key_id for value in values)

    client = LabelsAPIClient(organization.grafana_url, organization.api_token)
    for key_id in keys_ids:
        label_data, _ = client.get_values(key_id)
        if label_data:
            update_labels_cache.apply_async((label_data,))


import os
import zipfile
import shutil
import lmctl.files as files
import lmctl.project.handlers.interface as handlers_api
import lmctl.project.validation as project_validation
import lmctl.utils.descriptors as descriptor_utils
import lmctl.drivers.lm.base as lm_drivers
from .brent_autocorrect import BrentCorrectableValidation

class BrentPkgContentTree(files.Tree):

    RESOURCE_YAML_FILE = 'resource.yaml'

    def __init__(self, root_path=None):
        super().__init__(root_path)

    def gen_resource_package_file_path(self, resource_name):
        return self.resolve_relative_path('{0}.zip'.format(resource_name))

    @property
    def root_descriptor_file_path(self):
        return self.resolve_relative_path(BrentPkgContentTree.RESOURCE_YAML_FILE)

class BrentResourcePackageContentTree(files.Tree):

    DEFINITIONS_DIR_NAME = 'Definitions'
    INFRASTRUCTURE_DIR_NAME = 'infrastructure'
    LM_DIR_NAME = 'lm'
    DESCRIPTOR_FILE_NAME_YML = 'resource.yaml'
    LIFECYCLE_DIR_NAME = 'Lifecycle'
    INFRASTRUCTURE_MANIFEST_FILE_NAME = 'infrastructure.mf'
    LIFECYCLE_MANIFEST_FILE_NAME = 'lifecycle.mf'

    @property
    def definitions_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.DEFINITIONS_DIR_NAME)

    @property
    def infrastructure_definitions_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.DEFINITIONS_DIR_NAME, BrentResourcePackageContentTree.INFRASTRUCTURE_DIR_NAME)

    @property
    def lm_definitions_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.DEFINITIONS_DIR_NAME, BrentResourcePackageContentTree.LM_DIR_NAME)

    @property
    def descriptor_file_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.DEFINITIONS_DIR_NAME, BrentResourcePackageContentTree.LM_DIR_NAME, BrentResourcePackageContentTree.DESCRIPTOR_FILE_NAME_YML)

    @property
    def lifecycle_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.LIFECYCLE_DIR_NAME)

    @property
    def infrastructure_manifest_file_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.DEFINITIONS_DIR_NAME, BrentResourcePackageContentTree.INFRASTRUCTURE_DIR_NAME, BrentResourcePackageContentTree.INFRASTRUCTURE_MANIFEST_FILE_NAME)

    @property
    def lifecycle_manifest_file_path(self):
        return self.resolve_relative_path(BrentResourcePackageContentTree.LIFECYCLE_DIR_NAME, BrentResourcePackageContentTree.LIFECYCLE_MANIFEST_FILE_NAME)

class BrentContentHandlerDelegate(handlers_api.ResourceContentHandlerDelegate):

    def __init__(self, root_path, meta):
        super().__init__(root_path, meta)
        self.tree = BrentPkgContentTree(self.root_path)

    def validate_content(self, journal, env_sessions, validation_options):
        errors = []
        warnings = []
        self.__validate_descriptor(journal, errors, warnings)
        self.__validate_res_pkg(journal, validation_options, errors, warnings)
        return project_validation.ValidationResult(errors, warnings)

    def __validate_descriptor(self, journal, errors, warnings):
        descriptor_path = self.tree.root_descriptor_file_path
        if not os.path.exists(descriptor_path):
            msg = 'No descriptor found at: {0}'.format(descriptor_path)
            journal.error_event(msg)
            errors.append(project_validation.ValidationViolation(msg))

    def __validate_res_pkg(self, journal, validation_options, errors, warnings):
        journal.stage('Checking Resource package exists for {0}'.format(self.meta.name))
        res_pkg_path = self.tree.gen_resource_package_file_path(self.meta.full_name)
        if not os.path.exists(res_pkg_path):
            msg = 'No Resource package found at: {0}'.format(res_pkg_path)
            journal.error_event(msg)
            errors.append(project_validation.ValidationViolation(msg))
        else:
            extraction_path = os.path.join(os.path.dirname(res_pkg_path), 'tmp-extract')
            os.makedirs(extraction_path)
            with zipfile.ZipFile(res_pkg_path, "r") as res_pkg:
                res_pkg.extractall(extraction_path)
            try:
                tree = BrentResourcePackageContentTree(extraction_path)
                BrentCorrectableValidation().validate_and_autocorrect(journal, validation_options, errors, warnings, tree.descriptor_file_path, \
                tree.infrastructure_definitions_path, tree.infrastructure_manifest_file_path, tree.lifecycle_path, \
                    tree.lifecycle_manifest_file_path)
                with zipfile.ZipFile(res_pkg_path, "w") as res_pkg:
                    res_pkg_content_tree = BrentResourcePackageContentTree()
                    included_items = [
                        {'path': tree.definitions_path, 'alias': res_pkg_content_tree.definitions_path},
                        {'path': tree.lifecycle_path, 'alias': res_pkg_content_tree.lifecycle_path}
                    ]
                    for included_item in included_items:
                        self.__add_directory(journal, res_pkg, included_item)
            finally:
                if os.path.exists(extraction_path):
                    shutil.rmtree(extraction_path)

    def __add_directory(self, journal, res_pkg, included_item):
        path = included_item['path']
        res_pkg.write(path, arcname=included_item['alias'])
        rootlen = len(path) + 1
        for root, dirs, files in os.walk(path):
            for dirname in dirs:
                full_path = os.path.join(root, dirname)
                res_pkg.write(full_path, arcname=os.path.join(included_item['alias'], full_path[rootlen:]))
            for filename in files:
                full_path = os.path.join(root, filename)
                res_pkg.write(full_path, arcname=os.path.join(included_item['alias'], full_path[rootlen:]))

    def __clear_existing_descriptor(self, journal, env_sessions):
        lm_session = env_sessions.lm
        descriptor_path = self.tree.root_descriptor_file_path
        descriptor = descriptor_utils.DescriptorParser().read_from_file(descriptor_path)
        descriptor_name = descriptor.get_name()
        descriptor_version = descriptor.get_version()
        journal.event('Removing descriptor {0} from CP4NA orchestration ({1})'.format(descriptor_name, lm_session.env.address))
        descriptor_driver = lm_session.descriptor_driver
        try:
            descriptor_driver.delete_descriptor(descriptor_name)
            env_sessions.mark_lm_updated()
        except lm_drivers.NotFoundException:
            journal.event('Descriptor {0} not found'.format(descriptor_name))
        return descriptor_name, descriptor_version

    def push_content(self, journal, env_sessions, push_options):
        descriptor_name, descriptor_version = self.__clear_existing_descriptor(journal, env_sessions)
        self.__push_res_pkg(journal, env_sessions, descriptor_name, push_options)

    def __push_res_pkg(self, journal, env_sessions, descriptor_name, push_options):
        lm_session = env_sessions.lm
        pkg_driver = lm_session.resource_pkg_driver
        journal.event('Removing any existing Resource package named {0} (version: {1}) from Brent: {2} ({3})'.format(descriptor_name, self.meta.version, lm_session.env.name, lm_session.env.address))
        try:
            pkg_driver.delete_package(descriptor_name)
        except lm_drivers.NotFoundException:
            journal.event('No package named {0} found'.format(descriptor_name))
        res_pkg_path = self.tree.gen_resource_package_file_path(self.meta.full_name)
        journal.event('Pushing {0} (version: {1}) Resource package to Brent: {2} ({3})'.format(self.meta.full_name, self.meta.version, lm_session.env.name, lm_session.env.address))
        pkg_driver.onboard_package(res_pkg_path, object_group_id=push_options.object_group_id)
        env_sessions.mark_brent_updated()


from capstone import Cs, CS_ARCH_X86, CS_MODE_64, CS_MODE_32, CS_MODE_16

def convert_bin_to_asm(bin_path, output_file):
    # Load the binary file
    with open(bin_path, 'rb') as f:
        binary = f.read()

    # Initialize Capstone disassembler
    print("Select the architecture mode:")
    print("1. 64-bit")
    print("2. 32-bit")
    print("3. 16-bit")
    mode_choice = int(input("Enter your choice: "))
    if mode_choice == 1:
        md = Cs(CS_ARCH_X86, CS_MODE_64)
    elif mode_choice == 2:
        md = Cs(CS_ARCH_X86, CS_MODE_32)
    elif mode_choice == 3:
        md = Cs(CS_ARCH_X86, CS_MODE_16)
    else:
        print("Invalid choice. Defaulting to 64-bit mode.")
        md = Cs(CS_ARCH_X86, CS_MODE_64)

    # Disassemble the binary code into assembly instructions
    asm_code = ''
    for insn in md.disasm(binary, 0x0):
        asm_code += f'0x{insn.address:x}: {insn.mnemonic} {insn.op_str}\n'

    # Write the assembly code to the output file
    with open(output_file, 'w') as f:
        f.write(asm_code)

    print(f'Successfully converted {bin_path} to {output_file}.')

# Example usage
bin_path = input("Enter the path to the .bin file you want to convert: ")
output_file = input("Enter the path to the output file where the assembly code will be written: ")
convert_bin_to_asm(bin_path, output_file)


import abc
from collections import OrderedDict
import os

import gtimer as gt
import numpy as np
import torch

from rlkit.core import logger, eval_util
from rlkit.data_management.replay_buffer import ReplayBuffer
from rlkit.samplers.data_collector import DataCollector

import mlflow

def _get_epoch_timings():
    times_itrs = gt.get_times().stamps.itrs
    times = OrderedDict()
    epoch_time = 0
    for key in sorted(times_itrs):
        time = times_itrs[key][-1]
        epoch_time += time
        times['time/{} (s)'.format(key)] = time
    times['time/epoch (s)'] = epoch_time
    times['time/total (s)'] = gt.get_times().total
    return times

class BaseRLAlgorithm(object, metaclass=abc.ABCMeta):
    def __init__(
            self,
            trainer,
            exploration_env,
            evaluation_env,
            exploration_data_collector: DataCollector,
            evaluation_data_collector: DataCollector,
            replay_buffer: ReplayBuffer,
            early_stop_wait_epochs=None,
            early_stop_delta=None,
            early_stop_using_eval=True,
            use_gtimer=False,
    ):
        self.trainer = trainer
        self.expl_env = exploration_env
        self.eval_env = evaluation_env
        self.expl_data_collector = exploration_data_collector
        self.eval_data_collector = evaluation_data_collector
        self.replay_buffer = replay_buffer
        # Keep track of best expl and eval epochs.
        self._best_expl_epoch, self._best_eval_epoch = 0, 0
        self._best_expl_return, self._best_eval_return = float('-inf'), float('-inf')
        # Early stopping data.
        self._early_stop_wait_epochs = early_stop_wait_epochs
        self._early_stop_delta = early_stop_delta
        self._early_stop_using_eval = early_stop_using_eval
        self._early_stop_best_epoch = 0
        self._early_stop_best_return = float('-inf')
        self._should_early_stop = False
        # Gtimer stuff.
        self._use_gtimer = use_gtimer

        self.post_epoch_funcs = []

    def train(self, start_epoch=0):
        self._start_epoch = start_epoch
        self._train()

    def _train(self):
        """
        Train model.
        """
        raise NotImplementedError('_train must implemented by inherited class')

    def _mark_returns(self, epoch, paths, eval_returns=False):
        score = np.mean([np.sum(path['rewards']) for path in paths])
        if eval_returns:
            if score > self._best_eval_return:
                self._best_eval_return = score
                self._best_eval_epoch = epoch
            if not self._early_stop_using_eval:
                return
        else:
            if score > self._best_expl_return:
                self._best_expl_return = score
                self._best_expl_epoch = epoch
            if self._early_stop_using_eval:
                return
        if (self._early_stop_wait_epochs is not None
                and self._early_stop_delta is not None):
            if score - self._early_stop_best_return > self._early_stop_delta:
                self._early_stop_best_return = score
                self._early_stop_best_epoch = epoch
            elif epoch - self._early_stop_best_epoch >= self._early_stop_wait_epochs:
                self._should_early_stop = True

    def _time_stamp(self, name, **kwargs):
        if self._use_gtimer:
            gt.stamp(name, **kwargs)

    def _end_epoch(self, epoch):
        # Comment out snapshot because it doesn't even do anything for me anymore.
        # snapshot = self._get_snapshot()
        # logger.save_itr_params(epoch, snapshot)
        self._save_policy_weights('model.pt')
        if epoch == self._best_eval_epoch:
            self._save_policy_weights('best_eval_model.pt')
        if epoch == self._best_expl_epoch:
            self._save_policy_weights('best_expl_model.pt')
        self._time_stamp('saving', unique=False)
        self._log_stats(epoch)

        self.expl_data_collector.end_epoch(epoch)
        self.eval_data_collector.end_epoch(epoch)
        self.replay_buffer.end_epoch(epoch)
        self.trainer.end_epoch(epoch)

        for post_epoch_func in self.post_epoch_funcs:
            post_epoch_func(self, epoch)

    def _save_policy_weights(self, filename):
        save_path = os.path.join(logger.get_snapshot_dir(), filename)
        torch.save(self.expl_data_collector.policy.state_dict(), save_path)

    def _get_snapshot(self):
        snapshot = {}
        for k, v in self.trainer.get_snapshot().items():
            snapshot['trainer/' + k] = v
        for k, v in self.expl_data_collector.get_snapshot().items():
            snapshot['exploration/' + k] = v
        for k, v in self.eval_data_collector.get_snapshot().items():
            snapshot['evaluation/' + k] = v
        for k, v in self.replay_buffer.get_snapshot().items():
            snapshot['replay_buffer/' + k] = v
        return snapshot

    def _log_stats(self, epoch):
        logger.log("Epoch {} finished".format(epoch), with_timestamp=True)

        """
        Replay Buffer
        """
        logger.record_dict(
            self.replay_buffer.get_diagnostics(),
            prefix='replay_buffer/'
        )

        """
        Trainer
        """
        logger.record_dict(self.trainer.get_diagnostics(), prefix='trainer/')

        """
        Exploration
        """
        logger.record_dict(
            self.expl_data_collector.get_diagnostics(),
            prefix='exploration/'
        )
        expl_paths = self.expl_data_collector.get_epoch_paths()
        if hasattr(self.expl_env, 'get_diagnostics'):
            logger.record_dict(
                self.expl_env.get_diagnostics(expl_paths),
                prefix='exploration/',
            )
        logger.record_dict(
            eval_util.get_generic_path_information(expl_paths),
            prefix="exploration/",
        )
        """
        Evaluation
        """
        logger.record_dict(
            self.eval_data_collector.get_diagnostics(),
            prefix='evaluation/',
        )
        eval_paths = self.eval_data_collector.get_epoch_paths()
        if hasattr(self.eval_env, 'get_diagnostics'):
            logger.record_dict(
                self.eval_env.get_diagnostics(eval_paths),
                prefix='evaluation/',
            )
        logger.record_dict(
            eval_util.get_generic_path_information(eval_paths),
            prefix="evaluation/",
        )

        """
        Misc
        """
        self._time_stamp('logging', unique=False)
        if self._use_gtimer:
            logger.record_dict(_get_epoch_timings())
        logger.record_tabular('Epoch', epoch)
        snapshot = logger.get_table_dict()
        conv_snapshot = dict([a.replace('(','-').replace(')','-'), float(x)] for a, x in snapshot.items())
        mlflow.log_metrics(conv_snapshot, step=epoch)
        mlflow.log_artifacts(logger._snapshot_dir)
        logger.dump_tabular(with_prefix=False, with_timestamp=False)

    @abc.abstractmethod
    def training_mode(self, mode):
        """
        Set training mode to `mode`.
        :param mode: If True, training will happen (e.g. set the dropout
        probabilities to not all ones).
        """
        pass


from flask import Blueprint,render_template,request,session,make_response,redirect
from models import Zhilianjob,Bossjob,Lagoujob
import json
import datetime
from main import auth
companys = Blueprint('companys', __name__)

@companys.route('/companys/lagouweb',methods=["GET"])
@auth.require_login
def companys_lagou():
    if request.method == 'GET':
        return render_template('web_lagou.html')

@companys.route('/companys/bossweb',methods=["GET"])
@auth.require_login
def companys_boss():
    if request.method == 'GET':
        return render_template('web_boss.html')

@companys.route('/companys/zhilianweb',methods=["GET"])
@auth.require_login
def companys_zhilian():
    if request.method == 'GET':
        return render_template('web_zhilian.html')

@companys.route('/companys/zhilianweb_datas',methods=["GET"])
@auth.require_login
def companys_zhilian_dats():
    datas_zhilian = []
    zhilianjobs = Zhilianjob.query.filter(Zhilianjob.status=='0').all()
    for i in zhilianjobs:
         datas_zhilian.append([i.jobname,i.companyname,i.companytype,i.degree,i.money,i.worktime,i.address,i.platform,i.releasetime,i.id])
    data = {}
    data['data'] = datas_zhilian
    return json.dumps(data)

@companys.route('/companys/lagouweb_datas',methods=["GET"])
@auth.require_login
def companys_lagou_dats():
    datas_lagou = []
    lagoujobs = Lagoujob.query.filter(Lagoujob.status=='0').all()
    for i in lagoujobs:
        datas_lagou.append([i.jobname,i.companyname,i.companytype,i.degree,i.money,i.worktime,i.address,i.platform,i.releasetime,i.id])
    data = {}
    data['data'] = datas_lagou
    return json.dumps(data)

@companys.route('/companys/bossweb_datas',methods=["GET"])
@auth.require_login
def companys_boss_dats():
    datas_boss = []
    bossjobs = Bossjob.query.filter(Bossjob.status=='0').all()
    for i in bossjobs:
        datas_boss.append([i.jobname,i.companyname,i.companytype,i.degree,i.money,i.worktime,i.address,i.platform,i.releasetime,i.id])
    data = {}
    data['data'] = datas_boss
    return json.dumps(data)


# Create a simple String calculator with a method int Add(string numbers)
# The method can take 0, 1 or 2 numbers, and will return their sum
# python program

def add(list):
    if len(list) > 2:
        raise('More than two numbers can not be added')
    elif len(list) == 0:
        return 0
    else:
        sum = 0
        for n in list:
            if not int(n):
                raise('Input type needs to be number')
            elif int(n) < 0:
                raise('Negatives not allowed')
            elif int(n) > 1000:
                n = 0
                sum += int(n)
            else:
                sum += int(n)
        return sum


__all__ = ['remove_from_system_path', 'add_to_system_path', 'broadcast_environment_settings_change']

import sys
import os, ctypes
from os import path
from ctypes import wintypes
if sys.version_info[0] >= 3:
    import winreg as reg
else:
    import _winreg as reg

HWND_BROADCAST = 0xffff
WM_SETTINGCHANGE = 0x001A
SMTO_ABORTIFHUNG = 0x0002
SendMessageTimeout = ctypes.windll.user32.SendMessageTimeoutW
SendMessageTimeout.restype = None #wintypes.LRESULT
SendMessageTimeout.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM,
            wintypes.LPCWSTR, wintypes.UINT, wintypes.UINT, ctypes.POINTER(wintypes.DWORD)]

def sz_expand(value, value_type):
    if value_type == reg.REG_EXPAND_SZ:
        return reg.ExpandEnvironmentStrings(value)
    else:
        return value

def remove_from_system_path(pathname, allusers=True, path_env_var='PATH'):
    """Removes all entries from the path which match the value in 'pathname'

       You must call broadcast_environment_settings_change() after you are finished
       manipulating the environment with this and other functions.

       For example,
         # Remove Anaconda from PATH
         remove_from_system_path(r'C:\Anaconda')
         broadcast_environment_settings_change()
    """
    pathname = path.normcase(path.normpath(pathname))

    envkeys = [(reg.HKEY_CURRENT_USER, r'Environment')]
    if allusers:
        envkeys.append((reg.HKEY_LOCAL_MACHINE,
            r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment'))
    for root, keyname in envkeys:
        key = reg.OpenKey(root, keyname, 0,
                reg.KEY_QUERY_VALUE|reg.KEY_SET_VALUE)
        reg_value = None
        try:
            reg_value = reg.QueryValueEx(key, path_env_var)
        except WindowsError:
            # This will happen if we're a non-admin install and the user has
            # no PATH variable.
            reg.CloseKey(key)
            continue

        try:
            any_change = False
            results = []
            for v in reg_value[0].split(os.pathsep):
                vexp = sz_expand(v, reg_value[1])
                # Check if the expanded path matches the
                # requested path in a normalized way
                if path.normcase(path.normpath(vexp)) == pathname:
                    any_change = True
                else:
                    # Append the original unexpanded version to the results
                    results.append(v)

            modified_path = os.pathsep.join(results)
            if any_change:
                reg.SetValueEx(key, path_env_var, 0, reg_value[1], modified_path)
        except:
            # If there's an error (e.g. when there is no PATH for the current
            # user), continue on to try the next root/keyname pair
            reg.CloseKey(key)

def add_to_system_path(paths, allusers=True, path_env_var='PATH'):
    """Adds the requested paths to the system PATH variable.

       You must call broadcast_environment_settings_change() after you are finished
       manipulating the environment with this and other functions.

    """
    # Make sure it's a list
    if not issubclass(type(paths), list):
        paths = [paths]

    # Ensure all the paths are valid before we start messing with the
    # registry.
    new_paths = None
    for p in paths:
        p = path.abspath(p)
        if not path.isdir(p):
            raise RuntimeError(
                'Directory "%s" does not exist, '
                'cannot add it to the path' % p
            )
        if new_paths:
            new_paths = new_paths + os.pathsep + p
        else:
            new_paths = p

    if allusers:
        # All Users
        root, keyname = (reg.HKEY_LOCAL_MACHINE,
            r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment')
    else:
        # Just Me
        root, keyname = (reg.HKEY_CURRENT_USER, r'Environment')

    key = reg.OpenKey(root, keyname, 0,
            reg.KEY_QUERY_VALUE|reg.KEY_SET_VALUE)

    reg_type = None
    reg_value = None
    try:
        try:
            reg_value = reg.QueryValueEx(key, path_env_var)
        except WindowsError:
            # This will happen if we're a non-admin install and the user has
            # no PATH variable; in which case, we can write our new paths
            # directly.
            reg_type = reg.REG_EXPAND_SZ
            final_value = new_paths
        else:
            reg_type = reg_value[1]
            # If we're an admin install, put us at the end of PATH.  If we're
            # a user install, throw caution to the wind and put us at the
            # start.  (This ensures we're picked up as the default python out
            # of the box, regardless of whether or not the user has other
            # pythons lying around on their PATH, which would complicate
            # things.  It's also the same behavior used on *NIX.)
            if allusers:
                final_value = reg_value[0] + os.pathsep + new_paths
            else:
                final_value = new_paths + os.pathsep + reg_value[0]

        reg.SetValueEx(key, path_env_var, 0, reg_type, final_value)

    finally:
        reg.CloseKey(key)

def broadcast_environment_settings_change():
    """Broadcasts to the system indicating that master environment variables have changed.

    This must be called after using the other functions in this module to
    manipulate environment variables.
    """
    SendMessageTimeout(HWND_BROADCAST, WM_SETTINGCHANGE, 0, u'Environment',
                SMTO_ABORTIFHUNG, 5000, ctypes.pointer(wintypes.DWORD()))

def main():
    if len(sys.argv) < 3:
        sys.exit("Too few arguments: {}".format(sys.argv))
    elif len(sys.argv) > 3:
        sys.exit("Too many arguments: {}".format(sys.argv))

    if sys.argv[1] == 'add':
        add_to_system_path(sys.argv[2])
    elif sys.argv[1] == 'add_user':
        add_to_system_path(sys.argv[2], allusers=False)
    elif sys.argv[1] == 'remove':
        remove_from_system_path(sys.argv[2])
    elif sys.argv[1] == 'remove_user':
        remove_from_system_path(sys.argv[2], allusers=False)

    broadcast_environment_settings_change()

if __name__ == '__main__':
    main()


from torch.utils.data import Dataset
from DatasetMethods import DatasetMethods
import numpy as np
import torch
from random import random, randint
from pretrain_question_model import Constants

class PretrainDataSet(Dataset):
    def __init__(self, data_name, sq_len, step, question_size, skill_size):
        self.max_len = sq_len
        self.getPretrainDataSet = GetPretrainDataSet(data_name, sq_len, step, question_size, skill_size, self.max_len)
        self.batch_mask_questions, self.batch_question_label, self.batch_mask_total_skills, self.batch_total_skill_label,\
        self.batch_question, self.batch_total_skills, self.difficult_labels = self.getPretrainDataSet.getAllTensorData()

        assert (self.batch_mask_questions.size(0) == self.batch_mask_total_skills.size(0) == self.batch_question.size(0) == self.batch_total_skills.size(0))
        self.length = self.batch_mask_questions.size(0)

    def __len__(self):
        return self.length

    def __getitem__(self, idx):
        batch_mask_q = self.batch_mask_questions[idx]
        batch_q_label = self.batch_question_label[idx]
        batch_mask_s = self.batch_mask_total_skills[idx]
        batch_s_label = self.batch_total_skill_label[idx]
        batch_question = self.batch_question[idx]
        batch_total_skill = self.batch_total_skills[idx]
        batch_diff_label = self.difficult_labels[idx]

        return batch_mask_q, batch_q_label, batch_mask_s, batch_s_label, batch_question, batch_total_skill, batch_diff_label

class GetPretrainDataSet:
    def __init__(self, data_name, sq_len, step, question_size, skill_size, max_len):
        self.max_len = max_len
        self.question_size = question_size
        self.skill_size = skill_size
        self.dataset_methods = DatasetMethods(data_name, sq_len, step)

    def getAllTensorData(self):
        train_data, shuffle_total_skills, single_total_skills, difficult_dict = self.dataset_methods. \
            getTrainDataAndSkillDataAndDifficultDict()
        batch_mask_q, batch_q_label = self.getTrainDataTensor(train_data, self.question_size, self.max_len)
        batch_mask_s, batch_s_label = self.getTotalSkillDataTensor(shuffle_total_skills, self.skill_size)
        batch_question, batch_total_skill, batch_difficult_labels = self.getDifficultTrainDataTensor(train_data, single_total_skills, difficult_dict, self.max_len)

        return batch_mask_q, batch_q_label, batch_mask_s, batch_s_label, batch_question, batch_total_skill, batch_difficult_labels

    def getTrainDataTensor(self, data, question_size, max_len):
        # =======mask===========
        mask_questions, question_label = self.maskedTrainData(data, question_size)

        b_mask_questions = np.array([e + [Constants.PAD] * (max_len - len(e)) for e in mask_questions], dtype=float)
        b_question_label = np.array([e + [Constants.PAD] * (max_len - len(e)) for e in question_label], dtype=float)

        batch_mask_questions = torch.from_numpy(b_mask_questions).long()
        batch_question_label = torch.from_numpy(b_question_label).long()

        return batch_mask_questions, batch_question_label

    def maskedTrainData(self, data, question_size):
        mask_questions = []
        question_labels = []
        quest, skill = data
        data_zip = zip(quest, skill)
        for data_ in data_zip:
            mask_q, q_label = GetPretrainDataSet.maskedSingleData(data_, question_size, 'question')
            mask_questions.append(mask_q)
            question_labels.append(q_label)

        return mask_questions, question_labels

    def getTotalSkillDataTensor(self, data, skill_size):
        mask_total_skills = []
        total_skill_labels = []
        for data_ in data:
            mask_s, s_label = GetPretrainDataSet.maskedSingleData(data_, skill_size, 'skill')
            mask_total_skills.append(mask_s)
            total_skill_labels.append(s_label)

        b_mask_total_skills = np.array(mask_total_skills, dtype=float)
        b_total_skill_labels = np.array(total_skill_labels, dtype=float)

        batch_mask_total_skills = torch.from_numpy(b_mask_total_skills).long()
        batch_total_skill_labels = torch.from_numpy(b_total_skill_labels).long()

        return batch_mask_total_skills, batch_total_skill_labels

    def getDifficultTrainDataTensor(self, data, single_total_skills, difficult_dict, max_len):
        questions, total_skills, difficult_labels = self.getTrainDifficultData(data, single_total_skills, difficult_dict)
        b_questions = np.array([e + [Constants.PAD] * (max_len - len(e)) for e in questions], dtype=float)
        b_total_skills = np.array(total_skills, dtype=float)
        b_difficult_labels = np.array([e + [Constants.PAD_C] * (max_len - len(e)) for e in difficult_labels], dtype=float)

        batch_questions = torch.from_numpy(b_questions).long()
        batch_total_skills = torch.from_numpy(b_total_skills).long()
        batch_difficult_labels = torch.from_numpy(b_difficult_labels).float()

        return batch_questions, batch_total_skills, batch_difficult_labels

    def getTrainDifficultData(self, data, single_total_skills, difficult_dict):
        d_question = []
        d_skill = []
        difficult_labels = []
        quest, skill = data
        data_zip = zip(quest, skill)
        for data_ in data_zip:
            d_ques, dif_label = GetPretrainDataSet.MaskedSingleDifficult(data_, difficult_dict)
            d_question.append(d_ques)
            d_skill.append(single_total_skills)
            difficult_labels.append(dif_label)
        return d_question, d_skill, difficult_labels

    @staticmethod
    def MaskedSingleDifficult(data, difficult):
        question = []
        difficult_label = []
        quest, skill = data
        assert len(quest) == len(skill)
        num = len(quest)
        for i in range(num):
            question.append(quest[i])
            difficult_label.append(float(difficult[str(quest[i])]))
        return question, difficult_label

    @staticmethod
    def maskedSingleData(data, size, mask_name):
        mask_data = []
        data_label = []
        threshold = 0.15
        num = 0
        data_ = None
        if mask_name == 'question':
            quest, skill = data
            assert len(quest) == len(skill)
            num = len(quest)
            data_ = quest
        if mask_name == 'skill':
            num = len(data)
            data_ = data
        for i in range(num):
            r = random()
            if r < threshold:  # mask 15% of all token
                if r < threshold * 0.8:  # 80% replace [mask]=size+3
                    mask_data.append(Constants.mask_Q + size)
                elif r < threshold * 0.9:  # 10% random (1, size)
                    random_q_index = randint(1, size)
                    mask_data.append(random_q_index)
                else:
                    mask_data.append(data_[i])
                data_label.append(data_[i])
            else:
                mask_data.append(data_[i])
                data_label.append(Constants.PAD)

        return mask_data, data_label

# if __name__ == '__main__':
#     training_data = torch.utils.data.DataLoader(PretrainDataSet('2009', 256, 5, 16891, 101), batch_size=128)
#     for b in training_data:
#         batch_mask_q, batch_q_label, batch_mask_s, batch_s_label, batch_question, batch_total_skill, batch_diff_label = b
#         print(batch_question[:2])
#         print(batch_total_skill[:2])
#         print()
#         break


"""
V1 Beta Common Places
"""

from typing import Dict
from ....internals.get_resources import GetResources
from ....internals.list_resources import ListResources
from .... import exceptions

class Places(GetResources):
    """Class for Places APIs."""

    PLACES = "/places"

    def __init__(self, version, role):
        super().__init__(version, role, Places.PLACES)

    def get_by_id(self, id_) -> Dict:
        api_url = self.endpoint

        if not id_:
            raise exceptions.WrongParamsError("Invalid Parameters")

        query_params = {"id": id_}
        response = self.api.make_get_request(
            api_url=api_url,
            query_params=query_params,
        )

        if id_ and response["count"] > 1:
            raise exceptions.MultipleObjectReturned("Multiple Objects returned")

        return response.get("data")[0]

class PlacesAutocomplete(ListResources):
    """Class for Places autocomplete APIs."""

    PLACESAUTOCOMPLETE = "/places/autocomplete"

    def __init__(self, version, role):
        super().__init__(version, role, PlacesAutocomplete.PLACESAUTOCOMPLETE)

    def list(self, q, types=None, location=None) -> Dict:
        """
        Get Resources
        :param query_params:
        :return: List of Places Objects
        """
        if not q:
            raise exceptions.WrongParamsError("Invalid Parameters")
        query_params = {
            'q': q
        }

        if types:
            query_params['types'] = types

        if location:
            query_params['location'] = location

        return self.api.make_get_request(
            api_url=self.endpoint,
            query_params=query_params
        )