#!/usr/bin/env python
# pylint: disable=invalid-name,line-too-long,missing-function-docstring
"""sigtoaddr: Convert a Bitcoin signature into an address eg Bitcoin Segwit or Altcoin
"""

import base64
import sys
import urllib.request
from base64 import b64decode
from binascii import hexlify
from collections import namedtuple
from hashlib import sha256 as hashlib_sha256
from json import load as json_load
from typing import Optional

import click
from bitcoin.core import Hash160
from bitcoin.core.key import CPubKey
from bitcoin.core.script import CScript, OP_0
from bitcoin.signmessage import BitcoinMessage
from bitcoin.wallet import P2PKHBitcoinAddress, P2WPKHBitcoinAddress
from ecdsa import VerifyingKey, SECP256k1, ellipticcurve, numbertheory
from ecdsa.util import sigdecode_string
from eth_keys import KeyAPI as ETHKeyAPI
from litecoinutils.keys import PublicKey as LitecoinPublicKey
from litecoinutils.keys import add_magic_prefix as litecoin_add_magic_prefix
from litecoinutils.setup import setup as litecoin_setup
# Import pycoin used by the opendime trustme.py to verify the verify.txt. Rather than duplicating the logic.
from pycoin.contrib import msg_signing
from pycoin.ecdsa.secp256k1 import secp256k1_generator
from pycoin.symbols.btc import network as bitcoin_network
from sympy.ntheory import sqrt_mod

# VerifiedMessage: Simple wrapper for a verified bitcoin signature
VerifiedMessage = namedtuple('VerifiedMessage', ['address', 'signature', 'message', 'is_valid', 'public_key'])

# Addresses: Container for addresses derived from the public key
Addresses = namedtuple('Addresses', ['original', 'bitcoin_p2pkh', 'bitcoin_p2pkh_compressed', 'bitcoin_p2wpkh',
                                     'ethereum', 'litecoin_p2pkh', 'litecoin_p2pkh_compressed', 'litecoin_p2wpkh',
                                     'uncompressed', 'compressed'])

def compress_public_key(uncompressed_public_key_hex: str) -> str:
    if uncompressed_public_key_hex[0:2] in ('02', '03'):
        # Already compressed
        return uncompressed_public_key_hex

    xi = int(uncompressed_public_key_hex[2:66], 16)
    x = int.to_bytes(xi, length=32, byteorder='big', signed=False)
    yi = int(uncompressed_public_key_hex[66:], 16)
    header = b'\x03' if yi & 1 else b'\x02'
    return (header + x).hex()

def decompress_public_key(compressed_public_key_hex) -> str:
    if compressed_public_key_hex[0:2] == '04':
        # Already uncompressed
        return compressed_public_key_hex

    p = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F
    compressed_public_key_bytes = bytes.fromhex(compressed_public_key_hex)
    x = int.from_bytes(compressed_public_key_bytes[1:33], byteorder='big')
    y_sq = (pow(x, 3, p) + 7) % p
    y = pow(y_sq, (p + 1) // 4, p)
    if y % 2 != compressed_public_key_bytes[0] % 2:
        y = p - y
    y = y.to_bytes(32, byteorder='big')
    return (b'\x04' + compressed_public_key_bytes[1:33] + y).hex()

def verify_message(address: str, signature: str, message: BitcoinMessage) -> VerifiedMessage:
    if address.startswith('L'):
        return verify_message_litecoin(address, signature, message)

    sig_bytes = base64.b64decode(signature)
    msg_hash = message.GetHash()
    pubkey = CPubKey.recover_compact(msg_hash, sig_bytes)

    return VerifiedMessage(address, signature, message, str(P2PKHBitcoinAddress.from_pubkey(pubkey)) == str(address), pubkey)

def verify_textfile(addr: str, filename: str) -> VerifiedMessage:
    ms = msg_signing.MessageSigner(bitcoin_network, secp256k1_generator)
    with open(filename, encoding='ascii') as fp:
        msg = fp.read().strip()
    msg, sig_addr, sig = ms.parse_signed_message(msg)

    # force newlines to what we need.
    if '\r' not in msg:
        msg = msg.replace('\n', '\r\n')

    if addr and sig_addr != addr:
        raise ValueError('Not signed with correct address')

    if sig_addr.startswith('1'):
        # do math to verify msg
        ok = ms.verify_message(sig_addr, sig, msg)
        if not ok:
            raise ValueError('Invalid or incorrectly-signed verify.txt found.')
        msg = BitcoinMessage(msg)

    return verify_message(sig_addr, sig, msg)

def verify_message_litecoin(address: str, signature: str, message: str) -> VerifiedMessage:  # pylint: disable=too-many-locals
    # Copied from litecoinutils.key PublicKey class verify_message function so we can recover the public key
    """Creates a public key from a message signature and verifies message

    Bitcoin uses a compact format for message signatures (for tx sigs it
    uses normal DER format). The format has the normal r and s parameters
    that ECDSA signatures have but also includes a prefix which encodes
    extra information. Using the prefix the public key can be
    reconstructed from the signature.

    |  Prefix values:
    |      27 - 0x1B = first key with even y
    |      28 - 0x1C = first key with odd y
    |      29 - 0x1D = second key with even y
    |      30 - 0x1E = second key with odd y

    If key is compressed add 4 (31 - 0x1F, 32 - 0x20, 33 - 0x21, 34 - 0x22 respectively)

    Raises
    ------
    ValueError
        If signature is invalid
    """
    sig = b64decode(signature.encode('utf-8'))
    if len(sig) != 65:
        raise ValueError('Invalid signature size')

    # get signature prefix, compressed and recid (which key is odd/even)
    prefix = sig[0]
    if prefix < 27 or prefix > 35:
        return False
    if prefix >= 31:
        compressed = True
        recid = prefix - 31
    else:
        compressed = False
        recid = prefix - 27

    # create message digest -- note double hashing
    message_magic = litecoin_add_magic_prefix(message)
    message_digest = hashlib_sha256( hashlib_sha256(message_magic).digest() ).digest()

    #
    # use recid, r and s to get the point in the curve
    #

    # ECDSA curve using secp256k1 is defined by: y**2 = x**3 + 7
    # This is done modulo p which (secp256k1) is:
    # p is the finite field prime number and is equal to:
    # 2^256 - 2^32 - 2^9 - 2^8 - 2^7 - 2^6 - 2^4 - 1
    # Note that we could also get that from ecdsa lib from the curve, e.g.:
    # SECP256k1.__dict__['curve'].__dict__['_CurveFp__p']
    _p = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F
    # Curve's a and b are (y**2 = x**3 + a*x + b)
    _a = 0x0000000000000000000000000000000000000000000000000000000000000000
    _b = 0x0000000000000000000000000000000000000000000000000000000000000007
    # Curve's generator point is:
    _Gx = 0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798
    _Gy = 0x483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8
    # prime number of points in the group (the order)
    _order = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141

    # The ECDSA curve (secp256k1) is:
    # Note that we could get that from ecdsa lib, e.g.:
    # SECP256k1.__dict__['curve']
    _curve = ellipticcurve.CurveFp( _p, _a, _b )

    # The generator base point is:
    # Note that we could get that from ecdsa lib, e.g.:
    # SECP256k1.__dict__['generator']
    _G = ellipticcurve.Point( _curve, _Gx, _Gy, _order )

    # get signature's r and s
    r,s = sigdecode_string(sig[1:], _order)

    # ger R's x coordinate
    x = r + (recid // 2) * _order

    # get R's y coordinate (y**2 = x**3 + 7)
    y_values = sqrt_mod( (x**3 + 7) % _p, _p, True )
    if (y_values[0] - recid) % 2 == 0:
        y = y_values[0]
    else:
        y = y_values[1]

    # get R (recovered ephemeral key) from x,y
    R = ellipticcurve.Point(_curve, x, y, _order)

    # get e (hash of message encoded as big integer)
    e = int(hexlify(message_digest), 16)

    # compute public key Q = r^-1 (sR - eG)
    # because Point substraction is not defined we will instead use:
    # Q = r^-1 (sR + (-eG) )
    minus_e = -e % _order
    inv_r = numbertheory.inverse_mod(r, _order)
    Q = inv_r * ( s*R + minus_e*_G )

    # instantiate the public key and verify message
    public_key = VerifyingKey.from_public_point( Q, curve = SECP256k1 )
    key_hex = hexlify(public_key.to_string()).decode('utf-8')
    pubkey = LitecoinPublicKey.from_hex('04' + key_hex)

    valid = True
    if not pubkey.verify(signature, message):
        valid = False

    # confirm that the address provided corresponds to that public key
    if pubkey.get_address(compressed=compressed).to_string() != address:
        valid = False

    return VerifiedMessage(address, signature, message, valid, CPubKey.fromhex(compress_public_key('04' + key_hex)))

def public_key_to_bitcoin_p2pkh(public_key: CPubKey, compressed=False) -> str:
    if compressed:
        public_key = CPubKey.fromhex(compress_public_key(public_key.hex()))
    else:
        public_key = CPubKey.fromhex(decompress_public_key(public_key.hex()))
    return str(P2PKHBitcoinAddress.from_pubkey(public_key))

def public_key_to_bitcoin_p2wpkh(public_key: CPubKey) -> str:
    compressed_public_key = CPubKey.fromhex(compress_public_key(public_key.hex()))
    scriptPubKey = CScript([OP_0, Hash160(compressed_public_key)])
    return str(P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey))

def public_key_to_ethereum(public_key: CPubKey) -> str:
    uncompressed_public_key = CPubKey.fromhex(decompress_public_key(public_key.hex()))
    eth_pk = ETHKeyAPI.PublicKey(bytes.fromhex(uncompressed_public_key.hex()[2:]))
    return eth_pk.to_checksum_address()

def public_key_to_litecoin(public_key: CPubKey, compressed=False) -> str:
    litecoin_setup('mainnet')
    compressed_public_key = CPubKey.fromhex(compress_public_key(public_key.hex()))
    litecoin_public_key = LitecoinPublicKey(compressed_public_key.hex())
    return litecoin_public_key.get_address(compressed=compressed).to_string()

def public_key_to_litecoin_p2wpkh(public_key: CPubKey) -> str:
    litecoin_setup('mainnet')
    compressed_public_key = CPubKey.fromhex(compress_public_key(public_key.hex()))
    litecoin_public_key = LitecoinPublicKey(compressed_public_key.hex())
    return litecoin_public_key.get_segwit_address().to_string()

def get_addresses(verified_message: VerifiedMessage) -> Addresses:
    return Addresses(
        original=verified_message.address,
        bitcoin_p2pkh=public_key_to_bitcoin_p2pkh(verified_message.public_key, compressed=False),
        bitcoin_p2pkh_compressed=public_key_to_bitcoin_p2pkh(verified_message.public_key, compressed=True),
        bitcoin_p2wpkh=public_key_to_bitcoin_p2wpkh(verified_message.public_key),
        ethereum=public_key_to_ethereum(verified_message.public_key),
        litecoin_p2pkh=public_key_to_litecoin(verified_message.public_key, compressed=False),
        litecoin_p2pkh_compressed=public_key_to_litecoin(verified_message.public_key, compressed=True),
        litecoin_p2wpkh=public_key_to_litecoin_p2wpkh(verified_message.public_key),
        uncompressed=decompress_public_key(verified_message.public_key.hex()),
        compressed=compress_public_key(verified_message.public_key.hex()),
    )

def check_balance(address: str, fiat: str = 'usd') -> (Optional[float], Optional[float]):
    """check_balance: Basic usage of bitlaps API
    returns (float, float) eg Bitcoin amount (whole Bitcoin not Sats) and Fiat amount
    """
    # Using bitlaps https://developer.bitaps.com/blockchain which is free for 15 reqs in 5s currently
    currency = 'btc'  # if address[0] in ('1', '3', 'b'):
    if address[0].lower() == 'l':
        currency = 'ltc'
    if address[0] == '0':
        currency = 'eth'

    url = f'https://api.bitaps.com/{currency}/v1/blockchain/address/state/{address}'

    data = None
    with urllib.request.urlopen(url) as wp:
        try:
            data = json_load(wp)
        except:
            print('Error reading bitlaps API')
            return (None, None)

    balance = 0
    try:
        balance = data['data']['balance']
    except AttributeError:
        print('Error balance not found')

    if balance:
        if currency == 'eth':
            balance = balance * 0.000000000000000001
        else:
            # Bitcoin & Litecoin has 8 decimal places
            balance = balance * 0.00000001

    price_url = f'https://api.bitaps.com/market/v1/ticker/{currency}{fiat}'

    data = None
    with urllib.request.urlopen(price_url) as wp:
        try:
            data = json_load(wp)
        except:
            print('Error reading bitlaps API')
            return (None, None)

    fiat_value = None
    try:
        fiat_value = balance * float(data['data']['last'])
    except AttributeError:
        print('Error last price not found')

    return (balance, fiat_value)

def pretty_print_addresses(addresses: Addresses, show_balance: bool = False):
    friendly_name = {
        'bitcoin_p2pkh': 'Bitcoin P2PKH\t\t\t',
        'bitcoin_p2pkh_compressed': 'Bitcoin P2PKH (Compressed)\t',
        'bitcoin_p2wpkh': 'Bitcoin P2WPKH\t\t',
        'ethereum': 'Ethereum\t\t\t',
        'litecoin_p2pkh': 'Litecoin P2PKH\t\t',
        'litecoin_p2pkh_compressed': 'Litecoin P2PKH (Compressed)\t',
        'litecoin_p2wpkh': 'Litecoin P2WPKH\t\t',
    }

    print('Addresses for Opendime:\t', addresses.original)
    for name, value in addresses._asdict().items():
        if name not in friendly_name:
            continue

        pad = balance = spacer = fiat = ''
        if show_balance:
            pad = '\t\t'
            if name in ('bitcoin_p2wpkh', 'ethereum', 'litecoin_p2wpkh'):
                pad = '\t'

            spacer = ' = $'
            balance, fiat = check_balance(addresses.ethereum)

        print(f'- {friendly_name[name]} {value}{pad}{balance}{spacer}{fiat}')

def test_get_addresses(address: str, verifytxt: str, signature: str, message: Optional[BitcoinMessage], expected: Addresses):
    if verifytxt:
        verified_message = verify_textfile(address, verifytxt)
    else:
        verified_message = verify_message(address, signature, message)
    assert verified_message.is_valid

    addresses = get_addresses(verified_message)

    assert verified_message.public_key.hex() in (addresses.compressed, addresses.uncompressed)
    assert addresses == expected

def tests():
    test_get_addresses('1Nu1QpfegiGmqHS6YZxkaiGpnqAUXvZz2f', None,
        'HwPlEOxTxs62ruMHZvamv0wmUlbbaY/2ZSqw9Hpdw+FWfgXuSxQ9x55ceSiFyvnlpiZjt+KIhSYnhGnCv8iDe5o=', BitcoinMessage('Hello World!'),
        Addresses(
            original='1Nu1QpfegiGmqHS6YZxkaiGpnqAUXvZz2f',
            bitcoin_p2pkh='1GLfgL9yKVTRRG1D4fdKkEuEQqAE7ob1eB',
            bitcoin_p2pkh_compressed='1Nu1QpfegiGmqHS6YZxkaiGpnqAUXvZz2f',
            bitcoin_p2wpkh='bc1q7qcf63rtp20dsalcwmceucxs0kwn75l95nsxjf',
            ethereum='0x5D0a9F69035Be4275204f9eBbd5cC049e42429c6',
            litecoin_p2pkh='LaZcwYToQ9hUg4hNEocd2Fxzd3XWEMFnQ5',
            litecoin_p2pkh_compressed='Lh7xg2yUmNWq668Fihx3rjLb13XkbHuBMQ',
            litecoin_p2wpkh='ltc1q7qcf63rtp20dsalcwmceucxs0kwn75l9s02z2e',
            uncompressed='0471bb3ef523055565dd5f9864047b9fe93efa10151ff4bb3640f7de6dfdd76cea9d5cb2da17d725a835f25971818e54acc1db69e4866ea23c9dc33f57cb286315',
            compressed='0371bb3ef523055565dd5f9864047b9fe93efa10151ff4bb3640f7de6dfdd76cea',
        )
    )

    test_get_addresses('1Mmg2eycKHomhjAikEAVehHpCSHTREhLfR',
        './verify.txt_tips', None, None,
        Addresses(
            original='1Mmg2eycKHomhjAikEAVehHpCSHTREhLfR',
            bitcoin_p2pkh='1Mmg2eycKHomhjAikEAVehHpCSHTREhLfR',
            bitcoin_p2pkh_compressed='129azYLPaG55Kb7z1TgvBbj6nRjYFcNMqE',
            bitcoin_p2wpkh='bc1qpjtaggfhsnhkcyg967k3jmsxtm5hzg72q8ejr5',
            ethereum='0x76270d9D9afC0cf4EbfFBafE6401E01cb0F021Ce',
            litecoin_p2pkh='LfzdHsHSPx3pxXrsvN9nviMaQeejdnT81s',
            litecoin_p2pkh_compressed='LLNYFkeDevK8aPp9BbgDTcnrze6pQc7D6s',
            litecoin_p2wpkh='ltc1qpjtaggfhsnhkcyg967k3jmsxtm5hzg72ymrkmy',
            uncompressed='04f27deec87586e475f828cb3cd34d2a02a674c204875e91b90ce4ce1e8773289587979932eef0c5f76c5d5fc692db94749e4efba67b692f564190c4b36ca8763a',
            compressed='02f27deec87586e475f828cb3cd34d2a02a674c204875e91b90ce4ce1e87732895',
        )
    )

    test_get_addresses('LLsXEU59RyoMmjgCkUAghxTLr6FXoRCgQT', None,
        'H021r+HxbXZo2Vkuyq0D/pfz8kllqDzmOzczJXBanIytdsbZKPlg3q1NhytyLXp03DQa//0zoOjoJfVUjZORql8=', 'Hello World',
        Addresses(
            original='LLsXEU59RyoMmjgCkUAghxTLr6FXoRCgQT',
            bitcoin_p2pkh='1FZ33nWeZFk2qv8PnCL2VR2wA3KnGchNbZ',
            bitcoin_p2pkh_compressed='12eZyFmKMKZJWvz3aLBPRwPadstFaFGKAF',
            bitcoin_p2wpkh='bc1qzgfsnjuz7972nd9jtqh26qc00ltjns3tjdewkt',
            ethereum='0x33a5f5ff5d6Aeb3152d223C5407C1e71Bb202C76',
            litecoin_p2pkh='LZmzJzpUduz66ipYxLKKmS6hNFh4MgNPKy',
            litecoin_p2pkh_compressed='LLsXEU59RyoMmjgCkUAghxTLr6FXoRCgQT',
            litecoin_p2wpkh='ltc1qzgfsnjuz7972nd9jtqh26qc00ltjns3tk3r2wm',
            uncompressed='04a2e8f5aa9c46242cdc6463adac2ef8e6bb8b17202c06d17c647066ed143535ac1f93e66cc499170185ec79b2ef5c04119282544fea4c8072ff87711e13597bcf',
            compressed='03a2e8f5aa9c46242cdc6463adac2ef8e6bb8b17202c06d17c647066ed143535ac',
        )
    )

    test_get_addresses('LhNxvyyxBGv1Z9CKUaYPE5azvFCMnDMbRN',
        './litecoin_verify.txt_tips', None, None,
        Addresses(
            original='LhNxvyyxBGv1Z9CKUaYPE5azvFCMnDMbRN',
            bitcoin_p2pkh='1PA1fmg86cfxJLWAJSZ5x4XEi2q5kDxpBk',
            bitcoin_p2pkh_compressed='17tcs8A77LNzH3QqwdGjdKcVPiB1Ka3c2j',
            bitcoin_p2wpkh='bc1qfwf7s8qrlcjfulqymrrw3mejnwwas9y5wz5v8r',
            ethereum='0xDdb5Fc6f27921669FCd177f6877A69356dAe889C',
            litecoin_p2pkh='LhNxvyyxBGv1Z9CKUaYPE5azvFCMnDMbRN',
            litecoin_p2pkh_compressed='LS7a8LTwBzd3Xr717mG2uLgFbvYHQbbJ64',
            litecoin_p2wpkh='ltc1qfwf7s8qrlcjfulqymrrw3mejnwwas9y527wgln',
            uncompressed='04db8b0bc1bf85c9727d31b97fc7483b2d9bbc85d57f7e2ed8f617c98a96966271a41db637664355f9c490abd73b8e68a62afb1d40913fc1384f9edb2475009b89',
            compressed='03db8b0bc1bf85c9727d31b97fc7483b2d9bbc85d57f7e2ed8f617c98a96966271',
        )
   )

@click.command()
@click.option('--verifytxt', type=click.Path(exists=True, dir_okay=False, readable=True),
              help='Path to OPENDIME/advanced/verify.txt alternative to passing address, signature and message')
@click.option('--address', '-a', help='Bitcoin address. Optional with verify.txt.')
@click.option('--signature', '-s', help='Bitcoin signature (if verify.txt not used)')
@click.option('--message', '-m', help='Bitcoin message (if verify.txt not used)')
@click.option('--balance', '-b', help='Check balance', count=True)
def main(verifytxt, address, signature, message, balance):
    # Run the sanity tests first
    tests()

    if verifytxt:
        try:
            verified_message = verify_textfile(address, verifytxt)
        except ValueError as ex:
            print(f'Unable to verify text file: {ex}')
            sys.exit(1)
    elif address and signature and message:
        verified_message = verify_message(address, signature, BitcoinMessage(message))
    else:
        print('usage: sigtoaddr.py (--verifytxt and optional --address) OR (--address --signature --message)')
        sys.exit(1)

    addresses = get_addresses(verified_message)
    pretty_print_addresses(addresses, balance)

if __name__ == '__main__':
    sys.exit(main())  # pylint: disable=no-value-for-parameter


# https://www.codewars.com/kata/54e6533c92449cc251001667
# Unique in Order

# First Implementation 01/10/2021
def unique_in_order(iterable):
    unique_list=[]
    previous_item=""
    for item in iterable:
        if item != previous_item:
            unique_list.append(item)
        previous_item=item
    return unique_list


from CRM.Processors.Service.BWUsage import show_bw_usage
from CRM.Processors.Service.Dedicated import view_dedicated_service, add_new_dedicate_service, delete_dedicate_service, \
    get_dedicated_service_detail
from CRM.Processors.Service.Disconnect import kill_user
from CRM.Processors.Service.FrmServiceSummery import show_user_service_summery, report_service_problem
from CRM.Processors.Service.FrmTempRecharge import temp_recharge
from CRM.Processors.Service.IPStaticManagement import view_ip_static, add_new_static_ip, delete_static_ip, \
    request_ip_static, delete_user_static_ip, toggle_free_ip
from CRM.Processors.Service.ServiceGroupManagement import assign_service_to_group, \
    assign_package_to_group
from CRM.Processors.Service.ServiceManagement import view_float_service, view_service_formula, add_new_formula, \
    get_formula_detail, remove_formula, add_float_service, get_float_service_detail, remove_basic_service, \
    view_custom_option, add_float_custom_options, get_custom_option_detail, remove_custom_option, get_selected_options, \
    assign_custom_option, view_custom_option_group, add_custom_option_group, remove_custom_option_group, \
    get_custom_option_group, buy_float_service, calculate_float_price, get_service_options, get_hidden_groups, \
    validate_selection, add_float_template, show_all_service, update_normal_service, \
    import_ibs_services, delete_normal_service, get_normal_service_details, get_related_services, set_related_services, \
    assign_service_map, get_service_map_list, create_float_invoice, assign_default_group, view_float_service_discount, \
    get_float_service_discount, delete_float_service_discount, add_float_service_discount, view_service_templates, \
    delete_service_template, view_float_template_options, service_float_get_client_template, \
    service_float_assign_template, user_template_toggle_test, user_template_toggle_public_test, \
    service_kill_current_user, view_float_package_discount, add_float_package_discount, get_float_package_discount, \
    delete_float_package_discount
from CRM.Processors.Service.views import show_all_service_properties, service_group_management, assign_service_to_user, \
    service_switch_old
from CRM.Processors.Service.UpdateUserService import update_user_service

from django.conf.urls import patterns, url
__author__ = 'Administrator'
urlpatterns = patterns('',
                       url(r'^show/all/$', show_all_service, name='show all services'),
                       url(r'^dc_current/$', service_kill_current_user, name='service_kill_current'),
                       url(r'^import/', import_ibs_services, name='service_import_ibs'),
                       url(r'^create/$', update_normal_service, name='create service'),
                       url(r'^j/$', get_normal_service_details, name='service_normal_get'),
                       url(r'^delete/$', delete_normal_service, name='delete service'),
                       url(r'^summery/$', show_user_service_summery, name='user services summery'),
                       url(r'^reports/$', report_service_problem, name='report_service_problem'),
                       url(r'^assign/$', assign_service_to_user, name='assign service to user'),
                       url(r'^switch/(?P<user_id>\d+)/$', service_switch_old,name='service_switch_old'),
                       url(r'^temp/$', temp_recharge, name='temp recharge'),
                       url(r'^update/$', update_user_service, name='update_user_service'),
                       url(r'^ip/show/$', request_ip_static, name='view_ip_static_request'),
                       url(r'^ip/rmu/$', delete_user_static_ip, name='delete_user_static_ip'),
                       url(r'^dc/$', kill_user, name='kill_online_user'),
                       url(r'^graph/$', show_bw_usage, name='show_bw_usage'),
                       url(r'^groups/$', service_group_management, name='service_group_management'),
                       url(r'^groups/pack/$', assign_package_to_group, name='assign_package_to_group'),
                       url(r'^groups/service/$', assign_service_to_group, name='assign_service_to_group'),
                       url(r'^properties/$', show_all_service_properties, name='show_all_service_properties'),
                       url(r'^ip/pool/$', view_ip_static, name='view_ip_statics'),
                       url(r'^ip/pool/add/$', add_new_static_ip, name='add_ip_statics'),
                       url(r'^ip/pool/delete/$', delete_static_ip, name='delete_ip_static'),
                       url(r'^ip/mkf/$', toggle_free_ip, name='toggle_free_ip'),
                       url(r'^float/$', view_float_service, name='service_view_float'),
                       url(r'^float/discount/$', view_float_service_discount, name='service_float_discount_view'),
                       url(r'^float/discount/j/$', get_float_service_discount, name='service_float_discount_single'),
                       url(r'^float/discount/rm/$', delete_float_service_discount, name='service_float_discount_delete'),
                       url(r'^float/discount/add/$', add_float_service_discount, name='service_float_discount_add'),
                       url(r'^float/default/$', assign_default_group, name='service_float_assign_default'),
                       url(r'^float/related/j/$', get_related_services, name='service_float_related_get'),
                       url(r'^float/related/add/$', set_related_services, name='service_float_related_set'),
                       url(r'^float/add/$', add_float_service, name='service_float_add'),
                       url(r'^float/j/$', get_float_service_detail, name='service_float_detail'),
                       url(r'^float/rm/$', remove_basic_service, name='service_float_remove'),
                       url(r'^float/formula/$', view_service_formula, name='service_float_view_formula'),
                       url(r'^float/formula/add/$', add_new_formula, name='service_float_add_formula'),
                       url(r'^float/formula/j/$', get_formula_detail, name='service_float_formula_view_json'),
                       url(r'^float/formula/rm/$', remove_formula, name='service_float_formula_rm'),
                       url(r'^float/option/$', view_custom_option, name='service_float_option_view'),
                       url(r'^float/option/add/$', add_float_custom_options, name='service_float_option_add'),
                       url(r'^float/option/j/$', get_custom_option_detail, name='service_float_option_get'),
                       url(r'^float/option/rm/$', remove_custom_option, name='service_float_option_remove'),
                       url(r'^float/option/service/add/$', assign_service_map, name='service_float_option_add_map'),
                       url(r'^float/option/service/j/$', get_service_map_list, name='service_float_option_view_map'),
                       url(r'^float/options/$', get_selected_options, name='service_float_options_get'),
                       url(r'^float/option/group/$', view_custom_option_group, name='service_float_option_group_view'),
                       url(r'^float/option/group/add/$', add_custom_option_group,
                           name='service_float_option_group_add'),
                       url(r'^float/option/group/rm/$', remove_custom_option_group,
                           name='service_float_option_group_remove'),
                       url(r'^float/option/group/j/$', get_custom_option_group,
                           name='service_float_option_group_view_single'),
                       url(r'^float/assign/$', assign_custom_option, name='service_float_assign_option'),
                       url(r'^float/buy/$', buy_float_service, name='service_float_buy'),
                       url(r'^float/buy/invoice/', create_float_invoice, name='service_float_invoice'),
                       url(r'^float/buy/cal/$', calculate_float_price, name='service_float_buy_calculate'),
                       url(r'^float/buy/options/$', get_service_options, name='service_float_buy_options'),
                       url(r'^float/buy/related/$', get_hidden_groups, name='service_float_buy_related_group'),
                       url(r'^float/buy/validate/$', validate_selection, name='service_float_validate'),
                       url(r'^float/template/add/$', add_float_template, name='service_float_template_add'),
                       url(r'^float/template/$', view_service_templates, name='service_float_template_view'),
                       url(r'^float/template/toggle/$', user_template_toggle_test,
                           name='service_float_template_toggle_test'),
                       url(r'^float/template/toggle_public/$', user_template_toggle_public_test,
                           name='service_float_template_toggle_public_test'),
                       url(r'^float/template/rm/$', delete_service_template,
                           name='service_float_template_rm'),
                       url(r'^float/template/assign/$', service_float_assign_template,
                           name='service_float_template_assign'),
                       url(r'^float/template/user/$', service_float_get_client_template,
                           name='service_float_template_user_get'),
                       url(r'^float/template/options/$', view_float_template_options,
                           name='service_float_template_options_view'),
                       url(r'^dedicate/$', view_dedicated_service, name='service_dedicate_view'),
                       url(r'^dedicate/add/$', add_new_dedicate_service, name='service_dedicate_add'),
                       url(r'^dedicate/rm/$', delete_dedicate_service, name='service_dedicate_delete'),
                       url(r'^dedicate/j/$', get_dedicated_service_detail, name='service_dedicate_detail'),
                       url(r'^float/discount/package/$', view_float_package_discount,
                           name='service_float_discount_package_view'),
                       url(r'^float/discount/package/add/$', add_float_package_discount,
                           name='service_float_discount_package_add'),
                       url('^float/discount/package/j/$', get_float_package_discount,
                           name='service_float_discount_package_get'),
                       url('^float/discount/package/rm/$', delete_float_package_discount,
                           name='service_float_discount_package_delete')
                       )


# requires <https://github.com/Ninjaclasher/testcase-generator>
import random

from testcase_generator import (
    BaseConstraint, BoundedConstraint, ChoiceConstraint, NoArgumentConstraint, CustomGeneratorConstraint,
    Case, Batch, Generator, ConstraintParser,
    GraphGenerator, StringGenerator,
)

"""
 |  __init__(self, N, type, *args, **kwargs)
 |      N: a BoundedConstraint object or an integer for the number of nodes
 |      type:
 |               1: normal graph
 |               2: connected graph
 |               3: complete graph
 |               4: circle
 |               10: line
 |               11: normal tree
 |               12: tree, all nodes connected to one node
 |               13: caterpillar tree
 |               14: binary tree
 |      kwargs:
 |          M: number of edges, leave blank if it is a tree
 |          duplicates: allow for duplicate edges between nodes
 |          self_loops: allow for edges between the same node
"""

def set_constraints(self):
    ## Write main constraints here ##
    self.N = BoundedConstraint(1, 10**3)
    self.E = CustomGeneratorConstraint(generator=GraphGenerator)

def generate_input(self, **kwargs):
    ## Write generator here ##
    n = self.N.next
    yield n
    self.E.initialize(n, 11)
    for i in range(n-1):
        yield self.E.next

Case.SET_CONSTRAINTS = set_constraints
Case.SET_INPUT = generate_input

conf = """
- batch: 1
  constraints: {}
  cases:
    - constraints: {}
      repeat: 1
"""

p = ConstraintParser(data=conf)
p.parse()

batches = p.batches

Generator(batches=batches, exe="__RUN_COMMAND").start()


""" Boyuan Chen:
This file quickly trims a data file to your desired data range. You can use it either on raw data file, or processed
data file.
"""

import pandas as pd
import argparse

parser = argparse.ArgumentParser(description='Semantic Scan args parser')

parser.add_argument('--input_file', type=str, default="../data/ED_2016/ED_2016_full.txt")
parser.add_argument('--output_file', type=str, default="../data/ED_2016/ED_2016_comma.txt")
parser.add_argument('--start_time', type=str, default="12/01/2015 00:00")
parser.add_argument('--end_time', type=str, default="12/27/2016 23:59")

FLAGS = parser.parse_args()

def compress_by_date(input_file, output_file, start_date, end_date):
    chunk_idx = 0
    chunksize = 10000
    reader = pd.read_csv(input_file, sep='\t', iterator=True, chunksize=chunksize, encoding='latin1', low_memory=False)
    for cc_data in reader:
        chunk_idx += 1
        print(f"Chunk {chunk_idx} starts")
        cc_data['date'] = cc_data['date'].astype(str)
        cc_data['time'] = cc_data['time'].astype(str)
        cc_data['time'] = cc_data['time'].str.replace('24:00', '23:59')
        cc_data = cc_data.join(
            cc_data['date'].str.split('/', expand=True).rename(index=int, columns={0: 'month', 1: 'day', 2: 'year'}))
        cc_data = cc_data.join(
            cc_data['time'].str.split(':', expand=True).rename(index=int, columns={0: 'hour', 1: 'minute'}))
        # Drop the rows with invalid date and time
        cc_data.dropna(subset=['day', 'month', 'year', 'hour', 'minute'], how='any', inplace=True)

        # Reconstruct valid full date
        cc_data['year'] = '20' + cc_data['year'].astype(str)
        cc_data['month'] = cc_data['month'].astype(str).str.zfill(2)
        cc_data['day'] = cc_data['day'].astype(str).str.zfill(2)
        cc_data['hour'] = cc_data['hour'].astype(str).str.zfill(2)
        cc_data['full_date'] = pd.to_datetime(
            cc_data['year'] + '/' + cc_data['month'] + '/' + cc_data['day'] + ' ' + cc_data['hour'] + ':' + cc_data[
                'minute'], format='%Y/%m/%d %H:%M')

        cc_data = cc_data[(cc_data['full_date'] >= start_date) & (cc_data['full_date'] <= end_date)]
        cc_data = cc_data.drop(columns=['month','day','year','hour','minute','full_date'])
        cc_data.to_csv(output_file, sep=",", index=False, header=(chunk_idx==1), mode='a', encoding='utf-8')

def main(args):
    pd.set_option('expand_frame_repr', False)
    compress_by_date(args.input_file, args.output_file, args.start_time, args.end_time)

if __name__=="__main__":
    main(FLAGS)

MUSES Open Source Software Code/preprocessing/data_compressor.py


import tkinter as tk
from tkinter import ttk
from tkinter import messagebox
import Models
import re

class CreateProcessPage(tk.Frame):
    def __init__(self,parent, controller):
        super().__init__(parent)
        self.pageName = 'CreateProcessPage'
        self.controller = controller
        self.bg = self.controller.configVars['CreateProcessPageColor']
        self.pFolderTitleToId = {}
        self.addInfoVars = {}
        self.mandates = {}
        self.addInfoFields = {}
        self.currentMdt = Models.Mdt()

        self.editPageBut = tk.Button(self,
                                     text='Vorgangsmappen bearbeiten',
                                     command = self.showEditFrame)
        # Process Folder Selection Frame
        self.pFSelectionFrame = tk.Frame(self,bg=self.bg)
        self.pFSelectionVar = tk.StringVar()
        self.pFSelectionCombo = ttk.Combobox(self.pFSelectionFrame,
                                              textvariable = self.pFSelectionVar)
        self.pFSelectionCombo.bind('<<ComboboxSelected>>', self.createAddInfoFields)
        self.pFSelectionLab = tk.Label(self.pFSelectionFrame,
                                       text = "Vorgangsmappe:",
                                       bg = self.bg)
        self.addProcessFolderBut = tk.Button(self.pFSelectionFrame,
                                   text = "Vorgangsmappe zur Liste Hinzufügen",
                                   command = self.addPFToLbox)

        # Process Folder Listbox Frame
        # this need to be renamed to something more descriptive
        # since these are the PF to be added to the DATEV system
        self.pFListboxFrame = tk.Frame(self,bg=self.bg)
        self.pFListboxLab = tk.Label(self.pFListboxFrame,
                                     text = "Ausgewählte Vorgangsmappe",
                                     bg = self.bg)
        self.pFListboxVar = tk.StringVar(value = list(self.currentMdt.processes))
        self.pFListbox = tk.Listbox(self.pFListboxFrame,
                                    listvariable = self.pFListboxVar,
                                    width = 20, height = 5)
        self.pFListbox.bind("<<ListboxSelect>>",
                            lambda e: self.showPFAddInfo(
                                self.pFListbox.curselection()))

        # Additional Info Entry
        self.addInfoFrame = tk.Frame(self,bg=self.bg)
        for field in self.controller.placeholderChoices.values():
            self.addInfoVars[field] = tk.StringVar()

        # Mdt Listbox
        self.mdtListboxFrame = tk.Frame(self,bg=self.bg)
        self.mdtListboxLab = tk.Label(self.mdtListboxFrame,
                                     text = "Ausgewählte Mandanten",
                                     bg = self.bg)
        self.mdtListboxVar = tk.StringVar(value = list(self.mandates))
        self.mdtListbox = tk.Listbox(self.mdtListboxFrame,
                                     listvariable = self.mdtListboxVar,
                                     width = 20, height = 5,
                                     exportselection = False)
        self.addMdtEntVar = tk.StringVar()
        validateMdtWrapper = (self.controller.root.register(self.validateMdtEntry),
                           '%P')
        self.addMdtEnt = tk.Entry (self.mdtListboxFrame,
                                   textvariable = self.addMdtEntVar,
                                   validate = 'key',
                                   validatecommand = validateMdtWrapper)
        self.addMdtBut = tk.Button(self.mdtListboxFrame,
                                   text = "Mandanten hinzufügen",
                                   command = self.addMdt)
        self.deleteMdtBut = tk.Button(self.mdtListboxFrame,
                                   text = "Mandanten löschen",
                                   command = self.deleteMdt)

        # Grid
        self.editPageBut.pack()
        #
        self.pFSelectionFrame.pack()#
        self.pFSelectionLab.pack()
        self.pFSelectionCombo.pack()

        self.pFListboxFrame.pack()#
        self.pFListboxLab.pack()
        self.pFListbox.pack()

        self.mdtListboxFrame.pack()#
        self.mdtListboxLab.pack()
        self.mdtListbox.pack()
        self.addMdtEnt.pack()
        self.addMdtBut.pack()
        self.deleteMdtBut.pack()

        self.addInfoFrame.pack()#
        # Fields are added in the createAddInfoFields method

        self.addProcessFolderBut.pack()
        #

    def onRaise(self):
        self.updatePFSelectionBox()
        self.controller.root.title('This is CreateProcessPage')
        self.controller.root.geometry(self.controller.configVars['CreateProcessPageDimensions'])
        self.config(bg = self.controller.configVars['CreateProcessPageColor'])

    def update(self, textId,*args):
        pass
    def showEditFrame(self):
        self.controller.showFrame('ProcessFolderEditPage')
        self.controller.setLastFrame('CreateProcessPage')

    def updatePFSelectionBox(self):
        # Updates the Selectioncombo
        self.pFolderTitleToId = {}
        for folder in self.controller.processFolders.values():
            self.pFolderTitleToId[
                f'{folder.idNum}: {folder.title}'] = folder.idNum
        self.pFSelectionCombo['values'] = list(self.pFolderTitleToId)

    def createAddInfoFields(self,event):
        i = 0
        for label,entry in self.addInfoFields.values():
            label.destroy()
            entry.destroy()
        self.addInfoFields = {}
        selectedPFTitle = self.pFSelectionVar.get()
        selectedPFID = self.pFolderTitleToId[selectedPFTitle]
        selectedPF = self.controller.processFolders[selectedPFID]
        for field,stringvar in self.addInfoVars.items():
            if "{"+field+"}" in selectedPF.title:
                i += 1
                label = tk.Label(self.addInfoFrame,
                                 text = f'{self.controller.placeholders.__dict__[field]}:',
                                 bg = self.bg)

                entry = tk.Entry(self.addInfoFrame,
                                 textvariable = stringvar)
                label.grid(row = i, column = 1)
                entry.grid(row = i, column = 2)
                self.addInfoFields[field] = [label,entry]

    def addPFToLbox(self):
        if len(self.mandates) < 1:
            return
        mdtSelection = self.mdtListbox.curselection()
        mdt = self.mdtListbox.get(mdtSelection)
        self.currentMdt = self.mandates[mdt]
        print(self.currentMdt.mdtNr)

    def showPFAddInfo(self, selection):
        print(selection)
##        if len(self.mandates) < 1:
##            return

    def validateMdtEntry(self, newval):
        return re.match('^[0-9]*$', newval) is not None and len(newval) <= 5

    def addMdt(self):
        mdt = self.addMdtEnt.get()
        if mdt == "" or len(mdt)< 5:
            return
        newMdt = Models.Mdt()
        newMdt.mdtNr = mdt
        self.mandates[mdt] = newMdt
        self.mdtListboxVar.set(value = list(self.mandates))
##        self.mdtListbox['exportselection'] = True
        self.mdtListbox.selection_clear(0,tk.END)
        self.mdtListbox.selection_set(len(self.mandates)-1)
##        self.mdtListbox['exportselection'] = False

    def deleteMdt(self):
        selectedIndex = self.mdtListbox.curselection()
        if len(selectedIndex) < 1:
            return
        selectedMdt = self.mdtListbox.get(selectedIndex)
        del self.mandates[selectedMdt]
        self.mdtListboxVar.set(value = list(self.mandates))


# Licensed under a 3-clause BSD style license - see LICENSE.rst

from astropy.io import fits
import numpy as np
import pytest

from sofia_redux.instruments.hawc.datafits import DataFits
from sofia_redux.instruments.hawc.steps.stepmerge import StepMerge
from sofia_redux.instruments.hawc.tests.resources \
    import DRPTestCase, pol_bgs_data, scan_smp_data

class TestMerge(DRPTestCase):
    def make_data(self, tmpdir, nfiles=2, scan=False):
        inp = []
        for i in range(nfiles):
            if scan:
                hdul = scan_smp_data()
                hdul[0].header['NHWP'] = 1
            else:
                hdul = pol_bgs_data(idx=i)
            ffile = str(tmpdir.join('test{}.fits'.format(i)))
            hdul.writeto(ffile, overwrite=True)
            df = DataFits(ffile)
            inp.append(df)
        return inp

    def test_miso(self, tmpdir):
        # works for c2n style data
        inp = self.make_data(tmpdir, 2)
        step = StepMerge()
        out = step(inp)
        assert isinstance(out, DataFits)

        # and for scan data
        inp = self.make_data(tmpdir, 2, scan=True)
        step = StepMerge()
        out = step(inp)
        assert isinstance(out, DataFits)

    def test_read_fwhm(self, capsys):
        df = DataFits()
        df.setheadval('SPECTEL1', 'HAWE')
        step = StepMerge()
        step.datain = [df]
        step.runstart([df], {})

        expected = (step.getarg('fwhm')[-1],
                    step.getarg('radius')[-1],
                    step.getarg('cdelt')[-1],
                    step.getarg('beamsize')[-1])

        # test defaults
        result = step.read_fwhm_radius_cdelt_beam()
        assert result == expected

        # bad spectel
        df.setheadval('SPECTEL1', 'HAWQ')
        with pytest.raises(ValueError):
            step.read_fwhm_radius_cdelt_beam()
        capt = capsys.readouterr()
        assert 'Cannot parse waveband' in capt.err

        df.setheadval('SPECTEL1', '')
        with pytest.raises(ValueError):
            step.read_fwhm_radius_cdelt_beam()
        capt = capsys.readouterr()
        assert 'Cannot parse waveband' in capt.err

        # bad arglist
        df.setheadval('SPECTEL1', 'HAWE')
        step.runstart([df], {'cdelt': [1, 2, 3]})
        with pytest.raises(IndexError):
            step.read_fwhm_radius_cdelt_beam()
        capt = capsys.readouterr()
        assert 'Missing radius/fwhm values' in capt.err

    def test_tables(self, tmpdir, capsys):
        test_rec = np.array([(1, 2, 3)],
                            dtype=[('x', int), ('y', int), ('z', int)])
        test_tab = fits.TableHDU(test_rec).data

        # make some input data with tables to merge
        inp = []
        nfiles = 2
        for i in range(nfiles):
            hdul = pol_bgs_data(idx=i)
            ffile = str(tmpdir.join('test.fits'))
            hdul.writeto(ffile, overwrite=True)
            df = DataFits(ffile)
            df.tableset(test_tab, tablename='TABLE DATA')
            inp.append(df)

        step = StepMerge()
        out = step(inp)
        assert 'x' in out.table.names
        assert len(out.table['x']) == nfiles
        # some columns are added
        assert 'Right Ascension' in out.table.names
        assert 'Declination' in out.table.names
        assert 'Filename' in out.table.names

        # various table mismatch conditions

        # different units
        col1 = fits.Column(name='x', format='D',
                           unit='deg', array=np.array([1.0]))
        col = fits.Column(name='test1', format='D',
                          unit='deg', array=np.array([10.0]))
        tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([col1, col]))
        inp[0].table = tbhdu.data
        col = fits.Column(name='test1', format='D',
                          unit='rad', array=np.array([10.0]))
        tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([col1, col]))
        inp[1].table = tbhdu.data

        out = step(inp)
        capt = capsys.readouterr()
        assert 'different units' in capt.err
        assert 'x' in out.table.names
        assert 'test1' not in out.table.names

        # different dimensions
        xcol = fits.Column(name='x', format='D', dim='(1)',
                           unit='deg', array=np.array([10.0]))
        col = fits.Column(name='test1', format='D', dim='(1)',
                          unit='rad', array=np.array([10.0]))
        tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([xcol, col]))
        inp[1].table = tbhdu.data
        out = step(inp)
        capt = capsys.readouterr()
        assert 'different dimension' in capt.err
        assert 'x' not in out.table.names
        assert 'test1' not in out.table.names

        # different format
        col = fits.Column(name='test1', format='E',
                          unit='rad', array=np.array([10.0]))
        tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([col1, col]))
        inp[1].table = tbhdu.data
        out = step(inp)
        capt = capsys.readouterr()
        assert 'different format' in capt.err
        assert 'x' in out.table.names
        assert 'test1' not in out.table.names

        # missing column
        tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([col1]))
        inp[1].table = tbhdu.data
        out = step(inp)
        capt = capsys.readouterr()
        assert 'name not found' in capt.err
        assert 'x' in out.table.names
        assert 'test1' not in out.table.names

    def test_merge_options(self, tmpdir, capsys):
        inp = self.make_data(tmpdir)
        step = StepMerge()

        # set some default parameters to ensure consistent
        # reductions
        kwargs = {'cdelt': [1.00, 1.55, 1.55, 2.75, 3.7],
                  'fwhm': [2.57, 4.02, 4.02, 6.93, 9.43],
                  'radius': [2.57, 4.02, 4.02, 6.93, 9.43],
                  'fit_order': 0,
                  'adaptive_algorithm': None,
                  'edge_threshold': 0}

        # set nhwp = 1 to just do stokes i
        # and add some bad pixels to mask
        for df in inp:
            df.setheadval('LATPOLE', 0)
            df.setheadval('LONPOLE', 0)
            df.setheadval('NHWP', 1)
            imlist = ['STOKES Q', 'STOKES U',
                      'ERROR Q', 'ERROR U',
                      'COVAR Q I', 'COVAR U I',
                      'COVAR Q U']
            for im in imlist:
                df.imagedel(im)

            bpm = df.imageget('BAD PIXEL MASK')
            bpm[10:12, 10:12] = 1
            bpm[12:14, 12:14] = 2
            bpm[14:16, 14:16] = 3

        # test flux conservation
        out1 = step(inp, conserveflux=True, **kwargs)
        out2 = step(inp, conserveflux=False, **kwargs)
        flux_factor = 2.75 ** 2 / inp[0].getheadval('PIXSCAL') ** 2

        # difference is flux factor
        assert np.allclose(np.nanmean(out1.image / out2.image), flux_factor)

        # since oversampled, out2 will overestimate flux
        assert np.nansum(out1.image) < np.nansum(out2.image)

        # with flux conservation, summed flux should be similar
        assert np.allclose(np.nansum(out1.image),
                           np.nansum(inp[0].image),
                           rtol=0.3)
        assert not np.allclose(np.nansum(out2.image),
                               np.nansum(inp[0].image),
                               rtol=0.3)

        # also check that lat and lon pole are no longer in header
        assert 'LATPOLE' not in out1.header
        assert 'LONPOLE' not in out1.header

        # test widow pix -- if used, should be higher total
        # value in image map
        out1 = step(inp, widowstokesi=True, **kwargs)
        out2 = step(inp, widowstokesi=False, **kwargs)
        capsys.readouterr()
        assert np.nansum(out1.imageget('IMAGE MASK')) \
            > np.nansum(out2.imageget('IMAGE MASK'))

        # run with/without error weighting
        step(inp, errflag=True, **kwargs)
        capt = capsys.readouterr()
        assert 'Uncertainties used for weighting' in capt.out
        step(inp, errflag=False, **kwargs)
        capt = capsys.readouterr()
        assert 'Uncertainties NOT used for weighting' in capt.err

    def test_adaptive_fwhm(self, tmpdir, capsys):
        inp = self.make_data(tmpdir)
        step = StepMerge()

        # standard kwargs
        kwargs = {'cdelt': [1.00, 1.55, 1.55, 2.75, 3.7],
                  'radius': [2.57, 4.02, 4.02, 6.93, 9.43],
                  'fit_order': 1,
                  'edge_threshold': 0}

        # run with adaptive, fwhm = beam -- no warning
        expected = step(inp, adaptive_algorithm='scaled',
                        fwhm=[4.84, 7.80, 7.80, 13.6, 18.2],
                        beamsize=[4.84, 7.80, 7.80, 13.6, 18.2],
                        **kwargs)
        assert 'Setting smoothing FWHM to beam' not in capsys.readouterr().err

        # run with fwhm != beam -- warns
        testval = step(inp, adaptive_algorithm='scaled',
                       fwhm=[2.57, 4.02, 4.02, 6.93, 9.43],
                       beamsize=[4.84, 7.80, 7.80, 13.6, 18.2],
                       **kwargs)
        assert 'Setting smoothing FWHM to beam' in capsys.readouterr().err
        assert np.allclose(expected.image, testval.image,
                           equal_nan=True, rtol=.01)

        # run without adaptive, fwhm != beam -- no warning
        testval2 = step(inp, adaptive_algorithm=None,
                        fwhm=[2.57, 4.02, 4.02, 6.93, 9.43],
                        beamsize=[4.84, 7.80, 7.80, 13.6, 18.2],
                        **kwargs)
        assert 'Setting smoothing FWHM to beam' not in capsys.readouterr().err
        # result is not the same without adaptive
        assert not np.allclose(expected.image, testval2.image,
                               equal_nan=True, rtol=.01)

    def test_bin_cdelt(self, tmpdir, capsys):
        inp = self.make_data(tmpdir)
        step = StepMerge()

        # default: no binning, bin_cdelt on
        default = step(inp, bin_cdelt=True, fit_order=2)
        capt = capsys.readouterr().out
        assert 'Multiplying cdelt' not in capt
        assert 'Reducing fit order' not in capt

        # set binning to 2, bin_cdelt off - same result
        inp[0].setheadval('PIXELBIN', 2)
        bin_off = step(inp, bin_cdelt=False, fit_order=2)
        capt = capsys.readouterr().out
        assert 'Multiplying cdelt' not in capt
        assert 'Reducing fit order' not in capt
        assert np.allclose(default.image, bin_off.image, equal_nan=True)

        # turn bin_cdelt on: result is half the size, fit order is reduced
        bin_on = step(inp, bin_cdelt=True, fit_order=2)
        capt = capsys.readouterr().out
        assert 'Multiplying cdelt and radius by binning factor 2' in capt
        assert 'Reducing fit order to 1' in capt
        assert tuple([s // 2 for s
                      in default.image.shape]) == bin_on.image.shape
        # total flux should be pretty close
        assert np.allclose(np.nansum(default.image),
                           np.nansum(bin_on.image),
                           rtol=0.1)


#!/usr/bin/env python
import roslib
roslib.load_manifest('safety')
import rospy
from std_msgs.msg import Header
from msg_and_srv.msg import WatchDogMsg

class WatchDogPulse(object):

    def __init__(self):
        self.update_rate = rospy.get_param("watchdog_update_rate",10)
        self.rate = rospy.Rate(self.update_rate)

        # Setup watchdog pulse topic
        self.watchdog_msg = WatchDogMsg()
        self.watchdog_pub = rospy.Publisher('watchdog_pulse', WatchDogMsg)

    def sleep(self):
        self.rate.sleep()

    def update(self):
        self.watchdog_msg.header.stamp = rospy.get_rostime()
        self.watchdog_pub.publish(self.watchdog_msg)

# -----------------------------------------------------------------------------
if __name__ == '__main__':
    rospy.init_node('watchdog_pulse')
    pulse = WatchDogPulse()
    while not rospy.is_shutdown():
        pulse.update()
        pulse.sleep()


from django.urls import path
from . import views

app_name = 'blog'
urlpatterns = [
    path('', views.post_list, name='post_list'),
    # path('', views.PostListView.as_view(), name='post_list'),
    path('<int:year>/<int:month>/<int:day>/<slug:post>/', views.post_detail, name='post_detail'),
    path('category/<int:pk>/', views.category_list, name='category_list'),
    path('tag/<int:pk>/', views.tag_list, name='tag_list'),

    path('register/', views.register, name='register'),
    path('login/', views.login, name='login'),
]


#
# @lc app=leetcode id=172 lang=python3
#
# [172] Factorial Trailing Zeroes
#

# @lc code=start

class Solution:
    def trailingZeroes(self, n: int) -> int:
        '''
        方法1：求阶乘后的0，直接递归求阶乘数会导致溢出，所以应换思路
        最末尾出现一个0，表示有一组2*5，所以应看看质因数中有几个5
        '''
        result = 0
        while n > 0:
            n //= 5
            result += n
        return result
# @lc code=end


import numpy as np
import pandas as pd
from core import save

def get_marketing(save_df=False):
    # output files
    dataset = 'marketing'
    decision = 'subscribed'

    df = pd.read_csv('./raw_datasets/marketing/marketing.csv')

    df['age'] = df['age'].apply(lambda x: 'age:30-60' if ((x >= 30) & (x <=60))  else 'age:not30-60')

    if save_df:
        for rseed in range(10):
            save(df, dataset, decision, rseed)
    #return df


import tkinter as tk
from PIL import Image, ImageTk, ImageGrab
from copy import deepcopy
import imageio
import _thread

framesArray = []
currentFrame = -1
animationWidth = 500
animationHeight = 500
frameImages = []

class App(tk.Frame):
    def __init__(self, parent, initialFrame):
        tk.Frame.__init__(self, parent)
        self.parent = parent
        self.assets = []
        self.background = None
        self.sprites = []
        self.frames = []
        self.canvasView = None
        self.frames.append(initialFrame)
        self.currentFrame = 0
        self.loadFrame(0)

    # uses the current frame to add the canvas to the tkinter frame
    def loadFrame(self, index):
        self.currentFrame = index
        self.canvasView = self.frames[index].getCanvas(self.parent)
        self.canvasView.pack()
        self.canvasView.update_idletasks()

    def getCurrentFrame(self):
        return self.frames[self.currentFrame]

class Frame:
    def __init__(self, width, height):
        self.assets = []
        self.assetsLoaded = -1
        self.sprites = []
        self.spritesLoaded = -1
        self.background = None
        self.background2 = None
        self.width = width
        self.height = height
        self.canvas = None

    # prepares the canvas for use, adds the background, assets and sprites
    def getCanvas(self, parent):
        self.canvas = tk.Canvas(parent, height=self.height, width=self.width, bg="black",
                                highlightthickness=0, relief='ridge')
        self.loadBackground()
        self.loadSprites()
        self.loadAssets()
        return self.canvas

    # saves the canvas area as an image to a file
    def save(self, filename):
        x = self.canvas.winfo_rootx() + self.canvas.winfo_x()
        y = self.canvas.winfo_rooty() + self.canvas.winfo_y()
        x1 = x + self.canvas.winfo_width()
        y1 = y + self.canvas.winfo_height()
        box = (x, y, x1, y1)
        # os.system("screencapture -R"+str(x)+str(y)+str(x1)+str(y1) + filename + ".jpg")
        ImageGrab.grab(bbox=box).save(filename + ".jpg", format="JPEG")
        return filename + ".jpg"

    def addAsset(self, asset):
        self.assetsLoaded = self.assetsLoaded + 1
        self.assets.append(asset)

    # prepares and loads the assets added to the frame
    def loadAssets(self):
        for asset in self.assets:
            asset.setPhoto(ImageTk.PhotoImage(asset.image, name=asset.name))
            asset.setIdentifier(self.canvas.create_image(asset.posX,
                                                         asset.posY,
                                                         image=asset.photo,
                                                         anchor='nw'))

    def addSprite(self, sprite):
        self.spritesLoaded = self.spritesLoaded + 1
        self.sprites.append(sprite)

    # prepares and loads the sprites added to the frane
    def loadSprites(self):
        for sprite in self.sprites:
            sprite.setPhoto(ImageTk.PhotoImage(sprite.getCurrentSprite(), name=sprite.name))
            sprite.setIdentifier(self.canvas.create_image(sprite.posX, sprite.posY,
                                                          image=sprite.photo, anchor='nw'))

    def setBackground(self, background):
        self.background = Asset(str(background), "background")
        self.background2 = Asset(str(background), "background2")
        self.background2.move(-self.background2.centerX * 2, -self.background2.centerY * 2)

    # moves the background and uses a second image to make it repeat seamlessly
    def moveBackground(self, deltaX):
        self.background.move(deltaX, 0)
        if self.background.posX > 0:
            self.background2.moveAbs(0, 0)
            self.background2.move(self.background.posX - self.background2.centerX * 2, 0)
        elif self.background.posX + self.background.centerX * 2 < self.width:
            self.background2.moveAbs(0, 0)
            self.background2.move(self.background.posX + self.background.centerX * 2, 0)
        if (self.background.posX <= -self.background.centerX * 2) or (
                self.background.posX >= self.background.centerX * 2):
            self.background, self.background2 = self.background2, self.background
            if self.background.posX > 0:
                self.background2.moveAbs(0, 0)
                self.background2.move(self.background.posX - self.background2.centerX * 2, 0)
            elif self.background.posX + self.background.centerX * 2 < self.width:
                self.background2.moveAbs(0, 0)
                self.background2.move(self.background.posX + self.background.centerX * 2, 0)

    def loadBackground(self):
        if self.background is not None:
            self.background.setPhoto(ImageTk.PhotoImage(self.background.image))
            self.background.setIdentifier(self.canvas.create_image(self.background.posX, self.background.posY,
                                                                   image=self.background.photo,
                                                                   anchor='nw'))
        if self.background2 is not None:
            self.background2.setPhoto(ImageTk.PhotoImage(self.background2.image))
            self.background2.setIdentifier(self.canvas.create_image(self.background2.posX, self.background2.posY,
                                                                    image=self.background2.photo,
                                                                    anchor='nw'))

    # removes the asset form the frame
    def unloadAsset(self, asset):
        try:
            self.assets.remove(asset)
        except:
            print("Asset not loaded:", asset)

    # removes the sprite form the frame
    def unloadSprite(self, sprite):
        try:
            self.sprites.remove(sprite)
        except:
            print("Sprite not loaded:", sprite)

    def getAsset(self, assetName):
        asset = None
        for a in self.assets:
            if (a.name == assetName):
                asset = a
                break
        return asset

    def getSprite(self, spriteName):
        sprite = None
        for s in self.sprites:
            if s.name == spriteName:
                sprite = s
                break
        return sprite

    def __deepcopy__(self, memodict={}):
        newFrame = Frame(deepcopy(self.width), deepcopy(self.height))
        newFrame.assets = deepcopy(self.assets)
        newFrame.sprites = deepcopy(self.sprites)
        newFrame.assetsLoaded = deepcopy(self.assetsLoaded)
        newFrame.spritesLoaded = deepcopy(self.spritesLoaded)
        newFrame.background = deepcopy(self.background)
        newFrame.background2 = deepcopy(self.background2)
        return newFrame

class Asset:
    def __init__(self, fileName, name):
        self.name = name
        self.filename = fileName
        self.image = None
        self.canvasID = None
        self.centerX = None
        self.centerY = None
        self.multiplier = 1
        self.posX = 0
        self.posY = 0
        self.photo = None
        self.angle = 0
        self.load(fileName)

    # properly loads the passed filename to an asset
    def load(self, fileName):
        if isinstance(fileName, str):
            try:
                self.image = Image.open(fileName).convert('RGBA')
                self.centerX = self.image.size[0] // 2
                self.centerY = self.image.size[1] // 2
            except FileNotFoundError:
                print("File doesn't exist or couldn't be read:", fileName)
        else:
            print("Invalid Parameter", fileName)

    def resizeAssetMultiplier(self, multiplier):
        self.multiplier = multiplier
        self.image = self.image.resize((multiplier * self.image.size[0],
                                        multiplier * self.image.size[1]),
                                       Image.ANTIALIAS)
        self.centerX = self.image.size[0] // 2
        self.centerY = self.image.size[1] // 2

    def resizeAsset(self, newSizeX, newSizeY):
        self.image = self.image.resize((newSizeX, newSizeY), Image.ANTIALIAS)
        self.centerX = self.image.size[0] // 2
        self.centerY = self.image.size[1] // 2

    def move(self, deltaX, deltaY):
        self.posX = self.posX + deltaX
        self.posY = self.posY + deltaY

    def moveAbs(self, absX, absY):
        self.posX = absX
        self.posY = absY

    def rotate(self, angle):
        self.angle = self.angle + angle
        self.image = self.image.rotate(angle, expand=True, center=(self.centerX, self.centerY))

    def rotateAbs(self, angle):
        self.image = self.image.rotate(-self.angle, expand=True, center=(self.centerX, self.centerY))
        self.image = self.image.rotate(angle, expand=True, center=(self.centerX, self.centerY))
        self.angle = angle

    def setIdentifier(self, id):
        self.canvasID = id

    def setPhoto(self, photo):
        self.photo = photo

    # creates a deepcopy of the asset, creating a new asset object
    def __deepcopy__(self, memodict={}):
        newAsset = Asset(self.filename, deepcopy(self.name))
        newAsset.posX = deepcopy(self.posX)
        newAsset.posY = deepcopy(self.posY)
        newAsset.multiplier = deepcopy(self.multiplier)
        newAsset.resizeAsset(self.image.size[0], self.image.size[1])
        return newAsset

class Sprite:
    def __init__(self, name, fileName, spriteWidth, spriteHeight):
        self.name = name
        self.fileName = fileName
        self.image = None
        self.posX = 0
        self.posY = 0
        self.centerX = None
        self.centerY = None
        self.photo = None
        self.canvasID = None
        self.spritesArray = []
        self.spriteWidth = spriteWidth
        self.spriteHeight = spriteHeight
        self.selectedSprite = 0
        self.multiplier = 1
        self.angle = 0
        self.createSprites()

    # loads the passed file and divides it into the sprite states
    def createSprites(self):
        if isinstance(self.fileName, str):
            try:
                self.image = Image.open(self.fileName).convert('RGBA')
                y = 0
                while y < self.image.size[1]:
                    x = 0
                    while x < self.image.size[0]:
                        box = (x, y, (x + self.spriteWidth), (y + self.spriteHeight))
                        self.spritesArray.append(self.image.crop(box))
                        x = x + self.spriteWidth
                    y = y + self.spriteHeight

            except FileNotFoundError:
                print("File doesn't exist or couldn't be read:", self.fileName)
        else:
            print("Invalid Parameter", self.fileName)

    def move(self, deltaX, deltaY):
        self.posX = self.posX + deltaX
        self.posY = self.posY + deltaY

    def moveAbs(self, absX, absY):
        self.posX = absX
        self.posY = absY

    def changeSpriteName(self, name):
        self.name = name

    def resizeSpriteMultiplier(self, multiplier):
        self.multiplier = multiplier

    def rotate(self, angle):
        self.angle = self.angle + angle

    def rotateAbs(self, angle):
        self.angle = angle

    def setPhoto(self, photo):
        self.photo = photo

    def setIdentifier(self, id):
        self.canvasID = id

    # method used for getting the current sprites states, with proper resizing and rotation
    def getCurrentSprite(self):
        tempImage = self.spritesArray[self.selectedSprite]
        tempImage = tempImage.resize((self.multiplier * tempImage.size[0],
                                      self.multiplier * tempImage.size[1]),
                                     Image.ANTIALIAS)
        return tempImage.rotate(self.angle, expand=True, center=(tempImage.size[0] // 2, tempImage.size[1] // 2))

    def changeSelectedSprite(self, index):
        self.selectedSprite = index

    # creates a deepcopy of the sprite, creating a new sprite object
    def __deepcopy__(self, memodict={}):
        newSprite = Sprite(deepcopy(self.name), deepcopy(self.fileName),
                           deepcopy(self.spriteWidth), deepcopy(self.spriteHeight))
        newSprite.posX = deepcopy(self.posX)
        newSprite.posY = deepcopy(self.posY)
        newSprite.selectedSprite = deepcopy(self.selectedSprite)
        newSprite.multiplier = deepcopy(self.multiplier)
        newSprite.angle = deepcopy(self.angle)
        return newSprite

# creates and displays the tkinter frame with the current frame and canvas
def makeCanvas():
    root = tk.Tk()
    root.title('PAL-Project')
    root.resizable(False, False)

    app = App(root, framesArray[currentFrame])
    app.pack()
    app.update_idletasks()

    root.lift()
    root.attributes('-topmost', True)
    root.after_idle(root.attributes, '-topmost', False)
    app.mainloop()

# displays the tkinter frame long enough to save all the frames to files
def save(frameTime):
    count = 0

    for frame in framesArray:
        root = tk.Tk()
        root.title('PAL-Project')
        root.resizable(False, False)

        app = App(root, framesArray[count])
        app.pack()
        app.update_idletasks()
        root.lift()
        root.attributes('-topmost', True)
        root.after(500, lambda: saveFrameImage(frame, count))
        root.after(1000, lambda: root.destroy())

        app.mainloop()
        count = count + 1

    for x in range(0, count):
        frameImages.append(imageio.imread("frame" + str(x) + ".jpg"))
    imageio.mimsave('animation.gif', frameImages, duration=frameTime)

def saveFrameImage(frame, count):
    _thread.start_new(frame.save, ("frame" + str(count),))

# creates a new frame after the last one, with the same background, sprites and assets as the current one
def createFrame():
    global currentFrame
    if currentFrame != -1:
        framesArray.append(deepcopy(framesArray[currentFrame]))
    else:
        framesArray.append(Frame(animationWidth, animationHeight))
    currentFrame = currentFrame + 1


from .models import Lista
from rest_framework import serializers
from .models import Item

class ItemSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Item
        fields = ['name', 'list', 'comprado', 'url']

class ListaSerializer(serializers.HyperlinkedModelSerializer):
    item_set = ItemSerializer(many=True)

    class Meta:
        model = Lista
        fields = ['owner', 'name', 'item_set' , 'url']


import cv2
import os
import pandas as pd
import matplotlib.pyplot as plt
import tqdm
from tqdm import tqdm
import numpy as np
import pytesseract
import subprocess

# useful objects
directory = '/Users/IvanK/NFL/VideoData'
os.chdir(directory)

pytesseract.pytesseract.tesseract_cmd = (r'/Users/IvanK/tesseract/tesseract')

tessdata_dir_config = r'--tessdata-dir "/Users/IvanK/tesseract/tessdata" --psm 10 --oem 3 -c tessedit_char_whitelist=0123456789' # try # --oem 0?
# kernel = np.ones((2,2), np.uint8)

# binarisation constant
threshold = 180
# sizes and colours for expanding an image
top, bottom = int(20), int(20)
left, right = int(35), int(35)
white, black  = [255,255,255], [0,0,0]

def keywithmaxval(d):
    """ a) create a list of the dict's keys and values;
         b) return the key with the max value"""
    v=list(d.values())
    k=list(d.keys())
    return k[v.index(max(v))]

def basic_image_preproc(im_path, threshold = 180):
    # make sure that the path is correct
    tmp_img = cv2.imread(im_path, cv2.IMREAD_UNCHANGED)

    # we need to convert from BGR to RGB format/mode:
    img_rgb = cv2.cvtColor(tmp_img, cv2.COLOR_BGR2RGB)
    img_grey =  cv2.cvtColor(tmp_img, cv2.COLOR_BGR2GRAY)

    _, otsu_s = cv2.threshold(img_grey,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
    _, im_white = cv2.threshold(img_rgb,threshold,255,cv2.THRESH_BINARY_INV)

    # make the size uniform?
    otsu_s = cv2.copyMakeBorder(otsu_s,top,bottom,left,right, cv2.BORDER_CONSTANT, value = white)
    im_white = cv2.copyMakeBorder(im_white,top,bottom,left,right, cv2.BORDER_CONSTANT, value = white)

    return otsu_s, im_white

##############################################
##############################################
##############################################
############## Video to images ###############
##############################################
##############################################
##############################################

def video_to_images(v_name, max_frames = 2000):

    cap= cv2.VideoCapture(v_name)
    total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    freq = total // max_frames

    i = 0 # for naming
    n = 0 # iterator

    while(cap.isOpened()):
        ret, frame = cap.read()
        if ret == False:
            break
        if (n % freq == 0):
            cv2.imwrite(f'tmp/frames/F{i}.jpg',frame)
            i+=1
        n += 1

    cap.release()
    cv2.destroyAllWindows()

##############################################
##############################################
##############################################
############# Detecting Channel ##############
##############################################
##############################################
##############################################
'''does not work on this data'''

# retrieve keys
fox = cv2.imread('KeyImages/fox.jpg', cv2.IMREAD_UNCHANGED)
nbc = cv2.imread('KeyImages/nbc.jpg', cv2.IMREAD_UNCHANGED)
cbs = cv2.imread('KeyImages/cbs.jpg', cv2.IMREAD_UNCHANGED)
espn = cv2.imread('KeyImages/espn.jpg', cv2.IMREAD_UNCHANGED)

channels_keys = {'fox':fox, 'nbc':nbc, 'cbs':cbs, 'espn': espn}

def match_patterns(sample_img_names, video_id, keys):

    # allow for multiple key images per channel

    matching_results = {}

    for key_name, k in keys.items():

        tmp_placeholder = []

        for im_name in sample_img_names:

            im_name = f'Videos/{video_id}/frames/{im_name}'

            tmp_img = cv2.imread(im_name, cv2.IMREAD_UNCHANGED)
#             imageGray = cv2.cvtColor(tmp_img, cv2.COLOR_BGR2GRAY)
#             templateGray = cv2.cvtColor(k, cv2.COLOR_BGR2GRAY)

#             print(tmp_img.shape)
#             print(key_name)
#             print(k.shape)
            tmp_result = cv2.matchTemplate(tmp_img,k, cv2.TM_CCOEFF_NORMED)

            _, max_val, _, _ = cv2.minMaxLoc(tmp_result)
            tmp_placeholder.append(max_val)

            cv2.destroyAllWindows()
        tmp_max_val = np.median(tmp_placeholder)
        matching_results[key_name] = tmp_max_val

    print(matching_results)
    best_key = keywithmaxval(matching_results)

    return best_key

def get_matched_channel_for_video(v_id, S = 100):

    directory = f'Videos/{v_id}/frames'
    files = sorted(os.listdir(directory))
    N_files = len(files)

    files_sample = files[N_files//2 - S: N_files//2 + S]

    res = match_patterns(files_sample, v_id, channels_keys)
    # save results to df with video/matched with channel
    if 'video_channel_matched.csv' in os.listdir():
        df = pd.read_csv('video_channel_matched.csv')
        if v_id in list(df['video_id'].unique()):
            df.loc[df['video_id'] == v_id, 'channel_name'] =  res
            df.to_csv('video_channel_matched.csv', index = False)

        else:
            print('here')
            df = df.append({'video_id': v_id, 'channel_name':res}, ignore_index = True)
            print(df.head())
            df.to_csv('video_channel_matched.csv', index = False)

    else:
        df = pd.DataFrame(data = {'video_id': [v_id] ,'channel_name':  [res]})
        df.to_csv('video_channel_matched.csv', index = False)

##############################################
##############################################
##############################################
############## Finding Score Imgs ############
##############################################
##############################################
##############################################

channel_box_locations_d = {'fox': [(572,550), (572,643), [(51,87), (51,85)]], 'nbc': [(630,428), (630,682), [(50,64), (50,58)]],
                           'cbs': [(615,430), (615,618), [(45,53), (45,57)]], 'espn': [(640,405), (640,680), [(54,130),(54,130)]]}

def get_bounding_boxs_wth_scores(channel_name):

    team_A_loc, team_B_loc, box_size = channel_box_locations_d[channel_name][0], channel_box_locations_d[channel_name][1], channel_box_locations_d[channel_name][2]
    h_a, w_a  = box_size[0][0], box_size[0][1]
    h_b, w_b  = box_size[1][0], box_size[1][1]

    directory = f'tmp/frames'
    # get files and sort by date
    files = os.listdir(directory)
    files.sort(key=lambda s: os.path.getmtime(os.path.join(directory, s)))

    print(files)
    # DS STORE ISSUE
    N = len(files)

    for n, name in enumerate(tqdm(files)):

        im_name = f'tmp/frames/{name}'

        tmp_img = cv2.imread(im_name, cv2.IMREAD_UNCHANGED)

        score_A_img = tmp_img[team_A_loc[0]: team_A_loc[0] +h_a, team_A_loc[1]: team_A_loc[1] +w_a,:]

        score_B_img = tmp_img[team_B_loc[0]: team_B_loc[0] +h_b, team_B_loc[1]: team_B_loc[1] +w_b,:]

        cv2.imwrite(f'tmp/scoreboxes/F{n}_teamA.jpg',score_A_img)
        cv2.imwrite(f'tmp/scoreboxes/F{n}_teamB.jpg',score_B_img)

    cv2.destroyAllWindows()

##############################################
##############################################
##############################################
########## Get Score Data and PostProc #######
##############################################
##############################################
##############################################

def get_scores_for_video():

    directory = f'tmp/scoreboxes'

    # get files and sort by date (assuming ordered by time and team)
    files = os.listdir(directory)
    files.sort(key=lambda s: os.path.getmtime(os.path.join(directory, s)))
    files_A = files[1::2]
    files_B = files[2::2]

    # initialise score placeholders
    scores_A, scores_B = [],[]

    for n, name in enumerate(tqdm(files_A)):
        im_name = f'tmp/scoreboxes/{name}'
        blackened, whitened = basic_image_preproc(im_name)

        score_black = pytesseract.image_to_string(blackened, config=tessdata_dir_config)
        score_white = pytesseract.image_to_string(whitened, config=tessdata_dir_config)

        score_joint = score_black + '&' + score_white

#         if (score_black == score_white):
#             scores_A.append(score_black)
#         else:
#             scores_A.append(np.nan)

        scores_A.append(score_joint)

    for n, name in enumerate(tqdm(files_B)):
        im_name = f'tmp/scoreboxes/{name}'
        blackened, whitened = basic_image_preproc(im_name)

        score_black = pytesseract.image_to_string(blackened, config=tessdata_dir_config)
        score_white = pytesseract.image_to_string(whitened, config=tessdata_dir_config)

        score_joint = score_black + '&' + score_white

#         if (score_black == score_white):
#             scores_B.append(score_black)
#         else:
#             scores_B.append(np.nan)
        scores_B.append(score_joint)

    return scores_A, scores_B

##############################################
##############################################
##############################################
################## Big Loop ##################
##############################################
##############################################
##############################################

main_directory = '/Users/IvanK/NFL/VideoData'
video_table = pd.read_csv('merged_match_views_with_channel_2021.csv')
folder_paths = ['tmp/frames','tmp/scoreboxes']
n = 0

for index, v in video_table.iterrows():

    n += 1

    os.chdir(main_directory) # return to the main one in case we have moved

    v_url = 'https://www.youtube.com/' + v['videoId']
    v_channel = v['channel']

    v_id = v['videoId']

    extracted_videos = os.listdir('/Users/IvanK/NFL/VideoData/scores')
    v_in_extracted = f'{v_id}.csv' in extracted_videos

    if (v_channel == 'fox')|(v_in_extracted):
        pass

    else:

        subprocess.run(["pytube", v_url])

        files = os.listdir(directory)
        files.sort(key=lambda s: os.path.getmtime(os.path.join(directory, s)))
        print(files)
        v_name = files[-1]
        print(v_name)

        print('Converting to images!')
        video_to_images(v_name) # naming issue
        print('Getting bounding boxes!')
        get_bounding_boxs_wth_scores(v_channel)
        print('Getting scores boxes!')
        scores_A, scores_B = get_scores_for_video()

        # delete part
        os.unlink(v_name) # delete video

        for folder_path in folder_paths: # detele generated images
            for file_object in os.listdir(folder_path):
                file_object_path = os.path.join(folder_path, file_object)
                if os.path.isfile(file_object_path) or os.path.islink(file_object_path):
                    os.unlink(file_object_path)

        # post processing part
        # make score files the same length since +- 1 image can happen when cropping
        n_min = min(len(scores_A), len(scores_B))

        if len(scores_A) > n_min:
            scores_A = scores_A[:n_min]
        elif len(scores_B) > n_min:
            scores_B = scores_B[:n_min]

        scores_df = pd.DataFrame({'Score_A': scores_A, 'Score_B' :scores_B})
        scores_df.to_csv(f'scores/{v_id}.csv', index = False)
    #     scores_A_processed, scores_B_processed = post_processing(scores_A), post_processing(scores_B)

    #     if n >=1:
    #         break

        # saving results
        # ???

        # example 3 team AB reversed
    # if empty or something in one -> choose what the other says

    # figure out sorting/os and d.s store stuff


from functools import wraps
from flask import Blueprint, redirect, url_for, render_template, flash, g, current_app, request, get_flashed_messages
from flask.helpers import get_flashed_messages
from backends.brickowl import BrickOwl
from db import Session
from routes.auth import auth_request
from models import Op as SavedOp, User
from components.paginator import Paginator
from sqlalchemy import update
from backends.bricklink import Bricklink, InvalidRequest as BricklinkInvalidRequest
from backends.brickowl import BrickOwl, InvalidRequest as BrickowlInvalidRequest
import json
from typing import Optional
import syncer
from datetime import datetime

blueprint = Blueprint('settings', __name__)

def require_remote_keys(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        with Session.begin() as session:
            user = session.query(User) \
                .filter_by(id=g.user_id) \
                .first()

            if user.bl_credentials_approved and user.bo_credentials_approved:
                return current_app.ensure_sync(f)(*args, **kwargs)
            else:
                return redirect(url_for('settings.view'))
    return wrapper

@blueprint.route('/settings', methods=['GET'])
@auth_request
def view():
    with Session.begin() as session:
        user = session.query(User) \
            .filter_by(id=g.user_id) \
            .first()
        form_feedback = json.loads(get_flashed_messages()[0]) if get_flashed_messages() else {}
        return render_template('settings/index.j2', user=user, form_feedback=form_feedback)

@blueprint.route('/settings/stores', methods=['POST'])
@auth_request
def set_stores():
    with Session.begin() as session:
        user = session.query(User) \
            .filter_by(id=g.user_id) \
            .first()

        bl_customer_key = request.form.get('bl_customer_key')
        bl_customer_secret = request.form.get('bl_customer_secret')
        bl_token_value = request.form.get('bl_token_value')
        bl_token_secret = request.form.get('bl_token_secret')

        bo_key = request.form.get('bo_key')

        user.bl_customer_key = bl_customer_key
        user.bl_customer_secret = bl_customer_secret
        user.bl_token_value = bl_token_value
        user.bl_token_secret = bl_token_secret

        user.bo_key = bo_key

        user.bl_credentials_approved = False
        user.bo_credentials_approved = False

        bl = Bricklink(
            customer_key=bl_customer_key,
            customer_secret=bl_customer_secret,
            token_value=bl_token_value,
            token_secret=bl_token_secret,
        )
        try:
            bl.get_colors()
            user.bl_credentials_approved = True
        except BricklinkInvalidRequest as _:
            pass

        bo = BrickOwl(
            key=bo_key
        )
        try:
            bo.get_colors()
            user.bo_credentials_approved = True
        except BrickowlInvalidRequest as _:
            pass

    return redirect(url_for("settings.view"))

@blueprint.route('/settings/syncer/toggle', methods=['POST'])
@auth_request
def toggle_syncer():
    with Session.begin() as session:
        user = session.query(User) \
            .filter_by(id=g.user_id) \
            .first()

        if not (user.bl_credentials_approved and user.bo_credentials_approved):
            flash(json.dumps({ 'backends': { 'submit': 'Backends not approved.' }}))
            return redirect(url_for('settings.view'))

        should_enable = not user.syncer_enabled
        antiflood_interval = 2  # In seconds

        if should_enable:
            if user.syncer_enable_timestamp is None or \
                (datetime.now() - user.syncer_enable_timestamp).seconds >= antiflood_interval:
                syncer.start(session, user.id)

                user.syncer_enabled = True
                user.syncer_enable_timestamp = datetime.now().isoformat()
            else:
                # TODO flash error message
                pass
        else:
            user.syncer_enabled = False
            syncer.stop(user)

    return redirect(url_for("settings.view"))

@blueprint.route('/settings/syncer/start', methods=['POST'])
@auth_request
def start_syncer():
    with Session.begin() as session:

        syncer.start(session, g.user_id)

        return redirect(url_for('settings.view'))

@blueprint.route('/settings/syncer/stop', methods=['POST'])
@auth_request
def stop_syncer():
    with Session.begin() as session:

        syncer.stop(session, g.user_id)

        return redirect(url_for('settings.view'))


from datetime import datetime
from uuid import uuid4

from sqlalchemy import Column, DateTime, String
from sqlalchemy.sql import expression as sql

from db import Base

class User(Base):
    __tablename__ = "users"
    id = Column(String, primary_key=True)
    full_name = Column(String)
    created_at = Column(DateTime, index=True, default=datetime.utcnow)

    def __repr__(self):
        return (
            f"<{self.__class__.__name__}("
            f"id={self.id}, "
            f"full_name={self.full_name}, "
            f")>"
        )

    @classmethod
    async def create(cls, db, **kwargs) -> "User":
        query = (
            sql.insert(cls)
            .values(id=str(uuid4()), **kwargs)
            .returning(cls.id, cls.full_name)
        )
        users = await db.execute(query)
        await db.commit()
        return users.first()

    @classmethod
    async def update(cls, db, id, **kwargs) -> "User":
        query = (
            sql.update(cls)
            .where(cls.id == id)
            .values(**kwargs)
            .execution_options(synchronize_session="fetch")
            .returning(cls.id, cls.full_name)
        )
        users = await db.execute(query)
        await db.commit()
        return users.first()

    @classmethod
    async def get(cls, db, id) -> "User":
        query = sql.select(cls).where(cls.id == id)
        users = await db.execute(query)
        (user,) = users.first()
        return user

    @classmethod
    async def get_all(cls, db) -> list["User"]:
        query = sql.select(cls)
        users = await db.execute(query)
        users = users.scalars().all()
        return users

    @classmethod
    async def delete(cls, db, id) -> bool:
        query = (
            sql.delete(cls)
            .where(cls.id == id)
            .returning(
                cls.id,
                cls.full_name,
            )
        )
        await db.execute(query)
        await db.commit()
        return True


from flask import request

from xivo_dao.alchemy.callfiltermember import Callfiltermember as CallFilterMember
from xivo_dao.helpers import errors
from xivo_dao.helpers.exception import NotFoundError

from wazo_confd.auth import required_acl
from wazo_confd.helpers.restful import ConfdResource

from .schema import CallFilterRecipientUsersSchema, CallFilterSurrogateUsersSchema

class CallFilterRecipientUserList(ConfdResource):
    schema = CallFilterRecipientUsersSchema
    has_tenant_uuid = True

    def __init__(self, service, call_filter_dao, user_dao):
        self.service = service
        self.call_filter_dao = call_filter_dao
        self.user_dao = user_dao

    @required_acl('confd.callfilters.{call_filter_id}.recipients.users.update')
    def put(self, call_filter_id):
        tenant_uuids = self._build_tenant_list({'recurse': True})
        call_filter = self.call_filter_dao.get(
            call_filter_id, tenant_uuids=tenant_uuids
        )
        form = self.schema().load(request.get_json())
        try:
            recipients = []
            for user_form in form['users']:
                user = self.user_dao.get_by(
                    uuid=user_form['user']['uuid'], tenant_uuids=tenant_uuids
                )
                recipient = self.service.find_recipient_by_user(call_filter, user)
                if not recipient:
                    recipient = CallFilterMember()
                    recipient.user = user
                recipient.timeout = user_form['timeout']
                recipients.append(recipient)

        except NotFoundError as e:
            raise errors.param_not_found('users', 'User', **e.metadata)

        self.service.associate_recipients(call_filter, recipients)
        return '', 204

class CallFilterSurrogateUserList(ConfdResource):
    schema = CallFilterSurrogateUsersSchema
    has_tenant_uuid = True

    def __init__(self, service, call_filter_dao, user_dao):
        self.service = service
        self.call_filter_dao = call_filter_dao
        self.user_dao = user_dao

    @required_acl('confd.callfilters.{call_filter_id}.surrogates.users.update')
    def put(self, call_filter_id):
        tenant_uuids = self._build_tenant_list({'recurse': True})
        call_filter = self.call_filter_dao.get(
            call_filter_id, tenant_uuids=tenant_uuids
        )
        form = self.schema().load(request.get_json())
        try:
            surrogates = []
            for user_form in form['users']:
                user = self.user_dao.get_by(
                    uuid=user_form['user']['uuid'], tenant_uuids=tenant_uuids
                )
                surrogate = self.service.find_surrogate_by_user(call_filter, user)
                if not surrogate:
                    surrogate = CallFilterMember()
                    surrogate.user = user
                surrogates.append(surrogate)

        except NotFoundError as e:
            raise errors.param_not_found('users', 'User', **e.metadata)

        self.service.associate_surrogates(call_filter, surrogates)
        return '', 204


from statistics import mode
from unittest import result
from flask import Flask, render_template, request
import pickle
import numpy as np
import sys

app = Flask(__name__)

model = pickle.load(open('model.tflite', 'rb'))

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/predict", methods=['POST', 'GET'])
def predict():
    float_features = [float(x) for x in request.form.values()]

    #print('Hello world!' + float_features, file=sys.stderr)
    #input = [[0, 0, 0, 0, 700, 0, 3809677.791, 1796366, 0, 104]]
    final = [np.array(float_features)]
    output = model.predict(final)

    print(output[0], file=sys.stderr)

    if (output[0] == 0):
        return render_template("index.html", result="Not a attack")
    else:
        return render_template("index.html", result="Attack")

if (__name__ == "__main__"):
    app.run()


# recebe uma data e confirma se é ou não uma data válida
def valida_data(d,m,a):
    if m > 12 or m < 1:
        return False
    elif m == 1 or m == 3 or m == 5 or m == 7 or m == 8 or m == 10 or m == 12:
        if d <= 31:
            return True
    elif m == 4 or m == 6 or m == 9 or m == 11:
        if d <= 30:
            return True
    elif m == 1:
        if(a % 4 == 0 and a % 100 != 0) or a % 400 == 0:
            if d <= 29:
                return True
            elif d <= 28:
                return True
    else:
        return False

def main():
    print('Digite a data ')
    d = int(input('Dia: '))
    m = int(input('Mês: '))
    a = int(input('Ano: '))

    data = valida_data(d,m,a)

    if data == True:
        print('Data válida')
    else:
        print('Data inválida')

main()


from datasets import load_dataset
from transformers import BartForConditionalGeneration, AutoTokenizer
import torch
import pandas as pd

# for possible versions, see: https://huggingface.co/datasets/cnn_dailymail
# use this dataset instead of the original: https://huggingface.co/datasets/ccdv/cnn_dailymail
# see this bug for the reason why: https://github.com/huggingface/datasets/issues/996
cnn_versions = ['3.0.0', '1.0.0', '2.0.0']
cnn = load_dataset("ccdv/cnn_dailymail", cnn_versions[0])  # for general text summarization

model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-cnn', add_prefix_space=True)

premise = cnn['train'][0]['article']
hypothesis = cnn['train'][0]['highlights']

# run through model pre-trained on MNLI
device = "cuda"
x = tokenizer.encode(
    premise,
    hypothesis,
    return_tensors='pt',
    truncation_strategy='longest_first'
)
model.to(device)
with torch.no_grad():
    generated = model.generate(x.to(device))

def postprocess(token_ids: list) -> str:
    """
    Converts tokens to text.

    :param token_ids: tokens from a tokenizer
    """
    return ''.join([
        tokenizer.decode(x, skip_special_tokens=True, clean_up_tokenization_spaces=True)
        for x in token_ids
    ])

bart_abs_sum = postprocess(generated)

import nltk
from sentence_transformers import SentenceTransformer, util
import numpy as np
from LexRank import degree_centrality_scores

model = SentenceTransformer('all-MiniLM-L6-v2')

# Our input document we want to summarize
# As example, we take the first section from Wikipedia
document = premise

# Split the document into sentences
sentences = nltk.sent_tokenize(document)
print("Num sentences:", len(sentences))

# Compute the sentence embeddings
embeddings = model.encode(sentences, convert_to_tensor=True)

# Compute the pair-wise cosine similarities
cos_scores = util.cos_sim(embeddings, embeddings).cpu().numpy()

# Compute the centrality for each sentence
centrality_scores = degree_centrality_scores(cos_scores, threshold=None)

# We argsort so that the first element is the sentence with the highest score
most_central_sentence_indices = np.argsort(-centrality_scores)

# Summarize top 5 sentences
lexrank_sum = ' '.join([sentences[idx].strip() for idx in most_central_sentence_indices[0:5]])

import pytextrank
import spacy
from math import sqrt
from operator import itemgetter

nlp = spacy.load("en_core_web_sm")

# add PyTextRank to the spaCy pipeline
nlp.add_pipe("textrank")
doc = nlp(cnn['train'][0]['article'])

def get_textrank_summary(doc: spacy.tokens.doc.Doc, summary_len: int = 3):
    """Returns summary from a Spacy parsed document."""
    sent_text = {sent_id: sent.text for sent_id, sent in enumerate(doc.sents)}
    sent_bounds = [[s.start, s.end, set([])] for s in doc.sents]

    # iterate through the top ranked phrases and add them to the phrase vector for each sentence
    limit_phrases = 4

    phrase_id = 0
    unit_vector = []

    for p in doc._.phrases:
        unit_vector.append(p.rank)

        for chunk in p.chunks:
            for sent_start, sent_end, sent_vector in sent_bounds:
                if chunk.start >= sent_start and chunk.end <= sent_end:
                    sent_vector.add(phrase_id)
                    break

        phrase_id += 1

        if phrase_id == limit_phrases:
            break

    # normalize the unit vector
    sum_ranks = sum(unit_vector)
    unit_vector = [rank / sum_ranks for rank in unit_vector]

    # iterate through each sentence, calculating its euclidean distance from the unit vector
    sent_rank = {}
    sent_id = 0

    for sent_start, sent_end, sent_vector in sent_bounds:
        sum_sq = 0.0
        for phrase_id in range(len(unit_vector)):

            if phrase_id not in sent_vector:
                sum_sq += unit_vector[phrase_id] ** 2.0

        sent_rank[sent_id] = sqrt(sum_sq)
        sent_id += 1

    # extract the sentences with the lowest distance, up to the limit requested
    results = [
        (sent_id, rank, sent_text[sent_id])
        for sent_id, rank in sorted(sent_rank.items(), key=itemgetter(1))
    ]
    summary = ' '.join([s[2] for s in results[:summary_len]])
    return summary, results

textrank_sum, textrank_sentranks = get_textrank_summary(doc=doc)

from transformers import pipeline

summarizer = pipeline("summarization", model="t5-base", tokenizer="t5-base", framework="pt")
t5_abs_sum = summarizer(document, min_length=5, max_length=100)[0]['summary_text']

from rouge import Rouge

def format_rouge_scores(rouge_scores):
    """Flattens Rouge scores"""
    rs_new = {}
    for rouge_type_dict in rouge_scores:
        for rouge_type, metric in rouge_type_dict.items():
            for name, val in metric.items():
                rs_new[rouge_type + "_" + name] = val
    return rs_new

# evaluate CNN summaries
rouge = Rouge()
scores = {
    'bart': format_rouge_scores(rouge.get_scores(hyps=bart_abs_sum, refs=hypothesis)),
    't5': format_rouge_scores(rouge.get_scores(hyps=t5_abs_sum, refs=hypothesis)),
    'lexrank': format_rouge_scores(rouge.get_scores(hyps=lexrank_sum, refs=hypothesis)),
    'textrank': format_rouge_scores(rouge.get_scores(hyps=textrank_sum, refs=hypothesis)),
}

from sentence_transformers.cross_encoder import CrossEncoder

model = CrossEncoder('facebook/bart-large')
bart_cos_sim_scores = model.predict([
    [bart_abs_sum, hypothesis],
    [t5_abs_sum, hypothesis],
    [lexrank_sum, hypothesis],
    [textrank_sum, hypothesis],
])
for idx, m in enumerate(list(scores)):
    scores[m]['bart_cos_sim'] = bart_cos_sim_scores[idx]

scores_df = pd.DataFrame(scores)
scores_df.to_csv('summarizer_scores.csv')

breakpoint()


# 로그 남기기
import logging
from datetime import datetime

# logging.basicConfig(level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s")
#
# logging.debug("아 이거 누가 짠거야~~")
# logging.info("자동화 수행 준비")
# logging.warning("이 스크립트는 오래 되었습니다.")
# logging.error("에러 발생")
# logging.critical("치명적인 문제")

# 터미널과 파일에 함께 로그 남기기
logFormatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger()

# 로그 레벨 설정
logger.setLevel(logging.DEBUG)

# 스트림
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(logFormatter)
logger.addHandler(streamHandler)    # 로거에 스트림핸들러를 추가

# 파일
filename = datetime.now().strftime("mylogfile_%Y%m%d%H%M%S.log")    # mylog_20201010141011.log
fileHandler = logging.FileHandler(filename, encoding="utf-8")
fileHandler.setFormatter(logFormatter)
logger.addHandler(fileHandler)      # 로거에 파일 핸들러를 추가

logger.debug("로그를 남겨보는 테스트2")   # 로거는 스트림, 파일 핸들러로 출력