import json
import re
from datetime import datetime
from multiprocessing.pool import ThreadPool
from typing import Any, Dict, List, Optional, Union

from ccxt import Exchange, binance
from rp2.rp2_decimal import ZERO, RP2Decimal
from rp2.rp2_error import RP2RuntimeError

from dali.abstract_ccxt_input_plugin import (
    AbstractCcxtInputPlugin,
    ProcessOperationResult,
    Trade,
)
from dali.ccxt_pagination import (
    AbstractPaginationDetailSet,
    DateBasedPaginationDetailSet,
)
from dali.configuration import Keyword
from dali.in_transaction import InTransaction
from dali.intra_transaction import IntraTransaction
from dali.out_transaction import OutTransaction

# Native format keywords
_ACCOUNT_PROFITS: str = "accountProfits"
_ALGO: str = "algo"
_ALGO_NAME: str = "algoName"
_AMOUNT: str = "amount"
_ASSET: str = "asset"
_BEGIN_TIME: str = "beginTime"
_BUY: str = "buy"  # CCXT only variable
_COIN: str = "coin"
_COIN_NAME: str = "coinName"
_COST: str = "cost"  # CCXT only variable
_CREATE_TIME: str = "createTime"
_CRYPTOCURRENCY: str = "cryptoCurrency"
_CURRENCY: str = "currency"  # CCXT only variable
_DAILY: str = "DAILY"
_DATA: str = "data"
_DATE_TIME: str = "datetime"  # CCXT only variable
_DELIVER_DATE: str = "deliverDate"
_DEPOSIT: str = "deposit"  # CCXT only variable
_DIV_TIME: str = "divTime"
_END_TIME: str = "endTime"
_EN_INFO: str = "enInfo"
_FEE: str = "fee"
_FIAT_CURRENCY: str = "fiatCurrency"
_ID: str = "id"  # CCXT only variable
_INDICATED_AMOUNT: str = "indicatedAmount"
_INFO: str = "info"
_INSERT_TIME: str = "insertTime"
_INTEREST_PARAMETER: str = "INTEREST"
_INTEREST_FIELD: str = "interest"
_IS_DUST: str = "isDust"
_IS_FIAT_PAYMENT: str = "isFiatPayment"
_LEGAL_MONEY: str = "legalMoney"
_LENDING_TYPE: str = "lendingType"
_LIMIT: str = "limit"
_LOCK_PERIOD: str = "lockPeriod"
_OBTAIN_AMOUNT: str = "obtainAmount"
_ORDER: str = "order"  # CCXT only variable
_ORDER_NO: str = "orderNo"
_PAGE_INDEX: str = "pageIndex"
_PAGE_SIZE: str = "pageSize"
_POSITION_ID: str = "positionId"
_PRICE: str = "price"
_PRODUCT: str = "product"
_PROFIT_AMOUNT: str = "profitAmount"
_REDEMPTION: str = "REDEMPTION"
_ROWS: str = "rows"
_SELL: str = "sell"  # CCXT only variable
_SIDE: str = "side"  # CCXT only variable
_SIZE: str = "size"
_STAKING: str = "STAKING"
_START_TIME: str = "startTime"
_STATUS: str = "status"
_SOURCE_AMOUNT: str = "sourceAmount"
_SUBSCRIPTION: str = "SUBSCRIPTION"
_SYMBOL: str = "symbol"
_TIME: str = "time"
_TIMESTAMP: str = "timestamp"  # CCXT only variable
_TRAN_ID: str = "tranId"
_TRANSACTION_TYPE: str = "transactionType"
_TOTAL: str = "total"
_TOTAL_FEE: str = "totalFee"
_TOTAL_NUM: str = "totalNum"
_TYPE: str = "type"
_TX_ID: str = "txid"  # CCXT doesn't capitalize I
_TXN_TYPE: str = "txnType"
_UPDATE_TIME: str = "updateTime"
_USERNAME: str = "userName"
_WITHDRAWAL: str = "withdrawal"  # CCXT only variable

# Time period constants
_NINETY_DAYS_IN_MS: int = 7776000000
_THIRTY_DAYS_IN_MS: int = 2592000000
_ONE_DAY_IN_MS: int = 86400000
_MS_IN_SECOND: int = 1000

# Record limits
_DEPOSIT_RECORD_LIMIT: int = 1000
_DIVIDEND_RECORD_LIMIT: int = 500
_DUST_TRADE_RECORD_LIMIT: int = 100
_INTEREST_SIZE_LIMIT: int = 100
_MINING_PAGE_LIMIT: int = 200
_TRADE_RECORD_LIMIT: int = 1000
_WITHDRAWAL_RECORD_LIMIT: int = 1000

# Types of Binance Dividends
_BNB_VAULT = "BNB Vault"
_ETH_STAKING = "ETH 2.0 Staking"
_FLEXIBLE = "Flexible"
_FLEXIBLE_SAVINGS = "Flexible Savings"
_LAUNCH_POOL = "Launchpool"
_LOCKED = "Locked"
_LOCKED_SAVINGS = "Locked Savings"
_LOCKED_STAKING = "Locked Staking"
_SOLO_AIRDROP = "SOLO airdrop"
_GENERAL_STAKING = "STAKING"
# More types added
_CASH_VOUCHER = "Cash Voucher"
_LAUNCHPAD = "launchpad"
_SAVINGS_TRAIL_FUND = "Savings Trail Fund"

_AIRDROP_LIST = [_SOLO_AIRDROP]
_INTEREST_LIST = [_FLEXIBLE, _FLEXIBLE_SAVINGS, _LOCKED, _LOCKED_SAVINGS, _SAVINGS_TRAIL_FUND]
_STAKING_LIST = [_ETH_STAKING, _LOCKED_STAKING, _BNB_VAULT, _LAUNCH_POOL, _GENERAL_STAKING, _LAUNCHPAD]
_INCOME_LIST = [_CASH_VOUCHER]

class InputPlugin(AbstractCcxtInputPlugin):
    __EXCHANGE_NAME: str = "Binance.com"
    __PLUGIN_NAME: str = "Binance.com_REST"
    __DEFAULT_THREAD_COUNT: int = 1

    def __init__(
        self,
        account_holder: str,
        api_key: str,
        api_secret: str,
        native_fiat: str,
        username: Optional[str] = None,
        thread_count: Optional[int] = __DEFAULT_THREAD_COUNT,
    ) -> None:
        self.__api_key = api_key
        self.__api_secret = api_secret
        # We will have a default start time of July 13th, 2017 since Binance Exchange officially launched on July 14th Beijing Time.
        super().__init__(account_holder, datetime(2017, 7, 13, 0, 0, 0, 0), native_fiat, thread_count)
        self.__username = username

        # We have to know what markets and algos are on Binance so that we can pull orders using the market
        self.__algos: List[str] = []

    def exchange_name(self) -> str:
        return self.__EXCHANGE_NAME

    def plugin_name(self) -> str:
        return self.__PLUGIN_NAME

    def _initialize_client(self) -> binance:
        return binance(
            {
                "apiKey": self.__api_key,
                "enableRateLimit": True,
                "secret": self.__api_secret,
            }
        )

    @property
    def _client(self) -> binance:
        super_client: Exchange = super()._client
        if not isinstance(super_client, binance):
            raise RP2RuntimeError("Exchange is not instance of class binance.")
        return super_client

    def _get_algos(self) -> List[str]:
        if self.__algos:
            return self.__algos
        if self.__username:
            binance_algos = self._client.sapiGetMiningPubAlgoList()
            for algo in binance_algos[_DATA]:
                self._logger.debug("Algo: %s", json.dumps(algo))
                self.__algos.append(algo[_ALGO_NAME])
            return self.__algos
        return []

    def _get_process_deposits_pagination_detail_set(self) -> Optional[AbstractPaginationDetailSet]:
        return DateBasedPaginationDetailSet(
            limit=_DEPOSIT_RECORD_LIMIT,
            exchange_start_time=self._start_time_ms,
            window=_NINETY_DAYS_IN_MS,
        )

    def _get_process_withdrawals_pagination_detail_set(self) -> Optional[AbstractPaginationDetailSet]:
        return DateBasedPaginationDetailSet(
            limit=_WITHDRAWAL_RECORD_LIMIT,
            exchange_start_time=self._start_time_ms,
            window=_NINETY_DAYS_IN_MS,
        )

    def _get_process_trades_pagination_detail_set(self) -> Optional[AbstractPaginationDetailSet]:
        return DateBasedPaginationDetailSet(
            limit=_TRADE_RECORD_LIMIT,
            exchange_start_time=self._start_time_ms,
            markets=self._get_markets(),
        )

    ### Multiple transaction processing

    def _process_gains(
        self,
        in_transactions: List[InTransaction],
        out_transactions: List[OutTransaction],
    ) -> None:
        ### Regular Dividends from Staking (including Eth staking) and Savings (Lending) after around May 8th, 2021 01:00 UTC.

        # We need milliseconds for Binance
        current_start = self._start_time_ms
        now_time = int(datetime.now().timestamp()) * _MS_IN_SECOND
        processing_result_list: List[Optional[ProcessOperationResult]] = []

        # The exact moment when Binance switched to unified dividends is unknown/unpublished.
        # This allows us an educated guess.
        earliest_record_epoch: int = 0

        # We will pull in 30 day periods. This allows for 16 assets with daily dividends.
        current_end = current_start + _THIRTY_DAYS_IN_MS

        while current_start < now_time:
            self._logger.debug("Pulling dividends/subscriptions/redemptions from %s to %s", current_start, current_end)

            # CCXT doesn't have a standard way to pull income, we must use the underlying API endpoint
            dividends = self._client.sapiGetAssetAssetDividend(params=({_START_TIME: current_start, _END_TIME: current_end, _LIMIT: _DIVIDEND_RECORD_LIMIT}))
            # {
            #     "rows":[
            #         {
            #             "id":1637366104,
            #             "amount":"10.00000000",
            #             "asset":"BHFT",
            #             "divTime":1563189166000,
            #             "enInfo":"BHFT distribution",
            #             "tranId":2968885920
            #         },
            #         {
            #             "id":1631750237,
            #             "amount":"10.00000000",
            #             "asset":"BHFT",
            #             "divTime":1563189165000,
            #             "enInfo":"BHFT distribution",
            #             "tranId":2968885920
            #         }
            #     ],
            #     "total":2
            # }

            self._logger.debug("Pulled a total of %s records for %s to %s", dividends[_TOTAL], current_start, current_end)

            # If user received more than 500 dividends in a 30 day period we need to shrink the window.
            if int(dividends[_TOTAL]) <= _DIVIDEND_RECORD_LIMIT:
                current_start = current_end + 1
                current_end = current_start + _THIRTY_DAYS_IN_MS
                with ThreadPool(self._thread_count) as pool:
                    processing_result_list = pool.map(self._process_dividend, dividends[_ROWS])

                for processing_result in processing_result_list:
                    if processing_result is None:
                        continue
                    if processing_result.in_transactions:
                        in_transactions.extend(processing_result.in_transactions)
            else:
                # Using implicit API so we need to follow Binance order, which sends latest record first ([0])
                # CCXT standard API sorts by timestamp, so latest record is last ([499])
                number_of_excess_records = int(dividends[_TOTAL]) - _DIVIDEND_RECORD_LIMIT
                current_end = int(dividends[_ROWS][number_of_excess_records][_DIV_TIME])  # times are inclusive
                self._logger.debug("Readjusting time window end to %s from %s", current_end, current_start + _THIRTY_DAYS_IN_MS)
                # current_end = current_start + _THIRTY_DAYS_IN_MS

            if not earliest_record_epoch and int(dividends[_TOTAL]) > 0:
                earliest_record_epoch = int(dividends[_ROWS][-1][_DIV_TIME]) - 1

            # We need to track subscription and redemption amounts since Binance will take a fee equal to the amount of interest
            # earned during the lock period if the user prematurely redeems their funds.

        # Old system Locked Savings

        old_savings: bool = False

        # Reset window
        current_start = self._start_time_ms
        current_end = current_start + _THIRTY_DAYS_IN_MS

        # The cummulative interest from a positionID
        total_current_interest: Dict[int, RP2Decimal] = {}

        # The cummulative interest payments made to each positionID
        total_current_payments: Dict[int, int] = {}

        # Subscriptions organized [asset][amount] = timestamp in milliseconds
        current_subscriptions: Dict[str, Dict[str, Dict[str, str]]] = {}

        # We will step backward in time from the switch over
        while current_start < now_time:
            self._logger.debug("Pulling locked staking from older api system from %s to %s", current_start, current_end)

            locked_staking = self._client.sapi_get_staking_stakingrecord(
                params=({_START_TIME: current_start, _END_TIME: current_end, _PRODUCT: _STAKING, _TXN_TYPE: _INTEREST_PARAMETER, _SIZE: _INTEREST_SIZE_LIMIT})
            )
            # [
            #   {
            #       'positionId': '7146912',
            #       'time': '1624233772000',
            #       'asset': 'BTC',
            #       'amount': '0.017666',
            #       'status': 'SUCCESS'
            #   },
            #   {
            #       'positionId': '7147052',
            #       'time': '1624147893000',
            #       'asset': 'BTC',
            #       'amount': '0.0176665',
            #       'status': 'SUCCESS'
            #   }
            # ]
            # NOTE: All values are str
            processing_result_list = []
            for stake_dividend in locked_staking:
                if int(stake_dividend[_TIME]) < earliest_record_epoch:
                    (self._logger).debug("Locked Staking (OLD): %s", json.dumps(stake_dividend))
                    stake_dividend[_EN_INFO] = "Locked Staking/Savings (OLD)"
                    stake_dividend[_ID] = Keyword.UNKNOWN.value
                    stake_dividend[_DIV_TIME] = stake_dividend[_TIME]
                    processing_result_list.append(self._process_gain(stake_dividend, Keyword.STAKING))
                    old_savings = True

                # Early redemption penalty tracking. Needs to be recorded even for new system.
                position_id: int = stake_dividend[_POSITION_ID]
                total_current_interest[position_id] = total_current_interest.get(position_id, ZERO) + RP2Decimal(str(stake_dividend[_AMOUNT]))
                total_current_payments[position_id] = total_current_payments.get(position_id, 0) + 1

            for processing_result in processing_result_list:
                if processing_result is None:
                    continue
                if processing_result.in_transactions:
                    in_transactions.extend(processing_result.in_transactions)

            locked_subscriptions = self._client.sapi_get_staking_stakingrecord(
                params=({_START_TIME: current_start, _END_TIME: current_end, _PRODUCT: _STAKING, _TXN_TYPE: _SUBSCRIPTION, _SIZE: _INTEREST_SIZE_LIMIT})
            )
            # [
            #     {
            #         "time": "1624147893000",
            #         "asset": "BTC",
            #         "amount": "1",
            #         "lockPeriod": "10",
            #         "type": "NORMAL",
            #         "status": "SUCCESS",
            #     },
            #     {
            #         "time": "1624147893000",
            #         "asset": "BTC",
            #         "amount": "1",
            #         "lockPeriod": "10",
            #         "type": "NORMAL",
            #         "status": "SUCCESS",
            #     }
            # ]
            # NOTE: all values are str

            for subscription in locked_subscriptions:
                # If the dict already exists add another key, if not add new dict
                if current_subscriptions.get(subscription[_ASSET]):
                    current_subscriptions[subscription[_ASSET]][f"{RP2Decimal(subscription[_AMOUNT]):.13f}"] = subscription
                else:
                    current_subscriptions[subscription[_ASSET]] = {f"{RP2Decimal(subscription[_AMOUNT]):.13f}": subscription}

            locked_redemptions = self._client.sapi_get_staking_stakingrecord(
                params=({_START_TIME: current_start, _END_TIME: current_end, _PRODUCT: _STAKING, _TXN_TYPE: _REDEMPTION, _SIZE: _INTEREST_SIZE_LIMIT})
            )
            # [
            #         {
            #             "positionId": "12345",
            #             "time": "1624147893000"
            #             "asset": "BTC",
            #             "amount": "1",
            #             "deliverDate": "1624147895000"
            #             "status": "PAID",
            #         },
            #         {
            #             "positionId": "12346",
            #             "time": "1624147993000"
            #             "asset": "BTC",
            #             "amount": "0.95",
            #             "deliverDate": "1624148093000"
            #             "status": "PAID",
            #         }
            # ]
            # NOTE: all values are str

            for redemption in locked_redemptions:
                redemption_amount: str = f"{RP2Decimal(redemption[_AMOUNT]):.13f}"

                # Check if there is a subscription with this asset and if the redemption amount is equal to the subscription amount
                if redemption[_ASSET] in current_subscriptions and redemption_amount not in current_subscriptions[redemption[_ASSET]]:
                    # If they do not equal we need to calculate what the amended principal should be based on total interest paid to that productId
                    total_interest_earned: RP2Decimal = total_current_interest[redemption[_POSITION_ID]]
                    original_principal: str = f"{(RP2Decimal(redemption[_AMOUNT]) + RP2Decimal(str(total_interest_earned))):.13f}"
                    earliest_redemption_timestamp: int = 0

                    if str(original_principal) in current_subscriptions[redemption[_ASSET]]:
                        subscription_time: int = int(current_subscriptions[redemption[_ASSET]][str(original_principal)][_TIME])
                        lockperiod_in_ms: int = int(current_subscriptions[redemption[_ASSET]][str(original_principal)][_LOCK_PERIOD]) * _ONE_DAY_IN_MS
                        earliest_redemption_timestamp = subscription_time + lockperiod_in_ms

                    else:
                        raise RP2RuntimeError(
                            f"Internal Error: Principal ({original_principal}) minus paid interest ({RP2Decimal(str(total_interest_earned))}) does not equal"
                            f" returned principal ({RP2Decimal(redemption[_AMOUNT])}) on locked savings position ID - {redemption[_POSITION_ID]}."
                        )

                    # There is some lag time between application for the subscription and when the subscription actually starts ~ 2 days
                    if (int(redemption[_TIME]) - int(earliest_redemption_timestamp)) < 2 * _ONE_DAY_IN_MS:
                        out_transactions.append(
                            OutTransaction(
                                plugin=self.__PLUGIN_NAME,
                                unique_id=Keyword.UNKNOWN.value,
                                raw_data=json.dumps(redemption),
                                timestamp=self._rp2_timestamp_from_ms_epoch(redemption[_DELIVER_DATE]),
                                asset=redemption[_ASSET],
                                exchange=self.__EXCHANGE_NAME,
                                holder=self.account_holder,
                                transaction_type=Keyword.FEE.value,
                                spot_price=Keyword.UNKNOWN.value,
                                crypto_out_no_fee="0",
                                crypto_fee=str(total_interest_earned),
                                crypto_out_with_fee=str(total_interest_earned),
                                fiat_out_no_fee=None,
                                fiat_fee=None,
                                notes=(f"Penalty Fee for {redemption[_POSITION_ID]}"),
                            )
                        )

                    else:
                        raise RP2RuntimeError(
                            f"Internal Error: The redemption time ({self._rp2_timestamp_from_ms_epoch(redemption[_TIME])}) is not in the redemption window "
                            f"({self._rp2_timestamp_from_ms_epoch(str(earliest_redemption_timestamp))} + 2 days)."
                        )

                elif redemption[_ASSET] in current_subscriptions and redemption_amount in current_subscriptions[redemption[_ASSET]]:
                    self._logger.debug("Locked Savings positionId %s redeemed successfully.", redemption[_POSITION_ID])

                else:
                    raise RP2RuntimeError(f"Internal Error: Orphaned Redemption. Please open an issue at {self.ISSUES_URL}.")

            # if we returned the limit, we need to roll the window forward to the last time
            if len(locked_redemptions) < _INTEREST_SIZE_LIMIT:
                current_start = current_end + 1
                current_end = current_start + _THIRTY_DAYS_IN_MS
            else:
                current_start = now_time - 1  # int(locked_redemptions[0][_TIME]) + 1
                current_end = now_time  # current_start + _THIRTY_DAYS_IN_MS

        # Old system Flexible Savings

        # Reset window
        current_start = self._start_time_ms
        current_end = current_start + _THIRTY_DAYS_IN_MS

        # We will step backward in time from the switch over
        while current_start < earliest_record_epoch:
            self._logger.debug("Pulling flexible saving from older api system from %s to %s", current_start, current_end)

            flexible_saving = self._client.sapi_get_lending_union_interesthistory(
                params=({_START_TIME: current_start, _END_TIME: current_end, _LENDING_TYPE: _DAILY, _SIZE: _INTEREST_SIZE_LIMIT})
            )
            # [
            #     {
            #         "asset": "BUSD",
            #         "interest": "0.00006408",
            #         "lendingType": "DAILY",
            #         "productName": "BUSD",
            #         "time": 1577233578000
            #     },
            #     {
            #         "asset": "USDT",
            #         "interest": "0.00687654",
            #         "lendingType": "DAILY",
            #         "productName": "USDT",
            #         "time": 1577233562000
            #     }
            # ]
            processing_result_list = []
            for saving in flexible_saving:
                self._logger.debug("Flexible Saving: %s", json.dumps(saving))
                saving[_EN_INFO] = "Flexible Savings (OLD)"
                saving[_ID] = Keyword.UNKNOWN.value
                saving[_DIV_TIME] = saving[_TIME]
                saving[_AMOUNT] = saving[_INTEREST_FIELD]
                processing_result_list.append(self._process_gain(saving, Keyword.INTEREST))
                old_savings = True

            for processing_result in processing_result_list:
                if processing_result is None:
                    continue
                if processing_result.in_transactions:
                    in_transactions.extend(processing_result.in_transactions)

            # if we returned the limit, we need to roll the window forward to the last time
            if len(flexible_saving) < _INTEREST_SIZE_LIMIT:
                current_start = current_end + 1
                current_end = current_start + _THIRTY_DAYS_IN_MS
            else:
                current_start = int(flexible_saving[0][_TIME]) + 1
                current_end = current_start + _THIRTY_DAYS_IN_MS

            current_end = min(current_end, earliest_record_epoch)

        if old_savings:
            # Since we are making a guess at the cut off, there might be errors.
            self._logger.warning(
                "Pre-May 8th, 2021 savings detected. Please be aware that there may be duplicate or missing savings records around May 8th, 2021."
            )

        ### Mining Income

        # username is only required when pulling mining data
        for algo in self._get_algos():
            # Binance uses pages for mining payments
            current_page = 1
            while True:
                results = self._client.sapiGetMiningPaymentList(
                    params=({_ALGO: algo, _USERNAME: self.__username, _PAGE_INDEX: current_page, _PAGE_SIZE: _MINING_PAGE_LIMIT})
                )
                # {
                #   "code": 0,
                #   "msg": "",
                #   "data": {
                #     "accountProfits": [
                #       {
                #         "time": 1586188800000,            // Mining date
                #         "type": "31", // 0:Mining Wallet,5:Mining Address,7:Pool Savings,
                #           8:Transferred,31:Income Transfer ,32:Hashrate Resale-Mining Wallet 33:Hashrate Resale-Pool Savings
                #         "hashTransfer": null,            // Transferred Hashrate
                #         "transferAmount": null,          // Transferred Income
                #         "dayHashRate": 129129903378244,  // Daily Hashrate
                #         "profitAmount": 8.6083060304,   //Earnings Amount
                #         "coinName":"BTC",              // Coin Type
                #         "status": "2"    //Status：0:Unpaid， 1:Paying  2：Paid
                #       },
                #       {
                #         "time": 1607529600000,
                #         "coinName": "BTC",
                #         "type": "0", // String
                #         "dayHashRate": 9942053925926,
                #         "profitAmount": 0.85426469,
                #         "hashTransfer": 200000000000,
                #         "transferAmount": 0.02180958,
                #         "status": "2"
                #       },
                #       {
                #         "time": 1607443200000,
                #         "coinName": "BTC",
                #         "type": "31",
                #         "dayHashRate": 200000000000,
                #         "profitAmount": 0.02905916,
                #         "hashTransfer": null,
                #         "transferAmount": null,
                #         "status": "2"
                #       }
                #     ],
                #     "totalNum": 3,          // Total Rows
                #     "pageSize": 20          // Rows per page
                #   }
                # }

                if results[_DATA][_TOTAL_NUM] != "0":
                    profits: List[Dict[str, Union[int, str]]] = results[_DATA][_ACCOUNT_PROFITS]
                    processing_result_list = []
                    for result in profits:
                        self._logger.debug("Mining profit: %s", json.dumps(result))

                        # Currently the plugin only supports standard mining deposits
                        # Payment must also be made (status=2) in order to be counted
                        if result[_TYPE] == "0" and result[_STATUS] == "2":
                            processing_result_list.append(self._process_gain(result, Keyword.MINING))
                        else:
                            self._logger.error(
                                "WARNING: Unsupported Mining Transaction Type: %s.\nFull Details: %s\nPlease open an issue at %s.",
                                result[_TYPE],
                                json.dumps(result),
                                self.ISSUES_URL,
                            )

                    for processing_result in processing_result_list:
                        if processing_result is None:
                            continue
                        if processing_result.in_transactions:
                            in_transactions.extend(processing_result.in_transactions)

                    if len(profits) == _MINING_PAGE_LIMIT:
                        current_page += 1
                    else:
                        break
                else:
                    break

    def _process_implicit_api(  # pylint: disable=unused-argument
        self,
        in_transactions: List[InTransaction],
        out_transactions: List[OutTransaction],
        intra_transactions: List[IntraTransaction],
    ) -> None:
        # We need milliseconds for Binance
        now_time = int(datetime.now().timestamp()) * _MS_IN_SECOND

        processing_result_list: List[Optional[ProcessOperationResult]] = []

        # Crypto Bought with fiat. Technically this is a deposit of fiat that is used for a market order that fills immediately.
        # No limit on the date range
        # fiat (payments and deposits) takes the 'beginTime' param in contrast to other functions that take 'startTime'
        fiat_payments = self._client.sapiGetFiatPayments(params=({_TRANSACTION_TYPE: 0, _BEGIN_TIME: self._start_time_ms, _END_TIME: now_time}))
        # {
        #   "code": "000000",
        #   "message": "success",
        #   "data": [
        #   {
        #      "orderNo": "353fca443f06466db0c4dc89f94f027a",
        #      "sourceAmount": "20.0",  // Fiat trade amount
        #      "fiatCurrency": "EUR",   // Fiat token
        #      "obtainAmount": "4.462", // Crypto trade amount
        #      "cryptoCurrency": "LUNA",  // Crypto token
        #      "totalFee": "0.2",    // Trade fee
        #      "price": "4.437472",
        #      "status": "Failed",  // Processing, Completed, Failed, Refunded
        #      "createTime": 1624529919000,
        #      "updateTime": 1624529919000
        #   }
        #   ],
        #   "total": 1,
        #   "success": true
        # }
        if _DATA in fiat_payments:
            with ThreadPool(self._thread_count) as pool:
                processing_result_list = pool.map(self._process_fiat_payment, fiat_payments[_DATA])

            for processing_result in processing_result_list:
                if processing_result is None:
                    continue
                if processing_result.in_transactions:
                    in_transactions.extend(processing_result.in_transactions)
                if processing_result.out_transactions:
                    out_transactions.extend(processing_result.out_transactions)

        # Process actual fiat deposits (no limit on the date range)
        # Fiat deposits can also be pulled via CCXT fetch_deposits by cycling through legal_money
        # Using the underlying api endpoint is faster for Binance.
        # Note that this is the same endpoint as withdrawls, but with _TRANSACTION_TYPE set to 0 (for deposits)
        fiat_deposits = self._client.sapiGetFiatOrders(params=({_TRANSACTION_TYPE: 0, _BEGIN_TIME: self._start_time_ms, _END_TIME: now_time}))
        #    {
        #      "code": "000000",
        #      "message": "success",
        #      "data": [
        #        {
        #          "orderNo": "25ced37075c1470ba8939d0df2316e23",
        #          "fiatCurrency": "EUR",
        #          "indicatedAmount": "15.00",
        #          "amount": "15.00",
        #          "totalFee": "0.00",
        #          "method": "Card",
        #          "status": "Failed",  // Processing, Failed, Successful, Finished, Refunding, Refunded, Refund Failed, Order Partial credit Stopped
        #          "createTime": 1627501026000,
        #          "updateTime": 1627501027000
        #        }
        #      ],
        #      "total": 1,
        #      "success": True
        #    }
        if _DATA in fiat_deposits:
            with ThreadPool(self._thread_count) as pool:
                processing_result_list = pool.map(self._process_fiat_deposit_order, fiat_deposits[_DATA])

            for processing_result in processing_result_list:
                if processing_result is None:
                    continue
                if processing_result.in_transactions:
                    in_transactions.extend(processing_result.in_transactions)

        # Process actual fiat withdrawls (no limit on the date range)
        # Fiat deposits can also be pulled via CCXT fetch_withdrawls by cycling through legal_money
        # Using the underlying api endpoint is faster for Binance.
        # Note that this is the same endpoint as deposits, but with _TRANSACTION_TYPE set to 1 (for withdrawls)
        fiat_withdrawals = self._client.sapiGetFiatOrders(params=({_TRANSACTION_TYPE: 1, _BEGIN_TIME: self._start_time_ms, _END_TIME: now_time}))
        #    {
        #      "code": "000000",
        #      "message": "success",
        #      "data": [
        #        {
        #          "orderNo": "25ced37075c1470ba8939d0df2316e23",
        #          "fiatCurrency": "EUR",
        #          "indicatedAmount": "15.00",
        #          "amount": "15.00",
        #          "totalFee": "0.00",
        #          "method": "Card",
        #          "status": "Failed",  // Processing, Failed, Successful, Finished, Refunding, Refunded, Refund Failed, Order Partial credit Stopped
        #          "createTime": 1627501026000,
        #          "updateTime": 1627501027000
        #        }
        #      ],
        #      "total": 1,
        #      "success": True
        #    }
        if _DATA in fiat_withdrawals:
            with ThreadPool(self._thread_count) as pool:
                processing_result_list = pool.map(self._process_fiat_withdrawal_order, fiat_withdrawals[_DATA])

            for processing_result in processing_result_list:
                if processing_result is None:
                    continue
                if processing_result.out_transactions:
                    out_transactions.extend(processing_result.out_transactions)

        ### Dust Trades

        # We need milliseconds for Binance
        current_start = self._start_time_ms

        # We will pull in 30 day periods
        # If the user has more than 100 dust trades in a 30 day period this will break.
        # Maybe we can set a smaller window in the .ini file?
        current_end = current_start + _THIRTY_DAYS_IN_MS
        while current_start < now_time:
            dust_trades = self._client.fetch_my_dust_trades(params=({_START_TIME: current_start, _END_TIME: current_end}))
            # CCXT returns the same json as .fetch_trades()

            # Binance only returns 100 dust trades per call. If we hit the limit we will have to crawl
            # over each 'dribblet'. Each dribblet can have multiple assets converted into BNB at the same time.
            # If the user converts more than 100 assets at one time, we can not retrieve accurate records.
            if len(dust_trades) == _DUST_TRADE_RECORD_LIMIT:
                first_dribblet = [x for x in dust_trades if x[_DIV_TIME] == dust_trades[0][_DIV_TIME]]
                if len(first_dribblet) == _DUST_TRADE_RECORD_LIMIT:
                    raise RP2RuntimeError(
                        f"Internal error: too many assets dusted at the same time: " f"{self._rp2_timestamp_from_ms_epoch(str(dust_trades[0][_DIV_TIME]))}"
                    )

                with ThreadPool(self._thread_count) as pool:
                    processing_result_list = pool.map(self._process_dust_trade, first_dribblet)

                for processing_result in processing_result_list:
                    if processing_result is None:
                        continue
                    if processing_result.in_transactions:
                        in_transactions.extend(processing_result.in_transactions)
                    if processing_result.out_transactions:
                        out_transactions.extend(processing_result.out_transactions)

                # Shift the call window forward past this dribblet
                current_start = first_dribblet[len(first_dribblet) - 1][_DIV_TIME] + 1
                current_end = current_start + _THIRTY_DAYS_IN_MS
                break

            with ThreadPool(self._thread_count) as pool:
                processing_result_list = pool.map(self._process_dust_trade, dust_trades)

            for processing_result in processing_result_list:
                if processing_result is None:
                    continue
                if processing_result.in_transactions:
                    in_transactions.extend(processing_result.in_transactions)
                if processing_result.out_transactions:
                    out_transactions.extend(processing_result.out_transactions)

            current_start = current_end + 1
            current_end = current_start + _THIRTY_DAYS_IN_MS

    def _process_dividend(self, dividend: Any, notes: Optional[str] = None) -> ProcessOperationResult:
        self._logger.debug("Dividend: %s", json.dumps(dividend))
        if dividend[_EN_INFO] in _STAKING_LIST or re.search("[dD]istribution", dividend[_EN_INFO]) or re.search("[sS]taking", dividend[_EN_INFO]):
            return self._process_gain(dividend, Keyword.STAKING, notes)
        if dividend[_EN_INFO] in _INTEREST_LIST:
            return self._process_gain(dividend, Keyword.INTEREST, notes)
        if dividend[_EN_INFO] in _AIRDROP_LIST or re.search("[aA]ir[df]rop", dividend[_EN_INFO]):
            return self._process_gain(dividend, Keyword.AIRDROP, notes)
        if dividend[_EN_INFO] in _INCOME_LIST:
            return self._process_gain(dividend, Keyword.INCOME, notes)
        self._logger.error("WARNING: Unrecognized Dividend: %s. Please open an issue at %s", dividend[_EN_INFO], self.ISSUES_URL)
        return ProcessOperationResult(in_transactions=[], out_transactions=[], intra_transactions=[])

    def _process_dust_trade(self, dust: Any, notes: Optional[str] = None) -> ProcessOperationResult:
        self._logger.debug("Dust: %s", json.dumps(dust))
        # dust trades have a null id, and if multiple assets are dusted at the same time, all are assigned same ID
        dust_trade: Trade = self._to_trade(dust[_SYMBOL], str(dust[_AMOUNT]), str(dust[_COST]))
        # Some markets exist where BNB is the base asset (e.g. BTCBNB)
        if dust_trade.base_asset != "BNB":
            dust[_ID] = f"{dust[_ORDER]}{dust_trade.base_asset}"
        else:
            dust[_ID] = f"{dust[_ORDER]}{dust_trade.quote_asset}"
        return self._process_buy_and_sell(dust, notes)

    def _process_gain(self, transaction: Any, transaction_type: Keyword, notes: Optional[str] = None) -> ProcessOperationResult:
        self._logger.debug("Gain: %s", json.dumps(transaction))
        in_transaction_list: List[InTransaction] = []

        if transaction_type == Keyword.MINING:
            amount: RP2Decimal = RP2Decimal(str(transaction[_PROFIT_AMOUNT]))
            notes = f"{notes + '; ' if notes else ''}'Mining profit'"
            in_transaction_list.append(
                InTransaction(
                    plugin=self.__PLUGIN_NAME,
                    unique_id=Keyword.UNKNOWN.value,
                    raw_data=json.dumps(transaction),
                    timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_TIME]),
                    asset=transaction[_COIN_NAME],
                    exchange=self.__EXCHANGE_NAME,
                    holder=self.account_holder,
                    transaction_type=transaction_type.value,
                    spot_price=Keyword.UNKNOWN.value,
                    crypto_in=str(amount),
                    crypto_fee=None,
                    fiat_in_no_fee=None,
                    fiat_in_with_fee=None,
                    fiat_fee=None,
                    notes=notes,
                )
            )
        elif RP2Decimal(transaction[_AMOUNT]) != ZERO:  # Sometimes Binance reports interest payments with zero amounts
            amount = RP2Decimal(transaction[_AMOUNT])
            notes = f"{notes + '; ' if notes else ''}{transaction[_EN_INFO]}"

            in_transaction_list.append(
                InTransaction(
                    plugin=self.__PLUGIN_NAME,
                    unique_id=str(transaction[_ID]),  # Binance sometimes has two ids for one tranid
                    raw_data=json.dumps(transaction),
                    timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_DIV_TIME]),
                    asset=transaction[_ASSET],
                    exchange=self.__EXCHANGE_NAME,
                    holder=self.account_holder,
                    transaction_type=transaction_type.value,
                    spot_price=Keyword.UNKNOWN.value,
                    crypto_in=str(amount),
                    crypto_fee=None,
                    fiat_in_no_fee=None,
                    fiat_in_with_fee=None,
                    fiat_fee=None,
                    notes=notes,
                )
            )

        return ProcessOperationResult(in_transactions=in_transaction_list, out_transactions=[], intra_transactions=[])

    def _process_fiat_deposit_order(self, transaction: Any, notes: Optional[str] = None) -> ProcessOperationResult:
        return self._process_fiat_order(transaction, Keyword.BUY, notes)

    def _process_fiat_withdrawal_order(self, transaction: Any, notes: Optional[str] = None) -> ProcessOperationResult:
        return self._process_fiat_order(transaction, Keyword.SELL, notes)

    def _process_fiat_order(self, transaction: Any, transaction_type: Keyword, notes: Optional[str] = None) -> ProcessOperationResult:
        self._logger.debug("Fiat Order (%s): %s", transaction_type.value, json.dumps(transaction))
        in_transaction_list: List[InTransaction] = []
        out_transaction_list: List[OutTransaction] = []
        if transaction[_STATUS] == "Successful":
            amount: RP2Decimal = RP2Decimal(transaction[_INDICATED_AMOUNT])
            fee: RP2Decimal = RP2Decimal(transaction[_TOTAL_FEE])
            notes = f"{notes + '; ' if notes else ''}Fiat {transaction_type.value} of {transaction[_FIAT_CURRENCY]}"
            if transaction_type == Keyword.BUY:
                in_transaction_list.append(
                    InTransaction(
                        plugin=self.__PLUGIN_NAME,
                        unique_id=transaction[_ORDER_NO],
                        raw_data=json.dumps(transaction),
                        timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                        asset=transaction[_FIAT_CURRENCY],
                        exchange=self.__EXCHANGE_NAME,
                        holder=self.account_holder,
                        transaction_type=transaction_type.value,
                        spot_price="1",
                        crypto_in=str(amount),
                        crypto_fee=str(fee),
                        fiat_in_no_fee=None,
                        fiat_in_with_fee=None,
                        fiat_fee=None,
                        fiat_ticker=transaction[_FIAT_CURRENCY],
                        notes=notes,
                    )
                )
            elif transaction_type == Keyword.SELL:
                out_transaction_list.append(
                    OutTransaction(
                        plugin=self.__PLUGIN_NAME,
                        unique_id=transaction[_ORDER_NO],
                        raw_data=json.dumps(transaction),
                        timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                        asset=transaction[_FIAT_CURRENCY],
                        exchange=self.__EXCHANGE_NAME,
                        holder=self.account_holder,
                        transaction_type=Keyword.SELL.value,
                        spot_price="1",
                        crypto_out_no_fee=str(amount),
                        crypto_fee=str(fee),
                        fiat_out_no_fee=None,
                        fiat_fee=None,
                        fiat_ticker=transaction[_FIAT_CURRENCY],
                        notes=notes,
                    )
                )
        return ProcessOperationResult(in_transactions=in_transaction_list, out_transactions=out_transaction_list, intra_transactions=[])

    def _process_fiat_payment(self, transaction: Any, notes: Optional[str] = None) -> ProcessOperationResult:
        self._logger.debug("Fiat Payment: %s", json.dumps(transaction))
        in_transaction_list: List[InTransaction] = []
        out_transaction_list: List[OutTransaction] = []

        if transaction[_STATUS] == "Completed":
            if self.is_native_fiat(transaction[_FIAT_CURRENCY]):
                # For double entry accounting purposes we must create a fiat InTransaction
                in_transaction_list.append(
                    InTransaction(
                        plugin=self.__PLUGIN_NAME,
                        unique_id=transaction[_ORDER_NO],
                        raw_data=json.dumps(transaction),
                        timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                        asset=transaction[_CRYPTOCURRENCY],
                        exchange=self.__EXCHANGE_NAME,
                        holder=self.account_holder,
                        transaction_type=Keyword.BUY.value,
                        spot_price=str(RP2Decimal(transaction[_PRICE])),
                        crypto_in=transaction[_OBTAIN_AMOUNT],
                        crypto_fee=None,
                        fiat_in_no_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT]) - RP2Decimal(transaction[_TOTAL_FEE])),
                        fiat_in_with_fee=str(transaction[_SOURCE_AMOUNT]),
                        fiat_fee=str(RP2Decimal(transaction[_TOTAL_FEE])),
                        fiat_ticker=transaction[_FIAT_CURRENCY],
                        notes=(f"{notes + '; ' if notes else ''}Buy transaction for native fiat payment orderNo - {transaction[_ORDER_NO]}"),
                    )
                )

                # This is an OutTransaction for a buy or conversion based on what the native fiat is.
                out_transaction_list.append(
                    OutTransaction(
                        plugin=self.__PLUGIN_NAME,
                        unique_id=transaction[_ORDER_NO],
                        raw_data=json.dumps(transaction),
                        timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                        asset=transaction[_FIAT_CURRENCY],
                        exchange=self.__EXCHANGE_NAME,
                        holder=self.account_holder,
                        transaction_type=Keyword.SELL.value,
                        spot_price=str(RP2Decimal("1")),
                        crypto_out_no_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT]) - RP2Decimal(transaction[_TOTAL_FEE])),
                        crypto_fee=str(RP2Decimal(transaction[_TOTAL_FEE])),
                        crypto_out_with_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT])),
                        fiat_out_no_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT]) - RP2Decimal(transaction[_TOTAL_FEE])),
                        fiat_fee=None,
                        fiat_ticker=transaction[_FIAT_CURRENCY],
                        notes=(f"{notes + '; ' if notes else ''}Sell transaction conversion from native fiat orderNo - " f"{transaction[_ORDER_NO]}"),
                    )
                )
            else:
                in_transaction_list.append(
                    InTransaction(
                        plugin=self.__PLUGIN_NAME,
                        unique_id=transaction[_ORDER_NO],
                        raw_data=json.dumps(transaction),
                        timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                        asset=transaction[_CRYPTOCURRENCY],
                        exchange=self.__EXCHANGE_NAME,
                        holder=self.account_holder,
                        transaction_type=Keyword.BUY.value,
                        spot_price=Keyword.UNKNOWN.value,
                        crypto_in=transaction[_OBTAIN_AMOUNT],
                        crypto_fee=None,
                        fiat_in_no_fee=None,
                        fiat_in_with_fee=None,
                        fiat_fee=None,
                        notes=(f"{notes + '; ' if notes else ''}Buy transaction conversion from non-native_fiat orderNo - {transaction[_ORDER_NO]}"),
                    )
                )

                out_transaction_list.append(
                    OutTransaction(
                        plugin=self.__PLUGIN_NAME,
                        unique_id=transaction[_ORDER_NO],
                        raw_data=json.dumps(transaction),
                        timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                        asset=transaction[_FIAT_CURRENCY],
                        exchange=self.__EXCHANGE_NAME,
                        holder=self.account_holder,
                        transaction_type=Keyword.SELL.value,
                        spot_price=Keyword.UNKNOWN.value,
                        crypto_out_no_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT]) - RP2Decimal(transaction[_TOTAL_FEE])),
                        crypto_fee=str(RP2Decimal(transaction[_TOTAL_FEE])),
                        crypto_out_with_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT])),
                        fiat_out_no_fee=None,
                        fiat_fee=None,
                        notes=(f"{notes + '; ' if notes else ''}Sell transaction conversion from non-native_fiat orderNo - " f"{transaction[_ORDER_NO]}"),
                    )
                )

            # An InTransaction is needed for the fiat in order for the accounting to zero out
            in_transaction_list.append(
                InTransaction(
                    plugin=self.__PLUGIN_NAME,
                    unique_id=f"{transaction[_ORDER_NO]}/fiat_buy",
                    raw_data=json.dumps(transaction),
                    timestamp=self._rp2_timestamp_from_ms_epoch(transaction[_CREATE_TIME]),
                    asset=transaction[_FIAT_CURRENCY],
                    exchange=self.__EXCHANGE_NAME,
                    holder=self.account_holder,
                    transaction_type=Keyword.BUY.value,
                    spot_price=str(RP2Decimal("1")),
                    crypto_in=str(RP2Decimal(transaction[_SOURCE_AMOUNT])),
                    crypto_fee=None,
                    fiat_in_no_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT])),
                    fiat_in_with_fee=str(RP2Decimal(transaction[_SOURCE_AMOUNT])),
                    fiat_fee=None,
                    fiat_ticker=transaction[_FIAT_CURRENCY],
                    notes=(f"{notes + '; ' if notes else ''}Fiat deposit for orderNo - {transaction[_ORDER_NO]}"),
                )
            )

        return ProcessOperationResult(in_transactions=in_transaction_list, out_transactions=out_transaction_list, intra_transactions=[])


import os
import pyautogui

from time import sleep

def save_pic():
    cmd = """osascript -e 'tell app "Image Capture" to activate'"""
    sleep(0.1)
    os.system(cmd)
    pyautogui.click(x=370, y=116)
    pyautogui.click(x=370, y=116, button='right')
    sleep(0.1)
    pyautogui.click(x=393, y=150)
    sleep(0.3)
    cmd = """osascript -e 'tell app "iTerm2" to activate'"""
    os.system(cmd)


__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
__license__ = "AGPL v3"

from django.core.management.base import BaseCommand

from commentaries import factories

class Command(BaseCommand):
    help = "Create random Commentaries objects using the factories."

    def add_arguments(self, parser):
        parser.add_argument(
            "number",
            action="store",
            default=0,
            type=int,
            help="Number of Commentaries to add",
        )

    def handle(self, *args, **kwargs):
        self.create_commentaries(kwargs["number"])

    def create_commentaries(self, n):
        factories.CommentaryFactory.create_batch(n)
        self.stdout.write(
            self.style.SUCCESS("Successfully created {n} Commentaries.".format(n=n))
        )


import os
from db import db
from flask import session
from sqlalchemy import text

def lang_query(lng):
    return "SELECT COUNT(*) FROM Books b, Read r WHERE r.user_id=:id AND r.book_id=b.id AND b.lang=" + "'" + lng + "'"

def genre_query(genre):
    return "SELECT COUNT(*) FROM Genres g, Read r WHERE r.user_id=:id AND r.book_id=g.book_id AND g.genre="+"'"+genre+"'"

def execute_with_id(sql, id):
    return db.session.execute(sql, {"id": id}).fetchone()[0]

def get_basic_stats(id):
    sql = text("SELECT COUNT(*) FROM Read WHERE user_id=:id")
    n_books = execute_with_id(sql, id)
    sql = text(
        "SELECT SUM(b.pagenumber) FROM Read r, Books b WHERE r.user_id=:id AND r.book_id=b.id")
    n_pages = execute_with_id(sql, id)
    if not n_pages:
        n_pages = 0
    return [n_books, n_pages]

def get_lang_stats(id):
    total = get_basic_stats(id)[0]
    sql = text(lang_query("suomi"))
    n_fin = execute_with_id(sql, id)
    if not n_fin == 0:
        fin_per = round(n_fin/total*100)
    else:
        fin_per = 0
    sql = text(lang_query("englanti"))
    n_eng = execute_with_id(sql, id)
    if not n_eng == 0:
        eng_per = round(n_eng/total*100)
    else:
        eng_per = 0
    sql = text(lang_query("ruotsi"))
    n_sv = execute_with_id(sql, id)
    if not n_sv == 0:
        sv_per = round(n_sv/total*100)
    else:
        sv_per = 0
    return [(n_fin, fin_per), (n_eng, eng_per), (n_sv, sv_per)]

def get_genre_stats(id):
    total = get_basic_stats(id)[0]
    genres = ["Lapset", "Jännitys", "Mysteeri", "Klassikko",
              "Kauhu", "Fantasia", "Scifi", "Romantiikka"]
    n_genres = []
    for g in genres:
        sql = text(genre_query(g))
        n_genre = execute_with_id(sql, id)
        if not n_genre == 0:
            n_genres.append((n_genre, round(n_genre/total*100)))
        else:
            n_genres.append((n_genre, 0))
    return n_genres


# Hand detection
import cv2 as cv
import numpy as np

img = cv.imread("/media/techio/Study Materials/Python Projects/hand.jpg")
img = cv.resize(img,(300,400))
img_gray = cv.cvtColor(img,cv.COLOR_BGR2GRAY)

blur = cv.medianBlur(img_gray,5)
# Determining threshold value
ret,thres = cv.threshold(blur,235,255,cv.THRESH_BINARY_INV)

# Finding contours and drawing
cnts,h = cv.findContours(thres,cv.RETR_EXTERNAL,cv.CHAIN_APPROX_SIMPLE)
# cv.drawContours(img,cnts,-1,(50,50,150),2)
print("Total contours = ",len(cnts))

for cnt in cnts:
    epsilon = 0.0001*cv.arcLength(cnt,True)
    data = cv.approxPolyDP(cnt,epsilon,True)

    hull = cv.convexHull(data)
    cv.drawContours(img,[cnt],-1,(50,50,150),2)
    cv.drawContours(img,[hull],-1,(0,255,0),2)

# convexity defect
hull2 = cv.convexHull(cnts[0],returnPoints = False)
# defects return an array which contain value (start pt, end pt,farthest pt,approx)
defect = cv.convexityDefects(cnts[0],hull2)
for i in range(defect.shape[0]):
    s,e,f,d = defect[i,0]
    print(s,e,f,d)
    start = tuple(cnt[s][0])
    end = tuple(cnt[e][0])
    far = tuple(cnt[f][0])
    cv.circle(img,far,5,[0,0,255],-1)

# Extreme pts
# topmost,bottommost, rightmost, leftmost
cmax = max(cnts,key=cv.contourArea)
extLeft = tuple(cmax[cmax[:,:,0].argmin()][0])
extRight = tuple(cmax[cmax[:,:,0].argmax()][0])
extTop = tuple(cmax[cmax[:,:,1].argmin()][0])
extBot = tuple(cmax[cmax[:,:,1].argmax()][0])

# drawing outline
cv.circle(img,extLeft,8,(255,0,255),-1) #pink
cv.circle(img,extRight,8,(0,125,255),-1) # brown
cv.circle(img,extTop,8,(255,10,0),-1) # blue
cv.circle(img,extBot,8,(19,152,152),-1) # green

cv.imshow("Original Image",img)
cv.imshow("Grayscale image",img_gray)
cv.imshow("Threshold",thres)
cv.waitKey(0)
cv.destroyAllWindows()


"""
Train model on MNIST data and save to file
"""
import argparse
import keras
from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
from keras.datasets import mnist
from keras.models import Sequential

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--modelfile', default='mnist_model.h5', type=str, help='Trained model will be written to this file')
    parser.add_argument('--epochs', default=5, type=int, help='# of epochs to train for')
    parser.add_argument('--batchsize', default=64, type=int, help='Batch size to use for each epoch')

    args = parser.parse_args()

    # Data formatting

    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    x_train = x_train.reshape(60000, 28, 28, 1)
    x_train = x_train.astype('float32')

    x_test = x_test.reshape(10000, 28, 28, 1)
    x_test = x_test.astype('float32')

    x_train /= 255
    x_test /= 255

    y_train = keras.utils.to_categorical(y_train, 10)
    y_test = keras.utils.to_categorical(y_test, 10)

    # Setup model architecture

    model = Sequential()
    model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
    model.add(Conv2D(64, (3,3), activation='relu'))
    model.add(MaxPooling2D((2, 2), 2))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(10, activation='softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer=keras.optimizers.Adadelta(),
                  metrics=['accuracy'])

    model.fit(x_train,y_train,
              epochs=args.epochs,
              batch_size=args.batchsize,
              verbose=2,
              validation_data=(x_test,y_test) )

    model.save(args.modelfile)

    score, accuracy = model.evaluate(x_test,y_test)
    print('score', score)
    print('accuracy', accuracy)


## Importing Required Libraries ##
import pandas as pd
import os
import numpy as np

## Loading data ##
oct = pd.read_csv("oct.csv")
nov = pd.read_csv("nov.csv")

## Killing NA's ##
oct = oct.fillna(0)
nov = nov.fillna(0)

## Comparison between sets ##
comparison_values = oct.values == nov.values
print (comparison_values)

## Numpy where function ##
rows, cols=np.where(comparison_values==False)

## Identifying and marking changes ##
for item in zip(rows,cols):
    oct.iloc[item[0], item[1]] = '{} !!! {}'.format(oct.iloc[item[0], item[1]],nov.iloc[item[0], item[1]])

## Saving new sheet with differences noted ##
oct.to_excel('./novchange.xlsx',index=False,header=True)


import numpy as np
import re
import itertools
import os
from collections import Counter

"""
Original taken from https://github.com/dennybritz/cnn-text-classification-tf
"""

def add_Problem(problem):
    #print(problem)
    problem_words = problem.split()
    pw_len = len(problem_words)
    check_success = True            # New problem should be added
    errors = []
    positive_examples_filename = "../data/rt-polarity.pos"
    negative_examples_filename = "../data/rt-polarity.neg"
    negative_pool_filename = "../data/rt-polarity.pool"          # Take negative_examples from pool
    negative_pool_tmp_filename = "../data/rt-polarity.pooltmp"

    f2check=[positive_examples_filename, negative_examples_filename, negative_pool_filename]
    # check if files exists
    for f in f2check:
        if not(os.path.exists(f)):
            check_success = False
            errors.append("File %s does not exist" %f)

    # Step1: Check if problem is already a problem in our data
    positive_examples = list(open(positive_examples_filename).readlines())
    for sentence in positive_examples:
        pos_ex_word_list = sentence.split()
        ewl_len = len(pos_ex_word_list)
        # print("problem_len: %d sentence_len: %d \n" % (pw_len, ewl_len))
        if ewl_len == pw_len:           # if word length is equal, check if problem is new
            matches = 0
            for word in pos_ex_word_list:
                if word in problem_words:
                    matches = matches + 1
            # print("problem_len: %d matches: %d \n" % (pw_len, matches))
            if pw_len < (matches + 2):   # new problems should be slightly different from old ones
                errors.append("New problems should be different from old ones")
                check_success = False

    # Step2: Check if Pool size is big enough
    negative_pool = list(open(negative_pool_filename).readlines())
    if len(negative_pool) < 1: # Not enough negative_examples
        errors.append("Not enough negative_examples in negative_pool")
        check_success = False

    if check_success:
        # Step3: write new problem to positive_examples file
        with open(positive_examples_filename, "a") as f:
            f.write(problem + '\n')

        # Step4: get one none problem from pool file
        n = 1   # read n lines
        nfirstlines = []
        with open(negative_pool_filename) as f, open(negative_pool_tmp_filename, "w") as out:
            for x in xrange(n):                 # read first n lines
                nfirstlines.append(next(f))
            for line in f:                      # stream data to tmp file
                out.write(line)
        # remove old pool file and make tmp to old
        os.remove(negative_pool_filename)
        os.rename(negative_pool_tmp_filename, negative_pool_filename)

        # Step5: add non problem to negative_examples file
        with open(negative_examples_filename, "a") as f:
            for line in nfirstlines:
                f.write(line)
        errors.append("Problem successfully added")
    return (check_success, errors)

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def load_data_and_labels():
    """
    Loads MR polarity data from files, splits the data into words and generates labels.
    Returns split sentences and labels.
    """
    # Load data from files
    positive_examples = list(open("../data/rt-polarity.pos").readlines())
    positive_examples = [s.strip() for s in positive_examples]
    negative_examples = list(open("../data/rt-polarity.neg").readlines())
    negative_examples = [s.strip() for s in negative_examples]
    # Split by words
    x_text = positive_examples + negative_examples
    x_text = [clean_str(sent) for sent in x_text]
    x_text = [s.split(" ") for s in x_text]
    # Generate labels
    positive_labels = [[0, 1] for _ in positive_examples]
    negative_labels = [[1, 0] for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]

def pad_sentences(sentences, padding_word="<PAD/>"):
    """
    Pads all sentences to the same length. The length is defined by the longest sentence.
    Returns padded sentences.
    """
    sequence_length = max(len(x) for x in sentences)
    padded_sentences = []
    for i in range(len(sentences)):
        sentence = sentences[i]
        num_padding = sequence_length - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences

def build_vocab(sentences):
    """
    Builds a vocabulary mapping from word to index based on the sentences.
    Returns vocabulary mapping and inverse vocabulary mapping.
    """
    # Build vocabulary
    word_counts = Counter(itertools.chain(*sentences))
    # Mapping from index to word
    vocabulary_inv = [x[0] for x in word_counts.most_common()]
    # Mapping from word to index
    vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}
    return [vocabulary, vocabulary_inv]

def build_input_data(sentences, labels, vocabulary):
    """
    Maps sentencs and labels to vectors based on a vocabulary.
    """
    x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences])
    y = np.array(labels)
    return [x, y]

def load_data():
    """
    Loads and preprocessed data for the MR dataset.
    Returns input vectors, labels, vocabulary, and inverse vocabulary.
    """
    # Load and preprocess data
    sentences, labels = load_data_and_labels()
    sentences_padded = pad_sentences(sentences)
    vocabulary, vocabulary_inv = build_vocab(sentences_padded)
    x, y = build_input_data(sentences_padded, labels, vocabulary)
    return [x, y, vocabulary, vocabulary_inv]

def batch_iter(data, batch_size, num_epochs):
    """
    Generates a batch iterator for a dataset.
    """
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data) / batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        shuffle_indices = np.random.permutation(np.arange(data_size))
        shuffled_data = data[shuffle_indices]
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]