# coding=utf-8
from fem_base.exploit_fun import *
import os
from fem_base.exploit_fun import data_path

res_pathldhsbc = data_path

N_eig = 10
lnm = 100
B = 30
V_maxmeV = 0
h = 0.1
pot_version = 1
gauge = "Sym"
l = lnm * 10**-9

VmeV, Th = vth_data(lnm, h, pot_version)

getsave_eig(N_eig, lnm, B, VmeV, V_maxmeV, Th, h, pot_version, gauge)


class Solution:
    def trailingZeroes(self, n: int) -> int:
        if(n<0):
            return -1
        c=0
        i=5
        while n/i>=1:
            c+= n/i
            i*=5
        return c
a =Solution()
num=14
print(int(a.trailingZeroes(num)))


# 제무제표 항목명과 DB 필드명 매핑
dic_reports = {
    '코드': 'code',
    '종목코드': 'code',
    'rdate': 'rdate',
    '자산총계': 'total_assets',
    '유동자산': 'current_assets',
    '부채총계': 'total_liabilities',
    '자본총계': 'total_equity',
    '매출액': 'net_sales',
    '매출원가': 'cost_of_sales',
    '매출총이익': 'gross_profit',
    '발표영업이익': 'operating_income',
    '영업이익': 'operating_income',
    '순이익': 'ongoing_operating_income',
    '당기순이익': 'ongoing_operating_income',
    '반기순이익': 'ongoing_operating_income',
    '분기순이익': 'ongoing_operating_income',
    '영업현금흐름': 'cash_flows_from_operatings',
    '영업활동으로인한현금흐름': 'cash_flows_from_operatings',
    '유동비율': 'current_ratio',
    '부채비율': 'debt_to_equity_ratio'
}

dic_companies = {
    '코드': 'code',
    '종목코드': 'code',
    '시장': 'market',
    '회사명': 'company',
    '종목명': 'company',
    '산업코드': 'ind_code',
    '산업명': 'industry'
}

dic_prices = {
    '종목코드': 'code',
    '대비': 'movement',
    '등락률': 'movement_ratio',
    '거래량': 'trading_volume',
    '거래대금': 'trading_value',
    '시가': 'open',
    '고가': 'high',
    '저가': 'low',
    '현재가': 'close',
    '시가총액': 'market_cap',
    '시가총액비중(%)': 'market_ratio',
    '상장주식수': 'total_shares',
    '외국인 보유주식수': 'foreign_shares',
    '외국인 지분율(%)': 'foreign_ratio'
}


# Запись в файл. writer()

import csv

# with open('students.csv', 'w') as file:
#     csv_writer = csv.writer(file)
#     csv_writer.writerow(['First name', 'Last name', 'Age'])
#     csv_writer.writerow(['Jack', 'White', 24])
#     csv_writer.writerow(['Jane', 'Black', 23])

# Нужно считать данные и поместить их в какую то переменную

with open('cars.csv') as file:
    csv_reader = csv.reader(file)
    next(csv_reader)
    make_model_list = []
    for car in csv_reader:
        make_model = [car[1], car[2]]
        make_model_list.append(make_model)
        print(make_model_list)

# мы получаем список в списке с маркой и моделью

#  Теперь мы можем записать в файл значения, должен создаться файл cars_make_model.csv в котором будут записаны данные

with open('cars_make_model.csv', 'w') as file:
    csv_writer = csv.writer(file)
    for row in make_model_list:
        csv.writer.writerow(row)


"""
Django settings for ktotback project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from django.contrib.messages import constants as messages

SETUP = "PRODUCTION"
SITE_ID = 1
SITE_URL = "https://batondata.co.uk"
SITE_NAME = "Baton Data"

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
#ROOT_DIR = environ.Path(__file__) - 3  # (skorie/config/settings/base.py - 3 = skorie/)

VERSION = "0.0.6 June 2022"
API_VERSION = "006"
API_URL = "https://batondata.co.uk/"
LOGO_URL = ''

SECRET_KEY = "ABCD"

DEBUG = False
USE_ASSETS = True

# prevents this error: *kwasync)#012django.db.utils.OperationalError: FATAL:  remaining connection slots are reserved for non-replication superuser connections
CONN_MAX_AGE = None

ALLOWED_HOSTS = ['batondata.co.uk','www.batondata.co.uk',  ]

AUTHENTICATION_BACKENDS = (
    # Needed to login by username in Django admin, regardless of `keycloak`
    # this can create a problem where the webfront end knows the user is logged in but accessing an API does
    # not pass the user in.
    "django.contrib.auth.backends.ModelBackend",

)

AUTH_USER_MODEL = "web.CustomUser"
ACCOUNT_USER_MODEL_USERNAME_FIELD = None
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_USERNAME_REQUIRED = False
ACCOUNT_AUTHENTICATION_METHOD = 'email'

# # tastypie is trying to load geo libraries
# GDAL_LIBRARY_PATH = '/usr/lib/ogdi/libgdal.so'

POST_OFFICE = {
    'DEFAULT_PRIORITY': 'now',
    'LOG_LEVEL': 2,
}

DEFAULT_AUTO_FIELD='django.db.models.AutoField'
# DATABASE_ROUTERS = ['config.dbrouters.HelpdeskRouter', ]

MIDDLEWARE = [

    'django.middleware.security.SecurityMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    # 'web.middleware.IDSubomainRequestMiddleware',

    'django.middleware.common.CommonMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',

    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    # 'django_user_agents.middleware.UserAgentMiddleware',

]

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',
    'django.contrib.humanize',
    'django_countries',

    # "post_office",
    'django_filters',
    'rest_framework',
    "rest_framework_api_key",
    'csvexport',
    # 'theme',

    'gadgetdb',
    'web',

]

ROOT_URLCONF = 'config.urls'

WSGI_APPLICATION = 'config.wsgi.application'

LANGUAGE_CODE = 'en-GB'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

LOGIN_URL = '/login/'
LOGIN_REDIRECT_URL = "/"
LOGOUT_REDIRECT_URL = "/"
APPEND_SLASH = True

ADMINS = [('Phoebe','phoebebright310@gmail.com'),('Phoebe','phoebe@skor.ie'), ]
MANAGERS = ADMINS
DEFAULT_FROM_EMAIL = "Skor.ie <info@skor.ie>"
DEFAULT_TO_EMAIL = "info@skor.ie"
SUPPORT_EMAIL = "info@skor.ie"
NOTIFY_NEW_USER_EMAILS = "phoebebright310@gmail.com"

EMAIL_BACKEND = 'post_office.EmailBackend'

MESSAGE_TAGS = {
    messages.DEBUG: 'alert-secondary',
    messages.INFO: 'alert-info',
    messages.SUCCESS: 'alert-success',
    messages.WARNING: 'alert-warning',
    messages.ERROR: 'alert-danger',
}

# Static files (CSS, JavaScript, Images)

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',

)

STATIC_ROOT = os.path.join(BASE_DIR, 'shared_static')  # this may well be changed in settings_local in production
STATIC_URL = '/static/'

# Absolute path to the directory that holds media.
MEDIA_ROOT = os.path.join(BASE_DIR, 'uploads')
MEDIA_URL = '/uploads/'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates'),

                 ],
        'APP_DIRS': True,
        'OPTIONS': {
            'debug': False,
            'context_processors': [

                'django.template.context_processors.debug',

                'django.template.context_processors.request',
                # "allauth.account.context_processors.account",
                # "allauth.socialaccount.context_processors.socialaccount",
                # 'countries_plus.context_processors.add_request_country',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'web.context_processors.include_settings',
            ],
        },
    },
]

# # countries_plus - will only work once you have country code already -
#
# COUNTRIES_PLUS_COUNTRY_HEADER = 'HTTP_CF_COUNTRY'
# COUNTIRES_PLUS_DEFAULT_ISO = 'IE'

COUNTRIES_ONLY = ['US', 'GB', 'IE', 'AU', 'NZ','CA']

ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
DEFAULT_HTTP_PROTOCOL = "https"
ACCOUNT_EMAIL_VERIFICATION = "none"   # currently using custom signup if not facebook

SOCIAL_AUTH_POSTGRES_JSONFIELD = True

# TEMPLATE_CONTEXT_PROCESSORS = TEMPLATES[0]['OPTIONS']['context_processors']   # keeps allauth happy

# Registration settings
# REGISTRATION_AUTO_LOGIN = True
# ACCOUNT_ACTIVATION_DAYS = 7

REST_FRAMEWORK = {

    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.IsAuthenticated',
    ],
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.SessionAuthentication',
        'tools.auth.ApiKeyAuthentication',

    ),
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
    'DEFAULT_THROTTLE_CLASSES': (
        'rest_framework.throttling.ScopedRateThrottle',
    ),
    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',),

}
API_KEY_CUSTOM_HEADER = "HTTP_BAT_API_KEY"

# CACHES = {
#     'default': {
#         'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
#         'LOCATION': '127.0.0.1:11211',
#     }
# }

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,

    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'formatters': {
        'standard': {
            'format': '%(levelname)s %(asctime)s %(module)s  %(message)s'
        },
        "post_office": {
            "format": "[%(levelname)s]%(asctime)s PID %(process)d: %(message)s",
            "datefmt": "%d-%m-%Y %H:%M",
        },
    },

    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
        'logfile': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'maxBytes': 1024 * 1024 * 5,  # 5 MB
            'backupCount': 5,
            'filename': os.path.join(BASE_DIR, "logs/django.log"),
            'formatter': 'standard',
        },

        'email': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'maxBytes': 1024 * 1024 * 5,  # 5 MB
            'backupCount': 5,
            'filename': os.path.join(BASE_DIR, "logs/email.log"),
            'formatter': 'standard',
        },

        "post_office": {
            "level": "DEBUG",
            'class': 'logging.handlers.RotatingFileHandler',
            'maxBytes': 1024 * 1024 * 5,  # 5 MB
            'backupCount': 5,
            'filename': os.path.join(BASE_DIR, "logs/postoffice.log"),
            "formatter": "post_office"
        },

    },
    'loggers': {
        'django': {
            'handlers': ['logfile','console'],
            'level': 'INFO',
            'propagate': True,
        },

        'email': {
            'handlers': ['email', ],
            'level': 'DEBUG',
            'propagate': True,
        },

        "post_office": {
            "handlers": ["post_office",],
            "level": "INFO"
        },

        'django.utils.autoreload': {
            'level': 'INFO',
        },
    },
}

try:
    localpath=os.path.join(BASE_DIR, 'config/settings_local.py')
    print(localpath)
    if  os.path.exists(localpath):
        with open(localpath) as f:
            code = compile(f.read(), localpath, 'exec')
            exec(code, globals(), locals())

except Exception as e:
    print(e)
    print("No local settings found or error in settings_local.py")


import pandas as pd
new_data = pd.read_excel("D:\Car_sales.xlsx", sheet_name="car")

# Remove duplicates by variable
n2 = new_data.drop_duplicates(["Manufacturer","Horsepower"])

# row positon
n1= new_data.loc[20:40]
# or
new_data[20:40] # 39 will apppear it  always n-1 i.e 40-1 = 39 rows will come

#select all column between two variables
new_data.loc[:, "Sales_in_thousands" :"Vehicle_type"]

# keeping variables by their postion. 0 is first variable
new_data.iloc[:,[0,2,4,6,8,9]]

n1 = new_data.loc[20:40]
n2 =new_data.loc[60:70]
# appendding dataset with same columns names
n3 = pd.concat([n1,n2])

# Saving dataset in folder;
new_da = n2and
new_da.to_excel("D:\Python Video\df.xlsx")


# 647. 回文子串
# 给定一个字符串，你的任务是计算这个字符串中有多少个回文子串。

# 具有不同开始位置或结束位置的子串，即使是由相同的字符组成，也会被视作不同的子串。

class Solution:
    def countSubstrings(self, s: str) -> int:
        n = len(s)
        dp=[[0]*(n) for _ in range(n)]
        for i in range(n):
            dp[i][i]=1
        for k in range(1,n):
            for i in range(n):
                j = i+k
                if j>=n:
                    break
                if s[i] == s[j]:
                    if j-i==1 or dp[i+1][j-1]==1:
                        dp[i][j]=1
        return sum([sum(i) for i in dp])

sol = Solution()
s="aaaaa"
print(sol.countSubstrings(s))


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

def main(_):
  mnist = tf.contrib.learn.datasets.load_dataset("mnist")
  train_data = mnist.train.images
  train_labels = np.asarray(mnist.train.labels, dtype=np.int32)
  eval_data = mnist.test.images
  eval_labels = np.asarray(mnist.test.labels, dtype=np.int32)

  x = tf.placeholder(tf.float32, [None, 784])
  y_ = tf.placeholder(tf.int64, [None])

  input_layer = tf.reshape(x, [-1, 28, 28, 1])

  conv1 = tf.layers.conv2d(
    inputs=input_layer,
    filters=32,
    kernel_size=[5,5],
    padding="same",
    activation=tf.nn.relu,
  )

  pool1 = tf.layers.max_pooling2d(
    inputs=conv1,
    pool_size=[2,2],
    strides=2,
  )

  conv2 = tf.layers.conv2d(
    inputs=pool1,
    filters=64,
    kernel_size=[5,5],
    padding="same",
    activation=tf.nn.relu,
  )

  pool2 = tf.layers.max_pooling2d(
    inputs=conv2,
    pool_size=[2,2],
    strides=2,
  )

  pool2_flat = tf.reshape(pool2, [-1, 7*7*64])

  dense = tf.layers.dense(
    inputs=pool2_flat,
    units=256,
    activation=tf.nn.relu,
  )

  dropout = tf.layers.dropout(
    inputs=dense,
    rate=0.4,
    training=True,
  )

  logits = tf.layers.dense(
    inputs=dropout,
    units=10,
  )

  loss = tf.losses.sparse_softmax_cross_entropy(
    labels=y_,
    logits=logits,
  )

  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5)
  train_op = optimizer.minimize(
    loss=loss,
    global_step=tf.train.get_global_step(),
  )

  y = tf.nn.softmax(logits)
  correct_prediction = tf.equal(tf.argmax(y,1), y_)
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

  with tf.Session() as sess:
    tf.global_variables_initializer().run()
    for i in range(1000):
      batch_xs, batch_ys = mnist.train.next_batch(100)
      if i % 100 == 0:
        train_accuracy = accuracy.eval(feed_dict={x: batch_xs, y_: batch_ys})
        print("step %d, training accuracy %g" % (i, train_accuracy))
      sess.run(train_op, feed_dict={x: batch_xs, y_: batch_ys})

if __name__ == '__main__':
    tf.app.run()


from collections import defaultdict
class Solution:
    # O(n)
    def longestConsecutive(self, nums: List[int]) -> int:
        result = 0
        d = self.get_hashmap(nums)
        for num in nums:
            if (num-1) in d:
                continue
            else:
                count = 1 # One Because this number already counted
                number = num # temp variable
                # Run loop till next number in sequence exists
                while (number+1) in d:
                    number += 1
                    count += 1
                result = max(result,count)
        return result

    def get_hashmap(self, nums):
        d = defaultdict(int)
        for num in nums:
            d[num] = d[num] + 1
        return d

    # O(nlogn)
    def longestConsecutiveOld(self, nums: List[int]) -> int:
        if len(nums) == 0:
            return 0
        nums.sort()
        index = 0
        maxLength = -1
        count = 1
        # print('Sorted', nums)
        while index < len(nums)-1:
            if nums[index+1] == nums[index]:
                index += 1
            elif nums[index+1] == nums[index]+1:
                count += 1
                index += 1
            else:
                maxLength = max(maxLength, count)
                count = 1
                index += 1
        maxLength = max(maxLength, count)
        return maxLength


# -*- encoding:utf-8 -*-
# __author__=='Gan'

# Given a Binary Search Tree (BST), convert it to a Greater Tree such that every key of the original BST
# is changed to the original key plus sum of all keys greater than the original key in BST.
# Example:
# Input: The root of a Binary Search Tree like this:
#               5
#             /   \
#            2     13
#
# Output: The root of a Greater Tree like this:
#              18
#             /   \
#           20     13

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

# 212 / 212 test cases passed.
# Status: Accepted
# Runtime: 115 ms
# Your runtime beats 63.18 % of python submissions.
class Solution(object):
    def convertBST(self, root):
        """
        :type root: TreeNode
        :rtype: TreeNode
        """
        self.added_nums = 0
        def traverse_add(node):
            if node:
                traverse_add(node.right)
                node.val += self.added_nums
                self.added_nums = node.val
                traverse_add(node.left)

        traverse_add(root)
        return root

# 212 / 212 test cases passed.
# Status: Accepted
# Runtime: 155 ms
# Your runtime beats 14.20 % of python submissions.
class Solution(object):
    def convertBST(self, root):
        """
        :type root: TreeNode
        :rtype: TreeNode
        """
        def reverse(node):
            return reverse(node.right) + [node] + reverse(node.left) if node else []
        for a, b in zip(reverse(root), reverse(root)[1:]):
            b.val += a.val
        return root

if __name__ == '__main__':
    root = TreeNode(5)
    root.left = TreeNode(2)
    # root.left.left = TreeNode(1)
    root.right = TreeNode(13)
    print(Solution().convertBST(root))


# draw_cross_section.py

from email.base64mime import header_length
from matplotlib.collections import PathCollection
from class_show_curser import UISpectrumTilted2
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
from math import pi, sqrt
from create_hamiltonian import HamiltonianTilt
import scipy.linalg as scla
from matplotlib.widgets import Slider

def draw_kpath(M=2):
    ui_spectrum = UISpectrumTilted2()
    ui_spectrum.M = M
    k, k_folded_list, b2, b1 = ui_spectrum.return_kpath_wignercell()

    kx = [k[i][0] for i in range(len(k))]
    ky = [k[i][1] for i in range(len(k))]
    kfx = [k_folded_list[i][0] for i in range(len(k))]
    kfy = [k_folded_list[i][1] for i in range(len(k))]

    fig, ax = plt.subplots(1, 2)

    pol_xy = np.array([np.array([0,0]), b1, b1 + b2, b2])
    p = Polygon(pol_xy)
    p2 = Polygon(pol_xy)
    ax[0].add_patch(p)
    ax[1].add_patch(p2)

    xrange = b1[0]
    yrange = b2[1] - b1[1]

    # draw lines
    for i in range(1, M):
        x = b1[0] / M * i
        y1 = b1[1] / M * i
        y2 = b1[1] / M * i + b2[1]
        ax[0].plot([x, x], [y1, y2], linewidth=2, c='grey')
        ax[1].plot([x, x], [y1, y2], linewidth=2, c='grey')

    # draw arrows

    ax[0].arrow(0, 0, b1[0], b1[1], head_width=0.1, head_length=0.1, length_includes_head=True)
    ax[0].arrow(0, 0, b2[0], b2[1], head_width=0.1, head_length=0.1, length_includes_head=True)

    ax[1].arrow(0, 0, b1[0], b1[1], head_width=0.1, head_length=0.1, length_includes_head=True)
    ax[1].arrow(0, 0, b2[0], b2[1], head_width=0.1, head_length=0.1, length_includes_head=True)

    ax[0].set_xlim([-0.1 * xrange, b1[0] + 0.1 * xrange])
    ax[0].set_ylim([b1[1] - 0.1 * yrange, b2[1] + 0.1 * yrange])
    ax[1].set_xlim([-0.1 * xrange, b1[0] + 0.1 * xrange])
    ax[1].set_ylim([b1[1] - 0.1 * yrange, b2[1] + 0.1 * yrange])
    ax[0].plot(kx, ky, c='r', marker='*', ls='')
    ax[1].plot(kfx, kfy, c='b', marker='*', ls='')
    ax[0].set_xlabel(r"$k_x$")
    ax[0].set_ylabel(r"$k_y$")
    ax[1].set_xlabel(r"$k_x$")
    ax[1].set_ylabel(r"$k_y$")
    ax[0].set_title("Original path")
    ax[1].set_title("Path in folded part")
    plt.suptitle("Path between high symmetry points")

    #plt.show()

def return_eigenvalues_from_wigner(ui_spec):
    N = ui_spec.N
    M = ui_spec.M
    t = ui_spec.t
    tp = ui_spec.tp
    tx = ui_spec.tx
    t_bar = ui_spec.t_bar
    t_amp = ui_spec.t_amp
    t_til = ui_spec.t_til
    rA = np.sqrt(3) * ui_spec.a
    try:
        t_freq = 2 * pi * ui_spec.p / M / rA
    except:
        t_freq = 0
    delta_nnn = HamiltonianTilt.generate_nnn_const(N, M, tp, tx)
    delta_nn = HamiltonianTilt.generate_nn_vary_ttilda(N, M, t_bar, t_til, t_amp, t_freq, ui_spec.a)
    ham = HamiltonianTilt(N, M, t, delta_nn, delta_nnn, a=ui_spec.a)

    k_res = 100
    k1 = -ham.k1
    k2 = ham.k2

    en = np.zeros((k_res, k_res, 2 * M * N))

    k1f = k1 / N
    k2f = k2 / M

    X = np.array([[(k1[0] * i + k2[0] * j)/ k_res for j in range(k_res)] for i in range(k_res)])
    Y = np.array([[(k1[1] * i + k2[1] * j)/ k_res for j in range(k_res)] for i in range(k_res)])

    for i in range(k_res):
        for j in range(k_res):
            ham.construct_ham_inK(ui_spec.project_to_smaller_wigner_cell(k1f, k2f, i / k_res * k1 + j / k_res * k2))
            eigvals = scla.eigh(ham.H, eigvals_only=True)
            en[i, j, :] = eigvals

    return X, Y, en

def return_cross(t, width, height, b1, b2, result='X'):
    pos1 = b2 + (1-width)/2 * (-b2 + b1)
    pos2 = b2 + (1+width)/2 * (-b2 + b1)

    vec = -b2 + b1
    orth = np.array([vec[1], -vec[0]])

    X, Z = np.meshgrid(np.linspace(pos1[0] + t * orth[0], pos2[0] + t * orth[0]),
                        np.linspace(-height/2, height/2, 2))

    lenaorth = np.linalg.norm(t * orth)
    Y = -sqrt(3) * X + b2[1] - lenaorth * 2 * sqrt(3) / 3

    if result=="X":
        return X
    if result=="Y":
        return Y
    if result=="Z":
        return Z

def plot_cross_section_prev():
    a = 1
    b = 4 * pi / 3 / a
    b2 = np.array([0, b])
    b1 = np.array([np.sqrt(3) / 2 * b, -b / 2])

    # Add hamiltonian
    ui_spectrum = UISpectrumTilted2()
    M = 1
    N = 1
    ui_spectrum.M = M

    KX, KY, EN = return_eigenvalues_from_wigner(ui_spectrum)

    # size of patch
    width = 0.8
    height = 1

    pos1 = b2 + (1-width)/2 * (-b2 + b1)
    pos2 = b2 + (1+width)/2 * (-b2 + b1)

    X, Z = np.meshgrid(np.linspace(pos1[0], pos2[0], 2), np.linspace(-height/2, height/2, 2))

    Y = -sqrt(3) * X + b2[1]

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    surf = ax.plot_surface(X, Y, Z)
    # plt.subplots_adjust(left=0.25, bottom=0.25)

    # # slider
    # axt = fig.add_axes([0.25, 0.1, 0.65, 0.03])
    # t_slider = Slider(ax = axt, label="t", valmin=-0.1, valmax=0.1, valinit=0)

    # def update(val):
    #     surf.set_ydata(return_cross(t_slider.val, width, height, b1, b2, result="Y"))
    #     surf.set_xdata(return_cross(t_slider.val, width, height, b1, b2, result="X"))

    for i in range(2 * N * M):
        ax.plot_surface(KX, KY, EN[:, :, i], color='k')
    ax.set_zlim(-0.6 * height, 0.6 * height)

    #t_slider.on_changed(update)
    #plt.show()

def draw_en_from_pos1pos2(pos1, pos2, ui_spec, k_points):
    N = ui_spec.N
    M = ui_spec.M
    t = ui_spec.t
    tp = ui_spec.tp
    tx = ui_spec.tx
    t_bar = ui_spec.t_bar
    t_amp = ui_spec.t_amp
    t_til = ui_spec.t_til
    rA = np.sqrt(3) * ui_spec.a
    try:
        t_freq = 2 * pi * ui_spec.p / M / rA
    except:
        t_freq = 0
    delta_nnn = HamiltonianTilt.generate_nnn_const(N, M, tp, tx)
    delta_nn = HamiltonianTilt.generate_nn_vary_ttilda(N, M, t_bar, t_til, t_amp, t_freq, ui_spec.a)
    ham = HamiltonianTilt(N, M, t, delta_nn, delta_nnn, a=ui_spec.a)

    k1 = -ham.k1
    k2 = ham.k2

    k_res = k_points

    en = np.zeros((k_res+1, 2 * M * N))

    k1f = k1 / N
    k2f = k2 / M

    k = [pos1 + i/k_res * (-pos1 + pos2) for i in range(k_res+1)]
    for i in range(k_res+1):
        ham.construct_ham_inK(ui_spec.project_to_smaller_wigner_cell(k1f, k2f, k[i]))
        eigvals = scla.eigh(ham.H, eigvals_only=True)
        en[i, :] = eigvals

    return en, k

def plot_cross_section(M, width, height, k_points, close_on_oneWeyl):
    a = 1
    b = 4 * pi / 3 / a
    b2 = np.array([0, b])
    b1 = np.array([np.sqrt(3) / 2 * b, -b / 2])

    # Add hamiltonian
    ui_spectrum = UISpectrumTilted2()
    N = 1
    ui_spectrum.M = M

    if close_on_oneWeyl:
        pos1 = b2 + 1/3 * (-b2 + b1) - 1/3 * width * (-b2 + b1)
        pos2 = b2 + 1/3 * (-b2 + b1) + 1/3 * width * (-b2 + b1)
    else:
        pos1 = b2 + (1-width)/2 * (-b2 + b1)
        pos2 = b2 + (1+width)/2 * (-b2 + b1)

    vec = -b2 + b1
    orth = np.array([vec[1], -vec[0]])

    en, k = draw_en_from_pos1pos2(pos1, pos2, ui_spectrum, k_points)
    en = np.array(en)

    param = np.linspace(0, 1, len(k))

    fig, ax = plt.subplots()
    lines = ax.plot(param, en, ms=2, c='k', ls="", marker='s')
    ax.set_ylim(-height/2, height/2)
    ax.set_ylabel("Energy")
    ax.set_xlabel(r"$k_{parameterised}$")
    ax.set_title("Energy spectrum")

    plt.subplots_adjust(bottom=0.3)

    # Make four sliders

    axc = plt.axes([0.2, 0.16, 0.7, 0.03])
    c_slider = Slider(ax=axc, label='offset', valmin=-0.02, valmax=0.02, valinit=0)

    axttilda = plt.axes([0.2, 0.12, 0.7, 0.03])
    ttilda_slider = Slider(ax=axttilda, label=r'$\tilde{t}$ (const tilt)', valmin=-1, valmax=1, valinit=0)

    axtamp = plt.axes([0.2, 0.08, 0.7, 0.03])
    tamp_slider = Slider(ax=axtamp, label=r'$t_{amp}$ (var tilt)', valmin=-1, valmax=1, valinit=0)

    axp = plt.axes([0.2, 0.04, 0.7, 0.03])
    p_slider = Slider(ax=axp, label='freq', valmin=0, valmax=M-1, valinit=0, valstep=1)

    def update(val):
        c = c_slider.val

        if close_on_oneWeyl:
            pos1 = b2 + 1/3 * (-b2 + b1) - 1/3 * width * (-b2 + b1) + c * orth
            pos2 = b2 + 1/3 * (-b2 + b1) + 1/3 * width * (-b2 + b1) + c * orth
        else:
            pos1 = b2 + (1-width)/2 * (-b2 + b1) + c * orth
            pos2 = b2 + (1+width)/2 * (-b2 + b1) + c * orth

        ui_spectrum.t_amp = tamp_slider.val
        ui_spectrum.t_til = ttilda_slider.val
        ui_spectrum.p = p_slider.val

        en, k = draw_en_from_pos1pos2(pos1, pos2, ui_spectrum, k_points)
        for i, l in enumerate(lines):
            l.set_ydata(en[:, i])

    c_slider.on_changed(update)
    ttilda_slider.on_changed(update)
    tamp_slider.on_changed(update)
    p_slider.on_changed(update)

    plt.show()

if __name__ == "__main__":
    ################################################################################################
    # Select case = "draw mapping" for plotting part of Brillouin zone with path in k-space mapped
    # to smaller Brillouin zone (elongated system).
    #
    # Select case = "draw cross section" for plotting spectrum near mapped Dirac cones.
    #
    # Select case = "draw cross section zoomed in" for plotting spectrum near one of the Dirac cone.
    case = "draw cross section"
    ################################################################################################
    # Parameters
    M = 4                           # Size of unit cell in chosen direction
    width = 0.8                     # Range of k-points while plotting spectrum near Dirac cones
    height = 3                      # Range of energies included in the plotted spectrum
    k_points = 100                  # Number of plotted k-points
    ################################################################################################
    # Run code
    if case == "draw mapping":
        draw_kpath(M)
        plot_cross_section_prev()
        plt.show()
    elif case == "draw cross section":
        plot_cross_section(M, width, height, k_points, close_on_oneWeyl=False)
    elif case == "draw cross section zoomed in":
        plot_cross_section(M, width, height, k_points, close_on_oneWeyl=True)
    ################################################################################################


#!/usr/bin/env python3
from astropy.io import fits
import numpy as np
import matplotlib.pyplot as plt

def compare(fname_trace, fname_img=None, fname_spec=None, figname=None, dpi=240, fname_bpm=None, vmin=None, vmax=None):
    """ compare img and trace """
    trace = fits.open(fname_trace)
    if fname_img:
        img = fits.open(fname_img)
        linecol = "w"
    else:
        linecol = "k"
    if fname_spec:
        spec = fits.open(fname_spec)
    else:
        spec = None
    if fname_bpm:
        bpm = fits.open(fname_bpm)
    else:
        bpm = None

    X = np.arange(2048)
    FIG = plt.figure(figsize=(10, 4))

    for i in [1, 2, 3]:
        ax = FIG.add_subplot(1, 3, i)
        ax.set_xticks([])
        ax.set_yticks([])

        try:
            tdata = trace[i].data
        except:
            print("extension %s is missing, skipping." % i)
            continue
        if tdata is None:
            print("Data for CHIP%s is empty, skipping." % i)
            continue

        if fname_img:
            imgdata = img["CHIP%d.INT1" % i].data
            if imgdata is not None:
                # imgdata = np.ma.masked_where(np.isnan(imgdata),imgdata)
                # imgdata = np.nan_to_num(imgdata)
                if bpm is not None:
                    bpmdata = bpm["CHIP%d.INT1" % i].data
                    if bpmdata is not None:
                        imgdata[bpmdata != 0] = np.nan
                if vmin is None:
                    _vmin = np.nanpercentile(imgdata, 5)
                else:
                    _vmin = vmin
                if vmax is None:
                    _vmax = np.nanpercentile(imgdata, 98)
                else:
                    _vmax = vmax
                ax.imshow(imgdata, origin="lower", cmap="plasma", vmin=_vmin, vmax=_vmax, interpolation="none")

        for t in tdata:
            pol = np.polyval(t["Upper"][::-1], X)
            ax.plot(X, pol, ":" + linecol)
            pol = np.polyval(t["Lower"][::-1], X)
            ax.plot(X, pol, ":" + linecol)
            pol = np.polyval(t["All"][::-1], X)
            ax.plot(X, pol, "--" + linecol)
            if np.isnan(pol[1024]):
                continue
            ax.text(
                1024,
                pol[1024],
                "order: %s     trace: %s" % (t["order"], t["TraceNb"]),
                color=linecol,
                horizontalalignment="center",
                verticalalignment="center",
                size=8,
            )
        ax.axis((1, 2048, 1, 2048))

    FIG.tight_layout(pad=0.02)
    if figname is None:
        figname = fname_trace.replace(".fits", ".png")
    print(f"Saving plot: {figname}")
    plt.savefig(figname, dpi=dpi)

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("trace")
    parser.add_argument("img")
    parser.add_argument("--spec")
    parser.add_argument("--bpm")
    parser.add_argument("--out")
    parser.add_argument("--dpi", default=1000)
    parser.add_argument("--vmin", type=float)
    parser.add_argument("--vmax", type=float)
    args = parser.parse_args()

    fname_trace = args.trace
    fname_img = args.img
    fname_spec = args.spec
    fname_out = args.out
    fname_bpm = args.bpm
    dpi = args.dpi
    vmin = args.vmin
    vmax = args.vmax

    compare(fname_trace, fname_img, fname_spec, fname_out, fname_bpm=fname_bpm, dpi=dpi, vmin=vmin, vmax=vmax)


#
# @lc app=leetcode id=1249 lang=python3
#
# [1249] Minimum Remove to Make Valid Parentheses
#

# @lc code=start
class Solution:
    def minRemoveToMakeValid(self, s: str) -> str:
        #end stack is to track the closing parentesis after the opening parenthesis
        end = []
        #start stack to track the closing parenthesis in the start
        start = []
        # This is the final result string
        result = ""
        for i in range(len(s)):
            # checking whether the closing parenthesis is start or end
            if s[i] == ")" and len(end) == 0:
                start.append(i)
            # else tracking the index of the opening parenthesis
            elif s[i] == "(":
                end.append(i)
            # once found the closing parenthesis delete the last found open parenthesis
            elif s[i] == ")":
                end.pop()
        # merging the start and end list
        final = start + end
        #looping over the string and checking that the index is in the final stack and skipping them while adding into the final result
        for x in range(len(s)):
            if x not in final:
                result += s[x]

        return result

# @lc code=end


import xarray as xr
import numpy.ma as ma
import calendar
import time
import math
import scipy.spatial as sp
import subprocess

from rpy2.robjects import StrVector
from rpy2.robjects.packages import importr

from intense import gauge as ex
import pandas as pd
import numpy as np
import scipy.interpolate
import datetime
import os
import zipfile
import scipy.stats

# Selected upper bound for hourly world record - see:
# http://www.nws.noaa.gov/oh/hdsc/record_precip/record_precip_world.html
# http://www.bom.gov.au/water/designRainfalls/rainfallEvents/worldRecRainfall.shtml
# https://wmo.asu.edu/content/world-meteorological-organization-global-weather-climate-extremes-archive
world_records = {'hourly': 401.0, 'daily': 1825.0}  # mm

def world_record_check(val):
    """
    1h record1 with separate flags showing exceedance by <20%, >= 20%, 33%, or 50% -
    world record = 394.5mm in 1 hour (http://www.nws.noaa.gov/oh/hdsc/record_precip/record_precip_world.html)

    *** updated to use 401.0 mm in 1 hour - compare e.g.
    http://www.nws.noaa.gov/oh/hdsc/record_precip/record_precip_world.html
    http://www.bom.gov.au/water/designRainfalls/rainfallEvents/worldRecRainfall.shtml
    https://wmo.asu.edu/content/world-meteorological-organization-global-weather-climate-extremes-archive
    """

    if val >= world_records['hourly'] * 1.5:
        return 4
    elif val >= world_records['hourly'] * 1.33:
        return 3
    elif val >= world_records['hourly'] * 1.2:
        return 2
    elif val >= world_records['hourly']:
        return 1
    else:
        return 0

 # ETCCDI utility function 1
def prep_etccdi_variable(input_path, index_name, aggregation, data_source):
    ds = xr.open_dataset(input_path)

    # Omit final year (2010) of HADEX2 - suspiciously large CDD for Malaysia
    if data_source == 'HADEX2':
        ds = ds.sel(time=slice(datetime.datetime(1951, 1, 1, 0),
                               datetime.datetime(2009, 12, 31, 23)))

    # Calculate maximum rainfall value over whole period
    vals = ds[index_name].values
    if index_name in ['CWD', 'CDD']:
        vals = ds[index_name].values.astype('timedelta64[s]')
        vals = vals.astype('float32') / 86400.0
        vals[vals < 0.0] = np.nan
    vals = ma.masked_invalid(vals)
    if aggregation == 'max':
        data = ma.max(vals, axis=0)
    if aggregation == 'mean':
        data = ma.mean(vals, axis=0)

    # Convert back from to a xarray DataArray for easy plotting
    # - masked array seems to be interpreted as np array (i.e. nans are present
    # in the xarray DataArray
    data2 = xr.DataArray(data, coords={'Latitude': ds['lat'].values,
                                       'Longitude': ds['lon'].values}, dims=('Latitude', 'Longitude'),
                         name=index_name)

    ds.close()

    return data2

def get_etccdi_value(etccdi_data, index_name, lon, lat):
    lon = float(lon)
    lat = float(lat)

    # Check gauge longitude and convert to -180 - 180 range if necessary
    if lon > 180.0:
        lon = lon - 360.0

    # Array location indices for closest cell centre to gauge location
    location_indices = {'GHCNDEX': {}, 'HADEX2': {}}
    for data_source in location_indices.keys():
        location_indices[data_source]['lon'] = (np.argmin(
            np.abs(etccdi_data[data_source][index_name]['Longitude'].values - lon)))
        location_indices[data_source]['lat'] = (np.argmin(
            np.abs(etccdi_data[data_source][index_name]['Latitude'].values - lat)))

    # Maximum of ETCCDI index values from GHCNDEX and HADEX2 for cell
    etccdi_index_values = {}
    for data_source in location_indices.keys():
        yi = location_indices[data_source]['lat']
        xi = location_indices[data_source]['lon']
        etccdi_index_values[data_source] = etccdi_data[data_source][index_name].values[yi, xi]
    etccdi_vals = np.asarray(list(etccdi_index_values.values()))
    if np.any(np.isfinite(etccdi_vals)):
        max_index = np.max(etccdi_vals[np.isfinite(etccdi_vals)])
    else:
        max_index = np.nan

    # For cases where no value for the cell, look in 3x3 window and take the maximum
    if np.isnan(max_index):
        etccdi_index_window = {}
        for data_source in location_indices.keys():
            yi = location_indices[data_source]['lat']
            xi = location_indices[data_source]['lon']
            window = etccdi_data[data_source][index_name].values[yi - 1:yi + 2, xi - 1:xi + 2]
            if np.any(np.isfinite(window)):
                etccdi_index_window[data_source] = np.max(window[np.isfinite(window)])
            else:
                etccdi_index_window[data_source] = np.nan

        window_vals = np.asarray(list(etccdi_index_window.values()))
        if np.any(np.isfinite(window_vals)):
            max_index_window = np.max(window_vals[np.isfinite(window_vals)])
        else:
            max_index_window = np.nan

    else:
        max_index_window = np.nan

    return max_index, max_index_window

# ETCCDI utility function 3 - returns flag based on exceedence of parameters
    # Replaces Rx1dayCheck, R99pTOTCheck, PRCPTOTCheck
def day_check(val, p_max, p_max_filled):
    if np.isnan(p_max):
        if val >= p_max_filled * 1.5:
            return 8
        elif val >= p_max_filled * 1.33:
            return 7
        elif val >= p_max_filled * 1.2:
            return 6
        elif val >= p_max_filled:
            return 5
        else:
            return 0
    else:
        if val >= p_max * 1.5:
            return 4
        elif val >= p_max * 1.33:
            return 3
        elif val >= p_max * 1.2:
            return 2
        elif val >= p_max:
            return 1
        else:
            return 0

# Helper function, flags data based on various thresholds
def spell_check(val, longest_wet_period, longest_wet_period_filled):
    if np.isnan(longest_wet_period):
        if val >= longest_wet_period_filled * 24 * 1.5:
            return 8
        elif val >= longest_wet_period_filled * 24 * 1.33:
            return 7
        elif val >= longest_wet_period_filled * 24 * 1.2:
            return 6
        elif val >= longest_wet_period_filled * 24:
            return 5
        else:
            return 0
    else:
        if val >= longest_wet_period * 24 * 1.5:
            return 4
        elif val >= longest_wet_period * 24 * 1.33:
            return 3
        elif val >= longest_wet_period * 24 * 1.2:
            return 2
        elif val >= longest_wet_period * 24:
            return 1
        else:
            return 0

def get_dry_periods(vals):
    start_index_list = []
    duration_list = []

    dry_flag = 0
    hours_ticker = 0

    for i in range(len(vals)):
        v = vals[i]

        if v == 0:
            if dry_flag == 0:
                start_index_list.append(i)

            hours_ticker += 1
            dry_flag = 1
            if i == len(vals) - 1:
                duration_list.append(hours_ticker)

        else:
            if dry_flag == 1:
                duration_list.append(hours_ticker)

            hours_ticker = 0
            dry_flag = 0

        if i == len(vals):
            if dry_flag == 1:
                duration_list.append(hours_ticker)

    return [start_index_list, duration_list]

def daily_accums_day_check(day_list, mean_wet_day_val, mean_wet_day_val_filled):
    """
    Suspect daily accumulations flagged where a recorded rainfall amount at these times is preceded by 23 hours with no
    rain. A threshold of 2x the mean wet day amount for the corresponding month is applied to increase the chance of
    identifying accumulated values at the expense of genuine, moderate events.
    """

    if day_list[23] > 0:
        dry_hours = 0
        for i in range(23):
            if day_list[i] <= 0:
                dry_hours += 1
        if dry_hours == 23:
            if np.isnan(mean_wet_day_val):
                if day_list[23] > mean_wet_day_val_filled:
                    flag = 2
                else:
                    flag = 0
            else:
                if day_list[23] > mean_wet_day_val:
                    flag = 1
                else:
                    flag = 0
        else:
            flag = 0
    else:
        flag = 0

    return flag

# Coordinate system conversion
def geodetic_to_ecef(lat, lon, h):
    a = 6378137
    b = 6356752.3142
    f = (a - b) / a
    e_sq = f * (2 - f)
    lamb = math.radians(lat)
    phi = math.radians(lon)
    s = math.sin(lamb)
    n = a / math.sqrt(1 - e_sq * s * s)

    sin_lambda = math.sin(lamb)
    cos_lambda = math.cos(lamb)
    sin_phi = math.sin(phi)
    cos_phi = math.cos(phi)

    x = (h + n) * cos_lambda * cos_phi
    y = (h + n) * cos_lambda * sin_phi
    z = (h + (1 - e_sq) * n) * sin_lambda

    return x, y, z

def calculate_overlap(period1, period2):
    # taking period2 as the reference (daily) period
    r1, r2 = period2
    t1, t2 = period1

    if t1 >= r1 and t2 <= r2:
        p = 100
        rd = (t2 - t1).days

    elif t1 < r1 and t2 < r1:
        p = 0
        rd = 0

    elif t1 > r2 and t2 > r2:
        p = 0
        rd = 0

    elif t1 <= r1 and t2 >= r2:
        rd = (r2 - r1).days
        td = (t2 - t1).days

        try:
            p = rd * 100 / td
        except:
            p = -999

    elif t1 < r1 and t2 >= r1 and t2 < r2:
        rd = (t2 - r1).days
        td = (t2 - t1).days

        try:
            p = rd * 100 / td
        except:
            p = -999

    elif t1 > r1 and t1 <= r2 and t2 > r2:
        rd = (r2 - t1).days
        td = (t2 - t1).days

        try:
            p = rd * 100 / td
        except:
            p = -999

    else:
        p = -999

    return p, rd

# ++++++++++++++++++++++++++++++++++ GPCC functions +++++++++++++++++++++++++++++++++++ LIZ WILL CHANGE THIS

def get_daily_gpcc(path, start_year, end_year, gpcc_id):
    gpcc_filename = "tw_" + gpcc_id + ".dat"
    dat_path = os.path.join(path, "tw_" + gpcc_id + ".dat")
    zip_path = os.path.join(path, "tw_" + gpcc_id + ".zip")
    if not os.path.exists(zip_path):
        if not os.path.exists(dat_path):
            p = subprocess.Popen(["get_zeitreihe_tw_by_id.sh", str(start_year), str(end_year), gpcc_id],
                                 cwd="/media/nas/x21971/GPCC_daily2")
            p.wait()
            time.sleep(0.1)

            # Move retrieved .dat file to its own .zip folder
            with zipfile.ZipFile(zip_path, 'w') as zf:
                zf.write(dat_path, arcname=gpcc_filename, compress_type=zipfile.ZIP_DEFLATED)
            time.sleep(0.1)
            os.remove(dat_path)

    zf = zipfile.ZipFile(zip_path, "r")
    f = zf.open(gpcc_filename, "r")
    f.readline()

    dates = []
    vals = []

    file_format_error = 0
    try_again = 1
    for line in f:
        line_list = line.rstrip().split()
        # inserted try as at least one file has a dubious second header line
        try:
            dates.append(datetime.date(int(line_list[2]), int(line_list[1]), int(line_list[0])))
            read_values = 1
        except:
            if try_again == 1:
                try_again = 0
                read_values = 0
            elif try_again == 0:
                file_format_error = 1
                read_values = 0
                break
        if read_values == 1:
            got_one = 0
            for v in line_list[3:]:
                if float(v) >= 0:
                    vals.append(float(v))
                    got_one = 1
                    break
            if got_one == 0:
                vals.append(np.nan)

    f.close()
    zf.close()

    if file_format_error == 0:
        ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])

        # Remove duplicate rows if present and check against unique index values
        ddf['date_tmp'] = ddf.index.copy()
        idx = ddf.index.drop_duplicates().copy()
        ddf.drop_duplicates(inplace=True)
        ddf.sort_index(inplace=True)
        ddf.drop(['date_tmp'], axis=1, inplace=True)
        if ddf.shape[0] != idx.shape[0]:
            vals = []
            dates = []
            ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])

    elif file_format_error == 1:
        vals = []
        dates = []
        ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])

    return ddf

# Helper function to access Global Sub Daily Rainfall database (a.k.a. Intense Database)
def get_gsdr(gsdr_id, path):

    filename = gsdr_id + '.txt'

    if path.endswith('.zip'):
        f = zipfile.ZipFile(path).open(filename)
    else:
        f = os.path.join(path, filename)

    df = ex.read_intense(f, only_metadata=False).data.to_frame("GSDR")
    # convert hourly to daily 7am-7am
    df["roll"] = np.around(df.GSDR.rolling(window=24, center=False, min_periods=24).sum(), 1)

    dfd = df[df.index.hour == 7]
    dts = list(dfd.index)
    daily_vals = list(dfd.roll)
    dday = []
    for hday in dts:
        s0 = hday - datetime.timedelta(days=1)
        dday.append(datetime.date(s0.year, s0.month, s0.day))

    gsdr = pd.Series(daily_vals, index=dday).to_frame("ts2")

    return gsdr

# Helper function to access Global Precipitation Climatology Centre monthly data...
# Ask Liz what is needed to implement this!
def get_monthly_gpcc(path, start_year, end_year, gpcc_id):  # Hey liz! Check this once you have access to monthly!

    gpcc_filename = "mw_" + gpcc_id + ".dat"
    dat_path = os.path.join(path, "mw_" + gpcc_id + ".dat")
    zip_path = os.path.join(path, "mw_" + gpcc_id + ".zip")
    if not os.path.exists(zip_path):
        if not os.path.exists(dat_path):
            p = subprocess.Popen(["get_zeitreihe_mw_by_id.sh", str(start_year), str(end_year), gpcc_id],
                                 cwd="/media/nas/x21971/GPCC_monthly2")
            p.wait()
            time.sleep(0.1)

            # Move retrieved .dat file to its own .zip folder
            with zipfile.ZipFile(zip_path, 'w') as zf:
                zf.write(dat_path, arcname=gpcc_filename, compress_type=zipfile.ZIP_DEFLATED)
            time.sleep(0.1)
            os.remove(dat_path)

    zf = zipfile.ZipFile(zip_path, "r")
    f = zf.open(gpcc_filename, "r")
    f.readline()

    dates = []
    vals = []

    file_format_error = 0
    for line in f:
        line_list = line.rstrip().split()
        try:
            year = int(line_list[1])
            month = int(line_list[0])
            read_values = 1
        except:
            file_format_error = 1
            read_values = 0
            break
        if read_values == 1:
            day = calendar.monthrange(year, month)[1]
            dates.append(datetime.date(year, month, day))
            got_one = 0
            for v in line_list[2:]:
                if float(v) >= 0:
                    vals.append(float(v))
                    got_one = 1
                    break
            if got_one == 0:
                vals.append(np.nan)

    f.close()
    zf.close()

    # ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])

    if file_format_error == 0:
        ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])

        # Remove duplicate rows if present and check against unique index values
        ddf['date_tmp'] = ddf.index.copy()
        idx = ddf.index.drop_duplicates().copy()
        ddf.drop_duplicates(inplace=True)
        ddf.sort_index(inplace=True)
        ddf.drop(['date_tmp'], axis=1, inplace=True)
        if ddf.shape[0] != idx.shape[0]:
            vals = []
            dates = []
            ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])
            print(file_format_error, ddf.shape[0])

    elif file_format_error == 1:
        vals = []
        dates = []
        ddf = pd.DataFrame(vals, index=dates, columns=["GPCC"])
        print(file_format_error, ddf.shape[0])

    return ddf

def calculate_affinity_index_and_pearson(df1, df2):  # done

    df = pd.concat([df1, df2], axis=1, join='inner')
    df.columns = ["ts1", "ts2"]
    df = df.dropna()

    # dp 29/11/2019 - extra check to make sure there is some overlap between
    # hourly target and neighbour - implemented because at least one error in
    # GPCC statlex_daily
    # - also possibility that dropping nans from df reduces overlap
    if df.shape[0] >= 365:

        # p = 0.1 # 0.0 # 1.0
        a = np.around(df.loc[df['ts1'] >= 0.1, 'ts1'].min(), 1)
        b = np.around(df.loc[df['ts2'] >= 0.1, 'ts2'].min(), 1)
        p = max(a, b, 0.1)
        conditions = [
            (df['ts1'] > p) & (df['ts2'] > p),
            (df['ts1'] == p) & (df['ts2'] == p),
            (df['ts1'] == p) & (df['ts2'] > p),
            (df['ts1'] > p) & (df['ts2'] == p)]
        choices = [1, 1, 0, 0]

        df['duplicate'] = np.select(conditions, choices, default=np.nan)

        try:
            match = df["duplicate"].value_counts()[1]
        except:
            match = 0
        try:
            diff = df["duplicate"].value_counts()[0]
        except:
            diff = 0

        if (match > 0) or (diff > 0):
            perc = match / (match + diff)
            p_corr = df.ts1.corr(df.ts2)
            df["factor"] = df.ts1 / df.ts2
            f = np.mean(df.loc[(df.ts1 > 0) & (df.ts2 > 0), "factor"].values)
        else:
            perc = 0
            p_corr = 0
            f = 0

    else:
        perc = 0
        p_corr = 0
        f = 0

    return perc, p_corr, f

# Main helper function, used by check_neighbour and check_neighbourDry
def compare_target_to_neighbour(target, neighbour, high_or_dry, station=None, check_type=None,
                                neighbour_id=None):

    """ After Upton and Rahimi 2003 https://www.sciencedirect.com/science/article/pii/S0022169403001422

    last three args for output for
    normalised difference distribution checks
    """

    check_flag = 1  # default setting is to check

    # dp 31/12/2019 - this AI check should be redundant, because neighbours with
    # AI < 0.9 are filtered out before this function is called...
    if high_or_dry == "high":
        ai = calculate_affinity_index_and_pearson(target, neighbour)[0]
        if ai >= 0.9:
            check_flag = 1
        else:
            check_flag = 0

    if check_flag == 1:

        df = pd.concat([target, neighbour], axis=1, join='inner')
        df.columns = ["ts1", "ts2"]
        df = df.dropna()

        # There are cases where intermittent records cause problems in 15-days
        # windows for dry spell stuff, so try to ensure reasonable overlap
        if df.shape[0] >= 365:

            # Separate out high and dry checks as using slightly different approaches now
            if high_or_dry == "high":

                # Normalise target and neighbour series by their respective min/max and
                # find differences
                # - retained actual amounts too to help filter on wet days
                df['ts1n'] = (df['ts1'] - df['ts1'].min()) / (df['ts1'].max() - df['ts1'].min())
                df['ts2n'] = (df['ts2'] - df['ts2'].min()) / (df['ts2'].max() - df['ts2'].min())
                df["nd"] = df['ts1n'] - df['ts2n']

                # ----------
                # *** dp 31/12/2019 - rather than using a median/standard deviation approach to
                # identifying outlying differences, switching to an approach based on presumption
                # that normalised differences for wet day amounts are roughly exponentially
                # distributed (based on testing)

                # Filter for target wet days, no NAs and positive differences
                df1 = df.loc[(df['ts1'] >= 1.0) & (np.isfinite(df['ts1'])) &
                             (np.isfinite(df['ts2'])) & (df['nd'] > 0.0)]

                # Ensure still some data left to fit a distribution etc
                if df1.shape[0] >= 30:

                    # Fit exponential distribution
                    params = scipy.stats.expon.fit(df1['nd'])

                    # Calculate thresholds at key percentiles of fitted distribution
                    q95 = scipy.stats.expon.ppf(0.95, params[0], params[1])
                    q99 = scipy.stats.expon.ppf(0.99, params[0], params[1])
                    q999 = scipy.stats.expon.ppf(0.999, params[0], params[1])

                    # Assign flags
                    # - no need for an additional condition that target exceeds wet day threshold
                    # because the percentiles are defined based on just positive differences?
                    # -- left in for now...
                    conditions = [
                        (df['ts1'] >= 1.0) & (df['nd'] <= q95),
                        (df['ts1'] >= 1.0) & (df['nd'] > q95) & (df['nd'] <= q99),
                        (df['ts1'] >= 1.0) & (df['nd'] > q99) & (df['nd'] <= q999),
                        (df['ts1'] >= 1.0) & (df['nd'] > q999)]
                    choices = [0, 1, 2, 3]

                    df['temp_flags'] = np.select(conditions, choices, default=0)

                    temp_flags = df['temp_flags']
                    return temp_flags

                else:
                    return pd.Series([])

            elif high_or_dry == "dry":

                # Assign flags
                # - consider only whether dry 15-day periods at the target are
                # corroborated as dry by neighbours
                # - check based on whether 0, 1, 2 or >= 3 wet days are recorded at the
                # neighbour when the target is dry over the 15-day period
                # - dry flag works on the basis of fraction of dry days within 15-day
                # moving window, so 1 = all dry, 0 = all wet
                # -- truncating these fractions to 2 dp below and manipulating equalities
                # to work with these fractions, but could work in days not fractions if
                # change the convertToDrySpell function
                # - in dry day fraction calcs a threshold of 0 mm is currently used to
                # identify days as wet (i.e. any rainfall)
                frac_drydays = {}
                for d in range(1, 3 + 1):
                    frac_drydays[d] = np.trunc((1.0 - (float(d) / 15.0)) * 10 ** 2) / (10 ** 2)
                conditions = [
                    (df['ts1'] == 1.0) & (df['ts2'] == 1.0),
                    (df['ts1'] == 1.0) & (df['ts2'] < 1.0) & (df['ts2'] >= frac_drydays[1]),
                    (df['ts1'] == 1.0) & (df['ts2'] < frac_drydays[1]) & (df['ts2'] >= frac_drydays[2]),
                    (df['ts1'] == 1.0) & (df['ts2'] < frac_drydays[2])]  # & (df['ts2'] >= frac_drydays[3])
                choices = [0, 1, 2, 3]

                # *** dp 27/11/2019 *** - commented out line below so changed to default=0
                # normalized_df['temp_flags'] = np.select(conditions, choices, default=np.nan)
                # normalized_df['temp_flags'] = np.select(conditions, choices, default=0)
                df['temp_flags'] = np.select(conditions, choices, default=0)

                # tempFlags = normalized_df['temp_flags']
                temp_flags = df['temp_flags']
                return temp_flags

        else:
            return pd.Series([])

    else:
        return pd.Series([])

# Monthly checks ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
def compare_target_to_neighbour_monthly(target, neighbour):
    df = pd.concat([target, neighbour], axis=1, join='inner')

    df = df.dropna().rename(columns={"GPCC": "ts2", "target": "ts1"})

    df["perc_diff"] = (df.ts1 - df.ts2) * 100. / df.ts2
    df["factor_diff"] = df.ts1 / df.ts2
    conditions = [
        (df['perc_diff'] <= -100.0),
        (df['perc_diff'] <= -50.0) & (df['perc_diff'] > -100.0),
        (df['perc_diff'] <= -25.0) & (df['perc_diff'] > -50),
        (df['perc_diff'] < 25.0) & (df['perc_diff'] > -25.0),
        (df['perc_diff'] >= 25.0) & (df['perc_diff'] < 50.0),
        (df['perc_diff'] >= 50.0) & (df['perc_diff'] < 100.0),
        (df['perc_diff'] >= 100.0)]

    choices = [-3, -2, -1, 0, 1, 2, 3]

    df['temp_flags'] = np.select(conditions, choices, default=np.nan)

    df.loc[np.isnan(df['ts1']), 'temp_flags'] = np.nan
    temp_flags = df['temp_flags']

    conditions_f = [
        (df['factor_diff'] < 11) & (df['factor_diff'] > 9),  # hourly is approx 10x greater than monthly
        (df['factor_diff'] < 26) & (df['factor_diff'] > 24),  # hourly is approx 25.4x greater than monthly
        (df['factor_diff'] < 3) & (df['factor_diff'] > 2),  # hourly is approx 2.45x greater than monthly
        (df['factor_diff'] > 1 / 11) & (df['factor_diff'] < 1 / 9),
        (df['factor_diff'] > 1 / 26) & (df['factor_diff'] < 1 / 24),
        (df['factor_diff'] > 1 / 3) & (df['factor_diff'] < 1 / 2)]

    choices_f = [1, 2, 3, 4, 5, 6]

    df['factor_flags'] = np.select(conditions_f, choices_f, default=0)

    df.loc[np.isnan(df['ts1']), 'factor_flags'] = np.nan
    factor_flags = df['factor_flags']

    return [temp_flags, factor_flags]  # Hey Liz! Make sure theres a new thing for factor flags!

# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# Uses compare_target_to_neighbour function based on Upton and Rahimi(2003)
def check_neighbours(target, neighbours, station=None,
                     check_type=None):  # temporary extra args for checking neighbour stuff

    df = target
    concat_list = [df]

    nid = 1  # temporary for neighbours checking
    for n in neighbours:
        dfn = compare_target_to_neighbour(df, n, "high", station, check_type, nid)
        if dfn.empty:
            pass
        else:
            concat_list.append(dfn)
        nid += 1
    df = pd.concat(concat_list, axis=1, join='outer')

    cols = ["n" + str(i + 1) for i in range(len(concat_list) - 1)]
    cols2 = ["target"]
    cols2.extend(cols)

    df.columns = cols2

    # dp 28/11/2019 - changed assuming that looking for number of neighbours online
    # on any given day (similar to monthly neighbours)
    df["online"] = len(concat_list) - df[cols].isnull().sum(axis=1) - 1

    conditions = [
        ((df[cols] == 3).T.sum() == df.online),
        ((df[cols] >= 2).T.sum() == df.online),
        ((df[cols] >= 1).T.sum() == df.online)]

    choices = [3, 2, 1]
    df["flags"] = np.select(conditions, choices, default=0)

    df.loc[df.online < 3, "flags"] = np.nan
    dfr = df.flags

    dfr.index = pd.to_datetime(dfr.index) + datetime.timedelta(hours=8)

    return dfr

# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# Uses compareTargetToNeighbour function based on Upton and Rahimi(2003)
def check_neighbours_dry(target, neighbours):  # Liz check this

    df = convert_to_dry_spell(target)  # Liz, check column names

    concat_list = [df]
    for n in neighbours:
        nn = convert_to_dry_spell(n)  # Liz check column names
        dfn = compare_target_to_neighbour(df, nn, "dry")
        if dfn.empty:
            pass
        else:
            concat_list.append(dfn)

    df = pd.concat(concat_list, axis=1, join='outer')

    cols = ["n" + str(i + 1) for i in range(len(concat_list) - 1)]
    cols2 = ["target"]
    cols2.extend(cols)
    df.columns = cols2

    # dp 28/11/2019 - again assumed want count of number of neighbours online
    df["online"] = len(concat_list) - df[cols].isnull().sum(axis=1) - 1

    df["flags"] = np.floor(df[cols].sum(axis=1) / df.online)

    # *** dp 29/11/2019 - in last section below, why set flags to -999 when target equals 0? in this
    # case target is between 0 and 1 because it is the fraction of dry days in the 15-day period
    # ending on each date. 1 means all dry days in the period, so it should be values less than
    # 1 that are set to -999 i think - changed to that effect for now... ***
    # - may also be better to give a different flag than nan to show that the check has been done and
    #   not failed e.g. due to lack of neighbours?

    # *** dp 01/01/2020 - changed so that no adjustment to flags where period is
    # not totally dry (i.e. df.target < 1) on the basis that this is now handled in
    # the revised dry neighbours check (so just commented out line below)

    df.flags = df.flags.replace([np.inf, -np.inf], -999)
    df.loc[df.online < 3, "flags"] = -999
    df.flags = df.flags.astype(int)
    df.flags = df.flags.replace(-999, np.nan)
    # needs to be at hour=0800 to reconcile GPCC vs GSDR aggregation definitions
    df.index = pd.to_datetime(df.index) + datetime.timedelta(hours=8)
    return propagate_flags(df.flags)

# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
def check_m_neighbours(target, neighbours):
    df = target
    concat_list = [df]
    ticker = 0
    for n in neighbours:
        dfn, dff = compare_target_to_neighbour_monthly(df, n)
        concat_list.append(dfn)
        if ticker == 0:
            df_factor = dff.copy()
        ticker += 1

    df = pd.concat(concat_list, axis=1, join='outer')

    cols = ["n" + str(i + 1) for i in range(len(concat_list) - 1)]
    cols2 = ["target"]
    cols2.extend(cols)

    df.columns = cols2

    df["online"] = len(concat_list) - df[cols].isnull().sum(axis=1) - 1
    conditions = [
        ((df[cols] == -3).T.sum() == df.online),
        ((df[cols] <= -2).T.sum() == df.online),
        ((df[cols] <= -1).T.sum() == df.online),
        ((df[cols] == 3).T.sum() == df.online),
        ((df[cols] >= 2).T.sum() == df.online),
        ((df[cols] >= 1).T.sum() == df.online)]

    choices = [-3, -2, -1, 3, 2, 1]

    df["flags"] = np.select(conditions, choices, default=0)
    df.loc[np.isnan(df['target']), 'flags'] = np.nan

    # Add additional checks in relation to monthly neighbours climatology

    # Calculate neighbour monthly climatology (monthly min/max across all neighbours)
    tmp = []
    for n in neighbours:
        tmp.append(n)
    df_mc = pd.concat(tmp, axis=1, join='outer')
    cols3 = ["n" + str(i + 1) for i in range(len(tmp))]
    df_mc.columns = cols3
    df_mc.index = pd.to_datetime(df_mc.index, format='%Y-%m-%d')
    df_mc_max = df_mc.groupby(df_mc.index.month).max()
    df_mc_max['max'] = df_mc_max.max(axis=1)
    df_mc_min = df_mc.groupby(df_mc.index.month).min()
    df_mc_min['min'] = df_mc_min.min(axis=1)
    df_mc2 = pd.concat(
        [df_mc_min.drop(cols3, axis=1), df_mc_max.drop(cols3, axis=1)], axis=1,
        join='outer')
    df_mc2['month'] = df_mc2.index

    # Join monthly climatology to target series
    df['month'] = df.index.month
    df['date'] = df.index
    df2 = df.merge(df_mc2, left_on='month', right_on='month')
    df2.set_index('date', inplace=True)
    df2.sort_index(inplace=True)

    # Adjust flag where -3 or 3 and rainfall is outside neighbours min/max range
    df2.loc[(df2['flags'] == -3) & (df2['online'] >= 3) &
            ((df2['target'] < (0.75 * df2['min'])) | (df2['target'] > (1.25 * df2['max']))),
            'flags'] = -4
    df2.loc[(df2['flags'] == 3) & (df2['online'] >= 3) &
            ((df2['target'] < (0.75 * df2['min'])) | (df2['target'] > (1.25 * df2['max']))),
            'flags'] = 4

    # Flag where less than 3 neighbours but value > 2 * neighbour max or
    # < 0.5 * neighbour min
    df2.loc[(df2['online'] < 3) & (df2['target'] > (2.0 * df2['max'])),
            'flags'] = 5
    df2.loc[(df2['online'] < 3) & (df2['target'] < (0.5 * df2['min'])),
            'flags'] = -5

    # If less than 3 stations online than flag check as incomplete unless flag
    # has a value of -5/5 (i.e. outside climatology range)
    df2.loc[(df2.online < 3) & (np.absolute(df2['flags']) != 5), "flags"] = np.nan
    dfr = df2.flags

    return [dfr, df_factor]

def convert_to_dry_spell(daily_df):
    # dp 29/11/2019 - it would make sense to remove np.around so fractional not binary,
    # but this will require a change to how the flagging is done for it to make sense
    # i think (i.e. do all stations agree the period is wet when the target is dry?)
    # should the threshold for dry be larger than just zero?

    dry_periods = daily_df.copy()
    dry_periods[dry_periods > 0] = -1
    dry_periods += 1

    daily_df["fracDryDays"] = dry_periods.rolling(15, min_periods=15).sum() / 15

    converted_df = daily_df["fracDryDays"]
    converted_df.columns = ["ts1"]
    return converted_df

def propagate_flags(series, days=14):
    series = series.copy()
    # flag preceding periods, prioritising higher flag values
    for flag, series_filtered in [(flag, series[series == flag]) for flag in [1, 2, 3]]:
        for idx, value in series_filtered.iteritems():
            series[idx - datetime.timedelta(days=days):idx] = flag

    return series

# Prepare ETCCDI variables
def read_etccdi_data(etccdi_data_folder):
    etccdi_data = {"GHCNDEX": {}, "HADEX2": {}}
    etccdi_indices = ['CWD', 'CDD', 'R99p', 'PRCPTOT', 'SDII', 'Rx1day']
    periods = {"GHCNDEX": '1951-2018', "HADEX2": '1951-2010'}
    aggregations = {}
    for index in etccdi_indices:
        aggregations[index] = 'max'
    aggregations['SDII'] = 'mean'
    for data_source in etccdi_data.keys():
        for index in etccdi_indices:
            etccdi_data_path = (etccdi_data_folder + '/RawData_' + data_source +
                                '_' + index + '_' + periods[data_source] +
                                '_ANN_from-90to90_from-180to180.nc')
            etccdi_data[data_source][index] = prep_etccdi_variable(etccdi_data_path,
                                                                   index, aggregations[index], data_source)
    return etccdi_data

# create kd tree of monthly gauges ++++++++++++++++++++++++++++++++++++++
def create_kdtree_monthly_data(path):

    with open(path, "r") as monthly_info:

        monthly_names = []
        monthly_dates = []
        monthly_coords = []
        monthly_info.readline()

        for line in monthly_info:
            line_list = [line[0:10], line[11:54], line[54:62], line[62:73], line[73:79], line[125:135], line[137:147]]
            sid, name, lat, lon, elv, sd, ed = [l.strip() for l in line_list]
            try:
                sd = datetime.datetime.strptime(sd, "%d.%m.%Y")
                ed = datetime.datetime.strptime(ed, "%d.%m.%Y")
            except:
                sd = None
                ed = None

            if elv == "-999":
                elv = 100  # use 100m above seal level as a default elevation
            else:
                elv = float(elv)

            if sd is None or ed is None:
                pass
            else:
                monthly_names.append(sid)
                monthly_dates.append((sd, ed))
                monthly_coords.append((float(lat), float(lon), elv))

        converted_monthly_coords = [geodetic_to_ecef(a, b, c) for a, b, c in monthly_coords]
        monthly_tree = sp.KDTree(converted_monthly_coords)

    return monthly_names, monthly_dates, monthly_coords, monthly_tree

# create kd tree of daily gauges ++++++++++++++++++++++++++++++++++++++
def create_kdtree_daily_data(path):
    with open(path, "r") as daily_info:

        daily_names = []
        daily_dates = []
        daily_coords = []

        daily_info.readline()

        for line in daily_info:
            line_list = [line[0:10], line[11:54], line[54:62], line[62:73], line[73:79], line[125:135], line[137:147]]
            sid, name, lat, lon, elv, sd, ed = [l.strip() for l in line_list]
            try:
                sd = datetime.datetime.strptime(sd, "%d.%m.%Y")
                ed = datetime.datetime.strptime(ed, "%d.%m.%Y")
            except:
                sd = None
                ed = None

            if elv == "-999":
                elv = 100  # use 100m above sea level as a default elevation
            else:
                elv = float(elv)

            if sd is None or ed is None:
                pass
            else:
                daily_names.append(sid)
                daily_dates.append((sd, ed))
                daily_coords.append((float(lat), float(lon), elv))

        converted_dailyCoords = [geodetic_to_ecef(a, b, c) for a, b, c in daily_coords]
        tree = sp.KDTree(converted_dailyCoords)

    return daily_names, daily_dates, daily_coords, tree

# create kd tree of hourly gauges ++++++++++++++++++++++++++++++++++++++
def create_kdtree_hourly_data(path):
    with open(path, "r") as hourlyn_info:

        hourly_n_names = []
        hourly_n_dates = []
        hourly_n_coords = []
        hourly_n_paths = []
        converted_hourly_n_coords = []

        hourly_n_names_t = []
        hourly_n_dates_t = []
        hourly_n_coords_t = []
        hourly_n_paths_t = []

        hourlyn_info.readline()

        for line in hourlyn_info:
            sid, lat, lon, sd, ed, elv, hpath = line.rstrip().split(",")

            try:
                sd = datetime.datetime.strptime(sd, "%Y%m%d%H")
                ed = datetime.datetime.strptime(ed, "%Y%m%d%H")
            except:
                sd = None
                ed = None

            if elv.lower() == "na" or elv == "m" or elv == "nam" or elv == "nan":
                elv = 100  # use 100m above sea level as a default elevation
            else:
                if elv.endswith("m"):
                    elv = elv[:-1]
                try:
                    elv = float(elv)
                except:
                    elv = 100

            if sd is None or ed is None:
                pass
            else:

                # Only append if >=3 years of record (no point having potential neighbours
                # without substantial data)
                # - Also ensure that no duplicates arising from e.g. duplicates in Australia1min.zip
                date_diff = ed - sd
                if date_diff.days >= 3 * 365:
                    if sid not in hourly_n_names_t:
                        hourly_n_names_t.append(sid)
                        hourly_n_dates_t.append((sd, ed))
                        hourly_n_coords_t.append((float(lat), float(lon), elv))
                        hourly_n_paths_t.append(hpath)

        converted_hourly_n_coords_t = [geodetic_to_ecef(a, b, c) for a, b, c in hourly_n_coords_t]

        for i in range(len(converted_hourly_n_coords_t)):
            addIt = 1
            for j in converted_hourly_n_coords_t[i]:
                if np.isnan(j):
                    addIt = 0

            if addIt == 1:
                hourly_n_names.append(hourly_n_names_t[i])
                hourly_n_dates.append(hourly_n_dates_t[i])
                hourly_n_coords.append(hourly_n_coords_t[i])
                hourly_n_paths.append(hourly_n_paths_t[i])
                converted_hourly_n_coords.append(converted_hourly_n_coords_t[i])

        hourly_n_tree = sp.KDTree(converted_hourly_n_coords)

    return hourly_n_names, hourly_n_dates, hourly_n_coords, hourly_n_paths, hourly_n_tree

def try_float(test_val):
    try:
        v = float(test_val)
    except:
        v = np.nan
    return v

def try_strptime(test_val):
    try:
        v = datetime.datetime.strptime(test_val, '%Y%m%d%H')
    except:
        v = np.nan
    return v

def try_int(test_val):
    try:
        v = int(test_val)
    except:
        v = np.nan
    return v

def try_list(test_list):
    try:
        v = [try_int(i) for i in test_list[1:-1].split(", ")]
    except:
        v = np.nan
    return v

def install_r_package(package_name):
    """Installs """
    utils = importr('utils')
    utils.install_packages(StrVector([package_name]), repos='http://cran.us.r-project.org')


# -*- coding: utf-8 -*-
"""
Setup file for EDWIN
"""
import os
from setuptools import setup, find_packages
import pkg_resources

repo = os.path.dirname(__file__)
try:
    from git_utils import write_vers
    version = write_vers(vers_file='ed_win/__init__.py', repo=repo, skip_chars=1)
except Exception:
    version = '999'

try:
    from pypandoc import convert_file

    def read_md(f): return convert_file(f, 'rst', format='md')
except ImportError:
    print("warning: pypandoc module not found, could not convert Markdown to RST")

    def read_md(f): return open(f, 'r').read()

setup(name='ed_win',
      version=version,
      description='EDWIN an optimization and design package for electrical networks in windfarms',
      long_description=read_md('README.md'),
      url='https://gitlab.windenergy.dtu.dk/TOPFARM/EDWIN',
      author='DTU Wind Energy',
      author_email='juru@dtu.dk',
      license='MIT',
      packages=find_packages(),
      install_requires=[
          'matplotlib',  # for plotting
          'numpy',  # for numerical calculations
          'xarray',  # for WaspGridSite data storage
          'scipy',  # constraints
      ],
      extras_require={
          'test': [
              'pytest',  # for testing
              'pytest-cov',  # for calculating coverage
              'sphinx',  # generating documentation
              'sphinx_rtd_theme',  # docs theme
          ]},
      zip_safe=True)


import pytest
from src.Ej_4_3 import operaciones_conjuntos

@pytest.mark.parametrize(
    "inSet1, inSet2, outTupla",
    [  # interseccion, solo_prim, solo_seg
        ({"manzana", "pera", "naranja", "plátano", "uva"},
         {"manzana", "pera", "durazno", "sandía", "uva"},
         ({"manzana", "pera", "uva"}, {"naranja", "plátano"}, {"durazno", "sandía"})),
        ({"manzana", "kiwi", "naranja", "plátano", "uva"},
         {"manzana", "pera", "durazno", "sandía", "uva"},
         ({"manzana", "uva"}, {"kiwi", "naranja", "plátano"}, {"pera", "durazno", "sandía"})),
    ]
)
def test_operaciones_conjuntos(inSet1, inSet2, outTupla):
    assert operaciones_conjuntos(inSet1, inSet2) == outTupla


import configparser
import os
from pathlib import Path
import platform
import shutil
import tempfile
import warnings
from typing import MutableMapping, Optional

import lmdb

from . import __version__
from .constants import (
    CONFIG_USER_NAME,
    DIR_DATA_REMOTE,
    DIR_DATA_STAGE,
    DIR_DATA_STORE,
    DIR_DATA,
    LMDB_BRANCH_NAME,
    LMDB_HASH_NAME,
    LMDB_REF_NAME,
    LMDB_SETTINGS,
    LMDB_STAGE_HASH_NAME,
    LMDB_STAGE_REF_NAME,
    README_FILE_NAME,
)
from .records.commiting import unpack_commit_ref
from .records.heads import (
    create_branch,
    get_branch_head_commit,
    get_staging_branch_head,
    set_staging_branch_head,
)
from .records.parsing import repo_version_raw_spec_from_raw_string
from .records.vcompat import (
    is_repo_software_version_compatible,
    set_repository_software_version,
    startup_check_repo_version,
)
from .utils import readme_contents, is_64bits

class Environments(object):

    def __init__(self, pth: Path):

        self.repo_path: Path = pth
        self.refenv: Optional[lmdb.Environment] = None
        self.hashenv: Optional[lmdb.Environment] = None
        self.stageenv: Optional[lmdb.Environment] = None
        self.branchenv: Optional[lmdb.Environment] = None
        self.stagehashenv: Optional[lmdb.Environment] = None
        self.cmtenv: MutableMapping[str, lmdb.Environment] = {}
        self._startup()

    @property
    def repo_is_initialized(self) -> bool:
        """Property to check if the repository is initialized, read-only attribute

        Returns
        -------
        bool
            True if repo environments are initialized, False otherwise
        """
        ret = True if isinstance(self.refenv, lmdb.Environment) else False
        return ret

    def _startup(self) -> bool:
        """When first access to the Repo starts, attempt to open the db envs.

        This function is designed to fail if a repository does not exist at the
        :py:attribute:`repo_path` which is specified, so the user can
        explicitly choose to initialize the repo. Once opened, the lmdb
        environments should not be closed until the program terminates.

        Returns
        -------
        bool False if no repository exists at the given path, otherwise True

        Warns
        -----
        UserWarning Should the repository not exist at the provided repo path.

        Raises
        ------
        RuntimeError If the repository version is not compatible with the
        current software.
        """

        if not self.repo_path.joinpath(LMDB_BRANCH_NAME).is_file():
            msg = f'No repository exists at {self.repo_path}, please use `repo.init()` method'
            warnings.warn(msg, UserWarning)
            return False

        if not is_64bits():
            raise OSError(f'Hangar cannot run on 32 bit machines')

        repo_ver = startup_check_repo_version(self.repo_path)
        curr_ver = repo_version_raw_spec_from_raw_string(v_str=__version__)
        if not is_repo_software_version_compatible(repo_ver, curr_ver):
            msg = f'repository written version: {repo_ver} is not comatible '\
                  f'with the current Hangar software version: {curr_ver}'
            raise RuntimeError(msg)

        self._open_environments()
        return True

    def init_repo(self,
                  user_name: str,
                  user_email: str,
                  remove_old: bool = False) -> Path:
        """Create a new hangar repositiory at the specified environment path.

        Parameters
        ----------
        user_name : str
            Name of the repository user.
        user_email : str
            Email address of the respository user.
        remove_old : bool, optional(default value = False)
            DEVELOPER USE ONLY --- Remove all data and records stored in the
            repository if this opetion is enabled, defaults to False.

        Returns
        -------
        Path
            The path to the newly created repository on disk.

        Raises
        ------
        OSError
            If a hangar repository exists at the specified path, and `remove_old`
            was not set to ``True``.
        """
        if self.repo_path.joinpath(LMDB_BRANCH_NAME).is_file():
            if remove_old is True:
                shutil.rmtree(str(self.repo_path))
            else:
                raise OSError(f'Hangar Directory: {self.repo_path} already exists')

        self.repo_path.mkdir()
        self.repo_path.joinpath(DIR_DATA_STORE).mkdir()
        self.repo_path.joinpath(DIR_DATA_STAGE).mkdir()
        self.repo_path.joinpath(DIR_DATA_REMOTE).mkdir()
        self.repo_path.joinpath(DIR_DATA).mkdir()
        print(f'Hangar Repo initialized at: {self.repo_path}')

        userConf = {'USER': {'name': user_name, 'email': user_email}}
        CFG = configparser.ConfigParser()
        CFG.read_dict(userConf)
        with self.repo_path.joinpath(CONFIG_USER_NAME).open('w') as f:
            CFG.write(f)

        readmeTxt = readme_contents(user_name, user_email)
        with self.repo_path.joinpath(README_FILE_NAME).open('w') as f:
            f.write(readmeTxt.getvalue())

        self._open_environments()
        set_repository_software_version(branchenv=self.branchenv, ver_str=__version__)
        create_branch(self.branchenv, 'master', '')
        set_staging_branch_head(self.branchenv, 'master')
        return self.repo_path

    def checkout_commit(self, branch_name: str = '', commit: str = '') -> str:
        """Set up db environment with unpacked commit ref records.

        Parameters
        ----------
        branch_name : str, optional
            name of the branch to read, defaults to ''
        commit : str, optional
            name of the commit to read, defaults to ''

        Returns
        -------
        str
            commit hash which was checked out
        """
        if commit != '':
            commit_hash = commit
            txt = f' * Checking out COMMIT: {commit_hash}'
        elif branch_name != '':
            commit_hash = get_branch_head_commit(self.branchenv, branch_name)
            txt = f' * Checking out BRANCH: {branch_name} with current HEAD: {commit_hash}'
        else:
            head_branch = get_staging_branch_head(self.branchenv)
            commit_hash = get_branch_head_commit(self.branchenv, head_branch)
            txt = f'\n Neither BRANCH or COMMIT specified.'\
                  f'\n * Checking out writing HEAD BRANCH: {head_branch}'
        print(txt)

        # On UNIX-like system, an open process still retains ability to
        # interact with disk space allocated to a file when it is removed from
        # disk. Windows does not, and will not allow file to be removed if a
        # process is interacting with it. While the CM form is cleaner, this
        # hack allows similar usage on Windows platforms.

        if platform.system() != 'Windows':
            with tempfile.TemporaryDirectory() as tempD:
                tmpDF = os.path.join(tempD, f'{commit_hash}.lmdb')
                tmpDB = lmdb.open(path=tmpDF, **LMDB_SETTINGS)
                unpack_commit_ref(self.refenv, tmpDB, commit_hash)
                self.cmtenv[commit_hash] = tmpDB
        else:
            tempD = tempfile.mkdtemp()
            tmpDF = os.path.join(tempD, f'{commit_hash}.lmdb')
            tmpDB = lmdb.open(path=tmpDF, **LMDB_SETTINGS)
            unpack_commit_ref(self.refenv, tmpDB, commit_hash)
            self.cmtenv[commit_hash] = tmpDB

        return commit_hash

    def _open_environments(self):
        """Open the standard lmdb databases at the repo path.

        If any commits are checked out (in an unpacked state), read those in as
        well.
        """
        ref_pth = str(self.repo_path.joinpath(LMDB_REF_NAME))
        hash_pth = str(self.repo_path.joinpath(LMDB_HASH_NAME))
        stage_pth = str(self.repo_path.joinpath(LMDB_STAGE_REF_NAME))
        branch_pth = str(self.repo_path.joinpath(LMDB_BRANCH_NAME))
        stagehash_pth = str(self.repo_path.joinpath(LMDB_STAGE_HASH_NAME))

        self.refenv = lmdb.open(path=ref_pth, **LMDB_SETTINGS)
        self.hashenv = lmdb.open(path=hash_pth, **LMDB_SETTINGS)
        self.stageenv = lmdb.open(path=stage_pth, **LMDB_SETTINGS)
        self.branchenv = lmdb.open(path=branch_pth, **LMDB_SETTINGS)
        self.stagehashenv = lmdb.open(path=stagehash_pth, **LMDB_SETTINGS)

    def _close_environments(self):

        self.refenv.close()
        self.hashenv.close()
        self.stageenv.close()
        self.branchenv.close()
        self.stagehashenv.close()
        for env in self.cmtenv.values():
            if platform.system() == 'Windows':
                envpth = env.path()
                env.close()
                os.remove(envpth)
            else:
                env.close()


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo.exceptions import UserError
# from odoo.tools.misc import datetime
from datetime import datetime

class SaleOrderInh(models.Model):
    _inherit = 'sale.order'

    payment_count = fields.Integer(compute='compute_payments')
    advance_payment = fields.Integer("Advance Payment", compute='compute_payments')

    @api.depends("partner_id")
    def compute_payments(self):
        obj = self.env['account.payment'].search_count([('so_number', '=', self.name)])
        if obj:
            self.payment_count = obj
        else:
            self.payment_count = 0
        count = self.env['account.payment'].search([('so_number', '=', self.name)],limit=1)
        self.advance_payment = count.amount

    def action_register_payment(self):
        return {
            'type': 'ir.actions.act_window',
            'name': 'Apply Advance Payments',
            'view_id': self.env.ref('sale_order_payment.view_advance_payment_wizard_form', False).id,
            'context': {'default_ref': self.name, 'default_order_amount': self.amount_total, 'default_user_id': self.user_id.id},
            'target': 'new',
            'res_model': 'advance.payment.wizard',
            'view_mode': 'form',
        }

    def action_show_payments(self):
        return {
            'type': 'ir.actions.act_window',
            'name': 'Advance Payments',
            'view_id': self.env.ref('account.view_account_payment_tree', False).id,
            'target': 'current',
            'domain': [('communication', '=', self.name)],
            'res_model': 'account.payment',
            'views': [[False, 'tree'], [False, 'form']],
        }

    def action_confirm(self):
        flag = 0
        for rec in self:
            payments = self.env['account.payment'].search([('partner_id', '=', rec.partner_id.id), ('communication', '=', rec.name)])
            partner = self.env['res.partner'].search([('id', '=', rec.partner_id.id)], limit=1)
            print(partner.total_due)
            received_payment = 0
            for payment in payments:
                received_payment = received_payment + payment.amount
            if received_payment >= (rec.amount_total/2) or (partner.total_due*-1) >= (rec.amount_total/2):
                res = super(SaleOrderInh, self).action_confirm()
                flag = 1

        if flag == 0:
            raise UserError('There is no enough Advance Payment available to Confirm this Sale Order.')

#
# class StockPickingInh(models.Model):
#     _inherit = 'stock.picking'
#
#     state = fields.Selection([
#         ('draft', 'Draft'),
#         ('waiting', 'Waiting Another Operation'),
#         ('confirmed', 'Waiting'),
#         ('manager_approval', 'Approval from Manager'),
#         ('ceo_approval', 'Approval from CEO'),
#         ('reserve_manager_approvals', 'Reserve Approval from Manager'),
#         ('reserve_ceo_approval', 'Reserve Approval from CEO'),
#         ('assigned', 'Ready'),
#         ('done', 'Done'),
#         ('cancel', 'Cancelled'),
#     ], string='Status', compute='_compute_state',
#         copy=False, index=True, readonly=True, store=True, tracking=True,
#         help=" * Draft: The transfer is not confirmed yet. Reservation doesn't apply.\n"
#              " * Waiting another operation: This transfer is waiting for another operation before being ready.\n"
#              " * Waiting: The transfer is waiting for the availability of some products.\n(a) The shipping policy is \"As soon as possible\": no product could be reserved.\n(b) The shipping policy is \"When all products are ready\": not all the products could be reserved.\n"
#              " * Ready: The transfer is ready to be processed.\n(a) The shipping policy is \"As soon as possible\": at least one product has been reserved.\n(b) The shipping policy is \"When all products are ready\": all product have been reserved.\n"
#              " * Done: The transfer has been processed.\n"
#              " * Cancelled: The transfer has been cancelled.")
#     no_enough_amount = fields.Boolean(default=False, compute='compute_payment')
#
#     def compute_payment(self):
#         for rec in self:
#             partner = self.env['res.partner'].search([('id', '=', rec.partner_id.id)])
#             if partner:
#                 if abs(partner.total_due) >= (rec.sale_id.amount_total / 2):
#                     rec.no_enough_amount = False
#                 else:
#                     rec.no_enough_amount = True
#             else:
#                 rec.no_enough_amount = False
#
#
#     def action_reserve_do(self):
#         flag = 0
#         for rec in self:
#             partner = self.env['res.partner'].search([('id', '=', rec.partner_id.id)])
#             if partner:
#                 if abs(partner.total_due) >= (rec.sale_id.amount_total/2):
#                     rec.action_assign()
#                     flag = 1
#
#             if flag == 0:
#                 raise UserError('There is no enough Advance Payment available to Reserve this DO.')
#
#     def action_manager_approval(self):
#         self.state = 'ceo_approval'
#
#     def action_ceo_approval(self):
#         self.action_assign()
#
#     def action_get_approvals(self):
#         self.state = 'manager_approval'

class AccountMoveInh(models.Model):
    _inherit = 'account.move'

    advance_payment = fields.Float('Advance Payment', compute='compute_advance_payment')

    def compute_advance_payment(self):
        for rec in self:
            sale = self.env['sale.order'].search([('name', '=', rec.invoice_origin)])
            rec.advance_payment = sale.advance_payment

class AccountPaymentInh(models.Model):
    _inherit = 'account.payment'

    user_id = fields.Many2one('res.users')
    wiz_description = fields.Text('Description')
    cheque_no = fields.Char(string ="Cheque no")
    paid_by = fields.Char(string= "Paid by")
    received_by = fields.Char(string= "Received by")
    approved_by = fields.Char(string= "Paid by")
    so_number = fields.Char(string="SO Number")

    # def _compute_saleorder_number(self):
    #     for i in self:
    #         payments = self.env['sale.order'].search(
    #             [('name', '=', i.memo)])
    #         i.so_number=payments.name

    def get_amount_in_words(self,amount):
        amount_in_words = self.currency_id.amount_to_text(amount)
        amount_in_words = amount_in_words +" "+ "Only"
        return amount_in_words


import os, json
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm

def marked_spaces(L,space_length,params,ergodic): # UNTESTED!!!
    n_spaces = np.floor(L/space_length)
    spots = np.zeros(n_spaces,ergodic)
    for e in ergodic:
        for i in range(n_spaces):
            valid = False
            while not valid:
                l = np.random.normal(loc=params["mean_car_length"], scale=params["sigma_car_length"])
                if l <= space_length:
                    spots[i,e] = l
                    valid = True
    return spots

plt.style.use("transport-findings.mplstyle")

with open("json/strategies.json") as f:
    params = json.load(f)

if not os.path.exists('plots/'): os.makedirs('plots/')

labels = ["One end", "Either end", "Middle", "Random"]
fig, ax = plt.subplots(1, 2, figsize=[6, 1.75])
ax = ax.flatten()

for i, strategy in enumerate(params["strategy"]):
    d = np.load(
        f"output/strategy_{strategy}/L_{params['L'][0]}/mean_car_length_{params['mean_car_length'][0]}/sigma_car_length_{params['sigma_car_length'][0]}/linear_density_0.npy"
    )
    ax[0].semilogx(
        np.arange(len(d)) / params["L"],
        d,
        label=labels[i],
        c=cm.plasma(i / len(params["strategy"])),
    )

ls = ['dashed','dotted']
for i,N in enumerate([2,3]):
    # analytic solution for large L
    ax[0].semilogx(
        np.arange(len(d)) / params["L"],
        np.ones(len(d))*params['mean_car_length'][0]/(params['mean_car_length'][0] + N*params['sigma_car_length'][0]),
        ls=ls[i],
        c='k',
        # label=r'$l=\mu+'+str(N)+r'\sigma$'
    )
    # brute force method
    # space_length = params['mean_car_length'][0] + N*params['sigma_car_length'][0]
    # marked_spots = marked_spaces(L,space_length,params,ergodic)
    # ax[0].semilogx(
    # )

    ax[0].text(
        3e-1,
        0.01 + params['mean_car_length'][0]/(params['mean_car_length'][0] + N*params['sigma_car_length'][0]),
        '$l_'+str(i+1)+'=\mu+' + str(N) + r'\sigma$'
    )

# ax[0].semilogx(
#     np.arange(len(d)) / params["L"],
#     np.ones(len(d))*params['mean_car_length'][0]/(params['mean_car_length'][0] + 3*params['sigma_car_length'][0]),
#     ls='dotted',
#     c='k',
#     label=r'$\mu+3\sigma$'
# )
#
# ax[0].text(
#     1e0,
#     0.01 + params['mean_car_length'][0]/(params['mean_car_length'][0] + 3*params['sigma_car_length'][0]),
#     '$\mu+3\sigma$'
# )

ax[0].set_xlabel("Vehicles departed per metre (veh/m)")
ax[0].set_ylabel("Linear density (-)")
# ax[0].legend(loc=0)
# plt.savefig('plots/summary_density.pdf')

# plt.clf()
bins = np.logspace(-2, 1, 20)
bin_centres = np.sqrt(bins[1:] * bins[:-1])
for i, strategy in enumerate(params["strategy"]):
    d = np.load(
        f"output/strategy_{strategy}/L_{params['L'][0]}/mean_car_length_{params['mean_car_length'][0]}/sigma_car_length_{params['sigma_car_length'][0]}/gaps_0.npy"
    )
    hist, bin_edges = np.histogram(
        d,
        bins=bins,
        # density=True
    )
    ax[1].plot(
        bin_centres, hist, ls="-", label=labels[i], c=cm.plasma(i / len(params["strategy"]))
    )
ax[1].set_xscale("log")
ax[1].set_xlabel("Gap size (m)")
ax[1].set_ylabel("Number")
ax[1].legend(loc=0)

plt.subplots_adjust(left=0.08, right=0.99, bottom=0.24, top=0.99, wspace=0.25, hspace=0.3)
plt.savefig("plots/strategies.pdf")