from tkinter import *
from tkinter import ttk
from tkinter import messagebox

from programa_frecuencias import prueba_frecuencias

class gui_prueba_frecuencias:

    def gui_frecuencias(self):
        gui = Tk()

        ancho_ventana = 425
        alto_ventana = 400
        x_ventana = gui.winfo_screenwidth() // 2 - ancho_ventana // 2
        y_ventana = gui.winfo_screenheight()// 2 - alto_ventana // 2
        posicion = str(ancho_ventana) + "x" + str(alto_ventana) + "+" + str(x_ventana) + "+" + str(y_ventana)
        gui.geometry(posicion)

        gui.title('Simulacion - Prueba de frecuencias')
        gui.config(bg = '#88bea1')
        gui.resizable(0,0)
        gui.iconbitmap("logo.ico")

        labeltitiulo = Label(gui, text = "Prueba de frecuencias", bg = "#cdced0", font = "Times", width = "26", height= "2")
        labeltitiulo.place(relx=0.175, rely=0.125)

        label_archivo = Label(gui, text = "Nombre del archivo:", bg = "#cdced0", font = "Times", width = "26", height= "1")
        label_archivo.place(relx=0.176, rely=0.35)

        entry_archivo = StringVar()
        entry_archivo = Entry(gui, width = "28", font = "Times", textvariable=entry_archivo,justify="center")
        entry_archivo.place(relx=0.169, rely=0.45)

        label_alpha = Label(gui, text = "Alpha:", bg = "#cdced0", font = "Times", width = "12", height= "1")
        label_alpha.place(relx=0.175, rely=0.55)

        entry_alpha = StringVar()
        entry_alpha = Entry(gui, width = "13", font = "Times", textvariable=entry_alpha,justify="center")
        entry_alpha.place(relx=0.179, rely=0.65)

        label_iteraciones = Label(gui, text = "Iteraciones:", bg = "#cdced0", font = "Times", width = "12", height= "1")
        label_iteraciones.place(relx=0.52, rely=0.55)

        entry_iteraciones = StringVar()
        entry_iteraciones = Entry(gui, width = "13", font = "Times", textvariable=entry_iteraciones,justify="center")
        entry_iteraciones.place(relx=0.525, rely=0.65)

        def acciones():

            archivo = entry_archivo.get()
            alpha = float(entry_alpha.get())
            secciones = entry_iteraciones.get()

            if archivo == "":
                messagebox.showerror("Error", "Debe ingresar el nombre del archivo")
            if alpha == "":
                messagebox.showerror("Error", "Debe ingresar el valor de alpha")
            if secciones == "":
                messagebox.showerror("Error", "Debe ingresar el numero de secciones")
            else:
                archivo = entry_archivo.get()
                alpha = entry_alpha.get()
                alpha = float(alpha)
                secciones = int(secciones)

            archivo_completo = archivo + ".txt"

            try:
                archivo = open(archivo_completo, "r")
                lineas = archivo.readlines()
                archivo.close()

                aleatorios = []
                for linea in lineas:
                    linea = linea.strip()
                    aleatorios.append(float(linea))

                for i in range(len(aleatorios)):
                    new_string = str(aleatorios[i])
                    size_string = len(new_string)
                    new_string = new_string[2:size_string-1]
                    aleatorios[i] = int(new_string)
                print(aleatorios)

                pf = prueba_frecuencias()
                pf.procedimiento_frecuencias(aleatorios, alpha, secciones)

            except FileNotFoundError:
                messagebox.showerror("Error", "El archivo no existe")

        bunidad = Button(gui, text = "Seleccionar", bg = '#cbc9bb', command = acciones, font = "Times")
        bunidad.place(relx=0.275, rely=0.8)
        bunidad.config(activebackground="#94A9B9", width="16", height="1")

        gui.mainloop()
'''pf = gui_prueba_frecuencias()
pf.gui_frecuencias()'''


import sys
from itertools import permutations

def check(a, b, c, cnt):
    global ans
    if a<=0 and b<=0 and c<=0:
        ans=min(ans, cnt)
        return

    a=0 if a<=0 else a
    b=0 if b<=0 else b
    c=0 if c<=0 else c

    if dp[a][b][c] <= cnt and dp[a][b][c]!=0:
        return

    dp[a][b][c] = cnt

    for perm in permutations((9, 3, 1), 3):
        check(a-perm[0], b-perm[1], c-perm[2], cnt+1)

N = int(sys.stdin.readline())
SCV = list(map(int, sys.stdin.readline().split()))

ans = 100
while len(SCV)<3:
    SCV+=[0]

dp = [[[100]*(max(SCV)+1) for _ in range(max(SCV)+1)] for _ in range(max(SCV)+1)]
check(SCV[0],SCV[1],SCV[2],0)
print(ans)


# -*- coding: utf-8 -*-
# @Time    : 2021/9/19 下午9:49
# @Author  : gavin
# @FileName: 150.数组中重复的数据.py
# @Software: PyCharm
# @Blog    ：https://blog.csdn.net/weixin_35154281
"""
问题描述：
    给定一个整数数组 a，其中1 ≤ a[i] ≤ n （n为数组长度）, 其中有些元素
出现两次而其他元素出现一次。找到所有出现两次的元素。你可以不用到任何额外空
间并在O(n)时间复杂度内解决这个问题吗？

解题方法：
使用符号进行标记，下次出现可用符号变化进行判断
时间复杂度：O(N)
空间复杂度：O(1)

原题链接：https://leetcode-cn.com/problems/find-all-duplicates-in-an-array/
"""

class Solution:
    def findDuplicates(self, nums: List[int]) -> List[int]:

        if not nums: return []

        res = []
        n = len(nums)
        for i in range(n):
            num = abs(nums[i])
            if nums[num - 1] < 0:
                res.append(num)
            else:
                nums[num - 1] = -nums[num - 1]
        return res

150.数组中重复的数据.py


"""
阿里大鱼短信发送Api

req=AlibabaAliqinFcSmsNumSendRequest(appkey, secret, url, partner_id)

req.extend="123456"
req.sms_type="normal"
req.sms_free_sign_name="注册验证"
req.sms_param=json.dumps(params)
req.rec_num="13000000000"
req.sms_template_code="SMS_5410290"
try:
    resp= req.getResponse()
    print(resp)
except Exception,e:
    print(e)

"""

import requests
import hashlib
from time import time
import json
import logging

from  conf import  settings
logger = logging.getLogger(__name__)

class RestApi(object):
    def __init__(self, key, secret, url="https://eco.taobao.com/router/rest", partner_id=""):
        self.key = key
        self.secret = secret
        self.url = url
        self.partner_id = partner_id

    def sign(self, params):
        #===========================================================================
        # '''签名方法
        # @param parameters: 支持字典和string两种
        # '''
        #===========================================================================
        if isinstance(params, dict):
            params = "".join(["".join([k, v]) for k,v in sorted(params.items())])
            params = "".join([self.secret, params, self.secret])
        sign = hashlib.md5(params.encode("utf-8")).hexdigest().upper()
        return sign

    def get_api_params(self):
        params = {}
        try:
            [params.__setitem__(k, getattr(self, k)) for k in self.get_param_names()]
        except AttributeError:
            raise Exception("Some parameters is needed for this api call")
        [params.__setitem__(k, getattr(self, k)) for k in self.get_option_names() if hasattr(self, k)]
        return params

    def getResponse(self, authorize=None):
        sys_params = {
            "method": self.get_api_name(),
            "app_key": self.key,
            "timestamp": str(int(time() * 1000)),
            "format": "json",
            "v": "2.0",
            "partner_id": self.partner_id,
            "sign_method": "md5",
        }
        if authorize is not None:
            sys_params['session'] = authorize
        params = self.get_api_params()
        sign_params = sys_params.copy()
        sign_params.update(params)
        sys_params['sign'] = self.sign(sign_params)
        headers = {
            'Content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
            "Cache-Control": "no-cache",
            "Connection": "Keep-Alive",
        }
        # headers = {"Content-Type":"application/x-www-form-urlencoded;charset=utf-8"}

        sys_params.update(params)
        logger.debug(json.dumps(sys_params))
        r = requests.post(self.url, params=sys_params, headers=headers)
        r.raise_for_status()
        return r.json()

class AlibabaAliqinFcSmsNumSendRequest(RestApi):
    def get_api_name(self):
        return "alibaba.aliqin.fc.sms.num.send"

    def get_param_names(self):
        return ['sms_type', 'sms_free_sign_name', 'rec_num', 'sms_template_code']

    def get_option_names(self):
        return ['extend', 'sms_param']

def send_msg(**kwargs):
    """
    向用户发送短信

    kwargs 需含以下两个参数
    msg ：{"operate": "找回密码", "code": _code}
    tel : 用户手机号

    """
    msg = kwargs["msg"]
    tel =  kwargs["tel"]

    appkey = settings["Alidayu"]["appkey"]
    secret = settings["Alidayu"]["secret"]

    req = AlibabaAliqinFcSmsNumSendRequest(appkey, secret)

    req.extend = "123456"
    req.sms_type = "normal"
    req.sms_free_sign_name = settings["Alidayu"]["sign_name"]
    req.sms_template_code =  settings["Alidayu"]["template_code"]
    req.sms_param = json.dumps(msg)
    req.rec_num = tel

    try:
        resp = req.getResponse()
        return resp
    except Exception:
        return  False


'''
Fusking is the act of generating many strings by using a template with a range
of integers or a spinner of alternate strings.

Ranges:
    x[1-10]y -> x1y, x2y, x3y, x4y, x5y, x6y, x7y, x8y, x9y, x10y
    x[01-10]y -> x01y, x02y, x03y, x04y, x05y, x06y, x07y, x08y, x09y, x10y

Spinners:
    x{alpha|beta|charlie}y -> xalphay, xbetay, xcharliey

fusker.fusker('https://subdomain-{a|b|c}.website.com/image[01-99].jpg') ->
(
    'https://subdomain-a.website.com/image01.jpg',
    'https://subdomain-a.website.com/image02.jpg',
    'https://subdomain-a.website.com/image03.jpg',
    ...
    'https://subdomain-a.website.com/image99.jpg',
    'https://subdomain-b.website.com/image01.jpg',
    'https://subdomain-b.website.com/image02.jpg',
)
'''
import argparse
import collections
import itertools
import string
import sys

from voussoirkit import basenumber
from voussoirkit import pipeable

class Landmark:
    def __init__(self, opener, closer, parser):
        self.opener = opener
        self.closer = closer
        self.parser = parser

def barsplit(chars):
    wordlist = []
    wordbuff = []

    def flush():
        if not wordbuff:
            return
        word = fusk_join(wordbuff)
        wordlist.append(word)
        wordbuff.clear()

    for item in chars:
        if item == '|':
            flush()
        else:
            wordbuff.append(item)
    flush()
    return wordlist

def fusk_join(items):
    form = ''
    fusks = []
    result = []
    for item in items:
        if isinstance(item, str):
            form += item
        else:
            form += '{}'
            fusks.append(item)
    product = itertools.product(*fusks)
    for group in product:
        f = form.format(*group)
        result.append(f)
    return result

def fusk_spinner(items):
    for item in items:
        if isinstance(item, str):
            yield item
        else:
            yield from item

def parse_spinner(characters):
    words = barsplit(characters)
    spinner = fusk_spinner(words)
    return spinner

def fusk_range(lo, hi, padto=0, base=10, lower=False):
    for x in range(lo, hi+1):
        x = basenumber.to_base(x, base)
        x = x.rjust(padto, '0')
        if lower:
            x = x.lower()
        yield x

def parse_range(characters):
    r = ''.join(characters)
    (lo, hi) = r.split('-')
    lo = lo.strip()
    hi = hi.strip()

    lowers = string.digits + string.ascii_lowercase
    uppers = string.digits + string.ascii_uppercase
    lohi = lo + hi
    lower = False
    if all(c in string.digits for c in lohi):
        base = 10
    elif all(c in lowers for c in lohi):
        lower = True
        base = 36
    elif all(c in uppers for c in lohi):
        base = 36
    else:
        base = 62

    if (not lo) or (not hi):
        raise ValueError('Invalid range', r)
    if len(lo) > 1 and lo.startswith('0'):
        padto = len(lo)
        if len(hi) != padto:
            raise ValueError('Inconsistent padding', lo, hi)
    else:
        padto = 0
    lo = basenumber.from_base(lo, base)
    hi = basenumber.from_base(hi, base)

    frange = fusk_range(lo, hi, padto=padto, base=base, lower=lower)
    return frange

landmarks = {
    '{': Landmark('{', '}', parse_spinner),
    '[': Landmark('[', ']', parse_range),
}

def fusker(fstring, landmark=None, depth=0):
    escaped = False
    result = []
    buff = []

    if isinstance(fstring, str):
        fstring = collections.deque(fstring)
    while fstring:
        character = fstring.popleft()
        if escaped:
            buff.append('\\' + character)
            escaped = False
        elif character == '\\':
            escaped = True
        elif landmark and character == landmark.closer:
            buff = [landmark.parser(buff)]
            break
        elif character in landmarks:
            subtotal = fusker(fstring, landmark=landmarks[character])
            buff.extend(subtotal)
        else:
            buff.append(character)
    if not landmark:
        buff = parse_spinner(buff)
    return buff
    return result

def fusker_argparse(args):
    for result in fusker(args.pattern):
        pipeable.stdout(result)
    return 0

def main(argv):
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument('pattern')
    parser.set_defaults(func=fusker_argparse)

    args = parser.parse_args(argv)
    return args.func(args)

if __name__ == '__main__':
    raise SystemExit(main(sys.argv[1:]))


'''
DESCRIPTION

This module takes the cleaned data from the ETL module and trains a ML classification model
The output is a model which can be used to predict on new data

INPUTS

database_filepath -   path containing the processed data database

OUTPUTS

Saves the trained model to a pickle file

SCRIPT EXECUTION SAMPLE

python train_classifier.py ../data/Disaster_Response_Data.db classifier.pkl

'''
# import relevant libraries
import sys
import pandas as pd
import numpy as np
from sqlalchemy import create_engine
import re
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

from sklearn.metrics import confusion_matrix, classification_report
from sklearn.model_selection import train_test_split
from sklearn.model_selection import  GridSearchCV
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
from sklearn.multioutput import MultiOutputClassifier
from sklearn.base import BaseEstimator, TransformerMixin
import pickle

import nltk
nltk.download(['punkt', 'wordnet', 'averaged_perceptron_tagger'])

def load_data(database_filepath):

    '''
    DESCRIPTION
    Loads processed data from database at given file path

    INPUTS
    database_filepath - path containing the processed data database

    OUTPUTS
    X - dataframe of features for prediction model
    Y - Series of field we want to predict
    category_names - names of each column in dataset

    '''
    # load data from database

    engine = create_engine('sqlite:///' + database_filepath)
    df = pd.read_sql("SELECT * FROM messages", engine)

    # from inspection of data we see the 'related' column has some values set to 2. we impute those values to 1 since it's most dominant
    df['related']=df['related'].map(lambda x: 1 if x == 2 else x)

    # get X,y values to be used for classification

    X = df['message']
    Y = df.drop(['id', 'message', 'original', 'genre'], axis=1)

    # get column category list
    category_names = list(np.array(Y.columns))

    return X, Y, category_names

def tokenize(text):
    '''
    DESCRIPTION
    Normalizes and tokenizes text input given to it
    URLs and email addresses are also replaced with generic placeholders and tokens are lemmatized

    INPUTS
    text - sample text data from X dataframe

    OUTPUTS
    tokens_list - list of tokenized words from given input text

    '''
    # regex to identify URLs and email addresses

    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    email_regex = '^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$'

    # find and store all URLs and email addresses

    urls_found = re.findall(url_regex, text)
    emails_found = re.findall(email_regex, text)

    # replace all found URLs and email addresses with placeholder names

    for url in urls_found:
        text = text.replace(url, "urlplaceholder")

    for email in emails_found:
        text = text.replace(email, "emailplaceholder")

    # tokenize text input

    words = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    tokens_list = []

    # lemmatize text and convert text to lowercase

    for word in words:
        word_token = lemmatizer.lemmatize(word).lower().strip()
        tokens_list.append(word_token)

    return tokens_list

# Custom Transformer class to  get starting verb of text

class StartingVerbExtractor(BaseEstimator, TransformerMixin):

    def starting_verb(self, text):
        sentence_list = nltk.sent_tokenize(text)
        for sentence in sentence_list:
            pos_tags = nltk.pos_tag(tokenize(sentence))
            first_word, first_tag = pos_tags[0]
            if first_tag in ['VB', 'VBP'] or first_word == 'RT':
                return True
        return False

    def fit(self, x, y=None):
        return self

    def transform(self, X):
        X_tagged = pd.Series(X).apply(self.starting_verb)
        return pd.DataFrame(X_tagged)

def build_model():

    '''
    DESCRIPTION
    Creates a pipeline for building and training classification model

    OUTPUTS
    cv - best model output from Gridsearch operation on the pipeline

    '''
    pipeline = Pipeline([
        ('features', FeatureUnion([

            ('text_pipeline', Pipeline([
                ('count_vect', CountVectorizer(tokenizer=tokenize)),
                ('tfidf_transform', TfidfTransformer())
            ])),

             ('starting_verb', StartingVerbExtractor())

        ])),

        ('classifier', MultiOutputClassifier(AdaBoostClassifier()))
    ])

    parameters = {'classifier__estimator__learning_rate': [0.01, 0.1],
                   'classifier__estimator__n_estimators': [16, 32]}

    cv = GridSearchCV(pipeline, param_grid=parameters, verbose=1, n_jobs=-1)

    return cv

def evaluate_model(model, X_test, Y_test, category_names):

    '''
    DESCRIPTION
    Evaluates and prints scores of model on test set data

    INPUTS
    model - classification model
    X_test - test set for X
    Y_test - test set for Y
    category_names - names of each column in dataset

    '''
    y_pred = model.predict(X_test)

    print(classification_report(Y_test.values, y_pred, target_names=category_names))

def save_model(model, model_filepath):

    '''
    DESCRIPTION
    Saves classfication model for later use

    INPUTS
    model - classification model
    file_path - specified name of model

    '''
    with open(model_filepath, 'wb') as f:
        pickle.dump(model, f)

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


import numpy as np
import ephem
import datetime
import sys
sys.path.insert(0, '../../..')
from utils.rfi.sat_sim.sim_sat_paths import radec_to_lm, set_observer
from utils.helper.parallelize import parmap

locations = {'ska': (-30.712586, 21.442888),
             'canarvon': (-30.963501, 22.139584),
             'vanwyksvlei': (-30.332197, 21.818244),
             'brandvlei': (-30.472060, 20.465047),
             'williston': (-31.343608, 20.926702)}

def get_bearing(lat1, lon1, lat2, lon2):
    """
    Straight line bearing from 'lat1, lon1' to 'lat2, lon2'.

    Parameters
    ----------
    lat1 : float
        Latitude of the initial position in degrees or radians.
    lon1 : float
        Longitude of the initial position in degrees or radians.
    lat2 : float
        Latitude of the final position in degrees or radians.
    lon2 : float
        Longitude of the final position in degrees or radians.

    Returns
    -------
    bearing : float
        Straight line bearing from initial position pointing to final position.
        Units must be consitent between 'lat1, lon1, lat2, lon2'.
    """
    x, y = lon2-lon1, lat2-lat1
    bearing = np.arctan2(y, x)
    if bearing<0:
        bearing += 2*np.pi

    return bearing

def get_lm(args):
    """
    Get the direction cosine for a bearing on the horizon.
    """
    bearings, obs_date, phase_centre = args
    obs = set_observer(obs_date)
    lm = np.zeros((len(bearings), 2))
    for i, bearing in enumerate(bearings):
        ra, dec = np.rad2deg(obs.radec_of(bearing, 0))
        delta_ra = np.abs(ra - phase_centre[0])
        delta_dec = np.abs(dec - phase_centre[1])
        if delta_ra<90 and delta_dec<90:
            lm[i] = radec_to_lm(ra, dec, phase_centre)
        else:
            lm[i] = -0.7, -0.7

    return lm

def get_horizon_lm_tracks(phase_centre, transit, tracking_hours,
                          integration_secs=8):
    """
    Get the direction cosine tracks for a source on the horizon.
    """
    start_time = datetime.datetime.strptime(transit, '%a %b %d %H:%M:%S %Y') - \
                 datetime.timedelta(seconds=3600*tracking_hours/2)

    time_steps = int(3600*tracking_hours/integration_secs)

    ska = locations['ska']
    towns = ['canarvon', 'vanwyksvlei', 'brandvlei', 'williston']
    bearings = [get_bearing(ska[0], ska[1], *locations[town]) for town in towns]
    # bearings = np.deg2rad(np.arange(360))

    # Set up argument list for parallelization
    obs_times = [start_time + datetime.timedelta(seconds=i*8) for i in range(time_steps)]
    all_rfi = [bearings for i in range(len(obs_times))]
    centres = [phase_centre for i in range(len(obs_times))]
    arg_list = [list(i) for i in zip(all_rfi, obs_times, centres)]

    # Call parallelization function to get l,m and altitude for all time steps
    all_time = np.array(parmap(get_lm, arg_list, proc_power=0.8))

    return all_time


#import json

def flatten_tweets(tweets_in):
    """ Flattens out tweet dictionaries so relevant JSON
        is in a top-level dictionary."""
    tweets_list = []

    # Iterate through each tweet
    for tweet in tweets_in:
        #tweet_obj = json.loads(tweet)

        # Store the user screen name in 'user-screen_name'
        tweet['user-screen_name'] = tweet['user']['screen_name']

        # Check if this is a 140+ character tweet
        if 'extended_tweet' in tweet:
            # Store the extended tweet text in 'extended_tweet-full_text'
            tweet['extended_tweet-full_text'] = tweet['extended_tweet']['full_text']

        if 'retweeted_status' in tweet:
            # Store the retweet user screen name in 'retweeted_status-user-screen_name'
            tweet['retweeted_status-user-screen_name'] = tweet['retweeted_status']['user']['screen_name']

            # Store the retweet text in 'retweeted_status-text'
            tweet['retweeted_status-text'] = tweet['retweeted_status']['text']

            if 'extended_tweet' in tweet['retweeted_status']:
                tweet['retweeted_status-extended_tweet-full_text'] = tweet['retweeted_status']['extended_tweet']['full_text']

        if 'quoted_status' in tweet:
            # Store the quoted user screen name in 'retweeted_status-user-screen_name'
            tweet['quoted_status-user-screen_name'] = tweet['quoted_status']['user']['screen_name']

            # Store the retweet text in 'retweeted_status-text'
            tweet['quoted_status-text'] = tweet['quoted_status']['text']

            if 'extended_tweet' in tweet['quoted_status']:
                tweet['quoted_status-extended_tweet-full_text'] = tweet['quoted_status']['extended_tweet']['full_text']

        tweets_list.append(tweet)
    return tweets_list

def check_word_in_tweet(word, data):
    """Checks if a word is in a Twitter dataset's text.
    Checks text and extended tweet (140+ character tweets) for tweets,
    retweets and quoted tweets.
    Returns a logical pandas Series.
    """
    contains_column = data['text'].str.contains(word, case = False)
    contains_column |= data['extended_tweet-full_text'].str.contains(word, case = False)
    contains_column |= data['quoted_status-text'].str.contains(word, case = False)
    contains_column |= data['quoted_status-extended_tweet-full_text'].str.contains(word, case = False)
    contains_column |= data['retweeted_status-text'].str.contains(word, case = False)
    contains_column |= data['retweeted_status-extended_tweet-full_text'].str.contains(word, case = False)
    return contains_column


import datetime
from multiprocessing import context
from employees.forms import AdminAttendanceForm, AdminEmpAttendance, AdminLeaveForm, AdminRegularizationForm
from .tasks import calculate_salary, get_balance, get_month,get_year, leave_approval, leave_marked, leave_reject
from django.urls import reverse_lazy
from django.views.generic import DeleteView,UpdateView
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import get_object_or_404, redirect, render
from django.contrib.auth.decorators import login_required
from admin.forms import AddEmployeeForm, DesignationForm, EditEmployeeForm, HolidayForm, InterviewForm, JobForm, MessageForm, PayrollEditForm, SalaryForm
from django.contrib import messages
from admin.models import Designations, Holidays, Payroll, Salary
from applicants.models import Applications, Interviews, Messages
from base.models import Department, Jobs
from employees.models import Attendance, AttendanceRegularization, EmployeeDesignation, Leave, LeaveCounter, YearCounter
from users.models import ApplicantProfile, Newuser
from django.db.models import Q
from django.core.paginator import Paginator,EmptyPage
from django.contrib.auth.views import PasswordChangeView

@login_required
def admin_home(request):
    employees = Newuser.objects.filter(is_employee=True).count()
    applicants = Newuser.objects.filter(is_applicant=True).count()
    department = Department.objects.all().count()
    designations = Designations.objects.all().count()
    interviews = Interviews.objects.filter(interview_date__gt=datetime.date.today()).count()
    leave_count = Leave.objects.filter(approval = 'pending').count()
    leaves = Leave.objects.filter().order_by('-id')[:3]
    regular_count = AttendanceRegularization.objects.filter(status='pending').count()
    regular = AttendanceRegularization.objects.filter().order_by('-id')[:3]
    att_count = Attendance.objects.filter(attendance_date = datetime.date.today(),shift='morning').exclude(entry_time = None).count()
    mon_leave = Leave.objects.filter(applied_date__month = datetime.date.today().month,applied_date__year=datetime.date.today().year).count()
    mon_app = Leave.objects.filter(applied_date__month = datetime.date.today().month,applied_date__year=datetime.date.today().year,approval='approved').count()
    att_per = int((att_count/employees)*100)
    reg_mon = AttendanceRegularization.objects.filter(date__month=datetime.date.today().month,date__year=datetime.date.today().year).count()
    reg_app = AttendanceRegularization.objects.filter(date__month=datetime.date.today().month,date__year=datetime.date.today().year,status='approved').count()
    up_int = Interviews.objects.filter(interview_date__gt=datetime.date.today()).order_by('interview_date').first()
    holiday = Holidays.objects.filter(date__gt = datetime.date.today()).order_by('date').first()
    jobs = Jobs.objects.filter().order_by('-id')[:2]
    job_count = Jobs.objects.all().count()
    context = {
       'employees':employees,
       'applicants':applicants,
       'department':department,
       'designations':designations,
       'interviews':interviews,
       'leave_count':leave_count,
       'leaves':leaves,
       'regular_count':regular_count,
       'regular':regular,
       'att_per':att_per,
       'att_count':att_count,
       'mon_leave':mon_leave,
       'mon_app':mon_app,
       'reg_mon':reg_mon,
       'reg_app':reg_app,
       'up_int':up_int,
       'holiday':holiday,
       'jobs':jobs,
       'job_count':job_count

    }
    return render(request, 'admin/admin_dashboard.html',context)

class AdPasswordChangeView(SuccessMessageMixin,PasswordChangeView):
   template_name = 'admin/admin_password_change.html'
   success_message = "Password Changed"
   success_url = reverse_lazy('admin_password_change')

# adding the employee

@login_required
def add_employee(request):
    if request.method == "POST":
        reg_form = AddEmployeeForm(request.POST)    #containes detials for newuser table
        salary_form = SalaryForm(request.POST)      #contines details for salary table
        desig_form = DesignationForm(request.POST)  #contains details for EmployeeDesignation table

        if reg_form.is_valid() and salary_form.is_valid() and desig_form.is_valid():
            user = reg_form.save(False)
            user.is_employee = True
            user.save()

            salary = salary_form.save(False)
            salary.user = user

            designation = desig_form.save(False)
            designation.user =user

           #to create month counter
            year = get_year()
            i = 1
            while i<=12:
                date = datetime.date(year,i,1)
                LeaveCounter.objects.create(cl=0,el=0,lp=0,sl=0,date=date,user=user)
                i = i+1

            #to create year
            YearCounter.objects.create(cl=0,el=0,lp=0,sl=0,user=user)

            #to create the attendance for that day
            Attendance.objects.create(attendance_date=datetime.date.today(),entry_time=None,exit_time=None,shift='morning',user=user)

            salary.save()
            designation.save()

            messages.success(request,f'Employee added')
            reg_form = AddEmployeeForm()
            salary_form = SalaryForm()
            desig_form = DesignationForm()
            return render(request, 'admin/add-employee.html', {'reg_form':reg_form, 'salary_form':salary_form, 'desig_form':desig_form })

        else:

            return render(request, 'admin/add-employee.html', {'reg_form':reg_form, 'salary_form':salary_form, 'desig_form':desig_form })

    else:
        reg_form = AddEmployeeForm()
        salary_form = SalaryForm()
        desig_form = DesignationForm()
        return render(request, 'admin/add-employee.html', {'reg_form':reg_form, 'salary_form':salary_form, 'desig_form':desig_form })

# view the employees in table
@login_required
def view_employee(request):
    employees = Newuser.objects.filter(is_employee = True).order_by('first_name')
    obj = EmployeeDesignation.objects.all()

    return render(request, 'admin/employee-view.html',{'employees':employees,'obj':obj})

#detailed view of an employee
@login_required
def employee_detail(request,pk):
    employee = Newuser.objects.get(id = pk)
    #to get designation of employee from EmployeeDesignation
    desig_obj = EmployeeDesignation.objects.get(user = pk)
    designation = Designations.objects.get(id = desig_obj.designation_id )

    #to get department of employee from EmployeeDesignation

    dep_obj = EmployeeDesignation.objects.get(user = pk)
    department = Department.objects.get(id = dep_obj.department_id )

    salary = Salary.objects.get(user = pk)

    return render(request, 'admin/employee-detail.html',{ 'employee':employee, 'designation':designation, 'department':department,'salary':salary})

#edit employee
@login_required
def employee_edit(request,pk):

    #get the instance that we are updateing currently
    emp = Newuser.objects.get(id = pk)
    sal = Salary.objects.get(user = pk)
    desig = EmployeeDesignation.objects.get(user = pk)

    if request.method == "POST":
        employee = EditEmployeeForm(request.POST, instance=emp)
        salary = SalaryForm(request.POST,instance=sal)
        designation = DesignationForm(request.POST,instance=desig)

        if employee.is_valid() and salary.is_valid() and designation.is_valid():
            employee.save()
            salary.save()
            designation.save()
            messages.success(request,f'Changes Saved for the Employee')
            return redirect(employee_detail,pk)
        else:
            return redirect(employee_detail,pk)

    else:

        reg_form = EditEmployeeForm(instance = emp)
        salary_form = SalaryForm(instance = sal)
        desig_form = DesignationForm(instance=desig)

        return render(request, 'admin/edit_employee.html', {'reg_form':reg_form, 'salary_form':salary_form, 'desig_form':desig_form })

#  Delete an employee
@login_required
def delete_employee(requset,pk):
    employee = Newuser.objects.get(id=pk)
    employee.delete()
    messages.success(requset,f'Employee Deleted')
    return redirect('view_employee')

# department
@login_required
def department(request):
    if request.method == "POST":
        department = Department()
        department.department_name = request.POST['department']
        department.save()
        messages.success(request,f'Department Added')
        return redirect('department')
    else:
        departments = Department.objects.all()
        return render(request, 'admin/department-view.html',{'departments':departments})

# Designations
@login_required
def designations(request):
    if request.method == "POST":
        designation = Designations()
        designation.designation = request.POST['designation']
        designation.save()
        messages.success(request,f'Designation Added')
        return redirect('designations')
    else:
        designations = Designations.objects.all()
        return render(request, 'admin/designation.html',{'designations':designations})

# Edit Designation
@login_required
def designation_edit(request,pk):
    if request.method == "POST":
        designation = Designations(id = pk)
        designation.designation = request.POST['designation']
        designation.save()
        messages.success(request,f'Designation Updated')
        return redirect('designations')
    else:
        edit = Designations.objects.get(id = pk)
        return render(request, 'admin/designation_edit.html',{'edit': edit})

# Edit Department
@login_required
def department_edit(request,pk):
    if request.method == "POST":
        department = Department(id = pk)
        department.department_name = request.POST['department']
        department.save()
        messages.success(request,f'Department Updated')
        return redirect('department')
    else:
        edit = Department.objects.get(id = pk)
        return render(request, 'admin/department_edit.html',{'edit': edit})

# delete Department
class DeleteDepartment(SuccessMessageMixin,DeleteView):
    template_name = 'admin/department_delete.html'
    success_message = "Department Deleted"
    success_url = reverse_lazy('department')

    def get_object(self):
        id =  self.kwargs.get("pk")
        # print(id)
        return get_object_or_404(Department, id=id)

# delete Designation
class DeleteDesignation(SuccessMessageMixin,DeleteView):
    template_name = 'admin/designation_delete.html'
    success_message = "Designation Deleted"
    success_url = reverse_lazy('designations')

    def get_object(self):
        id =  self.kwargs.get("pk")
        print(id)
        return get_object_or_404(Designations, id=id)

# job add for applicant
@login_required
def add_jobs(request):
    if request.method == "POST":
        form = JobForm(request.POST)
        if form.is_valid():
            form.save()
            messages.success(request,f'Job Added')
            return redirect('add_jobs')
        else:
            return render(request, 'admin/add_jobs.html',{'form':form })
    else:
        form = JobForm()
        return render(request, 'admin/add_jobs.html',{'form':form })

# view all the jobs
@login_required
def view_all_jobs(request):
    jobs = Jobs.objects.all()
    return render(request, 'admin/view_jobs.html',{ 'jobs':jobs })

#detailed job view
@login_required
def job_detail_view(request,slug):
    job_id = Jobs.objects.get(id = slug)
    return render(request, 'admin/admin_jobdetail_view.html',{'job_id': job_id})

#edit Job
class EditJob(SuccessMessageMixin,UpdateView):
   model = Jobs
   form_class = JobForm
   template_name = 'admin/add_jobs.html'
   success_message = "Job Updated"

#delete job by admin
@login_required
def delete_job(request,pk):
    job = Jobs.objects.get(id=pk)
    job.delete()
    messages.success(request,f'Job Deleted')
    return redirect('view_all_jobs')

#view applicants by admin
@login_required
def view_applicants(request):
    applicants = Newuser.objects.filter(is_applicant=True)
    return render(request,'admin/view_applicants.html',{ 'applicants':applicants})

#view applicant details
@login_required
def applicant_details(request,pk):
    applicant = ApplicantProfile.objects.get(user=pk)
    return render(request, 'admin/view_applicant_details.html',{'applicant':applicant})

#  Delete an applicant
@login_required
def delete_applicants(requset,pk):
    applicant = Newuser.objects.get(id=pk)
    applicant.delete()
    messages.success(requset,f'Applicant Deleted')
    return redirect('view_applicants')

@login_required
def applications(request,pk):
    applications = Applications.objects.filter(job = pk).order_by('-applied_date')
    job = Jobs.objects.get(id = pk)
    context = {
        'applications':applications,
        'job':job
    }
    return render(request, 'admin/view_applications.html',context)

#to accept the applicant
@login_required
def applicant_status(request,id,jobid,btn):
    applicant = Applications.objects.get(id = id)
    user = applicant.user.first_name +' '+ applicant.user.middle_name +' '+ applicant.user.last_name
    if btn == 'accept':
        applicant.selected = 'accepted'
        messages.success(request,f'{user} Accepted')
    elif btn == 'reject':
        applicant.selected = 'rejected'
        messages.warning(request,f'{user} Rejected')
    applicant.save()
    return redirect('applications', pk= jobid)

#to check if any interview is ready to be scheduled
@login_required
def check_schedule(request):
    jobs = Jobs.objects.filter(withdraw_date__lt=datetime.date.today()).order_by('-withdraw_date')
    context = {
        'jobs':jobs
    }
    return render(request,'admin/check_schedule.html',context)

#scheule interview
@login_required
def add_interview(request,pk):
    job = Jobs.objects.get(id=pk)
    if request.method =="POST":
        form = InterviewForm(request.POST)
        if form.is_valid():
            interview = form.save(False)
            interview.job = job
            interview.department = job.department
            job.scheduled = True
            job.save()
            interview.save()
            messages.success(request,f'Interview added')
            return redirect('check_schedule')
        else:
            return render(request,'admin/add_interview.html',{'form':form,'job':job})
    else:
        form = InterviewForm()
        return render(request,'admin/add_interview.html',{'form':form,'job':job})

#view scheduled interviews
@login_required
def scheduled_interviews(request,pk):
    interview = Interviews.objects.get(job=pk)
    return render(request,'admin/view_scheduled_interviews.html',{'interview':interview})

#edit scheduled interviews
@login_required
def edit_interviews(request,pk):
    interview = Interviews.objects.get(job=pk)  #take the interview with the given job id
    job = Jobs.objects.get(id =pk)  #get job to display job title in template
    if request.method == "POST":
        form = InterviewForm(request.POST,instance=interview)
        if form.is_valid():
            form.save()
            messages.success(request,f'Interview Edited')
            return redirect('scheduled_interviews',pk=pk)
        else:
            return render(request,'admin/add_interview.html',{'form':form,'job':job})

    else:
        form = InterviewForm(instance=interview)
        return render(request,'admin/add_interview.html',{'form':form,'job':job})

@login_required
def selected_applicants(request,pk):
    applications = Applications.objects.filter(job = pk, selected ='accepted')
    return render(request, 'admin/view_selected_applicants.html',{'applications':applications})

#cancel a scheduled interview
@login_required
def cancel_interview(request,pk):
    if request.method == "POST":
        interview = Interviews.objects.get(job=pk)
        job = Jobs.objects.get(id=pk)
        job.scheduled = False
        job.save()
        interview.delete()
        messages.success(request,f'Interview for {job.job_title} cancelled')
        return redirect('check_schedule')

    else:
        interview = Interviews.objects.get(job=pk)
        return render(request, 'admin/cancel_interview.html',{'interview':interview})

#to view the leaves
@login_required
def admin_view_leaves(request,user=None):
    obj = EmployeeDesignation.objects.all() #for the department of employee
    if request.method == "POST":
        filter = request.POST['filter']
        if filter == 'month':
            if user:
                leaves = Leave.objects.filter(from_date__month = get_month(),user=user).order_by('-applied_date')
            else:
                leaves = Leave.objects.filter(from_date__month = get_month()).order_by('-applied_date')
        elif filter =='all':
            if user:
                leaves = Leave.objects.filter(user=user).order_by('-applied_date')
            else:
                leaves = Leave.objects.all().order_by('-applied_date')
        return render(request,'admin/admin_view_leaves.html',{'leaves':leaves,'obj':obj,'filter':filter})

    else:
        if user:
                leaves = Leave.objects.filter(user=user).order_by('-applied_date')
        else:
                leaves = Leave.objects.all().order_by('-applied_date')
        return render(request,'admin/admin_view_leaves.html',{'leaves':leaves,'obj':obj})

@login_required
def admin_leave_detail(request,pk):
    leave = Leave.objects.get(id = pk)
    #get current months leave counter
    month_counter = LeaveCounter.objects.get(date__month=get_month(),date__year=get_year(),user=leave.user)
    year_counter = YearCounter.objects.get(date__year=get_year(),user=leave.user)
    context = {
        'leave':leave,
        'month_counter':month_counter,
        'year_counter':year_counter
    }
    return render(request,'admin/admin_leave_detail.html',context)

# to edit admin added leave
@login_required
def admin_leave_edit(request,pk):
    attendance = Leave.objects.get(id = pk)  #leave is renamed to attendance
    #get current months leave counter
    leave_form = AdminLeaveForm(instance=attendance)
    month_counter = LeaveCounter.objects.get(date__month=get_month(),date__year=get_year(),user=attendance.user)
    year_counter = YearCounter.objects.get(date__year=get_year(),user=attendance.user)
    context = {
        'form':leave_form,
        'attendance':attendance,
        'month':month_counter,
        'count':year_counter
    }
    if request.method == "POST":
        leave_form = AdminLeaveForm(request.POST,instance=attendance)
        if leave_form.is_valid():
            #leave balance check
            balance = get_balance(leave_form,attendance.user)
            leave_type = leave_form.cleaned_data['leave_type']
            if balance:
                messages.warning(request, f'Please check your {leave_type.upper()} availability')
                return render(request, 'admin/admin_leave_apply.html',context)
            leave_form.save()
            return redirect('admin_leave_detail',pk=pk)
    else:
        return render(request,'admin/admin_leave_apply.html',context)

@login_required
def approve_leave(request,pk):
    leave = Leave.objects.get(id=pk)
    if leave.approval != 'approved':
        leave_approval(leave)
        messages.success(request,f'Leave Approved')
        return redirect('admin_leave_detail',pk=pk)

    else:
        messages.info(request,f'Already Approved')
        return redirect('admin_leave_detail',pk=pk)

@login_required
def reject_leave(request,pk):
    leave = Leave.objects.get(id=pk)
    if leave.approval == 'approved':
        leave_reject(leave)
        messages.warning(request,f'Leave Rejected')
        return redirect('admin_leave_detail',pk=pk)

    else:
        if leave.approval == 'rejected':
            messages.info(request,f'Already Rejected')
        else:
            leave.approval = 'rejected'
            leave.save()
            messages.warning(request,f'Leave Rejected')

        return redirect('admin_leave_detail',pk=pk)

def leave_history(request):
    leaves = YearCounter.objects.filter(date__year=get_year())
    obj = EmployeeDesignation.objects.all()
    context = {
        'leaves':leaves,
        'obj':obj
    }
    return render(request, 'admin/leave_history.html', context)

@login_required
def leave_history_detail(request,pk):
    year = YearCounter.objects.get(date__year=get_year(),user=pk)
    month = LeaveCounter.objects.filter(date__year=get_year(),user=pk)
    employee = Newuser.objects.get(id=pk)
    context = {
        'year':year,
        'month':month,
        'employee':employee
    }
    return render(request,'admin/leave_history_detail.html', context)

@login_required
def admin_attendance_view(request,user=None):
    if request.method == "POST":
      shift = request.POST['shift']
      if user:
            attendance =  Attendance.objects.filter(shift = shift,user=user).exclude(entry_time = None).order_by('-attendance_date')
      else:
            attendance =  Attendance.objects.filter(shift = shift).exclude(entry_time = None).order_by('-attendance_date')
      return render(request, 'admin/admin_attendance_view.html',{'attendance':attendance,'shift':shift})

    else:
        if user:
            attendance =  Attendance.objects.filter(shift='morning',user=user).exclude(entry_time = None).order_by('-attendance_date')
        else:
            attendance =  Attendance.objects.filter(shift='morning').exclude(entry_time = None).order_by('-attendance_date')

        return render(request, 'admin/admin_attendance_view.html',{'attendance':attendance})

@login_required
def admin_missing_regular(request):
    if request.method == "POST":
        filter = request.POST['filter']
        if filter == 'today':
            #todays attendance missing
            attendance =  Attendance.objects.filter(attendance_date =datetime.date.today(),shift='morning',entry_time=None).order_by('-attendance_date')
            for attendance in attendance:
                flag = leave_marked(attendance.user,attendance.attendance_date)
                if flag:
                    attendance.leave = True
                    attendance.save()
                else:          #if flag is false and leave is True (when rejected)
                    if attendance.leave == True:
                        attendance.leave = False
                        attendance.save()
                attendance =  Attendance.objects.filter(attendance_date =datetime.date.today(),shift='morning',entry_time=None).order_by('-attendance_date')

        elif filter == 'month':
            #this months attendance missing
            attendance =  Attendance.objects.filter(entry_time=None, attendance_date__month=get_month(),shift='morning').order_by('-attendance_date')
            for attendance in attendance:
                flag = leave_marked(attendance.user,attendance.attendance_date)
                if flag:
                    attendance.leave = True
                    attendance.save()
                else:          #if flag is false and leave is True (when rejected)
                    if attendance.leave == True:
                        attendance.leave = False
                        attendance.save()
            attendance =  Attendance.objects.filter(entry_time=None, attendance_date__month=get_month(),shift='morning').order_by('-attendance_date')

        elif filter == 'all':
            attendance =  Attendance.objects.filter(entry_time=None, shift='morning').order_by('-attendance_date')
            for attendance in attendance:
                flag = leave_marked(attendance.user,attendance.attendance_date)
                if flag:
                    attendance.leave = True
                    attendance.save()
                else:          #if flag is false and leave is True (when rejected)
                    if attendance.leave == True:
                        attendance.leave = False
                        attendance.save()
            attendance =  Attendance.objects.filter(entry_time=None, shift='morning').order_by('-attendance_date')

        return render(request, 'admin/admin_missing_regular.html',{'attendance':attendance,'filter':filter})

    else:
        attendance =  Attendance.objects.filter(entry_time=None, attendance_date = datetime.date.today(),shift='morning').order_by('-attendance_date')
        for attendance in attendance:
                flag = leave_marked( attendance.user,attendance.attendance_date)
                if flag:
                    attendance.leave = True
                    attendance.save()
                else:          #if flag is false and leave is True (when rejected)
                    if attendance.leave == True:
                        attendance.leave = False
                        attendance.save()
        attendance =  Attendance.objects.filter(entry_time=None, attendance_date = datetime.date.today(),shift='morning').order_by('-attendance_date')
        return render(request, 'admin/admin_missing_regular.html',{'attendance':attendance})

# to add leave by admin
@login_required
def admin_leave_apply(request,pk):
    attendance = Attendance.objects.get(id=pk)
    count = YearCounter.objects.get(user = attendance.user)
    month = LeaveCounter.objects.get(user = attendance.user, date__month=get_month())
    if request.method == "POST":
      leave_form = AdminLeaveForm(request.POST)
      if leave_form.is_valid():
         #leave balance check
         balance = get_balance(leave_form,attendance.user)
         leave_type = leave_form.cleaned_data['leave_type']
         if balance:
            messages.warning(request, f'Please check your {leave_type.upper()} availability')
            return render(request, 'admin/admin_leave_apply.html',{ 'form': leave_form,'attendance':attendance,'count':count,'month':month })

         leave = leave_form.save(False)
         leave.from_date = attendance.attendance_date
         leave.to_date = attendance.attendance_date
         leave.reason = "Leave Added by admin"
         leave.admin = True
         leave.user = attendance.user
         leave.save()
         return redirect('approve_leave', pk=leave.pk)
      else:
         return render(request, 'admin/admin_leave_apply.html',{ 'form': leave_form, 'attendance':attendance,'count':count,'month':month })
    else:
      form = AdminLeaveForm()
      return render(request, 'admin/admin_leave_apply.html',{ 'form': form, 'attendance':attendance,'count':count,'month':month })

#exit time not marked
@login_required
def exit_missing(request):
    if request.method == "POST":
        filter = request.POST['filter']
        if filter == 'today':
            #todays attendance missing
            attendance =  Attendance.objects.filter(attendance_date =datetime.date.today(),shift='morning',exit_time=None).exclude(entry_time = None).order_by('attendance_date')

        elif filter == 'month':
            #this months attendance missing
            attendance =  Attendance.objects.filter(exit_time=None, attendance_date__month=get_month(),shift='morning').exclude(entry_time = None).order_by('attendance_date')

        elif filter == 'all':
            attendance =  Attendance.objects.filter(exit_time=None, shift='morning').exclude(entry_time = None).order_by('attendance_date')

        return render(request, 'admin/exit_time_missing.html',{'attendance':attendance,'filter':filter})

    else:
        attendance =  Attendance.objects.filter(exit_time=None, attendance_date = datetime.date.today(),shift='morning').exclude(entry_time = None).order_by('attendance_date')
        return render(request, 'admin/exit_time_missing.html',{'attendance':attendance})

# Attendance Mark for Exit missing
@login_required
def exit_missing_mark(request,pk):
    attendance = Attendance.objects.get(id=pk)
    form = AdminAttendanceForm(instance=attendance)
    context = {
        'attendance':attendance,
        'form':form
    }
    if request.method == "POST":
        form = AdminAttendanceForm(request.POST ,instance=attendance)
        if form.is_valid():
            form.save()
            emp = attendance.user.first_name+' '+attendance.user.middle_name+' '+attendance.user.last_name
            messages.success(request,f'Exit time marked for {emp.upper()}')
            return redirect('admin_attendance_view')
    else:
        return render(request,'admin/admin_add_attendance.html',context)

#view night shift unmarked attendance:
@login_required
def night_unmarked(request):
    if request.method == "POST":
        filter = request.POST['filter']
        if filter == 'month':
            #this months attendance missing
            attendance =  Attendance.objects.filter(exit_time=None, attendance_date__month=get_month(),shift='night').exclude(entry_time = None).order_by('attendance_date')

        elif filter == 'all':
            attendance =  Attendance.objects.filter(exit_time=None, shift='night').exclude(entry_time = None).order_by('attendance_date')

        return render(request, 'admin/night_shift_unmarked.html',{'attendance':attendance,'filter':filter})

    else:
        attendance =  Attendance.objects.filter(exit_time=None, attendance_date__month=get_month(),shift='night').exclude(entry_time = None).order_by('attendance_date')
        return render(request, 'admin/night_shift_unmarked.html',{'attendance':attendance})

# Add an attendance to employee
@login_required
def add_emp_attendance(request,pk):
    emp = Newuser.objects.get(id=pk)
    form = AdminEmpAttendance()
    context = {
        'form':form,
        'emp':emp
    }
    if request.method == "POST":
        form = AdminEmpAttendance(request.POST)
        if form.is_valid():
            shift = form.cleaned_data['shift']
            date = form.cleaned_data['attendance_date']

            #to update the leve feild by checking if leave applied or not
            try:
                attendance = Attendance.objects.get(attendance_date=date,user=pk,shift='morning')
                flag = leave_marked(pk,date)
                if flag:
                        attendance.leave = True
                        attendance.save()
                else:          #if flag is false and leave is True (when rejected)
                    if attendance.leave == True:
                        attendance.leave = False
                        attendance.save()
            except:
                messages.warning(request,f'Check your attendance date')
                return render(request,'admin/admin_emp_attendance.html',{'form':form,'emp':emp})

            if shift == 'morning':
                attendance = Attendance.objects.get(attendance_date=date, user=pk, shift='morning')
                if attendance:
                    if attendance.entry_time == None and attendance.exit_time == None and attendance.leave == False:
                        attendance.entry_time = form.cleaned_data['entry_time']
                        attendance.exit_time = form.cleaned_data['exit_time']
                        attendance.save()
                        messages.success(request,f'Attendance Marked')
                        return redirect('admin_attendance_view',user=pk)
                    else:
                        messages.warning(request,f'Failed to mark attendance')
                else:
                    messages.warning(request,f'Failed to mark attendance')
                return render(request,'admin/admin_emp_attendance.html',{'form':form,'emp':emp})

            #  night shift
            else:
                if date > datetime.date.today():
                    messages.warning(request,f'Check your attendance date')
                    return render(request,'admin/admin_emp_attendance.html',{'form':form,'emp':emp})
                else:
                    leave = form.save(False)
                    leave.user = emp
                    leave.save()
                    messages.success(request,f'Attendance Added')
                    return redirect('admin_attendance_view',user=pk)
    else:
        return render(request,'admin/admin_emp_attendance.html',context)

# Attendance regularization
@login_required
def admin_regularize(request):
    attendance = AttendanceRegularization.objects.all().order_by('-id')
    context = {
        'attendance':attendance
    }
    return render(request,'admin/admin_regularization.html',context)

#apply regualrization
def apply_regularization(request,pk):
    attendance = AttendanceRegularization.objects.get(id=pk)
    id = attendance.attendance_id
    print(id)
    form = AdminRegularizationForm(instance=attendance)
    context = {
        'attendance':attendance,
        'form':form
    }
    if request.method == "POST":
        form = AdminRegularizationForm(request.POST, instance=attendance)
        if form.is_valid():
            old_attendance = Attendance.objects.get(id=id)
            old_attendance.entry_time = form.cleaned_data['new_entry']
            old_attendance.exit_time = form.cleaned_data['new_exit']
            old_attendance.save()
            attendance.status = 'approved'
            attendance.save()
            messages.success(request, f'Regularization Approved')
            return redirect('admin_regularize')

    return render(request,'admin/admin_regularization_detail.html', context)

#reject regularization
@login_required
def reject_regularization(request,pk):
    attendance = AttendanceRegularization.objects.get(id=pk)
    id = attendance.attendance_id
    if attendance.status == 'approved':
        old_attendance = Attendance.objects.get(id=id)
        old_attendance.entry_time = attendance.old_entry
        old_attendance.exit_time = attendance.old_exit
        old_attendance.save()
        attendance.status = 'rejected'
    else:
        attendance.status = 'rejected'
    attendance.save()
    messages.warning(request, f'Regularization rejected')
    return redirect('admin_regularize')

#notification
@login_required
def add_notification(request):
    form = MessageForm()
    message = Messages.objects.all().order_by('-id')
    count = Messages.objects.all().count()
    context = {
        'message':message,
        'count':count,
        'form':form
    }
    if request.method == "POST":
        if "search" in request.POST:
            value = request.POST['search_msg']
            message = Messages.objects.filter(Q (title__icontains=value) | Q(date__icontains=value)).order_by('-id')
            c = message.count()
            if c == 0:
                messages.info(request,f'No results found')
                return redirect('add_notification')
            else:
                p = Paginator(message,5)  # second argument is no of items to be displayed
                page_num = request.GET.get('page',1 ) #get the page no by url  and 1 is default
                try:
                    page = p.page(page_num)
                except EmptyPage:
                    page = p.page(1)
                return render(request, 'admin/add_notifications.html',{'message':page,'count':count,'form':form})

        elif "add" in request.POST:
            form = MessageForm(request.POST)
            if form.is_valid():
                msg = form.save(False)
                msg.date = datetime.datetime.now()
                msg.save()
                messages.success(request,'Message Added')
                return redirect('add_notification')

    else:
        if count != 0:
            p = Paginator(message,5)  # second argument is no of items to be displayed
            page_num = request.GET.get('page',1 ) #get the page no by url  and 1 is default
            try:
               page = p.page(page_num)
            except EmptyPage:
               page = p.page(1)
            return render(request, 'admin/add_notifications.html',{'message':page,'count':count,'form':form})
        else:
            return render(request, 'admin/add_notifications.html',context)

# edit notification
@login_required
def edit_notification(request,pk):
    message = Messages.objects.get(id=pk)
    if request.method == "POST":
        form = MessageForm(request.POST, instance=message)
        if form.is_valid():
            form.save()
            messages.success(request, f'Message Edited')
            return redirect('add_notification')
    else:
        form = MessageForm(instance=message)
        return render(request, 'admin/edit_notification.html',{'form':form})

#view message
@login_required
def view_notification(request,pk):
    message = Messages.objects.get(id=pk)
    return render(request, 'admin/view_message.html',{'message':message})

#delete message
@login_required
def delete_notification(request,pk):
    message = Messages.objects.get(id=pk)
    if request.method == "POST":
        message.delete()
        messages.warning(request,f'Message Deleted')
        return redirect('add_notification')
    else:
        return render(request, 'admin/delete_message.html',{'message':message})

#payroll month select
@login_required
def payroll_month(request):
    if request.method == "POST":
        date = request.POST['date']
        datem = datetime.datetime.strptime(date, "%Y-%m-%d")
        today = datetime.date.today()
        if  datem.year == today.year:
                if datem.month < today.month:
                    payroll = Payroll.objects.filter(date__month__gte=datem.month, date__year__gte=datem.year).count()
                    if payroll == 0:
                        return redirect('calculate_payroll',cdate=datem)

                    else:
                        messages.info(request,f'Payroll Already Calculated')
                        return redirect('payroll_month')
                else:
                    messages.warning(request,f'Select a valid month and year')
                    return redirect('payroll_month')

        elif  datem.year < today.year:
                payroll = Payroll.objects.filter(date__month__gte=datem.month, date__year__gte=datem.year).count()
                if payroll == 0:
                    return redirect('calculate_payroll',cdate=datem)
                else:
                    messages.info(request,f'Payroll Already Calculated')
                    return redirect('payroll_month')

        else:
                    messages.warning(request,f'Select a valid month and year')
                    return redirect('payroll_month')

    else:
         return render(request,'admin/payroll_month_selector.html')

#calculate payroll
@login_required
def calculate_payroll(request,cdate):
    cdate = datetime.datetime.strptime(cdate, "%Y-%m-%d %H:%M:%S")
    calculate_salary(cdate)
    messages.success(request, f'Calculation completed')
    return redirect('view_payroll', vdate = cdate)

#payroll view month select
@login_required
def view_payroll_month(request):
    if request.method == "POST":
        date = request.POST['date']
        datem = datetime.datetime.strptime(date, "%Y-%m-%d")
        payroll = Payroll.objects.filter(date__month=datem.month, date__year=datem.year).count()
        if payroll != 0:
            return redirect('view_payroll', vdate=datem)
        else:
            messages.warning(request,f'Not Available, please check your date')
            return redirect('view_payroll_month')
    else:
        return render(request, 'admin/payroll_view_month.html')

#view payroll
@login_required
def view_payroll(request,vdate):
    try:
        vdate = datetime.datetime.strptime(vdate, "%Y-%m-%d %H:%M:%S")
    except:
        vdate = datetime.datetime.strptime(vdate, "%Y-%m-%d")

    payroll = Payroll.objects.filter(date__month=vdate.month, date__year=vdate.year)
    obj = EmployeeDesignation.objects.all()
    context = {
        'payroll':payroll,
        'obj':obj,
        'vdate':vdate
    }
    return render(request, 'admin/view_payroll.html',context)

#hold salary
@login_required
def hold_salary(request, pk):
    salary = Payroll.objects.get(id=pk)
    emp = salary.user.first_name+' '+salary.user.middle_name+' '+salary.user.last_name
    if salary.status == 'hold':
        salary.status = 'pending'
        messages.success(request,f'Salary unholded for {emp.upper()}')
    else:
        salary.status = 'hold'
        messages.warning(request,f'Salary holded for {emp.upper()}')
    vdate = salary.date
    salary.save()
    return redirect('view_payroll', vdate = vdate)

#approve salary
@login_required
def approve_salary(request,pk):
    salary = Payroll.objects.get(id=pk)
    emp = salary.user.first_name+' '+salary.user.middle_name+' '+salary.user.last_name
    salary.status = 'approved'
    salary.save()
    vdate = salary.date
    messages.success(request,f'Salary Approved for {emp.upper()}')
    return redirect('view_payroll', vdate = vdate)

#view salary slip
@login_required
def salary_slip(request, pk):
    payroll = Payroll.objects.get(id=pk)
    emp = Newuser.objects.get(id = payroll.user.id)
    designation = EmployeeDesignation.objects.get(user = payroll.user.id)
    context = {
        'payroll':payroll,
        'emp':emp,
        'designation':designation
    }
    return render(request, 'admin/salary_slip.html',context)

#edit payroll
@login_required
def edit_payroll(request,pk):
    payroll = Payroll.objects.get(id=pk)
    form = PayrollEditForm()
    vdate = payroll.date
    if request.method == "POST":
        form = PayrollEditForm(request.POST)
        if form.is_valid():
            payroll.net_salary = payroll.net_salary + float(form.cleaned_data['other_benefits'])

            payroll.net_salary = payroll.net_salary - float(form.cleaned_data['other_deductions'])

            payroll.other_benefits = payroll.other_benefits + float(form.cleaned_data['other_benefits'])
            payroll.other_deductions = payroll.other_deductions +  float(form.cleaned_data['other_deductions'])

            payroll.save()
            messages.success(request, f'Payroll Edited Successfully')
            return redirect('view_payroll', vdate = vdate)
    else:
        return render(request, 'admin/edit_payroll.html',{'payroll':payroll,'form':form})

# to approve all pending payrolls
@login_required
def approve_all_payroll(request, vdate):
    vdate = datetime.datetime.strptime(vdate, "%Y-%m-%d %H:%M:%S")
    payrolls = Payroll.objects.filter(status='pending',date=vdate)
    for payroll in payrolls:
        payroll.status = 'approved'
        payroll.save()
    messages.success(request, f'Payrolls Approved')
    return redirect('view_payroll', vdate = vdate)

#add holidays
@login_required
def holidays(request):
    holidays = Holidays.objects.all().order_by('-date')
    form = HolidayForm()
    if request.method == "POST":
        form = HolidayForm(request.POST)
        if form.is_valid():
            date = form.cleaned_data['date']
            attendance = Attendance.objects.filter(attendance_date = date)
            if attendance:
                for attendance in attendance:
                    attendance.holiday = True
                    attendance.save()
            form.save()
            messages.success(request,f'Holiday Added')
            return redirect('holidays')
    else:
        return render(request,'admin/admin_holidays.html',{'holidays':holidays,'form':form})

#edit holidays
@login_required
def holiday_edit(request,pk):
    holiday = Holidays.objects.get(id = pk)
    form = HolidayForm(instance=holiday)
    if request.method == "POST":
        att = Attendance.objects.filter(attendance_date = holiday.date)
        if att:
            for att in att:
                att.holiday = False
                att.save()
        form = HolidayForm(request.POST,instance=holiday)
        if form.is_valid():
            date = form.cleaned_data['date']
            attendance = Attendance.objects.filter(attendance_date = date)
            if attendance:
                for attendance in attendance:
                    attendance.holiday = True
                    attendance.save()
            form.save()
            messages.success(request,f'Holiday Edited')
            return redirect('holidays')
    else:
        return render(request, 'admin/holiday_edit.html',{'form':form})

#delete holiday
@login_required
def delete_holiday(request,pk):
    holiday = Holidays.objects.get(id = pk)
    if request.method == "POST":
        att = Attendance.objects.filter(attendance_date = holiday.date)
        if att:
            for att in att:
                att.holiday = False
                att.save()
        holiday.delete()
        messages.success(request,f'Holiday Deleted')
        return redirect('holidays')
    else:
        return render(request,'admin/delete_holiday.html',{'holiday':holiday})


import gym
import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
from cascade.utils import EnvWithTerminal, Sampler, merge_data_, update_logging_stats, softmax_policy
from cascade.nn import CascadeQ
from cascade.utils import clone_lin_model, norm_squared_lin, stable_kl_div

def main():
    # env_id = 'MountainCar-v0'
    env_id = 'CartPole-v1'
    # env_id = 'Acrobot-v1'
    # env_id = 'LunarLander-v2'
    torch.set_num_threads(1)

    print('learning on', env_id)
    env = EnvWithTerminal(gym.make(env_id))
    env_sampler = Sampler(env)
    gamma = .99

    nb_act = env.get_nb_act()
    dim_s = env.get_dim_obs()
    cascade_qfunc = CascadeQ(dim_s, nb_act)
    nb_iter = 100
    nb_samp_per_iter = 10000
    min_grad_steps_per_iter = 10000
    min_td0_steps_per_iter = 0
    nb_add_neurone_per_iter = 10
    norm_weight = 0.
    neurone_nonlinearity = torch.nn.Tanh()

    batch_size = 64
    lr_model = 1e-3
    lr_td0 = 1e-3

    data = {}
    max_replay_memory_size = nb_samp_per_iter
    total_ts = 0
    curr_cum_rwd = 0
    returns_list = []
    eta = .1
    for iter in range(nb_iter):

        roll = env_sampler.rollouts(lambda x: softmax_policy(x, cascade_qfunc, eta), min_trans=nb_samp_per_iter, max_trans=nb_samp_per_iter)
        curr_cum_rwd, returns_list, total_ts = update_logging_stats(roll['rwd'], roll['done'], curr_cum_rwd, returns_list, total_ts)
        merge_data_(data, roll, max_replay_memory_size)

        obs, act, rwd, nobs, nact, not_terminal = torch.FloatTensor(data['obs']), torch.LongTensor(data['act']), \
                              torch.FloatTensor(data['rwd']), torch.FloatTensor(data['nobs']), torch.LongTensor(data['nact']),\
                                        1 - torch.FloatTensor(data['terminal'])

        print(f'iter {iter} ntransitions {total_ts} avr_return_last_20 {np.mean(returns_list[-20:])}')

        # add new neurone to cascade
        with torch.no_grad():
            obs_feat = cascade_qfunc.get_features(obs)
            nobs_feat = cascade_qfunc.get_features(nobs)
            nobs_q_all = cascade_qfunc.get_q(nobs)
            obs_q = cascade_qfunc.get_q(obs).gather(dim=1, index=act)
            nobs_q = nobs_q_all.gather(dim=1, index=nact)
            obs_old_distrib = torch.distributions.Categorical(logits=eta * cascade_qfunc(obs))
            nobs_old_distrib = torch.distributions.Categorical(logits=eta * cascade_qfunc(nobs))
            nobs_v = (nobs_q_all * nobs_old_distrib.probs).sum(1, keepdim=True)
            old_out = clone_lin_model(cascade_qfunc.output)
            # q_target = rwd + gamma * nobs_q * not_terminal

        cascade_qfunc.add_n_neurones(obs_feat, n_neurones=nb_add_neurone_per_iter, non_linearity=neurone_nonlinearity)
        # data_loader = DataLoader(TensorDataset(obs_feat, act, obs_q, q_target), batch_size=batch_size, shuffle=True, drop_last=True)
        grad_steps = 0
        # FQI
        while grad_steps < min_grad_steps_per_iter:
            # train
            train_losses = []
            with torch.no_grad():
                newqsp = cascade_qfunc.forward_from_old_cascade_features(nobs_feat).gather(dim=1, index=nact) # q-value for the next state and actions taken in the next states
                q_target = rwd + gamma * (nobs_q + newqsp) * not_terminal

                # newqsp_all = cascade_qfunc.forward_from_old_cascade_features(nobs_feat)
                # newvsp = (newqsp_all * nobs_old_distrib.probs).sum(1, keepdim=True)
                # q_target = rwd + gamma * (newvsp + nobs_v) * not_terminal

                data_loader = DataLoader(TensorDataset(obs_feat, act, obs_q, q_target), batch_size=batch_size, shuffle=True, drop_last=True)

            optim = torch.optim.Adam([*cascade_qfunc.cascade_neurone_list[-1].parameters(), *cascade_qfunc.output.parameters()], lr=lr_model)
            for s, a, oldq, tq in data_loader:
                optim.zero_grad()
                newqs = cascade_qfunc.forward_from_old_cascade_features(s)
                qs = newqs.gather(dim=1, index=a) + oldq
                loss = (qs - tq).pow(2).mean() + norm_weight * (norm_squared_lin(cascade_qfunc.output) + norm_squared_lin(cascade_qfunc.cascade_neurone_list[-1].f[0]))
                train_losses.append(loss.item())
                loss.backward()
                optim.step()
                grad_steps += 1
            print(f'\t grad_steps {grad_steps} q_error_train {np.mean(train_losses)} norm_squared_output '
                  f'{cascade_qfunc.output.weight.pow(2).sum() + cascade_qfunc.output.bias.pow(2).sum()}')

        # TD(0)
        grad_steps_td = 0
        optim_td0 = torch.optim.SGD(cascade_qfunc.output.parameters(), lr=lr_td0)
        while grad_steps_td < min_td0_steps_per_iter:
            # train
            train_losses = []
            with torch.no_grad():
                new_obs_feat = cascade_qfunc.get_features(obs)
                new_nobs_feat = cascade_qfunc.get_features(nobs)
                data_loader = DataLoader(TensorDataset(new_obs_feat, act, rwd, new_nobs_feat, nact, not_terminal, obs_q, nobs_q), batch_size=batch_size, shuffle=True, drop_last=True)

            for phis, a, r, phisp, ap, nt, oldqs, oldqsp in data_loader:
                optim_td0.zero_grad()
                qs = cascade_qfunc.output(phis).gather(dim=1, index=a) + oldqs
                qsp = cascade_qfunc.output(phisp).gather(dim=1, index=ap) + oldqsp
                td = (r + gamma * qsp * nt).detach() - qs
                loss = td.pow(2).mean() + norm_weight * (norm_squared_lin(cascade_qfunc.output) + norm_squared_lin(cascade_qfunc.cascade_neurone_list[-1].f[0]))
                train_losses.append(loss.item())

                loss.backward()
                optim_td0.step()

                grad_steps_td += 1
            print(f'\t grad_steps_td {grad_steps_td} q_error_train {np.mean(train_losses)} norm_squared_output '
                  f'{cascade_qfunc.output.weight.pow(2).sum() + cascade_qfunc.output.bias.pow(2).sum()}')

        cascade_qfunc.merge_q(old_out)
        with torch.no_grad():
            new_distrib = torch.distributions.Categorical(logits=eta * cascade_qfunc(obs))
            # kl = torch.distributions.kl_divergence(obs_old_distrib, new_distrib).mean().item()
            kl = stable_kl_div(obs_old_distrib.probs, new_distrib.probs).mean().item()
            if kl > 1e30:
                print("KL is too large!")
                print("obs_old_distrib.probs", obs_old_distrib.probs)
                print("new_distrib", new_distrib.probs)
            normalized_entropy = new_distrib.entropy().mean().item() / np.log(nb_act)
            print(f'grad_steps {grad_steps} q_error_train last epoch {np.mean(train_losses)} kl {kl} entropy (in (0, 1)) {normalized_entropy}')

if __name__ == '__main__':
    main()


import numpy as np
from pymoo.core.algorithm import Algorithm
from pymoo.core.population import Population
from pymoo.util.termination.no_termination import NoTermination

from pyallocation.allocation import FastAllocation
from pyallocation.problem import AllocationProblem

def exhaustively(problem):
    alloc = FastAllocation(problem, debug=False)
    k = 0
    sols = []
    rec_exhaustively(problem, alloc, k, sols)
    sols.sort(key=lambda x: (x[1], x[2]))
    return sols[:100]

def rec_exhaustively(problem, alloc, k, sols):
    if not alloc.feas:
        return

    if k == problem.n_var:
        x, cv, f = np.copy(alloc.x), alloc.CV, (alloc.F * problem.w).sum()
        sols.append((x, cv, f))

        if len(sols) > 1000:
            sols.sort(key=lambda x: (x[1], x[2]))
            while len(sols) > 100:
                sols.pop()

    else:
        for val in range(problem.xl[k], problem.xu[k] + 1):
            alloc.set(k, val)
            rec_exhaustively(problem, alloc, k + 1, sols)
            alloc.set(k, -1)

class ExhaustiveAlgorithm(Algorithm):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.default_termination = NoTermination()

    def setup(self, problem, **kwargs):
        super().setup(problem, **kwargs)
        assert isinstance(problem, AllocationProblem)
        return self

    def _initialize(self):
        self._next()

    def _next(self):
        solutions = exhaustively(self.problem)

        self.pop = Population.new(X=np.array([x for x, _, _ in solutions]))
        self.evaluator.eval(self.problem, self.pop)

        for ind in self.pop:
            print(ind.F[0], ind.X)

        self.termination.force_termination = True


import os
import sys
import asyncio

sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))

from presenter import bot  # noqa: E402
from common import setup_logging, PayloadProducer, PayloadConsumer  # noqa: E402
from common.constants import TOPICS  # noqa: E402

setup_logging('bot')

loop = asyncio.new_event_loop()
try:
    prod = PayloadProducer(loop)
    cons = PayloadConsumer(loop, TOPICS.RESULTS)

    bot.set_async_callback(prod.produce)
    cons.set_async_callback(bot.on_result)

    loop.run_until_complete(asyncio.gather(
        loop.create_task(prod.start()),
        loop.create_task(cons.run()),
        loop.create_task(bot.run())
    ))
finally:
    loop.close()


import argparse
import os
import re
import ConfigParser
from bs4 import BeautifulSoup
from docutils import core, io, nodes
from docutils.parsers import rst
from docutils.nodes import Special, Invisible, FixedTextElement
from anerist.rst import custom_directives

def slugify(title):
    """
    Give it a string, and slugify makes it a slug. Spaces are converted to
    hyphens, some basic language processing is done - for now, removing
    articles - and all characters are lower-cased.
    """
    title = title.lower()
    words = title.split(' ')
    articles = "a", "an", "the"
    words_removed = 0
    for word in words:
        if word in articles:
            words.remove(word)
    title = '-'.join(words)
    return title

# Thanks Alex Martelli!
# https://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788
class FakeSecHead(object):
    """
    This simply helps ConfigParser read perl config::simple config files.
    """
    def __init__(self, fp):
        self.fp = fp
        self.sechead = '[pants]\n'
    def readline(self):
        if self.sechead:
            try:
                return self.sechead
            finally:
                self.sechead = None
        else:
            return self.fp.readline()
class rest():
    """
    Extracts metadata from a ReStructuredText document.  Some 'custom' directives are
    expected, refer to rst/custom_directives for details.
    """
    parser = rst.Parser(rfc2822=True)

    def _read_rst_config(self, contents, source_path=None, destination_path=None):
        """
        Pass in a ReStructuredText document, as a string, and get back a docutils
        document object.
        """
        overrides = {}
        overrides['input_encoding'] = 'unicode'
        output, pub = core.publish_programmatically(
            source_class=io.StringInput,
            source=contents,
            source_path=source_path,
            destination_class=io.NullOutput, destination=None,
            destination_path=destination_path,
            reader=None, reader_name='standalone',
            parser=self.parser, parser_name='restructuredtext', #parser=None, parser_name='restructuredtext',
            writer=None, writer_name='null',
            settings=None, settings_spec=None, settings_overrides=overrides,
            config_section=None, enable_exit_status=None
            )
        document = pub.writer.document
        return document
    def _parse_metadata(self, document):
        """
        Give an entire ReStructuredText document, as a docutils 'document' object.
        Returns specific, targeted document attributes.
        Use like:
            title, slug, abstract, tags, taxonomy = _parse_metadata(document)
        """
        title = document.get('title')
        for element in document:
            if element.tagname == 'slug':
                slug = element.astext()
                # we don't have anything to reder this, so get it out!
                document.pop(document.index(element))
            elif element.tagname is 'abstract':
                abstract = element.astext()
            elif element.tagname is 'tags':
                tags = element.astext().split(',')
            elif element.tagname is 'taxonomy':
                taxonomy = element.astext()
            elif element.tagname is 'title':
                explicit_title = element.astext()
        if not 'abstract' in locals():
            abstract = ""
        if not 'tags' in locals():
            tags = []
        if not 'taxonomy' in locals():
            taxonomy = "uncategorized"
        if 'explicit_title' in locals():
            title = explicit_title
        if not 'slug' in locals():
            slug = slugify(title)

        return title, slug, abstract, tags, taxonomy
    def _read_file(self, inputfile, lang, extra_args):
        """
        Give a path to a ReStructuredText file, receive the file's metadata.
        """
        output = {}
        f = open(inputfile, 'r')
        print("reading " + os.path.basename(inputfile))
        contents = unicode(f.read(), errors='replace')
        f.close()
        document = self._read_rst_config(contents)
        output['title'], output['slug'], output['abstract'], output['tags'], output['taxonomy'] = self._parse_metadata(document)
        output['source_type'] = "rest"
        output['lang'] = lang
        if extra_args:
            for key in extra_args.keys():
                output[key] = extra_args[key]
        return output
    def read_broker(self, target, lang, extra_args):
        """
        Give it a list of files or paths, and it does all the things
        to retrieve metadata from ReStructuredText files it finds.

        The read_broker expects three arguments:

        - target: a list of files or paths to search.  Given paths,
            looks only for files ending in ".rst"

        - lang: The language code of the target document.  Usefulness is
            dubious here.

        - extra_args: A dictionary of extra metadata to slap on after
            reading the file.

        """
        metadata = []
        file_list = []
        for item in target:
            if os.path.isdir(item):
                for root, dirs, files in os.walk(item):
                    for name in files:
                        if name.endswith("rst"):
                            file_list.append(os.path.join(root, name))
            elif os.path.isfile(item) or os.path.islink(item):
                file_list.append(item)
            else:
                print("Invalid target type specified")
                sys.exit()
        for inputfile in file_list:
            metadatum = self._read_file(inputfile, lang, extra_args)
            metadata.append(metadatum)

        return metadata

class docbook():
    """
    Extracts metadata from a docbook project.  A few assumptions are made:

    - XML files for a given language are stored in a folder named by the language code.
      For example, US English sources are stored in an en-US directory.

    - This does not currently interpolate translation strings.  It expects fully prepared
      sources in the target language.

    - Content is mostly or wholly derived from either Book_Info.xml or Article_Info.xml.

    - Document taxonomy is represented in a <subjectterm> value.

    - Document tags are represented in a <keywordset> set.

    The first three asumptions are met with any publican-created projects.
    Other docbook source paradigms will need to have support added in the future.
    """
    def _get_xml_filelists(self, target, lang, scope='all'):
        """
        Given a list of paths, this walks the paths and returns a list of
        XML files, and a list of XML entity files, that are discovered there.

        It isn't smart, filename will need to end with ".xml" or ".ent", respectively.
        This could probably be improved with mime type detection, or whatever
        /usr/bin/file does.  It doesn't know about entities within XML files; the
        entity files are presumed to be in a format like::

            <!ENTITY PRODUCT "Anerist">
            <!ENTITY BOOKID "Anerist Guide">

        """

        xml_files = []
        entity_files = []
        for item in target:
#            item = unicode(item, errors='replace')
            for root, dirs, files in os.walk(item, lang):
                for name in files:
                    if name.endswith("xml"):
                        xml_files.append(os.path.join(root, name))
                    elif name.endswith("ent"):
                       entity_files.append(os.path.join(root, name))
        return entity_files, xml_files
         #placeholder - maybe later we might only want xml or entities

    def _get_xmldoc_info(self, xml_files):
        """
        Given a list of xml files, this finds either Book_Info.xml or Article_Info.xml
        and returns their content as a string.

        This could be improved by finding the actual <bookinfo> or <articleinfo> tag set,
        even if those files aren't available.
        """
        pub_types = "Book", "Article"
        for name in xml_files:
            if os.path.basename(name).endswith("Info.xml") and os.path.basename(name).startswith(pub_types): #&& if info_file is not set?
                info_file = name
        xml = open(info_file)
        xml_string = unicode(xml.read())
        xml.close()
        return xml_string

    def _substitute_entities(self, xml_string, entity_dict):
        """
        Give it an XML string that potentially contains entities, and a dictionary
        of entity values, and it interpolates the entity values and returns the
        processed string.
        """
        for item in entity_dict.keys():
            xml_string = re.sub('&%s;' % item, entity_dict[item], xml_string)
        return xml_string

    def _get_entity_dict(self, entity_filelist):
        """
        Takes a list of files containing entity declarations, and returns a dictionary
        containing entity names and their values.
        """

        def remove_member(l, member):
            if member in l:
                l.remove(member)
            return l
        entity_dict = {}
        for entity_file in entity_filelist:
            f = open(entity_file)
            e = unicode(f.read())
            e = re.sub('\n', ' ', e)
            for ee in remove_member(e.split('ENTITY'), '<!'):
                ll = remove_member(ee.split(), '<!')
                entity_name = ll[0]
                entity_value = re.sub('>$', '', ' '.join(ll[1:]))
                entity_dict[entity_name] = entity_value
        for item in entity_dict.keys():
            for value in entity_dict.keys():
                entity_dict[item] = re.sub('&%s;' % value, entity_dict[value], entity_dict[item])

        # add some known values
        entity_dict['nbsp'] = " "
        entity_dict['amp'] = "&"
        return entity_dict

    def _get_docbook_metadata(self, interpolated_xml_string, lang, extra_args):
        """
        Returns a dictionary of document metadata from an XML string.  This presumes some
        tags are present:

        - <title>
        - <abstract>
        - <keywordset>, with <keyword> children
        - <subjectset>, with a single <subjectterm> child

        Improperly prepared sources will probably fail.
        """
        docsoup = BeautifulSoup(interpolated_xml_string, "lxml")
        metadata = []
        output = {}
        # TODO: error handling for books without the desired attributes
        output['title'] = docsoup.title.string
        output['slug'] = slugify(docsoup.title.string)
        output['abstract'] = docsoup.subtitle.string
        output['taxonomy'] = docsoup.subjectterm.string
        output['source_type'] = 'docbook'
        output['lang'] = lang
        output['tags'] = []
        for tag in docsoup.keywordset.findChildren():
            output['tags'].append(tag.string)
        if extra_args:
            for key in extra_args.keys():
                output[key] = extra_args[key]
        metadata.append(output)
        return metadata

    def read_broker(self, target, lang, extra_args):
        """
        Given a list of paths, this does all the things and returns a
        python object with their metadata.
        """
        entity_filelist, xml_filelist = self._get_xml_filelists(target, lang)
        info = self._get_xmldoc_info(xml_filelist)
        entity_dict = self._get_entity_dict(entity_filelist)
        interpolated_xml = self._substitute_entities(info, entity_dict)
        meta = self._get_docbook_metadata(interpolated_xml, lang, extra_args)
        return meta


import numpy as np

def iou(gt, prior):
    """
    # Compute the Intersection over Union
    Note: function iou(a, b) used in match_priors
    :param a: bounding boxes, dim: (n_items, 4)
    :param b: bounding boxes, dim: (n_items, 4) or (1, 4) if b is a reference
    :return: iou value: dim: (n_item)
    """

    # Compute the top left and bottom right coordinates for intersection
    bottom_right = np.minimum(gt[:, 2:], prior[2:])
    top_left = np.maximum(gt[:, :2], prior[:2])

    # Compute the intersection
    intersection = np.clip((bottom_right - top_left), a_min=0.0, a_max=np.inf)
    intersection = np.prod(intersection, axis=1)

    # Compute union
    area_gt = np.prod((gt[:, 2:] - gt[:, :2]), axis=1)
    area_prior = np.prod((prior[2:] - prior[:2]))
    union = area_gt + area_prior - intersection

    # Compute the iou
    jaccard_sim = intersection / union

    return jaccard_sim

def expansion(img, sample_bboxes, sample_labels, mean_img):

    if np.random.randint(2):
        return img, sample_bboxes, sample_labels

    h, w, c = img.shape
    exp_ratio = np.random.uniform(1, 4.0)
    x = np.random.uniform(0, w * exp_ratio - w)
    y = np.random.uniform(0, h * exp_ratio - h)

    expand_img = np.zeros((int(h * exp_ratio),
                           int(w * exp_ratio), c),
                          dtype=img.dtype)
    expand_img[:, :, :] = mean_img
    expand_img[int(y):int(y + h), int(x):int(x + w)] = img

    sample_bboxes = sample_bboxes.copy()
    sample_bboxes[:, :2] += (int(x), int(y))
    sample_bboxes[:, 2:] += (int(x), int(y))

    return expand_img, sample_bboxes, sample_labels

# This will randomly crop the input image
def random_crop(img, sample_bboxes, sample_labels, max_itr=50, min_crop_ratio=0.3):
    while True:
        # Choose min jaccard similarity randomly
        sim_thresholds = np.array([0.1, 0.3, 0.5, 0.7, 0.9])
        min_sim = np.random.choice(sim_thresholds)
        # if min_sim is None:
            # return img, sample_bboxes, sample_labels

        h, w, _ = img.shape
        for i in range(max_itr):
            # choose width and height for the crop
            length = np.minimum(h, w)
            if np.random.randint(2):
                min_crop_ratio = 1
            crop_length = np.random.uniform(min_crop_ratio * length, length)

            # h_crop = np.random.uniform(0.3 * h, h)
            # if (w_crop / h_crop < 0.5) or (w_crop / h_crop > 2):
                # continue

            # Find the coordinates of the crop
            x = np.random.uniform(w-crop_length)
            y = np.random.uniform(h-crop_length)
            crop_coord = np.asarray([x, y, x+crop_length, y+crop_length])

            # Check if the jaccard similarity meets minimum requirement
            # pos_labels = sample_labels > 0
            # jaccard_sim = iou(sample_bboxes[pos_labels, :], crop_coord)
            # mask = jaccard_sim.min() > min_sim
            # if not mask.any():
                # continue
            # Crop the image
            new_img = img[int(y):int(y+crop_length), int(x):int(x+crop_length), :]

            # Compute the center of the objects
            pos_labels = sample_labels > 0
            centers = (sample_bboxes[:, :2] + sample_bboxes[:, 2:]) / 2.0

            # Take the ground truth bounding box if the center is in the
            # cropped image
            I1 = (centers[:, 0] > crop_coord[0]) & (centers[:, 0] < crop_coord[2])
            I2 = (centers[:, 1] > crop_coord[1]) & (centers[:, 1] < crop_coord[3])
            mask = I1 * I2 * pos_labels
            if not mask.any():
                continue
            bboxes_masked = sample_bboxes[mask, :].copy()
            labels_masked = sample_labels[mask]

            # Take the overlap as the new ground truth bounding box
            bboxes_masked[:, :2] = np.maximum(bboxes_masked[:, :2], crop_coord[:2])
            bboxes_masked[:, 2:] = np.minimum(bboxes_masked[:, 2:], crop_coord[2:])

            # Adjust the ground truth coordinates wrt to the cropped image
            bboxes_masked[:, :2] -= crop_coord[:2]
            bboxes_masked[:, 2:] -= crop_coord[:2]
            # print(min_sim)

            return new_img, bboxes_masked, labels_masked