from msilib.schema import ComboBox
from urllib.parse import ParseResultBytes
import requests as r
import math
import os
from tkinter import *
from tkinter import ttk
from tkinter import messagebox
from tkinter import filedialog
import threading
from moviepy.editor import *
import webbrowser
headers = {
    "user-agent":"BDownload.exe v1.0",
    "referer":"https://www.bilibili.com",
    }
def BvToAv(Bv):
    BvNo1 = Bv[2:]
    keys = {
        '1':'13', '2':'12', '3':'46', '4':'31', '5':'43', '6':'18', '7':'40', '8':'28', '9':'5',
        'A':'54', 'B':'20', 'C':'15', 'D':'8', 'E':'39', 'F':'57', 'G':'45', 'H':'36', 'J':'38', 'K':'51', 'L':'42', 'M':'49', 'N':'52', 'P':'53', 'Q':'7', 'R':'4', 'S':'9', 'T':'50', 'U':'10', 'V':'44', 'W':'34', 'X':'6', 'Y':'25', 'Z':'1',
        'a': '26', 'b': '29', 'c': '56', 'd': '3', 'e': '24', 'f': '0', 'g': '47', 'h': '27', 'i': '22', 'j': '41', 'k': '16', 'm': '11', 'n': '37', 'o': '2',
        'p': '35', 'q': '21', 'r': '17', 's': '33', 't': '30', 'u': '48', 'v': '23', 'w': '55', 'x': '32', 'y': '14','z':'19'
    }
    BvNo2 = []
    for index, ch in enumerate(BvNo1):
        BvNo2.append(int(str(keys[ch])))
    BvNo2[0] = int(BvNo2[0] * math.pow(58, 6));
    BvNo2[1] = int(BvNo2[1] * math.pow(58, 2));
    BvNo2[2] = int(BvNo2[2] * math.pow(58, 4));
    BvNo2[3] = int(BvNo2[3] * math.pow(58, 8));
    BvNo2[4] = int(BvNo2[4] * math.pow(58, 5));
    BvNo2[5] = int(BvNo2[5] * math.pow(58, 9));
    BvNo2[6] = int(BvNo2[6] * math.pow(58, 3));
    BvNo2[7] = int(BvNo2[7] * math.pow(58, 7));
    BvNo2[8] = int(BvNo2[8] * math.pow(58, 1));
    BvNo2[9] = int(BvNo2[9] * math.pow(58, 0));
    sum = 0
    for i in BvNo2:
        sum += i
    sum -= 100618342136696320
    temp = 177451812

    return str(sum ^ temp)

def AvToCid(Av):
    try:
       r.get("https://www.bilibili.com",headers=headers)
    except:
        cid_list = "connection error"
    else:
        url = "https://api.bilibili.com/x/player/pagelist?aid="+Av+"&jsonp=jsonp"
        text = r.get(url,headers=headers).text
        list_ = eval(text)
        try:
            data_list = list_["data"]
            cid_list = []
        except:
            cid_list = "cannot get value"
        else:
            for i in data_list:
                cid = i["cid"]
                cid_list.append(str(cid))
    return cid_list

def download(Av,Cid,Dpi,Paths,P,L):
    try:
       r.get("https://www.bilibili.com",headers=headers)
    except:
        return_ = "connection error"
    else:
        url = "https://api.bilibili.com/x/player/playurl?fnval=80&avid="+Av+"&cid="+Cid
        text = r.get(url,headers=headers).text
        print(url)
        null = None
        dic = eval(text)
        data = dic['data']
        #print(data)
        dash = data['dash']
        video = dash['video']
        #print(video)
        for item in video:
            if item["id"] == Dpi:# and item["codecid"] == 7:
                base_url = item['base_url']
                print(base_url)
            if item["id"] != Dpi:
                pass
        audio = dash['audio']
        for item in audio:
            if item["id"] == 30232:
                base_url2 = item['base_url']

        exists = os.path.exists(Paths)
        try:
            print(base_url)
        except:
            return_ = "Dpi isnt exists"
        else:
            if exists:
                video_content = r.get(base_url,headers=headers).content
                audio_content = r.get(base_url2,headers=headers).content
                P = str(P)
                try:
                    with open(Paths+"Av"+Av+"Part"+P+".mp4","wb+") as file1,open(Paths+"AV"+Av+"Part"+P+".mp3","wb+") as file2:
                        file1.write(video_content)
                        file2.write(audio_content)
                    ButtomLabel['text'] = '正在合成中……（'+str(int(P))+'/'+str(L)+'）请耐心等待。'
                    video_path = VideoFileClip(Paths+"Av"+Av+"Part"+P+".mp4")
                    audio_path = AudioFileClip(Paths+"AV"+Av+"Part"+P+".mp3")
                    video = video_path.set_audio(audio_path)
                    video.write_videofile(f''+Paths+"Downloaded_Av"+Av+"Part"+P+".mp4",audio_codec='aac')

                except PermissionError:
                    return_ = "permission error"
                else:
                    os.remove(Paths+"Av"+Av+"Part"+P+".mp4")
                    os.remove(Paths+"AV"+Av+"Part"+P+".mp3")
                    return_ = "successfully download"

            if exists == False:
                return_ = "paths error"
    return return_
pink = "#FB7299"
pink2 = "#C7425D"
root = Tk()
root.geometry("310x282")
root.title("BDownloder 必下载 v1.0")
Frame0 = Frame(root,bg="white")
Frame0.pack(fill=BOTH,expand=True)
Label1 = Label(root,text="视频Bv号：",bg="white")
Label1.place(x=20,y=15)
Entry1 = ttk.Entry(root,width=20)
Entry1.place(x=90,y=15)
def Question():
    messagebox.showinfo('说明','''关于Bv号的说明：
    例：https://www.bilibili.com/video/BV1GW411g7mc?spm_id_from=333.337.search-card.all.click
    其中www.bilibili.com/video/后BV**********就是BV号（*有十位）''')
def ChangeColor(event):
    Button0['image'] = Photo12
def ChangeColor2(event):
    Button0['image'] = Photo11
Photo11 = PhotoImage(file="Img/b1-1.gif")
Photo12 = PhotoImage(file="Img/b1-2.gif")
Button0 = Button(root,image=Photo11,bd=0,bg="white",command=Question)
Button0.place(x=250,y=13)
Button0.bind("<Enter>",ChangeColor)
Button0.bind("<Leave>",ChangeColor2)
Label2 = Label(root,text="下载路径：",bg="white",)
Label2.place(x=20,y=55)
Entry2 = ttk.Entry(root,width=20)
Entry2.place(x=90,y=55)
#Entry2.insert(END,"")
Entry2["state"] = "readonly"
def ViewFolder():
    #tf = True
    path = filedialog.askdirectory()
    if path == '':
        pass
    if path != '':
        Entry2['state'] = 'normal'
        Entry2.delete(0,END)
        if len(path) == 3:
            Entry2.insert(END,path)
        if len(path) != 3:
            Entry2.insert(END,path+'/')
        Entry2['state'] = 'readonly'
def ChangeColor11(event):
    Button1['image'] = Photo22
def ChangeColor12(event):
    Button1['image'] = Photo21
Photo21 = PhotoImage(file="Img/b2-1.gif")
Photo22 = PhotoImage(file="Img/b2-2.gif")
Button1 = Button(root,bd=0,image=Photo21,bg="white",command=ViewFolder)
Button1.place(x=250,y=53)
Button1.bind("<Enter>",ChangeColor11)
Button1.bind("<Leave>",ChangeColor12)
Label3 = Label(root,text="选择画质：",bg="white")
Label3.place(x=20,y=95)
Combobox0 = ttk.Combobox(root,width=18,values=("高清 1080P","高清 720P","清晰 480P","流畅 360P"),state="readonly")
Combobox0.place(x=90,y=95)
Combobox0.current(3)
def thread_it(func, *args):
    '''将函数打包进线程'''
    t = threading.Thread(target=func, args=args)
    t.setDaemon(True)
    t.start()
def none():
    messagebox.showerror('错误','正在下载中，请不要操作。')
def download_command():
    global success
    Button2['command'] = none
    Bv = Entry1.get()
    Path = Entry2.get()
    Dpi_ =Combobox0.get()
    if Dpi_ == "高清 1080P":
        Dpi = 80
    elif Dpi_ == "高清 720P":
        Dpi = 64
    elif Dpi_ == "清晰 480P":
        Dpi = 32
    elif Dpi_ == "流畅 360P":
        Dpi = 16
    try:
        Av = BvToAv(Bv)
    except:
        messagebox.showerror('错误','Bv号参数错误。')
    else:
        Cid = AvToCid(Av)
        if Cid =="connection error":
            messagebox.showerror('错误','连接错误。')
        elif Cid == "cannot get value":
            messagebox.showerror('错误','Bv号参数错误。')
        elif Cid != "connection error" and Cid != "cannot get value":
            j = 0
            len_ = len(Cid)
            ButtomLabel['text'] = '状态：正在下载中……（'+str(j+1)+'/'+str(len_)+'）'
            #messagebox.showinfo('提示','开始下载，请点击“确定”继续。')
            for i in Cid:
                j = j + 1
                download_ = download(Av,i,Dpi,Path,j,len_)
                if download_ == "Dpi isnt exists":
                    messagebox.showerror('错误','该视频不支持该清晰度。')
                    success = False
                    break
                elif download_ == "paths error":
                    messagebox.showerror('错误','该路径不存在。')
                    success = False
                    break
                elif download_ == "permission error":
                    messagebox.showerror('错误','本应用程序无权限访问此目录，请以管理员身份运行本应用。')
                    success = False
                    break
                elif download_ != "Dpi isnt exists" and download_ != "paths error" and download_ != "permission error":
                    if j == len_:
                        pass
                    else:
                        ButtomLabel['text'] = '状态：正在下载中……（'+str(j+1)+'/'+str(len_)+'）'
                        #messagebox.showinfo('提示','已下载'+str(j)+'/'+str(len_)+'，请点击“确定”继续。')
                    success = True
            if success == True:
                messagebox.showinfo('提示','下载成功。')
        ButtomLabel['text'] = '状态：等待中……'
        Entry1.delete(0,END)
    Button2['command'] = download_command
photo31 = PhotoImage(file="Img/b3-1.gif")
photo32 = PhotoImage(file="Img/b3-2.gif")
def ChangeColor21(event):
    Button2['image'] = photo32
def ChangeColor22(event):
    Button2['image'] = photo31
Button2 = Button(root,image=photo31,bg="white",bd=0,command=lambda :thread_it(download_command))
Button2.bind("<Enter>",ChangeColor21)
Button2.bind("<Leave>",ChangeColor22)
Button2.place(x=120,y=150)
grey_ = "#F5F5F5"
ButtomLabel = Label(root,text="    状态：等待中……", bd=0, relief=SUNKEN,anchor=W,height=2,bg=grey_)
ButtomLabel.pack(side=BOTTOM, fill=X)
about_ = '''【关于】
应用名称：BDownloder必下载.exe （bilibili站视频下载工具）
应用版本：v1.0
开发者：SbyYz-x00（伞兵营营长）
时间：2022.6
开发者邮箱：stoneyueryouyou@outlook.com
开发者QQ：1932972859

【用户许可协议】
在安装、使用本应用之前，为了保障您和他人的权利，请务必仔细阅读本《用户许可协议》。
1. 本应用为开源、免费软件，
（1） 开发者不接受任何形式对本应用的售卖；
（2） 源码已在Github公布，开发者谢绝任何反编译、反破解以及对应用的注入、篡改的行为；
（3） 欢迎交流，转载，版权归开发者所有，请标明来源。
2. 本应用的使用建立在遵守《中华人民共和国著作权法》的前提条件下，
（1） 请务必依照bilibili和作者对视频版权的说明使用以本应用下载的视频。未经作者允许，不得转载、篡改等操作；
（2） 本应用仅供交流娱乐，为了保护作者的著作权，请务必在24小时内删除本应用下载的视频；
（3） 严禁将由本应用下载的视频用于除娱乐以外的用途，尤其是盈利目的。
3. 本应用有权利对您的计算机作出更改。当涉及权限问题时，会自动请求管理员权限，当管理员同意后，才将继续操作。
如果同意本《用户许可协议》，请继续使用本应用；若不同意，请停止下载/将本应用从您的计算机中卸载。

【免责声明】
1. 本应用旨在倡导文明绿色上网，若用户将由本应用下载的视频用于非法用途，以及不当使用本应用导致的损失，均与开发者无关。
2. 若用户使用本应用下载的视频内容非法，与开发者无关。
3. 若有侵权，请事先协商解决。
开发者邮箱：stoneyueryouyou@outlook.com 开发者QQ：1932972859'''
photo41 = PhotoImage(file="Img/b4-1.gif")
photo42 = PhotoImage(file="Img/b4-2.gif")
def about():
    messagebox.showinfo("关于",about_)
def ChangeColor31(event):
    Button3['image'] = photo42
def ChangeColor32(event):
    Button3['image'] = photo41
Button3 = Button(root,image=photo41,bg=grey_,bd=0,command=about)
Button3.bind("<Enter>",ChangeColor31)
Button3.bind("<Leave>",ChangeColor32)
Button3.place(x=270,y=250)
def openweb():
    webbrowser.open("https://space.bilibili.com/1635778169",new=0,autoraise=True)
    webbrowser.open("https://github.com/SbyYz-x00",new=0,autoraise=True)
    webbrowser.open("https://blog.csdn.net/yueryouyou_sbyyz",new=0,autoraise=True)
def ChangeColor41(event):
    Button4['image'] = photo52
def ChangeColor42(event):
    Button4['image'] = photo51
photo51 = PhotoImage(file="Img/b5-1.gif")
photo52 = PhotoImage(file="Img/b5-2.gif")
Button4 = Button(root,image=photo51,bg=grey_,bd=0,command=openweb)
Button4.bind("<Enter>",ChangeColor41)
Button4.bind("<Leave>",ChangeColor42)
Button4.place(x=240,y=250)
root.mainloop()


# Importing numpy library for using exponential function.

import numpy as np

# Parameters for the logistic regression model is mentioned in the given problem.
para = {'B01': 0.1, 'B1': 0.5, 'B2': 0.5, 'B02': 1, 'B03': 0}

# User input for the data

data = {
    'X1': list(map(float, input("Enter values for X1 separated by space: ").split())),
    'X2': list(map(float, input("Enter values for X2 separated by space: ").split())),
    'Sero': list(map(float, input("Enter values for Sero separated by space: ").split()))}

# converting  the inputs into lists for ease
list1 = data['X1']
list2 = data['X2']
list3 = data['Sero']

# Function to calculate probabilities using the logistic function
def logistic_function(logitsv1, logitsv2, logitsv3):
    P1 = np.exp(logitsv1) / (np.exp(logitsv1) + np.exp(logitsv2) + np.exp(logitsv3))
    P2 = np.exp(logitsv2) / (np.exp(logitsv1) + np.exp(logitsv2) + np.exp(logitsv3))
    P3 = np.exp(logitsv3) / (np.exp(logitsv1) + np.exp(logitsv2) + np.exp(logitsv3))
    return P1, P2, P3

# Iterate through the lists and calculate probabilities
for y in range(len(list1)):
    try:
        x1 = list1[y]
        x2 = list2[y]
        x3 = list3[y]
    except IndexError:
        # Handle the expection when the index is out of range
        print(f"p{y + 1}")
        print("List index out of range. Skipping this iteration.")
        print()
        continue

    # Features and weights for logistic regression that are taken from the user
    features = [1, x1, x2]
    weights0 = [para['B01'], para['B1'], para['B2']]
    weights1 = [para['B02'], para['B1'], para['B2']]
    Sero = [1, x3, x3]  # Corrected Sero values
    weights2 = [para['B03'], para['B1'], para['B2']]

    # Calculate logits for each alternative
    logitsv1 = np.dot(features, weights0)
    logitsv2 = np.dot(features, weights1)
    # calculating the dot product using dot function of numpy
    logitsv3 = np.dot(Sero, weights2)

    # Calculate probabilities using the logistic function
    probabilities = logistic_function(logitsv1, logitsv2, logitsv3)

    # Print the results for each alternative
    print(f"P{y + 1} =")
    for idx, prob in enumerate(probabilities, start=1):
        print(f"  P{idx} = {round(prob, 4)}")
    print()


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 19 20:28:47 2022

@author: Willirin
"""

x = int (input ())

y = int (input ())

x = x% y

x = x% y

y = y% x

print (y)


import os
import time

def clear():
    os.system(["clear", "cls"][os.name == "nt"])

def art():
    print(f"""{fCiano}
           ,   ,
          /////|
         ///// |
        |~~~|  |
        |===|  |
        |   |  |  {fReset}theb3lial{fCiano}
        |   |  |
        |   | /
        |===|/
        '---'\n{fReset}""")

fPreto = '\033[2;30m'
fBranco = '\033[1;97m'
fVermelho = '\033[1;91m'
fAzul = '\033[1;34m'
fAmarelo = '\033[1;33m'
fVerde = '\033[1;92m'
fCiano = '\033[1;36m'
fReset = '\033[m'

file = open("path.txt", "r")
path = file.readlines()
path = list(map(lambda a: a.strip(), path))

linhas = 0
for x in path:
    linhas = linhas + 1

clear()
time.sleep(1)
art()

inurl = input(f"Nome do diretório {fCiano}‒ {fReset}")

with open("dork.txt", "w") as f:
    for x in range(0, linhas):
        f.write(f"{inurl}{path[x]}\n")

print("Salvo em dork.txt")


import httpx
import hashlib
import urllib.parse
from typing import Dict, Tuple, Union
import asyncio
from datetime import datetime

class EpusdtException(Exception):
    def __init__(self, message: str, *args: object) -> None:
        super().__init__(message, *args)
        self.message = message

class EpusdtSDK(object):
    def __init__(self, base_url: str, callback_url: str, sign_key: str) -> None:
        self.base_url = base_url
        self.callback_url = callback_url
        self.sign_key = sign_key

    def epusdtSign(self, parameter: Dict[str, str], signKey: str) -> str:
        parameter = dict(sorted(parameter.items()))
        sign = ""
        urls = ""
        for key, val in parameter.items():
            if val == "":
                continue
            if key != "signature":
                if sign != "":
                    sign += "&"
                    urls += "&"
                sign += f"{key}={val}"
                urls += f"{key}={urllib.parse.quote(str(val).encode())}"
        sign = hashlib.md5(f"{sign}{signKey}".encode()).hexdigest()
        return sign

    def handleAmount(self, amount: Union[str, float, int]) -> Union[int, float]:
        try:
            return int(amount)
        except:
            pass

        try:
            return float(amount)
        except:
            pass

        raise EpusdtException(f"{amount} 格式化错误!")

    async def createPay(
        self, amount: Union[str, float, int]
    ) -> Tuple[str, str, str]:
        """
        请求 EPusdt API 新建一个交易,返回交易的 trade_id:str actual_amount:str 实际交易的金额 token:str 交易的 token
        """
        data: dict = {
            "order_id": datetime.now().strftime("%Y%m%d%H%M%S%f")[:-3],
            "amount": self.handleAmount(amount),
            "notify_url": self.callback_url,
            "redirect_url": "",
        }
        sign = self.epusdtSign(parameter=data, signKey=self.sign_key)
        data.update({"signature": sign})
        async with httpx.AsyncClient() as client:
            try:
                result = await client.post(
                    url=f"{self.base_url}/api/v1/order/create-transaction",
                    json=data,
                )
            except Exception as exc:
                raise EpusdtException(message=f"HTTP Requests ERROR {exc}")

            try:
                jdata = result.json()
            except:
                raise EpusdtException(
                    f"json encode error:{result.text} url:{result.url}"
                )
            if jdata["status_code"] == 200:
                return (
                    jdata["data"]["trade_id"],
                    jdata["data"]["actual_amount"],
                    jdata["data"]["token"],
                )

            raise EpusdtException(f"{data} Response: {result.text}")

async def main():
    epusdtSDK = EpusdtSDK(
        base_url="http://192.168.8.1:8966",
        callback_url="http://192.168.8.219:8000/pay/callback",
        sign_key="lovehyy9420",
    )
    await epusdtSDK.createPay(amount=10.88)

if __name__ == "__main__":
    asyncio.run(main())


'''
bitmask O(n 2^n)
backtracking O(4^n)
dp[mask | (1<<i)] = (dp[mask] + nums[i]) % tar

'''

class Solution_bitmask_top_down:
    def canPartitionKSubsets(self, nums: List[int], k: int) -> bool:
        s, n = sum(nums), len(nums)
        if s < k or n < k or s % k != 0 :
            return False
        nums.sort(reverse=True)

        @lru_cache(None)
        def dfs(mask, target):
            # convert 1111 to 0000
            # print(bin(mask)[2:], target)
            if mask == 0:
                return target == 0 # to the end, judge target sum == 0
            if target == 0:
                return dfs(mask, s // k) # reset target for others
                # while target is 0 and mask is not 0, we found an equal subset.
                # But we need to keep doing this to find k equal subset.
            res = False
            for i in range(n):
                if mask & (1 << i):
                    if nums[i] > target:
                        continue
                    if dfs(mask ^ (1 << i), target - nums[i]):
                        res = True
                        break
            return res

        @lru_cache(None)
        def dfs1(mask, target):
            if target < 0:
                return False
            if mask == 0:
                return target == 0
            if target == 0:
                return dfs1(mask, s // k) # reset

            for i in range(n):
                if mask & (1 << i): # i is not taken
                    if dfs1(mask ^ (1 << i), target - nums[i]):
                        return True
            return False

        return dfs((1 << n) - 1, s // k)

class Solution_bitmask_bottom_up:
    def canPartitionKSubsets(self, nums: List[int], k: int) -> bool:
        # dp[mask|(1<<i)] = (dp[mask]+nums[i]) % target
        s, n = sum(nums), len(nums)
        if s < k or n < k or s % k != 0 :
            return False
        nums.sort(reverse=True)
        target = s // k

        full_mask = (1 << n) - 1
        dp = [0] + [-1] * full_mask
        for mask in range(1 << n):
            if dp[mask] == -1:
                continue
            for i in range(n):
                if not mask & (1<<i) and dp[mask] + nums[i] <= target:
                    dp[mask ^ (1<<i)] = (dp[mask] + nums[i]) % target
        return dp[(1<<n) - 1] == 0

class Solution_bitmask_short:
    def canPartitionKSubsets(self, nums: List[int], k: int) -> bool:
        target = sum(nums) // k

        @lru_cache(None)
        def dp(total,mask):
            if mask == 0: return total == 0
            return any(mask&(1<<i) and dp(total-nums[i],mask ^ (1<<i))
                            for i in range(len(nums))
                                if (total-1)%target >= nums[i]-1)

        return dp(sum(nums),(1<<len(nums))-1)

class Solution_bt(object):
    def canPartitionKSubsets(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: bool
        """
        s, n = sum(nums), len(nums)
        if s < k or n < k or s % k != 0 :
            return False
        nums.sort(reverse=True)
        target = [s / k] * k

        def dfs(target, pos):
            if pos == n:
                return True
            # seen = set() # pruning
            for i in range(k):
                if target[i] >= nums[pos]:
                # if target[i] >= nums[pos] and target[i] not in seen:
                    target[i] -= nums[pos]
                    if dfs(target, pos + 1):
                        return True
                    target[i] += nums[pos]
                    # seen.add(target[i]) # pruning

                    # If a single number couldn't fit into one bucket, it is a waste of time to put it into the other bucket.
                    if target[i] == s / k: break # another game changer pruning
            return False

        return dfs(target, 0)


# Script for distributed training with synchronized batch norm
# Guide for converting a regular training script to a distributed one can
# be found at https://github.com/dougsouza/pytorch-sync-batchnorm-example

import argparse
import datetime
import numbers
import pathlib
import warnings
import logging

import torch
import tqdm
import yaml
import numpy as np

import loss
import options.segment
import datasets.nrw
import datasets.dfc
import models.unet

logger = logging.getLogger(__name__)

def train_and_eval(
    model, optimizer, criterion, train_dataloader, test_dataloader, num_epochs, device
):

    pbar = tqdm.tqdm(total=num_epochs)
    for n_epoch in range(num_epochs):

        loss_train, iou_val_train = train(model, optimizer, criterion, train_dataloader, 'rgb', device)

        loss_test, iou_val_test = evaluate(
            model, criterion, test_dataloader, 'rgb', device
        )

        info_str = "epoch {:3d} train: loss={:6.3f} iou={:6.3f}, test loss={:6.3f} iou={:6.3f}".format(
            n_epoch, loss_train, loss_test, iou_val_train, iou_val_test
        )

        logger.info(info_str)

        pbar.update(1)
        pbar.set_description(info_str)
        pbar.write(info_str)
    return model

def evaluate(model, criterion, dataloader, src="rgb", device='cuda'):
    """ evaluates a model on the given dataset

    Parameters
    ----------

    model: (torch.nn.Module)
        the neural network
    optimizer: (torch.optim)
        optimizer for parameters of model
    criterion: unction
        takes batch_output and batch_labels and computes the loss for the batch
    dataloader: torch.utils.data.DataLoader
        fetches training data

    """

    model.eval()

    running_loss = 0.0
    iou_val = 0.0
    for sample in dataloader:
        sample = {k: v.to(device) for k, v in sample.items()}

        with torch.no_grad():
            output = model(sample[src])

        pred = (output > 0).long()

        loss_t = criterion(output, sample["seg"])
        running_loss += loss_t.item()
        iou_val += loss.iou(pred, sample["seg"]).item()

    running_loss /= len(dataloader)
    iou_val /= len(dataloader)

    return running_loss, iou_val

def train(model, optimizer, criterion, dataloader, src="rgb", device='cuda'):
    """ trains a model for one epoch

    Parameters
    ----------

    model: (torch.nn.Module)
        the neural network
    optimizer: (torch.optim)
        optimizer for parameters of model
    loss_fn: unction
        takes batch_output and batch_labels and computes the loss for the batch
    dataloader: torch.utils.data.DataLoader
        fetches training data

    """

    model.train()

    running_loss = 0.0
    iou_val = 0.0
    for sample in dataloader:
        optimizer.zero_grad()

        sample = {k: v.to(device) for k, v in sample.items()}

        output = model(sample[src])
        pred = (output > 0).long()

        loss_t = criterion(output, sample["seg"])

        loss_t.backward()
        running_loss += loss_t.item()
        iou_val += loss.iou(pred, sample["seg"]).item()
        optimizer.step()

    running_loss /= len(dataloader)
    iou_val /= len(dataloader)

    return running_loss, iou_val

if __name__ == "__main__":

    parser = options.segment.get_parser()
    args = parser.parse_args()

    # Reproducibilty config https://pytorch.org/docs/stable/notes/randomness.html
    if args.seed is not None:
        torch.manual_seed(args.seed)
        np.random.seed(args.seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
        warnings.warn(
            "You have chosen to seed training. "
            "This will turn on the CUDNN deterministic setting, "
            "which can slow down your training considerably! "
            "You may see unexpected behavior when restarting "
            "from checkpoints."
        )

    if len(args.crop) == 1:
        args.crop = args.crop[0]

    if len(args.resize) == 1:
        args.resize = args.resize[0]

    if torch.cuda.is_available():
        device = torch.device("cuda")
    else:
        raise RuntimeError("This scripts expects CUDA to be available")
    OUT_DIR = args.out_dir / options.segment.args2str(args)
    # All process make the directory.
    # This avoids errors when setting up logging later due to race conditions.
    OUT_DIR.mkdir(exist_ok=True)

    ###########
    #         #
    # Logging #
    #         #
    ###########

    logging.basicConfig(
        format="%(asctime)s [%(levelname)-8s] %(message)s",
        level=logging.INFO,
        filename=OUT_DIR / "log_training.txt",
    )
    logger = logging.getLogger()
    logger.info("Saving logs, configs and models to %s", OUT_DIR)

    CONFIG = options.segment.args2dict(args)
    with open(OUT_DIR / "config.yml", "w") as cfg_file:
        yaml.dump(CONFIG, cfg_file)

    #########################
    #                       #
    # Dataset configuration #
    #                       #
    #########################

    train_transforms, test_transforms = options.common.get_transforms(CONFIG)

    dataset_train = options.common.get_dataset(CONFIG, split='train', transforms=train_transforms)
    dataset_test = options.common.get_dataset(CONFIG, split='test', transforms=test_transforms)

    print("training dataset statistics")
    print(dataset_train)

    print("testing dataset statistics")
    print(dataset_test)

    dataloader_train = torch.utils.data.DataLoader(
        dataset_train,
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=args.num_workers,
    )

    dataloader_test = torch.utils.data.DataLoader(
        dataset_test,
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=args.num_workers,
    )

    ###############
    #             #
    # Model setup #
    #             #
    ###############

    criterion = torch.nn.BCEWithLogitsLoss()

    dset_class = getattr(datasets, CONFIG["dataset"]["name"])
    n_labels = dset_class.N_LABELS
    input_nc = dset_class.N_CHANNELS[CONFIG["dataset"]["input"]]
    model = models.unet.UNet(input_nc, n_labels).to(device)

    ############
    #          #
    # TRAINING #
    #          #
    ############

    optimizer = torch.optim.Adam(
        model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay,
    )

    train_and_eval(
        model,
        optimizer,
        criterion,
        dataloader_train,
        dataloader_test,
        num_epochs=args.epochs,
        device=device,
    )

    ##############
    #            #
    # save model #
    #            #
    ##############

    torch.save(model, OUT_DIR / "{}.pt".format(options.segment.args2str(args)))


from django.db import models
from django.contrib.auth.models import AbstractUser

# Create your models here.

class User(AbstractUser):
    is_rider = models.BooleanField('Rider', default=False)
    is_driver = models.BooleanField('Driver', default=False)

class RideHistory(models.Model):

    SHARE = 1
    MINI_GO = 2
    RIDE_TYPE = (
        (SHARE, 'share'),
        (MINI_GO, 'mini_go'),
    )
    driver = models.ForeignKey(User, related_name="driver", on_delete=models.CASCADE, blank=True, null=True)
    ride_type = models.IntegerField(default=MINI_GO, choices=RIDE_TYPE)
    active = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

class RideTrack(models.Model):

    REQUESTED = 1
    ACCEPTED = 2
    ONGOING = 3
    COMPLETED = 4
    REJECTED = 5
    RIDE_STATE = (
        (REQUESTED, 'requested'),
        (ACCEPTED, 'accepted'),
        (ONGOING, 'ongoing'),
        (COMPLETED, 'completed'),
        (REJECTED, 'rejected'),
    )
    ride_state = models.IntegerField(default=0, choices=RIDE_STATE)
    history = models.ForeignKey(RideHistory, related_name="history", on_delete=models.CASCADE, blank=True, null=True)
    rider = models.ForeignKey(User, related_name="rider", on_delete=models.CASCADE, blank=True, null=True)
    seats = models.IntegerField(default=0, blank=True, null=True)
    created_at = models.DateTimeField(auto_now_add=True)


import socket
import random
import re
import logging
"""
class Sink(object):
    def write(self, *args, **kwargs):
        pass
    def writeln(self, *args, **kwargs):
        pass

class Base(object):
    def __init__(self, **kwargs):
        super(Base,self).__init__()
        if "logTo" in kwargs:
            if kwargs["logTo"] == None:
                self.log = Sink()
                self.errlog = Sink()
            else:
                self.log = kwargs["logTo"]
                self.errlog = kwargs["logTo"]
        else:
            self.log = sys.stdout
            self.errlog = sys.stderr
"""
def wsUrl_str(t):
    if t.startswith("ws://"):
        return t
    else:
        raise ValueError("Not a ws:// format url")

def udpUrl_str(t):
    m = re.match("^udp://([^:/]+):([0-9]+)$",t)
    if m:
        return iphost_str(m.group(1)), int(m.group(2))
    else:
        raise ValueError("Not a udp://<host>:<port> url")

def iphost_str(t):
    """Pass through a string provide it represents an ip address, or convert from a host name to string representation of ip address
    otherwise raise an exception if not possible"""
    try:
        socket.inet_aton(t)
        return t
    except socket.error:
        try:
            return socket.gethostbyname(t)
        except:
            raise ValueError("Not a recognised/resolvable host name or is an IP address string not of the form nnn.nnn.nnn.nnn")

def ipaddr_str(t):
    """Pass through a string provided it represents an ip address (not a host name) otherwise raise an exception"""
    try:
        socket.inet_aton(t)
    except socket.error:
        raise ValueError("IP address string not of the form nnn.nnn.nnn.nnn")
    return t

def port_int(p):
    """Pass through a port number (as a number or a string) provided it is valid and in range, otherwise raise an exception"""
    try:
        port=int(p)
    except:
        raise ValueError("Invalid port number")

    if port>=0 and port<=65535:
        return port
    else:
        raise ValueError("Port number out of range")

def port_int_or_random(p):
    """Same as port_int() except will also accept "random" in which case a random port between 10000 and 20000 is chosen"""
    if p=="random":
        return random.randrange(10000,20000)
    else:
        return port_int(p)

def parse_logLevel(levelName):
    levelName=levelName.lower()
    for n in range(logging.NOTSET,logging.CRITICAL+1):
        name=logging.getLevelName(n).lower()
        if levelName == name:
            return n
    raise ValueError("Logging level name not recognised")


import re

from gi.repository import Gdk
from gi.repository import Gtk
from gi.repository import GES
from gi.repository import Gio
from gi.repository import GObject

from gettext import gettext as _

from pitivi.timeline import elements
from pitivi.utils.loggable import Loggable
from pitivi.utils import ui
from pitivi.utils import timeline as timelineUtils

class BaseLayerControl(Gtk.Box, Loggable):

    """
    Base Layer control classes
    """

    __gtype_name__ = 'LayerControl'

    def __init__(self, layer, app, type_name):
        Gtk.Box.__init__(self, spacing=0)
        Loggable.__init__(self)

        self._app = app
        self.layer = layer
        self._selected = False
        self.__type_name = type_name
        self.__meta_name = type_name + "::name"
        self.props.no_show_all = True

        context = self.get_style_context()

        # get the default color for the current theme
        self.UNSELECTED_COLOR = context.get_background_color(
            Gtk.StateFlags.NORMAL)
        # use base instead of bg colors so that we get the lighter color
        # that is used for list items in TreeView.
        self.SELECTED_COLOR = context.get_background_color(
            Gtk.StateFlags.SELECTED)

        self.set_orientation(Gtk.Orientation.VERTICAL)

        table = Gtk.Table(n_rows=2, n_columns=2)
        table.set_border_width(ui.PADDING)
        table.set_row_spacings(3)
        table.set_col_spacings(3)

        self.pack_start(table, True, True, 0)

        self.name_entry = Gtk.Entry()
        self.name_entry.set_tooltip_text(
            _("Set a personalized name for this layer"))
        self.name_entry.set_property("secondary-icon-name", self._getIconName())
        self.name_entry.connect("key-press-event", self._keyPressCb)

        layer.bLayer.connect("notify::priority", self.__layerPriorityChangedCb)
        self.__resetLayerName()
        table.attach(self.name_entry, 0, 2, 0, 2)

        self.show_all()

    def __del__(self):
        self.layer.bLayer.disconnect_by_func(self.__layerPriorityChangedCb)

    def __layerPriorityChangedCb(self, bLayer, pspec):
        self.__resetLayerName()

    def __nameIsDefault(self, name):
        if name is None:
            return False

        return re.findall("%s [0-9]+$" % self.__type_name, name)

    def __resetLayerName(self):
        name = self.layer.bLayer.get_meta(self.__meta_name)
        if name is None or self.__nameIsDefault(name):
            name = '%s %d' % (self.__type_name, self.layer.bLayer.get_priority())
            self.layer.bLayer.set_meta(self.__meta_name, name)
        self.name_entry.set_text(name)

    def _getIconName(self):
        return None

    def getSelected(self):
        return self._selected

    def setSelected(self, selected):
        if selected != self._selected:
            self._selected = selected
            self._selectionChangedCb()

    def force_show_all(self):
        self.props.no_show_all = False
        self.show_all()
        self.props.no_show_all = True

    selected = property(getSelected, setSelected, None, "Selection state")

    def _selectionChangedCb(self):
        """
        Called when the selection state changes
        """
        if self.selected:
            self.name_entry.override_background_color(
                Gtk.StateType.NORMAL, self.SELECTED_COLOR)
        else:
            self.name_entry.override_background_color(
                Gtk.StateType.NORMAL, self.UNSELECTED_COLOR)

        # continue GTK signal propagation
        return True

    def _keyPressCb(self, unused_widget, event):
        self.layer.bLayer.set_meta(self.__meta_name, self.name_entry.get_text())
        self._app.project_manager.current_project.setModificationState(True)

    def getHeight(self):
        return self.get_allocation().height

    def getSeparatorHeight(self):
        return self.sep.get_allocation().height

    def getControlHeight(self):
        return self.getHeight() - self.getSeparatorHeight()

    def setSoloState(self, state):
        self.solo_button.set_active(state)

    def setSeparatorVisibility(self, visible):
        if visible:
            self.sep.show()
        else:
            self.sep.hide()

class VideoLayerControl(BaseLayerControl):
    """
    Layer control class for video layers
    """

    __gtype_name__ = 'VideoLayerControl'

    def __init__(self, layer, app):
        BaseLayerControl.__init__(self, layer, app, "video")

    def _getIconName(self):
        return "video-x-generic"

class AudioLayerControl(BaseLayerControl):
    """
    Layer control class for audio layers
    """

    __gtype_name__ = 'AudioLayerControl'

    def __init__(self, layer, app):
        BaseLayerControl.__init__(self, layer, app, "audio")

    def _getIconName(self):
        return "audio-x-generic"

class TwoStateButton(Gtk.Button):
    """
    Button with two states and according labels/images
    """

    __gsignals__ = {
        "changed-state": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_PYOBJECT,),)
    }

    def __init__(self, state1="", state2=""):
        Gtk.Button.__init__(self)
        self.set_relief(Gtk.ReliefStyle.NONE)
        self.connect("clicked", self._clickedCb)

        self.set_states(state1, state2)
        self._state = True

        self.set_label(self.states[self._state])

    def set_states(self, state1, state2):
        self.states = {True: state1, False: state2}

    def _clickedCb(self, unused_widget):
        self._state = not self._state

        self.set_label(self.states[self._state])
        self.emit("changed-state", self._state)

class SpacedSeparator(Gtk.EventBox):
    """
    A Separator with vertical spacing

    Inherits from EventBox since we want to change background color
    """

    def __init__(self, position):
        Gtk.EventBox.__init__(self)

        self.box = Gtk.Box()
        self.box.set_orientation(Gtk.Orientation.VERTICAL)
        self.add(self.box)
        self.__position = position

        self.get_style_context().add_class("SpacedSeparator")
        self.box.get_style_context().add_class("SpacedSeparator")
        self.props.height_request = 1
        self.props.margin_bottom = 4
        self.props.margin_top = 4

    def do_state_flags_changed(self, old_flags):
        HIGLIGHTED_PADDING = 3
        total_height = ui.PADDING + HIGLIGHTED_PADDING
        if not self.get_state_flags() & Gtk.StateFlags.PRELIGHT:
            self.props.height_request = 1
            self.props.margin_bottom = (total_height - 1) / 2
            self.props.margin_top = (total_height - 1) / 2
        else:
            self.props.height_request = ui.PADDING
            if self.__position == Gtk.PositionType.TOP:
                self.props.margin_bottom = HIGLIGHTED_PADDING
                self.props.margin_top = 0
            else:
                self.props.margin_bottom = 0
                self.props.margin_top = HIGLIGHTED_PADDING
        Gtk.EventBox.do_state_flags_changed(self, old_flags)

class LayerControls(Gtk.EventBox, Loggable):

    __gtype_name__ = 'PitiviLayerControls'

    def __init__(self, bLayer, app):
        Gtk.EventBox.__init__(self)
        Loggable.__init__(self)

        self.bLayer = bLayer
        self.bTimeline = self.bLayer.get_timeline()
        self.app = app

        self.props.width_request = ui.CONTROL_WIDTH
        # Half the height because we display only the video strip when empty.
        self.props.height_request = ui.LAYER_HEIGHT / 2
        self.props.hexpand = True

        content = Gtk.Grid()
        self.add(content)

        self.before_sep = SpacedSeparator(Gtk.PositionType.TOP)
        content.attach(self.before_sep, 0, 0, 2, 1)

        self.video_control = VideoLayerControl(self, self.app)
        self.video_control.force_show_all()
        self.video_control.props.height_request = ui.LAYER_HEIGHT / 2
        self.video_control.props.hexpand = True
        content.attach(self.video_control, 0, 1, 1, 1)

        self.audio_control = AudioLayerControl(self, self.app)
        self.audio_control.hide()
        self.audio_control.props.height_request = ui.LAYER_HEIGHT / 2
        self.audio_control.props.hexpand = True
        content.attach(self.audio_control, 0, 2, 1, 1)

        menubutton = Gtk.MenuButton.new()
        menubutton.props.valign = Gtk.Align.START
        menubutton.props.margin_top = 3 * ui.PADDING
        menubutton.props.relief = Gtk.ReliefStyle.NONE
        menubutton.props.direction = Gtk.ArrowType.RIGHT
        menubutton.props.margin_end = ui.PADDING
        model, action_group = self.__createMenuModel()
        popover = Gtk.Popover.new_from_model(menubutton, model)
        popover.insert_action_group("layer", action_group)
        popover.props.position = Gtk.PositionType.LEFT
        menubutton.set_popover(popover)
        content.attach(menubutton, 1, 1, 1, 2)

        self.after_sep = SpacedSeparator(Gtk.PositionType.BOTTOM)
        content.attach(self.after_sep, 0, 3, 2, 1)

        sep = Gtk.Separator.new(Gtk.Orientation.VERTICAL)
        sep.props.margin_top = ui.PADDING / 2
        sep.props.margin_bottom = ui.PADDING / 2
        content.attach(sep, 2, 0, 1, 4)

        self.bLayer.connect("notify::priority", self.__layerPriorityChangedCb)
        self.bTimeline.connect("layer-added", self.__timelineLayerAddedCb)
        self.bTimeline.connect("layer-removed", self.__timelineLayerRemovedCb)
        self.__updateActions()

        # When the window property is set, specify the mouse cursor.
        self.connect("notify::window", self.__windowSetCb)

    def __windowSetCb(self, unused_window, unused_pspec):
        self.props.window.set_cursor(Gdk.Cursor.new(Gdk.CursorType.HAND1))

    def __del__(self):
        self.bLayer.disconnect_by_func(self.__layerPriorityChangedCb)
        self.bTimeline.disconnect_by_func(self.__timelineLayerAddedCb)
        self.bTimeline.disconnect_by_func(self.__timelineLayerRemovedCb)
        super(LayerControls, self).__del__()

    def __layerPriorityChangedCb(self, unused_bLayer, unused_pspec):
        self.__updateActions()

    def __timelineLayerAddedCb(self, unused_timeline, unused_bLayer):
        self.__updateActions()

    def __timelineLayerRemovedCb(self, unused_timeline, unused_bLayer):
        self.__updateActions()

    def __updateActions(self):
        priority = self.bLayer.get_priority()
        first = priority == 0
        self.__move_layer_up_action.props.enabled = not first
        self.__move_layer_top_action.props.enabled = not first
        layers_count = len(self.bTimeline.get_layers())
        last = priority == layers_count - 1
        self.__move_layer_down_action.props.enabled = not last
        self.__move_layer_bottom_action.props.enabled = not last
        self.__delete_layer_action.props.enabled = layers_count > 1

    def __createMenuModel(self):
        action_group = Gio.SimpleActionGroup()
        menu_model = Gio.Menu()

        self.__move_layer_top_action = Gio.SimpleAction.new("move_layer_to_top", None)
        action = self.__move_layer_top_action
        action.connect("activate", self._moveLayerCb, -2)
        action_group.insert(action)
        menu_model.append(_("Move layer to top"), "layer.%s" % action.get_name().replace(" ", "."))

        self.__move_layer_up_action = Gio.SimpleAction.new("move_layer_up", None)
        action = self.__move_layer_up_action
        action.connect("activate", self._moveLayerCb, -1)
        action_group.insert(action)
        menu_model.append(_("Move layer up"), "layer.%s" % action.get_name().replace(" ", "."))

        self.__move_layer_down_action = Gio.SimpleAction.new("move_layer_down", None)
        action = self.__move_layer_down_action
        action.connect("activate", self._moveLayerCb, 1)
        action_group.insert(action)
        menu_model.append(_("Move layer down"), "layer.%s" % action.get_name().replace(" ", "."))

        self.__move_layer_bottom_action = Gio.SimpleAction.new("move_layer_to_bottom", None)
        action = self.__move_layer_bottom_action
        action.connect("activate", self._moveLayerCb, 2)
        action_group.insert(action)
        menu_model.append(_("Move layer to bottom"), "layer.%s" % action.get_name().replace(" ", "."))

        self.__delete_layer_action = Gio.SimpleAction.new("delete_layer", None)
        action = self.__delete_layer_action
        action.connect("activate", self._deleteLayerCb)
        action_group.insert(action)
        menu_model.append(_("Delete layer"), "layer.%s" % action.get_name())

        return menu_model, action_group

    def _deleteLayerCb(self, unused_action, unused_parametter):
        self.app.action_log.begin("delete layer")
        bLayer = self.bLayer
        bTimeline = bLayer.get_timeline()
        bTimeline.remove_layer(bLayer)
        bTimeline.get_asset().pipeline.commit_timeline()
        self.app.action_log.commit()

    def _moveLayerCb(self, unused_simple_action, unused_parametter, step):
        index = self.bLayer.get_priority()
        if abs(step) == 1:
            index += step
        elif step == -2:
            index = 0
        else:
            index = len(self.bLayer.get_timeline().get_layers()) - 1
            # if audio, set last position
        self.bLayer.get_timeline().ui.moveLayer(self.bLayer, index)
        self.app.project_manager.current_project.pipeline.commit_timeline()

class LayerLayout(Gtk.Layout, Loggable):
    """
    A GtkLayout that exclusivly container Clips.
    This allows us to properly handle the z order of
    """
    __gtype_name__ = "PitiviLayerLayout"

    def __init__(self, timeline):
        super(LayerLayout, self).__init__()
        Loggable.__init__(self)

        self._children = []
        self._changed = False
        self.timeline = timeline

        self.props.hexpand = True
        self.get_style_context().add_class("LayerLayout")

    def do_add(self, widget):
        self._children.append(widget)
        self._children.sort(key=lambda clip: clip.z_order)
        Gtk.Layout.do_add(self, widget)
        self._changed = True

        for child in self._children:
            if isinstance(child, elements.TransitionClip):
                window = child.get_window()
                if window is not None:
                    window.raise_()

    def do_remove(self, widget):
        self._children.remove(widget)
        self._changed = True
        Gtk.Layout.do_remove(self, widget)

    def put(self, child, x, y):
        self._children.append(child)
        self._children.sort(key=lambda clip: clip.z_order)
        Gtk.Layout.put(self, child, x, y)
        self._changed = True

    def do_draw(self, cr):
        if self._changed:
            self._children.sort(key=lambda clip: clip.z_order)
            for child in self._children:

                if isinstance(child, elements.TransitionClip):
                    window = child.get_window()
                    window.raise_()
            self._changed = False

        self.props.width = max(self.timeline.layout.get_allocation().width,
                               timelineUtils.Zoomable.nsToPixel(self.timeline.bTimeline.props.duration))
        self.props.width_request = max(self.timeline.layout.get_allocation().width,
                                       timelineUtils.Zoomable.nsToPixel(self.timeline.bTimeline.props.duration))

        for child in self._children:
            self.propagate_draw(child, cr)

class Layer(Gtk.EventBox, timelineUtils.Zoomable, Loggable):

    __gtype_name__ = "PitiviLayer"

    __gsignals__ = {
        "remove-me": (GObject.SignalFlags.RUN_LAST, None, (),)
    }

    def __init__(self, bLayer, timeline):
        super(Layer, self).__init__()
        Loggable.__init__(self)

        self.bLayer = bLayer
        self.bLayer.ui = self
        self.timeline = timeline
        self.app = timeline.app

        self.bLayer.connect("clip-added", self._clipAddedCb)
        self.bLayer.connect("clip-removed", self._clipRemovedCb)

        # FIXME Make the layer height user setable with 'Paned'
        self.props.height_request = ui.LAYER_HEIGHT / 2
        self.props.valign = Gtk.Align.START

        self._layout = LayerLayout(self.timeline)
        self._layout.connect("remove", self.__childWidgetRemovedCb)
        self.add(self._layout)

        self.media_types = GES.TrackType(0)
        for clip in bLayer.get_clips():
            self._addClip(clip)

        self.before_sep = SpacedSeparator(Gtk.PositionType.TOP)
        self.after_sep = SpacedSeparator(Gtk.PositionType.BOTTOM)

    def release(self):
        for clip in self.bLayer.get_clips():
            self._removeClip(clip)
        self.bLayer.disconnect_by_func(self._clipAddedCb)
        self.bLayer.disconnect_by_func(self._clipRemovedCb)

    def checkMediaTypes(self):
        if self.timeline.editing_context:
            self.info("Not updating media types as"
                      " we are editing the timeline")
            return
        old_media_types = self.media_types
        self.media_types = GES.TrackType(0)
        bClips = self.bLayer.get_clips()

        """
        FIXME: That produces segfault in GES/GSequence
        if not bClips:
            self.emit("remove-me")
            return
        """

        for bClip in bClips:
            for child in bClip.get_children(False):
                self.media_types |= child.get_track().props.track_type
                if self.media_types == (GES.TrackType.AUDIO | GES.TrackType.VIDEO):
                    break

        if not (self.media_types & GES.TrackType.AUDIO) and not (self.media_types & GES.TrackType.VIDEO):
            # An empty layer only shows the video strip.
            self.media_types = GES.TrackType.VIDEO

        height = 0
        if self.media_types & GES.TrackType.AUDIO:
            height += ui.LAYER_HEIGHT / 2
            self.bLayer.control_ui.audio_control.force_show_all()
        else:
            self.bLayer.control_ui.audio_control.hide()

        if self.media_types & GES.TrackType.VIDEO:
            self.bLayer.control_ui.video_control.force_show_all()
            height += ui.LAYER_HEIGHT / 2
        else:
            self.bLayer.control_ui.video_control.hide()

        self.props.height_request = height
        self.bLayer.control_ui.props.height_request = height

        if old_media_types != self.media_types:
            self.updatePosition()

    def move(self, child, x, y):
        self._layout.move(child, x, y)

    def _childAddedCb(self, bClip, child):
        self.checkMediaTypes()

    def _childRemovedCb(self, bClip, child):
        self.checkMediaTypes()

    def _clipAddedCb(self, layer, bClip):
        self._addClip(bClip)

    def _addClip(self, bClip):
        ui_type = elements.GES_TYPE_UI_TYPE.get(bClip.__gtype__, None)
        if ui_type is None:
            self.error("Implement UI for type %s?" % bClip.__gtype__)
            return

        if not hasattr(bClip, "ui") or bClip.ui is None:
            clip = ui_type(self, bClip)
        else:
            clip = bClip.ui

        self._layout.put(clip, self.nsToPixel(bClip.props.start), 0)
        self.show_all()
        bClip.connect_after("child-added", self._childAddedCb)
        bClip.connect_after("child-removed", self._childRemovedCb)
        self.checkMediaTypes()

    def _clipRemovedCb(self, bLayer, bClip):
        self._removeClip(bClip)

    def _removeClip(self, bClip):
        if not bClip.ui:
            return

        ui_type = elements.GES_TYPE_UI_TYPE.get(bClip.__gtype__, None)
        if ui_type is None:
            self.error("Implement UI for type %s?" % bClip.__gtype__)
            return

        bClip.ui.release()
        self._layout.remove(bClip.ui)

    def __childWidgetRemovedCb(self, layout, clip):
        bClip = clip.bClip
        bClip.ui.layer = None
        if self.timeline.draggingElement is None:
            bClip.ui.release()
            bClip.ui = None

        bClip.disconnect_by_func(self._childAddedCb)
        bClip.disconnect_by_func(self._childRemovedCb)
        self.checkMediaTypes()

    def updatePosition(self):
        for bClip in self.bLayer.get_clips():
            if hasattr(bClip, "ui"):
                bClip.ui.updatePosition()

    def do_draw(self, cr):
        Gtk.Box.do_draw(self, cr)


import azure.functions as func
import datetime
import json
import base64
import hashlib
import hmac
import requests
import re
import os
import logging
from .state_manager import StateManager

token = os.environ['SentinelOneAPIToken']
domain = os.environ['SentinelOneUrl']
table_name = "SentinelOne"
chunksize = 10000
customer_id = os.environ['WorkspaceID']
shared_key = os.environ['WorkspaceKey']
connection_string = os.environ['AzureWebJobsStorage']
logAnalyticsUri = os.environ.get('logAnalyticsUri')

if ((logAnalyticsUri in (None, '') or str(logAnalyticsUri).isspace())):
    logAnalyticsUri = 'https://' + customer_id + '.ods.opinsights.azure.com'

pattern = r'https:\/\/([\w\-]+)\.ods\.opinsights\.azure.([a-zA-Z\.]+)$'
match = re.match(pattern,str(logAnalyticsUri))
if(not match):
    raise Exception("SentinelOne Data Connector: Invalid Log Analytics Uri.")

class SOne():

    def __init__(self):
        self.domain = domain
        self.header = {
            'Authorization': 'ApiToken {}'.format(token),
            'Content-Type': 'application/json',
            }
        self.from_date, self.to_date = self.generate_date()
        self.results_array = []

    def generate_date(self):
        current_time = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
        state = StateManager(connection_string)
        past_time = state.get()
        if past_time is not None:
            logging.info("The last time point is: {}".format(past_time))
        else:
            logging.info("There is no last time point, trying to get events for last day.")
            past_time = (current_time - datetime.timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
        state.post(current_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))
        return (past_time, current_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))

    def get_report(self, report_type_suffix, report_type_name, params = None):
        try:
            r = requests.get(self.domain + report_type_suffix, headers = self.header, params = params)
            if r.status_code == 200:
                self.results_array_join(r.json(), report_type_name)
                next_page_token = (r.json().get('pagination')).get('nextCursor')
                return next_page_token
            elif r.status_code == 400:
                logging.error("Invalid user input received. See error details for further information."
                      " Error code: {}".format(r.status_code))
            elif r.status_code == 401:
                logging.error("Unauthorized access - please sign in and retry. Error code: {}".format(r.status_code))
            else:
                logging.error("Something wrong. Error code: {}".format(r.status_code))
        except Exception as err:
            logging.error("Something wrong. Exception error text: {}".format(err))

    def results_array_join(self, result_element, api_req_name):
        for element in result_element['data']:
            element['event_name'] = api_req_name
            if "data" in element:
                element["DataFields"] = str(json.dumps(element['data']))
                del element['data']
            self.results_array.append(element)

    def reports_list(self):
        reports_api_requests_dict = \
            {
                "activities_created_events": {"api_req": "/web/api/v2.1/activities", "name": "Activities."},
                "agents_created_events": {"api_req": "/web/api/v2.1/agents", "name": "Agents."},
                "agents_updated_events": {"api_req": "/web/api/v2.1/agents", "name": "Agents."},
                "groups_updated_events": {"api_req": "/web/api/v2.1/groups", "name": "Groups."},
                "threats_created_events": {"api_req": "/web/api/v2.1/threats", "name": "Threats."},
                "threats_updated_events": {"api_req": "/web/api/v2.1/threats", "name": "Threats."},
                "alerts_created_events": {"api_req": "/web/api/v2.1/cloud-detection/alerts", "name": "Alerts."}
            }
        for api_req_id, api_req_info in reports_api_requests_dict.items():
            api_req = api_req_info['api_req']
            api_req_name = api_req_info['name']
            if "created_events" in api_req_id:
                api_req_params = {
                    "limit": 1000,
                    "createdAt__gt": self.from_date,
                    "createdAt__lt": self.to_date
                }
            elif "updated_events" in api_req_id:
                api_req_params = {
                    "limit": 200,
                    "updatedAt__gt": self.from_date,
                    "updatedAt__lt": self.to_date
                }
            logging.info("Getting report: {}".format(api_req_id))
            next_page_token = self.get_report(report_type_suffix = api_req, report_type_name = api_req_name, params = api_req_params)
            while next_page_token:
                api_req_params.update({"cursor": next_page_token})
                next_page_token = self.get_report(report_type_suffix=api_req, report_type_name=api_req_name,
                                                  params = api_req_params)

class Sentinel:

    def __init__(self):
        self.logAnalyticsUri = logAnalyticsUri
        self.success_processed = 0
        self.fail_processed = 0
        self.table_name = table_name
        self.chunksize = chunksize
    def gen_chunks_to_object(self, data, chunksize=100):
        chunk = []
        for index, line in enumerate(data):
            if (index % chunksize == 0 and index > 0):
                yield chunk
                del chunk[:]
            chunk.append(line)
        yield chunk

    def gen_chunks(self, data):
        for chunk in self.gen_chunks_to_object(data, chunksize=self.chunksize):
            obj_array = []
            for row in chunk:
                if row != None and row != '':
                    obj_array.append(row)
            body = json.dumps(obj_array)
            self.post_data(body, len(obj_array))

    def build_signature(self, date, content_length, method, content_type, resource):
        x_headers = 'x-ms-date:' + date
        string_to_hash = method + "\n" + str(content_length) + "\n" + content_type + "\n" + x_headers + "\n" + resource
        bytes_to_hash = bytes(string_to_hash, encoding="utf-8")
        decoded_key = base64.b64decode(shared_key)
        encoded_hash = base64.b64encode(
            hmac.new(decoded_key, bytes_to_hash, digestmod=hashlib.sha256).digest()).decode()
        authorization = "SharedKey {}:{}".format(customer_id, encoded_hash)
        return authorization

    def post_data(self, body, chunk_count):
        method = 'POST'
        content_type = 'application/json'
        resource = '/api/logs'
        rfc1123date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
        content_length = len(body)
        signature = self.build_signature(rfc1123date, content_length, method, content_type,
                                         resource)
        uri = self.logAnalyticsUri + resource + '?api-version=2016-04-01'
        headers = {
            'content-type': content_type,
            'Authorization': signature,
            'Log-Type': self.table_name,
            'x-ms-date': rfc1123date
        }
        response = requests.post(uri, data=body, headers=headers)
        if (response.status_code >= 200 and response.status_code <= 299):
            logging.info("Chunk was processed({} events)".format(chunk_count))
            self.success_processed = self.success_processed + chunk_count
        else:
            logging.info("Error during sending events to Azure Sentinel. Response code:{}".format(response.status_code))
            self.fail_processed = self.fail_processed + chunk_count

def main(mytimer: func.TimerRequest) -> None:
    utc_timestamp = datetime.datetime.utcnow().replace(
        tzinfo=datetime.timezone.utc).isoformat()
    if mytimer.past_due:
        logging.info("The timer is past due!")
    logging.info("Python timer trigger function ran at %s", utc_timestamp)
    logging.info("Starting program")
    SO = SOne()
    sentinel = Sentinel()
    SO.reports_list()
    SOne_class_vars = vars(SO)
    from_date, to_date = SOne_class_vars["from_date"], SOne_class_vars["to_date"]
    results_array = SOne_class_vars["results_array"]
    sentinel.gen_chunks(results_array)
    sentinel_class_vars = vars(sentinel)
    success_processed, fail_processed = sentinel_class_vars["success_processed"], \
                                        sentinel_class_vars["fail_processed"]
    logging.info("Total events processed successfully: {}, failed: {}. Period: {} - {}"
          .format(success_processed, fail_processed, from_date, to_date))


from common.python.storage.impl.clickhouse_storage import ClickHouseStorage
from common.python.utils.log_utils import schedule_logger
from flow.cycle_actions.flow_action_queue.worker.base_flow_action_worker import BaseFlowActionWorker

class SaveOutputDataWorker(BaseFlowActionWorker):
    """
    The processor when the save_output_data signal is received
    """

    def work(self, params):
        name = params.get('name')
        namespace = params.get('namespace')
        # fcs
        fcs_name = params.get('fcs_info').get('name')
        fcs_namespace = params.get('fcs_info').get('namespace')
        partitions = params.get('partitions')

        schedule_logger().info('schedule save data {}'.format(params))

        self.save_to_ck(name, namespace, fcs_name, fcs_namespace, partitions)

    @staticmethod
    def save_to_ck(name, namespace, fcs_name, fcs_namespace, partitions):
        """
            save fc data into clickHouse table
        Parameters
        ----------
        name: clickHouse table name
        namespace: clickHouse namespace
        fcs_name: fc table name
        fcs_namespace: fc namespace
        partitions: data is divided into several partitions on fc table

        Returns
        -------

        """
        from common.python.calculation.fc.fc_storage import FCStorage
        # get fcs object
        fcs = FCStorage(namespace=fcs_namespace, name=fcs_name, partitions=partitions)
        data = fcs.collect()
        ck_storage = ClickHouseStorage(_type=None, namespace=namespace, name=name)
        ck_storage.put_all(data)


import os
import torch
import loaddatas as lds
import torch.nn.functional as F
import numpy as np
from baselines import TLCGNN as TLCGNN
from sklearn.metrics import roc_auc_score,average_precision_score
from torch.nn.init import xavier_normal_ as xavier

def train():
    model.train()
    optimizer.zero_grad()
    emb = model.encode(data)
    x, y = model.decode(data, emb)
    loss = F.binary_cross_entropy(x,y)
    loss.backward()
    optimizer.step()
    return x

def test():
    model.eval()
    accs = []
    emb = model.encode(data)
    for type in ["val", "test"]:
        pred,y = model.decode(data,emb,type=type)
        pred,y = pred.cpu(),y.cpu()
        if type == "val":
            accs.append(F.binary_cross_entropy(pred, y))
            pred = pred.data.numpy()
            roc = roc_auc_score(y, pred)
            accs.append(roc)
            acc = average_precision_score(y,pred)
            accs.append(acc)
        else:
            pred = pred.data.numpy()
            roc = roc_auc_score(y, pred)
            accs.append(roc)
            acc = average_precision_score(y, pred)
            accs.append(acc)
    return accs

def weights_init(m):
    if isinstance(m, torch.nn.Linear):
        xavier(m.weight)
        if not m.bias is None:
            torch.nn.init.constant_(m.bias, 0)

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    torch.backends.cudnn.deterministic = True

#d_names = ['PPI']; times=range(20)
d_names = ["Photo", "PubMed", "Computers"]; times=range(50)

wait_total= 200
total_epochs = 2000

pipelines=['TLCGNN']
pipeline_acc={'TLCGNN':[i for i in times]}
pipeline_acc_sum={'TLCGNN':0}
pipeline_roc={'TLCGNN':[i for i in times]}
pipeline_roc_sum={'TLCGNN':0}
pipeline_acc_same={'TLCGNN':[i for i in times]}
pipeline_acc_same_sum={'TLCGNN':0}
pipeline_roc_same={'TLCGNN':[i for i in times]}
pipeline_roc_same_sum={'TLCGNN':0}
pipeline_acc_diff={'TLCGNN':[i for i in times]}
pipeline_acc_diff_sum={'TLCGNN':0}
pipeline_roc_diff={'TLCGNN':[i for i in times]}
pipeline_roc_diff_sum={'TLCGNN':0}

if not os.path.exists("./scores"):
    os.mkdir("./scores")

for d_name in d_names:
    f2 = open('scores/pipe_benchmark_' + d_name + '_LP_scores.txt', 'w+')
    f2.write('{0:7} {1:7}\n'.format(d_name, 'TLCGNN'))
    f2.flush()
    dataset = lds.loaddatas(d_name)
    for data_cnt in times:
        for Conv_method in pipelines:
            if d_name in ['Rand_nnodes_github1000', 'PPI']:
                data = dataset[data_cnt]
            else:
                data = dataset[0]
            if d_name in ['Rand_nnodes_github1000']:
                data.x = data.x[:, :10]
            #data.x = torch.ones(data.x.size())
            index = [i for i in range(len(data.y))]
            if d_name != "PPI":
                model, data = locals()[Conv_method].call(data, dataset.name, data.x.size(1), dataset.num_classes,
                                                     data_cnt)
            else:
                model, data = locals()[Conv_method].call(data, 'PPI', data.x.size(1), dataset.num_classes,
                                                         data_cnt)
            model.apply(weights_init)
            optimizer = torch.optim.Adam(model.parameters(), lr=0.005, weight_decay=0)
            best_val_acc = test_acc_same = test_acc_diff = test_acc = 0.0
            best_val_roc = test_roc_same = test_roc_diff = test_roc = 0.0
            best_val_loss = np.inf
            # train and val/test
            wait_step = 0

            # train and test
            for epoch in range(1, total_epochs + 1):
                pred = train()
                val_loss, val_roc, val_acc, tmp_test_roc, tmp_test_acc = test()
                if val_roc >= best_val_roc:
                    test_acc = tmp_test_acc
                    test_roc = tmp_test_roc
                    best_val_acc = val_acc
                    best_val_roc = val_roc
                    best_val_loss = val_loss
                    wait_step = 0
                else:
                    wait_step += 1
                    if wait_step == wait_total:
                        print('Early stop! Min loss: ', best_val_loss, ', Max accuracy: ', best_val_acc,
                              ', Max roc: ', best_val_roc)
                        break
            del model
            del data
            # print result

            pipeline_acc[Conv_method][data_cnt] = test_acc
            pipeline_roc[Conv_method][data_cnt] = test_roc

            log = 'Epoch: ' + str(
                total_epochs) + ', dataset name: ' + d_name + ', Method: ' + Conv_method + ' Test pr: {:.4f}, roc: {:.4f} \n'
            print((log.format(pipeline_acc[Conv_method][data_cnt], pipeline_roc[Conv_method][data_cnt])))
            #print(pred)

            f2.write('{}, {:.4f}, {:.4f}\n'.format(data_cnt, pipeline_acc[Conv_method][data_cnt],
                                                     pipeline_roc[Conv_method][data_cnt],))
            f2.flush()
    f2.write('{0:4} {1:4f}\n'.format('std', np.std(pipeline_acc[Conv_method])))
    f2.write('{0:4} {1:4f}\n'.format('mean', np.mean(pipeline_acc[Conv_method])))
    f2.write('{0:4} {1:4f}\n'.format('std', np.std(pipeline_roc[Conv_method])))
    f2.write('{0:4} {1:4f}\n'.format('mean', np.mean(pipeline_roc[Conv_method])))
    f2.close()


# -*- coding: utf-8 -*-
"""
hpack/hpack
~~~~~~~~~~~

Implements the HPACK header compression algorithm as detailed by the IETF.
"""
import logging

from .table import HeaderTable, table_entry_size
from .compat import to_byte, to_bytes
from .exceptions import (
    HPACKDecodingError, OversizedHeaderListError, InvalidTableSizeError
)
from .huffman import HuffmanEncoder
from .huffman_constants import (
    REQUEST_CODES, REQUEST_CODES_LENGTH
)
from .huffman_table import decode_huffman
from .struct import HeaderTuple, NeverIndexedHeaderTuple

log = logging.getLogger(__name__)

INDEX_NONE = b'\x00'
INDEX_NEVER = b'\x10'
INDEX_INCREMENTAL = b'\x40'

# Precompute 2^i for 1-8 for use in prefix calcs.
# Zero index is not used but there to save a subtraction
# as prefix numbers are not zero indexed.
_PREFIX_BIT_MAX_NUMBERS = [(2 ** i) - 1 for i in range(9)]

try:  # pragma: no cover
    str = str
except NameError:  # pragma: no cover
    str = (str, bytes)

# We default the maximum header list we're willing to accept to 64kB. That's a
# lot of headers, but if applications want to raise it they can do.
DEFAULT_MAX_HEADER_LIST_SIZE = 2 ** 16

def _unicode_if_needed(header, raw):
    """
    Provides a header as a unicode string if raw is False, otherwise returns
    it as a bytestring.
    """
    name = to_bytes(header[0])
    value = to_bytes(header[1])
    if not raw:
        name = name.decode('utf-8')
        value = value.decode('utf-8')
    return header.__class__(name, value)

def encode_integer(integer, prefix_bits):
    """
    This encodes an integer according to the wacky integer encoding rules
    defined in the HPACK spec.
    """
    log.debug("Encoding %d with %d bits", integer, prefix_bits)

    if integer < 0:
        raise ValueError(
            "Can only encode positive integers, got %s" % integer
        )

    if prefix_bits < 1 or prefix_bits > 8:
        raise ValueError(
            "Prefix bits must be between 1 and 8, got %s" % prefix_bits
        )

    max_number = _PREFIX_BIT_MAX_NUMBERS[prefix_bits]

    if integer < max_number:
        return bytearray([integer])  # Seriously?
    else:
        elements = [max_number]
        integer -= max_number

        while integer >= 128:
            elements.append((integer & 127) + 128)
            integer >>= 7

        elements.append(integer)

        return bytearray(elements)

def decode_integer(data, prefix_bits):
    """
    This decodes an integer according to the wacky integer encoding rules
    defined in the HPACK spec. Returns a tuple of the decoded integer and the
    number of bytes that were consumed from ``data`` in order to get that
    integer.
    """
    if prefix_bits < 1 or prefix_bits > 8:
        raise ValueError(
            "Prefix bits must be between 1 and 8, got %s" % prefix_bits
        )

    max_number = _PREFIX_BIT_MAX_NUMBERS[prefix_bits]
    index = 1
    shift = 0
    mask = (0xFF >> (8 - prefix_bits))

    try:
        number = to_byte(data[0]) & mask
        if number == max_number:
            while True:
                next_byte = to_byte(data[index])
                index += 1

                if next_byte >= 128:
                    number += (next_byte - 128) << shift
                else:
                    number += next_byte << shift
                    break
                shift += 7

    except IndexError:
        raise HPACKDecodingError(
            "Unable to decode HPACK integer representation from %r" % data
        )

    log.debug("Decoded %d, consumed %d bytes", number, index)

    return number, index

def _dict_to_iterable(header_dict):
    """
    This converts a dictionary to an iterable of two-tuples. This is a
    HPACK-specific function becuase it pulls "special-headers" out first and
    then emits them.
    """
    assert isinstance(header_dict, dict)
    keys = sorted(
        list(header_dict.keys()),
        key=lambda k: not _to_bytes(k).startswith(b':')
    )
    for key in keys:
        yield key, header_dict[key]

def _to_bytes(string):
    """
    Convert string to bytes.
    """
    if isinstance(string, bytes):
        return string
    if not isinstance(string, str):  # pragma: no cover
        string = str(string)

    return string if isinstance(string, bytes) else string.encode('utf-8')

class Encoder(object):
    """
    An HPACK encoder object. This object takes HTTP headers and emits encoded
    HTTP/2 header blocks.
    """

    def __init__(self):
        self.header_table = HeaderTable()
        self.huffman_coder = HuffmanEncoder(
            REQUEST_CODES, REQUEST_CODES_LENGTH
        )
        self.table_size_changes = []

    @property
    def header_table_size(self):
        """
        Controls the size of the HPACK header table.
        """
        return self.header_table.maxsize

    @header_table_size.setter
    def header_table_size(self, value):
        self.header_table.maxsize = value
        if self.header_table.resized:
            self.table_size_changes.append(value)

    def encode(self, headers, huffman=True):
        """
        Takes a set of headers and encodes them into a HPACK-encoded header
        block.

        :param headers: The headers to encode. Must be either an iterable of
                        tuples, an iterable of :class:`HeaderTuple
                        <hpack.struct.HeaderTuple>`, or a ``dict``.

                        If an iterable of tuples, the tuples may be either
                        two-tuples or three-tuples. If they are two-tuples, the
                        tuples must be of the format ``(name, value)``. If they
                        are three-tuples, they must be of the format
                        ``(name, value, sensitive)``, where ``sensitive`` is a
                        boolean value indicating whether the header should be
                        added to header tables anywhere. If not present,
                        ``sensitive`` defaults to ``False``.

                        If an iterable of :class:`HeaderTuple
                        <hpack.struct.HeaderTuple>`, the tuples must always be
                        two-tuples. Instead of using ``sensitive`` as a third
                        tuple entry, use :class:`NeverIndexedHeaderTuple
                        <hpack.struct.NeverIndexedHeaderTuple>` to request that
                        the field never be indexed.

                        .. warning:: HTTP/2 requires that all special headers
                            (headers whose names begin with ``:`` characters)
                            appear at the *start* of the header block. While
                            this method will ensure that happens for ``dict``
                            subclasses, callers using any other iterable of
                            tuples **must** ensure they place their special
                            headers at the start of the iterable.

                            For efficiency reasons users should prefer to use
                            iterables of two-tuples: fixing the ordering of
                            dictionary headers is an expensive operation that
                            should be avoided if possible.

        :param huffman: (optional) Whether to Huffman-encode any header sent as
                        a literal value. Except for use when debugging, it is
                        recommended that this be left enabled.

        :returns: A bytestring containing the HPACK-encoded header block.
        """
        # Transforming the headers into a header block is a procedure that can
        # be modeled as a chain or pipe. First, the headers are encoded. This
        # encoding can be done a number of ways. If the header name-value pair
        # are already in the header table we can represent them using the
        # indexed representation: the same is true if they are in the static
        # table. Otherwise, a literal representation will be used.
        log.debug("HPACK encoding %s", headers)
        header_block = []

        # Turn the headers into a list of tuples if possible. This is the
        # natural way to interact with them in HPACK. Because dictionaries are
        # un-ordered, we need to make sure we grab the "special" headers first.
        if isinstance(headers, dict):
            headers = _dict_to_iterable(headers)

        # Before we begin, if the header table size has been changed we need
        # to signal all changes since last emission appropriately.
        if self.header_table.resized:
            header_block.append(self._encode_table_size_change())
            self.header_table.resized = False

        # Add each header to the header block
        for header in headers:
            sensitive = False
            if isinstance(header, HeaderTuple):
                sensitive = not header.indexable
            elif len(header) > 2:
                sensitive = header[2]

            header = (_to_bytes(header[0]), _to_bytes(header[1]))
            header_block.append(self.add(header, sensitive, huffman))

        header_block = b''.join(header_block)

        log.debug("Encoded header block to %s", header_block)

        return header_block

    def add(self, to_add, sensitive, huffman=False):
        """
        This function takes a header key-value tuple and serializes it.
        """
        log.debug("Adding %s to the header table", to_add)

        name, value = to_add

        # Set our indexing mode
        indexbit = INDEX_INCREMENTAL if not sensitive else INDEX_NEVER

        # Search for a matching header in the header table.
        match = self.header_table.search(name, value)

        if match is None:
            # Not in the header table. Encode using the literal syntax,
            # and add it to the header table.
            encoded = self._encode_literal(name, value, indexbit, huffman)
            if not sensitive:
                self.header_table.add(name, value)
            return encoded

        # The header is in the table, break out the values. If we matched
        # perfectly, we can use the indexed representation: otherwise we
        # can use the indexed literal.
        index, name, perfect = match

        if perfect:
            # Indexed representation.
            encoded = self._encode_indexed(index)
        else:
            # Indexed literal. We are going to add header to the
            # header table unconditionally. It is a future todo to
            # filter out headers which are known to be ineffective for
            # indexing since they just take space in the table and
            # pushed out other valuable headers.
            encoded = self._encode_indexed_literal(
                index, value, indexbit, huffman
            )
            if not sensitive:
                self.header_table.add(name, value)

        return encoded

    def _encode_indexed(self, index):
        """
        Encodes a header using the indexed representation.
        """
        field = encode_integer(index, 7)
        field[0] |= 0x80  # we set the top bit
        return bytes(field)

    def _encode_literal(self, name, value, indexbit, huffman=False):
        """
        Encodes a header with a literal name and literal value. If ``indexing``
        is True, the header will be added to the header table: otherwise it
        will not.
        """
        if huffman:
            name = self.huffman_coder.encode(name)
            value = self.huffman_coder.encode(value)

        name_len = encode_integer(len(name), 7)
        value_len = encode_integer(len(value), 7)

        if huffman:
            name_len[0] |= 0x80
            value_len[0] |= 0x80

        return b''.join(
            [indexbit, bytes(name_len), name, bytes(value_len), value]
        )

    def _encode_indexed_literal(self, index, value, indexbit, huffman=False):
        """
        Encodes a header with an indexed name and a literal value and performs
        incremental indexing.
        """
        if indexbit != INDEX_INCREMENTAL:
            prefix = encode_integer(index, 4)
        else:
            prefix = encode_integer(index, 6)

        prefix[0] |= ord(indexbit)

        if huffman:
            value = self.huffman_coder.encode(value)

        value_len = encode_integer(len(value), 7)

        if huffman:
            value_len[0] |= 0x80

        return b''.join([bytes(prefix), bytes(value_len), value])

    def _encode_table_size_change(self):
        """
        Produces the encoded form of all header table size change context
        updates.
        """
        block = b''
        for size_bytes in self.table_size_changes:
            size_bytes = encode_integer(size_bytes, 5)
            size_bytes[0] |= 0x20
            block += bytes(size_bytes)
        self.table_size_changes = []
        return block

class Decoder(object):
    """
    An HPACK decoder object.

    .. versionchanged:: 2.3.0
       Added ``max_header_list_size`` argument.

    :param max_header_list_size: The maximum decompressed size we will allow
        for any single header block. This is a protection against DoS attacks
        that attempt to force the application to expand a relatively small
        amount of data into a really large header list, allowing enormous
        amounts of memory to be allocated.

        If this amount of data is exceeded, a `OversizedHeaderListError
        <hpack.OversizedHeaderListError>` exception will be raised. At this
        point the connection should be shut down, as the HPACK state will no
        longer be useable.

        Defaults to 64kB.
    :type max_header_list_size: ``int``
    """
    def __init__(self, max_header_list_size=DEFAULT_MAX_HEADER_LIST_SIZE):
        self.header_table = HeaderTable()

        #: The maximum decompressed size we will allow for any single header
        #: block. This is a protection against DoS attacks that attempt to
        #: force the application to expand a relatively small amount of data
        #: into a really large header list, allowing enormous amounts of memory
        #: to be allocated.
        #:
        #: If this amount of data is exceeded, a `OversizedHeaderListError
        #: <hpack.OversizedHeaderListError>` exception will be raised. At this
        #: point the connection should be shut down, as the HPACK state will no
        #: longer be usable.
        #:
        #: Defaults to 64kB.
        #:
        #: .. versionadded:: 2.3.0
        self.max_header_list_size = max_header_list_size

        #: Maximum allowed header table size.
        #:
        #: A HTTP/2 implementation should set this to the most recent value of
        #: SETTINGS_HEADER_TABLE_SIZE that it sent *and has received an ACK
        #: for*. Once this setting is set, the actual header table size will be
        #: checked at the end of each decoding run and whenever it is changed,
        #: to confirm that it fits in this size.
        self.max_allowed_table_size = self.header_table.maxsize

    @property
    def header_table_size(self):
        """
        Controls the size of the HPACK header table.
        """
        return self.header_table.maxsize

    @header_table_size.setter
    def header_table_size(self, value):
        self.header_table.maxsize = value

    def decode(self, data, raw=False):
        """
        Takes an HPACK-encoded header block and decodes it into a header set.

        :param data: A bytestring representing a complete HPACK-encoded header
                     block.
        :param raw: (optional) Whether to return the headers as tuples of raw
                    byte strings or to decode them as UTF-8 before returning
                    them. The default value is False, which returns tuples of
                    Unicode strings
        :returns: A list of two-tuples of ``(name, value)`` representing the
                  HPACK-encoded headers, in the order they were decoded.
        :raises HPACKDecodingError: If an error is encountered while decoding
                                    the header block.
        """
        log.debug("Decoding %s", data)

        if not isinstance(data, memoryview):
            data_mem = memoryview(data)
        else:
            data_mem = data
        headers = []
        data_len = len(data)
        inflated_size = 0
        current_index = 0

        while current_index < data_len:
            # Work out what kind of header we're decoding.
            # If the high bit is 1, it's an indexed field.
            current = to_byte(data[current_index])
            indexed = True if current & 0x80 else False

            # Otherwise, if the second-highest bit is 1 it's a field that does
            # alter the header table.
            literal_index = True if current & 0x40 else False

            # Otherwise, if the third-highest bit is 1 it's an encoding context
            # update.
            encoding_update = True if current & 0x20 else False

            if indexed:
                header, consumed = self._decode_indexed(
                    data_mem[current_index:]
                )
            elif literal_index:
                # It's a literal header that does affect the header table.
                header, consumed = self._decode_literal_index(
                    data_mem[current_index:]
                )
            elif encoding_update:
                # It's an update to the encoding context. These are forbidden
                # in a header block after any actual header.
                if headers:
                    raise HPACKDecodingError(
                        "Table size update not at the start of the block"
                    )
                consumed = self._update_encoding_context(
                    data_mem[current_index:]
                )
                header = None
            else:
                # It's a literal header that does not affect the header table.
                header, consumed = self._decode_literal_no_index(
                    data_mem[current_index:]
                )

            if header:
                headers.append(header)
                inflated_size += table_entry_size(*header)

                if inflated_size > self.max_header_list_size:
                    raise OversizedHeaderListError(
                        "A header list larger than %d has been received" %
                        self.max_header_list_size
                    )

            current_index += consumed

        # Confirm that the table size is lower than the maximum. We do this
        # here to ensure that we catch when the max has been *shrunk* and the
        # remote peer hasn't actually done that.
        self._assert_valid_table_size()

        try:
            return [_unicode_if_needed(h, raw) for h in headers]
        except UnicodeDecodeError:
            raise HPACKDecodingError("Unable to decode headers as UTF-8.")

    def _assert_valid_table_size(self):
        """
        Check that the table size set by the encoder is lower than the maximum
        we expect to have.
        """
        if self.header_table_size > self.max_allowed_table_size:
            raise InvalidTableSizeError(
                "Encoder did not shrink table size to within the max"
            )

    def _update_encoding_context(self, data):
        """
        Handles a byte that updates the encoding context.
        """
        # We've been asked to resize the header table.
        new_size, consumed = decode_integer(data, 5)
        if new_size > self.max_allowed_table_size:
            raise InvalidTableSizeError(
                "Encoder exceeded max allowable table size"
            )
        self.header_table_size = new_size
        return consumed

    def _decode_indexed(self, data):
        """
        Decodes a header represented using the indexed representation.
        """
        index, consumed = decode_integer(data, 7)
        header = HeaderTuple(*self.header_table.get_by_index(index))
        log.debug("Decoded %s, consumed %d", header, consumed)
        return header, consumed

    def _decode_literal_no_index(self, data):
        return self._decode_literal(data, False)

    def _decode_literal_index(self, data):
        return self._decode_literal(data, True)

    def _decode_literal(self, data, should_index):
        """
        Decodes a header represented with a literal.
        """
        total_consumed = 0

        # When should_index is true, if the low six bits of the first byte are
        # nonzero, the header name is indexed.
        # When should_index is false, if the low four bits of the first byte
        # are nonzero the header name is indexed.
        if should_index:
            indexed_name = to_byte(data[0]) & 0x3F
            name_len = 6
            not_indexable = False
        else:
            high_byte = to_byte(data[0])
            indexed_name = high_byte & 0x0F
            name_len = 4
            not_indexable = high_byte & 0x10

        if indexed_name:
            # Indexed header name.
            index, consumed = decode_integer(data, name_len)
            name = self.header_table.get_by_index(index)[0]

            total_consumed = consumed
            length = 0
        else:
            # Literal header name. The first byte was consumed, so we need to
            # move forward.
            data = data[1:]

            length, consumed = decode_integer(data, 7)
            name = data[consumed:consumed + length]
            if len(name) != length:
                raise HPACKDecodingError("Truncated header block")

            if to_byte(data[0]) & 0x80:
                name = decode_huffman(name)
            total_consumed = consumed + length + 1  # Since we moved forward 1.

        data = data[consumed + length:]

        # The header value is definitely length-based.
        length, consumed = decode_integer(data, 7)
        value = data[consumed:consumed + length]
        if len(value) != length:
            raise HPACKDecodingError("Truncated header block")

        if to_byte(data[0]) & 0x80:
            value = decode_huffman(value)

        # Updated the total consumed length.
        total_consumed += length + consumed

        # If we have been told never to index the header field, encode that in
        # the tuple we use.
        if not_indexable:
            header = NeverIndexedHeaderTuple(name, value)
        else:
            header = HeaderTuple(name, value)

        # If we've been asked to index this, add it to the header table.
        if should_index:
            self.header_table.add(name, value)

        log.debug(
            "Decoded %s, total consumed %d bytes, indexed %s",
            header,
            total_consumed,
            should_index
        )

        return header, total_consumed