# encdoing: utf-8
"""
@Project: pm25_v1
@File:    plotmap2
@Author:  Jiachen Zhao
@Time:    2021/8/2 14:30
@Description:
"""

import os
# noinspection PyUnresolvedReferences
import numpy as np
import pandas as pd
import folium
import datetime
from IPython.display import HTML
from src.utils.utils import *
from folium.plugins import HeatMap

root_path = os.path.abspath('.')
data_path = rf'{root_path}/Datasets'
# data_path = rf'F:/data-list/PycharmProjects/Datasets/'
# data_path = r'/data1/zhaojiachen/Datasets/PM25/'
data_path_2021 = data_path + 'pm25-2021/'
data_path_2020 = data_path + 'pm25-2020/'
preprocess_data_path = data_path + 'pm25-preprocess_data/'
preprocess_EachHour_2021 = preprocess_data_path + '21-EachHourEachFile'
preprocess_EachStation_2021 = preprocess_data_path + '21-EachStationEachFile'

preprocess_EachHour_2020 = preprocess_data_path + '20-EachHourEachFile'
preprocess_EachStation_2020 = preprocess_data_path + '20-EachStationEachFile'
preprocess_EachStation_NotFilled_2020 = preprocess_EachStation_2020 + '/NotFilled'
preprocess_EachStation_Filled_2020 = preprocess_EachStation_2020 + '/Filled'

project_folder = "F:/data-list/PycharmProjects/pm25_v1"
lookup_folder = project_folder + "/lookup"
stationMap_folder = project_folder + "/StationMap"
neighborMap_folder = project_folder + "/StationNeighborMap"

def plot_baseMap():
    ch_map = folium.Map(location=[35, 100], zoom_start=4)
    # ch_map = folium.Map(
    #     location=[38.96, 117.78],
    #     zoom_start=4,
    #     # tiles='http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}', # 高德街道图
    #     # tiles='http://webst02.is.autonavi.com/appmaptile?style=6&x={x}&y={y}&z={z}', # 高德卫星图
    #     tiles='https://mt.google.com/vt/lyrs=s&x={x}&y={y}&z={z}',  # google 卫星图
    #     # tiles='https://mt.google.com/vt/lyrs=h&x={x}&y={y}&z={z}', # google 地图
    #     attr='default'
    # )

    folium.Choropleth(
        geo_data='china.json',
        name='China regions',
        fill_opacity=0.1,
        line_opacity=0.2,
    ).add_to(ch_map)
    return ch_map

def plot_stationIcon_fun(query="华北地区", map=None):  # 将输入地区省份的所有站点，在地图上标注出来
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = provinces_fun()  # csv中的所有省份
    print(provinces)

    if query in provinces:
        query = [query]
    elif query == "全国":
        query = provinces
    elif query == "华北地区":
        query = ('北京市', '天津市', '河北省', '山西省', '内蒙古自治区')
    elif query == "华东地区":
        query = ('上海市', '江苏省', '浙江省', '山东省', '安徽省')
    elif query == "东北地区":
        query = ('辽宁省', '吉林省', '黑龙江省')
    elif query == "华中地区":
        query = ('湖北省', '湖南省', '河南省', '江西省')
    elif query == "华南地区":
        query = ('广东省', '广西壮族自治区', '海南省', '福建省')
    elif query == "西南地区":
        query = ('四川省', '重庆市', '贵州省', '云南省', '西藏自治区')
    elif query == '西北地区':
        query = ('陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区')
    else:
        print(f"{query}, 错误的省份")
    print(query)
    stations_query_df = stations_df[stations_df['province'].isin(query)]  # 过滤出所选地区的省份数据
    print(stations_query_df.shape)

    folium.LayerControl().add_to(map)
    for idx, row in stations_query_df.iterrows():  # 编号，行信息
        s1 = f"ID: {int(row['id'])}"
        s2 = f"{row['latitude'], row['longitude']}"
        s3 = f"{row['province'], row['city'], row['district']}"
        WIDTH = max(len(s1.encode('utf-8')), len(s2.encode('utf-8')), len(s3.encode('utf-8')))
        pop = folium.Popup(html=folium.Html("""
                            {}

                            {}

                            {}

                            """.format(s1, s2, s3),
                                            script=True,
                                            width=WIDTH * 4),
                           parse_html=True,
                           max_width=3000)
        folium.Marker(location=[row['latitude'], row['longitude']],
                      popup=pop).add_to(map)
    return map

def provinces_fun():  # 获取csv中所有的省份名，返回省份名列表
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = []
    for ii in stations_df['province'].unique().tolist():
        if ii is not np.NAN:
            if '\u4e00' <= ii[0] <= '\u9fff':
                provinces.append(ii)
    return provinces

def plot_stationMaps():  # 画出所有的站点数据点
    queries = [
        "全国",
        "华北地区", "华东地区", "东北地区", "华中地区", "华南地区", "西南地区", "西北地区",
        # "北京市", '天津市', '河北省', '山西省', '内蒙古自治区',
        # '上海市', '江苏省', '浙江省', '山东省', '安徽省',
        # '辽宁省', '吉林省', '黑龙江省',
        # '湖北省', '湖南省', '河南省', '江西省',
        # '广东省', '广西壮族自治区', '海南省', '福建省',
        # '四川省', '重庆市', '贵州省', '云南省', '西藏自治区',
        # '陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区'
    ]
    for query in queries:
        map_base = plot_baseMap()  # 多张图，所有区域都画出
        plot_stationIcon_fun(query)
        # map_query = plot_stationIcon_fun(query, map_base)
        # map_query.save(f'{stationMap_folder}/{query}.html')

def add_StationByDF(map, df):  # 将传入的 dataframe 中的所有站点数据点显示在Map上
    for idx, row in df.iterrows():
        s1 = f"ID: {int(row['id'])}"
        s2 = f"{row['latitude'], row['longitude']}"
        s3 = f"{row['province'], row['city'], row['district']}"
        WIDTH = max(len(s1.encode('utf-8')), len(s2.encode('utf-8')), len(s3.encode('utf-8')))
        pop = folium.Popup(html=folium.Html("""
                            {}

                            {}

                            {}

                            """.format(s1, s2, s3),
                                            script=True,
                                            width=WIDTH * 4),
                           parse_html=True,
                           max_width=3000)
        folium.Marker(location=[row['latitude'], row['longitude']],
                      popup=pop).add_to(map)
    return map

def plot_OneStationNeighbor_fun(query_id=None):
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    station_net_df = pd.read_csv(lookup_folder + '/StationNeighbors.csv', index_col=0)
    # print(station_net_df.head())
    # print(stations_df.head())
    map_base = plot_baseMap()
    folium.LayerControl().add_to(map_base)
    query_station = stations_df[stations_df['id'] == query_id]  # 选出输入序号站点的数据
    map = add_StationByDF(map_base, query_station)  # 将输入序号站点加入地图
    for i in range(1, 10):  # 若干相邻点
        neighbor_id_i = station_net_df[station_net_df['id'] == query_id][f'{i}_neighbor'].values[0]  # 得到目标站点对应的附近站点id值
        neighbor_dist_i = station_net_df[station_net_df['id'] == query_id][f'{i}_distance'].values[0]
        # print(neighbor_id_i, neighbor_dist_i)
        neighbor_station_i = stations_df[stations_df['id'] == neighbor_id_i]  # 得到附近站点的相关信息
        # print(neighbor_station_i)
        # print([query_station['latitude'].values[0], query_station['longitude'].values[0]])
        # print([neighbor_station_i['latitude'].values[0], neighbor_station_i['longitude'].values[0]])
        folium.PolyLine([[query_station['latitude'].values[0], query_station['longitude'].values[0]],
                         [neighbor_station_i['latitude'].values[0], neighbor_station_i['longitude'].values[0]]],
                        color='red').add_to(map)
        map = add_StationByDF(map, neighbor_station_i)  # 给出对应附近站点的标记
    return map

def plot_DotMap(query=None, Time=None):  # 某个时间下，某个地区范围内所有站点的空气质量等级颜色图（某站点一定半径范围内）
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = provinces_fun()

    if query in provinces:
        querys = [query]
    elif query == "全国":
        querys = provinces
    elif query == "华北地区":
        querys = ('北京市', '天津市', '河北省', '山西省', '内蒙古自治区')
    elif query == "华东地区":
        querys = ('上海市', '江苏省', '浙江省', '山东省', '安徽省')
    elif query == "东北地区":
        querys = ('辽宁省', '吉林省', '黑龙江省')
    elif query == "华中地区":
        querys = ('湖北省', '湖南省', '河南省', '江西省')
    elif query == "华南地区":
        querys = ('广东省', '广西壮族自治区', '海南省', '福建省')
    elif query == "西南地区":
        querys = ('四川省', '重庆市', '贵州省', '云南省', '西藏自治区')
    elif query == '西北地区':
        querys = ('陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区')
    else:
        print(f"{query}, 错误的省份")
    stations_query_df = stations_df[stations_df['province'].isin(querys)]

    map_base = plot_baseMap()
    # folium.LayerControl().add_to(map_base)
    aqi_map_feature = folium.map.FeatureGroup()

    for idx, row in stations_query_df.iterrows():
        id = row['id']
        try:
            file = f'{preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv'
            df_station_observation = pd.read_csv(file, index_col=0)
            # print(df_station_observation.head())
            df_station_time = df_station_observation[df_station_observation['time'] == Time]  # 获取某个时间点下站点的数据
            pm25_value = df_station_time['pm25'].values[0]
            aqi, rank, quality, alarm_color = compute_iaqi(cp=pm25_value, return_rank=True)  # 空气质量指数aqi，等级，质量评价，警报等级
            # print(f"AQI: {np.round(aqi, 2)}, rank: {rank}, quality: {quality}, alarm_color: {alarm_color}")
            # print(pm25_value)
            # print([row['latitude'], row['longitude']])
            aqi_map_feature.add_child(  # 站点附近一定范围内，显示空气质量aqi对应颜色
                folium.CircleMarker(
                    [row['latitude'], row['longitude']],
                    radius=7,
                    color=None,
                    fill=True,
                    fill_color=alarm_color,
                    fillopacity=10
                )
            )
        except FileNotFoundError:
            print(f'There is no fill {preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv')
            continue
    map_base.add_child(aqi_map_feature)
    map_base.save(f'{project_folder}/AQI_Dot_map/{query}_dotmap_{Time}.html')

def plot_HeatMap(query='全国', Time=20020520):
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = provinces_fun()

    if query in provinces:
        querys = [query]
    elif query == "全国":
        querys = provinces
    elif query == "华北地区":
        querys = ('北京市', '天津市', '河北省', '山西省', '内蒙古自治区')
    elif query == "华东地区":
        querys = ('上海市', '江苏省', '浙江省', '山东省', '安徽省')
    elif query == "东北地区":
        querys = ('辽宁省', '吉林省', '黑龙江省')
    elif query == "华中地区":
        querys = ('湖北省', '湖南省', '河南省', '江西省')
    elif query == "华南地区":
        querys = ('广东省', '广西壮族自治区', '海南省', '福建省')
    elif query == "西南地区":
        querys = ('四川省', '重庆市', '贵州省', '云南省', '西藏自治区')
    elif query == '西北地区':
        querys = ('陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区')
    else:
        print(f"{query}, 错误的省份")
    stations_query_df = stations_df[stations_df['province'].isin(querys)]

    map_base = plot_baseMap()

    df_haetmap = pd.DataFrame({'id': [], 'latitude': [], 'longitude': [], 'aqi': []})
    for idx, row in stations_query_df.iterrows():
        id = row['id']
        try:
            file = f'{preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv'
            df_station_observation = pd.read_csv(file, index_col=0)
            df_station_time = df_station_observation[df_station_observation['time'] == Time]  # 所选区域中某站点id，在所选时间下的站点数据
            pm25_value = df_station_time['pm25'].values[0]  # 取PM2.5浓度数据
            aqi, rank, quality, alarm_color = compute_iaqi(cp=pm25_value, return_rank=True)
            df_haetmap = df_haetmap.append({
                                            'id': id,
                                            'latitude': row['latitude'],
                                            'longitude': row['longitude'],
                                            'aqi': aqi
                                            }, ignore_index=True)  # 重建 df ，专为热力图，是输入 Time 下的某id站点的空气质量
        except FileNotFoundError:
            print(f'There is no fill {preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv')
            continue
    data = df_haetmap[['latitude', 'longitude', 'aqi']].values.tolist()  # 从 df 转为 array,最后转为 list
    map_base.add_child(HeatMap(data, radius=10, gradient={0.16: 'green', 0.32: 'yellow', 0.48: 'orange',
                                                         0.49: 'red', 0.8: 'purple', 1: 'maroon'}))
    print(df_haetmap)
    map_base.save(f'{project_folder}/AQI_heat_map/{query}_heatmap_{Time}.html')

def read_one_observation(id=None, Time=None, columns=['pm25', 'pm10']):  # 某站点id，单个时间点下，某些特征
    file = f'{preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv'
    df_station_observation = pd.read_csv(file, index_col=0)
    dfrow_station_time = df_station_observation[df_station_observation['time'] == Time]
    df_observation = dfrow_station_time[columns]
    return df_observation

def read_multiple_time(id=None, start_t=None, hours=6, direction='backward', columns=['time','pm25', 'pm10']):
    # a = '/'.join(['20'+str(start_t)[i:i+2] if i==0 else str(start_t)[i:i+2] for i in range(0, len(str(start_t)), 2)])
    begin = datetime.datetime.strptime(f'20{start_t}', '%Y%m%d%H')
    if direction == 'forward':
        delta = datetime.timedelta(hours=1)  # 时间差往前一小时
        tt = [(begin + delta * i).strftime('%Y%m%d%H')[2:] for i in range(hours)]  # 得到一个时间列表 [20010100, 20010101, ……]之类的
    elif direction == 'backward':
        delta = datetime.timedelta(hours=-1)
        tt = [(begin + delta * i).strftime('%Y%m%d%H')[2:] for i in range(hours)][::-1]  # 列表取反，时间仍然转为正序，与 'forward' 保持一致：时间点靠前的序号小
    tt = list(map(int, tt))  # 将字符串转换为int型
    dfs = pd.concat([read_one_observation(id=id, Time=t, columns=columns) for t in tt])
    dfs.reset_index(inplace=True, drop=True)  # 重置索引
    print(dfs)  # 时间前靠前的数据，索引小

def plot_OneStation_pm25_pop(id=None, Time=None):
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    station_info = stations_df[stations_df['id']==id]
    for idx, row in station_info.iterrows():
        observation = read_one_observation(id=id, Time=Time, columns=['pm25'])
        aqi, rank, quality, alarm_color = compute_iaqi(cp=observation.values, return_rank=True)
        print(f"AQI: {np.round(aqi, 2)}, rank: {rank}, quality: {quality}, alarm_color: {alarm_color}")
        map_base = plot_baseMap()
        folium.LayerControl().add_to(map_base)
        s1 = f"ID: {int(row['id'])}"
        s2 = f"{row['latitude'], row['longitude']}"
        s3 = f"{row['province'], row['city'], row['district']}"
        s4 = f"AQI:      {np.round(aqi, 2)}"
        s5 = f"RANK:     {rank}"
        s6 = f"QUALITY:  {quality}"
        s7 = f"ALARM:    {alarm_color}"
        WIDTH = max(len(s1.encode('utf-8')), len(s2.encode('utf-8')), len(s3.encode('utf-8')))

        pop = folium.Popup(html=folium.Html("""
                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    """.format(s1, s2, s3, '-'*len(s2), s4, s5, s6, s7),
                                            script=True,
                                            width=WIDTH * 4),
                           parse_html=True,
                           max_width=3000)
        folium.Marker(location=[row['latitude'], row['longitude']],
                      popup=pop).add_to(map_base)

        map_base.save(f'{project_folder}/AQI_pop_map/{id}_{Time}.html')

if __name__ == "__main__":
    print('-----')
    # plot_stationMaps()
    # query_plotOneStationNeighbor = [54499, 54511, 58367, 58035]
    # for query_id in query_plotOneStationNeighbor:
    #     map_OneStationNeighbor = plot_OneStationNeighbor_fun(query_id)
    #     map_OneStationNeighbor.save(f'{neighborMap_folder}/{query_id}_neighbors.html')
    # plot_DotMap(query='全国', Time=20020520)
    # read_one_observation(id = 54499, Time=20010508, columns=['pm25', 'pm10'])
    # read_multiple_time(id=54499, start_t=20010501, hours=6)
    # plot_OneStation_pm25_pop(id=58035, Time=20070107)
    # plot_OneStation_pm25_pop(id=54499, Time=20121109)
    # plot_HeatMap(query='河北省', Time=20112520)


from zoo import Zoo
from time import sleep

z = Zoo()

while True:

  sleepval = 0.00001

  for x in range(z.NODE_COUNT):

    z.set_node(x, [255, 0, 0])
    z.send_frame()

    sleep(sleepval)

    z.set_node(x, [0, 255, 0])
    z.send_frame()

    sleep(sleepval)

    z.set_node(x, [0, 0, 255])
    z.send_frame()

    sleep(sleepval)

    z.set_node(x, [255, 255, 255])
    z.send_frame()

    sleep(sleepval)


from django.urls import path
from . import views
from django.contrib.staticfiles.urls import staticfiles_urlpatterns

urlpatterns = [
    path('', views.home, name='home'),
    path('about/', views.about, name='about'),
    path('goals/', views.goals_index, name='goals'),
    path('goals/<int:goal_id>/', views.goal_detail, name='goal_detail'),
    path('goals/create/', views.GoalCreate.as_view(), name='goal_create')
]

urlpatterns += staticfiles_urlpatterns()


#Anthony Bernardo Kamers
#Campo Minado (15 x 15)

from random import randint

class CampoMinado:
    def __init__(self):
        self.qtdBombas = randint(35, 55) #bombas aleatórias (de 35 a 55 bombas)
        self.campo = []
        self.bombas = []

        self.qtdJogadas = 225 - self.qtdBombas #225 porque é 15 * 15
        self.qtd = 0 #cada jogada do usuário aumenta 1 (se não for bomba)

        self.abertos = [] #campos q são abertos a cada abrirCampos()

        #preencher campo minado
        for i in range(0, 15):
            x = []
            for j in range(0, 15):
                x.append([0, 0])
            self.campo.append(x)

        #preencher campo minado com bombas
        i = 0
        while True:
            linha = randint(0, 14)
            coluna = randint(0, 14)

            while(self.campo[linha][coluna][1] == 'X'):
                linha = randint(0, 14)
                coluna = randint(0, 14)

            self.bombas.append([linha, coluna])
            self.campo[linha][coluna][1] = 'X'
            i += 1

            if i == self.qtdBombas:
                break

        #preencher com números (perto das bombas)
        for i in range(0, len(self.bombas)):
            for linha in range(0, len(self.campo)):
                for coluna in range(0, len(self.campo)):
                    if self.campo[linha][coluna][1] == 'X':
                        pass
                    else:
                        if ((linha == self.bombas[i][0] - 1 or linha == self.bombas[i][0] or linha == self.bombas[i][0] + 1) and (coluna == self.bombas[i][1] - 1 or coluna == self.bombas[i][1] or coluna == self.bombas[i][1] + 1)):
                            self.campo[linha][coluna][1] += 1

        #chamar função do jogo
        self.play()

    #show do campo minado (abertos ou não)
    def show(self):
        print('''      0   1   2   3   4   5   6   7   8   9  10  11  12  13  14''')
        for linha in range(0, len(self.campo)):
            if linha < 10:
                print(f'{linha}  - ', end='')
            else:
                print(f'{linha} - ', end='')

            for coluna in range(0, len(self.campo)):
                if self.campo[linha][coluna][0] == 0:
                    print('[ ]', end=' ')
                else:
                    print(f'[{self.campo[linha][coluna][1]}]', end=' ')
            print()

    def atacar(self, x, y):

        if self.campo[x][y][0] == 1:
            return 'preenchido'

        else:
            if self.campo[x][y][1] == 'X':
                return 'bomba'

            elif self.campo[x][y][1] == 0:
                self.campo[x][y][0] = 1 #abre campo
                self.qtd += 1 #aumenta, dizendo q abriu mais um campo
                return 'vazio'

            else:
                self.campo[x][y][0] = 1 #abre campo
                self.qtd += 1 #aumenta, dizendo q abriu mais um campo
                return 'numero'

    def abrirCampos(self, x, y):
        for linha in range(0, len(self.campo)):
            for coluna in range(0, len(self.campo)):
                if ((linha == x - 1 or linha == x or linha == x + 1) and (coluna == y - 1 or coluna == y or coluna == y + 1)):
                    resp = self.atacar(linha, coluna)

                    if resp == 'vazio': #or resp == 'numero':
                        self.abertos.append([linha, coluna])

    #sempre abre o primeiro item (index = 0)
    def abrirAbertos(self):
        while len(self.abertos) > 0:
            self.abrirCampos(self.abertos[0][0], self.abertos[0][1])

            self.abertos.pop(0)

    #método q faz o usuário jogar
    def play(self):
        while True:
            self.show()
            campo = input("Diga as coordenadas do campo que deseja atacar (x,y): ")

            campo1 = campo.split(',')
            x = int(campo1[0])
            y = int(campo1[1])

            resposta = self.atacar(x, y)

            if resposta == 'bomba':
                self.perdeu()
                break

            elif resposta == 'vazio':
                self.abrirCampos(x, y)
                self.abrirAbertos()

            elif resposta == 'numero':
                pass #só abre o próprio campo

            elif resposta == 'preenchido':
                print('já preenchido')

            if(self.qtd == self.qtdJogadas):
                self.ganhou()
                break

    def perdeu(self):
        print('Você explodiu uma bomba! Você perdeu')

        #mostrar para o usuário o self.campo inteiro
        for linha in range(0, len(self.campo)):
            print('', end='')
            for coluna in range(0, len(self.campo)):
                print(self.campo[linha][coluna][1], end=' ')
            print()

        self.campoInteiro()

    def ganhou(self):
        print('Parabéns!! Você ganhou!! Abriu todos os campos sem abrir nenhuma bomba!!')

        self.campoInteiro()

    def campoInteiro(self):
        for linha in range(0, len(self.campo)):
            print('', end='')
            for coluna in range(0, len(self.campo)):
                print(self.campo[linha][coluna][1], end=' ')
            print()

if __name__ == "__main__":
    campoMinado = CampoMinado()


import os
import platform
import sys
from methods import get_compiler_version, using_gcc
from platform_methods import detect_arch

from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from SCons import Environment

def get_name():
    return "LinuxBSD"

def can_build():
    if os.name != "posix" or sys.platform == "darwin":
        return False

    pkgconf_error = os.system("pkg-config --version > /dev/null")
    if pkgconf_error:
        print("Error: pkg-config not found. Aborting.")
        return False

    return True

def get_opts():
    from SCons.Variables import BoolVariable, EnumVariable

    return [
        EnumVariable("linker", "Linker program", "default", ("default", "bfd", "gold", "lld", "mold")),
        BoolVariable("use_llvm", "Use the LLVM compiler", False),
        BoolVariable("use_x11", "Use X11 display server", True),
        BoolVariable("use_wayland", "Use Wayland display server", False),
        BoolVariable("use_gles", "Use OpenGL ES 3.2 instead of OpenGL 3.2", False),
    ]

def get_flags():
    return [
        ("arch", detect_arch()),
    ]

def configure(env: "Environment"):
    # Validate arch.
    supported_arches = ["x86_32", "x86_64", "arm32", "arm64"]
    if env["arch"] not in supported_arches:
        print(
            'Unsupported CPU architecture "%s" for Linux / *BSD. Supported architectures are: %s.'
            % (env["arch"], ", ".join(supported_arches))
        )
        sys.exit(255)

    if env["debug_symbols"]:
        env.Append(LINKFLAGS=["-rdynamic"])

    host_is_64_bit = sys.maxsize > 2**32
    if host_is_64_bit and env["arch"] == "x86_32":
        env.Append(CCFLAGS=["-m32"])
        env.Append(LINKFLAGS=["-m32"])
    elif not host_is_64_bit and env["arch"] == "x86_64":
        env.Append(CCFLAGS=["-m64"])
        env.Append(LINKFLAGS=["-m64"])

    ## Compiler configuration

    if "CC" in env and "clang" in os.path.basename(env["CC"]):
        # Convenience check to enforce the use_llvm overrides when CC is clang(++)
        env["use_llvm"] = True

    if env["use_llvm"]:
        if "clang" not in os.path.basename(env["CC"]):
            env["CC"] = "clang"
        env.extra_suffix = ".llvm" + env.extra_suffix

    if env["linker"] != "default":
        print("Using linker program: " + env["linker"])
        env.Append(LINKFLAGS=["-fuse-ld=%s" % env["linker"]])

    # LTO

    if env["lto"] == "auto":  # Full LTO for production.
        env["lto"] = "full"

    if env["lto"] != "none":
        if env["lto"] == "thin":
            if not env["use_llvm"]:
                print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.")
                sys.exit(255)
            env.Append(CCFLAGS=["-flto=thin"])
            env.Append(LINKFLAGS=["-flto=thin"])
        elif not env["use_llvm"] and env.GetOption("num_jobs") > 1:
            env.Append(CCFLAGS=["-flto"])
            env.Append(LINKFLAGS=["-flto=" + str(env.GetOption("num_jobs"))])
        else:
            env.Append(CCFLAGS=["-flto"])
            env.Append(LINKFLAGS=["-flto"])

        if not env["use_llvm"]:
            env["RANLIB"] = "gcc-ranlib"
            env["AR"] = "gcc-ar"

    env.Append(CCFLAGS=["-pipe"])

    ## Dependencies
    ## TODO: Need Deps

    env.Append(
        CPPDEFINES=[
            "LINUXBSD_ENABLED",
            "UNIX_ENABLED",
        ]
    )

    env.Append(LIBS=["pthread"])

    if platform.system() == "Linux":
        env.Append(LIBS=["dl"])

    if platform.system() == "FreeBSD":
        env.Append(LINKFLAGS=["-lkvm"])

    if env["use_wayland"]:
        env.Append(CPPDEFINES=["WAYLAND_ENABLED"])

        if os.system("pkg-config --exists wayland-client"):
            print("Error: Wayland libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-client --cflags --libs")

        if os.system("pkg-config --exists wayland-cursor"):
            print("Error: Wayland cursor libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-cursor --cflags --libs")

        if os.system("pkg-config --exists wayland-egl"):
            print("Error: Wayland EGL libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-egl --cflags --libs")

        if os.system("pkg-config --exists wayland-protocols"):
            print("Error: Wayland protocols not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-protocols --cflags --libs")

        wayland_protocols_dir = os.popen("pkg-config wayland-protocols --variable=pkgdatadir").read().strip().replace("//", "/")
        print("Wayland protocols dir: " + wayland_protocols_dir)
        os.system("wayland-scanner client-header %s/stable/xdg-shell/xdg-shell.xml %s/xdg-shell.gen.h" % (wayland_protocols_dir, "platform/linuxbsd/wayland"))
        os.system("wayland-scanner private-code %s/stable/xdg-shell/xdg-shell.xml %s/xdg-shell.gen.c" % (wayland_protocols_dir, "platform/linuxbsd/wayland"))

    if env["use_x11"]:
        env.Append(CPPDEFINES=["X11_ENABLED"])

        if os.system("pkg-config --exists x11"):
            print("Error: X11 libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config x11 --cflags --libs")


import logging
from collections import namedtuple
from enum import Enum
from functools import lru_cache
from pathlib import Path
from typing import FrozenSet, List, Optional

import lief
from lief.ELF import E_TYPE

from .binary import BinarySecurity
from .errors import ErrorParsingFailed
from .utils import LibcNotFoundError, find_libc

FORTIFIED_END_MARKER = "_chk"
FORTFIED_START_MARKER = "__"
LIBC_OBJ = None
ELFChecksecData = namedtuple(
    "ELFChecksecData",
    [
        "relro",
        "canary",
        "nx",
        "pie",
        "rpath",
        "runpath",
        "symbols",
        "fortify_source",
        "fortified",
        "fortifiable",
        "fortify_score",
    ],
)

__LIBC_OBJ = {}

def get_libc(libc_path: Optional[Path] = None) -> Optional["Libc"]:
    """This function initializes a Libc using LIEF

    :param libc_path: an optional Path to the libc library. if None, the Libc will be auto detected using various
    methods
    :return a Libc object
    """
    # Note: this weird function is designed as a poor singleton lru_cache wasn't possible, since get_libc(path) and
    # get_libc(None) should return the same data once the first call has been made we need to maintain a global
    # object as we can't pass the Libc object to ELFSecurity, as LIEF's objects are not picklable
    global __LIBC_OBJ
    try:
        __LIBC_OBJ["libc"]
    except KeyError:
        logging.debug("Libc object not set")
        try:
            libc = Libc(libc_path)
        except (LibcNotFoundError, ErrorParsingFailed) as e:
            logging.debug("Failed to init Libc object: %s", e)
            __LIBC_OBJ["libc"] = None
        else:
            logging.debug("Libc object initialized")
            __LIBC_OBJ["libc"] = libc
    logging.debug(__LIBC_OBJ)
    return __LIBC_OBJ["libc"]

def is_elf(filepath: Path) -> bool:
    return lief.is_elf(str(filepath))

class RelroType(Enum):
    No = 1
    Partial = 2
    Full = 3

class PIEType(Enum):
    No = 1
    DSO = 2
    PIE = 3
    REL = 4

class Libc:
    def __init__(self, libpath: Path = None):
        if libpath is None:
            libpath = Path(find_libc())
            if not libpath:
                raise LibcNotFoundError
        logging.debug("Initializing Libc from %s", libpath)
        self.libc = lief.parse(str(libpath))
        if not self.libc:
            raise ErrorParsingFailed(libpath)

    @property
    @lru_cache()
    def fortified_symbols(self) -> FrozenSet[str]:
        """Get the list of libc symbols who have been fortified"""
        return frozenset({s.name for s in self.libc.symbols if s.name.endswith(FORTIFIED_END_MARKER)})

    @property
    @lru_cache()
    def fortified_symbols_base(self) -> FrozenSet[str]:
        """Get the list of fortified libc symbols, keeping only the function basename"""
        return frozenset(
            {sym[len(FORTFIED_START_MARKER) : -len(FORTIFIED_END_MARKER)] for sym in self.fortified_symbols}
        )

class ELFSecurity(BinarySecurity):
    def __init__(self, elf_path: Path):
        super().__init__(elf_path)
        self._libc = get_libc()

    @property
    @lru_cache()
    def set_dyn_syms(self) -> FrozenSet[str]:
        return frozenset(f.name for f in self.bin.dynamic_symbols)

    @property
    def relro(self) -> RelroType:
        if self.bin.get(lief.ELF.SEGMENT_TYPES.GNU_RELRO) is None:
            return RelroType.No

        flags = self.bin.get(lief.ELF.DYNAMIC_TAGS.FLAGS)
        if flags is None:
            bind_now = False
        else:
            bind_now = lief.ELF.DYNAMIC_FLAGS.BIND_NOW in flags

        flags_1 = self.bin.get(lief.ELF.DYNAMIC_TAGS.FLAGS_1)
        if flags_1 is None:
            now = False
        else:
            now = lief.ELF.DYNAMIC_FLAGS_1.NOW in flags_1

        if bind_now or now:
            return RelroType.Full
        else:
            return RelroType.Partial

    @property
    def has_canary(self) -> bool:
        canary_sections = ["__stack_chk_fail", "__intel_security_cookie"]
        for section in canary_sections:
            try:
                if self.bin.get_symbol(section):
                    return True
            except lief.not_found:
                pass
        return False

    @property
    def pie(self) -> PIEType:
        if self.bin.header.file_type == E_TYPE.DYNAMIC:
            if self.bin.has(lief.ELF.DYNAMIC_TAGS.DEBUG):
                return PIEType.PIE
            else:
                return PIEType.DSO
        elif self.bin.header.file_type == E_TYPE.RELOCATABLE:
            return PIEType.REL
        return PIEType.No

    @property
    def has_rpath(self) -> bool:
        try:
            if self.bin.get(lief.ELF.DYNAMIC_TAGS.RPATH):
                return True
        except lief.not_found:
            pass
        return False

    @property
    def has_runpath(self) -> bool:
        try:
            if self.bin.get(lief.ELF.DYNAMIC_TAGS.RUNPATH):
                return True
        except lief.not_found:
            pass
        return False

    @property
    @lru_cache()
    def symbols(self) -> List[str]:
        return [symbol.name for symbol in self.bin.static_symbols]

    @property
    def is_stripped(self) -> bool:
        # TODO: hwo to reset static_symbols iterator for the next call to symbols() ?
        # consumes only the first symbol from iterator, saving CPU cycles
        try:
            next(self.bin.static_symbols)
        except StopIteration:
            return True
        else:
            return False

    @property
    def is_fortified(self) -> bool:
        return True if self.fortified else False

    @property
    @lru_cache()
    def fortified(self) -> Optional[FrozenSet[str]]:
        """Get the list of fortified symbols"""
        if not self._libc:
            return None
        return self.set_dyn_syms & self._libc.fortified_symbols

    @property
    @lru_cache()
    def fortifiable(self) -> Optional[FrozenSet[str]]:
        """Get the list of fortifiable symbols (fortified + unfortified)"""
        if not self._libc:
            return None
        return self.set_dyn_syms & (self.fortified | self._libc.fortified_symbols_base)

    @property
    def checksec_state(self) -> ELFChecksecData:
        fortify_source = None
        fortified_count = None
        fortifiable_count = None
        score = None
        if self._libc:
            fortified_count = len(self.fortified)
            fortifiable_count = len(self.fortifiable)
            if not self.is_fortified:
                score = 0
            else:
                # fortified
                if fortified_count == 0:
                    # all fortified !
                    score = 100
                else:
                    score = (fortified_count * 100) / fortifiable_count
                    score = round(score)
            fortify_source = True if fortified_count != 0 or fortifiable_count == 0 else False
        return ELFChecksecData(
            relro=self.relro,
            canary=self.has_canary,
            nx=self.has_nx,
            pie=self.pie,
            rpath=self.has_rpath,
            runpath=self.has_runpath,
            symbols=not self.is_stripped,
            fortify_source=fortify_source,
            fortified=fortified_count,
            fortifiable=fortifiable_count,
            fortify_score=score,
        )


# Count the Number of matching characters in a pair of string

def count_number_matching(string1, string2):
    matching_chars = []
    for char in string1:
        for match in string2:
            if match == char:
                if match not in matching_chars:
                    matching_chars.append(match)
    return {
        'output': len(matching_chars),
        'matching': matching_chars
    }

user_string = input('Enter the first string : ')
user_string_2 = input('Enter the second string : ')
result = count_number_matching(user_string, user_string_2)

print(f'The count of matching characters : {result["output"]}')
print(f'The matching characters are : {result["matching"]}')


#!/usr/bin/env python3
""" module test_utils.py """

import unittest
from unittest.mock import patch, Mock
from parameterized import parameterized
from typing import Any, Dict, Tuple, Union
from utils import access_nested_map, get_json, memoize

class TestAccessNestedMap(unittest.TestCase):
    """Unit tests for the access_nested_map function."""
    @parameterized.expand([
        ({"a": 1}, ("a",), 1),
        ({"a": {"b": 2}}, ("a",), {"b": 2}),
        ({"a": {"b": 2}}, ("a", "b"), 2),
    ])
    def test_access_nested_map(
        self,
        nested_map: Dict[str, Any],
        path: Tuple[str, ...],
        expected_output: Any
    ) -> None:
        """
        Test that the access_nested_map function returns the expected result.
        Args:
            nested_map: The nested map to access.
            path: The path to the value in the nested map.
            expected_output: The expected result of accessing the nested map.
        Returns:
            None.
        """
        self.assertEqual(access_nested_map(nested_map, path), expected_output)

    @parameterized.expand([
        ({}, ("a",), KeyError),
        ({"a": 1}, ("a", "b"), KeyError),
    ])
    def test_access_nested_map_exception(
        self,
        nested_map: Dict[str, Any],
        path: Tuple[str, ...],
        exception_msg: Exception
    ) -> None:
        """
        Test that a KeyError is raised when accessing a non-existent key
         in the nested map.
        Args:
            nested_map: The nested map to access.
            path: The path to the value in the nested map.
            exception_msg: The expected exception message.
        Returns:
            None.
        """
        with self.assertRaises(exception_msg):
            access_nested_map(nested_map, path)

class TestGetJson(unittest.TestCase):
    """Unit tests for the get_json function."""
    @parameterized.expand([
        ("http://example.com", {"payload": True}),
        ("http://holberton.io", {"payload": False}),
    ])
    @patch('test_utils.get_json')
    def test_get_json(
        self,
        test_url: str,
        test_payload: Dict[str, Any],
        mock_get: Any
    ) -> None:
        """
        Test that get_json returns the expected result.
        Args:
            test_url: The URL to use in the test.
            test_payload: The expected payload to be returned.
            mock_get: Mocked requests.get object.
        Returns:
            None.
        """
        mock_get.return_value = test_payload
        result = get_json(test_url)
        self.assertEqual(result, test_payload)

class TestMemoize(unittest.TestCase):
    """Unit tests for the memoize decorator."""
    class TestClass:
        """Class to test the memoize decorator."""
        def a_method(self) -> int:
            return 42

        @memoize
        def a_property(self) -> int:
            return self.a_method()

    def test_memoize(self) -> None:
        """
        Test that the memoize decorator caches the result and avoids
        redundant method calls.
        """
        obj = self.TestClass()
        with patch.object(obj, 'a_method') as mock_method:
            result1: int = obj.a_property
            result2: int = obj.a_property
            mock_method.assert_called_once()
            self.assertEqual(result1, result2)


#!/usr/bin/env python2

from itertools import groupby
import json
import os

import cv2

from dataset_converters.ConverterBase import ConverterBase

class TDG2COCOConverter(ConverterBase):

    formats = ['TDGSEGM2COCO', 'TDG2COCO']

    def __init__(self, copy_fn):
        ConverterBase.__init__(self, copy_fn)

    def _read_annotations(self, filename):
        with open(filename, 'r') as f:
            lines = [l.strip().split(' ') for l in f.readlines()]
        instances = {}

        def group(lst, n):
            return zip(*[lst[i::n] for i in range(n)])

        for line in lines:
            filename = line[0]
            instances[filename] = []
            for instance in group(line[1:], 5):
                bbox = [int(s) for s in instance[1:]]
                x, y, w, h = bbox
                segmentation = [[x, y, x + w, y, x + w, y + h, x, y + h]]
                instances[filename].append({'id': int(instance[0]), 'bbox': bbox, 'segmentation': segmentation})

        return instances

    def _read_annotations_with_segmentations(self, filename):
        with open(filename, 'r') as f:
            lines = [l.strip().split(' ') for l in f.readlines()]
        instances = {}
        for line in lines:
            filename = line[0]
            instances[filename] = []
            label = int(line[1])
            next_label = -1
            for instance in [list(group) for k, group in groupby(line[2:], lambda x: x == 'segm') if not k]:
                assert(label >= 0)
                # If the number of elements is odd there is a label for next instance
                if (len([i for i in instance if i.isdigit()]) % 2 == 1):
                    next_label = int(instance[-1])
                    instance = instance[:-1]
                components = []
                while(len(instance) > 0):
                    if '|' in instance:
                        index = instance.index('|')
                    else:
                        index = len(instance)

                    component = [int(s) for s in instance[:index]]
                    instance = instance[index+1:]
                    min_x = min(component[::2])
                    max_x = max(component[::2])
                    min_y = min(component[1::2])
                    max_y = max(component[1::2])
                    components.append(component)

                bbox = [min_x, min_y, max_x - min_x, max_y - min_y]
                instances[filename].append({'id': label, 'bbox': bbox, 'segmentation': components})
                label = next_label

        return instances

    def _read_labels(self, filename):
        with open(filename, 'r') as f:
            lines = [l.strip().split(' ') for l in f.readlines()]
        labels = []
        for line in lines:
            labels.append({'supercategory': 'none', 'id': int(line[0]), 'name': line[1]})

        return labels

    def _run(self, input_folder, output_folder, FORMAT):
        images_folder = os.path.join(output_folder, 'train')
        annotations_folder = os.path.join(output_folder, 'annotations')

        self._ensure_folder_exists_and_is_clear(output_folder)
        self._ensure_folder_exists_and_is_clear(images_folder)
        self._ensure_folder_exists_and_is_clear(annotations_folder)

        labels_file = os.path.join(input_folder, 'labels.txt')

        if FORMAT == 'TDGSEGM2COCO':
            instances = self._read_annotations_with_segmentations(os.path.join(input_folder, 'segms.txt'))
        else:
            instances = self._read_annotations(os.path.join(input_folder, 'bboxes.txt'))
        labels = self._read_labels(labels_file)

        to_dump = {'images': [], 'type': 'instances', 'annotations': [], 'categories': labels}
        image_counter = 1
        instance_counter = 1
        for filename, single_image_instances in instances.items():
            full_image_path = os.path.join(input_folder, filename)
            image = cv2.imread(full_image_path)
            to_dump['images'].append(
                {
                    'file_name': filename,
                    'height': image.shape[0],
                    'width': image.shape[1],
                    'id': image_counter
                }
            )
            for instance in single_image_instances:
                _, _, w, h = instance['bbox']
                to_dump['annotations'].append(
                    {
                        'segmentation': instance['segmentation'],
                        'area': w * h,
                        'iscrowd': 0,
                        'image_id': image_counter,
                        'bbox': instance['bbox'],
                        'category_id': instance['id'],
                        'id': instance_counter,
                        'ignore': 0
                    }
                )
                instance_counter += 1
            self.copy(full_image_path, images_folder)
            image_counter += 1

        with open(os.path.join(annotations_folder, 'train.json'), 'w') as f:
            json.dump(to_dump, f, indent=4)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right

# Flattern a tree, result in a linked-list, with root.right point to the next node

# It's not hard to flattern a sub-tree, but the difficulty is how to make the last node of this sub-tree
# point to the right flatterned tree

# Solution: Return the last node every time when flatterning the tree
class Solution:
    def flattern_tree_and_return_last(self, root):
        if root is None:
            return None

        left_last = self.flattern_tree_and_return_last(root.left)
        right_last = self.flattern_tree_and_return_last(root.right)

        if left_last:
            left_last.right = root.right
            root.right = root.left
            root.left = None

        return right_last or left_last or root

    def flatten(self, root: TreeNode) -> None:
        """
        Do not return anything, modify root in-place instead.
        """
        self.flattern_tree_and_return_last(root)


import cv2
from Sem5_miniproj_Fruit_detection_and_classification import fruittracker, HTracker
import Speak

if __name__ =='__main__':
    # ft = FruitTracker(model_path="C:\\Users\\hp\\Desktop\\yolo\\yolov7\\runs\\train\\yolov7\\weights\\epoch_049.pt",repo="C:\\Users\\hp\\Desktop\\yolo\\yolov7")
    ft = fruittracker.FruitTracker()
    hd = HTracker.HandDetector(detectionCon=0.4, max_hands=1)
    sp = Speak.speak
    cap= cv2.VideoCapture(0)
    while True:
        success, img = cap.read()
        # img = cv2.flip(img, -1)
        img,stx,sty,endx,endy,pred = ft.predict(img,True)
        img,hlandmarks = hd.detectHands(img, True)
        # print(hlandmarks)
        if pred!=[]:
            img, direction = hd.withinRegionAndHandNavigator(img, hlandmarks[0], hlandmarks[5], hlandmarks[17],
                                                           [int(stx[0]), int(sty[0]), int(endx[0]), int(endy[0])])
            print(pred,direction)
            sp(direction)

        cv2.imshow("Image", img)
        cv2.waitKey(1)


import socket
import select
import errno
import os
import time
from threading import Thread
import json
import hashlib
import sys
import hashlib
import datetime
from pathlib import Path
from _thread import *

# get configurations
config = json.load(open(f"{os.path.dirname(os.path.abspath(__file__))}/config.json"))

CLIENT_ID = int(os.path.basename(Path(os.path.realpath(__file__)).parent).split('_')[1])
IP = config['client']['ip_address']
PORT = config['server']['ports'][CLIENT_ID]
HEADER_LENGTH = config['header_length']
META_LENGTH = config['meta_length']
THREAD_PORTS = config['client']['ports']
DOWNLOAD_FOLDER_NAME = config['client']['download_folder_name']
REDOWNLOAD_TIME = config['redownload_times']
LOG = open(f"{os.path.dirname(os.path.abspath(__file__))}/{config['client']['log_file']}", "a")

# Logs messages
def log_this(msg):
    # print(msg)
    LOG.write(f"{datetime.datetime.now()} {msg}\n")
    LOG.flush()

# Verbose function
def help():
    print("\n*** INFO ON FUNCTIONS ***\n")
    print("[function_name] [options] [parameters] - [description]\n")
    print("get_files_list - gets the file names from all the clients\n")
    print("download -p <client_number> <file_name> ... <file_name> - downloads one or more files serially or parallely from the target client. To download serially, use it without the -p option\n")
    print("help - prints verbose for functions\n")
    print("quit - exits client interface\n")

# send updated directory to server
def update_server():
    try:
        list_of_dir = os.listdir(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/")
        list_of_dir = '\n'.join(list_of_dir)
        list_of_dir = f"update_list {list_of_dir}".encode('utf-8')
        list_of_dir_header = f"{len(list_of_dir):<{HEADER_LENGTH}}".encode('utf-8')
        meta = f"{f'{CLIENT_ID}':<{META_LENGTH}}".encode('utf-8')
        central_socket.send(list_of_dir_header + meta + list_of_dir)

    except:
        # client closed connection, violently or by user
        return False

# daemon that updates the directory to the server whenever a new file is added or an file is deleted
def folder_watch_daemon(current_file_directory):
    while True:
        temp = os.listdir(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/")
        if current_file_directory != temp:
            update_server()
            current_file_directory = temp

# Waiting for a list of directories from the server
def wait_for_list(full_command):
    # Encode command to bytes, prepare header and convert to bytes, like for username above, then send
    full_command = full_command.encode('utf-8')
    command_header = f"{len(full_command):<{HEADER_LENGTH}}".encode('utf-8')
    meta = f"{'':<{META_LENGTH}}".encode('utf-8')
    central_socket.send(command_header + meta + full_command)

    log_this(f"Client sent command: {full_command}")

    start = time.time()

    # Keep trying to recieve until client recieved returns from the server
    while True:
        try:
            # Receive our "header" containing username length, it's size is defined and constant
            header = central_socket.recv(HEADER_LENGTH)

            # If we received no data, server gracefully closed a connection, for example using socket.close() or socket.shutdown(socket.SHUT_RDWR)
            if not len(header):
                log_this(f"Connection closed by the server")
                return

            # Convert header to int value
            header = int(header.decode('utf-8').strip())

            # Get meta data
            meta = central_socket.recv(META_LENGTH)
            meta = meta.decode('utf-8').strip()

            # Receive and decode msg
            dir_list = central_socket.recv(header).decode('utf-8')
            dir_list = json.loads(dir_list)

            # Print List
            for client in dir_list:
                print(f"Client with id {client}:")
                for file in dir_list[client]:
                    print(f"\t{file}")

            end = time.time()

            log_this(f"FileQueryComplete: {(end-start)*1000} ms.")
            # Break out of the loop when list is recieved
            break

        except IOError as e:

            if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:
                log_this('Reading error: {}'.format(str(e)))
                return

            # We just did not receive anything
            continue

        except Exception as e:
            # Any other exception - something happened, exit
            log_this('Reading error: '.format(str(e)))
            return

# Handles command receiving
def receive_command(client_socket):

    try:

        # Receive our "header" containing command length, it's size is defined and constant
        command_header = client_socket.recv(HEADER_LENGTH)

        # If we received no data, client gracefully closed a connection, for example using socket.close() or socket.shutdown(socket.SHUT_RDWR)
        if not len(command_header):
            return False

        # Get meta data
        meta = client_socket.recv(META_LENGTH)
        meta = meta.decode('utf-8').strip()

        # Convert header to int value
        command_length = int(command_header.decode('utf-8').strip())

        # Return an object of command header and command data
        return {'header': command_header, 'meta': meta, 'data': client_socket.recv(command_length)}

    except:
        # client closed connection, violently or by user
        return False

# waiting function for parallelized/serial file download
def parallelize_wait_for_file_download(client_socket, files):

    # Encode command to bytes, prepare header and convert to bytes, like for username above, then send
    full_command = f"download {' '.join(files)}".encode('utf-8')
    command_header = f"{len(full_command):<{HEADER_LENGTH}}".encode('utf-8')
    meta = f"{'':<{META_LENGTH}}".encode('utf-8')
    client_socket.send(command_header + meta + full_command)

    log_this(f"Client sent command: {full_command}")

    # open files
    fds = [open(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/{files[i]}",'w') for i in range(len(files))]
    files_closed = 0
    redownload_count = 0

    # md5 reconstruction
    m = [hashlib.md5() for _ in range(len(files))]

    # Keep trying to recieve until client recieved returns from the server
    while True:
        try:
            start = time.time()
            header = client_socket.recv(HEADER_LENGTH)

            # If we received no data, server gracefully closed a connection, for example using socket.close() or socket.shutdown(socket.SHUT_RDWR)
            if not len(header):
                log_this('Connection closed by the server')
                return

            # Convert header to int value
            header = int(header.decode('utf-8').strip())

            # Get meta data
            meta = client_socket.recv(META_LENGTH)
            meta = meta.decode('utf-8').strip()
            meta = meta.split(' ')

            # Recieve line and convert to string
            line = client_socket.recv(header).decode('utf-8')

            # if there is any error, remove all files
            if meta[0] == 'ERROR':
                log_this(line)

                for i in range(len(files)):
                    fds[i].flush()
                    fds[i].close()
                    os.remove(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/{files[i]}")
                break

            # Flush and close and files is finished recieving
            elif meta[0] == 'END':
                fds[int(meta[1])].flush()
                fds[int(meta[1])].close()
                files_closed += 1

                # if there is contamination in the checksum, log and delete file
                if m[int(meta[1])].hexdigest() != line:
                    log_this(f"Incorrect checksum for file : {files[int(meta[1])]}")
                    log_this(f"Deleting file : {files[int(meta[1])]}")
                    os.remove(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/{files[int(meta[1])]}")

            # continue to write and flush to files
            else:
                m[int(meta[0])].update(line.encode('utf-8'))
                fds[int(meta[0])].write(line)
                fds[int(meta[0])].flush()

            # when all files are closed/downloaded sucessfully then we can break from the loop
            if files_closed == len(fds):
                end = time.time()
                log_this(f"DownloadComplete: {(end-start)*1000} ms. Downloaded Files are {' '.join(files)}")
                break

        except IOError as e:

            if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:

                if redownload_count < REDOWNLOAD_TIME:
                    redownload_count += 1
                    continue

                log_this('Reading error: {}'.format(str(e)))
                return

            # We just did not receive anything
            continue

        except Exception as e:
            # Any other exception - something happened, exit
            log_this('Reading error: {}'.format(str(e)))
            return

# Waiting for the file contents from the server
def wait_for_file_download(full_command):

    parameters = full_command.split(' ')[1:]

    # check for parallelism option
    if parameters[0] == '-p':
        if len(parameters) <= 2:
            log_this("ParameterError: Too less parameters")
            return
        else:
            parallelize = True
            target_client = int(parameters[1])
            files = parameters[2:]
    else:
        if len(parameters) <= 1:
            log_this("ParameterError: Too less parameters")
            return
        else:
            parallelize = False
            target_client = int(parameters[0])
            files = parameters[1:]

    # if the target client is itself, don't do anything
    if target_client == CLIENT_ID:
        log_this("WrongClient: Target Client is Current Client.")
        return

    # Compute ports that 'this' peer will try to connect
    client_thread_ports = [i+((len(config['client']['ports'])+1)*target_client) for i in THREAD_PORTS]

    # initialize connections with the other peer
    client_sockets = []

    for i in range(len(client_thread_ports)):
        temp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        temp.connect((IP, client_thread_ports[i]))
        temp.setblocking(False)

        # Initialize conneciton with the server
        temp.send(username_header + meta + username)
        client_sockets.append(temp)

    # starts waiting for file download
    if parallelize:
        for i in range(0, len(files), len(client_thread_ports)):
            thread_idx = 0
            threads = []
            for j in range(i,i+ len(client_thread_ports)):
                if j < len(files):
                    t = Thread(target=parallelize_wait_for_file_download, args=(client_sockets[thread_idx], [files[j]],))
                    t.start()
                    threads.append(t)
                    thread_idx += 1

            for t in threads:
                t.join()
    else:
        # start = time.time()
        t = Thread(target=parallelize_wait_for_file_download, args=(client_sockets[0], files,))
        t.start()

        # end = time.time()
        # log_this(f"DownloadComplete: {(end-start)*1000} ms. Downloaded Files are {' '.join(files)}")

    return

# Sends file to the peer
def send_files(peer_socket, peers, files):
    try:
        fds = [open(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/{files[i]}",'r') for i in range(len(files))]

        for i in range(len(files)):
            # using md5 checksum
            m = hashlib.md5()

            while True:

                # read line
                line = fds[i].readline()

                if not line:
                    line = m.hexdigest().encode('utf-8')
                    line_header = f"{len(line):<{HEADER_LENGTH}}".encode('utf-8')
                    meta = f"{f'END {i}':<{META_LENGTH}}".encode('utf-8')
                    peer_socket.send(line_header + meta + line)

                    log_this(f"{files[i]} was sent to {peers[peer_socket]['data']}")
                    break

                line = line.encode('utf-8')

                # update md5 checksum
                m.update(line)

                line_header = f"{len(line):<{HEADER_LENGTH}}".encode('utf-8')
                meta = f"{f'{i}':<{META_LENGTH}}".encode('utf-8')
                peer_socket.send(line_header + meta + line)

            fds[i].close()

    except Exception as e:
        # client closed connection, violently or by user
        error = str(e).encode('utf-8')
        header = f"{len(error):<{HEADER_LENGTH}}".encode('utf-8')
        meta = f"{'ERROR':<{META_LENGTH}}".encode('utf-8')
        peer_socket.send(header + meta + error)
        return False

# A daemon that listens for download requests from any other clients/peers
def server_daemon():
    # Create list of listening server sockets
    server_sockets = []

    # Compute ports that 'this' peer will listen to as a server
    server_thread_ports = [i+((len(config['client']['ports'])+1)*CLIENT_ID) for i in THREAD_PORTS]

    for i in range(len(server_thread_ports)):
        temp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        temp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        temp.bind((IP, server_thread_ports[i]))
        temp.listen()
        server_sockets.append(temp)

    # Create list of sockets for select.select()
    sockets_list = [i for i in server_sockets]

    # List of connected clients - socket as a key, user header and name as data
    peers = {}

    for port in server_thread_ports:
        log_this(f'Listening for connections on {IP}:{port}...')

    while True:
        read_sockets, _, exception_sockets = select.select(sockets_list, [], sockets_list)

        for notified_socket in read_sockets:

            # If notified socket is a server socket - new connection, accept it
            if notified_socket in server_sockets:

                client_socket, client_address = server_sockets[server_sockets.index(notified_socket)].accept()

                # Client should send his name right away, receive it
                user = receive_command(client_socket)

                # If False - client disconnected before he sent his name
                if user is False:
                    continue

                # Add accepted socket to select.select() list
                sockets_list.append(client_socket)

                # Also save username and username header
                peers[client_socket] = user

                #logging
                log_msg = 'Accepted new connection from {}:{}, username: {}'.format(*client_address, user['data'].decode('utf-8'))
                log_this(log_msg)

            # Else existing socket is sending a command
            else:

                # Recieve command
                command = receive_command(notified_socket)

                # If False, client disconnected, cleanup
                if command is False:
                    log_msg = '{} Closed connection from: {}'.format(datetime.datetime.now(), peers[notified_socket]['data'].decode('utf-8'))
                    log_this(log_msg)

                    # remove connections
                    sockets_list.remove(notified_socket)
                    del peers[notified_socket]
                    continue

                # Get user by notified socket, so we will know who sent the command
                user = peers[notified_socket]

                # Get command
                command_msg = command["data"].decode("utf-8")
                command_msg = command_msg.split(' ')

                # logging
                log_msg = f'{datetime.datetime.now()} Recieved command from {user["data"].decode("utf-8")}: {command_msg[0]}\n'
                log_this(log_msg)

                # Handle commands
                if command_msg[0] == 'download':
                    start_new_thread(send_files, (notified_socket,peers,command_msg[1:],))

            # handle some socket exceptions just in case
            for notified_socket in exception_sockets:

                # remove connections
                sockets_list.remove(notified_socket)
                del peers[notified_socket]

if __name__ == "__main__":

    # Start the peer's server daemon
    start_new_thread(server_daemon,())

    # allow time for the server daemon to start up
    time.sleep(3)

    # Manual Mode of Client Interface
    if len(sys.argv) == 1:

        # create username to connect to the server
        my_username = input("Username: ")
        username = my_username.encode('utf-8')
        username_header = f"{len(username):<{HEADER_LENGTH}}".encode('utf-8')
        meta = f"{'':<{META_LENGTH}}".encode('utf-8')

        # Initialize connection with the server with central socket
        central_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        central_socket.connect((IP, PORT))
        central_socket.setblocking(False)
        central_socket.send(username_header + meta + username)

        # Initialize file directory to server
        update_server()

        # get current file directories
        current_file_directory = os.listdir(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/")

        # Start folder watch daemon to automatically update to server
        start_new_thread(folder_watch_daemon,(current_file_directory,))

        # Print verbose client shell begins
        help()

        # Does Client Things
        while True:

            # Wait for user to input a command
            full_command = input(f'{my_username} > ').strip()
            command = full_command.split(' ')[0]
            parameters = full_command.split(' ')[1:]

            if command == "download":
                if len(parameters) != 0:
                    wait_for_file_download(full_command)
                else:
                    log_this("ParameterError: Too less parameters")

            elif command == "get_files_list":
                if len(parameters) == 0:
                    wait_for_list(full_command)
                else:
                    log_this("ParameterError: Too many parameters")

            elif command == "help":
                help()

            elif command == "quit":
                # Encode command to bytes, prepare header and convert to bytes, like for username above, then send
                full_command = f"unregister".encode('utf-8')
                command_header = f"{len(full_command):<{HEADER_LENGTH}}".encode('utf-8')
                meta = f"{f'{CLIENT_ID}':<{META_LENGTH}}".encode('utf-8')
                central_socket.send(command_header + meta + full_command)

                log_this(f"Client sent command: {full_command}")
                LOG.close()
                sys.exit()

    # Automatic Mode of Client Interface

    #Args
    #python client.py username command1 command2 ... commandn

    else:

        # create username to connect to the server
        my_username = sys.argv[1]
        username = my_username.encode('utf-8')
        username_header = f"{len(username):<{HEADER_LENGTH}}".encode('utf-8')
        meta = f"{'':<{META_LENGTH}}".encode('utf-8')

        # Initialize connection with the server with central socket
        central_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        central_socket.connect((IP, PORT))
        central_socket.setblocking(False)
        central_socket.send(username_header + meta + username)

        # Initialize file directory to server
        update_server()

        # get current file directories
        current_file_directory = os.listdir(f"{os.path.dirname(os.path.abspath(__file__))}/{DOWNLOAD_FOLDER_NAME}/")

        # Start folder watch daemon to automatically update to server
        start_new_thread(folder_watch_daemon,(current_file_directory,))

        # Does Client Things
        for i in sys.argv[2:]:

            # give time to process command
            time.sleep(1)

            # Wait for user to input a command
            full_command = i
            command = full_command.split(' ')[0]
            parameters = full_command.split(' ')[1:]

            if command == "download":
                if len(parameters) != 0:
                    wait_for_file_download(full_command)
                else:
                    log_this("ParameterError: Too less parameters")

            elif command == "get_files_list":
                if len(parameters) == 0:
                    wait_for_list(full_command)
                else:
                    log_this("ParameterError: Too many parameters")

            elif command == "help":
                help()

            elif command == "quit":
                # Encode command to bytes, prepare header and convert to bytes, like for username above, then send
                full_command = f"unregister".encode('utf-8')
                command_header = f"{len(full_command):<{HEADER_LENGTH}}".encode('utf-8')
                meta = f"{f'{CLIENT_ID}':<{META_LENGTH}}".encode('utf-8')
                central_socket.send(command_header + meta + full_command)

                log_this(f"Client sent command: {full_command}")
                LOG.close()
                sys.exit()


from chalice import Chalice, CustomAuthorizer, CORSConfig, Response

app = Chalice(app_name='core-api-authorizers-sample')

WAITING_ROOM_AUTH = CustomAuthorizer('WaitingRoomAuthorizer',
                                     header='Authorization',
                                     authorizer_uri='PLACEHOLDER')

CORS_CONFIG = CORSConfig(allow_origin='*',
                         allow_headers=['*'],
                         max_age=600,
                         expose_headers=['*'],
                         allow_credentials=True)

RESPONSE_HEADERS = {
    'Content-Type': 'application/json',
    'Access-Control-Allow-Origin': '*'
}

@app.route('/checkout',
           methods=['GET'],
           authorizer=WAITING_ROOM_AUTH,
           cors=CORS_CONFIG)
def checkout():
    """
    This function represents a /checkout API endpoint.
    """
    return Response(status_code=200,
                    body={"result": "Success"},
                    headers=RESPONSE_HEADERS)

@app.route('/search',
           methods=['GET'],
           authorizer=WAITING_ROOM_AUTH,
           cors=CORS_CONFIG)
def search():
    """
    This function represents a /search API endpoint.
    """
    return Response(status_code=200,
                    body={"result": "Success"},
                    headers=RESPONSE_HEADERS)

@app.route('/layaway',
           methods=['GET'],
           authorizer=WAITING_ROOM_AUTH,
           cors=CORS_CONFIG)
def layaway():
    """
    This function represents a /layaway API endpoint.
    """
    return Response(status_code=200,
                    body={"result": "Success"},
                    headers=RESPONSE_HEADERS)


import json
import sys
from collections import defaultdict
from optparse import make_option
from urllib.parse import urlparse

import ckanapi
import pandas as pd
import lxml
from django.db.models import Count

from . import InventoryCommand
from inventory.models import Dataset, Distribution
from inventory.scrapers import CKAN

class Command(InventoryCommand):
    help = 'Analyzes catalogs'

    option_list = InventoryCommand.option_list + (
        make_option('--access', action='append_const', dest='reports', const='access',
                    help='Classification of direct download domain names, for import into R.'),
        make_option('--api', action='append_const', dest='reports', const='api',
                    help='Usage of catalog API technologies.'),
        make_option('--dcat', action='append_const', dest='reports', const='dcat',
                    help='Usage of DCAT by CKAN catalogs.'),
        make_option('--pod', action='append_const', dest='reports', const='pod',
                    help='Usage of Project Open Data Metadata Schema.'),
        make_option('--schemaorg', action='append_const', dest='reports', const='schemaorg',
                    help='Usage of Schema.org.'),
        make_option('--federation', action='append_const', dest='reports', const='federation',
                    help='Usage of Federation technologies.'),
        make_option('--licenses', action='append_const', dest='reports', const='licenses',
                    help='Usage of licenses.'),
        make_option('--csv', action='store_const', dest='format', const='csv',
                    default='table',
                    help='Prints the results as CSV.'),
    )

    def handle(self, *args, **options):
        self.setup(*args, **options)

        for report in options['reports']:
            result = getattr(self, report)()
            if result is not None:
                if options['format'] == 'table':
                    result.to_string(sys.stdout)
                elif options['format'] == 'csv':
                    result.to_csv(sys.stdout)

    def series(self, getter):
        series = {}
        for catalog in self.catalogs:
            series[catalog.division_id] = getter(catalog)
        return pd.Series(series)

    def api(self):
        def getter(catalog):
            if issubclass(catalog.scraper, CKAN):
                try:
                    client = ckanapi.RemoteCKAN(catalog.url, get_only=catalog.get_only)
                    status_show = client.call_action('status_show', verify=catalog.verify)
                    return int('datastore' in status_show['extensions'])
                except ckanapi.errors.CKANAPIError:
                    pass

        return self.series(getter)

    def dcat(self):
        def getter(catalog):
            if issubclass(catalog.scraper, CKAN):
                datasets = Dataset.objects.filter(division_id=catalog.division_id)
                if datasets.exists():
                    response = self.get(catalog.dataset_url(datasets[0]))
                    if response.status_code == 200:
                        response = self.get(catalog.dataset_rdf_url(datasets[0]))
                        return int(response.status_code == 200)

        return self.series(getter)

    def pod(self):
        def getter(catalog):
            response = self.get(catalog.data_json_url)
            return int(response.status_code == 200)

        return self.series(getter)

    def schemaorg(self):
        def getter(catalog):
            datasets = Dataset.objects.filter(division_id=catalog.division_id)
            if datasets.exists():
                url = catalog.dataset_url(datasets[0])
                if url:
                    response = self.get(url)
                    if response.status_code == 200:
                        return int('http://schema.org/Dataset' in response.text)

        return self.series(getter)

    def access(self):
        frame = defaultdict(lambda: defaultdict(int))
        for catalog in self.catalogs:
            urls = Distribution.objects.filter(division_id=catalog.division_id).values_list('accessURL', flat=True)
            if urls:
                frame['count'][catalog.division_id] = urls.count()
                for url in urls:
                    host = urlparse(url).netloc.split(':', 1)[0]
                    if host.endswith(catalog_domains[catalog.division_id]):
                        key = 'Catalog'
                    elif any(host.endswith(suffix) for suffix in government_domains[catalog.division_id]):
                        key = 'Government'
                    else:
                        key = 'Other'
                        # frame[host][catalog.division_id] += 1
                    frame[key][catalog.division_id] += 1
        return pd.DataFrame(frame)

    def federation(self):
        frame = defaultdict(lambda: defaultdict(int))
        for catalog in self.catalogs:
            # Assumes we don't need to paginate.
            if issubclass(catalog.scraper, CKAN):
                client = ckanapi.RemoteCKAN(catalog.url, get_only=catalog.get_only)
                package_search = client.call_action('package_search', {'fq': 'type:harvest', 'rows': 300000}, verify=catalog.verify)

                if package_search['results']:
                    for package in package_search['results']:
                        source_type = self.source_type(catalog, package)
                        if source_type:
                            frame[source_type][catalog.division_id] += 1
                        else:
                            self.warning('could not determine source type of {}'.format(catalog.dataset_api_url(package)))
                # GB
                else:
                    try:
                        for package in client.call_action('harvest_source_list', verify=catalog.verify):
                            if package['active']:
                                source_type = normalize_source_type(package, package['type'])
                                if source_type:
                                    frame[source_type][catalog.division_id] += 1
                                else:
                                    self.warning('could not determine source type of {}'.format(catalog.harvest_api_url(package)))
                    except ckanapi.errors.CKANAPIError:
                        pass
            elif catalog.scraper.__name__ == 'Socrata':
                if 'federation_filter' in self.get(catalog.url).text:
                    frame['socrata'][catalog.division_id] = 1
        return pd.DataFrame(frame)

    def licenses(self):
        frame = defaultdict(lambda: defaultdict(int))
        for catalog in self.catalogs:
            for value in Dataset.objects.filter(division_id=catalog.division_id).values('license').annotate(count=Count('id', distinct=True)).order_by('count').iterator():
                frame[catalog.division_id][value['license']] = value['count']
        return pd.DataFrame(frame)

    def report(self, klass, field, *, distinct):
        for catalog in self.catalogs:
            count = Dataset.objects.filter(division_id=catalog.division_id).count()
            print('{} ({})'.format(catalog.division_id, count))
            for value in klass.objects.filter(division_id=catalog.division_id).values(field).annotate(count=Count(distinct, distinct=True)).order_by('count').iterator():
                print('  {:7.2%} {} ({})'.format(value['count'] / count, value[field], value['count']))

    def source_type(self, catalog, package):
        # AU, FI, IE, IT, MX, PY
        if package.get('source_type'):
            return normalize_source_type(package, package['source_type'])

        # IT
        elif '/api/rest/dataset/' in package['url']:
            url, name = package['url'].split('api/rest/dataset/', 1)
            return self.source_type(catalog, ckanapi.RemoteCKAN(url).call_action('package_show', {'id': name}))

        # US
        # @see https://github.com/ckan/ckanext-spatial/blob/master/doc/harvesters.rst
        # @see https://github.com/GSA/ckanext-geodatagov/tree/master/ckanext/geodatagov/harvesters
        elif package.get('extras'):
            source_type = next(extra['value'] for extra in package['extras'] if extra['key'] == 'source_type')
            # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/base.py#L174
            if source_type == 'single-doc':
                response = self.get(package['url'])
                if response.status_code == 200:
                    try:
                        return normalize_metadata_scheme(response)
                    except lxml.etree.XMLSyntaxError:
                        pass
            # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/waf_collection.py
            elif source_type == 'waf-collection':
                # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/validation/__init__.py
                config = json.loads(next(extra['value'] for extra in package['extras'] if extra['key'] == 'config'))
                if config.get('validator_profiles'):
                    if len(config['validator_profiles']) > 1:
                        self.warning('multiple validator_profiles for {}'.format(catalog.dataset_api_url(package)))
                    else:
                        return 'waf-{}'.format(validators[config['validator_profiles'][0]])
                else:
                    response = self.get(config['collection_metadata_url'])
                    if response.status_code == 200:
                        scheme = normalize_metadata_scheme(response)
                        if scheme:
                            return 'waf-{}'.format(scheme)
            else:
                normalized = normalize_source_type(package, source_type)
                if normalized:
                    return normalized

        # BR
        else:
            try:
                if ckanapi.RemoteCKAN(package['url']).call_action('site_read'):
                    return 'ckan'
            except ckanapi.errors.CKANAPIError:
                pass

def normalize_source_type(package, source_type):
    if source_type in source_types:
        return source_types[source_type]
    elif package['url'].endswith('/csw') or '/csw/' in package['url']:
        return 'csw'

def normalize_metadata_scheme(response):
    if 'FGDC-STD-001-1998' in response.text:
        return 'fgdc'
    elif lxml.etree.fromstring(response.content).xpath('/MD_Metadata|/gmi:MI_Metadata', namespaces={'gmi': 'http://www.isotc211.org/2005/gmi'}):
        return 'iso19139'

source_types = {
    # Dynamic API
    'arcgis': 'arcgis',
    'ckan': 'ckan',
    'csw': 'csw',
    'waf': 'waf',
    # GB
    'gemini-waf': 'waf-gemini',
    # US
    # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/base.py#L185
    'geoportal': 'csw',
    # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/z3950.py
    'z3950': 'iso23950',

    # Static file
    # GB
    'dcat_rdf': 'dcat_rdf',
    'data_json': 'dcat_json',
    'gemini-single': 'gemini',
    'inventory': 'datashare',
    # MX
    'dcat_json': 'pod',  # v1.0
    # US
    'datajson': 'pod',
}

validators = {
    'fgdc_minimal': 'fgdc',
    'iso19139ngdc': 'iso19139',
}

catalog_domains = {
    'ocd-division/country:ar': 'datospublicos.gob.ar',
    'ocd-division/country:au': 'data.gov.au',
    'ocd-division/country:br': 'dados.gov.br',
    'ocd-division/country:ca': 'data.gc.ca',
    'ocd-division/country:cl': 'datos.gob.cl',
    'ocd-division/country:cr': 'gobiernodigitalcr.cloudapi.junar.com',
    'ocd-division/country:ee': 'opendata.riik.ee',
    'ocd-division/country:es': 'datos.gob.es',
    'ocd-division/country:fi': 'avoindata.fi',
    'ocd-division/country:fr': 'data.gouv.fr',
    'ocd-division/country:gb': 'data.gov.uk',
    'ocd-division/country:gh': 'data.gov.gh',
    'ocd-division/country:gr': 'data.gov.gr',
    'ocd-division/country:id': 'data.id',
    'ocd-division/country:ie': 'data.gov.ie',
    'ocd-division/country:it': 'dati.gov.it',
    'ocd-division/country:ke': 'opendata.go.ke',
    'ocd-division/country:md': 'data.gov.md',
    'ocd-division/country:mx': 'datos.gob.mx',
    'ocd-division/country:nl': 'data.overheid.nl',
    'ocd-division/country:ph': 'data.gov.ph',
    'ocd-division/country:py': 'datos.gov.py',
    'ocd-division/country:ro': 'data.gov.ro',
    'ocd-division/country:se': 'oppnadata.se',
    'ocd-division/country:sk': 'data.gov.sk',
    'ocd-division/country:tz': 'opendata.go.tz',
    'ocd-division/country:us': 'data.gov',
    'ocd-division/country:uy': 'catalogodatos.gub.uy',
}

# We only categorize the most popular domains, i.e. >=0.1% of distributions.
government_domains = {
    'ocd-division/country:ar': ['.gob.ar'],
    'ocd-division/country:au': ['.gov.au'],
    'ocd-division/country:br': [
        '.gov.br',
        '.jus.br',
        '.leg.br',
    ],
    'ocd-division/country:ca': [
        '.gc.ca',
        'geobase.ca',  # Natural Resources Canada
    ],
    'ocd-division/country:cl': ['.gob.cl'],
    'ocd-division/country:cr': ['.go.cr'],
    'ocd-division/country:ee': [
        '.riik.ee',  # "state"
        '.muinas.ee',  # http://et.wikipedia.org/wiki/Muinsuskaitseamet
    ],
    'ocd-division/country:es': [
        '.gob.es',
        '.cnig.es',  # Centro Nacional de Información Geográfica
        '.chj.es',  #  Confederación Hidrográfica del Júcar
        '.agenciatributaria.es',  # Agencia Tributaria
        '.fega.es',  # Fondo Español de Garantía Agraria
        '.ign.es',  #  Instituto Geográfico Nacional
        '.imserso.es',  # Instituto de Mayores y Servicios Sociales
        '.ine.es',  # Instituto Nacional de Estadística
        '.ipyme.org',  # Dirección General de Industria y de la Pequeña y Mediana Empresa
        '.cis.es',  # Centro de investigaciones sociológicas
    ],
    'ocd-division/country:fi': [  # 0.005
        '.gtk.fi',  # Geological Survey of Finland
        '.liikennevirasto.fi',  # Finnish Transport Agency
        '.maanmittauslaitos.fi',  # National Land Survey
        '.paikkatietoikkuna.fi',  # National Land Survey
        '.sfs.fi',  # Finnish Standards Association
        '.stat.fi',  # Statistics Finland
        '.vahtiohje.fi',  # Ministry of Finance
    ],
    'ocd-division/country:fr': ['.gouv.fr'],
    'ocd-division/country:gb': [
        '.gov.uk',
        '.hpa.org.uk',  # Public Health England
        '.isdscotland.org',  # Scottish Government
        '.nhs.uk',  # Department of Health
        '.ordnancesurvey.co.uk',  # Department for Business, Innovation and Skills
        '.slc.co.uk',  # http://en.wikipedia.org/wiki/Student_Loans_Company
        '.uktradeinfo.com',  # HM Revenue and Customs
        'opendatacommunities.org',  # Department for Communities and Local Government
    ],
    'ocd-division/country:gh': ['.gov.gh'],
    'ocd-division/country:gr': [  # 0.005
        '.gov.gr',
        '.astynomia.gr',  # http://en.wikipedia.org/wiki/Hellenic_Police
        '.ekdd.gr',  # http://en.wikipedia.org/wiki/National_Centre_for_Public_Administration_and_Local_Government
    ],
    'ocd-division/country:id': ['.go.id'],
    'ocd-division/country:ie': [  # 0.005
        '.gov.ie',
        '.marine.ie',  # Marine Institute
        '.cso.ie',  # Central Statistics Office Ireland
        '.hea.ie',  # Higher Education Authority
        '.buildingsofireland.ie',  # Department of Arts, Heritage and the Gaeltacht
        '.epa.ie',  # Environmental Protection Agency
        '.education.ie',  # Department of Education and Skills
        '.environ.ie',  # Department of the Environment, Community and Local Government
        '.fishingnet.ie',  # Department of Agriculture, Food and the Marine
        '.gsi.ie',  # Geological Survey of Ireland
        '.infomar.ie',  # Geological Survey of Ireland / Marine Institute
        '.seai.ie',  # Sustainable Energy Authority
    ],
    'ocd-division/country:it': [
        '.gov.it',
        '.inail.it',  # Istituto Nazionale Assicurazione contro gli Infortuni sul Lavoro
        '.inps.it',  # Istituto Nazionale Previdenza Sociale
        '.istat.it',  # Istituto nazionale di statistica
        '.politicheagricole.it',  # Ministero delle politiche agricole alimentari e forestali
    ],
    'ocd-division/country:ke': [
        '.go.ke',
        '.kenyalaw.org',  # National Council for Law Reporting
    ],
    'ocd-division/country:md': [
        '.gov.md',
        '.statistica.md',  # Biroul Naţional de Statistică al Republicii Moldova
        '.knbs.or.ke',  # Kenya National Bureau of Statistics
    ],
    'ocd-division/country:mx': [
        '.gob.mx',
        '.nafin.com',  # Nacional Financiera, state bank
        '.pemex.com',  # Petróleos Mexicanos, state-owned
    ],
    'ocd-division/country:nl': [
        '.duo.nl',  # Ministerie van Onderwijs, Cultuur en Wetenschap
        '.gdngeoservices.nl',  # Geologische Dienst Nederland
        '.kaartenbalie.nl',  # Ministerie van Economische Zaken
        '.kadaster.nl',  # Cadastre, Land Registry and Mapping Agency
        '.knmi.nl',  # Ministry of Infrastructure and the Environment
        '.nationaalgeoregister.nl',
        '.overheid.nl', # "government"
        '.pbl.nl',  # Planbureau voor de Leefomgeving
        '.rijkswaterstaat.nl',  # Ministerie van Infrastructuur en Milieu
        '.risicokaart.nl',
        '.rivm.nl',  # Ministerie van Volksgezondheid, Welzijn en Sport
        '.rwsgeoweb.nl',
        '.waterschapservices.nl',
    ],
    'ocd-division/country:ph': ['.gov.ph'],
    'ocd-division/country:py': ['.gov.py'],
    'ocd-division/country:ro': [  # 0.005
        '.gov.ro',
        '.edu.ro',  # Ministerul Educației Naționale
        '.inforegio.ro',  # Ministry of Regional Development and Public Administration
        '.mfinante.ro',  # Ministerul Finanţelor Publice
        '.mmuncii.ro',  # Ministerul Muncii, Familiei, Protecţiei Sociale şi Persoanelor Vârstnice
    ],
    'ocd-division/country:se': [  # 0.025
        '.gov.se',
        '.sgu.se',  # http://en.wikipedia.org/wiki/Geological_Survey_of_Sweden
        '.socialstyrelsen.se',  # http://en.wikipedia.org/wiki/National_Board_of_Health_and_Welfare_%28Sweden%29
        '.verksamt.se',
    ],
    'ocd-division/country:sk': [  # 0.005
        '.gov.sk',
        '.justice.sk',  # Ministerstvo Spravodlivosti
        '.mfa.sk',  # Ministry of Foreign and European Affairs of the Slovak Republic
        '.mhsr.sk',  # Ministry of Economy of the Slovak Republic
        '.minv.sk',  # Ministry of Interior of the Slovak Republic
        '.mosr.sk',  # Ministry of Defense of Slovak Republic
        '.mpsr.sk',  # Ministry of Agriculture and Rural Development of the Slovak Republic
        '.nrsr.sk',  # National Council of the Slovak Republic
        '.sazp.sk',  #  Slovenská agentúra životného prostredia
        '.skgeodesy.sk',  # Geodesy, Cartography and Cadastre Authority of Slovak Republic
        '.statistics.sk',  # Statistical Office of the Slovak Republic
        '.svssr.sk',  # State Veterinary and Food Administration of the Slovak Republic
    ],
    'ocd-division/country:tz': ['.go.tz'],
    'ocd-division/country:us': [
        '.gov',
        '.mil',
    ],
    'ocd-division/country:uy': [
        '.gub.uy',
        '.precios.uy',  #  Ministerio de Economía y Finanzas
    ],
}


# Giovanni Thsibangu
# Python Indeed Web Scraper
# 3/16/2022

import requests
from bs4 import BeautifulSoup
import pandas as pd
import csv
from datetime import *
import smtplib, ssl
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import smtplib
import mimetypes
from email import encoders
from email.message import Message
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
import time

# In[46]:

# Date and time object
today =date.today()
now =datetime.now().strftime("%H:%M:%S")

# Data frame and series Utilities
template  = {"Job Title":[],"Company Name":[],"Location":[],"Salaries":[],"Link":[]}
errorTemplate={"Eror":[],"Location":[],"Date":[],"Time":[]}
errorDataFrame= pd.DataFrame(errorTemplate)
Main_DataFrame =pd.DataFrame(template)

# Initiliaze variables
error_reporter=""
error_location=""
url=""
names=[]
titles=[]
links=[]
locations=[]
salaries=[]

# In[ ]:

# In[47]:

def getPage(url):

    try:
        page = requests.get(url)
        page.raise_for_status()
        soup = BeautifulSoup(page.content,'html.parser')
        result = soup.find(id="mosaic-provider-jobcards")
        job_elements = result.find_all("div", class_="job_seen_beacon")

        error_reporter="Success Soup"
        error_location="GetPage URL"
        errorLog_file(error_reporter,error_location,today,now)
        soup = BeautifulSoup(page.content,'html.parser')
        result = soup.find(id="mosaic-provider-jobcards")
        job_elements = result.find_all("div", class_="job_seen_beacon")

    except Exception as ex:

        error_reporter= str(ex)
        error_location="Soup"
        errorLog_file(error_reporter,error_location,today,now)
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)

        email_Update("error","getPage "+error_reporter)
        print("An error as occured in the Soup: ")
    else:
        print("Soup elements have been successfuly scraped.")
        return soup

def jobCard(url):
    try:
        soup =getPage(url)
    except Exception as ex:
        error_reporter=str(ex)
        error_location="getPage Function Failed"
        errorLog_file(error_reporter,error_location,today,now)
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)
        email_Update("error","jobCard "+error_reporter)
    else:
        title_element = soup.find_all("h2", class_="jobTitle")
        company_element = soup.find_all("span", class_="companyName")
        location_element = soup.find_all("div", class_="companyLocation")
        salary = soup.find_all("div", class_="attribute_snippet")
        job_link = soup.find_all("a", class_="tapItem")

        for title in title_element:
            titles.append(title.text)
        for name in company_element:
            names.append(name.text)
        for location in location_element:
            locations.append(location.text)
        for salar in salary:
            salaries.append(salar.text)
        for link in job_link:
            links.append("https://www.indeed.com"+link.get("href"))

        print("Success Job card  data")

        error_reporter="Success Jobcard"
        error_location="JobCard"
        errorLog_file(error_reporter,error_location,today,now)
        return (titles,names,locations,salaries,links)

def createDataFrame(url):
    try:
        jobCard(url)
        df_titles=pd.Series(titles,name="Job Title")
        df_names=pd.Series(names,name="Company Name")
        df_locations=pd.Series(locations,name="Location")
        df_salaries=pd.Series(salaries,name="Salaries")
        df_job_link=pd.Series(links,name="Links")
        # Frame contains all the row required for our dataFrame
        frames =[df_titles,df_names,df_locations,df_salaries,df_job_link]
        # Concat all.
        job_card =pd.concat(frames,axis=1)
        print("Data Frame successfull created")
    except Exception as ex:
        error_reporter=str(ex)
        error_location="createDataFrame"
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)
        errorLog_file(error_reporter,error_location,today,now)
        email_Update("error","createDataFrame "+error_reporter)
        print(ex)
    else:
            # Return dataFrame
        return job_card
# In[54]:

# Email Function
def email_Update(file,subject):
    emailfrom = "<Your email>"
    emailto ="<Recipient email>"
    fileToSend = file+".csv"
    username = "user"
    password = "<Your password>"

    msg = MIMEMultipart()
    msg["From"] = emailfrom
    msg["To"] = emailto
    msg["Subject"] = subject
    msg.preamble = subject

    ctype, encoding = mimetypes.guess_type(fileToSend)
    if ctype is None or encoding is not None:
        ctype = "application/octet-stream"

    maintype, subtype = ctype.split("/", 1)

    if maintype == "text":
        fp = open(fileToSend)
        # Note: we should handle calculating the charset
        attachment = MIMEText(fp.read(), _subtype=subtype)
        fp.close()
    elif maintype == "image":
        fp = open(fileToSend, "rb")
        attachment = MIMEImage(fp.read(), _subtype=subtype)
        fp.close()
    elif maintype == "audio":
        fp = open(fileToSend, "rb")
        attachment = MIMEAudio(fp.read(), _subtype=subtype)
        fp.close()
    else:
        fp = open(fileToSend, "rb")
        attachment = MIMEBase(maintype, subtype)
        attachment.set_payload(fp.read())
        fp.close()
        encoders.encode_base64(attachment)
    attachment.add_header("Content-Disposition", "attachment", filename=fileToSend)
    msg.attach(attachment)

    server = smtplib.SMTP("smtp.gmail.com:587")
    server.starttls()
    server.login(emailfrom,password)
    server.sendmail(emailfrom, emailto, msg.as_string())
    print("Quitting the Server!!!")
    server.quit()
# In[55]:

def errorLog_file(error,loc,date,time):
    try:
        errorTemplate={"Eror":[error],"Location":[loc],"Date":[date],"Time":[time]}
        errorDataFrame= pd.DataFrame(errorTemplate)

    except Exception as ex:
        error_reporter=str(ex)
        error_location="Driver main"
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)

        email_Update("error","ErrorLog_File "+error_reporter)
    else:
        errorDataFrame.to_csv("error.csv")
# In[56]:
def DriverMain(listOfposition):
    try:
        for job_title in listOfposition:
            city="waterbury"
            state="CT"
            url ="https://www.indeed.com/jobs?q={job_title}&l={city}%2C%20{state}&fromage=1&".format(job_title=job_title,city=city,state=state)

            file = createDataFrame(url)

            file.to_csv(job_title+".csv")
            email_Update(job_title,"Job Update!")
            print(url)
            print(job_title+" Job  CSV created Email sent!")
            time.sleep(10)
    except Exception as ex:
        print("Failed link: "+url)
        print(ex)
        error_reporter=str(ex)
        error_location="Driver main"
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)
        email_Update("error","DriverMain "+error_reporter)
        errorLog_file(error_reporter,error_location,today,now)
    else:
        print(job_title," Job file has been searched and saved as csv ")
        print()
# In[59]:

listOfposition=["Developer","Programmer","software"] # List of job titles to search
DriverMain(listOfposition)      # Driver function


class RunnerHighlighter:
    def highlight(self, half_inning):
        self.bases_scored_from = []
        for atbat in reversed(half_inning.atbats):
            self.bases_new = []
            self.parse_atbat_ending_runners(atbat)
            self.parse_mid_pa_runners(atbat)
            self.bases_scored_from = self.bases_new

    def parse_atbat_ending_runners(self, atbat):
        for runner in atbat.runners:
            if ((runner.end in self.bases_scored_from or
                 runner.end == 4) and not runner.out):
                self.mark_and_save(runner)

    def parse_mid_pa_runners(self, atbat):
        for runner in reversed(atbat.mid_pa_runners):
            if runner.end in self.bases_new and not runner.out:
                self.bases_new.remove(runner.end)
                self.mark_and_save(runner)
            elif runner.end == 4 and not runner.out:
                self.mark_and_save(runner)

    def mark_and_save(self, runner):
        runner.to_score = True
        if runner.start != 0:
            self.bases_new.append(runner.start)