from datetime import datetime, timedelta
import os
import operator
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators.python_operator import PythonOperator
from operators.stage_from_s3 import StageFromS3Operator
from operators.get_data import GetDataOperator
from operators.load_fact import LoadFactOperator
from operators.data_quality import DataValidationOperator
from airflow.hooks.S3_hook import S3Hook
from helpers import LoadFactQueries, LoadDimensionQueries

# Arguments for this dag
default_args = {
    'owner': 'hiroo',
    'start_date': datetime(2021,1,3),
    'depends_on_past': False,
    'retries': 3,
    'retry_delay': timedelta(minutes=3),
    'catchup': False,
    'email':['kifa0422@gmail.com'],
    'email_on_failure':False,
    'email_on_retry':False
}

# dag definition, the dag is scheduled to run daily
dag = DAG('covid_warehouse_dag',
          default_args=default_args,
          description='Load and transform data in Redshift with Airflow',
          schedule_interval= '@daily'
        )

# address for temporary storage in disk
OUTPUT_PATH = '/Users/hiroakioshima/Desktop/Hiro_Covid_Project/tables'

# helper function to move files to AWS S3
def upload_to_S3(filename, key, bucket_name):
	hook = S3Hook('my_S3_conn')
	hook.load_file(filename, key, bucket_name, replace=True)

start_operator = DummyOperator(task_id='Begin_execution',  dag=dag)

# get data about county's cases, make copy in local disk
get_county_case = GetDataOperator(
		task_id='Get_county_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_county_case',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/latimes-county-totals.csv',
		output_path= OUTPUT_PATH
	)

# get data about hospitalized cases
get_hospitalized_case = GetDataOperator(
		task_id='Get_hospitalized_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_hospitalized',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-hospital-patient-county-totals.csv',
		output_path=OUTPUT_PATH
	)

# get data about every nursing home cases
get_nursing_home = GetDataOperator(
		task_id='Get_nursing_home_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_nursing_home',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-skilled-nursing-facilities.csv',
		output_path=OUTPUT_PATH
	)

# get data about senior home cases
get_senior_home = GetDataOperator(
		task_id='Get_senior_home_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_senior_facilities',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-adult-and-senior-care-facilities.csv',
		output_path=OUTPUT_PATH
	)

# get data about specific city/places
get_place_data = GetDataOperator(
		task_id='Get_place_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_place_totals',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-adult-and-senior-care-facilities.csv',
		output_path=OUTPUT_PATH
	)

# get data about every prison's cases
get_prison_case_data = GetDataOperator(
		task_id='Get_prison_case_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_prison_cases',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdcr-prison-totals.csv',
		output_path=OUTPUT_PATH
	)

# get data about reopening metrics
get_reopening_metrics = GetDataOperator(
		task_id='Get_reopening_metrics_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_reopening_metrics',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-reopening-metrics.csv',
		output_path=OUTPUT_PATH
	)

# get data about nationwide cases
get_nationwide_case = GetDataOperator(
		task_id='Get_nationwide_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='nationwide_cases',
		data_url='https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-counties.csv',
		output_path=OUTPUT_PATH
	)

# get data about prioson's information, used for dimension tables
get_prison_info = GetDataOperator(
		task_id='Get_prison_info',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_prisons',
		data_url='https://raw.githubusercontent.com/nrjones8/cdcr-population-data/master/data/monthly_cdcr_population.csv',
		output_path=OUTPUT_PATH
	)

# get data about reopening tier for each county
get_reopening_tier = GetDataOperator(
		task_id='Get_reopening_tier',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_reopening_tier',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-reopening-tiers.csv',
		output_path=OUTPUT_PATH
	)

# move county data to S3
county_case_toS3 = PythonOperator(
		task_id='Upload_county_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':'/Users/hiroakioshima/Desktop/Hiro_Covid_Project/tables/staging_county_case.csv',
			'key':'tables/staging_county_case.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move hospitalized case data to S3
hospitalized_case_toS3 = PythonOperator(
		task_id='Upload_hospitalized_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':'/Users/hiroakioshima/Desktop/Hiro_Covid_Project/tables/staging_hospitalized.csv',
			'key':'tables/staging_hospitalized.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move nursing home data to S3
nursing_case_toS3 = PythonOperator(
		task_id='Upload_nursinghome_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_nursing_home.csv',
			'key':'tables/staging_nursing_home.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move senior cases to S3
senior_case_toS3 = PythonOperator(
		task_id='Upload_senior_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_senior_facilities.csv',
			'key':'tables/staging_senior_facilities.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move place data to S3
place_totals_tos3 = PythonOperator(
		task_id='Upload_places_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_place_totals.csv',
			'key':'tables/staging_place_totals.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move prison case data to S3
prison_case_tos3 = PythonOperator(
		task_id='Upload_prison_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_prison_cases.csv',
			'key':'tables/staging_prison_cases.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move reopening data to S3
reopening_metrics_tos3 = PythonOperator(
		task_id='Upload_reopening_metrics_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_reopening_metrics.csv',
			'key':'tables/staging_reopening_metrics.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move nationwide case data to S3
nationwide_tos3 = PythonOperator(
		task_id='Upload_nationwide_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/nationwide_cases.csv',
			'key':'tables/nationwide_cases.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move prison informaiton data to S3
prison_info_tos3 = PythonOperator(
		task_id='Upload_prisoninfo_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_prisons.csv',
			'key':'tables/staging_prisons.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move reopening tier data to S3
reopening_tier_tos3 = PythonOperator(
		task_id='Upload_reopening_tier_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_reopening_tier.csv',
			'key':'tables/staging_reopening_tier.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# load hospitalized data to staging table on redshift server
load_hospitalized_redshift = StageFromS3Operator(
		task_id='Load_hospitalized_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_hospitalized',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_hospitalized.csv'
	)

# load county case data to staging table on redshift server
load_county_redshift = StageFromS3Operator(
		task_id='Load_county_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_county_case',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_county_case.csv'
	)

# load nursing home cases data to staging table on redshift server
load_nursing_redshift = StageFromS3Operator(
		task_id='Load_nursing_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_nursing_home',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_nursing_home.csv'
	)

# load senior home cases data to staging table on redshift server
load_seniorhome_redshift = StageFromS3Operator(
		task_id='Load_senior_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_senior_facilities',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_senior_facilities.csv'
	)

# load place data to staging table on redshift server
load_places_redshift = StageFromS3Operator(
		task_id='Load_places_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_place_totals',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_place_totals.csv'
	)

# load prison cases data to staging table on redshift server
load_prisoncase_redshift = StageFromS3Operator(
		task_id='Load_prisoncase_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_prison_cases',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_prison_cases.csv'
	)

# load openmetrics data to staging table on redshift server
load_openmetrics_redshift = StageFromS3Operator(
		task_id='Load_openmetrics_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_reopening_metrics',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_reopening_metrics.csv'
	)

# load nationwide cases data to staging table on redshift server
load_nationwide_redshift = StageFromS3Operator(
		task_id='Load_nationwide_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='nationwide_cases',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/nationwide_cases.csv'
	)

# load prison information data to staging table on redshift server
load_prisoninfo_redshift = StageFromS3Operator(
		task_id='Load_prisoninfo_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_prisons',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_prisons.csv'
	)

# load open tier data to staging table on redshift server
load_opentier_redshift = StageFromS3Operator(
		task_id='Load_opentier_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_reopening_tier',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_reopening_tier.csv'
	)

# load information about healthcare faculty from data lake to redshift staging table
load_healthcarefacs_redshift = StageFromS3Operator(
		task_id='Load_healthcarefacs_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_healthcare_listing',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='static/healthcare_facility_locations.csv'
	)

# load census data from data lake to redshift staging table
load_census_redshift = StageFromS3Operator(
		task_id='Load_census_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='census_data2019',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='static/census_2019.csv'
	)

# transform the staging tables and load county case fact table
loadfact_county = LoadFactOperator(
		task_id='Loadfact_county',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'county_case',
		source_table = 'staging_county_case',
		sql = LoadFactQueries.load_county_case
	)

# transform the staging tables and load the hospitalized fact table on redshift
loadfact_hospitalized = LoadFactOperator(
		task_id='Loadfact_hospitalized',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'hospitalized_case',
		source_table = 'staging_hospitalized',
		sql = LoadFactQueries.load_hospitalized_case
	)

# transform the staging tables and load nursing home cases fact table on redshift
loadfact_nursing = LoadFactOperator(
		task_id='Loadfact_nursing',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'nursing_home_case',
		source_table = 'staging_nursing_home',
		sql = LoadFactQueries.load_nursing_home
	)

# transform the staging tables and load senior home cases fact table on redshift
loadfact_senior = LoadFactOperator(
		task_id='Loadfact_senior',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'senior_facility_case',
		source_table = 'staging_senior_facilities',
		sql = LoadFactQueries.load_senior_facs
	)

# transform the staging tables and load place cases fact table on redshift
loadfact_place = LoadFactOperator(
		task_id='Loadfact_place',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'place_total',
		source_table = 'staging_place_totals',
		sql = LoadFactQueries.load_places
	)

# transform the staging tables and load prison cases fact table on redshift
loadfact_prison = LoadFactOperator(
		task_id='Loadfact_prison',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'prison_case',
		source_table = 'staging_prison_cases',
		sql = LoadFactQueries.load_prison
	)

# transform the staging tables and load reopening metrics fact table on redshift
loadfact_open_metrics = LoadFactOperator(
		task_id='Loadfact_open_metrics',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'reopening_metrics',
		source_table = 'staging_reopening_metrics',
		sql = LoadFactQueries.load_open_metrics
	)

# transform the staging tables and load reopening tiers fact table on redshift
loadfact_open_tiers = LoadFactOperator(
		task_id='Loadfact_open_tiers',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'reopening_tier',
		source_table = 'staging_reopening_tier',
		sql = LoadFactQueries.load_open_tiers
	)

# transform the staging tables and load nationwide cases fact table on redshift
loadfact_nationwide = LoadFactOperator(
		task_id='Loadfact_nationwide',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'other_states_cases',
		source_table = 'nationwide_cases',
		sql = LoadFactQueries.load_nationwide
	)

# transform the staging tables and load healthcare facilities dimension table on redshift
loaddim_healthcare = PostgresOperator(
	task_id='Loaddim_healthcare',
    dag=dag,
    sql=LoadDimensionQueries.load_healthcare_facs,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load county dimension table on redshift
loaddim_county = PostgresOperator(
	task_id='Loaddim_county',
    dag=dag,
    sql=LoadDimensionQueries.load_county,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load prison dimension table on redshift
loaddim_prison = PostgresOperator(
	task_id='Loaddim_prison',
    dag=dag,
    sql=LoadDimensionQueries.load_prison,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load nursing homes dimension table on redshift
loaddim_nursing = PostgresOperator(
	task_id='Loaddim_nursing',
    dag=dag,
    sql=LoadDimensionQueries.load_nursing,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load dimension table on redshift
loaddim_senior = PostgresOperator(
	task_id='Loaddim_senior',
    dag=dag,
    sql=LoadDimensionQueries.load_senior,
    postgres_conn_id='redshift'
	)

# perfom data validation on fact tables, check if the latest data is loaded correctly
dataval_fact = DataValidationOperator(
	task_id='data_validation_facttables',
	dag=dag,
	redshift_conn_id='redshift',
	dq_checks= [{'testsql':'SELECT MAX(date) FROM county_case', 'expected':datetime(2021,1,20), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM hospitalized_case', 'expected':datetime(2021,1,20), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM reopening_metrics', 'expected':datetime(2021,1,22), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM nursing_home_case', 'expected':datetime(2021,1,21), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM senior_facility_case', 'expected':datetime(2021,1,21), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM place_total', 'expected':datetime(2021,1,21), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM prison_case', 'expected':datetime(2021,1,22), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM reopening_tier', 'expected':datetime(2021,1,22), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM other_states_cases', 'expected':datetime(2021,1,21), 'op':operator.eq}]
	)

# perform data validation on dimension tables, check if the number of records matches the expected number for each table
dataval_dim = DataValidationOperator(
	task_id='data_validation_dimtables',
	dag=dag,
	redshift_conn_id='redshift',
	dq_checks= [{'testsql':'SELECT COUNT(*) FROM county', 'expected':50, 'op':operator.ge},
			   {'testsql':'SELECT COUNT(*) FROM healthcare_facility', 'expected':1000, 'op':operator.ge},
			   {'testsql':'SELECT COUNT(*) FROM prison', 'expected':200, 'op':operator.le},
			   {'testsql':'SELECT COUNT(*) FROM nursing_facility', 'expected':200, 'op':operator.ge},
			   {'testsql':'SELECT COUNT(*) FROM adult_senior_care', 'expected':200, 'op':operator.ge}]
	)

end_operator = DummyOperator(task_id='Stop_execution',  dag=dag)

# Dependency of this dag
start_operator >> [get_county_case, get_hospitalized_case, get_nursing_home, get_senior_home, get_place_data, \
					get_prison_case_data, get_reopening_metrics, get_nationwide_case, get_prison_info, get_reopening_tier]

start_operator >> [load_healthcarefacs_redshift, load_census_redshift]

get_county_case >> county_case_toS3 >> load_county_redshift >> loadfact_county
get_hospitalized_case >> hospitalized_case_toS3 >> load_hospitalized_redshift >> loadfact_hospitalized
get_nursing_home >> nursing_case_toS3 >> load_nursing_redshift >> loadfact_nursing
get_senior_home >> senior_case_toS3 >> load_seniorhome_redshift >> loadfact_senior
get_place_data >> place_totals_tos3 >> load_places_redshift >> loadfact_place
get_prison_case_data >> prison_case_tos3 >> load_prisoncase_redshift >> loadfact_prison
get_reopening_metrics >> reopening_metrics_tos3 >> load_openmetrics_redshift >> loadfact_open_metrics
get_nationwide_case >> nationwide_tos3 >> load_nationwide_redshift >> loadfact_nationwide
get_prison_info >> prison_info_tos3 >> load_prisoninfo_redshift
get_reopening_tier >> reopening_tier_tos3 >> load_opentier_redshift >> loadfact_open_tiers

[loadfact_county, loadfact_hospitalized, loadfact_nursing, loadfact_senior, loadfact_place, loadfact_prison, \
 loadfact_open_metrics, loadfact_nationwide, loadfact_open_tiers] >> dataval_fact

[loaddim_healthcare, loaddim_county, loaddim_prison, loaddim_nursing, loaddim_senior] >> dataval_dim

[dataval_fact, dataval_dim] >> end_operator

[load_census_redshift, loadfact_open_tiers, load_healthcarefacs_redshift] >> loaddim_county
load_healthcarefacs_redshift >> [loaddim_healthcare, loaddim_nursing, loaddim_senior]
[load_prisoninfo_redshift, load_prisoncase_redshift] >> loaddim_prison


#coding:utf-8
'''
Created on 2016年5月30日 下午12:00:30

@author: TianD

@E-mail: tiandao_dunjian@sina.cn

@Q    Q: 298081132

@Description: define ui

'''
import sys

from PyQt4 import QtCore, QtGui

from _functools import partial

from kx_submission_tool import Ui_SubmissionMainWindow
from kx_submission_tool_delegate import ProgressBarDelegate, ComboBoxDelegate
from kx_submission_tool_model import TableModel
from kx_submission_tool_thread import SubmitWorker, LoadWorker

class SubmissionWindow(QtGui.QMainWindow, Ui_SubmissionMainWindow):

    def __init__(self, parent = None):
        super(SubmissionWindow, self).__init__(parent)
        self.setupUi(self)

        self.broswePushButton.clicked.connect(self.brosweCmd)
        self.submitPushButton.clicked.connect(self.submitCmd)
        self.removeButton.clicked.connect(self.removeCmd)
        self.resetPushButton.clicked.connect(self.resetCmd)
        self.pathLineEdit.editingFinished.connect(self.loadCmd)

        #self.connect(self.tableView, QtCore.SIGNAL("dropped"), self.droppedCmd)
        self.tableView.dropped.connect(self.droppedCmd)

        self.initTableView()

    def removeCmd(self):
        selected = self.tableView.selectedIndexes()
        rows = list(set([sel.row() for sel in selected]))
        rows.sort(reverse=True)
        for row in rows:
            self.contentModel.removeRow(row)

    def resetCmd(self):
        rowCount = self.contentModel.rowCount()
        for row in range(rowCount-1, -1,-1):
            self.contentModel.removeRow(row)

    def loadCmd(self):
        self.loadThread = LoadWorker()
        imagePath = u"{0}".format(self.pathLineEdit.text())
        self.loadThread.start(imagePath)
        self.loadThread.fileSignal.connect(self.setTableData)
        self.pathLineEdit.setText('')

    def brosweCmd(self):
        pathDialog = QtGui.QFileDialog(parent = self)

        pathDialog.setAcceptMode(0)
        pathDialog.fileSelected.connect(self.pathLineEdit.setText)
        pathDialog.fileSelected.connect(self.pathLineEdit.editingFinished)

        pathDialog.setFileMode(4)

        pathDialog.show()

    def initTableView(self):
        headerLabels = [u'状态', u'名称', u'帧数', u'选项', u'进度']
        options = [QtCore.QVariant('Normal'), QtCore.QVariant('Left'), QtCore.QVariant('Right')]
        tabledata = []
        self.contentModel = TableModel(headerLabels, tabledata)
        self.tableView.setModel(self.contentModel)
        pbd = ProgressBarDelegate(self.tableView)
        self.tableView.setItemDelegateForColumn(4, pbd)
        cbd = ComboBoxDelegate(options, self.tableView)
        self.tableView.setItemDelegateForColumn(3, cbd)

    def setTableData(self, data):
        rowCount = self.contentModel.rowCount()
        self.contentModel.insertRows(rowCount, data)

    def submitCmd(self):
        '''
        submit images from sw. to production path
        '''
        model = self.tableView.model()
        row = model.rowCount()
        column = model.columnCount()
        indexLst = [model.index(i,4) for i in range(row)]
        threads = [SubmitWorker() for i in range(row)]
        for i in range(row):

            index = indexLst[i]
            subThread = threads[i]
            subThread.progressSignal.connect(partial(model.setData, index))
            subThread.start(model.getData()[i])
            subThread.finishSignal.connect(partial(self.setFlagCmd, index))

    def setFlagCmd(self, index, int):
        model = index.model()
        row = index.row()
        column = 0
        flagIndex = model.index(row, 0)
        if int == 100:
            model.setData(flagIndex, 2)
        else :
            model.setData(flagIndex, 1)

    def droppedCmd(self, str):
        self.loadThread = LoadWorker()
        imagePath = [u"{0}".format(ipath) for ipath in str]
        self.loadThread.start(imagePath)
        self.loadThread.fileSignal.connect(self.setTableData)

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)
    ui=SubmissionWindow()
    ui.show()
    app.exec_()


#!/usr/bin/env python3
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import math

def mw_expert(dataset_path, T):
    data = pd.read_csv(dataset_path,header=None)
    n_servers = data.shape[0] # Number of servers
    weights = np.ones(n_servers)
    regrets = []
    eta=np.sqrt(np.log(n_servers)/T)
    for t in range(T):
        # pick a server with probability proportional to the weights
        choice = np.random.choice(n_servers, p=weights/np.sum(weights))
        # observe the loss of all servers
        losses = data.iloc[:,t].values
        # calculate the regret
        regret = losses[choice] -np.min(losses)
        regrets.append(regret)
        # update the weights of all servers
        weights = weights * np.exp(-eta* losses)
    return np.cumsum(regrets)

def mw_adversarial(dataset_path, T):
    data = pd.read_csv(dataset_path, header=None)
    n_servers = data.shape[0] # Number of servers
    weights = np.ones(n_servers)
    regrets = []
    eta = np.sqrt(np.log(n_servers) / T)
    for t in range(T):
        # calculate the probability of selecting each action i
        pi = weights / np.sum(weights)
        # select an action i with probability proportional to pi
        choice = np.random.choice(n_servers, p=pi)
        # observe the loss of the chosen action
        loss = data.iloc[choice,t]
        # calculate the regret
        regret = np.min(data.iloc[:,t].values) - loss
        regrets.append(np.abs(regret))
        # update the weights of all actions using the MW update rule
        for i in range(n_servers):
            if i == choice:
                weights[i] *= np.exp(-eta * loss / pi[i])
            else:
                weights[i] *= np.exp(eta * loss / ((1 - pi[choice]) * (n_servers - 1)))

        # renormalize the weights to ensure they sum to 1
        weights /= np.sum(weights)
    return np.cumsum(regrets)

def ucb_losses(dataset_path, T):
    data = pd.read_csv(dataset_path, header=None)
    n_servers = data.shape[0] # Number of servers
    n = np.zeros(n_servers) # Number of times each server has been selected
    losses = np.zeros(n_servers) # Cumulative losses for each server
    est_means = np.zeros(n_servers) # Estimated mean loss for each server
    regrets = np.zeros(T) # Initialize regrets array

    # Initialize the estimates and UCB values randomly
    for i in range(n_servers):
        loss = data.iloc[:,i].sum()
        n[i] += 1
        losses[i] += loss
        est_means[i] = losses[i] / n[i]

    # Run the UCB algorithm for T time steps
    for t in range(n_servers, T):
        # Choose the server with the highest UCB value
        ucb_values = est_means + np.sqrt(2 * np.log(t) / (n + 1e-6))
        server = np.argmax(ucb_values)

        # Update the estimates and UCB values
        loss = data.iloc[server,t]
        n[server] += 1
        losses[server] -= loss
        est_means[server] = losses[server] / n[server]

        # Calculate and record the regret
        optimal_loss = data.iloc[:,t].min()
        regret = optimal_loss - loss
        regrets[t] = np.abs(regret)

    # Calculate and return the cumulative regrets
    return np.cumsum(regrets)

# Compare performance of MW algorithm in expert and bandit cases
T = [1000, 7000]
for t in T:
    regrets_expert = mw_expert("Milano_timeseries.csv",t)
    regrets_bandit = mw_adversarial("Milano_timeseries.csv",t)
    regrets_ucb = ucb_losses("Milano_timeseries.csv",t)

    plt.figure(figsize=(8, 6))
    plt.plot(regrets_expert, label='MW Expert')
    plt.plot(regrets_bandit, label='MW Bandit')
    plt.plot(regrets_ucb, label='UCB')
    # plt.semilogy(regrets_expert, label='MW Expert')
    # plt.semilogy(regrets_bandit, label='MW Bandit')
    # plt.semilogy(regrets_ucb, label='UCB')
    plt.legend()
    plt.title(f"T={t}")
    plt.xlabel('Time')
    plt.ylabel('Cumulative regret')
    plt.show()


import csv
import datetime as dt
import json
import logging
import os
from collections import defaultdict
from datetime import timedelta, date, datetime
from json.decoder import JSONDecodeError

import requests
import xlrd
import xlwt
from django.contrib import messages
from django.contrib.auth import update_session_auth_hash
from django.contrib.auth.decorators import login_required, user_passes_test
from django.contrib.auth.forms import PasswordChangeForm
from django.contrib.auth.models import Group
from django.db.models import Sum
from django.http import HttpResponse
from django.http import JsonResponse
from django.shortcuts import render, redirect
from django.utils import timezone
from requests.exceptions import Timeout, ConnectionError

from dashboard.forms import FranchiseBaseForm, FranchiseUserForm
from utility.models import Ads, District, MyAds, Depo, Route
from utility.models import bus_Detail
from .models import Franchise

logger = logging.getLogger(__name__)
# Set the timeout value to 5 seconds
timeout = 200

# Create your views here.
def is_patner(user):
    return user.groups.filter(name='Franchise').exists()

@login_required()
@user_passes_test(is_patner)
def dash(request):
    # Get the current user's franchise and district
    franchise = Franchise.objects.get(user=request.user)
    districts = franchise.district.all()  # get all associated districts
    ads = Ads.objects.filter(District__in=districts, display=True).distinct()
    today = timezone.now().date()
    # Get the value of display_after_end attribute
    first_ad = ads.first()
    if first_ad is not None:
        display_after_end = first_ad.display_after_end
    else:
        display_after_end = None

    # Calculate five days ago using the attribute value
    if display_after_end is not None:
        five_days_ago = today - timezone.timedelta(days=display_after_end)
    else:
        # Handle the case when display_after_end is None
        five_days_ago = today - timezone.timedelta(days=5)

    # ads = ads.exclude(EndDate__lt=today)  # Exclude ads that have ended in the past
    ads = ads.exclude(EndDate__lte=five_days_ago)  # Exclude ads that have ended more than n days ago

    csv_path = os.path.join(os.getcwd(), 'static', 'data')
    total_spots = None  # initialize total to None
    filled_spots = None  # initialize filled to None
    last_modified = None  # initialize last modified time to None

    for district in districts:
        # Assuming that each district's Excel file is named after the district's name
        file_name = f"{district.District}_summary.csv"
        file_path = os.path.join(csv_path, file_name)

        try:
            with open(file_path, newline='') as csvfile:
                reader = csv.reader(csvfile)
                row = next(reader)
                # Split column names by colon and extract parts
                total_col = row[0]  # get the first column name
                total_spots = (total_spots or 0) + int(total_col.split(':')[1])  # extract and add total spots
                filled_col = row[1]  # get the second column name
                filled_spots = (filled_spots or 0) + int(filled_col.split(':')[1])  # extract and add filled spots

            # Get the last modified time of the file
            mod_time = os.path.getmtime(file_path)
            if last_modified is None or mod_time > last_modified:
                last_modified = mod_time

        except FileNotFoundError:
            print(f"File {file_name} not found for district {district.District}")
        except Exception as e:
            print(f"Error processing file {file_name} for district {district.District}: {str(e)}")

    # Convert the last modified time to a human-readable format
    last_modified_str = None
    if last_modified is not None:
        last_modified_str = datetime.fromtimestamp(last_modified).strftime('%d/%m/%y')

    # Check if any files were processed
    if total_spots is None or filled_spots is None:
        # No files were processed, set values to None or any other default value you want
        total_spots = filled_spots = percentage = free = "No data"
    else:
        # Calculate percentage and free spots
        free = total_spots - filled_spots
        percentage = "{:.2f}".format((filled_spots / total_spots) * 100)
    # print(percentage)
    ten_days = []
    five_days = []
    for ad in ads:
        if ad.diff <= 10 and ad.diff >= 5:
            ten_days.append(ad)
            # print(ten_days)
        elif ad.diff <= 5:
            five_days.append(ad)
            # print(five_days)

        ad.myads_count = \
            MyAds.objects.filter(adname=ad.AdName, date_time__range=[ad.StartDate, ad.EndDate]).aggregate(Sum('Count'))[
                'Count__sum']

        ad.myads_count = ad.myads_count if ad.myads_count is not None else 0  # To Print the total count is 0
        statuss = ad.day * ad.ECPD
        diff = abs(ad.myads_count - statuss)
        if ad.myads_count == 0:
            ad.status = "crtical"
        elif diff == 0:
            ad.status = "up"
        elif diff <= 200:
            ad.status = "up"
        elif diff > 200 and diff <= 400:
            ad.status = "down"
        elif ad.myads_count > statuss:
            ad.status = "up"
        else:
            ad.status = "error"

        # print(ad.day, ad.ECPD, statuss, ad.status, diff)
        if ad.myads_count is not None:  # To handle the total count is 0
            if ad.TotalCount:
                # print(ad.AdName, ad.myads_count, ad.TotalCount)
                ad.percentage = (ad.myads_count / ad.TotalCount) * 100
            else:
                ad.percentage = 0
        else:
            ad.percentage = 0
    # getupdate(request)

    context = {
        'ads': ads,
        'ten_days': ten_days,
        'five_days': five_days,
        'total': total_spots,
        'filled': filled_spots,
        'free': free,
        'percentage': percentage,
        'last': last_modified_str,
        'franchise': franchise
    }
    return render(request, 'Dashboard/dashboard.html', context)

@login_required()
@user_passes_test(is_patner)
def view_ad(request, ad_id):
    franchise = Franchise.objects.get(user=request.user)
    ad = Ads.objects.get(id=ad_id)
    date_count_array = []
    ad.myads_count = \
        MyAds.objects.filter(adname=ad.AdName, date_time__range=[ad.StartDate, ad.EndDate]).aggregate(Sum('Count'))[
            'Count__sum']
    ad.myads_count = ad.myads_count if ad.myads_count is not None else 0  # To Print the total count is 0

    day = timezone.now().date() - timedelta(days=1)
    today = date.today()
    # To get last 7 days data by default
    end_date = datetime.now().date()
    start_date = end_date - timedelta(days=7)
    start_str = start_date.strftime('%d/%m/%y')
    end_str = end_date.strftime('%d/%m/%y')

    param = {
        'name': ad.AdUsername,
        'from': start_date.strftime("%Y-%m-%d"),
        'to': end_date.strftime("%Y-%m-%d"),
    }
    # Query the MyAds model to get the required data
    districts = ad.District.all()
    district_names = [d.District for d in districts]
    if 'NGL' in district_names:
        urls1 = ['https://delta.busads.in/get_adcountv2.php', 'https://tvl.busads.in/get_adcountv2.php']
    elif any(name in district_names for name in ['TVL', 'TUT', 'TNK']):
        urls1 = ['https://delta.busads.in/get_adcountv2.php', 'https://tvl.busads.in/get_adcountv2.php']
    elif any(name in district_names for name in ['Madurai', 'MADURAI']):
        urls1 = ['https://delta.busads.in/get_adcountv2.php', 'https://track.siliconharvest.net/get_adcountv2.php']
    else:
        urls1 = ['https://delta.busads.in/get_adcountv2.php']
    for url1 in urls1:
        response = requests.get(url1, params=param, timeout=timeout)
        if response.status_code != 200:
            print(f"Error response received with status code {response.status_code}")
            logger.error(f"Error response received with status code {response.status_code}")
            continue
        try:
            data = response.json()
        except JSONDecodeError:
            print(f"Error decoding JSON: {response.text}")
            logger.warning(f"Error decoding JSON: {response.text}")
            continue
        except Timeout:
            print("Request TimeOut")
            logger.error(f"Timeout Error: {url1}")
            continue
        except ConnectionError:
            continue
        if data is None:
            print("API returned None")
            logger.warning('API returned None')
            continue

        day_counts = defaultdict(int)

        for item in data:
            for key, value in item.items():
                if key in ['imei', 'bus_no', 'route_no', 'route_name']:
                    continue
                day = dt.datetime.strptime(key, "%d/%m/%Y").date().strftime("%d/%m")
                count = int(value)
                day_counts[day] += count

        date_count_array1 = [{'date': date, 'count': count} for date, count in day_counts.items()]

        # print(date_count_array)
        # Query the MyAds model to get the required data
        if 'NGL' in district_names:
            urls2 = ['https://delta.busads.in/get_adcountv3.php', 'https://tvl.busads.in/get_adcountv3.php']
        elif any(name in district_names for name in ['TVL', 'TUT', 'TNK']):
            urls2 = ['https://delta.busads.in/get_adcountv3.php', 'https://tvl.busads.in/get_adcountv3.php']
        elif any(name in district_names for name in ['Madurai', 'MADURAI']):
            urls2 = ['https://delta.busads.in/get_adcountv3.php', 'https://track.siliconharvest.net/get_adcountv3.php']
        else:
            urls2 = ['https://delta.busads.in/get_adcountv3.php']
        counts_by_url = []
        for url2 in urls2:
            response = requests.get(url2, params=param, timeout=timeout)
            if response.status_code != 200:
                print(f"Error response received with status code {response.status_code}")
                logger.error(f"Error response received with status code {response.status_code}")
                continue
            try:
                data = response.json()
            except JSONDecodeError:
                print(f"Error decoding JSON: {response.text}")
                continue
            except Timeout:
                print(f"Error Timeout: {url2}")
                continue
            except ConnectionError:
                continue
            if data is None:
                print("API returned None")
                continue
            day_counts = defaultdict(int)
            for item in data:
                for key, value in item.items():
                    if key in ['imei', 'bus_no']:
                        continue
                    day = dt.datetime.strptime(key, "%d/%m/%Y").date().strftime("%d/%m")
                    count = int(value)
                    day_counts[day] += count
            counts_by_url.append(day_counts)

        # Merge the daily counts for each URL into a single dictionary
        merged_counts = defaultdict(int)
        for counts in counts_by_url:
            for day, count in counts.items():
                merged_counts[day] += count

        date_count_array = [{'date': date, 'count': count} for date, count in merged_counts.items()]

        # Sort the date_count_array by date in ascending order
        sorted_date_count_array = sorted(date_count_array, key=lambda x: dt.datetime.strptime(x['date'], "%d/%m"))

    if ad.myads_count is not None:  # To handle the total count is 0
        if ad.TotalCount:
            # print(ad.AdName, ad.myads_count, ad.TotalCount)
            ad.percentage = (ad.myads_count / ad.TotalCount) * 100
        else:
            ad.percentage = 0
    else:
        ad.percentage = 0

    # Fetch API data and give inital data
    params1 = {
        'name': ad.AdUsername,
        'from': today.strftime("%Y-%m-%d"),
        'length': 1,
    }
    if 'NGL' in district_names:
        urls3 = ['https://delta.busads.in/get_adcountv3.php', 'https://tvl.busads.in/get_adcountv3.php']
    elif any(name in district_names for name in ['TVL', 'TUT', 'TNK']):
        urls3 = ['https://delta.busads.in/get_adcountv3.php', 'https://tvl.busads.in/get_adcountv3.php']
    elif any(name in district_names for name in ['Madurai', 'MADURAI']):
        urls3 = ['https://delta.busads.in/get_adcountv3.php', 'https://track.siliconharvest.net/get_adcountv3.php']
    else:
        urls3 = ['https://delta.busads.in/get_adcountv3.php']
    api_data = 0

    for url3 in urls3:
        try:
            response = requests.get(url3, params=params1, timeout=timeout)
            if response.status_code != 200:
                print(f"Error response received with status code {response.status_code}")
                logger.error(f"Error response received with status code {response.status_code}")
                continue
            api_data += int(response.json())
        except Timeout:
            print(f"Timeout error while making request to {url3}")
            logger.error(f"Timeout error while making request to {url3}")
            continue
        except Exception as e:
            print(f"Error while making request to {url3}: {e}")
            logger.error(f"Error while making request to {url3}: {e}")
            continue
        except ConnectionError:
            continue

    # Create a dictionary with the result
    today_count = api_data

    # Convert dictionary to JSON
    json_data = json.dumps(today_count)

    params3 = {
        'name': ad.AdUsername,
        'from': ad.StartDate.strftime("%Y-%m-%d"),
        'to': ad.EndDate.strftime("%Y-%m-%d"),
        'all': 0,
    }
    # print(params3)
    if 'NGL' in district_names:
        urls88 = ['https://delta.busads.in/get_status.php', 'https://tvl.busads.in/get_status.php']
    elif any(name in district_names for name in ['TVL', 'TUT', 'TNK']):
        urls88 = ['https://delta.busads.in/get_status.php', 'https://tvl.busads.in/get_status.php']
    elif any(name in district_names for name in ['Madurai', 'MADURAI']):
        urls88 = ['https://delta.busads.in/get_status.php',
                  'https://track.siliconharvest.net/get_status.php?city=madurai']
    else:
        urls88 = ['https://delta.busads.in/get_status.php']

    get33 = []

    for url8 in urls88:
        try:
            response = requests.get(url8, timeout=timeout)
            if response.status_code != 200:
                print(f"Error response received with status code {response.status_code}")
                logger.error(f"Error response received with status code {response.status_code}")
                continue
            # Append the response data to the 'get2' list
            get33 += response.json()
        except Timeout:
            print(f"Timeout error while making request to {url8}")
            logger.error(f"Timeout error while making request to {url8}")
            continue
        except ValueError:
            print('Value error')
            logger.warning('Value error')
            continue
        except Exception as e:
            print(f"Error while making request to {url8}: {e}")
            logger.error(f"Error while making request to {url8}: {e}")
            continue
        except ConnectionError:
            continue

    # Extract required information from data1 and data2
    data1 = []
    for api__data in [get33]:
        for item in api__data:
            d = {
                'imei': item['imei'],
                'bus_no': item['bus_no'],
                'route_no': item['route_no'],
                'route_name': item['route_name'],
                'city': item['city']  # Assuming 'city' is a key in the original data
            }
            data1.append(d)

    filtered_data1 = []

    for item in data1:
        city = item['city']
        if city not in ['Testing', 'OFFICE', 'new_testing', 'TV-Testing', 'Testing']:
            filtered_data1.append(item)

    # fetch count data from API
    if 'NGL' in district_names:
        urls4 = ['https://delta.busads.in/get_adcountv3.php', 'https://tvl.busads.in/get_adcountv3.php']
    elif any(name in district_names for name in ['TVL', 'TUT', 'TNK']):
        urls4 = ['https://delta.busads.in/get_adcountv3.php', 'https://tvl.busads.in/get_adcountv3.php']
    elif any(name in district_names for name in ['Madurai', 'MADURAI']):
        urls4 = ['https://delta.busads.in/get_adcountv3.php', 'https://track.siliconharvest.net/get_adcountv3.php']
    else:
        urls4 = ['https://delta.busads.in/get_adcountv3.php']
    get = []

    for url4 in urls4:
        try:
            response = requests.get(url4, params=params3, timeout=timeout)
            if response.status_code != 200:
                print(f"Error response received with status code {response.status_code}")
                logger.error(f"Error response received with status code {response.status_code}")
                continue
            get += response.json()
        except Timeout:
            print(f"Timeout error while making request to {url4}")
            logger.error(f"Timeout error while making request to {url4}")
            continue
        except ValueError:
            print('Value error')
            logger.warning('Value error')
            continue
        except Exception as e:
            print(f"Error while making request to {url4}: {e}")
            logger.error(f"Error while making request to {url4}: {e}")
            continue
        except ConnectionError:
            continue

    # Extract required information from data1 and data2
    report = []
    start_date = ad.StartDate
    end_date = ad.EndDate
    dates = []
    while start_date <= end_date:
        dates.append(start_date.strftime('%d/%m/%Y'))
        start_date += timedelta(days=1)  # increment start_date by 1 day

    total_counts = {}  # initialize counts for each date
    for api_data1 in [get]:
        for data in api_data1:
            bus_no = data["bus_no"]
            imei = data["imei"]
            counts = {}  # initialize counts for this bus
            for key, value in data.items():
                if key == "bus_no" or key == "imei":
                    continue
                da, count = key, value
                da = datetime.strptime(da, "%d/%m/%Y").strftime("%d/%m/%Y")
                if da in counts:
                    counts[da] += count
                else:
                    counts[da] = count
                if da in total_counts:
                    total_counts[da] += count
                else:
                    total_counts[da] = count

            date_counts = [{"date": da, "count": counts.get(da, 0)} for da in
                           dates]  # create date_counts list for this bus
            report.append(
                {"bus_no": bus_no, "route_no": "", "date_counts": date_counts, "total_count": sum(counts.values())})

            # Compare imei with data1 and update route_no in report
            for item in data1:
                if item['bus_no'] == bus_no:
                    city = item['route_name']
                    # Check if city does not contain the specified keywords
                    if 'Testing' not in city and 'TV-Testing' not in city and 'new_testing' not in city and 'OFFICE' not in city and 'testing' not in city:
                        report[-1]["route_no"] = item['route_no']

        total_count_all_buses = sum(total_counts.values())
        total_date_counts = [{"date": da, "count": total_counts.get(da, 0)} for da in
                             dates]  # create date_counts list for total count
        report.append({"bus_no": "Total", "route_no": "", "date_counts": total_date_counts,
                       "total_count": total_count_all_buses})  # add total count of all buses to report

    merged_report = []

    # Merge duplicate entries based on bus_no
    for entry in report:
        existing_entry = next((item for item in merged_report if item['bus_no'] == entry['bus_no']), None)
        if existing_entry:
            existing_entry['total_count'] += entry['total_count']
            for date_count in entry['date_counts']:
                existing_date_count = next(
                    (item for item in existing_entry['date_counts'] if item['date'] == date_count['date']),
                    None)
                if existing_date_count:
                    existing_date_count['count'] += date_count['count']
                else:
                    existing_entry['date_counts'].append(date_count)
        else:
            merged_report.append(entry)

    params2 = {
        'name': ad.AdUsername,
        'from': today.strftime("%Y-%m-%d"),
        'all': 1,
    }
    # print(data)

    get2 = []

    for url5 in urls4:
        try:
            response = requests.get(url5, params=params2, timeout=timeout)
            if response.status_code != 200:
                print(f"Error response received with status code {response.status_code}")
                logger.error(f"Error response received with status code {response.status_code}")
                continue
            # Append the response data to the 'get2' list
            get2 += response.json()
        except Timeout:
            print(f"Timeout error while making request to {url5}")
            logger.error(f"Timeout error while making request to {url5}")
            continue
        except ValueError:
            print('Value error')
            logger.warning('Value error')
            continue
        except Exception as e:
            print(f"Error while making request to {url5}: {e}")
            logger.error(f"Error while making request to {url5}: {e}")
            continue
        except ConnectionError:
            continue

    # Extract required information from data1 and data2
    result = []
    current_time = dt.datetime.now().time()
    #print(current_time)
    for api__data in [get2]:
        for item in api__data:
            for data_item in data1:
                city = data_item['route_name']  # Get the city from data_item
                # Check if city does not contain the specified keywords
                if 'Testing' not in city and 'TV-Testing' not in city and 'new_testing' not in city and 'OFFICE' not in city and 'testing' not in city:
                    if data_item['imei'] == item['imei']:
                        received_time = dt.datetime.strptime(item['date_time'], "%H:%M:%S").time()
                        if received_time > current_time:
                            # Update 'date_time' in the dictionary with the current time
                            item['date_time'] = current_time.strftime("%H:%M:%S")
                            logger.error(f"Received time ({received_time}) is greater than current time({current_time.strftime('%H:%M:%S')}).")
                        d = {
                            'imei': item['imei'],
                            'bus_no': item['bus_no'],
                            'route_no': data_item['route_no'],  # Get route_no from data_item
                            'route_name': data_item['route_name'],  # Get route_name from data_item
                            'date_time': item['date_time'],
                            'count': item['count'],
                        }
                        result.append(d)
                        break

    labels = []
    data = []
    for item in sorted_date_count_array:
        labels.append(item['date'])
        data.append(item['count'])
    context = {
        'ad': ad,
        'result': result,
        'json_data': json_data,
        'labels': labels,
        'data': data,
        'start_date': start_str,
        'end_date': end_str,
        'franchise': franchise,
        'report': merged_report,
        'dates': dates
    }
    return render(request, 'Dashboard/detail.html', context)

@login_required()
@user_passes_test(is_patner)
def route_summary(request):
    # Get the current user's franchise and district
    franchise = Franchise.objects.get(user=request.user)
    districts = franchise.district.all()  # get all associated districts

    # Construct the path to the Excel file based on the districts
    csv_path = os.path.join(os.getcwd(), 'static', 'data')
    sheets = {}  # initialize sheets as a dictionary

    for district in districts:
        # Assuming that each district's Excel file is named after the district's name
        file_name = f"{district.District}_filllist.xls"
        file_path = os.path.join(csv_path, file_name)

        try:
            # Open the Excel file using xlrd
            workbook = xlrd.open_workbook(file_path)
            # Replace NaN with empty strings
            sheets_district = {}
            for sheet_name in workbook.sheet_names()[3:]:
                sheet = workbook.sheet_by_name(sheet_name)
                header = [str(sheet.cell(0, col).value).split('.')[0] for col in range(sheet.ncols)]
                sheet_data = []
                for row in range(1, sheet.nrows):
                    row_data = [sheet.cell(row, col).value for col in range(sheet.ncols)]
                    sheet_data.append(row_data)
                sheets_district[sheet_name] = {'header': header, 'data': sheet_data}
                # print(sheet_data)

            # Merge the sheets for all districts
            for sheet_name, sheet_data in sheets_district.items():
                if not sheet_name.startswith('sheet'):
                    if sheet_name in sheets:
                        sheets[sheet_name]['data'].extend(sheet_data['data'])
                    else:
                        sheets[sheet_name] = sheet_data
        except:
            # handle the case where no Excel file is found for the district
            logger.warning('Sheet Not Found')
            pass

    sorted_sheets = sorted(sheets.keys())  # sort the keys

    selected_sheet = request.GET.get('route')
    sheet_data = None
    if selected_sheet and selected_sheet in sheets:
        sheet_data = sheets[selected_sheet]

    select = request.GET.get('select')
    sheet_data1 = None
    if select in sheets or select == 'All':
        if select == 'All':
            # Export all sheets to Excel including headers
            response = HttpResponse(content_type='application/ms-excel')
            response['Content-Disposition'] = 'attachment; filename="fill_list.xls"'
            wb = xlwt.Workbook(encoding='utf-8')

            for sheet_name, sheet_data in sheets.items():
                ws = wb.add_sheet(sheet_name)
                row_num = 0

                # Write headers to Excel sheet
                header_style = xlwt.XFStyle()
                header_style.font.bold = True
                for col_num, header_value in enumerate(sheet_data['header']):
                    ws.write(row_num, col_num, header_value, style=header_style)
                row_num += 1

                # Write data to Excel sheet
                for row in sheet_data['data']:
                    for col_num, cell_value in enumerate(row):
                        ws.write(row_num, col_num, cell_value)
                    row_num += 1

            wb.save(response)
            return response
        else:
            # Export selected sheet to Excel including headers
            sheet_data = sheets[select]
            response = HttpResponse(content_type='application/ms-excel')
            response['Content-Disposition'] = f'attachment; filename="{select}.xls"'
            wb = xlwt.Workbook(encoding='utf-8')
            ws = wb.add_sheet(select)
            row_num = 0

            # Write headers to Excel sheet
            header_style = xlwt.XFStyle()
            header_style.font.bold = True
            for col_num, header_value in enumerate(sheet_data['header']):
                ws.write(row_num, col_num, header_value, style=header_style)
            row_num += 1

            # Write data to Excel sheet
            for row in sheet_data['data']:
                for col_num, cell_value in enumerate(row):
                    ws.write(row_num, col_num, cell_value)
                row_num += 1

            wb.save(response)
            return response

    context = {
        'sheets': sheets,
        'selected_sheet': selected_sheet,
        'sheet_data': sheet_data,
        'list': sorted_sheets,
        'franchise': franchise
    }

    return render(request, 'Dashboard/route.html', context)

@login_required()
@user_passes_test(is_patner)
def spot(request):
    # Get the current user's franchise and district
    franchise = Franchise.objects.get(user=request.user)
    districts = franchise.district.all()  # get all associated districts

    # Construct the path to the Excel file based on the districts
    csv_path = os.path.join(os.getcwd(), 'static', 'data')
    sheets = {}  # initialize sheets as a dictionary

    for district in districts:
        # Assuming that each district's Excel file is named after the district's name
        file_name = f"{district.District}_spot.xls"
        file_path = os.path.join(csv_path, file_name)

        try:
            # Open the Excel file using xlrd
            workbook = xlrd.open_workbook(file_path)
            # Replace NaN with empty strings
            sheets_district = {}
            for sheet_name in workbook.sheet_names():
                sheet = workbook.sheet_by_name(sheet_name)
                header = [str(sheet.cell(0, col).value).split('.')[0] for col in range(sheet.ncols)]
                sheet_data = []
                for row in range(1, sheet.nrows):
                    row_data = [str(sheet.cell(row, col).value).split('.')[0] for col in range(sheet.ncols)]
                    sheet_data.append(row_data)
                sheets_district[sheet_name] = {'header': header, 'data': sheet_data}
                # print(sheet_data)

            # Merge the sheets for all districts
            for sheet_name, sheet_data in sheets_district.items():
                if not sheet_name.startswith('Sheet'):
                    if sheet_name in sheets:
                        sheets[sheet_name]['data'].extend(sheet_data['data'])
                    else:
                        sheets[sheet_name] = sheet_data
        except:
            # handle the case where no Excel file is found for the district
            logger.warning('Sheet Not Found')
            pass

    selected_sheet = request.GET.get('ad')
    sheet_data = None
    if selected_sheet and selected_sheet in sheets:
        sheet_data = sheets[selected_sheet]

    select = request.GET.get('select')
    sheet_data1 = None
    if select in sheets or select == 'All':
        if select == 'All':
            # Export all sheets to Excel including headers
            response = HttpResponse(content_type='application/ms-excel')
            response['Content-Disposition'] = 'attachment; filename="fill_list.xls"'
            wb = xlwt.Workbook(encoding='utf-8')

            # Write each sheet to a separate Excel sheet
            for sheet_name, sheet_data in sheets.items():
                ws = wb.add_sheet(sheet_name)
                row_num = 0

                # Write headers to Excel sheet
                header_value = sheet_data['header']
                for col_num, header in enumerate(header_value):
                    ws.write(row_num, col_num, header)
                row_num += 1

                # Write data to Excel sheet
                for row in sheet_data['data']:
                    for col_num, cell_value in enumerate(row):
                        ws.write(row_num, col_num, cell_value)
                    row_num += 1

            wb.save(response)
            return response
        else:
            # Export selected sheet to Excel including headers
            sheet_data = sheets[select]
            response = HttpResponse(content_type='application/ms-excel')
            response['Content-Disposition'] = f'attachment; filename="{select}.xls"'
            wb = xlwt.Workbook(encoding='utf-8')
            ws = wb.add_sheet(select)
            row_num = 0

            # Write headers to Excel sheet
            header_value = sheet_data['header']
            for col_num, header in enumerate(header_value):
                ws.write(row_num, col_num, header)
            row_num += 1

            # Write data to Excel sheet
            for row in sheet_data['data']:
                for col_num, cell_value in enumerate(row):
                    ws.write(row_num, col_num, cell_value)
                row_num += 1

            wb.save(response)
            return response

    context = {
        'sheets': sheets,
        'selected_sheet': selected_sheet,
        'sheet_data': sheet_data,
        'franchise': franchise
    }

    return render(request, 'Dashboard/spotadd.html', context)

@login_required()
@user_passes_test(is_patner)
def route_summary_filled(request):
    # Get the current user's franchise and district
    franchise = Franchise.objects.get(user=request.user)
    districts = franchise.district.all()  # get all associated districts

    # Construct the path to the Excel file based on the districts
    csv_path = os.path.join(os.getcwd(), 'static', 'data')

    data = []  # initialize data to an empty list
    data2 = []  # initialize data2 to an empty list

    # Loop through the districts and read the data from the corresponding CSV files
    for district in districts:
        # Assuming that each district's Excel file is named after the district's name
        file_name = f"{district.District}_summary.csv"
        file_path = os.path.join(csv_path, file_name)

        try:
            with open(file_path, 'r') as csv_file:
                reader = csv.reader(csv_file)
                next(reader)  # skip the header row
                for row in reader:
                    data.append(row)
        except FileNotFoundError:
            messages.warning(request, f"File not found for district {district.District}")

        file_name = f"{district.District}.csv"
        file_path = os.path.join(csv_path, file_name)

        try:
            with open(file_path, 'r') as csv_file:
                reader = csv.reader(csv_file)
                next(reader)  # skip the header row
                for row in reader:
                    data2.append(row)
        except FileNotFoundError:
            messages.warning(request, f"File not found for district {district.District}")
    # Get all unique values of row.0 in data
    all_routes = sorted(set(row[0] for row in data))

    # Filter the data and data2 based on the user's selection from the dropdown
    routes = request.GET.getlist('routes[]')  # get the selected values from the dropdown as a list
    # print(routes)

    if routes:
        if '' in routes or 'All' in routes:
            # Include all rows in data and data2
            pass
        else:
            data = [row for row in data if row[0] in routes]
            data2 = [row for row in data2 if row[1] in routes]

    # Sort the data based on row.0
    data = sorted(data, key=lambda row: row[0])
    # Sort the data2 based on row.0
    data2 = sorted(data2, key=lambda row: row[0])

    select = request.GET.get('route_select')
    if select:
        if select == 'All':
            # Export all data
            selected_routes = [row[0] for row in data]
        else:
            selected_routes = select.split(',')  # parse comma-separated values

        # filter data based on selected_routes
        data = [row for row in data if row[0] in selected_routes]
        if not data:
            messages.warning(request, f"No data found for '{select}'")
        else:
            response = HttpResponse(content_type='application/ms-excel')
            if select == 'All':
                filename = 'all_routes'
            else:
                filename = select

            header = ['Route', 'Available slot', 'Filled slot', 'Free slot']
            sheet_name = 'Route Summary'

            response['Content-Disposition'] = f'attachment; filename="{filename}_route_summary.xls"'
            wb = xlwt.Workbook(encoding='utf-8')
            ws = wb.add_sheet(sheet_name)

            # Write header to Excel sheet
            for col_num, cell_value in enumerate(header):
                ws.write(0, col_num, cell_value)

            # Write data to Excel sheet
            for row_num, row in enumerate(data):
                for col_num, cell_value in enumerate(row):
                    ws.write(row_num + 1, col_num, cell_value)

            wb.save(response)
            return response
    bus_select = request.GET.get('bus_select')
    if bus_select:
        if bus_select == 'All':
            # Export all data
            data = data2
            filename = 'bus_routes'
        else:
            selected_routes = bus_select.split(',')  # parse comma-separated values
            # filter data based on selected_routes
            data = [row for row in data2 if row[1] in selected_routes]
            filename = bus_select

        header = ['Bus no', 'Route', 'Available slot', 'Filled slot', 'Free slot']
        sheet_name = 'Bus Summary'

        response = HttpResponse(content_type='application/ms-excel')
        response['Content-Disposition'] = f'attachment; filename="{filename}_bus_summary.xls"'
        wb = xlwt.Workbook(encoding='utf-8')
        ws = wb.add_sheet(sheet_name)

        # Write header to Excel sheet
        for col_num, cell_value in enumerate(header):
            ws.write(0, col_num, cell_value)

        # Write data to Excel sheet
        for row_num, row in enumerate(data):
            for col_num, cell_value in enumerate(row):
                ws.write(row_num + 1, col_num, cell_value)

        wb.save(response)
        return response

    context = {
        'data': data,
        'data2': data2,
        'route': routes,
        'all': all_routes,
        'franchise': franchise
    }

    return render(request, 'Dashboard/route_filed.html', context)

def fetch_data_from_api(url, timeout):
    try:
        response = requests.get(url, timeout=timeout)
        data = response.json()
        filtered_data = [item for item in data if item.get('city', '').lower() not in ['testing', 'new_testing']]
        return filtered_data
    except (ValueError, requests.exceptions.Timeout) as e:
        logger.warning('Error for {}: {}'.format(url, e))
        return []

@login_required()
@user_passes_test(is_patner)
def tracking(request):
    franchise = Franchise.objects.get(user=request.user)
    districts = franchise.district.all()
    day = date.today()
    today = day.strftime("%Y/%m/%d")
    depo = []
    route = []
    dist = request.GET.get('district')
    depot = request.GET.get('depo')
    routes = request.GET.get('route')
    district_data = None

    if dist and dist.lower() != 'all':
        # Filter Depo and Route objects based on selected district
        district_obj = District.objects.get(District=dist)
        depo = Depo.objects.filter(city=district_obj)
        route = Route.objects.filter(city=district_obj)
        urls = [
            'https://track.siliconharvest.net/get_status.php',
            'https://delta.busads.in/get_status.php',
            'https://tvl.busads.in/get_status.php',
        ]
        speaker_issue = []
        error_data = []
        not_run = []
        halt = []
        district_data = [
            {
                'imei': item['imei'],
                'sno': item['sno'],
                'bus_no': item['bus_no'],
                'route_no': item['route_no'],
                'route_name': item['route_name'],
                'destination': station_dest[0].strip(),
                'station': station_dest[1].strip(),
                'last_played': item['last_played'],
                'date_time': item['date_time'],
                'depo': item['depo'],
                'volume': item['volume'],
            }
            for url in urls
            for item in fetch_data_from_api(url, timeout)
            if item.get('city') == dist and item.get('city').lower() not in ['testing', 'new_testing']
            if not depot or item.get('depo') == depot
            if not routes or item.get('route_no') == routes
            for station_dest in [item['station'].split(',')]
        ]
        today_date = datetime.now().strftime('%Y/%m/%d')
        for item in district_data:
            try:
                d1 = datetime.strptime(item['date_time'], '%Y/%m/%d %H:%M:%S')
            except ValueError:
                item['myClass'] = 'text-warning'
                item['icon'] = 'fas fa-circle'
                not_run.append(item)
                continue  # skip this item if date_time string cannot be parsed

            item_date = item['date_time'][:10]  # extract date from date_time string
            item_time = item['date_time'][11:] if len(item['date_time']) > 10 else None
            d2 = datetime.now()
            diff = (d2 - d1).total_seconds() // 60  # get time difference in minutes

            if item_date != today_date:
                item['myClass'] = 'text-danger'
                item['icon'] = 'fas fa-circle'
                not_run.append(item)
            elif item_time is None:
                item['myClass'] = 'text-warning'
                item['icon'] = 'fas fa-circle'
                not_run.append(item)
            elif diff > 15:
                item['myClass'] = 'text-warning'
                item['icon'] = 'fas fa-circle'
                halt.append(item)
            elif ' ' in item['volume'] or '-' in item['volume']:
                if item['volume'][0] == 'S':
                    item['myClass'] = 'text-success'
                    item['icon'] = 'fas fa-circle'
                elif item['volume'][0] == 'x':
                    item['myClass'] = 'text-warning'
                    item['icon'] = 'fas fa-volume-mute'
                    speaker_issue.append(item)
                elif item['volume'][0] == 'L':
                    item['myClass'] = 'text-success'
                    item['icon'] = 'fas fa-volume-off'
                    speaker_issue.append(item)
                else:
                    item['myClass'] = 'text-success'
                    item['icon'] = 'fas fa-volume-off'
                if 'error' in item['station'].lower():
                    item['myClass'] = 'text-warning'
                    item['icon'] = 'fas fa-circle'
                    error_data.append(item)
                elif 'GPS' in item['station']:
                    item['myClass'] = 'text-warning'
                    item['icon'] = 'fas fa-circle'
                    error_data.append(item)
                elif 'ERROR' in item['station']:
                    item['myClass'] = 'text-warning'
                    item['icon'] = 'fas fa-circle'
                    error_data.append(item)
                elif 'depot' in item['station'].lower():
                    item['myClass'] = 'text-warning'
                    item['icon'] = 'fas fa-home'
                    halt.append(item)
                elif item['station'] == "":
                    item['myClass'] = 'text-warning'
                    item['icon'] = 'fas fa-circle'

        # print(error_data)

        length = len(district_data)
        not_len = len(not_run)
        sp_len = len(speaker_issue)
        halt_len = len(halt)
        error_len = len(error_data)
        total_halt_len = not_len + halt_len
        active_lent = length - total_halt_len

        # print(active_lent)
    else:
        district_data = []
        speaker_issue = []
        not_run = []
        error_data = []
        halt = []
        length = len(district_data)
        not_len = len(not_run)
        sp_len = len(speaker_issue)
        halt_len = len(halt)
        error_len = len(error_data)
        total_halt_len = not_len + halt_len
        active_lent = length - total_halt_len

    context = {
        'sp_len': sp_len,
        'halt_len': halt_len,
        'error_len': error_len,
        'not_len': not_len,
        'length': length,
        'total_halt_len': total_halt_len,
        'active_lent': active_lent,
        'districts': districts,
        'depo': depo,
        'route': route,
        'district_data': district_data,
        'dist': dist,
        'depot': depot,
        'routes': routes,
        's_l': speaker_issue,
        'error': error_data,
        'not_run': not_run,
        'halt': halt,
        'franchise': franchise
    }
    return render(request, 'Dashboard/track.html', context)

@login_required()
@user_passes_test(is_patner)
def change_password(request):
    if request.method == 'POST':
        form = PasswordChangeForm(user=request.user, data=request.POST)
        if form.is_valid():
            user = form.save()
            update_session_auth_hash(request, user)  # Important to maintain user session
            messages.success(request, 'Your password was successfully updated!')
            return redirect('Dashboard:dashboard')
        else:
            messages.error(request, 'Please correct the error below.')
    else:
        form = PasswordChangeForm(user=request.user)
    context = {'form': form}
    return render(request, 'common/change_password.html', context)

def Franchise_signup_view(request):
    userForm = FranchiseBaseForm()
    FranchiseUser = FranchiseUserForm()
    dist = District.objects.all().filter(is_Active=True)
    mydict = {'userForm': userForm, 'FranchiseUser': FranchiseUser, 'dist': dist}
    if request.method == 'POST':
        userForm = FranchiseBaseForm(request.POST)
        FranchiseUser = FranchiseUserForm(request.POST)
        if userForm.is_valid() and FranchiseUser.is_valid():
            user = userForm.save()
            user.set_password(user.password)
            user.save()
            Franchise = FranchiseUser.save(commit=False)
            Franchise.user = user
            # get the selected districts and add them to the FranchiseUser object
            selected_districts = FranchiseUser.cleaned_data.get('district')
            Franchise.save()
            Franchise.district.add(*selected_districts)
            my_group = Group.objects.get_or_create(name='Franchise')
            my_group[0].user_set.add(user)
        return redirect('Login')
    return render(request, 'Dashboard/signup.html', context=mydict)


import json
from urllib.request import proxy_bypass

from constants.constants import FORBIDDEN_REQUEST_FOUND, MODEL_ALREADY_EXIST, MODEL_CREATION_FAILED, MODEL_PARAM_MISSED, MODEL_RECORD_NOT_FOUND
from core.models import  Batch, Group, ProjectTitle, TopProject,User, Voter
from django.db import transaction
from django_filters import rest_framework as filters
from django_filters.rest_framework import DjangoFilterBackend
from pkg.util import error_response, success_response
from rest_framework.decorators import action
from rest_framework import viewsets
from rest_framework.filters import OrderingFilter, SearchFilter
from rest_framework.response import Response
from core.permissions import IsAdmin
from django.db.models import F
from rest_framework.decorators import action
from django.db.models import  Sum
# from django.core import serializers

from .serializer import TopProjectSerializer,VoterSerializer
class TopProjectFilter(filters.FilterSet):
    project_id = filters.NumberFilter(field_name="project_id")
    user_id = filters.NumberFilter(field_name="user_id")
    group = filters.NumberFilter(field_name="group")
    batch = filters.CharFilter(field_name="batch")

    class Meta:
        model = TopProject
        fields = ["title", "batch", "group","doc_path","vote","description","is_approved"]

class TopProjectViewSet(viewsets.ModelViewSet):
    serializer_class = TopProjectSerializer
    filter_backends = [DjangoFilterBackend, OrderingFilter, SearchFilter]
    filter_class = TopProjectFilter
    ordering_fields = ["batch", "group"]
    search_fields = ["batch", "group"]
    filterset_fields = [
        "batch","group","id"
    ]
    queryset=None
    def get_queryset(self):
        print("fetching ...")
        queryset = TopProject.objects.all().order_by('id')
        return queryset

    @transaction.atomic
    def create(self, request, *args, **kwargs):
        print("creating ...")
        form_data = request.data
        global user_obj
        global project_obj
        global batch_obj
        global title_obj
        global group_obj
        usr=self.request.user.id
        print("user id =>",usr)
        try:
            user_obj = User.objects.get(id=int(usr))
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "User")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        try:
            batch_obj = Batch.objects.get(name=form_data["batch"])
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "Batch")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        try:
            group_obj = Group.objects.get(id=form_data["group"])
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "Group")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        try:
            title_obj = ProjectTitle.objects.get(id=form_data["id"])
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "Title")
            return Response(res,status=int(res['status_code']), content_type="application/json")

        project_obj = TopProject.objects.create(
            group=group_obj,
            batch=batch_obj,
            title=title_obj,
            description=form_data["description"]
        )
        serializer = TopProjectSerializer(project_obj)
        data = success_response(serializer.data)
        return Response((data))

    def retrieve(self, request, pk=None):
        if not pk.isdigit():
            res = error_response(request, MODEL_PARAM_MISSED, "TopProject")
            res['message']='Invalid request parameter found.'
            return Response(res,status=int(res['status_code']), content_type="application/json")
        try:
            queryset = TopProject.objects.get(pk=pk)
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "TopProject")
            res['message']='TopProject not found with id '+pk+"."
            return Response(res,status=int(res['status_code']), content_type="application/json")
        serializer=TopProjectSerializer(queryset)
        return Response(serializer.data)
    @action(
        detail=True,
        methods=["PUT"],
        permission_classes=[IsAdmin],
        url_path="vote",
    )
    def vote(self, request,*args, **kwargs):
        print("updating ...")
        pk = int(kwargs["pk"])
        usr=self.request.user.id
        voter_count=0
        top_project_obj = None
        user_obj = None
        voter_count = Voter.objects.filter(user_id=usr).count()
        if voter_count==1:
            voter = Voter.objects.get(user_id=usr)
            # print("User Information => ","user_id ->",voter.user_id.id,"project_id ->",voter.project_id.id,"pk ->",pk," voter.project_id.id != pk ->" ,voter.project_id.id != pk)
            if voter.user_id.id == usr and voter.project_id.id == pk:
                print("line-106")
                TopProject.objects.filter(id=pk).update(vote=F('vote')-1)
                Voter.objects.filter(user_id=usr,project_id=pk).delete()
                return Response({"message":"You unvoted successfully!"})

            elif (voter.user_id.id == usr and voter.project_id.id != pk):
                print("line-111")
                TopProject.objects.filter(id=voter.project_id.id).update(vote=F('vote')-1)
                Voter.objects.filter(user_id=voter.user_id.id,project_id=voter.project_id.id).delete()
                print("line-113")
                try:
                    user_obj = User.objects.get(id=usr)
                except:
                    res = error_response(request, MODEL_RECORD_NOT_FOUND, "User")
                    return Response(res,status=int(res['status_code']), content_type="application/json")
                try:
                    top_project_obj = TopProject.objects.get(id=pk)
                except:
                    res = error_response(request, MODEL_RECORD_NOT_FOUND, "TopProject")
                    return Response(res,status=int(res['status_code']), content_type="application/json")
                Voter.objects.create(user_id=user_obj,project_id=top_project_obj)
                TopProject.objects.filter(id=pk).update(vote=F('vote')+1)
                return Response({"message":"You Voted Successfully!"})

            else:
                print("line-132")
                pass
        else:
            print("line-119")
            top_project_obj = TopProject.objects.filter(id=pk).update(vote=F('vote')+1)
            try:
                user_obj = User.objects.get(id=usr)
            except:
                res = error_response(request, MODEL_RECORD_NOT_FOUND, "User")
                return Response(res,status=int(res['status_code']), content_type="application/json")
            try:
                top_project_obj = TopProject.objects.get(id=pk)
            except:
                res = error_response(request, MODEL_RECORD_NOT_FOUND, "TopProject")
                return Response(res, status=int(res['status_code']),content_type="application/json")
            Voter.objects.create(user_id=user_obj,project_id=top_project_obj)
            return Response({"message":"You Voted Successfully!"})

    def destroy(self, request, *args, **kwargs):
        print("deleting ...")
        instance = TopProject.objects.filter(id=int(kwargs["pk"]))
        if len(instance) != 1:
            res = error_response(self.request, MODEL_RECORD_NOT_FOUND, "TopProject")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        instance.delete()
        return Response({"result": "TopProject instance was successfuly deleted!"})


#!/usr/bin/env python3

import json,sys

data=json.load(sys.stdin)

if len(data) == 0:
    print("No data")
else:
    for issue in data:
        fileName = ".git/issues/{0}".format(issue["number"])
        f = open(fileName, "w")
        f.write(issue["title"])
        f.close()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from collections import namedtuple

GenericURISpec = namedtuple('GenericURISpec', [
    'namespace',
    'doctype',
    'docversion',
    'docid',
])

# See: https://docs.telemetry.mozilla.org/concepts/pipeline/http_edge_spec.html#postput-request
TelemetryURISpec = namedtuple('TelemetryURISpec', [
    'namespace',
    'docid',
    'doctype',
    'appName',
    'appVersion',
    'appUpdateChannel',
    'appBuildId'
])

def build_route(spec):
    # drop everything after the first None
    if None in spec:
        spec = spec[:spec.index(None)]
    return '/'.join(['/submit'] + list(map(str, spec)))


# https://github.com/Practical-CV/EYE-BLINK-DETECTION-WITH-OPENCV-AND-DLIB

from scipy.spatial import distance as dist
from imutils import face_utils

import numpy as np

import time
import dlib
import os
import cv2

# load facial landmark detector
model_root = os.getcwd() + '/models'

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(model_root + "/shape_predictor_68_face_landmarks_GTX.dat")

(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS['left_eye']
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS['right_eye']

# cnn detector + predictor
import caffe
os.environ['GLOG_minloglevel'] = '2'

model_weight = model_root + "/mmod_human_face_detector.dat"
detector_cnn = dlib.cnn_face_detection_model_v1(model_weight)

net_work_path = model_root + '/landmark_deploy.prototxt'
weight_path = model_root + '/VanFace.caffemodel'

net = caffe.Net(net_work_path, weight_path, caffe.TEST)
net.name = 'FaceThink_face_landmark'

def current_time():
    return int(round(time.time() * 1000))

def eye_aspect_ratio(eye):
    A = dist.euclidean(eye[1], eye[5])
    B = dist.euclidean(eye[2], eye[4])
    C = dist.euclidean(eye[0], eye[3])

    return (A + B) / (2.0 * C)

def face_rect(frame, rect):

    x1 = max(0, rect.left())
    y1 = max(0, rect.top())
    x2 = min(rect.right(), frame.shape[1])
    y2 = min(rect.bottom(), frame.shape[0])

    gray = frame[y1:y2+1, x1:x2+1,]
    # gray = cv2.cvtColor(roi, cv2.COLOR_GRAY2GRAY)

    w, h = 60, 60
    res = cv2.resize(gray, (w, h), 0.0, 0.0, interpolation=cv2.INTER_CUBIC)
    resize_mat = np.float32(res)

    m = np.zeros((w, h))
    sd = np.zeros((w, h))
    mean, std_dev = cv2.meanStdDev(resize_mat, m, sd)
    new_m = mean[0][0]
    new_sd = std_dev[0][0]
    img = (resize_mat - new_m) / (0.000001 + new_sd)

    return img

# get 68 points
def landmarks(points, rect, frame):
    x1 = max(0, rect.left())
    y1 = max(0, rect.top())
    x2 = min(rect.right(), frame.shape[1])
    y2 = min(rect.bottom(), frame.shape[0])

    res = []
    for i in range(len(points) // 2):
        x = points[2*i] * (x2 - x1) + x1
        y = points[2*i+1] * (y2 - y1) + y1
        res.append([x, y])

    return np.array(res)

def extract_ears(img, grayed=False, cnn=False, debug=False):
    gray = img
    if not grayed:
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    start_ms = current_time()
    ear = -1.0
    center = np.array([.0, .0])

    if cnn:
        rects = [r.rect for r in detector_cnn(gray, 0)]
    else:
        rects = detector(gray, 0)

    # take only the first
    for rect in rects[:1]:
        # if cnn:
        #     net.blobs['data'].data[...] = face_rect(gray, rect)
        #     net.forward()
        #     shape = net.blobs['Dense3'].data[0].flatten()
        #     shape = landmarks(shape, rect, gray)

        shape = predictor(gray, rect)
        shape = face_utils.shape_to_np(shape)

        leftEye = shape[lStart:lEnd]
        rightEye = shape[rStart:rEnd]

        leftEAR = eye_aspect_ratio(leftEye)
        rightEAR = eye_aspect_ratio(rightEye)

        l_center = sum(leftEye) / len(leftEye)
        r_center = sum(rightEye) / len(rightEye)

        ear = (leftEAR + rightEAR) / 2.0
        center = (l_center + r_center) / 2.0

        if debug:
            leftEyeHull = cv2.convexHull(leftEye)
            rightEyeHull = cv2.convexHull(rightEye)

            cv2.drawContours(img, [leftEyeHull], -1, (0, 255, 0), 1)
            cv2.drawContours(img, [rightEyeHull], -1, (0, 255, 0), 1)

        duration_ms = current_time() - start_ms
        print("Eye aspect ratio took:     ", str(duration_ms / 1000) + "s")

    return ear, center


import stat

import pytest
from ruamel import yaml

from ansiblediscover.utils.fs import FS

def test_load_yaml_raises_on_invalid_yaml(tmpdir):
    content = 'this: is: invalid'

    task_file = tmpdir.join("mydata.yml")
    task_file.write(content)

    with pytest.raises(TypeError):
        FS.load_yaml(str(task_file))

def test_load_yaml_raises_on_missing_file(tmpdir):
    task_file = tmpdir.join('mydata.yml')

    with pytest.raises(OSError):
        FS.load_yaml(str(task_file))

def test_load_yaml_raises_on_wrong_file_permissions(tmpdir):
    task_file = tmpdir.join("mydata.yml")
    task_file.write('irrelevant')
    task_file.chmod(stat.S_IWUSR)

    with pytest.raises(OSError):
        FS.load_yaml(str(task_file))

def test_load_yaml_succeeds_on_valid_yaml(tmpdir):
    content = ['hello', {'this': 'is valid yaml'}]

    task_file = tmpdir.join("mydata.yml")
    task_file.write(yaml.dump(content))

    assert content == FS.load_yaml(str(task_file))

def test_load_yaml_succeeds_on_tags(tmpdir):
    def default_ctor(loader, tag_suffix, node):
        return tag_suffix + ' ' + node.value

    yaml.add_multi_constructor('', default_ctor)
    content = yaml.load("""\
        data_contains: !vault |
            $ANSIBLE_VAULT;1.1;AES256
            1234567890ABCDEFGHIJKLMNO
    """)

    task_file = tmpdir.join("mydata.yml")
    task_file.write(yaml.dump(content))

    assert content == FS.load_yaml(str(task_file))


import os

import sys

from mai_experiments.experiment_settings import DebugPrintingOptions, FileNameData, FoldController
from mai_experiments.fold_file_parser import main_cross_validation

# CHANGE THESE TWO FOR EACH TEST
test_name = 'bongard4'
logic_name = 'bongard'

# --- command-line printing settings ---
debug_printing_options = DebugPrintingOptions()

filter_out_unlabeled_examples = False
hide_printouts = True

# --- directories ---
droot = '/home/joschout/Repos/tilde/mai_experiments/data'
dlogic_relative = 't-0-0-0'
dfold_relative = 'folds'
dout_relative = 'output'

file_name_data = FileNameData(root_dir=droot,
                              logic_relative_dir=dlogic_relative,
                              fold_relative_dir=dfold_relative,
                              output_relative_dir=dout_relative,
                              test_name=test_name,
                              logic_name=logic_name)

# --- fold settings ---
fname_prefix_fold = 'test'
fold_start_index = 0
nb_folds = 10
fold_suffix = '.txt'

fold_settings = FoldController(
    fold_file_directory=file_name_data.fold_dir,
    fold_fname_prefix=fname_prefix_fold,
    fold_start_index=fold_start_index,
    nb_folds=nb_folds,
    fold_suffix=fold_suffix)

# -- create output directory
if not os.path.exists(file_name_data.output_dir):
    os.makedirs(file_name_data.output_dir)

print("start bongard4")
save_stdout = sys.stdout
if hide_printouts:
    sys.stdout = open(os.devnull, "w")

main_cross_validation(file_name_data,
                      fold_settings,
                      filter_out_unlabeled_examples,
                      debug_printing_options)
if hide_printouts:
    sys.stdout = save_stdout
print("finished bongard4")


from django import template
from django.utils.safestring import mark_safe

register = template.Library()

def _review_score_number(context, score):
    if score is None:
        return '×'

    max_score = context['request'].event.settings.get('review_max_score') if context else None
    if isinstance(score, int) or (isinstance(score, float) and score.is_integer()):
        score = int(score)
        name = context and context['request'].event.settings.get(f'review_score_name_{score}')
        if name:
            return f'{score}/{max_score} (“{name}”)'
    else:
        score = round(score, 1)

    return f'{score}/{max_score}'

def _review_score_override(positive_overrides, negative_overrides):
    result = ''
    if positive_overrides:
        result += f''
        if positive_overrides > 1 or negative_overrides:
            result += f' {positive_overrides}'
    if negative_overrides:
        result += f''
        if negative_overrides > 1 or positive_overrides:
            result += f' {negative_overrides}'
    return result

@register.simple_tag(takes_context=True)
def review_score(context, submission):
    score = submission.current_score
    if score is None:
        return '-'
    if hasattr(submission, 'has_override') and not submission.has_override:
        return _review_score_number(context, score)
    positive_overrides = submission.reviews.filter(override_vote=True).count()
    negative_overrides = submission.reviews.filter(override_vote=False).count()
    if positive_overrides or negative_overrides:
        return mark_safe(_review_score_override(positive_overrides, negative_overrides))
    return _review_score_number(context, score)


import cv2
import time
import os

SAMPLE_VIDEO_PATH = os.environ["SAMPLE_VIDEO_PATH"]

frames  = []

start = time.time()

video_source = cv2.VideoCapture(SAMPLE_VIDEO_PATH)
while video_source.isOpened():
    ret, frame = video_source.read()
    if not ret:
        break
    frames.append(frame)

end = time.time()
print(f"Total time taken for reading the complete video is : {abs(end-start)} seconds")

for frame in frames:
    cv2.imshow("Frame", frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

video_source.release()
cv2.destroyAllWindows()