#!/usr/bin/env python

import os
import sys
from config import Config, ConfigMerger
from sortedcontainers import SortedDict
import logging.handlers
import signal
import glob2
import psutil
import time
import subprocess
import requests
import traceback
import schedule
import daemon
import argparse

cfg_dir = 'cfg'
cfg_filename = 'main.cfg'

class Cam:
    cfg = Config()
    cam_cfg = []
    cam_cfg_resolver_dict = {}
    cam_streamer = []
    cam_streamer_pid = []
    cam_streamer_start_time = []
    cam_streamer_start_flag = []
    cam_streamer_poll_flag = []
    cam_capturer = []
    cam_capturer_pid = []
    cam_capturer_start_flag = []
    cam_capturer_check_flag = []
    log = logging.getLogger()
    log_handler_file = None
    main_loop_active_flag = True
    signals_name = {}

    def __init__(self, config_dir, config_filename, log_level=None):
        self.cfg_dir = config_dir
        self.cfg_filename = config_filename
        self.log_level = log_level
        self.cfg_file = os.path.join(self.cfg_dir, self.cfg_filename)

        self.read_main_config()
        self.create_dirs()
        self.setup_logging()
        self.pid_file = os.path.join(self.cfg['pid_dir'], self.cfg['pid_filename'])

        for sig in dir(signal):
            if sig.startswith('SIG') and not sig.startswith('SIG_'):
                self.signals_name[getattr(signal, sig)] = sig

    def read_main_config(self):
        if os.path.isfile(self.cfg_file):
            self.cfg.load(open(self.cfg_file))
        else:
            print('Failed to open the file: ' + self.cfg_file)
            sys.exit(1)

    def create_dirs(self):
        if not os.path.exists(self.cfg['log_dir']):
            os.makedirs(self.cfg['log_dir'])

        if not os.path.exists(self.cfg['pid_dir']):
            os.makedirs(self.cfg['pid_dir'])

    @staticmethod
    def get_log_level(log_level):
        log_levels = {
            'DEBUG': logging.DEBUG,
            'INFO': logging.INFO,
            'WARNING': logging.WARNING,
            'ERROR': logging.ERROR,
            'CRITICAL': logging.CRITICAL
        }

        if log_level is None:
            log_level = 'INFO'

        return log_levels.get(log_level.strip().upper(), logging.INFO)

    def setup_logging(self):
        if self.log_level is None:
            self.log_level = self.cfg['log_level']

        self.log_level = self.get_log_level(self.log_level)
        self.log.setLevel(self.log_level)

        logging_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')

        log_handler_stream = logging.StreamHandler()
        log_handler_stream.setFormatter(logging_formatter)
        log_handler_stream.setLevel(self.log_level)

        self.log_handler_file = logging.handlers.TimedRotatingFileHandler(
            os.path.join(self.cfg['log_dir'], self.cfg['log_filename']),
            when='midnight')
        self.log_handler_file.setFormatter(logging_formatter)
        self.log_handler_file.setLevel(self.log_level)

        self.log.addHandler(log_handler_stream)
        self.log.addHandler(self.log_handler_file)

        logging.getLogger('requests').setLevel(logging.WARNING)
        logging.getLogger('schedule').setLevel(logging.WARNING)
        sys.excepthook = self.exception_handler

    def write_main_pid(self):
        if os.path.isfile(self.pid_file):
            pid_file_content = open(self.pid_file, 'r').read()

            if len(pid_file_content.strip()):
                pid = int(pid_file_content)

                if psutil.pid_exists(pid):
                    print('Error. Already running, PID: %i' % pid)
                    sys.exit(1)

        open(self.pid_file, 'w').write(str(os.getpid()))

    def exit_handler(self, s, frame, log_signal=True, exit_code=0):
        if log_signal:
            self.log.warning('Caught signal: ' + self.signals_name[s])

        self.kill_cams_process()

        self.log.debug('Remove own PID file: ' + self.pid_file)
        if os.path.isfile(self.pid_file):
            os.remove(self.pid_file)
        else:
            self.log.warning('PID file not found: ' + self.pid_file)

        if exit_code == 0:
            self.main_loop_active_flag = False
        else:
            sys.exit(exit_code)

    def exit_child(self, s, frame, log_signal=True):
        while True:
            try:
                pid, status = os.waitpid(-1, os.WNOHANG)

                if pid != 0:
                    logging.warning('Received "%s" signal for "%s" PID with "%s" status' %
                                    (self.signals_name[s], pid, status))
                else:
                    break
            except ChildProcessError:
                self.log.debug('ChildProcessError: No child processes')
                break

    def exception_handler(self, *exception_data):
        self.log.critical('Unhandled exception:\n%s', ''.join(traceback.format_exception(*exception_data)))
        self.exit_handler(None, None, log_signal=False, exit_code=1)

    def kill_process(self, pid_file, remove_pid_file):
        pid = None

        if os.path.isfile(pid_file):
            pid_file_content = open(pid_file, 'r').read()
            self.log.debug('PID file content: "%s"' % pid_file_content)

            if len(pid_file_content.strip()):
                pid = int(pid_file_content)
                self.log.debug('Process id: %i' % pid)

                if psutil.pid_exists(pid):
                    self.log.debug('Kill process: %i' % pid)
                    try:
                        os.kill(pid, signal.SIGTERM)
                    except OSError:
                        self.log.warning('Failed to kill process: %i' % pid)
                else:
                    self.log.info('Process not found: %i' % pid)
            else:
                self.log.warning('PID is empty')

            if remove_pid_file:
                os.remove(pid_file)
        else:
            self.log.debug('Skip kill, PID file not found: ' + pid_file)

        return pid

    def kill_cam_processes(self, cam_index, cam_reset_flag=False, kill_streamer_flag=True, kill_capturer_flag=True):
        self.log.info('Stop cam: ' + self.cam_cfg[cam_index]['name'])

        if kill_capturer_flag:
            self.log.debug('Kill %s capturer' % self.cam_cfg[cam_index]['name'])
            self.kill_process(self.cam_capturer_pid[cam_index], True)

        if kill_streamer_flag:
            self.log.debug('Kill %s streamer' % self.cam_cfg[cam_index]['name'])
            self.kill_process(self.cam_streamer_pid[cam_index], True)

        if cam_reset_flag:
            try:
                self.cam_cfg[cam_index]['reset_cmd']
            except AttributeError:
                self.log.debug('Cam reset command not found. Skip reset')
            else:
                self.log.info('Resetting cam: ' + self.cam_cfg[cam_index]['name'])
                self.log.debug('Reset command: ' + self.cam_cfg[cam_index]['reset_cmd'])
                return_code = subprocess.call(self.cam_cfg[cam_index]['reset_cmd'], shell=True)
                self.log.info('Reseted with exit code: %s' % return_code)

    def kill_cams_process(self, cam_reset_flag=False):
        for iterator, _ in enumerate(self.cam_cfg):
            self.kill_cam_processes(iterator, cam_reset_flag=cam_reset_flag)

    def bg_run(self, cmd, pid_file=None):
        self.log.debug('Running:\n' + cmd)

        subproc = subprocess.Popen(cmd, shell=True,
                                   stdout=subprocess.DEVNULL,
                                   stdin=subprocess.DEVNULL,
                                   stderr=subprocess.DEVNULL)

        self.log.debug('Started PID: %s' % subproc.pid)

        if pid_file is not None:
            open(pid_file, 'w').write(str(subproc.pid))

        return subproc

    def replacer(self, s, cam_index):
        s = s.replace('[cam_name]', self.cam_cfg[cam_index]['name'])
        s = s.replace('[cams_number]', str(len(self.cam_cfg)))
        return s

    def cleaner(self):
        self.log.debug('Cleaner started')
        clean_flag = False
        store_file_list = None

        if int(self.cfg['cleaner_store_max_gb']) != 0:
            store_file_list = glob2.glob(os.path.join(self.cfg['cap_dir'], '**'))
            self.log.debug('Found files: %s' % store_file_list)

            store_file_total_size_bytes = 0
            for store_file in store_file_list:
                store_file_total_size_bytes += os.path.getsize(store_file)

            store_file_total_size_gigabytes = 1.0 * store_file_total_size_bytes / 1024 / 1024 / 1024
            self.log.debug('Store files size, Gb: %f' % store_file_total_size_gigabytes)

            if store_file_total_size_gigabytes > float(self.cfg['cleaner_store_max_gb']):
                self.log.info('Current store size / Configured max store size, Gb: %.3f/%.3f' %
                              (store_file_total_size_gigabytes, self.cfg['cleaner_store_max_gb']))
                clean_flag = True

        if int(self.cfg['cleaner_store_keep_free_gb']) != 0:
            store_stat = os.statvfs(self.cfg['cap_dir'])
            store_free_gb = 1.0 * store_stat.f_bavail * store_stat.f_frsize / 1024 / 1024 / 1024
            self.log.debug('Store free space, Gb: %f' % store_free_gb)

            if store_free_gb < float(self.cfg['cleaner_store_keep_free_gb']):
                self.log.info('Current store free space / Configured keep store free space, Gb: %.3f/%.3f' %
                              (store_free_gb, self.cfg['cleaner_store_keep_free_gb']))
                clean_flag = True

        if clean_flag:
            self.log.info('Clean is active')

            if store_file_list is None:
                store_file_list = glob2.glob(os.path.join(self.cfg['cap_dir'], '**'))

            store_file_list_sorted = SortedDict()
            for store_file in store_file_list:
                store_file_list_sorted.update({os.path.getmtime(store_file): store_file})

            self.log.debug('Sorted files list (with last modification date): %s' % store_file_list_sorted)
            self.log.debug('Sorted files list (by last modification date): %s' % store_file_list_sorted.values())

            removes = 0
            for file_name in store_file_list_sorted.values():
                if os.path.isfile(file_name):
                    file_size = os.path.getsize(file_name)
                    self.log.info('Remove file: ' + file_name)
                    os.remove(file_name)

                    if file_size > int(self.cfg['cleaner_force_remove_file_less_bytes']):
                        removes += 1
                    else:
                        self.log.warning('Removed "%s" file with the "%s" bytes size' % (file_name, file_size))

                    if removes == int(self.cfg['cleaner_max_removes_per_run']):
                        self.log.debug('Max removes reached: ' + self.cfg['cleaner_max_removes_per_run'])
                        break

        self.log.debug('Cleaner finished')

    def configs_resolver(self, map1, map2, key):
        self.cam_cfg_resolver_dict[key] = map1[key]
        return "overwrite"

    def main(self):
        self.log.info('Start')
        self.log.debug('Started: ' + os.path.abspath(__file__))
        self.log.debug('Setting SIGTERM, SIGINT, SIGCHLD handlers')
        signal.signal(signal.SIGTERM, self.exit_handler)
        signal.signal(signal.SIGINT, self.exit_handler)
        signal.signal(signal.SIGCHLD, self.exit_child)

        # Read cam configs
        cam_cfg_dir = os.path.join(self.cfg_dir, self.cfg['cam_cfg_mask'])
        self.log.debug('Configs search path: ' + cam_cfg_dir)

        cam_cfg_list = glob2.glob(os.path.join(self.cfg_dir, self.cfg['cam_cfg_mask']))
        cam_cfg_list.remove(self.cfg_file)
        self.log.debug('Found configs: %s' % cam_cfg_list)

        if len(cam_cfg_list) == 0:
            self.log.critical('No cam config found. Exit')
            sys.exit(0)

        for cur_cam_cfg in cam_cfg_list:
            self.log.debug('Read cam config: ' + cur_cam_cfg)
            tmp_cfg = Config(open(cur_cam_cfg))
            cur_cam_cfg_active_flag = True

            try:
                tmp_cfg['active']
            except AttributeError:
                self.log.debug('active flag not found')
            else:
                cur_cam_cfg_active_flag = tmp_cfg['active']

            if cur_cam_cfg_active_flag:
                self.cam_cfg.append(tmp_cfg)
                self.cam_cfg_resolver_dict.clear()
                merger = ConfigMerger(resolver=self.configs_resolver)
                merger.merge(self.cam_cfg[-1], self.cfg)

                for key in self.cam_cfg_resolver_dict:
                    self.cam_cfg[-1][key] = self.cam_cfg_resolver_dict[key]

                self.log.debug('Loaded settings for: ' + self.cam_cfg[-1]['name'])
            else:
                self.log.debug('Cam config is skipped due active flag: ' + cur_cam_cfg)
        # End Read cam configs

        # Cleaner
        self.cfg['cleaner_max_removes_per_run'] = self.replacer(str(self.cfg['cleaner_max_removes_per_run']), 0)
        schedule.every(self.cfg['cleaner_run_every_minutes']).minutes.do(self.cleaner)
        # End Cleaner

        # PIDs full path
        for iterator, cam in enumerate(self.cam_cfg):
            try:
                pid_streamer = cam['pid_streamer']
            except AttributeError:
                self.log.debug('pid_streamer not found for cam: ' + cam['name'])
                try:
                    pid_streamer = self.cfg['pid_streamer']
                except AttributeError:
                    self.log.critical("Can't find pid_streamer in config")
                    sys.exit(1)

            try:
                pid_capturer = cam['pid_capturer']
            except AttributeError:
                self.log.debug('pid_capturer not found for cam: ' + cam['name'])
                try:
                    pid_capturer = self.cfg['pid_capturer']
                except AttributeError:
                    self.log.critical("Can't find pid_capturer in config")
                    sys.exit(1)

            self.cam_streamer_pid.append(self.replacer(os.path.join(self.cfg['pid_dir'], pid_streamer), iterator))
            self.cam_capturer_pid.append(self.replacer(os.path.join(self.cfg['pid_dir'], pid_capturer), iterator))
        # End PIDs full path

        self.kill_cams_process()
        self.write_main_pid()

        while self.main_loop_active_flag:
            for iterator, cam in enumerate(self.cam_cfg):
                if len(self.cam_streamer) == iterator:

                    # Create cam cap dir only if cap_cmd is not False
                    try:
                        cap_cmd = self.cam_cfg[iterator]['cap_cmd']
                    except AttributeError:
                        cap_cmd = None
                        self.log.debug('Capture command not found')

                    if cap_cmd is not False:
                        cap_dir_cam = self.replacer(self.cfg['cap_dir_cam'], iterator)
                        if not os.path.exists(cap_dir_cam):
                            try:
                                os.makedirs(cap_dir_cam)
                            except OSError:
                                self.log.critical('Failed to create directory: ' + cap_dir_cam)
                                sys.exit(1)
                    # End Create cam cap dir

                    self.cam_streamer_start_flag.append(True)

                    self.cam_streamer.append(None)
                    self.cam_streamer_start_time.append(0)
                    self.cam_streamer_poll_flag.append(False)

                    self.cam_capturer.append(None)
                    self.cam_capturer_start_flag.append(False)
                    self.cam_capturer_check_flag.append(False)
                else:
                    if self.cam_streamer[iterator].poll() is None:
                        self.log.debug('Streamer "%s" is alive' % cam['name'])
                    else:
                        self.log.warning('Streamer "%s" is dead (exit code: %s)' %
                                         (cam['name'], self.cam_streamer[iterator].returncode))
                        self.cam_streamer_start_flag[iterator] = True

                # Capturer alive check
                if self.cam_capturer_check_flag[iterator]:
                    if self.cam_capturer[iterator].poll() is None:
                        self.log.debug('Capturer "%s" is alive' % cam['name'])
                    else:
                        self.log.warning('Capturer "%s" is dead (exit code: %s)' %
                                         (cam['name'], self.cam_capturer[iterator].returncode))
                        self.cam_streamer_poll_flag[iterator] = True
                        self.cam_capturer_check_flag[iterator] = False
                # End Capturer alive check

                # Run streamer
                if self.cam_streamer_start_flag[iterator]:
                    self.log.info('Run "%s" streamer in background' % cam['name'])
                    self.cam_streamer[iterator] = self.bg_run(cam['cmd'].strip(), self.cam_streamer_pid[iterator])
                    self.cam_streamer_start_time[iterator] = time.time()
                    self.cam_streamer_poll_flag[iterator] = True
                    self.cam_streamer_start_flag[iterator] = False
                # End Run streamer

                # Poll streamer
                if self.cam_streamer_poll_flag[iterator]:
                    cap_url = self.cfg['cap_url']
                    cap_url = self.replacer(cap_url, iterator)

                    self.log.debug('Getting HTTP status: ' + cap_url)
                    http_code = 0

                    try:
                        http_code = requests.head(cap_url, timeout=1).status_code
                    except requests.exceptions.RequestException:
                        self.log.warning('Failed to connect: ' + cap_url)

                    if http_code != 0:
                        self.log.info('Checked "%s", status: %s' % (cam['name'], http_code))

                        if http_code == 200:
                            self.cam_capturer_start_flag[iterator] = True
                            self.cam_streamer_poll_flag[iterator] = False

                    start_time_delta = time.time() - self.cam_streamer_start_time[iterator]
                    if self.cam_streamer_poll_flag[iterator]:
                        if start_time_delta > cam['max_start_seconds']:
                            self.log.warning('Time outed waiting data from: ' + cam['name'])
                            self.log.info('Kill: ' + cam['name'])
                            self.kill_cam_processes(iterator, cam_reset_flag=True)
                            self.cam_streamer_start_flag[iterator] = True
                        else:
                            self.log.info('Attempt "%s": [%i/%i]' %
                                          (cam['name'], start_time_delta, cam['max_start_seconds']))
                # End Poll streamer

                # Run capturer
                if self.cam_capturer_start_flag[iterator]:
                    if self.cam_capturer[iterator] is not None and self.cam_capturer[iterator].poll() is None:
                        self.log.warning('Capturer "%s" is STILL alive' % cam['name'])
                    else:
                        cap_cmd = None
                        try:
                            cap_cmd = self.cam_cfg[iterator]['cap_cmd']
                        except AttributeError:
                            self.log.debug('Capture command not found in cam config. Using global')
                            try:
                                cap_cmd = self.cfg['cap_cmd']
                            except AttributeError:
                                self.log.critical('Capture command not found. Exit')
                                self.exit_handler(None, None, log_signal=False, exit_code=1)

                        if cap_cmd is not False:
                            cap_cmd = self.replacer(cap_cmd, iterator)

                            self.log.info('Run "%s" capturer in background' % cam['name'])
                            self.cam_capturer[iterator] = self.bg_run(cap_cmd, self.cam_capturer_pid[iterator])
                            self.cam_capturer_check_flag[iterator] = True
                        else:
                            self.log.info('Capturer "%s" is turned off' % cam['name'])

                    self.cam_capturer_start_flag[iterator] = False
                # End Run capturer

            schedule.run_pending()
            time.sleep(1)

        self.log.info('Finish')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-daemon', choices=['start', 'stop', 'restart'],
                        help='Daemon mode startup options')
    parser.add_argument('-log_level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
                        help='Override config log_level')
    args = parser.parse_args()

    script_dir = os.path.dirname(os.path.abspath(__file__))
    os.chdir(script_dir)

    if args.log_level:
        c = Cam(cfg_dir, cfg_filename, args.log_level)
    else:
        c = Cam(cfg_dir, cfg_filename)

    if args.daemon:
        if args.daemon == 'stop' or args.daemon == 'restart':
            c.log.debug('[Daemon] Stopping')
            main_pid = c.kill_process(c.pid_file, False)

            if main_pid:
                kill_time = time.time()
                killed_flag = False

                timeout = 20

                while time.time() - kill_time < timeout:
                    if not psutil.pid_exists(main_pid):
                        killed_flag = True
                        break

                if not killed_flag:
                    c.log.warning('[Daemon] Time outed waiting process to exit (timeout: "%i" seconds). PID: "%i"' %
                                  (timeout, main_pid))

        if args.daemon == 'start' or args.daemon == 'restart':
            c.log.debug('[Daemon] Starting from working directory: ' + script_dir)
            with daemon.DaemonContext(working_directory=script_dir, files_preserve=[c.log_handler_file.stream]):
                c.main()
    else:
        c.main()


from JupyterReviewer.DataTypes.GenericData import GenericData
from JupyterReviewer.Data import Data, DataAnnotation

import pandas as pd
import numpy as np
from typing import Union, List, Dict

class MutationData(Data):
    """
    This Data object is Deprecated. See GeneratMutationData
    """
    def __init__(
        self,
        index,
        description,
        mutations_df: pd.DataFrame,
        bams_df: pd.DataFrame,
        mutations_df_sample_col: str,
        chrom_col: str,
        start_pos_col: str,
        bam_df_sample_col: str,
        bam_col: str,
        bai_col: str = None,
        annot_df: pd.DataFrame = None,
        annot_col_config_dict: Dict = None,
        history_df: pd.DataFrame = None,
    ):
        """
        Parameters
        ==========
        mutations_df: pd.DataFrame
            Dataframe with mutations for each sample

        """
        super().__init__(
            index=index,
            description=description,
            annot_df=annot_df,
            annot_col_config_dict=annot_col_config_dict,
            history_df=history_df,
        )

        required_mutations_df_cols = [mutations_df_sample_col, chrom_col, start_pos_col]
        if not set(
            required_mutations_df_cols
        ).issubset(mutations_df.columns):
            missing_cols = [c for c in required_mutations_df_cols if c not in mutations_df.columns]
            raise ValueError(f'Following columns do not exist in mutations_df dataframe: {missing_cols}')

        required_bam_df_cols = [bam_df_sample_col, bam_col]
        if not set(
            required_bam_df_cols
        ).issubset(bams_df.columns):
            missing_cols = [c for c in required_bam_df_cols if c not in bams_df.columns]
            raise ValueError(f'Following columns do not exist in bams_df dataframe: {missing_cols}')

        self.mutations_df = mutations_df
        self.bams_df = bams_df

        self.mutations_df_sample_col = mutations_df_sample_col
        self.chrom_col = chrom_col
        self.start_pos_col = start_pos_col

        self.bam_df_sample_col = bam_df_sample_col
        self.bam_col = bam_col
        self.bai_col = bai_col

        self.group_muts_cols = [chrom_col, start_pos_col]
        # self.index = mutations_df[self.group_muts_cols].unique()


import time
import threading
try:
    from greenlet import getcurrent as get_ident
except ImportError:
    try:
        from thread import get_ident
    except:
        from _thread import get_ident

class CameraEvent(object):
    '''
      An event-like class signaling all active
      clients when a new frame is available
    '''

    def __init__(self):
        self.events = {}

    def wait(self):
        '''
          invoked from each client's thread
          to wait for the next frame
        '''
        ident = get_ident()
        if ident not in self.events:
            # for a new client -> add entry
            self.events[ident] = [threading.Event(), time.time()]
        return self.events[ident][0].wait()

    def set(self):
        '''
          invoked by camera thread when
          a new frame is available
        '''
        now = time.time()
        remove = None
        for ident, event in self.events.items():
            if not event[0].isSet():
                # if the client's evnet is not set, set it
                # and also update the last set timestamp to now
                event[0].set()
                event[1] = now
            else:
                # if it's already set, it means the client
                # didn't process a previous frame
                # if the event stays set for more than 5 seconds,
                # then assume the client is gone and remove it
                if now - event[1] > 5:
                    remove = ident
        if remove:
            del self.events[remove]

    def clear(self):
        '''
          invoked from each client's thread
          after a frame was processed
        '''
        self.events[get_ident()][0].clear()

class CameraBase(object):
    thread = None  # the thread that reads from camera
    frame = None  # current frame of the camera
    last_access_time = 0  # time of last client access
    event = CameraEvent()

    def __init__(self):
        '''
          initialize and start camera thread
        '''
        if CameraBase.thread is None:
            CameraBase.last_access_time = time.time()

            # start camera
            CameraBase.thread = threading.Thread(target=self._thread)
            CameraBase.thread.start()

            # wait till frames become available
            while self.get_frame() is None:
                time.sleep(0)

    def get_frame(self):
        '''
          get new frame
        '''
        CameraBase.last_access_time = time.time()

        CameraBase.event.wait()
        CameraBase.event.clear()

        return CameraBase.frame

    @staticmethod
    def capture():
        '''
            functionality to capture a new image independently
        '''
        raise RuntimeError('Camera capture is not implemented')

    @staticmethod
    def frames():
        '''
          the main generator that produces the camera frames
        '''
        raise RuntimeError('Camera frames is not implemented')

    @classmethod
    def _thread(cls):
        '''
            camera background thread
        '''
        print('-> starting camera thread for new viewer')
        frame_iter = cls.frames()
        for frame in frame_iter:
            CameraBase.frame = frame
            CameraBase.event.set()
            time.sleep(0)

            if time.time() - CameraBase.last_access_time > 10:
                frame_iter.close()
                print('>>> camera goes to sleep due to inactivity')
                break
        CameraBase.thread = None


# parabolic equation
import math

#### CHANGE THESE VALUES
r0 = 2 			# 2 mm (radius of quadrupole)
height = 1.8	# put height in mm
#### END OF CHANGING VALUES

x = math.sqrt((r0+height)**2 - r0**2)
x_total_height =  x * 2
# x = math.sqrt(y**2-r0**2)
print("x: ", x_total_height, "mm")
print("height: ", height, "mm")


# Measure the distance or depth with an grove Ultrasonic sound
# sensor and a Raspberry Pi.  Imperial and Metric measurements are available

# Al Audet
# MIT License
from __future__ import division

import time
import math
import warnings
import RPi.GPIO as GPIO
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from constant_values import *

class Measurement(object):
    # Create a measurement using a HC-SR04 Ultrasonic Sensor connected to
    # the GPIO pins of a Raspberry Pi.
    # Metric values are used by default. For imperial values use
    # unit='imperial'
    # temperature=<Desired temperature in Fahrenheit>

    def __init__(
        self, trig_pin = TRIG_PIN, temperature=20, unit="metric", gpio_mode=GPIO.BCM
    ):
        self.trig_pin = trig_pin
        self.temperature = temperature
        self.unit = unit
        self.gpio_mode = gpio_mode
        self.pi = math.pi
        self.median_reading = 0
        self.distance_cm = 0
        self.signal_off = 0
        self.signal_on = 0
        self.speed_of_sound = 331.3 * math.sqrt(1 + (self.temperature / 273.15))
        self.sample = []

        # setup input/output pins
        GPIO.setwarnings(False)
        GPIO.setmode(self.gpio_mode)
        # GPIO.setup(self.trig_pin, GPIO.IN)
        # GPIO.add_event_detect(self.trig_pin, GPIO.BOTH, callback = self.detect_edge)

    def raw_distance(self, sample_size=11, sample_wait=0.1):
        # Return an error corrected unrounded distance, in cm, of an object
        # adjusted for temperature in Celcius.  The distance calculated
        # is the median value of a sample of `sample_size` readings.
        # Speed of readings is a result of two variables.  The sample_size
        # per reading and the sample_wait (interval between individual samples).
        # Example: To use a sample size of 5 instead of 11 will increase the
        # speed of your reading but could increase variance in readings;
        # value = sensor.Measurement(trig_pin, echo_pin)
        # r = value.raw_distance(sample_size=5)
        # Adjusting the interval between individual samples can also
        # increase the speed of the reading.  Increasing the speed will also
        # increase CPU usage.  Setting it too low will cause errors.  A default
        # of sample_wait=0.1 is a good balance between speed and minimizing
        # CPU usage.  It is also a safe setting that should not cause errors.
        # e.g.
        # r = value.raw_distance(sample_wait=0.03)

        if self.unit == "imperial":
            self.temperature = (self.temperature - 32) * 0.5556
        elif self.unit == "metric":
            pass
        else:
            raise ValueError("Wrong Unit Type. Unit Must be imperial or metric")

        speed_of_sound = 331.3 * math.sqrt(1 + (self.temperature / 273.15))
        # sample = []
        # time.sleep(sample_wait)
        # GPIO.remove_event_detect(self.trig_pin)
        GPIO.setup(self.trig_pin, GPIO.OUT)
        GPIO.output(self.trig_pin, GPIO.LOW)
        time.sleep(sample_wait)
        GPIO.output(self.trig_pin, True)
        time.sleep(0.00001)
        GPIO.output(self.trig_pin, False)
        self.signal_on = time.time()
        signal_sent = self.signal_on
        GPIO.setup(self.trig_pin, GPIO.IN)
        # add event detect is too slow and misses the signal sent and returned
        # GPIO.add_event_detect(self.trig_pin, GPIO.BOTH, callback = self.detect_edge)
        while GPIO.input(self.trig_pin) == 0:
            self.signal_on = time.time()
            if self.signal_on - signal_sent > 0.1:
                self.signal_on = None
                break
        while GPIO.input(self.trig_pin) == 1:
            self.signal_off = time.time()
            if self.signal_off - signal_sent > 0.1:
                self.signal_off = None
                break
        if self.signal_on is not None and self.signal_off is not None:
            signal_duration = self.signal_off - self.signal_on
            self.distance_cm = signal_duration * ((self.speed_of_sound * 100) / 2)
            # print('Distance: {:.1f}cm' .format(self.distance_cm))
        else:
            self.distance_cm = 999
        # lowpass filter version
        # if signal_duration < 0.04:
        #     self.distance_cm = signal_duration * ((self.speed_of_sound * 100) / 2)
        #     print('Distance: {:.1f}cm' .format(self.distance_cm))
        #     self.sample.append(self.distance_cm)

        # no lowpass filter version

        # self.sample.append(self.distance_cm)

    def distance(self):
        # Calculate the distance from the sensor to an object.
        self.raw_distance()
        if self.unit == "imperial":
            return self.median_reading * 0.394
        else:
            # don't need this method if using metric. Use raw_distance
            # instead.  But it will return median_reading anyway if used.
            self.median_reading = self.distance_cm
            return self.median_reading

    # def detect_edge(self,channel):
    #     # print("detected edge")
    #     if GPIO.input(self.trig_pin) == 1:
    #         self.signal_on = time.time()
    #     else:
    #         self.signal_off = time.time()
    #         signal_duration = self.signal_off - self.signal_on
    #         if signal_duration < 0.04:
    #             self.distance_cm = signal_duration * ((self.speed_of_sound * 100) / 2)
    #             # print('Distance: {:.1f}cm' .format(self.distance_cm))
    #             self.sample.append(self.distance_cm)

    # @staticmethod
    # def basic_distance(trig_pin, echo_pin, celsius=20):
    #     # Return an unformatted distance in cm's as read directly from
    #     # RPi.GPIO.
    #     # currently not working

    #     sonar_signal_off = 0
    #     sonar_signal_on = 0

    #     speed_of_sound = 331.3 * math.sqrt(1 + (celsius / 273.15))
    #     GPIO.setup(trig_pin, GPIO.OUT)
    #     GPIO.setup(echo_pin, GPIO.IN)
    #     GPIO.output(trig_pin, GPIO.LOW)
    #     time.sleep(0.1)
    #     GPIO.output(trig_pin, True)
    #     time.sleep(0.00001)
    #     GPIO.output(trig_pin, False)
    #     echo_status_counter = 1
    #     while GPIO.input(echo_pin) == 0:
    #         if echo_status_counter < 1000:
    #             sonar_signal_off = time.time()
    #             echo_status_counter += 1
    #         else:
    #             raise SystemError("Echo pulse was not received")
    #     while GPIO.input(echo_pin) == 1:
    #         sonar_signal_on = time.time()

    #     time_passed = sonar_signal_on - sonar_signal_off
    #     return time_passed * ((speed_of_sound * 100) / 2)

if __name__ == "__main__":
    grove = Measurement(TRIG_PIN)
    sample9 = []
    tmp9 = []
    sample7 = []
    tmp7 = []
    sample5 = []
    tmp5 = []
    sample3 = []
    tmp3 = []
    nine_samples = open("nine_samples4.txt", "a")
    seven_samples = open("seven_samples4.txt", "a")
    five_samples = open("five_samples4.txt", "a")
    three_samples = open("three_samples4.txt", "a")
    one_samples = open("one_samples4.txt", "a")

    # how_far = grove.distance()
    # time.sleep(120)
    # GPIO.cleanup()
    try:
        while(True):
            # for i in range(9):
            #     how_far = grove.distance()
            # grove.sample.sort()
            # size_grove_sample = len(grove.sample)
            # sample.append(grove.sample[size_grove_sample//2])
            # grove.sample.clear()
            ###############################
            how_far = grove.distance()

    except KeyboardInterrupt:
        ##########################################
        # uncomment end of raw distance function #
        # for sample[] to be made                #
        ##########################################

        GPIO.cleanup()
        size_all_samples = len(grove.sample)
        for i in range(size_all_samples):
            tmp3.append(grove.sample[i])
            if 2 == i%3:
                size3 = len(tmp3)
                tmp3.sort()
                sample3.append(tmp3[size3//2])
                tmp3.clear()
            tmp5.append(grove.sample[i])
            if 4 == i%5:
                size5 = len(tmp5)
                tmp5.sort()
                sample5.append(tmp5[size5//2])
                tmp5.clear()
            tmp7.append(grove.sample[i])
            if 6 == i%7:
                size7 = len(tmp7)
                tmp7.sort()
                sample7.append(tmp7[size7//2])
                tmp7.clear()
            tmp9.append(grove.sample[i])
            if 8 == i%9:
                size9 = len(tmp9)
                tmp9.sort()
                sample9.append(tmp9[size9//2])
                tmp9.clear()
        sample3.sort()
        num_bins = 51
        n, bins, patches = plt.hist(sample3, num_bins, facecolor='blue', alpha=0.5)
        plt.savefig("sample3_4.png")
        plt.clf()
        size3 = len(sample3)
        for i in range(size3):
            three_samples.write('Distance: {:.1f}cm\n' .format(sample3[i]))
        three_samples.write('5%: {:.1f}cm\nQuartile: {:.1f}cm\nMedian: {:.1f}cm\nThird quartile: {:.1f}cm\n95%: {:.1f}cm' .format(sample3[size3//20], sample3[size3//4], sample3[size3//2], sample3[size3*3//4], sample3[size3*19//20]))
        three_samples.close()
        sample5.sort()
        num_bins = 51
        n, bins, patches = plt.hist(sample5, num_bins, facecolor='blue', alpha=0.5)
        plt.savefig("sample5_4.png")
        plt.clf()
        size5 = len(sample5)
        for i in range(size5):
            five_samples.write('Distance: {:.1f}cm\n' .format(sample5[i]))
        five_samples.write('5%: {:.1f}cm\nQuartile: {:.1f}cm\nMedian: {:.1f}cm\nThird quartile: {:.1f}cm\n95%: {:.1f}cm' .format(sample5[size5//20], sample5[size5//4], sample5[size5//2], sample5[size5*3//4], sample5[size5*19//20]))
        five_samples.close()
        sample7.sort()
        num_bins = 51
        n, bins, patches = plt.hist(sample7, num_bins, facecolor='blue', alpha=0.5)
        plt.savefig("sample7_4.png")
        plt.clf()
        size7 = len(sample7)
        for i in range(size7):
            seven_samples.write('Distance: {:.1f}cm\n' .format(sample7[i]))
        seven_samples.write('5%: {:.1f}cm\nQuartile: {:.1f}cm\nMedian: {:.1f}cm\nThird quartile: {:.1f}cm\n95%: {:.1f}cm' .format(sample7[size7//20], sample7[size7//4], sample7[size7//2], sample7[size7*3//4], sample7[size7*19//20]))
        seven_samples.close()
        sample9.sort()
        num_bins = 51
        n, bins, patches = plt.hist(sample9, num_bins, facecolor='blue', alpha=0.5)
        plt.savefig("sample9_4.png")
        plt.clf()
        size9 = len(sample9)
        for i in range(size9):
            nine_samples.write('Distance: {:.1f}cm\n' .format(sample9[i]))
        nine_samples.write('5%: {:.1f}cm\nQuartile: {:.1f}cm\nMedian: {:.1f}cm\nThird quartile: {:.1f}cm\n95%: {:.1f}cm' .format(sample9[size9//20], sample9[size9//4], sample9[size9//2], sample9[size9*3//4], sample9[size9*19//20]))
        nine_samples.close()
        # raw samples
        grove.sample.sort()
        num_bins = 51
        n, bins, patches = plt.hist(grove.sample, num_bins, facecolor='blue', alpha=0.5)
        plt.savefig("sample1_4.png")
        plt.clf()
        size = len(grove.sample)
        for i in range(size):
            one_samples.write('Distance: {:.1f}cm\n' .format(grove.sample[i]))
        one_samples.write('5%: {:.1f}cm\nQuartile: {:.1f}cm\nMedian: {:.1f}cm\nThird quartile: {:.1f}cm\n95%: {:.1f}cm' .format(grove.sample[size//20], grove.sample[size//4], grove.sample[size//2], grove.sample[size*3//4], grove.sample[size*19//20]))
        one_samples.close()
        #######################
        # sample.sort()
        # size = len(sample)
        # for i in range(size):
        #     print('Distance: {:.1f}cm' .format(sample[i]))
        # print('5%: {:.1f}cm\nQuartile: {:.1f}cm\nMedian: {:.1f}cm\nThird quartile: {:.1f}cm\n95%: {:.1f}cm' .format(sample[size//20], sample[size//4], sample[size//2], sample[size*3//4], sample[size*19//20]))


from . import Widget

class Field(Widget):

	def __init__(self, char_size=1):
		self.width = 0
		self.height = 0
		self.char_size = char_size
		self.pad = None
		self.center = (0, 0)
		self.redraw = False
		self.offset = (0, 0)

	def set_backend(self, backend):
		self.backend = backend
		self.pad = self.backend.create_pad(self.width * self.char_size, self.height)

	def set_char_size(self, char_size):
		self.char_size = char_size
		self.pad = self.backend.create_pad(self.width * self.char_size, self.height)

	def set_dimensions(self, offset, width, height, keep=False):
		new_pad = self.backend.create_pad(width * self.char_size, height)
		if self.width > 0 and self.height > 0:
			new_pad.draw_pad(self.pad, (self.offset[0] - offset[0]) * self.char_size, self.offset[1] - offset[1], self.width * self.char_size, self.height, 0, 0)
		self.pad = new_pad
		self.offset = offset
		self.width = width
		self.height = height
		self.redraw = True
		self.change()

	def change_cell(self, x, y, char, style=None):
		x -= self.offset[0]
		y -= self.offset[1]
		if x < 0 or y < 0 or x >= self.width or y >= self.height:
			return
		self.pad.write(x * self.char_size, y, char, style)
		self.change()

	def draw_all(self, values, mapping, area=None):
		if area is None:
			xmin = 0
			ymin = 0
			w = self.width
			h = self.height
		else:
			((xmin, ymin), (w, h)) = area
		xmin -= self.offset[0]
		ymin -= self.offset[1]
		# This code is hot. Performance gains can be worth the price of code quality
		brushes = [(char, self.pad.get_raw_style(style)) for (char, style) in mapping]
		for x in range(0, w):
			sized_x = (x + xmin) * self.char_size
			for y in range(0, h):
				value = values[x+w*y]
				brush = brushes[value]
				self.pad.set_char(sized_x, y + ymin, brush[0], brush[1])
		self.change()

	def set_center(self, x, y):
		self.center = (x, y)
		self.change()

	def _round_width(self, x):
		return x // self.char_size * self.char_size

	def draw(self, target):
		center_x = self.center[0] - self.offset[0]
		center_y = self.center[1] - self.offset[1]
		target.draw_pad(
			self.pad,
			src_x = max(0, min(
				self._round_width(self.pad.width - target.width),
				self._round_width(center_x * self.char_size - target.width // 2)
			)),
			src_y = max(0, min(self.pad.height - target.height, center_y - target.height // 2)),
			width = self._round_width(target.width),
			dest_x = max(0, (target.width - self.pad.width) // 2),
			dest_y = max(0, (target.height - self.pad.height) // 2)
		)

	@classmethod
	def from_xml(cls, children, attr, text):
		return cls(char_size=int(attr.get("char-size", 1)))


birth = lambda a , b : a[0] + b[0]
print(birth("30" ,"40"))

cont = lambda num :  "high" if num > 50 else "low"
print(cont(49.99))

ch = lambda n1,n2 : (n1*n2) + (n1+n2)

def check(operation , n1 , n2):
    return operation(n1,n2)

result = check(ch,2,3)
print(result)

seq = [1,2,3,4,5,6,7,8,9]
even = filter(lambda num : num % 2 == 0 , seq )
print(even)

a = int(input(3))
b = int(input(3))
c = a+b
c1 = a-b
c2 = a*b
print(c)
print(c1)
print(c2)


# 백준 - 색종이

import sys

sys.stdin = open("input.txt")

def how_many_1s(x: int, y: int) -> int:
    count_new_1s = 0
    for i in range(10):
        for j in range(10):
            if white_paper[x - i][y + j] != 1:
                white_paper[x - i][y + j] = 1
                count_new_1s += 1
    return count_new_1s

color_paper_count: int = int(input())

white_paper = [[0] * 100 for _ in range(100)]
result = 0
for _ in range(color_paper_count):
    idx_x, idx_y = map(int, input().split())
    result += how_many_1s(idx_x, idx_y)

print(result)


from datetime import datetime

try:
    from config import config_page_name
except ModuleNotFoundError:
    raise ModuleNotFoundError("config.py")

import json
import os
import re
import time
import mwparserfromhell
import pywikibot

defaultTime = datetime(1, 1, 1)
os.environ['PYWIKIBOT_DIR'] = os.path.dirname(os.path.realpath(__file__))

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg["enable"]:
    exit("disabled")

rxpage = pywikibot.Page(site, cfg["main_page_name"])
text = rxpage.text

wikicode = mwparserfromhell.parse(text)

archivelist = {}
count = 0

for template in wikicode.filter_templates():
    if template.name.lower() == "status2":
        if template.has(1):
            status = template.get(1)
        else:
            status = "(empty)"
        print(f"Status: {status}")
        if status in cfg["publicizing_status"]:
            publicizing = True
            print("Publicizing...")
            break
        elif status in cfg["done_status"]:
            processed = True
            print("Processed.")
            break

lasttime = datetime(1, 1, 1)
for m in re.findall(r"(\d{4})年(\d{1,2})月(\d{1,2})日 \(.\) (\d{2}):(\d{2}) \(UTC\)", str(wikicode)):
    d = datetime(int(m[0]), int(m[1]), int(m[2]), int(m[3]), int(m[4]))
    lasttime = max(lasttime, d)
print(f"var: lasttime: {lasttime}")

processed = False
if re.search(cfg["processed_regex"], str(wikicode)) and not re.search(cfg["not_processed_regex"], str(wikicode)):
    processed = True
    print("Processed.")
else:
    print("Not processed.")

t1 = time.time() - lasttime.timestamp()
if (processed and t1 > cfg["time_to_live_for_processed"]) or (not processed and t1 > cfg["time_to_live_for_not_processed"]) and (lasttime != datetime(1, 1, 1)):
    target = (lasttime.year, lasttime.month)

if ((processed and not publicizing and t1 > cfg["time_to_live_for_processed"]) or (not processed and not publicizing and t1 > cfg["time_to_live_for_not_processed"])) and lasttime != datetime(1, 1, 1):
    target = (lasttime.year, lasttime.month)
    if target not in archivelist:
        archivelist[target] = []
    archivestr = str(wikicode).strip()
    archivestr = re.sub(
        r"{{bot-directive-archiver\|no-archive-begin}}[\s\S]+?{{bot-directive-archiver\|no-archive-end}}\n?", "", archivestr)
    archivelist[target].append(archivestr)
    count += 1
    wikicode.remove(wikicode)
    print("Archive to " + str(target))

pywikibot.showDiff(rxpage.text, text)
rxpage.text = text
summary = cfg["main_page_summary"].format(count)
print(summary)
rxpage.save(summary=summary, minor=False)

for target in archivelist:
    archivepage = pywikibot.Page(site, cfg["archive_page_name"].format(target[0], target[1]))
    text = archivepage.text
    print(archivepage.title())
    if not archivepage.exists():
        text = cfg["archive_page_preload"]
    text += "\n\n" + "\n\n".join(archivelist[target])

    pywikibot.showDiff(archivepage.text, text)
    archivepage.text = text
    summary = cfg["archive_page_summary"].format(len(archivelist[target]))
    print(summary)
    archivepage.save(summary=summary, minor=False)


from flask import Flask, render_template
from sqlalchemy import create_engine, MetaData, Table, Column, DECIMAL, Integer, String
import boto3
import os
import time
from flask import request

# ugly globals but this isn't MVC so I don't have a good way to share data across pages atm
numfiles = ""
bucket = ""
region = ""
endpoint = ""
accesskey = ""
secretkey = ""
saveobjects = False
postresults = {}

application = Flask(__name__)

@application.before_first_request
def db_setup():
    dbconn = os.environ['DATABASE_URL']
    engine = create_engine(dbconn, echo=True)
    meta = MetaData()

    results = Table(
        'results', meta,
        Column('bucket', String),
        Column('endpoint', String),
        Column('minblock', Integer),
        Column('maxblock', Integer),
        Column('numfiles', Integer),
        Column('postmin', DECIMAL),
        Column('postmax', DECIMAL),
        Column('postmean', DECIMAL),
        Column('posttotal', DECIMAL),
        Column('getmin', DECIMAL),
        Column('getmax', DECIMAL),
        Column('getmean', DECIMAL),
        Column('gettotal', DECIMAL),
    )
    meta.create_all(engine)

@application.route("/", methods=['GET'])
def index():
    return render_template("index.html")

@application.route("/post-testing", methods=['POST', 'GET'])
def posttesting():
    global numfiles
    global bucket
    global region
    global endpoint
    global accesskey
    global secretkey
    global saveobjects
    global postresults
    global minblock
    global maxblock

    # timer storage
    time_to_upload = []

    try:
        for x in range(numfiles):
            command = "dd if=/dev/urandom of=/tmp/file.{} bs=$(shuf -i{}-{} -n1) count=1024".format(x, minblock, maxblock)
            os.system(command)

    except Exception as error:
        message = "Error generating objects, details potentially below:"
        return render_template("error.html", message=message, error=error)
    try:
        session = boto3.session.Session()
        client = session.client('s3',
                                region_name=region,
                                endpoint_url= endpoint,
                                aws_access_key_id=accesskey,
                                aws_secret_access_key=secretkey)
    except Exception as error:
        message = "Error initializing session, details potentially below:"
        return render_template("error.html", message=message, error=error)

    try:
        total_start = time.time()
        for x in range(numfiles):
            local_start = time.time()
            client.upload_file('/tmp/file.{}'.format(x),  # Path to local file
                bucket,  # Name of Space
                'file.{}'.format(x))  # Name for remote file
            local_end = time.time()
            local_time = local_end - local_start
            time_to_upload.append(local_time)

        total_end = time.time()
        total_time = total_end - total_start
    except Exception as error:
        message = "Error occurred during upload, details potentially below:"
        return render_template("error.html", message=message, error=error)
    # cleanup /tmp
    os.system("rm -f /tmp/file.*")

    mean = sum(time_to_upload) / int(numfiles)
    lowest = min(time_to_upload)
    highest = max(time_to_upload)

    postresults['total'] = total_time
    postresults['mean'] = mean
    postresults['lowest'] = lowest
    postresults['highest'] = highest

    return render_template("partial-results.html", total_time=total_time, mean=mean, lowest=lowest, highest=highest)

@application.route("/inprogress", methods=['POST'])
def inprogress():

    # Form variables
    global numfiles
    global bucket
    global region
    global endpoint
    global accesskey
    global secretkey
    global saveobjects
    global postresults
    global minblock
    global maxblock

    minblock = request.form['minblock']
    maxblock = request.form['maxblock']
    numfiles = int(request.form['numfiles'])
    bucket = request.form['bucket']
    region = request.form['region']
    endpoint = request.form['endpoint']
    accesskey = request.form['accesskey']
    secretkey = request.form['secretkey']

    # saveobjects logic, default to False, only flip on box check
    if request.form.get('saveobjects'):
        saveobjects = True

    return render_template("inprogress.html")

@application.route("/get-testing", methods=['GET'])
def gettesting():
    global numfiles
    global bucket
    global region
    global endpoint
    global accesskey
    global secretkey
    global saveobjects
    global postresults

    # timer storage
    time_to_download = []

    try:
        session = boto3.session.Session()
        client = session.client('s3',
                                region_name=region,
                                endpoint_url=endpoint,
                                aws_access_key_id=accesskey,
                                aws_secret_access_key=secretkey)
    except Exception as error:
        message = "Error initializing session, details potentially below:"
        return render_template("error.html", message=message, error=error)

    try:
        total_start = time.time()
        for x in range(numfiles):
            local_start = time.time()
            client.download_file(bucket,  # Name of Space
                'file.{}'.format(x),  # Name for remote file
                '/tmp/file.{}'.format(x))  # Name and path for local file

            local_end = time.time()
            local_time = local_end - local_start
            time_to_download.append(local_time)

        total_end = time.time()
        total_time = total_end - total_start
    except Exception as error:
        message = "Error occurred during download, details potentially below:"
        return render_template("error.html", message=message, error=error)

    # cleanup /tmp
    os.system("rm -f /tmp/file.*")

    mean = sum(time_to_download) / int(numfiles)
    lowest = min(time_to_download)
    highest = max(time_to_download)

    # If saveobjects is FALSE then delete, otherwise skip
    if not saveobjects:
        try:
            # cleanup logic goes here
            filedict = [{"Key": f"file.{n}"} for n in range(numfiles)]
            response = client.delete_objects(
                Bucket=bucket,
                Delete={
                    'Objects': filedict,
                    'Quiet': True
                },
            )
        except Exception as error:
            message = "Error occurred during object cleanup, details potentially below:"
            return render_template("error.html", error=error)

    # Database work goes here

    try:
        dbconn = os.environ['DATABASE_URL']
        engine = create_engine(dbconn, echo=True)
        meta = MetaData(bind=engine, reflect=True)

        results = meta.tables['results']
        ins = results.insert().values(bucket=bucket, endpoint=endpoint, postmin=postresults['lowest'], postmax=postresults['highest'],
                                      postmean=postresults['mean'], posttotal=postresults['total'],
                                      getmin=lowest, getmax=highest, getmean=mean, gettotal=total_time,
                                      minblock=minblock, maxblock=maxblock, numfiles=numfiles)
        conn = engine.connect()
        result = conn.execute(ins)
    except Exception as error:
        message = "Error occurred while writing to the database, details potentially below:"
        return render_template("error.html", message=message, error=error)
    return render_template("results.html", post_total_time=postresults['total'],
                           post_mean=postresults['mean'],
                           post_lowest=postresults['lowest'],
                           post_highest=postresults['highest'],
                           get_total_time=total_time,
                           get_mean=mean,
                           get_lowest=lowest,
                           get_highest=highest
                           )

if __name__ == "__main__":
    application.run(host="0.0.0.0", port=80)


from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase

class ComicData(ComicDataBase):
    name = "A Softer World"
    language = "en"
    url = "http://www.asofterworld.com/"
    start_date = "2003-02-07"
    rights = "Joey Comeau, Emily Horne"
    active = False

class Crawler(CrawlerBase):
    schedule = None
    time_zone = "US/Pacific"

    def crawl(self, pub_date):
        page = self.parse_page("http://www.asofterworld.com/")
        url = page.src("#comicimg img")
        text = page.title("#comicimg img")
        return CrawlerImage(url, None, text)


import requests
from requests.exceptions import Timeout
import dateparser
import json
from shapely.geometry import Point
from shapely.ops import transform
from functools import partial
import pyproj

SHUTTLE_SPEED = 15 #km/hr

def coordinates_to_geography(shapely_point):
    project = partial(
        pyproj.transform,
        pyproj.Proj(init='EPSG:4326'),
        pyproj.Proj(init='EPSG:32633')
    )
    return transform(project, shapely_point)

class Stop():
    '''doc'''
    all_stops = {}
    def __init__(self, _id, name, station, lat, lon):
        self._id = _id
        self.name = name
        self.station = station
        self.location = Point(lon, lat)
        self.projected_location = coordinates_to_geography(Point(lon, lat))
        self.lines = set()
        type(self).all_stops[_id] = self

class Station():
    '''doc'''
    all_stations = {}
    def __init__(self, _id, name, lat, lon, weight):
        self._id = _id
        self.name = name
        self.projected_location = coordinates_to_geography(Point(lon, lat))
        self.location = Point(lon, lat)
        self.weight = float(weight)
        self.schedules = []
        self.linked_stations = []
        self.station = self
        self.stops = []
        self.lines = set()
        type(self).all_stations[_id] = self

    def departures_between(self, start, end):
        """tbd"""
        result = []
        for schedule in sorted(self.schedules):
            if schedule >= start and schedule <= end:
                result.append(schedule)
        return result

    def time_to_station(self, station, when=None):
        url = f'http://localhost:3000/journeys?from={self._id}&to={station._id}&results=1'
        if when:
            url += f'&when={when}'
        try:
            journeys_data = requests.get(url, timeout=3).json()
            if len(journeys_data) > 0:
                try:
                    journey = journeys_data[0]
                    return int((dateparser.parse(journey['arrival']).timestamp() - when)/60)
                except Exception as e:
                    print('error!!!!!!!!!')
                    print(e)
                    print(journeys_data)
        except Timeout:
            print('request timed out')
        except (KeyboardInterrupt, SystemExit):
            raise
        return 100000000

    def time_by_shuttle_in_minutes(self, station):
        origin = self.projected_location
        destination = station.projected_location
        distance = origin.distance(destination)/1000
        return (distance/SHUTTLE_SPEED)*60

    def contiguous_stations(self):
        contiguous = []
        for line in self.lines:
            contiguous.extend(line.next_stations(self))
        return contiguous

    def next_departure(self, current_time):
        """tbd"""
        for schedule in sorted(self.schedules):
            if schedule >= current_time:
                return schedule

    def next_arrival_to(self, station_id, current_time):
        """tbd"""
        pass

    def departures(self, timestamp):
        """tbd"""
        journeys = []
        directions = self.contiguous_stations()
        for direction in directions:
            url = f'http://localhost:3000/stations/{self._id}/departures?when={int(timestamp)}&nextStation={direction._id}'
            journeys_data = requests.get(url, timeout=3).json()
            for journey in journeys_data:
                if journey.get('cancelled', False):
                    continue
                journeys.append(Journey(journey['journeyId'], self, journey['when'], direction, journey['line'], journey['trip'], journey['delay']))
        return journeys

    @classmethod
    def common_linked_stations(cls, station_a, station_b):
        """to be documented"""
        return set(station_a.linked_stations).intersection(set(station_b.linked_stations))

    @classmethod
    def minutes_between(cls, station_a, station_b):
        """to be documented"""
        distance = station_a.projected_location.distance(station_b.projected_location)/1000
        return (distance/SHUTTLE_SPEED)*60

    @classmethod
    def can_reach(cls, station_a, station_b, current_time):
        """doc"""
        common_stations = cls.common_linked_stations(station_a, station_b)
        reachable_stations = []
        time_between = cls.time_between(station_a, station_b)
        for station in common_stations:
            next_arrival_a = station_a.next_arrival_to(station, current_time)
            next_arrival_b = station_b.next_arrival_to(station, current_time + time_between)
            if  next_arrival_a >= next_arrival_b:
                reachable_stations.append(station)
        return reachable_stations

    @classmethod
    def get_stations(cls):
        '''doc'''
        stations_data = requests.get('http://localhost:3000/stations/all', timeout=3).json()
        as_dictionary = {}
        as_list = []
        for key in stations_data:
            data = stations_data[key]
            station = cls(key, data['name'],
                          data['location']['latitude'],
                          data['location']['longitude'],
                          data['weight'])
            for stop_data in data['stops']:
                stop = Stop(stop_data['id'], stop_data['name'], station,
                            stop_data['location']['latitude'],
                            stop_data['location']['longitude'])
                station.stops.append(stop)
            as_list.append(station)
            as_dictionary[key] = station
        return as_list

    @classmethod
    def find_by_name(cls, name):
        for station_id in cls.all_stations:
            station = cls.all_stations[station_id]
            if station.name == name:
                return station
        return None

    def modes(self):
        modes_set = set()
        for line in self.lines:
            modes_set.add(line.mode)
        return modes_set

class Journey():
    '''doc'''
    def __init__(self, _id, station, when, direction, line, trip, delay):
        self._id = _id
        self.station = station
        self.when = dateparser.parse(when)
        self.direction = direction # station
        self.line = Line.find_by_name(line['name'])
        self.line_name = line['name']
        self.trip = trip
        self.delay = delay

class Line():
    '''doc'''
    all_lines = {}
    def __init__(self, _id, name, operator, mode, product, variants):
        self._id = _id
        type(self).all_lines[_id] = self
        self.name = name
        self.operator = operator
        self.mode = mode
        self.product = product
        self.variants = []
        self.stations_set = set()
        self.stops_set = set()
        for variant in variants:
            new_variant = []
            for stop_id in variant:
                stop = None
                try:
                    stop = Stop.all_stops[stop_id]
                    self.stops_set.add(stop)
                    self.stations_set.add(stop.station)
                    stop.station.lines.add(self)
                except:
                    stop = Station.all_stations[stop_id]
                    self.stations_set.add(stop)
                stop.lines.add(self)
                new_variant.append(stop)
            self.variants.append(new_variant)

    def station_from_direction_steps(self, origin, direction, steps=1):
        for variant in self.variants:
            index_origin = None
            index_direction = None
            index_result = None
            for i in range(len(variant)):
                if variant[i].station == origin.station:
                    index_origin = i
                elif variant[i].station == direction.station:
                    index_direction = i
                if index_direction and index_origin:
                    if index_direction > index_origin:
                        index_result = index_origin + steps
                    else:
                        index_result = index_origin - steps
                if index_result and index_result < len(variant) and index_result >= 0:
                    return variant[index_result]
        return None

    @classmethod
    def get_lines(cls, return_dictionary=False):
        '''doc'''
        lines_data = requests.get('http://localhost:3000/lines?variants=true', timeout=3)
        as_dictionary = {}
        as_list = []
        for line_data in lines_data.text.split('\n'):
            if len(line_data) < 2:
                continue
            data = json.loads(line_data)
            line = cls(data['id'], data['name'], data['operator'], data['mode'],
                       data['product'], data['variants'])
            as_list.append(line)
            as_dictionary[data['id']] = line
        if return_dictionary:
            return as_dictionary
        return as_list

    def contains_stop_or_station(self, stop_or_station):
        return stop_or_station in self.stations_set

    def next_stations(self, station):
        result = set()
        for variant in self.variants:
            for i in range(len(variant)):
                if variant[i].station == station:
                    if i > 0:
                        result.add(variant[i - 1])
                    if i + 1 < len(variant):
                        result.add(variant[i + 1])
                    continue
        return list(result)

    def intersecting_lines(self, origin, dest):
        for variant in self.variants:
            try:
                origin_index = variant.index(origin)
                dest_index = variant.index(dest)
            except:
                continue
            if dest_index > origin_index:
                origin_index = origin_index + 1
            else:
                origin_index = origin_index - 1

            intersections = []
            for i in range(origin_index, dest_index):
                current_station = variant[i]
                intersections.append([current_station, current_station.lines.remove(self)])
            return intersections
        return None

    @classmethod
    def find_by_name(cls, name):
        name = name.split()
        if len(name)>1:
            name = name[1]
        else:
            name = name[0]
        for line_id in cls.all_lines:
            line = cls.all_lines[line_id]
            if line.name == name:
                return line
        return None